From 8518e53484b3374bfec1325354aa9f1bfe8bcf2a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:27:39 -0800 Subject: [PATCH 01/14] feat(novelty): ground answers on a C API's names from its headers and libraries `skill novelty ground` could only veto long options from a tool's --help. A closed C library (oneMKL) has no such help, so an answer naming an invented function (cblas_dgemmx, LAPACKE_dgesvv) went to the judge. --header PATH (glob ok) and --library PATH (read with nm -D --defined-only), both repeatable, add the API's names as the known set of an identifiers veto. --prefix (a C identifier ending in '_', repeatable) is required with them and scopes which names are checked, so an ordinary word is never read. Each input is pinned by the sha256 of its bytes, as the help snapshot is. A set with no prefixed name is refused, so a wrong header cannot veto everything. Key verification (meta.key_status) covers these names too. The flags are registered as the api_header and symbol_prefix concepts. Signed-off-by: Nikolay Petrov --- ...000_napetrov_novelty-ground-c-api-names.md | 9 + dokimi/docs/answer-formats.md | 2 +- dokimi/docs/grading-tiers.md | 12 +- dokimi/docs/reference/cli-interface.json | 5 +- dokimi/docs/skill-novelty.md | 3 +- dokimi/dokimi/cli_flags.py | 13 + dokimi/dokimi/commands/novelty_ground.py | 38 ++- dokimi/dokimi/commands/skill.py | 13 +- dokimi/dokimi/novelty/grounding.py | 31 ++- dokimi/dokimi/novelty/symbols.py | 149 ++++++++++++ dokimi/tests/test_novelty_checks.py | 18 +- dokimi/tests/test_novelty_ground_symbols.py | 226 ++++++++++++++++++ 12 files changed, 489 insertions(+), 30 deletions(-) create mode 100644 dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md create mode 100644 dokimi/dokimi/novelty/symbols.py create mode 100644 dokimi/tests/test_novelty_ground_symbols.py diff --git a/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md b/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md new file mode 100644 index 000000000..eea927033 --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md @@ -0,0 +1,9 @@ +### Added + +- **`skill novelty ground` vetoes C API names** — `--header PATH` (glob ok) and `--library PATH` + (a shared library, read with `nm -D --defined-only`), each repeatable, with `--prefix cblas_` + (a C identifier ending in `_`, repeatable). An answer naming a prefixed name no header declares + and no library exports (`cblas_dgemmx`) is vetoed without a judge, and `meta.key_status` is + `not_found` for a key that names one. Each input is pinned by its sha256. A `--header` or + `--library` set with no prefixed name is refused, and `--prefix` needs one of them. No MKL + install is needed: any header set or shared library of the API works. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 2551924c7..254ac3e0d 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -60,7 +60,7 @@ question is `` when the config has one prompt and | `json` | Parses as JSON. With `values`, it must equal one of them. With `contains`, the answer may be any JSON value, or else it is the first JSON object or array anywhere in the answer; the scan tries the first 1000 `{` or `[` and stops once the tries that failed have scanned 1,000,000 characters in all (a try that fails on an overlong number, the depth limit or an unterminated string counts as scanning to the end of the text), so a value that starts after more brackets or more failed text than that reads as not JSON. | OpenAI `JsonMatch`; promptfoo `is-json`, `contains-json`, `equals` with an object value | | `judge` | One required point (`criterion`, `reference` or both), asked of the [checklist](benchmarking-and-comparison.md) judge in both polarities. 1 when the two agree that it is covered, 0 when they agree that it is not, 0.5 when they split. | Inspect `model_graded_qa`/`model_graded_fact`; a custom Inspect `@scorer`, on its docstring's first paragraph; OpenAI `ModelBasedClassify` specs; promptfoo `llm-rubric`, `g-eval`, `factuality`, `model-graded-closedqa`; every skill-creator expectation | | `unsupported` | Listed with the native name and never counted | promptfoo `javascript`, `python`, `similar`, `cost`, `latency`, negated judge asserts (`not-llm-rubric`) and other asserts; Inspect `pattern` (it compares an extracted group with the target) and Inspect scorers that are neither built in nor a `@scorer` of the file with a docstring; a row whose `ideal` or `target` is blank or missing | -| `identifiers` | Every long option the answer names (`--x`, or the `pattern`, inside `within`) is in `known`. An answer that names no option passes, since the check only rules names out. Not `sufficient`: a `sufficient` identifiers check is refused. | None: built from a tool's `--help` by `skill novelty ground`, for novelty probes | +| `identifiers` | Every long option the answer names (`--x`, or the `pattern`, inside `within`) is in `known`; with `skill novelty ground --header/--library`, every name with a `--prefix` prefix is. An answer that names no option passes, since the check only rules names out. Not `sufficient`: a `sufficient` identifiers check is refused. | None: built from a tool's `--help` by `skill novelty ground`, for novelty probes | `role` (`veto`, `sufficient`, `evidence`) and `tier` are read by novelty probes only, as [grading-tiers](grading-tiers.md) describes. `eval score reference` does not use `role` to score, diff --git a/dokimi/docs/grading-tiers.md b/dokimi/docs/grading-tiers.md index 6f384de8e..ecfee9329 100644 --- a/dokimi/docs/grading-tiers.md +++ b/dokimi/docs/grading-tiers.md @@ -67,15 +67,25 @@ with the docs, not knowledge of the skill) and is still a model grading free tex | Roles: a failed `veto` gives `wrong_other`; a `sufficient` check decides; `evidence` is recorded only; an abstention is never overruled into a pass | `measure.py` `resolve_verdict` | | `judge_verdict`, `verdict_source`, `checks` on each checked run; a record-level `checks` block (`checked_runs`, `overruled`, `vetoed`, `judge_passed_vetoed`, `decided`, `judge_agreed`) | `measure.py` `ProbeRun`, `MeasureResult.checks_summary`, `report.py` | | `identifiers` check: every identifier the answer names (long options by default, inside `within`) is in `known`; `--no-x` counts when `--x` is known and the check sets `negations: true` or the help lists `--[no-]x`. Two words and an option after a command read as another command, so `and add --x` after it is not checked. `role` and `tier` validated on every check | `dokimi/eval/reference_checks.py` | -| `skill novelty ground --atoms A --tool "perf report" [--tool ...]`: runs the tool's `--help`, writes an `identifiers` veto with the help's sha256 into the atoms that name the tool | `dokimi/novelty/grounding.py`, `commands/novelty_ground.py` | +| `skill novelty ground --atoms A --tool "perf report" [--tool ...]`: runs the tool's `--help`, writes an `identifiers` veto with the help's sha256 into the atoms that name the tool. `--header PATH` (glob ok) and `--library PATH` with `--prefix cblas_` (repeatable) do the same for a C API: the prefixed names a header declares, and the exported symbols of a shared library (`nm -D --defined-only`), each input pinned by its sha256 | `dokimi/novelty/grounding.py`, `novelty/symbols.py`, `commands/novelty_ground.py` | | Key verification: the same command writes `meta.key_status` on each checked atom (`verified`, `not_found` with the `missing` names, `unverifiable` when the key names no option or one of its checks could not finish) and prints the counts. A `not_found` is a review flag, not an error | `grounding.py` `verify_keys` | A `--tool` check applies only where the tool's name is followed directly by an option, so a `perf` check does not judge `perf report --stdio`: give each subcommand as its own `--tool`. An answer that names no option of the tool passes the veto. +A C API has no `--help`, so its check reads names instead. `--prefix` (a C identifier ending in +`_`, repeatable) says which names are checked: an answer's `cblas_dgemmx` is vetoed when no +`--header` declares it and no `--library` exports it, and an ordinary word is never read. A +header's names are every prefixed identifier of its code (comments and strings aside), so a +name a macro expands to counts too; a header's `#include` of another header is not followed, so +pass each header. A `--header` or `--library` set with no prefixed name is refused, so a wrong +file cannot veto every name. Key verification works the same way: a key naming `LAPACKE_dgesvv` +is `not_found`. + ```bash python cli.py skill novelty ground --atoms atoms.json --tool "perf report" --tool "perf record" --tool-version 6.17 +python cli.py skill novelty ground --atoms atoms.json --header "include/mkl_*.h" --library libmkl_rt.so --prefix cblas_ --prefix LAPACKE_ python cli.py skill novelty probe --artifact skill:linux-perf --atoms atoms.json --out probes.json ``` diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..f92b56297 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1672,9 +1672,12 @@ "allow_abbrev": true, "params": [ {"key": "--atoms", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, + {"key": "--header", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--library", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--out", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, - {"key": "--tool", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, + {"key": "--prefix", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--tool", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--tool-version", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} ], "path": "dokimi skill novelty ground", diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index c4c2723c7..d89855976 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -931,7 +931,8 @@ python cli.py skill novelty items --probes m1.json m2.json m3.json [reference checks](answer-formats.md); every probe built from it except the reversal gets it, and its non-judge checks run on each answer next to the judge. `skill novelty ground --atoms atoms.json --tool "perf report"` writes one such check: a veto that fails an answer naming a -`perf report --flag` the tool's own `--help` does not list. The judge's verdict is kept on +`perf report --flag` the tool's own `--help` does not list. With `--header` and `--library` and a +`--prefix` (`cblas_`), it vetoes a C API name no header declares and no library exports. The judge's verdict is kept on each checked run and the record's `checks` block counts how often the judge passed an answer a veto failed. Design and evidence: [grading-tiers.md](grading-tiers.md). diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index ab3985679..0e6edf7ce 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -288,6 +288,19 @@ class Concept: banned=("--doc-source", "--context7-id"), note="A documentation source is an arm: --arm docs:SOURCE.", ), + Concept( + name="api_header", + help="a C header whose declared names a veto may check", + canonical=(Flag("--header", "PATH", "path", repeatable=True, + meaning="a C header of the API (glob ok)"),), + note="Used with --prefix (and --library) by skill novelty ground.", + ), + Concept( + name="symbol_prefix", + help="the C identifier prefix that scopes the names a veto checks", + canonical=(Flag("--prefix", "PREFIX", "text", repeatable=True, + meaning="a C identifier prefix ending in '_' (cblas_)"),), + ), Concept( name="force", help="re-run over existing output", diff --git a/dokimi/dokimi/commands/novelty_ground.py b/dokimi/dokimi/commands/novelty_ground.py index c9de726c5..961d5d884 100644 --- a/dokimi/dokimi/commands/novelty_ground.py +++ b/dokimi/dokimi/commands/novelty_ground.py @@ -14,15 +14,42 @@ from dokimi.utils import InputError +def _refuse_bad_sources(tools: list[str], headers: list[str], libraries: list[str], + prefixes: list[str]) -> None: + """Refuse a source set the flags cannot read: ``--prefix`` goes with a header or a library.""" + if prefixes and not (headers or libraries): + raise InputError("--prefix scopes the names of --header and --library: give one of them") + if (headers or libraries) and not prefixes: + raise InputError("--header and --library need --prefix: it says which names are checked") + if not (tools or headers or libraries): + raise InputError("give --tool, --header or --library: nothing to ground the atoms on") + + +def _describe(check: dict) -> str: + """The stderr line for one check: its source, how many names it lists, and the pinned sha256.""" + source = check["source"] + if source.get("kind") == "symbols": + inputs = source["inputs"] + return (f"symbols {' '.join(source['prefixes'])}: {len(check['known'])} names from " + f"{len(inputs)} files (sha256 {'; '.join(i['sha256'][:12] for i in inputs)})") + return (f"{source['tool']}: {len(check['known'])} options " + f"(help sha256 {source['sha256'][:12]})") + + def cmd_novelty_ground(args: argparse.Namespace) -> None: - """Write ``identifiers`` vetoes from each ``--tool``'s own ``--help`` into an atoms file.""" + """Write ``identifiers`` vetoes from each ``--tool``'s ``--help`` and each source's names.""" from dokimi.novelty.atoms import load_atoms_record from dokimi.novelty.grounding import add_checks, help_text, identifiers_check, verify_keys + from dokimi.novelty.symbols import symbols_check + tools, headers = list(args.tool or []), list(args.header or []) + libraries, prefixes = list(args.library or []), list(args.prefix or []) + _refuse_bad_sources(tools, headers, libraries, prefixes) _, _, data = load_atoms_record(args.atoms) - checks = [identifiers_check(tool, help_text(tool), version=args.tool_version) - for tool in args.tool] - changed = add_checks(data.get("atoms", []), checks, args.tool) + checks = [identifiers_check(tool, help_text(tool), version=args.tool_version) for tool in tools] + if headers or libraries: + checks.append(symbols_check(prefixes, headers, libraries)) + changed = add_checks(data.get("atoms", []), checks) keys = verify_keys(data.get("atoms", [])) out = Path(args.out or args.atoms).resolve() # a symlink: its target is what is rewritten try: # as save_atoms does, an output in a new directory creates it @@ -43,8 +70,7 @@ def cmd_novelty_ground(args: argparse.Namespace) -> None: finally: tmp.unlink(missing_ok=True) for check in checks: - print(f"{check['source']['tool']}: {len(check['known'])} options " - f"(help sha256 {check['source']['sha256'][:12]})", file=sys.stderr) + print(_describe(check), file=sys.stderr) print(f"{changed} of {len(data.get('atoms', []))} atoms got a veto check -> {out}", file=sys.stderr) print("answer keys: " + ", ".join(f"{n} {status}" for status, n in keys.items()) diff --git a/dokimi/dokimi/commands/skill.py b/dokimi/dokimi/commands/skill.py index 9c71e3fc4..782fbf301 100644 --- a/dokimi/dokimi/commands/skill.py +++ b/dokimi/dokimi/commands/skill.py @@ -136,15 +136,24 @@ def register_skill(skill_sub, positive_int) -> None: probe_p.set_defaults(func=stops_at_cap(cmd_novelty_probe)) ground_p = novelty_sub.add_parser( - "ground", help="Add identifier vetoes from a tool's own --help to an atoms file" + "ground", help="Add identifier vetoes from a tool's --help or a C API to an atoms file" ) ground_p.add_argument("--atoms", required=True, help="atoms.json from `skill novelty atoms`") - ground_p.add_argument("--tool", action="append", required=True, + ground_p.add_argument("--tool", action="append", default=None, help="Command whose --help lists the valid options, e.g. " "'perf report'. Repeatable; an atom gets the check of each " "tool it names") ground_p.add_argument("--tool-version", default=None, dest="tool_version", help="Version recorded with the help snapshot") + ground_p.add_argument("--header", action="append", default=None, metavar="PATH", + help="C header whose API names are known (glob ok). Repeatable; " + "needs --prefix") + ground_p.add_argument("--library", action="append", default=None, metavar="PATH", + help="Shared library whose exported symbols (nm -D) are known. " + "Repeatable; needs --prefix") + ground_p.add_argument("--prefix", action="append", default=None, metavar="PREFIX", + help="C identifier prefix ending in '_' (e.g. cblas_) that scopes " + "--header and --library: only names with it are checked. Repeatable") ground_p.add_argument("--out", default=None, help="Default: rewrite --atoms in place") from dokimi.commands.novelty_ground import cmd_novelty_ground diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index f6799c18b..98b93224e 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -175,26 +175,39 @@ def _require_valid_gradings(atoms: list[dict]) -> None: raise InputError(f"atom {atom.get('id')}: invalid grading: {'; '.join(problems)}") -def add_checks(atoms: list[dict], checks: list[dict], tools: list[str]) -> int: - """Attach each tool's check to the atoms that name the tool; how many atoms changed. +def _mention_of(check: dict) -> str: + """What an atom must name for ``check`` to attach: a tool's command line, or a symbol prefix.""" + source = check.get("source") or {} + if source.get("kind") == "symbols": + return str(check["within"]) + return _mention(source["tool"]) - An atom that does not name the tool gets no check: its answers would rarely carry the - tool's command line, and a check that never fires is noise in the report. A check for - the same tool already on the atom is replaced, so re-running refreshes the snapshot. + +def _source_of(check: dict) -> str: + """The source a check replaces on re-run: its tool's name, or ``symbols`` for a symbol check.""" + source = check.get("source") or {} + return "symbols" if source.get("kind") == "symbols" else str(source.get("tool")) + + +def add_checks(atoms: list[dict], checks: list[dict]) -> int: + """Attach each check to the atoms that name its tool or prefix; how many atoms changed. + + An atom that names none gets no check: its answers would rarely carry the tool's command + line, and a check that never fires is noise in the report. A check from the same source + already on the atom is replaced, so re-running refreshes the snapshot. """ _require_valid_gradings(atoms) changed = 0 for atom in atoms: text = " ".join(str(atom.get(k, "")) for k in ("statement", "question", "answer")) - mine = [c for c, tool in zip(checks, tools, strict=True) - if re.search(_mention(tool), text)] + mine = [c for c in checks if re.search(_mention_of(c), text)] if not mine: continue meta = atom["meta"] = atom.get("meta") or {} grading = meta["grading"] = meta.get("grading") or {"checks": []} + replaced = {_source_of(m) for m in mine} kept = [c for c in grading.get("checks", []) - if not (c.get("type") == "identifiers" - and (c.get("source") or {}).get("tool") in {m["source"]["tool"] for m in mine})] + if not (c.get("type") == "identifiers" and _source_of(c) in replaced)] grading["checks"] = kept + mine changed += 1 return changed diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py new file mode 100644 index 000000000..842b6d80d --- /dev/null +++ b/dokimi/dokimi/novelty/symbols.py @@ -0,0 +1,149 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Known identifiers of a C library's API, from its headers and its exported symbols. + +``skill novelty ground --header`` and ``--library`` pin the names a C API declares, so an answer +that names ``cblas_dgemmx`` (not in the library) is vetoed the way an invented ``--option`` is. +Only names with a prefix from ``--prefix`` are read, so an ordinary word is never checked. +""" + +from __future__ import annotations + +import glob +import hashlib +import re +from pathlib import Path +from typing import Sequence + +from dokimi.process import Env, run +from dokimi.utils import InputError + +#: A C identifier prefix: a C identifier that ends in an underscore (``cblas_``, ``LAPACKE_``). +_PREFIX = re.compile(r"[A-Za-z_]\w*_") +#: Comments and string or character literals of C code: no declaration is named in them. +_C_SKIP = re.compile(r'/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.DOTALL) +_CHUNK = 1 << 20 + + +def check_prefixes(prefixes: Sequence[str]) -> tuple[str, ...]: + """The prefixes, sorted and without repeats; refused unless each is a C identifier prefix. + + Raises: + InputError: A prefix is not a C identifier ending in ``_`` (``mkl`` is refused: it + would also match ordinary words). + """ + bad = [p for p in prefixes if not _PREFIX.fullmatch(p)] + if bad: + raise InputError(f"--prefix {bad[0]!r} is not a C identifier prefix ending in '_' " + "(for example cblas_)") + return tuple(sorted(set(prefixes))) + + +def symbol_pattern(prefixes: Sequence[str]) -> str: + """A regex for the names with one of ``prefixes`` as C identifiers, not as parts of a word. + + A name after ``.``, ``>``, ``/`` or ``-`` is a member, a path or an option, and ``mkl_types`` + in ``mkl_types.h`` is a file name: none of those is an API name. + """ + alternatives = "|".join(re.escape(p) for p in prefixes) + return rf"(?-])(?:{alternatives})\w+(?!\.?\w)" + + +def header_names(text: str, prefixes: Sequence[str]) -> set[str]: + """The names with a prefix that a C header's code mentions: macros, types, declarations. + + Comments and literals are skipped. A name a macro expands to counts too, so the set is a + superset of the declared API: the check then misses an invented name only if the header + itself spells it. + """ + code = _C_SKIP.sub(" ", text) + return set(re.findall(symbol_pattern(prefixes), code)) + + +def nm_names(output: str, prefixes: Sequence[str]) -> set[str]: + """The names with a prefix in ``nm -D --defined-only`` output, without a ``@@VER`` tag.""" + names: set[str] = set() + for line in output.splitlines(): + fields = line.split() + if len(fields) != 3: # address, type, name: an undefined name has no address + continue + name = fields[2].split("@", 1)[0] + if re.fullmatch(r"\w+", name) and name.startswith(tuple(prefixes)): + names.add(name) + return names + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(_CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _files(patterns: Sequence[str], flag: str) -> list[Path]: + """The files the ``--header`` or ``--library`` patterns name: globs expanded, sorted. + + Raises: + InputError: A pattern names no file. + """ + found: set[Path] = set() + for pattern in patterns: + matches = [Path(m) for m in glob.glob(pattern, recursive=True)] + files = [m for m in matches if m.is_file()] + if not files: + raise InputError(f"--{flag} {pattern!r} matches no file") + found.update(files) + return sorted(found) + + +def _library_output(path: Path, *, timeout: float = 120.0) -> str: + """``nm -D --defined-only`` of ``path``, the whole output (a large library lists thousands).""" + try: + done = run(["nm", "-D", "--defined-only", str(path)], timeout=timeout, env=Env.inherit(), + capture_limit=None) + except FileNotFoundError: + raise InputError(f"nm is not installed; cannot read the symbols of {path}") from None + except OSError as exc: + raise InputError(f"nm cannot be run on {path}: {exc.strerror or exc}") from None + if done.timed_out or done.returncode != 0: + lines = done.stderr.strip().splitlines() + reason = lines[-1] if lines else ("timed out" if done.timed_out else f"exit {done.returncode}") + raise InputError(f"nm cannot read {path}: {reason}") + return done.stdout + + +def symbols_check(prefixes: Sequence[str], headers: Sequence[str], + libraries: Sequence[str]) -> dict: + """An ``identifiers`` veto over the prefixed names of ``headers`` and ``libraries``. + + The check's ``within`` and ``pattern`` are :func:`symbol_pattern`, so each name the answer + mentions with a prefix is one scope: a denial or a negation before it makes no claim, as + for a tool's options. ``known`` is the union of every input's names; each input is recorded + with the sha256 of its bytes, as a tool's ``--help`` is. + + Raises: + InputError: A prefix is not valid, a pattern names no file, nm cannot read a library, + or no input has a name with a prefix (a wrong header would veto every name). + """ + prefixes = check_prefixes(prefixes) + known: set[str] = set() + inputs: list[dict] = [] + sources = [("header", _files(headers, "header")), ("library", _files(libraries, "library"))] + for kind, paths in sources: + for path in paths: + if kind == "header": + names = header_names(path.read_text(encoding="utf-8", errors="replace"), prefixes) + else: + names = nm_names(_library_output(path), prefixes) + known |= names + inputs.append({"kind": kind, "path": str(path), "sha256": _sha256(path)}) + if not known: # a wrong header would otherwise veto every name the answer mentions + raise InputError(f"no name with the prefix {', '.join(prefixes)} in the given files") + pattern = symbol_pattern(prefixes) + return { + "type": "identifiers", "tier": "verifiable", "role": "veto", + "pattern": pattern, "within": pattern, + "known": sorted(known), + "source": {"kind": "symbols", "prefixes": list(prefixes), "inputs": inputs}, + } diff --git a/dokimi/tests/test_novelty_checks.py b/dokimi/tests/test_novelty_checks.py index dde09b26e..7516bb68d 100644 --- a/dokimi/tests/test_novelty_checks.py +++ b/dokimi/tests/test_novelty_checks.py @@ -186,8 +186,8 @@ def test_add_checks_targets_atoms_naming_the_tool_and_replaces_its_own(): atoms = [{"statement": "Use `perf report --stdio`.", "answer": "x"}, {"statement": "Unrelated", "answer": "y"}] check = _veto() - assert add_checks(atoms, [check], ["perf report"]) == 1 - assert add_checks(atoms, [check], ["perf report"]) == 1 + assert add_checks(atoms, [check]) == 1 + assert add_checks(atoms, [check]) == 1 assert len(atoms[0]["meta"]["grading"]["checks"]) == 1 assert "meta" not in atoms[1] @@ -279,7 +279,7 @@ def test_a_check_attaches_to_an_atom_that_names_the_tool_without_an_option(): # The answer "perf report" names the tool with no option yet: the veto must still attach, # or a later answer's invented --stdin would go unchecked. atoms = [{"statement": "perf report displays recorded samples", "answer": "perf report"}] - assert add_checks(atoms, [_veto()], ["perf report"]) == 1 + assert add_checks(atoms, [_veto()]) == 1 assert len(atoms[0]["meta"]["grading"]["checks"]) == 1 @@ -492,7 +492,7 @@ def test_novelty_ground_creates_the_output_directory(tmp_path, monkeypatch): monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) out = tmp_path / "new-dir" / "atoms.json" args = argparse.Namespace(atoms=str(atoms_path), out=str(out), tool=["perf report"], - tool_version=None) + tool_version=None, header=None, library=None, prefix=None) novelty_ground.cmd_novelty_ground(args) _, _, data = load_atoms_record(out) assert data["atoms"][0]["meta"]["grading"]["checks"][0]["type"] == "identifiers" @@ -511,7 +511,7 @@ def test_novelty_ground_keeps_the_mode_of_the_atoms_file(tmp_path, monkeypatch): atoms_path.chmod(0o640) monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None) + tool_version=None, header=None, library=None, prefix=None) novelty_ground.cmd_novelty_ground(args) assert atoms_path.stat().st_mode & 0o777 == 0o640 @@ -553,7 +553,7 @@ def failing_write(self, data, *args, **kwargs): monkeypatch.setattr(Path, "write_text", failing_write) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None) + tool_version=None, header=None, library=None, prefix=None) # A failed write is a usage error with a one-line message, not a traceback. with pytest.raises(InputError, match="cannot write"): novelty_ground.cmd_novelty_ground(args) @@ -651,7 +651,7 @@ def test_an_atom_with_null_meta_or_grading_gets_its_check(meta): atoms = [{"id": "a1", "statement": "perf report", "question": "q", "answer": "perf report --stdio", "meta": meta}, {"id": "a2", "statement": "other", "question": "q", "answer": "a", "meta": None}] - assert add_checks(atoms, [check], ["perf report"]) == 1 + assert add_checks(atoms, [check]) == 1 assert atoms[0]["meta"]["grading"]["checks"] == [check] assert atoms[1]["meta"] is None @@ -689,7 +689,7 @@ def test_novelty_ground_refuses_a_malformed_grading_as_probe_does(tmp_path, monk before = atoms_path.read_bytes() monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None) + tool_version=None, header=None, library=None, prefix=None) with pytest.raises(InputError, match="invalid grading"): novelty_ground.cmd_novelty_ground(args) assert atoms_path.read_bytes() == before @@ -698,7 +698,7 @@ def test_novelty_ground_refuses_a_malformed_grading_as_probe_does(tmp_path, monk @pytest.mark.parametrize("step", ["add_checks", "verify_keys"]) def test_each_grounding_step_refuses_a_malformed_grading(step): atoms = [{"id": "a1", "statement": "x", "answer": "y", "meta": {"grading": {"checks": None}}}] - run = {"add_checks": lambda: add_checks(atoms, [_veto()], ["perf report"]), + run = {"add_checks": lambda: add_checks(atoms, [_veto()]), "verify_keys": lambda: verify_keys(atoms)}[step] with pytest.raises(InputError, match="atom a1: invalid grading: grading has no checks"): run() diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py new file mode 100644 index 000000000..6b118734a --- /dev/null +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -0,0 +1,226 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Identifier vetoes from a C library's API: its header declarations and its exported symbols.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +from pathlib import Path + +import pytest + +from dokimi.eval.reference_checks import grade_check, validate_grading +from dokimi.novelty import symbols +from dokimi.novelty.atoms import Atom, Span, load_atoms_record, save_atoms +from dokimi.novelty.grounding import add_checks, identifiers_check, verify_keys +from dokimi.novelty.symbols import header_names, nm_names, symbols_check +from dokimi.utils import InputError + +PREFIXES = ("cblas_", "LAPACKE_", "mkl_") + +CBLAS_H = """ +/* cblas_bogus in a comment declares nothing */ +#include +#define CBLAS_INDEX int +void cblas_dgemm(const int order, const int m, const double *a); +double cblas_ddot(const int n, const double *x, const int incx); +const char *banner = "cblas_dgemmx in a string"; +""" + +LAPACKE_H = "lapack_int LAPACKE_dgesv(int matrix_layout, lapack_int n, double *a);\n" + +NM_OUTPUT = """\ +0000000000001139 T cblas_dgemm@@MKL_1.0 +0000000000001150 T cblas_ddot +0000000000004010 D LAPACKE_dgesv@MKL_1.0 +0000000000000000 A Q +0000000000003000 T other_fn + U cblas_undefined +""" + + +def _headers(tmp_path, *, cblas: str = CBLAS_H, lapacke: str | None = LAPACKE_H): + """A header directory: ``cblas.h`` and, when given, ``lapacke.h``.""" + include = tmp_path / "include" + include.mkdir(exist_ok=True) + (include / "cblas.h").write_text(cblas, encoding="utf-8") + if lapacke is not None: + (include / "lapacke.h").write_text(lapacke, encoding="utf-8") + return include + + +def _check(tmp_path, prefixes=PREFIXES) -> dict: + return symbols_check(list(prefixes), [str(_headers(tmp_path) / "*.h")], []) + + +def test_header_names_are_the_prefixed_names_the_code_declares(): + names = header_names(CBLAS_H + LAPACKE_H, ("cblas_", "LAPACKE_", "mkl_")) + assert names == {"cblas_dgemm", "cblas_ddot", "LAPACKE_dgesv"} + + +def test_a_header_file_name_is_no_identifier_of_the_prefix(): + # ``mkl_types`` in ``#include `` is a file name, not an API name. + assert "mkl_types" not in header_names(CBLAS_H, ("mkl_",)) + + +@pytest.mark.parametrize("answer, passed", [ + ("Call `cblas_dgemm(order, m, a)` first.", True), + ("Call `cblas_dgemmx(order, m, a)` first.", False), # an invented name + ("Use `LAPACKE_dgesvv` for the solve.", False), # the nonexistent LAPACKE name of the study + ("Use `LAPACKE_dgesv` for the solve.", True), + ("Link libmkl_rt.so and include mkl_cblas.h.", True), # not a prefixed name at the start + ("The cblas word alone names no function.", True), + ("Use mkl_dfti_unknown here.", False), # a prefixed name no header declares + ("There is no cblas_dgemmx in this library.", True), # a negated mention is no claim + ("cblas_dgemmx does not exist; use cblas_dgemm.", True), # a denial right after the name + ("cblas_dgemm and cblas_dgemmx both work.", False), +]) +def test_symbol_check_vetoes_only_prefixed_names_no_header_declares(tmp_path, answer, passed): + result = grade_check(_check(tmp_path), "q", answer, None) + assert result.passed is passed + if not passed: + assert result.detail.startswith("not in source: ") + + +def test_a_symbol_check_validates_and_is_never_sufficient(tmp_path): + check = _check(tmp_path) + assert validate_grading({"checks": [check]}) == [] + assert validate_grading({"checks": [{**check, "role": "sufficient"}]}) + + +def test_a_key_naming_a_missing_api_name_is_not_found_and_a_real_one_verified(tmp_path): + check = _check(tmp_path) + atoms = [{"id": "a1", "statement": "cblas_dgemmx computes the product", "question": "q", + "answer": "Call cblas_dgemmx.", "meta": None}, + {"id": "a2", "statement": "cblas_ddot computes a dot product", "question": "q", + "answer": "Call cblas_ddot.", "meta": None}] + assert add_checks(atoms, [check]) == 2 + assert verify_keys(atoms) == {"verified": 1, "not_found": 1, "unverifiable": 0} + assert atoms[0]["meta"]["key_status"] == {"status": "not_found", "missing": ["cblas_dgemmx"]} + assert atoms[1]["meta"]["key_status"] == {"status": "verified"} + + +def test_the_symbol_check_attaches_to_atoms_naming_a_prefix_and_keeps_tool_checks(tmp_path): + tool = identifiers_check("perf report", " --stdio Use the stdio interface\n") + atoms = [{"statement": "Call cblas_ddot, then perf report", "question": "q", "answer": "a"}, + {"statement": "The BLAS library is fast", "question": "q", "answer": "a"}] + assert add_checks(atoms, [tool, _check(tmp_path)]) == 1 + assert add_checks(atoms, [_check(tmp_path)]) == 1 # a re-run replaces its own check + kinds = [c["source"].get("kind", c["source"].get("tool")) + for c in atoms[0]["meta"]["grading"]["checks"]] + assert kinds == ["perf report", "symbols"] + assert "meta" not in atoms[1] + + +def test_each_input_is_pinned_by_the_sha256_of_its_bytes(tmp_path): + check = _check(tmp_path) + inputs = check["source"]["inputs"] + assert [i["kind"] for i in inputs] == ["header", "header"] + header = tmp_path / "include" / "cblas.h" + assert inputs[0]["sha256"] == hashlib.sha256(header.read_bytes()).hexdigest() + changed = symbols_check(["cblas_"], [str(tmp_path / "include" / "cblas.h")], []) + (tmp_path / "include" / "cblas.h").write_text(CBLAS_H + "/* edit */\n", encoding="utf-8") + again = symbols_check(["cblas_"], [str(tmp_path / "include" / "cblas.h")], []) + assert again["source"]["inputs"][0]["sha256"] != changed["source"]["inputs"][0]["sha256"] + + +def test_known_names_are_the_prefixed_names_of_all_inputs(tmp_path): + check = _check(tmp_path) + assert check["known"] == ["LAPACKE_dgesv", "cblas_ddot", "cblas_dgemm"] + + +@pytest.mark.parametrize("prefix", ["mkl", "cblas-", "1_", "", "x y_", "_"]) +def test_a_prefix_must_be_a_c_identifier_ending_in_an_underscore(tmp_path, prefix): + with pytest.raises(InputError, match="prefix"): + symbols_check([prefix], [str(_headers(tmp_path) / "*.h")], []) + + +def test_a_header_glob_expands_to_sorted_files_and_a_pattern_matching_none_is_refused(tmp_path): + check = _check(tmp_path) + names = [Path(i["path"]).name for i in check["source"]["inputs"]] + assert names == ["cblas.h", "lapacke.h"] + with pytest.raises(InputError, match="matches no file"): + symbols_check(["cblas_"], [str(tmp_path / "nowhere" / "*.h")], []) + + +def test_a_header_set_with_no_prefixed_name_is_refused_so_nothing_is_vetoed_by_mistake(tmp_path): + with pytest.raises(InputError, match="no name with the prefix"): + symbols_check(["mkl_"], [str(_headers(tmp_path, lapacke=None) / "cblas.h")], []) + + +def test_nm_names_reads_defined_symbols_and_drops_their_version_tags(): + assert nm_names(NM_OUTPUT, ("cblas_", "LAPACKE_")) == {"cblas_dgemm", "cblas_ddot", + "LAPACKE_dgesv"} + + +def test_library_names_are_the_exported_symbols_of_a_shared_library(tmp_path): + if shutil.which("nm") is None or shutil.which("cc") is None: + pytest.skip("needs nm and cc to build a fixture library") + source = tmp_path / "fixture.c" + source.write_text("int cblas_dgemm(int x) { return x; }\n" + "int cblas_ddot(int x) { return x; }\n" + "int other_fn(int x) { return x; }\n" + "static int cblas_hidden(int x) { return x; }\n", encoding="utf-8") + library = tmp_path / "libfixture.so" + subprocess.run(["cc", "-shared", "-fPIC", "-o", str(library), str(source)], check=True) + check = symbols_check(["cblas_"], [], [str(library)]) + assert check["known"] == ["cblas_ddot", "cblas_dgemm"] + assert check["source"]["inputs"][0]["kind"] == "library" + + +def test_a_missing_nm_is_an_input_error_that_names_it(tmp_path, monkeypatch): + def no_nm(*args, **kwargs): + raise FileNotFoundError("nm") + monkeypatch.setattr(symbols, "run", no_nm) + library = tmp_path / "libfixture.so" + library.write_bytes(b"not read") + with pytest.raises(InputError, match="nm is not installed"): + symbols_check(["cblas_"], [], [str(library)]) + + +def test_a_file_nm_cannot_read_is_an_input_error(tmp_path): + if shutil.which("nm") is None: + pytest.skip("needs nm") + library = tmp_path / "libnot.so" + library.write_text("this is text, not an object file\n", encoding="utf-8") + with pytest.raises(InputError, match="nm cannot read"): + symbols_check(["cblas_"], [], [str(library)]) + + +def _atoms_file(tmp_path, answer="Call cblas_dgemm."): + span = Span(id="s1", kind="prose", text="cblas_dgemm is the product", start=0, end=25) + path = tmp_path / "atoms.json" + save_atoms([Atom(statement="cblas_dgemm is the product", question="How?", answer=answer, + span_ids=("s1",))], [span], path) + return path + + +def test_novelty_ground_writes_the_symbol_check_and_verifies_the_key(tmp_path, capsys): + from dokimi.commands import novelty_ground + atoms_path = _atoms_file(tmp_path) + args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=None, tool_version=None, + header=[str(_headers(tmp_path) / "*.h")], library=[], + prefix=["cblas_", "LAPACKE_"]) + novelty_ground.cmd_novelty_ground(args) + _, _, data = load_atoms_record(atoms_path) + atom = data["atoms"][0] + assert atom["meta"]["grading"]["checks"][0]["source"]["kind"] == "symbols" + assert atom["meta"]["key_status"] == {"status": "verified"} + err = capsys.readouterr().err + assert "symbols" in err and "1 of 1 atoms got a veto check" in err + + +@pytest.mark.parametrize("kwargs, message", [ + ({"tool": None, "header": [], "library": [], "prefix": []}, "--tool, --header or --library"), + ({"tool": None, "header": ["x.h"], "library": [], "prefix": []}, "--prefix"), + ({"tool": None, "header": [], "library": [], "prefix": ["cblas_"]}, "--prefix scopes"), +]) +def test_novelty_ground_refuses_symbol_flags_that_do_not_go_together(tmp_path, kwargs, message): + from dokimi.commands import novelty_ground + args = argparse.Namespace(atoms=str(_atoms_file(tmp_path)), out=None, tool_version=None, + **kwargs) + with pytest.raises(InputError, match=message): + novelty_ground.cmd_novelty_ground(args) From 5b79668f663e93db4a6e932517f76ccd73d4839b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:40:53 -0800 Subject: [PATCH 02/14] fix(novelty): refuse an empty prefix list and an unreadable header as input errors symbols_check([]) built an empty alternation that matches every word of an answer; the CLI refused this already, the function did not. An unreadable --header raised a bare PermissionError, which printed a traceback instead of the one-line error the other input refusals give. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/symbols.py | 12 +++++++++++- dokimi/tests/test_novelty_ground_symbols.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index 842b6d80d..f8656c07e 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -32,6 +32,8 @@ def check_prefixes(prefixes: Sequence[str]) -> tuple[str, ...]: InputError: A prefix is not a C identifier ending in ``_`` (``mkl`` is refused: it would also match ordinary words). """ + if not prefixes: # an empty alternation matches every word of the answer + raise InputError("give at least one --prefix: it says which names are checked") bad = [p for p in prefixes if not _PREFIX.fullmatch(p)] if bad: raise InputError(f"--prefix {bad[0]!r} is not a C identifier prefix ending in '_' " @@ -97,6 +99,14 @@ def _files(patterns: Sequence[str], flag: str) -> list[Path]: return sorted(found) +def _header_text(path: Path) -> str: + """The text of a header; an unreadable one is an input error, not a traceback.""" + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + raise InputError(f"cannot read the header {path}: {exc.strerror or exc}") from None + + def _library_output(path: Path, *, timeout: float = 120.0) -> str: """``nm -D --defined-only`` of ``path``, the whole output (a large library lists thousands).""" try: @@ -133,7 +143,7 @@ def symbols_check(prefixes: Sequence[str], headers: Sequence[str], for kind, paths in sources: for path in paths: if kind == "header": - names = header_names(path.read_text(encoding="utf-8", errors="replace"), prefixes) + names = header_names(_header_text(path), prefixes) else: names = nm_names(_library_output(path), prefixes) known |= names diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 6b118734a..757e72623 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -190,6 +190,21 @@ def test_a_file_nm_cannot_read_is_an_input_error(tmp_path): symbols_check(["cblas_"], [], [str(library)]) +def test_no_prefix_at_all_is_refused_because_an_empty_pattern_names_every_word(tmp_path): + with pytest.raises(InputError, match="prefix"): + symbols_check([], [str(_headers(tmp_path) / "*.h")], []) + + +def test_an_unreadable_header_is_an_input_error_not_a_traceback(tmp_path, monkeypatch): + header = _headers(tmp_path) / "cblas.h" + + def denied(self, *args, **kwargs): + raise PermissionError(13, "Permission denied") + monkeypatch.setattr(Path, "read_text", denied) + with pytest.raises(InputError, match="cannot read the header"): + symbols_check(["cblas_"], [str(header)], []) + + def _atoms_file(tmp_path, answer="Call cblas_dgemm."): span = Span(id="s1", kind="prose", text="cblas_dgemm is the product", start=0, end=25) path = tmp_path / "atoms.json" From 57f29fa8bf135d95abea92d19a5b72345d60abb6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:47:35 -0800 Subject: [PATCH 03/14] fix(novelty): keep a tool named symbols apart from the symbol check; state the refusal per set add_checks keyed a symbol check by the literal "symbols", so a --tool named symbols was deleted on a re-run that added --header or --library. The key is now a tuple. grading-tiers.md now says the refusal applies to the sources together, and that one file with no prefixed name is not reported. Signed-off-by: Nikolay Petrov --- dokimi/docs/grading-tiers.md | 5 +++-- dokimi/dokimi/novelty/grounding.py | 9 ++++++--- dokimi/tests/test_novelty_ground_symbols.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/grading-tiers.md b/dokimi/docs/grading-tiers.md index ecfee9329..029b514cb 100644 --- a/dokimi/docs/grading-tiers.md +++ b/dokimi/docs/grading-tiers.md @@ -79,8 +79,9 @@ A C API has no `--help`, so its check reads names instead. `--prefix` (a C ident `--header` declares it and no `--library` exports it, and an ordinary word is never read. A header's names are every prefixed identifier of its code (comments and strings aside), so a name a macro expands to counts too; a header's `#include` of another header is not followed, so -pass each header. A `--header` or `--library` set with no prefixed name is refused, so a wrong -file cannot veto every name. Key verification works the same way: a key naming `LAPACKE_dgesvv` +pass each header. The sources together are refused when no file in them has a prefixed name, so a wrong +file set cannot veto every name. One file with no prefixed name adds nothing and is not +reported, so a wrong `--library` next to correct headers is not caught. Key verification works the same way: a key naming `LAPACKE_dgesvv` is `not_found`. ```bash diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index 98b93224e..6c9eb267b 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -183,10 +183,13 @@ def _mention_of(check: dict) -> str: return _mention(source["tool"]) -def _source_of(check: dict) -> str: - """The source a check replaces on re-run: its tool's name, or ``symbols`` for a symbol check.""" +def _source_of(check: dict) -> tuple[str, str]: + """The source a check replaces on re-run: ``("symbols", "")`` for a symbol check, else + ``("tool", )``. The tuple keeps a tool named ``symbols`` apart from the symbol check.""" source = check.get("source") or {} - return "symbols" if source.get("kind") == "symbols" else str(source.get("tool")) + if source.get("kind") == "symbols": + return ("symbols", "") + return ("tool", str(source.get("tool"))) def add_checks(atoms: list[dict], checks: list[dict]) -> int: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 757e72623..a9ef1b5c9 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -115,6 +115,17 @@ def test_the_symbol_check_attaches_to_atoms_naming_a_prefix_and_keeps_tool_check assert "meta" not in atoms[1] +def test_a_tool_named_symbols_keeps_its_check_when_the_symbol_check_is_replaced(tmp_path): + tool = identifiers_check("symbols", " --verbose Print more\n") + atoms = [{"statement": "Call cblas_ddot, then run symbols --verbose", "question": "q", + "answer": "a"}] + add_checks(atoms, [tool, _check(tmp_path)]) + add_checks(atoms, [_check(tmp_path)]) # a re-run replaces the symbol check only + sources = [c["source"] for c in atoms[0]["meta"]["grading"]["checks"]] + assert sorted(s.get("kind", "tool:" + s.get("tool", "")) for s in sources) == [ + "symbols", "tool:symbols"] + + def test_each_input_is_pinned_by_the_sha256_of_its_bytes(tmp_path): check = _check(tmp_path) inputs = check["source"]["inputs"] From 7fc13174888d08e7ce0c306e6ece3898bda7a7c9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:57:53 -0800 Subject: [PATCH 04/14] fix(novelty): a symbol check per prefix set, so a re-run with other prefixes keeps the first veto add_checks replaced every symbol check on the atom, so a second ground run with other --prefix values dropped the first run's veto. Symbol checks are now keyed by their prefixes. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/grounding.py | 7 ++++--- dokimi/tests/test_novelty_ground_symbols.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index 6c9eb267b..f28fb7a0a 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -184,11 +184,12 @@ def _mention_of(check: dict) -> str: def _source_of(check: dict) -> tuple[str, str]: - """The source a check replaces on re-run: ``("symbols", "")`` for a symbol check, else - ``("tool", )``. The tuple keeps a tool named ``symbols`` apart from the symbol check.""" + """The source a check replaces on re-run: ``("symbols", )`` for a symbol check, + else ``("tool", )``. The tuple keeps a tool named ``symbols`` apart from the symbol + check, and a run with other prefixes from the one before it.""" source = check.get("source") or {} if source.get("kind") == "symbols": - return ("symbols", "") + return ("symbols", " ".join(source.get("prefixes") or [])) return ("tool", str(source.get("tool"))) diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index a9ef1b5c9..f1fbd27ab 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -126,6 +126,14 @@ def test_a_tool_named_symbols_keeps_its_check_when_the_symbol_check_is_replaced( "symbols", "tool:symbols"] +def test_symbol_checks_with_other_prefixes_both_survive_a_rerun(tmp_path): + atoms = [{"statement": "Call cblas_ddot and LAPACKE_dgesv", "question": "q", "answer": "a"}] + add_checks(atoms, [_check(tmp_path, ("cblas_",))]) + add_checks(atoms, [_check(tmp_path, ("LAPACKE_",))]) + kept = sorted(tuple(c["source"]["prefixes"]) for c in atoms[0]["meta"]["grading"]["checks"]) + assert kept == [("LAPACKE_",), ("cblas_",)] + + def test_each_input_is_pinned_by_the_sha256_of_its_bytes(tmp_path): check = _check(tmp_path) inputs = check["source"]["inputs"] From 74acda766d2747c0cf7178114d82407c59708b75 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:28:48 -0800 Subject: [PATCH 05/14] fix(novelty): read existing header paths, C comment splices, dash-named libraries A header or library path that exists is read as it is, even with glob characters in its name. A `//` comment continued by a backslash ends with its next line, so a name in that line is not declared. A library path starting with `-` reaches nm after `--`. A symbol check replaces any older one that shares a prefix, not only one with the same prefix set, so a stale older check cannot veto a name the newer headers declare. Thread: PRRT_kwDOVBg7g8630sjS Thread: PRRT_kwDOVBg7g8630sjc Thread: PRRT_kwDOVBg7g8630sji Thread: PRRT_kwDOVBg7g8630YKj Review: PRR_kwDOVBg7g88AAAABYKZxaA Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/grounding.py | 24 ++++++++++--- dokimi/dokimi/novelty/symbols.py | 10 ++++-- dokimi/tests/test_novelty_ground_symbols.py | 37 +++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index f28fb7a0a..5eb39af8d 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -193,12 +193,28 @@ def _source_of(check: dict) -> tuple[str, str]: return ("tool", str(source.get("tool"))) +def _replaced_by(old: dict, mine: list[dict]) -> bool: + """Whether a check ``old`` on an atom is replaced by one of ``mine``. + + A tool's check is replaced by the same tool's. A symbol check is replaced by any symbol check + sharing a prefix with it: both would read that prefix's names, from different inputs, so the + older one would veto a name the newer one declares. A disjoint prefix set is kept. + """ + if old.get("type") != "identifiers": + return False + if _source_of(old)[0] == "tool": + return any(_source_of(m) == _source_of(old) for m in mine) + shared = set((old.get("source") or {}).get("prefixes") or []) + return any((m.get("source") or {}).get("kind") == "symbols" + and shared & set(m["source"].get("prefixes") or []) for m in mine) + + def add_checks(atoms: list[dict], checks: list[dict]) -> int: """Attach each check to the atoms that name its tool or prefix; how many atoms changed. An atom that names none gets no check: its answers would rarely carry the tool's command - line, and a check that never fires is noise in the report. A check from the same source - already on the atom is replaced, so re-running refreshes the snapshot. + line, and a check that never fires is noise in the report. A check replaced by a new one + (see :func:`_replaced_by`) is dropped, so re-running refreshes the snapshot. """ _require_valid_gradings(atoms) changed = 0 @@ -209,9 +225,7 @@ def add_checks(atoms: list[dict], checks: list[dict]) -> int: continue meta = atom["meta"] = atom.get("meta") or {} grading = meta["grading"] = meta.get("grading") or {"checks": []} - replaced = {_source_of(m) for m in mine} - kept = [c for c in grading.get("checks", []) - if not (c.get("type") == "identifiers" and _source_of(c) in replaced)] + kept = [c for c in grading.get("checks", []) if not _replaced_by(c, mine)] grading["checks"] = kept + mine changed += 1 return changed diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index f8656c07e..4975957de 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -21,7 +21,7 @@ #: A C identifier prefix: a C identifier that ends in an underscore (``cblas_``, ``LAPACKE_``). _PREFIX = re.compile(r"[A-Za-z_]\w*_") #: Comments and string or character literals of C code: no declaration is named in them. -_C_SKIP = re.compile(r'/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.DOTALL) +_C_SKIP = re.compile(r'/\*.*?\*/|//(?:\\\n|[^\n])*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.DOTALL) _CHUNK = 1 << 20 @@ -91,7 +91,11 @@ def _files(patterns: Sequence[str], flag: str) -> list[Path]: """ found: set[Path] = set() for pattern in patterns: - matches = [Path(m) for m in glob.glob(pattern, recursive=True)] + # A path that exists is read as it is: glob would take ``[1]`` in ``inc[1]/`` for a class. + if Path(pattern).is_file(): + matches = [Path(pattern)] + else: + matches = [Path(m) for m in glob.glob(pattern, recursive=True)] files = [m for m in matches if m.is_file()] if not files: raise InputError(f"--{flag} {pattern!r} matches no file") @@ -110,7 +114,7 @@ def _header_text(path: Path) -> str: def _library_output(path: Path, *, timeout: float = 120.0) -> str: """``nm -D --defined-only`` of ``path``, the whole output (a large library lists thousands).""" try: - done = run(["nm", "-D", "--defined-only", str(path)], timeout=timeout, env=Env.inherit(), + done = run(["nm", "-D", "--defined-only", "--", str(path)], timeout=timeout, env=Env.inherit(), capture_limit=None) except FileNotFoundError: raise InputError(f"nm is not installed; cannot read the symbols of {path}") from None diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index f1fbd27ab..305b7993e 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -258,3 +258,40 @@ def test_novelty_ground_refuses_symbol_flags_that_do_not_go_together(tmp_path, k **kwargs) with pytest.raises(InputError, match=message): novelty_ground.cmd_novelty_ground(args) + + +def test_a_header_path_with_glob_characters_that_exists_is_read(tmp_path): + include = tmp_path / "inc[1]" + include.mkdir() + (include / "cblas.h").write_text(CBLAS_H, encoding="utf-8") + check = symbols_check(["cblas_"], [str(include / "cblas.h")], []) + assert "cblas_dgemm" in check["known"] + + +def test_a_line_comment_continued_by_a_backslash_declares_no_name(): + text = "void cblas_real(void);\n// unavailable APIs: \\\n cblas_fake\n" + assert header_names(text, ("cblas_",)) == {"cblas_real"} + + +def test_a_library_path_starting_with_a_dash_is_read_as_a_file(tmp_path, monkeypatch): + if shutil.which("nm") is None or shutil.which("cc") is None: + pytest.skip("needs nm and cc to build a fixture library") + source = tmp_path / "fixture.c" + source.write_text("int cblas_ddot(int x) { return x; }\n", encoding="utf-8") + subprocess.run(["cc", "-shared", "-fPIC", "-o", str(tmp_path / "-api.so"), str(source)], + check=True) + monkeypatch.chdir(tmp_path) + check = symbols_check(["cblas_"], [], ["./-api.so"]) + assert check["known"] == ["cblas_ddot"] + + +def test_an_overlapping_prefix_set_replaces_the_stale_symbol_check(tmp_path): + old_dir = tmp_path / "old" + old_dir.mkdir() + (old_dir / "cblas.h").write_text("void cblas_ddot(void);\n", encoding="utf-8") + old = symbols_check(["cblas_"], [str(old_dir / "cblas.h")], []) + atoms = [{"statement": "Call cblas_dgemm.", "question": "q", "answer": "a"}] + add_checks(atoms, [old]) + add_checks(atoms, [_check(tmp_path, ("LAPACKE_", "cblas_"))]) + checks = atoms[0]["meta"]["grading"]["checks"] + assert [tuple(c["source"]["prefixes"]) for c in checks] == [("LAPACKE_", "cblas_")] From d04342d74e19bce3aa18c9aa15f4867e541098ca Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:28:48 -0800 Subject: [PATCH 06/14] docs(novelty): say which header names a symbol check reads A macro's replacement list counts as a name the header spells, but a name the preprocessor builds (token pasting) does not, so a library is the workaround. A header with no prefixed name is recorded with its sha256 and gives no warning. Thread: PRRT_kwDOVBg7g8630YKe Thread: PRRT_kwDOVBg7g8630sju Signed-off-by: Nikolay Petrov --- dokimi/docs/grading-tiers.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/grading-tiers.md b/dokimi/docs/grading-tiers.md index 029b514cb..26df0106c 100644 --- a/dokimi/docs/grading-tiers.md +++ b/dokimi/docs/grading-tiers.md @@ -77,11 +77,13 @@ An answer that names no option of the tool passes the veto. A C API has no `--help`, so its check reads names instead. `--prefix` (a C identifier ending in `_`, repeatable) says which names are checked: an answer's `cblas_dgemmx` is vetoed when no `--header` declares it and no `--library` exports it, and an ordinary word is never read. A -header's names are every prefixed identifier of its code (comments and strings aside), so a -name a macro expands to counts too; a header's `#include` of another header is not followed, so -pass each header. The sources together are refused when no file in them has a prefixed name, so a wrong -file set cannot veto every name. One file with no prefixed name adds nothing and is not -reported, so a wrong `--library` next to correct headers is not caught. Key verification works the same way: a key naming `LAPACKE_dgesvv` +header's names are every prefixed identifier its code spells (comments and strings aside), so +a macro's replacement list counts too. A name the preprocessor builds (`cblas_##n`) does not, so +a valid function can be vetoed: supply a `--library` that exports it. A header's `#include` of +another header is not followed, so pass each header. The sources together are refused when no +file in them has a prefixed name, so a wrong file set cannot veto every name. A file with no +prefixed name is recorded with its sha256 and adds no name and no warning, so a wrong `--library` +next to correct headers is not caught. Key verification works the same way: a key naming `LAPACKE_dgesvv` is `not_found`. ```bash From 1d5ef97710711bd09b1d18ffc1dd4330c8be218a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 13:04:10 -0800 Subject: [PATCH 07/14] Address review: symbol checks overlap by prefix, and operator-adjacent API names are checked A nested prefix (mkl_ and mkl_sparse_) now replaces the stale check, and a name after an operator is an API name when it is called or follows an operand; a member, option or path stays excluded. Thread: PRRT_kwDOVBg7g864GSzn Thread: PRRT_kwDOVBg7g864GSzs Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/grounding.py | 11 +++--- dokimi/dokimi/novelty/symbols.py | 7 ++-- dokimi/tests/test_novelty_ground_symbols.py | 38 ++++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index 5eb39af8d..256375158 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -197,16 +197,19 @@ def _replaced_by(old: dict, mine: list[dict]) -> bool: """Whether a check ``old`` on an atom is replaced by one of ``mine``. A tool's check is replaced by the same tool's. A symbol check is replaced by any symbol check - sharing a prefix with it: both would read that prefix's names, from different inputs, so the - older one would veto a name the newer one declares. A disjoint prefix set is kept. + whose prefix starts with one of its prefixes, or that one starts with: both would read the same + names (``mkl_`` and ``mkl_sparse_`` overlap), from different inputs, so the older one would veto + a name the newer one declares. A disjoint prefix set is kept. """ if old.get("type") != "identifiers": return False if _source_of(old)[0] == "tool": return any(_source_of(m) == _source_of(old) for m in mine) - shared = set((old.get("source") or {}).get("prefixes") or []) + old_prefixes = (old.get("source") or {}).get("prefixes") or [] return any((m.get("source") or {}).get("kind") == "symbols" - and shared & set(m["source"].get("prefixes") or []) for m in mine) + and any(a.startswith(b) or b.startswith(a) + for a in old_prefixes for b in m["source"].get("prefixes") or []) + for m in mine) def add_checks(atoms: list[dict], checks: list[dict]) -> int: diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index 4975957de..18bb0ca41 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -44,11 +44,12 @@ def check_prefixes(prefixes: Sequence[str]) -> tuple[str, ...]: def symbol_pattern(prefixes: Sequence[str]) -> str: """A regex for the names with one of ``prefixes`` as C identifiers, not as parts of a word. - A name after ``.``, ``>``, ``/`` or ``-`` is a member, a path or an option, and ``mkl_types`` - in ``mkl_types.h`` is a file name: none of those is an API name. + A name after ``.`` or ``->`` is a member, and ``mkl_types`` in ``mkl_types.h`` is a file name: + neither is an API name. A name after ``/`` is one only when it is called (``a/cblas_x(``), and + a name after ``-`` is one when it is called or follows an operand (``f(x)-cblas_x``). """ alternatives = "|".join(re.escape(p) for p in prefixes) - return rf"(?-])(?:{alternatives})\w+(?!\.?\w)" + return rf"(?)(?:(? set[str]: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 305b7993e..b4339aa44 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -6,6 +6,7 @@ import argparse import hashlib +import re import shutil import subprocess from pathlib import Path @@ -16,7 +17,7 @@ from dokimi.novelty import symbols from dokimi.novelty.atoms import Atom, Span, load_atoms_record, save_atoms from dokimi.novelty.grounding import add_checks, identifiers_check, verify_keys -from dokimi.novelty.symbols import header_names, nm_names, symbols_check +from dokimi.novelty.symbols import header_names, nm_names, symbol_pattern, symbols_check from dokimi.utils import InputError PREFIXES = ("cblas_", "LAPACKE_", "mkl_") @@ -295,3 +296,38 @@ def test_an_overlapping_prefix_set_replaces_the_stale_symbol_check(tmp_path): add_checks(atoms, [_check(tmp_path, ("LAPACKE_", "cblas_"))]) checks = atoms[0]["meta"]["grading"]["checks"] assert [tuple(c["source"]["prefixes"]) for c in checks] == [("LAPACKE_", "cblas_")] + + +def test_a_nested_prefix_refresh_replaces_the_stale_symbol_check(tmp_path): + old_dir = tmp_path / "old" + old_dir.mkdir() + (old_dir / "mkl.h").write_text("void mkl_sparse_old(void);\n", encoding="utf-8") + old = symbols_check(["mkl_"], [str(old_dir / "mkl.h")], []) + atoms = [{"statement": "Call mkl_sparse_new.", "question": "q", "answer": "a"}] + add_checks(atoms, [old]) + new_dir = tmp_path / "new" + new_dir.mkdir() + (new_dir / "mkl.h").write_text("void mkl_sparse_old(void);\nvoid mkl_sparse_new(void);\n", encoding="utf-8") + add_checks(atoms, [symbols_check(["mkl_sparse_"], [str(new_dir / "mkl.h")], [])]) + checks = atoms[0]["meta"]["grading"]["checks"] + assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_sparse_",)] + + +@pytest.mark.parametrize("code", [ + "return cblas_ddot(n,x,1)-cblas_dgemmx(n,x,1);", + "return cblas_ddot(n,x,1)/cblas_dgemmx(n,x,1);", + "if (x>cblas_dgemmx(n,x,1)) return 0;", + "return a/cblas_dgemmx(n,x,1);", +]) +def test_an_operator_before_a_called_api_name_keeps_the_name(code): + assert "cblas_dgemmx" in re.findall(symbol_pattern(("cblas_",)), code) + + +@pytest.mark.parametrize("code, name", [ + ("x = s.cblas_dgemmx;", "cblas_dgemmx"), + ("x = p->cblas_dgemmx;", "cblas_dgemmx"), + ("run --cblas_dgemmx", "cblas_dgemmx"), + ("inc/cblas_api", "cblas_api"), +]) +def test_a_member_option_or_path_is_not_an_api_name(code, name): + assert name not in re.findall(symbol_pattern(("cblas_",)), code) From 9f03f9beaf5eda3d12b389df49c96ac5557af6c6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 14:51:19 -0800 Subject: [PATCH 08/14] Address review: a called API name may have whitespace before its paren An invented call after an operator, written with a space or a newline before its argument list, was not in the key scope, so the veto missed it. Thread: PRRT_kwDOVBg7g864Ix9M Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/symbols.py | 2 +- dokimi/tests/test_novelty_ground_symbols.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index 18bb0ca41..5e8b259b8 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -49,7 +49,7 @@ def symbol_pattern(prefixes: Sequence[str]) -> str: a name after ``-`` is one when it is called or follows an operand (``f(x)-cblas_x``). """ alternatives = "|".join(re.escape(p) for p in prefixes) - return rf"(?)(?:(?)(?:(? set[str]: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index b4339aa44..01098d087 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -318,11 +318,20 @@ def test_a_nested_prefix_refresh_replaces_the_stale_symbol_check(tmp_path): "return cblas_ddot(n,x,1)/cblas_dgemmx(n,x,1);", "if (x>cblas_dgemmx(n,x,1)) return 0;", "return a/cblas_dgemmx(n,x,1);", + "return cblas_ddot(n,x,1)/cblas_dgemmx (n,x,1);", + "return cblas_ddot(n,x,1)/cblas_dgemmx\n(n,x,1);", + "return a/cblas_dgemmx\t(n,x,1);", ]) def test_an_operator_before_a_called_api_name_keeps_the_name(code): assert "cblas_dgemmx" in re.findall(symbol_pattern(("cblas_",)), code) +def test_a_called_api_name_with_whitespace_before_the_paren_is_in_the_key_scope(): + # The bot's case: an invented call after an operator, with a space before its argument list. + from dokimi.novelty.symbols import header_names + assert header_names("return cblas_ddot(n,x,1)/cblas_dgemmx (n,x,1);", ("cblas_",)) == {"cblas_ddot", "cblas_dgemmx"} + + @pytest.mark.parametrize("code, name", [ ("x = s.cblas_dgemmx;", "cblas_dgemmx"), ("x = p->cblas_dgemmx;", "cblas_dgemmx"), From 3e5ed8d0d5153b88be26b69d04dad63d97422fe9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 22:59:19 -0800 Subject: [PATCH 09/14] Address review: keep the broader prefix check on a nested refresh A symbol check was dropped whole when a new symbol check overlapped one of its prefixes, so names in its other scopes (for example mkl_vml_*) were no longer checked. Now a symbol check with a prefix the new check does not cover is kept, and takes the new check's names for the shared scope; names outside that scope stay as they were. A check whose prefixes are all covered by the new check is still replaced, and a kept check left with no name is dropped. Thread: PRRT_kwDOVBg7g864GSzn Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/grounding.py | 52 +++++++++++++---- dokimi/tests/test_novelty_ground_symbols.py | 62 ++++++++++++++++++++- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index 256375158..c64898fca 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -196,20 +196,47 @@ def _source_of(check: dict) -> tuple[str, str]: def _replaced_by(old: dict, mine: list[dict]) -> bool: """Whether a check ``old`` on an atom is replaced by one of ``mine``. - A tool's check is replaced by the same tool's. A symbol check is replaced by any symbol check - whose prefix starts with one of its prefixes, or that one starts with: both would read the same - names (``mkl_`` and ``mkl_sparse_`` overlap), from different inputs, so the older one would veto - a name the newer one declares. A disjoint prefix set is kept. + A tool's check is replaced by the same tool's. A symbol check is replaced by a symbol check + whose prefixes include all of its own: its scope lies inside the new one, so nothing is lost. + A symbol check with a prefix the new one does not cover is kept, and :func:`_merged` carries the + new names into it; a disjoint one is kept as it is. """ if old.get("type") != "identifiers": return False if _source_of(old)[0] == "tool": return any(_source_of(m) == _source_of(old) for m in mine) old_prefixes = (old.get("source") or {}).get("prefixes") or [] - return any((m.get("source") or {}).get("kind") == "symbols" - and any(a.startswith(b) or b.startswith(a) - for a in old_prefixes for b in m["source"].get("prefixes") or []) - for m in mine) + return bool(old_prefixes) and any( + _source_of(m)[0] == "symbols" and all( + any(a.startswith(b) for b in m["source"].get("prefixes") or []) for a in old_prefixes) + for m in mine) + + +def _merged(old: dict, mine: list[dict]) -> dict | None: + """``old`` with the names of ``mine``'s symbol checks merged in, as a new dict (``old`` is not changed). + + Only for a symbol check that overlaps one of ``mine``; None when no name is left. The names with + the new prefixes are replaced by what the new checks read now, so a name no header declares any + more drops out; the names with ``old``'s prefixes that the new checks read are added; the other + names stay. The inputs of the new checks are added to ``old``'s, a path once with the later sha. + """ + symbol = [m for m in mine if _source_of(m)[0] == "symbols"] + if old.get("type") != "identifiers" or _source_of(old)[0] != "symbols" or not symbol: + return old + old_prefixes = tuple((old.get("source") or {}).get("prefixes") or []) + new_prefixes = tuple(p for m in symbol for p in m["source"].get("prefixes") or []) + if not any(a.startswith(b) or b.startswith(a) for a in old_prefixes for b in new_prefixes): + return old + fresh = {n for m in symbol for n in m.get("known") or [] if n.startswith(old_prefixes)} + kept = {n for n in old.get("known") or [] if not n.startswith(new_prefixes)} + if not kept | fresh: # a check with no name would veto every mention of its prefixes + return None + source = dict(old.get("source") or {}) + inputs = {(i["kind"], i["path"]): i for i in source.get("inputs") or []} + for m in symbol: + inputs.update({(i["kind"], i["path"]): i for i in m["source"].get("inputs") or []}) + source["inputs"] = list(inputs.values()) + return dict(old, known=sorted(kept | fresh), source=source) def add_checks(atoms: list[dict], checks: list[dict]) -> int: @@ -217,7 +244,8 @@ def add_checks(atoms: list[dict], checks: list[dict]) -> int: An atom that names none gets no check: its answers would rarely carry the tool's command line, and a check that never fires is noise in the report. A check replaced by a new one - (see :func:`_replaced_by`) is dropped, so re-running refreshes the snapshot. + (see :func:`_replaced_by`) is dropped, so re-running refreshes the snapshot; a kept one has + the new names merged in (see :func:`_merged`). """ _require_valid_gradings(atoms) changed = 0 @@ -228,7 +256,11 @@ def add_checks(atoms: list[dict], checks: list[dict]) -> int: continue meta = atom["meta"] = atom.get("meta") or {} grading = meta["grading"] = meta.get("grading") or {"checks": []} - kept = [c for c in grading.get("checks", []) if not _replaced_by(c, mine)] + kept = [] + for old in grading.get("checks", []): + merged = None if _replaced_by(old, mine) else _merged(old, mine) + if merged is not None: + kept.append(merged) grading["checks"] = kept + mine changed += 1 return changed diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 01098d087..f2055000e 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -298,10 +298,11 @@ def test_an_overlapping_prefix_set_replaces_the_stale_symbol_check(tmp_path): assert [tuple(c["source"]["prefixes"]) for c in checks] == [("LAPACKE_", "cblas_")] -def test_a_nested_prefix_refresh_replaces_the_stale_symbol_check(tmp_path): +def test_a_nested_prefix_refresh_keeps_the_broader_check_for_its_other_scopes(tmp_path): old_dir = tmp_path / "old" old_dir.mkdir() - (old_dir / "mkl.h").write_text("void mkl_sparse_old(void);\n", encoding="utf-8") + (old_dir / "mkl.h").write_text("void mkl_sparse_old(void);\nvoid mkl_vml_add(void);\n" + "void mkl_sparse_gone(void);\n", encoding="utf-8") old = symbols_check(["mkl_"], [str(old_dir / "mkl.h")], []) atoms = [{"statement": "Call mkl_sparse_new.", "question": "q", "answer": "a"}] add_checks(atoms, [old]) @@ -310,7 +311,13 @@ def test_a_nested_prefix_refresh_replaces_the_stale_symbol_check(tmp_path): (new_dir / "mkl.h").write_text("void mkl_sparse_old(void);\nvoid mkl_sparse_new(void);\n", encoding="utf-8") add_checks(atoms, [symbols_check(["mkl_sparse_"], [str(new_dir / "mkl.h")], [])]) checks = atoms[0]["meta"]["grading"]["checks"] - assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_sparse_",)] + assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_",), ("mkl_sparse_",)] + broad = checks[0] + # the unaffected scope keeps its names; the refreshed scope takes the new header's names + assert broad["known"] == ["mkl_sparse_new", "mkl_sparse_old", "mkl_vml_add"] + # no veto on a newly declared name, nor on one only the broader header declared + for answer in ("Call mkl_sparse_new().", "Call mkl_vml_add()."): + assert all(grade_check(c, "q", answer, None).passed for c in checks), answer @pytest.mark.parametrize("code", [ @@ -340,3 +347,52 @@ def test_a_called_api_name_with_whitespace_before_the_paren_is_in_the_key_scope( ]) def test_a_member_option_or_path_is_not_an_api_name(code, name): assert name not in re.findall(symbol_pattern(("cblas_",)), code) + + +def test_a_narrower_old_check_is_replaced_by_the_broader_new_one(tmp_path): + old_dir = tmp_path / "old" + old_dir.mkdir() + (old_dir / "mkl.h").write_text("void mkl_sparse_old(void);\n", encoding="utf-8") + old = symbols_check(["mkl_sparse_"], [str(old_dir / "mkl.h")], []) + atoms = [{"statement": "Call mkl_sparse_old.", "question": "q", "answer": "a"}] + add_checks(atoms, [old]) + new_dir = tmp_path / "new" + new_dir.mkdir() + (new_dir / "mkl.h").write_text("void mkl_sparse_old(void);\nvoid mkl_vml_add(void);\n", encoding="utf-8") + add_checks(atoms, [symbols_check(["mkl_"], [str(new_dir / "mkl.h")], [])]) + checks = atoms[0]["meta"]["grading"]["checks"] + assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_",)] + + +def test_a_refresh_leaves_a_check_shared_with_another_atom_unchanged(tmp_path): + old_dir = tmp_path / "old" + old_dir.mkdir() + (old_dir / "mkl.h").write_text("void mkl_sparse_old(void);\nvoid mkl_vml_add(void);\n", encoding="utf-8") + old = symbols_check(["mkl_"], [str(old_dir / "mkl.h")], []) + before = repr(old) + atoms = [{"statement": "Call mkl_sparse_new.", "question": "q", "answer": "a"}, + {"statement": "Call mkl_vml_add.", "question": "q", "answer": "a"}] + add_checks(atoms, [old]) + new_dir = tmp_path / "new" + new_dir.mkdir() + (new_dir / "mkl.h").write_text("void mkl_sparse_new(void);\n", encoding="utf-8") + new = symbols_check(["mkl_sparse_"], [str(new_dir / "mkl.h")], []) + add_checks([atoms[0]], [new]) # only the first atom names the narrower scope + assert repr(old) == before # the check the second atom still holds is not rewritten + assert atoms[1]["meta"]["grading"]["checks"][0]["known"] == ["mkl_sparse_old", "mkl_vml_add"] + + +def test_a_refresh_that_leaves_an_old_check_with_no_names_drops_it(tmp_path): + old_dir = tmp_path / "old" + old_dir.mkdir() + (old_dir / "mkl.h").write_text("void mkl_sparse_a(void);\n", encoding="utf-8") + old = symbols_check(["mkl_sparse_", "cblas_"], [str(old_dir / "mkl.h")], []) + atoms = [{"statement": "Call mkl_sparse_a.", "question": "q", "answer": "a"}] + add_checks(atoms, [old]) + new_dir = tmp_path / "new" + new_dir.mkdir() + (new_dir / "mkl.h").write_text("void mkl_vml_add(void);\n", encoding="utf-8") + add_checks(atoms, [symbols_check(["mkl_"], [str(new_dir / "mkl.h")], [])]) + checks = atoms[0]["meta"]["grading"]["checks"] + assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_",)] + assert all(c["known"] for c in checks) From 989b9f53c0babd7d7445e5f34f835a0f5f78f4be Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:06:01 -0800 Subject: [PATCH 10/14] Address review: keep the symbol checks sound on refresh and parsing A partial refresh over a same-path overwrite keeps both snapshot hashes for the names that still depend on the old one. A subset-prefix refresh no longer adds a second check over the same names. A member access after -> may have blanks before its name; a name after . is not vetoed, since a sentence's period and a space look the same. A C short character literal is the only literal skipped; an apostrophe in a directive is not. The scan is linear in the answer length, and a clause window cut inside a word negates nothing. --tool-version without --tool is refused. Thread: PRRT_kwDOVBg7g864UA60 Thread: PRRT_kwDOVBg7g864UA67 Thread: PRRT_kwDOVBg7g864aBl- Thread: PRRT_kwDOVBg7g864aBl4 Thread: PRRT_kwDOVBg7g864aBly Thread: PRRT_kwDOVBg7g864aBmJ Review: PRR_kwDOVBg7g88AAAABYelOBQ Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty_ground.py | 6 ++- dokimi/dokimi/eval/reference_checks.py | 10 +++- dokimi/dokimi/novelty/grounding.py | 18 +++++-- dokimi/dokimi/novelty/symbols.py | 12 +++-- dokimi/tests/test_novelty_ground_symbols.py | 52 +++++++++++++++++++++ dokimi/tests/test_reference_checks.py | 6 +++ 6 files changed, 94 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/commands/novelty_ground.py b/dokimi/dokimi/commands/novelty_ground.py index 961d5d884..41c212cea 100644 --- a/dokimi/dokimi/commands/novelty_ground.py +++ b/dokimi/dokimi/commands/novelty_ground.py @@ -15,8 +15,10 @@ def _refuse_bad_sources(tools: list[str], headers: list[str], libraries: list[str], - prefixes: list[str]) -> None: + prefixes: list[str], tool_version: str | None = None) -> None: """Refuse a source set the flags cannot read: ``--prefix`` goes with a header or a library.""" + if tool_version and not tools: + raise InputError("--tool-version names the version of --tool: give --tool with it") if prefixes and not (headers or libraries): raise InputError("--prefix scopes the names of --header and --library: give one of them") if (headers or libraries) and not prefixes: @@ -44,7 +46,7 @@ def cmd_novelty_ground(args: argparse.Namespace) -> None: tools, headers = list(args.tool or []), list(args.header or []) libraries, prefixes = list(args.library or []), list(args.prefix or []) - _refuse_bad_sources(tools, headers, libraries, prefixes) + _refuse_bad_sources(tools, headers, libraries, prefixes, args.tool_version) _, _, data = load_atoms_record(args.atoms) checks = [identifiers_check(tool, help_text(tool), version=args.tool_version) for tool in tools] if headers or libraries: diff --git a/dokimi/dokimi/eval/reference_checks.py b/dokimi/dokimi/eval/reference_checks.py index e8caa9adf..96bd8ffcc 100644 --- a/dokimi/dokimi/eval/reference_checks.py +++ b/dokimi/dokimi/eval/reference_checks.py @@ -432,9 +432,17 @@ def scope_pairs(within: str, answer: str, *, timeout: float) -> list[tuple[str, re.IGNORECASE) +_CLAUSE_WINDOW = 400 # characters before a mention that can make up its clause + + def _negated_before(answer: str, start: int) -> bool: """Whether the clause just before ``start`` negates or warns against what follows it.""" - clause = re.split(r"[.;!?,\n]|\b(?:and|but|then|while)\b", answer[:start], flags=re.I)[-1] + # the clause is read back from a bounded window: a long answer with many mentions stays linear + first = max(0, start - _CLAUSE_WINDOW) + window = answer[first:start] + if first and re.match(r"\w\w", answer[first - 1:first + 1]): # cut inside a word: that partial word is no negation + window = re.sub(r"^\S*", "", window, count=1) + clause = re.split(r"[.;!?,\n]|\b(?:and|but|then|while)\b", window, flags=re.I)[-1] # "not only" and "no problem" affirm: they are no negation of what follows. clause = re.sub(r"(?i)\bnot\s+only\b|\bno\s+problem\b", " ", clause) clause = re.sub(r"(? dict | None: Only for a symbol check that overlaps one of ``mine``; None when no name is left. The names with the new prefixes are replaced by what the new checks read now, so a name no header declares any more drops out; the names with ``old``'s prefixes that the new checks read are added; the other - names stay. The inputs of the new checks are added to ``old``'s, a path once with the later sha. + names stay. The inputs of the new checks are added to ``old``'s; a path overwritten since keeps both + snapshots: every snapshot is kept, so the list grows by one per overwrite of the path (no pruning). """ symbol = [m for m in mine if _source_of(m)[0] == "symbols"] if old.get("type") != "identifiers" or _source_of(old)[0] != "symbols" or not symbol: @@ -232,13 +233,22 @@ def _merged(old: dict, mine: list[dict]) -> dict | None: if not kept | fresh: # a check with no name would veto every mention of its prefixes return None source = dict(old.get("source") or {}) - inputs = {(i["kind"], i["path"]): i for i in source.get("inputs") or []} + inputs = {(i["kind"], i["path"], i.get("sha256")): i for i in source.get("inputs") or []} for m in symbol: - inputs.update({(i["kind"], i["path"]): i for i in m["source"].get("inputs") or []}) + inputs.update({(i["kind"], i["path"], i.get("sha256")): i for i in m["source"].get("inputs") or []}) source["inputs"] = list(inputs.values()) return dict(old, known=sorted(kept | fresh), source=source) +def _absorbed(new: dict, kept: list[dict]) -> bool: + """Whether a kept symbol check already holds every name of ``new`` (a subset-prefix refresh merged it in).""" + if _source_of(new)[0] != "symbols": + return False + new_prefixes = set(new["source"].get("prefixes") or []) + return any(_source_of(k)[0] == "symbols" and set(k["source"].get("prefixes") or []) >= new_prefixes + and set(new.get("known") or []) <= set(k.get("known") or []) for k in kept) + + def add_checks(atoms: list[dict], checks: list[dict]) -> int: """Attach each check to the atoms that name its tool or prefix; how many atoms changed. @@ -261,7 +271,7 @@ def add_checks(atoms: list[dict], checks: list[dict]) -> int: merged = None if _replaced_by(old, mine) else _merged(old, mine) if merged is not None: kept.append(merged) - grading["checks"] = kept + mine + grading["checks"] = kept + [m for m in mine if not _absorbed(m, kept)] changed += 1 return changed diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index 5e8b259b8..c0629ae9e 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -21,7 +21,7 @@ #: A C identifier prefix: a C identifier that ends in an underscore (``cblas_``, ``LAPACKE_``). _PREFIX = re.compile(r"[A-Za-z_]\w*_") #: Comments and string or character literals of C code: no declaration is named in them. -_C_SKIP = re.compile(r'/\*.*?\*/|//(?:\\\n|[^\n])*|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', re.DOTALL) +_C_SKIP = re.compile(r'/\*.*?\*/|//(?:\\\n|[^\n])*|"(?:\\.|[^"\\\n])*"|\'(?:\\(?:x[0-9A-Fa-f]+|[0-7]{1,3}|.)|[^\'\\\n])\'', re.DOTALL) _CHUNK = 1 << 20 @@ -41,15 +41,21 @@ def check_prefixes(prefixes: Sequence[str]) -> tuple[str, ...]: return tuple(sorted(set(prefixes))) +_MEMBER_GAP = 8 # the blanks a member access may have between its `->` and the name + + def symbol_pattern(prefixes: Sequence[str]) -> str: """A regex for the names with one of ``prefixes`` as C identifiers, not as parts of a word. - A name after ``.`` or ``->`` is a member, and ``mkl_types`` in ``mkl_types.h`` is a file name: + A name after ``.`` or after ``->`` (with up to 8 blanks) is a member, and ``mkl_types`` in ``mkl_types.h`` is a file name: neither is an API name. A name after ``/`` is one only when it is called (``a/cblas_x(``), and a name after ``-`` is one when it is called or follows an operand (``f(x)-cblas_x``). """ alternatives = "|".join(re.escape(p) for p in prefixes) - return rf"(?)(?:(?` may be followed by up to _MEMBER_GAP blanks (not newlines): one fixed-width lookbehind each. + # a name after `. ` stays an API name: a sentence's period and a space look the same. + member = "".join(rf"(?[ \t]{{{n}}})" for n in range(_MEMBER_GAP + 1)) + return rf"(? set[str]: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index f2055000e..62308b2d0 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -320,6 +320,18 @@ def test_a_nested_prefix_refresh_keeps_the_broader_check_for_its_other_scopes(tm assert all(grade_check(c, "q", answer, None).passed for c in checks), answer +def test_a_same_path_overwrite_keeps_the_old_snapshot_too(tmp_path): + header = tmp_path / "mkl.h" + header.write_text("void mkl_sparse_old(void);\nvoid mkl_vml_add(void);\n", encoding="utf-8") + atoms = [{"statement": "Call mkl_sparse_new.", "question": "q", "answer": "a"}] + add_checks(atoms, [symbols_check(["mkl_"], [str(header)], [])]) + header.write_text("void mkl_sparse_new(void);\n", encoding="utf-8") # the same path, overwritten + add_checks(atoms, [symbols_check(["mkl_sparse_"], [str(header)], [])]) + broad = atoms[0]["meta"]["grading"]["checks"][0] + assert "mkl_vml_add" in broad["known"] # kept: its only pinned snapshot is the old header + # both snapshots of the path stay pinned while a retained name depends on the old one + assert len({i["sha256"] for i in broad["source"]["inputs"] if i["path"] == str(header)}) == 2 + @pytest.mark.parametrize("code", [ "return cblas_ddot(n,x,1)-cblas_dgemmx(n,x,1);", "return cblas_ddot(n,x,1)/cblas_dgemmx(n,x,1);", @@ -342,6 +354,7 @@ def test_a_called_api_name_with_whitespace_before_the_paren_is_in_the_key_scope( @pytest.mark.parametrize("code, name", [ ("x = s.cblas_dgemmx;", "cblas_dgemmx"), ("x = p->cblas_dgemmx;", "cblas_dgemmx"), + ("x = p-> cblas_dgemmx;", "cblas_dgemmx"), ("run --cblas_dgemmx", "cblas_dgemmx"), ("inc/cblas_api", "cblas_api"), ]) @@ -349,6 +362,11 @@ def test_a_member_option_or_path_is_not_an_api_name(code, name): assert name not in re.findall(symbol_pattern(("cblas_",)), code) +@pytest.mark.parametrize("text", ["Use the API. cblas_dgemmx handles it.", "Done.\ncblas_dgemmx(a);", "1. cblas_dgemmx"]) +def test_a_name_after_a_period_in_prose_is_an_api_name(text): + assert "cblas_dgemmx" in re.findall(symbol_pattern(("cblas_",)), text) + + def test_a_narrower_old_check_is_replaced_by_the_broader_new_one(tmp_path): old_dir = tmp_path / "old" old_dir.mkdir() @@ -396,3 +414,37 @@ def test_a_refresh_that_leaves_an_old_check_with_no_names_drops_it(tmp_path): checks = atoms[0]["meta"]["grading"]["checks"] assert [tuple(c["source"]["prefixes"]) for c in checks] == [("mkl_",)] assert all(c["known"] for c in checks) + + +def test_an_apostrophe_in_a_directive_text_does_not_hide_the_names_between_two_of_them(): + # Two apostrophes on one line are not a character literal: the names between them are the header's. + from dokimi.novelty.symbols import header_names + text = "#error Don't call cblas_dgemm_batch directly; include mkl's header\n" + assert "cblas_dgemm_batch" in header_names(text, ("cblas_",)) + assert header_names("char c = '\\n'; int cblas_dgemm;", ("cblas_",)) == {"cblas_dgemm"} + + +def test_a_long_answer_with_many_mentions_is_scanned_in_linear_time(): + # Every mention used to re-read the whole text before it: 126 KB took about 12 s, past the grader's 1 s. + import time + from dokimi.eval.reference_checks import scope_pairs + unit = "Then call cblas_dgemm(a, b) on the block. " + answer = (unit * 3000)[:126_000] + started = time.monotonic() + assert len(scope_pairs(symbol_pattern(("cblas_",)), answer, timeout=30)) > 2000 + assert time.monotonic() - started < 5.0 # a quadratic scan takes 11 s here; the box is shared + + +def test_a_refresh_with_a_subset_of_the_prefixes_does_not_veto_the_same_names_twice(tmp_path): + atoms = [{"statement": "Call cblas_dgemm and LAPACKE_dgesv.", "question": "q", "answer": "a"}] + add_checks(atoms, [symbols_check(["cblas_", "LAPACKE_"], [str(_headers(tmp_path) / "*.h")], [])]) + add_checks(atoms, [symbols_check(["cblas_"], [str(_headers(tmp_path) / "*.h")], [])]) + checks = atoms[0]["meta"]["grading"]["checks"] + assert [sorted(c["source"]["prefixes"]) for c in checks] == [["LAPACKE_", "cblas_"]] # one check, not two + assert "cblas_dgemm" in checks[0]["known"] and "LAPACKE_dgesv" in checks[0]["known"] + + +def test_a_tool_version_without_a_tool_is_refused_not_dropped(): + from dokimi.commands.novelty_ground import _refuse_bad_sources + with pytest.raises(InputError, match="--tool-version"): + _refuse_bad_sources([], ["cblas.h"], [], ["cblas_"], "2025.1") diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index d3ea08045..8cec783d5 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -524,3 +524,9 @@ def test_a_reference_cell_carries_its_answer_error_as_an_arm_cell_does(tmp_path) assert scores["skill"]["error"] == "generation_failed: HTTP 500" assert "error" not in scores["baseline"] save_artifact(kind, payload, tmp_path / "arms.json") # validates arms.v1 with the error on the cell + + +def test_a_window_cut_inside_a_word_makes_no_negation(): + # The window starts at "not zzz…": the clipped word "not" must not negate the mention after it. + answer = "can" + "not " + "z" * 395 + " mkl" + assert reference_checks._negated_before(answer, answer.index("mkl")) is False From 06912109d57ed918e20c549df90d7240e65c7b76 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:32:37 -0800 Subject: [PATCH 11/14] Address review: --shared-lib names the library; ground no longer takes --library The shared library of skill novelty ground gets its own concept (shared_library, next to api_header) and the spelling --shared-lib PATH. --library is not on the ground command any more, so the inputs concept's ban needs no allowlist row for it. Thread: PRRT_kwDOVBg7g864aBlm Signed-off-by: Nikolay Petrov --- ...000_napetrov_novelty-ground-c-api-names.md | 4 ++-- dokimi/docs/answer-formats.md | 2 +- dokimi/docs/grading-tiers.md | 10 +++++----- dokimi/docs/reference/cli-interface.json | 2 +- dokimi/docs/skill-novelty.md | 2 +- dokimi/dokimi/cli_flags.py | 9 ++++++++- dokimi/dokimi/commands/novelty_ground.py | 10 +++++----- dokimi/dokimi/commands/skill.py | 4 ++-- dokimi/tests/test_novelty_checks.py | 8 ++++---- dokimi/tests/test_novelty_ground_symbols.py | 19 +++++++++++++++---- 10 files changed, 44 insertions(+), 26 deletions(-) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md b/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md index eea927033..58fae8b12 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_novelty-ground-c-api-names.md @@ -1,9 +1,9 @@ ### Added -- **`skill novelty ground` vetoes C API names** — `--header PATH` (glob ok) and `--library PATH` +- **`skill novelty ground` vetoes C API names** — `--header PATH` (glob ok) and `--shared-lib PATH` (a shared library, read with `nm -D --defined-only`), each repeatable, with `--prefix cblas_` (a C identifier ending in `_`, repeatable). An answer naming a prefixed name no header declares and no library exports (`cblas_dgemmx`) is vetoed without a judge, and `meta.key_status` is `not_found` for a key that names one. Each input is pinned by its sha256. A `--header` or - `--library` set with no prefixed name is refused, and `--prefix` needs one of them. No MKL + `--shared-lib` set with no prefixed name is refused, and `--prefix` needs one of them. No MKL install is needed: any header set or shared library of the API works. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 254ac3e0d..995c8b7c2 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -60,7 +60,7 @@ question is `` when the config has one prompt and | `json` | Parses as JSON. With `values`, it must equal one of them. With `contains`, the answer may be any JSON value, or else it is the first JSON object or array anywhere in the answer; the scan tries the first 1000 `{` or `[` and stops once the tries that failed have scanned 1,000,000 characters in all (a try that fails on an overlong number, the depth limit or an unterminated string counts as scanning to the end of the text), so a value that starts after more brackets or more failed text than that reads as not JSON. | OpenAI `JsonMatch`; promptfoo `is-json`, `contains-json`, `equals` with an object value | | `judge` | One required point (`criterion`, `reference` or both), asked of the [checklist](benchmarking-and-comparison.md) judge in both polarities. 1 when the two agree that it is covered, 0 when they agree that it is not, 0.5 when they split. | Inspect `model_graded_qa`/`model_graded_fact`; a custom Inspect `@scorer`, on its docstring's first paragraph; OpenAI `ModelBasedClassify` specs; promptfoo `llm-rubric`, `g-eval`, `factuality`, `model-graded-closedqa`; every skill-creator expectation | | `unsupported` | Listed with the native name and never counted | promptfoo `javascript`, `python`, `similar`, `cost`, `latency`, negated judge asserts (`not-llm-rubric`) and other asserts; Inspect `pattern` (it compares an extracted group with the target) and Inspect scorers that are neither built in nor a `@scorer` of the file with a docstring; a row whose `ideal` or `target` is blank or missing | -| `identifiers` | Every long option the answer names (`--x`, or the `pattern`, inside `within`) is in `known`; with `skill novelty ground --header/--library`, every name with a `--prefix` prefix is. An answer that names no option passes, since the check only rules names out. Not `sufficient`: a `sufficient` identifiers check is refused. | None: built from a tool's `--help` by `skill novelty ground`, for novelty probes | +| `identifiers` | Every long option the answer names (`--x`, or the `pattern`, inside `within`) is in `known`; with `skill novelty ground --header/--shared-lib`, every name with a `--prefix` prefix is. An answer that names no option passes, since the check only rules names out. Not `sufficient`: a `sufficient` identifiers check is refused. | None: built from a tool's `--help` by `skill novelty ground`, for novelty probes | `role` (`veto`, `sufficient`, `evidence`) and `tier` are read by novelty probes only, as [grading-tiers](grading-tiers.md) describes. `eval score reference` does not use `role` to score, diff --git a/dokimi/docs/grading-tiers.md b/dokimi/docs/grading-tiers.md index 26df0106c..bfe8e1633 100644 --- a/dokimi/docs/grading-tiers.md +++ b/dokimi/docs/grading-tiers.md @@ -67,7 +67,7 @@ with the docs, not knowledge of the skill) and is still a model grading free tex | Roles: a failed `veto` gives `wrong_other`; a `sufficient` check decides; `evidence` is recorded only; an abstention is never overruled into a pass | `measure.py` `resolve_verdict` | | `judge_verdict`, `verdict_source`, `checks` on each checked run; a record-level `checks` block (`checked_runs`, `overruled`, `vetoed`, `judge_passed_vetoed`, `decided`, `judge_agreed`) | `measure.py` `ProbeRun`, `MeasureResult.checks_summary`, `report.py` | | `identifiers` check: every identifier the answer names (long options by default, inside `within`) is in `known`; `--no-x` counts when `--x` is known and the check sets `negations: true` or the help lists `--[no-]x`. Two words and an option after a command read as another command, so `and add --x` after it is not checked. `role` and `tier` validated on every check | `dokimi/eval/reference_checks.py` | -| `skill novelty ground --atoms A --tool "perf report" [--tool ...]`: runs the tool's `--help`, writes an `identifiers` veto with the help's sha256 into the atoms that name the tool. `--header PATH` (glob ok) and `--library PATH` with `--prefix cblas_` (repeatable) do the same for a C API: the prefixed names a header declares, and the exported symbols of a shared library (`nm -D --defined-only`), each input pinned by its sha256 | `dokimi/novelty/grounding.py`, `novelty/symbols.py`, `commands/novelty_ground.py` | +| `skill novelty ground --atoms A --tool "perf report" [--tool ...]`: runs the tool's `--help`, writes an `identifiers` veto with the help's sha256 into the atoms that name the tool. `--header PATH` (glob ok) and `--shared-lib PATH` with `--prefix cblas_` (repeatable) do the same for a C API: the prefixed names a header declares, and the exported symbols of a shared library (`nm -D --defined-only`), each input pinned by its sha256 | `dokimi/novelty/grounding.py`, `novelty/symbols.py`, `commands/novelty_ground.py` | | Key verification: the same command writes `meta.key_status` on each checked atom (`verified`, `not_found` with the `missing` names, `unverifiable` when the key names no option or one of its checks could not finish) and prints the counts. A `not_found` is a review flag, not an error | `grounding.py` `verify_keys` | A `--tool` check applies only where the tool's name is followed directly by an option, so a @@ -76,19 +76,19 @@ An answer that names no option of the tool passes the veto. A C API has no `--help`, so its check reads names instead. `--prefix` (a C identifier ending in `_`, repeatable) says which names are checked: an answer's `cblas_dgemmx` is vetoed when no -`--header` declares it and no `--library` exports it, and an ordinary word is never read. A +`--header` declares it and no `--shared-lib` exports it, and an ordinary word is never read. A header's names are every prefixed identifier its code spells (comments and strings aside), so a macro's replacement list counts too. A name the preprocessor builds (`cblas_##n`) does not, so -a valid function can be vetoed: supply a `--library` that exports it. A header's `#include` of +a valid function can be vetoed: supply a `--shared-lib` that exports it. A header's `#include` of another header is not followed, so pass each header. The sources together are refused when no file in them has a prefixed name, so a wrong file set cannot veto every name. A file with no -prefixed name is recorded with its sha256 and adds no name and no warning, so a wrong `--library` +prefixed name is recorded with its sha256 and adds no name and no warning, so a wrong `--shared-lib` next to correct headers is not caught. Key verification works the same way: a key naming `LAPACKE_dgesvv` is `not_found`. ```bash python cli.py skill novelty ground --atoms atoms.json --tool "perf report" --tool "perf record" --tool-version 6.17 -python cli.py skill novelty ground --atoms atoms.json --header "include/mkl_*.h" --library libmkl_rt.so --prefix cblas_ --prefix LAPACKE_ +python cli.py skill novelty ground --atoms atoms.json --header "include/mkl_*.h" --shared-lib libmkl_rt.so --prefix cblas_ --prefix LAPACKE_ python cli.py skill novelty probe --artifact skill:linux-perf --atoms atoms.json --out probes.json ``` diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index f6722f3fc..b461b6a3e 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1689,9 +1689,9 @@ {"key": "--atoms", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--header", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, - {"key": "--library", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--out", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--prefix", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--shared-lib", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--tool", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--tool-version", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} ], diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index f3508070e..19c9bba2e 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -999,7 +999,7 @@ python cli.py skill novelty items --probes m1.json m2.json m3.json [reference checks](answer-formats.md); every probe built from it except the reversal gets it, and its non-judge checks run on each answer next to the judge. `skill novelty ground --atoms atoms.json --tool "perf report"` writes one such check: a veto that fails an answer naming a -`perf report --flag` the tool's own `--help` does not list. With `--header` and `--library` and a +`perf report --flag` the tool's own `--help` does not list. With `--header` and `--shared-lib` and a `--prefix` (`cblas_`), it vetoes a C API name no header declares and no library exports. The judge's verdict is kept on each checked run and the record's `checks` block counts how often the judge passed an answer a veto failed. Design and evidence: [grading-tiers.md](grading-tiers.md). diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index 389bfae17..39e19864b 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -295,7 +295,14 @@ class Concept: help="a C header whose declared names a veto may check", canonical=(Flag("--header", "PATH", "path", repeatable=True, meaning="a C header of the API (glob ok)"),), - note="Used with --prefix (and --library) by skill novelty ground.", + note="Used with --prefix (and --shared-lib) by skill novelty ground.", + ), + Concept( + name="shared_library", + help="a shared library whose exported symbols a veto may check", + canonical=(Flag("--shared-lib", "PATH", "path", repeatable=True, + meaning="a shared library whose exported (nm -D) names are known"),), + note="Used with --prefix (and --header) by skill novelty ground.", ), Concept( name="symbol_prefix", diff --git a/dokimi/dokimi/commands/novelty_ground.py b/dokimi/dokimi/commands/novelty_ground.py index 41c212cea..4a8726a95 100644 --- a/dokimi/dokimi/commands/novelty_ground.py +++ b/dokimi/dokimi/commands/novelty_ground.py @@ -16,15 +16,15 @@ def _refuse_bad_sources(tools: list[str], headers: list[str], libraries: list[str], prefixes: list[str], tool_version: str | None = None) -> None: - """Refuse a source set the flags cannot read: ``--prefix`` goes with a header or a library.""" + """Refuse a source set the flags cannot read: ``--prefix`` goes with a header or a shared library.""" if tool_version and not tools: raise InputError("--tool-version names the version of --tool: give --tool with it") if prefixes and not (headers or libraries): - raise InputError("--prefix scopes the names of --header and --library: give one of them") + raise InputError("--prefix scopes the names of --header and --shared-lib: give one of them") if (headers or libraries) and not prefixes: - raise InputError("--header and --library need --prefix: it says which names are checked") + raise InputError("--header and --shared-lib need --prefix: it says which names are checked") if not (tools or headers or libraries): - raise InputError("give --tool, --header or --library: nothing to ground the atoms on") + raise InputError("give --tool, --header or --shared-lib: nothing to ground the atoms on") def _describe(check: dict) -> str: @@ -45,7 +45,7 @@ def cmd_novelty_ground(args: argparse.Namespace) -> None: from dokimi.novelty.symbols import symbols_check tools, headers = list(args.tool or []), list(args.header or []) - libraries, prefixes = list(args.library or []), list(args.prefix or []) + libraries, prefixes = list(args.shared_lib or []), list(args.prefix or []) _refuse_bad_sources(tools, headers, libraries, prefixes, args.tool_version) _, _, data = load_atoms_record(args.atoms) checks = [identifiers_check(tool, help_text(tool), version=args.tool_version) for tool in tools] diff --git a/dokimi/dokimi/commands/skill.py b/dokimi/dokimi/commands/skill.py index 044fb3281..103b0aee5 100644 --- a/dokimi/dokimi/commands/skill.py +++ b/dokimi/dokimi/commands/skill.py @@ -149,12 +149,12 @@ def register_skill(skill_sub, positive_int) -> None: ground_p.add_argument("--header", action="append", default=None, metavar="PATH", help="C header whose API names are known (glob ok). Repeatable; " "needs --prefix") - ground_p.add_argument("--library", action="append", default=None, metavar="PATH", + ground_p.add_argument("--shared-lib", action="append", default=None, dest="shared_lib", metavar="PATH", help="Shared library whose exported symbols (nm -D) are known. " "Repeatable; needs --prefix") ground_p.add_argument("--prefix", action="append", default=None, metavar="PREFIX", help="C identifier prefix ending in '_' (e.g. cblas_) that scopes " - "--header and --library: only names with it are checked. Repeatable") + "--header and --shared-lib: only names with it are checked. Repeatable") ground_p.add_argument("--out", default=None, help="Default: rewrite --atoms in place") from dokimi.commands.novelty_ground import cmd_novelty_ground diff --git a/dokimi/tests/test_novelty_checks.py b/dokimi/tests/test_novelty_checks.py index 7516bb68d..6694a1ee5 100644 --- a/dokimi/tests/test_novelty_checks.py +++ b/dokimi/tests/test_novelty_checks.py @@ -492,7 +492,7 @@ def test_novelty_ground_creates_the_output_directory(tmp_path, monkeypatch): monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) out = tmp_path / "new-dir" / "atoms.json" args = argparse.Namespace(atoms=str(atoms_path), out=str(out), tool=["perf report"], - tool_version=None, header=None, library=None, prefix=None) + tool_version=None, header=None, shared_lib=None, prefix=None) novelty_ground.cmd_novelty_ground(args) _, _, data = load_atoms_record(out) assert data["atoms"][0]["meta"]["grading"]["checks"][0]["type"] == "identifiers" @@ -511,7 +511,7 @@ def test_novelty_ground_keeps_the_mode_of_the_atoms_file(tmp_path, monkeypatch): atoms_path.chmod(0o640) monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None, header=None, library=None, prefix=None) + tool_version=None, header=None, shared_lib=None, prefix=None) novelty_ground.cmd_novelty_ground(args) assert atoms_path.stat().st_mode & 0o777 == 0o640 @@ -553,7 +553,7 @@ def failing_write(self, data, *args, **kwargs): monkeypatch.setattr(Path, "write_text", failing_write) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None, header=None, library=None, prefix=None) + tool_version=None, header=None, shared_lib=None, prefix=None) # A failed write is a usage error with a one-line message, not a traceback. with pytest.raises(InputError, match="cannot write"): novelty_ground.cmd_novelty_ground(args) @@ -689,7 +689,7 @@ def test_novelty_ground_refuses_a_malformed_grading_as_probe_does(tmp_path, monk before = atoms_path.read_bytes() monkeypatch.setattr(grounding, "help_text", lambda tool, **k: PERF_HELP) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=["perf report"], - tool_version=None, header=None, library=None, prefix=None) + tool_version=None, header=None, shared_lib=None, prefix=None) with pytest.raises(InputError, match="invalid grading"): novelty_ground.cmd_novelty_ground(args) assert atoms_path.read_bytes() == before diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 62308b2d0..edd5bbbb4 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -237,7 +237,7 @@ def test_novelty_ground_writes_the_symbol_check_and_verifies_the_key(tmp_path, c from dokimi.commands import novelty_ground atoms_path = _atoms_file(tmp_path) args = argparse.Namespace(atoms=str(atoms_path), out=None, tool=None, tool_version=None, - header=[str(_headers(tmp_path) / "*.h")], library=[], + header=[str(_headers(tmp_path) / "*.h")], shared_lib=[], prefix=["cblas_", "LAPACKE_"]) novelty_ground.cmd_novelty_ground(args) _, _, data = load_atoms_record(atoms_path) @@ -249,9 +249,9 @@ def test_novelty_ground_writes_the_symbol_check_and_verifies_the_key(tmp_path, c @pytest.mark.parametrize("kwargs, message", [ - ({"tool": None, "header": [], "library": [], "prefix": []}, "--tool, --header or --library"), - ({"tool": None, "header": ["x.h"], "library": [], "prefix": []}, "--prefix"), - ({"tool": None, "header": [], "library": [], "prefix": ["cblas_"]}, "--prefix scopes"), + ({"tool": None, "header": [], "shared_lib": [], "prefix": []}, "--tool, --header or --shared-lib"), + ({"tool": None, "header": ["x.h"], "shared_lib": [], "prefix": []}, "--prefix"), + ({"tool": None, "header": [], "shared_lib": [], "prefix": ["cblas_"]}, "--prefix scopes"), ]) def test_novelty_ground_refuses_symbol_flags_that_do_not_go_together(tmp_path, kwargs, message): from dokimi.commands import novelty_ground @@ -261,6 +261,17 @@ def test_novelty_ground_refuses_symbol_flags_that_do_not_go_together(tmp_path, k novelty_ground.cmd_novelty_ground(args) +def test_the_shared_library_flag_is_shared_lib_and_library_is_refused(): + from dokimi import cli + parser = cli.build_parser() + args = parser.parse_args(["skill", "novelty", "ground", "--atoms", "a.json", "--shared-lib", "libx.so", + "--prefix", "cblas_"]) + assert args.shared_lib == ["libx.so"] + with pytest.raises(SystemExit): + parser.parse_args(["skill", "novelty", "ground", "--atoms", "a.json", "--library", "libx.so", + "--prefix", "cblas_"]) + + def test_a_header_path_with_glob_characters_that_exists_is_read(tmp_path): include = tmp_path / "inc[1]" include.mkdir() From 1f10991a227c0b2ee2263df0396b0038786d6b5c Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Wed, 18 Nov 2026 00:25:54 +0000 Subject: [PATCH 12/14] fix(novelty): read a name after a spaced minus as an API name `Return x -cblas_fake;` passed: the minus lookbehind in symbol_pattern read the character before the dash, so a space there excluded the name. A blank before the dash now counts as an operand, as in `x - cblas_fake`. Signed-off-by: vchamarthi --- dokimi/dokimi/novelty/symbols.py | 4 ++-- dokimi/tests/test_novelty_ground_symbols.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index c0629ae9e..e9ef65ceb 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -49,13 +49,13 @@ def symbol_pattern(prefixes: Sequence[str]) -> str: A name after ``.`` or after ``->`` (with up to 8 blanks) is a member, and ``mkl_types`` in ``mkl_types.h`` is a file name: neither is an API name. A name after ``/`` is one only when it is called (``a/cblas_x(``), and - a name after ``-`` is one when it is called or follows an operand (``f(x)-cblas_x``). + a name after ``-`` is one when it is called or follows an operand (``f(x)-cblas_x``, ``x -cblas_x``). """ alternatives = "|".join(re.escape(p) for p in prefixes) # `->` may be followed by up to _MEMBER_GAP blanks (not newlines): one fixed-width lookbehind each. # a name after `. ` stays an API name: a sentence's period and a space look the same. member = "".join(rf"(?[ \t]{{{n}}})" for n in range(_MEMBER_GAP + 1)) - return rf"(? set[str]: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index edd5bbbb4..5e66db814 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -351,6 +351,8 @@ def test_a_same_path_overwrite_keeps_the_old_snapshot_too(tmp_path): "return cblas_ddot(n,x,1)/cblas_dgemmx (n,x,1);", "return cblas_ddot(n,x,1)/cblas_dgemmx\n(n,x,1);", "return a/cblas_dgemmx\t(n,x,1);", + "Return x -cblas_dgemmx;", + "Return f(x) -cblas_dgemmx;", ]) def test_an_operator_before_a_called_api_name_keeps_the_name(code): assert "cblas_dgemmx" in re.findall(symbol_pattern(("cblas_",)), code) From 32f03a7a9c621d435dae88c6f135bffe11108a6f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 17:32:23 -0800 Subject: [PATCH 13/14] fix(novelty): a link context is no veto under a shared prefix With --prefix mkl_, a library name in a link sentence (`Link with mkl_core`, `target_link_libraries(app MKL::mkl_core)`, `Use the mkl_rt single dynamic library`) was vetoed as "not in source": library names share the prefix but export no symbol of their own. A name after -l or ::, or in a sentence about linking or a library, is no longer vetoed; a called name (x( ) still is, and a library file's stem (mkl_rt for libmkl_rt.so) is a known name. Signed-off-by: Nikolay Petrov --- ...00_napetrov_novelty-ground-link-context.md | 13 +++++ dokimi/docs/grading-tiers.md | 4 +- dokimi/dokimi/eval/reference_checks.py | 34 ++++++++++++-- dokimi/dokimi/novelty/symbols.py | 14 +++++- dokimi/tests/test_novelty_ground_symbols.py | 47 +++++++++++++++++++ 5 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md diff --git a/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md b/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md new file mode 100644 index 000000000..1a5d58bc7 --- /dev/null +++ b/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md @@ -0,0 +1,13 @@ + + +### Fixed + +- **`skill novelty ground --prefix mkl_` no longer vetoes library names in a link context** — `Link with + mkl_intel_lp64, mkl_sequential and mkl_core.`, `Use the mkl_rt single dynamic library.` and + `target_link_libraries(app MKL::mkl_core)` were refused as "not in source". A name after `-l` or `::`, + or in a sentence about linking or a library, is not vetoed, and a library file's stem (`mkl_rt` for + `libmkl_rt.so`) is a known name. A called name (`mkl_x(`) and a name no header or library declares + elsewhere are still vetoed. diff --git a/dokimi/docs/grading-tiers.md b/dokimi/docs/grading-tiers.md index bfe8e1633..551734991 100644 --- a/dokimi/docs/grading-tiers.md +++ b/dokimi/docs/grading-tiers.md @@ -84,7 +84,9 @@ another header is not followed, so pass each header. The sources together are re file in them has a prefixed name, so a wrong file set cannot veto every name. A file with no prefixed name is recorded with its sha256 and adds no name and no warning, so a wrong `--shared-lib` next to correct headers is not caught. Key verification works the same way: a key naming `LAPACKE_dgesvv` -is `not_found`. +is `not_found`. A prefix a library file or a link line shares (`--prefix mkl_` with `libmkl_rt.so`) does not +veto the library's name: a file's stem (`mkl_rt`) is a known name, and a name in a link context (a `-l` or `::` +operand, or a sentence about linking or a library) is not vetoed. A called name (`mkl_x(`) is always an API claim. ```bash python cli.py skill novelty ground --atoms atoms.json --tool "perf report" --tool "perf record" --tool-version 6.17 diff --git a/dokimi/dokimi/eval/reference_checks.py b/dokimi/dokimi/eval/reference_checks.py index 96bd8ffcc..1e7609bf3 100644 --- a/dokimi/dokimi/eval/reference_checks.py +++ b/dokimi/dokimi/eval/reference_checks.py @@ -386,20 +386,22 @@ def identifier_names(check: Mapping[str, Any], answer: str) -> tuple[Optional[li # Both scans share one deadline, as the regex check does: a pathological pattern is unscored. deadline = time.monotonic() + REGEX_TIMEOUT try: - scopes = scope_pairs(str(within), answer, timeout=_left(deadline)) if within else [(answer, "")] + spans = scope_spans(str(within), answer, timeout=_left(deadline)) if within else [(0, answer, "")] unknown: list[str] = [] - for scope, rest in scopes: + for start, scope, rest in spans: for name in scope_options(pattern, scope, timeout=_left(deadline), negated=bool(within), rest=rest): if name in known or (check.get("negations", False) and name.startswith("--no-") and "--" + name[5:] in known): continue + if check.get("link_context") and _link_mention(answer, start, rest): + continue if name not in unknown: unknown.append(name) except TimeoutError: return None, False except regex.error as exc: raise CheckError(f"identifiers pattern does not compile: {exc}") from exc - return unknown, bool(scopes) + return unknown, bool(spans) def scope_pairs(within: str, answer: str, *, timeout: float) -> list[tuple[str, str]]: @@ -408,10 +410,34 @@ def scope_pairs(within: str, answer: str, *, timeout: float) -> list[tuple[str, A mention the answer negates or warns against (``do not use --x``) is no claim to veto. The text after a scope is kept too: a denial may follow a code span that ends the scope ("`--x` does not exist"). """ - return [(m.group(0), answer[m.end():]) for m in regex.finditer(within, answer, timeout=timeout) + return [(scope, rest) for _, scope, rest in scope_spans(within, answer, timeout=timeout)] + + +def scope_spans(within: str, answer: str, *, timeout: float) -> list[tuple[int, str, str]]: + """As :func:`scope_pairs`, each scope with its offset in ``answer`` first.""" + return [(m.start(), m.group(0), answer[m.end():]) for m in regex.finditer(within, answer, timeout=timeout) if not _negated_before(answer, m.start())] +#: A sentence about linking or a library (``target_link_libraries`` too, its words run on). +_LINK_CONTEXT = re.compile(r"link|librar", re.IGNORECASE) +_SENTENCE_END = re.compile(r"[.;!?\n]") +_LINK_OPERAND = re.compile(r"(?:-l|::)\s*$") + + +def _link_mention(answer: str, start: int, rest: str) -> bool: + """Whether a mention is a link operand (``-l x``, ``MKL::x``) or in a sentence about linking or a library. + + Read only for a check with ``link_context``: a library file's name is no API name, and a call + (``x(``) is always an API claim, so a called name is never read as a link operand. + """ + if re.match(r"\s*\(", rest): + return False + before = _SENTENCE_END.split(answer[max(0, start - _CLAUSE_WINDOW):start])[-1] + after = _SENTENCE_END.split(rest[:_CLAUSE_WINDOW], maxsplit=1)[0] + return _LINK_OPERAND.search(before) is not None or _LINK_CONTEXT.search(before + after) is not None + + _NEGATED = re.compile(r"(?i)\bnot\b|n['’]t\b|\bnever\b|\bno\b|\bavoid\w*|\bwithout\b|\binstead of\b|\brather than\b") #: An existence denial right after a flag, in its code span or not, denies it: only these two phrases, on one #: line; other words after a flag leave its claim standing. diff --git a/dokimi/dokimi/novelty/symbols.py b/dokimi/dokimi/novelty/symbols.py index e9ef65ceb..6c154cb18 100644 --- a/dokimi/dokimi/novelty/symbols.py +++ b/dokimi/dokimi/novelty/symbols.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Known identifiers of a C library's API, from its headers and its exported symbols. -``skill novelty ground --header`` and ``--library`` pin the names a C API declares, so an answer +``skill novelty ground --header`` and ``--shared-lib`` pin the names a C API declares, so an answer that names ``cblas_dgemmx`` (not in the library) is vetoed the way an invented ``--option`` is. Only names with a prefix from ``--prefix`` are read, so an ordinary word is never checked. """ @@ -82,6 +82,12 @@ def nm_names(output: str, prefixes: Sequence[str]) -> set[str]: return names +def library_stem(file_name: str) -> str: + """The name a library file is linked by: ``libmkl_rt.so.2`` and ``mkl_rt.2.dll`` are both ``mkl_rt``.""" + stem = file_name.split(".", 1)[0] + return stem[3:] if stem.startswith("lib") else stem + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -149,6 +155,7 @@ def symbols_check(prefixes: Sequence[str], headers: Sequence[str], """ prefixes = check_prefixes(prefixes) known: set[str] = set() + stems: set[str] = set() inputs: list[dict] = [] sources = [("header", _files(headers, "header")), ("library", _files(libraries, "library"))] for kind, paths in sources: @@ -157,14 +164,17 @@ def symbols_check(prefixes: Sequence[str], headers: Sequence[str], names = header_names(_header_text(path), prefixes) else: names = nm_names(_library_output(path), prefixes) + stems |= {library_stem(path.name)} known |= names inputs.append({"kind": kind, "path": str(path), "sha256": _sha256(path)}) if not known: # a wrong header would otherwise veto every name the answer mentions raise InputError(f"no name with the prefix {', '.join(prefixes)} in the given files") + # A library file's name is no symbol, but it is a name the answer may give (``mkl_rt`` for libmkl_rt.so). + known |= {s for s in stems if re.fullmatch(r"\w+", s) and s.startswith(prefixes)} pattern = symbol_pattern(prefixes) return { "type": "identifiers", "tier": "verifiable", "role": "veto", - "pattern": pattern, "within": pattern, + "pattern": pattern, "within": pattern, "link_context": True, "known": sorted(known), "source": {"kind": "symbols", "prefixes": list(prefixes), "inputs": inputs}, } diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index 5e66db814..b9e12c81b 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -461,3 +461,50 @@ def test_a_tool_version_without_a_tool_is_refused_not_dropped(): from dokimi.commands.novelty_ground import _refuse_bad_sources with pytest.raises(InputError, match="--tool-version"): _refuse_bad_sources([], ["cblas.h"], [], ["cblas_"], "2025.1") + + +MKL_H = "void mkl_set_num_threads(int n);\n" + + +@pytest.mark.parametrize("answer", [ + "Link with mkl_intel_lp64, mkl_sequential and mkl_core.", # a link sentence names library files + "Use the mkl_rt single dynamic library.", # a library file named in a sentence about the library + "target_link_libraries(app MKL::mkl_core)", # a CMake link operand, a namespace-scoped library + "Add -l mkl_core to the link line.", # a -l operand +]) +def test_library_names_in_link_context_are_not_vetoed_under_a_shared_prefix(tmp_path, answer): + check = symbols_check(["mkl_"], [str(_mkl_header(tmp_path))], []) + result = grade_check(check, "q", answer, None) + assert result.passed is True, result.detail + + +@pytest.mark.parametrize("answer", [ + "Call mkl_bogus_fn(1) to set the threads.", # a call is an API claim, link words or not + "Use mkl_bogus_fn to set the threads.", # a prefixed name no header declares, no link context + "Link against mkl_bogus_fn(1) for the threads.", # a called name stays vetoed in a link sentence +]) +def test_a_real_missing_symbol_stays_vetoed_under_a_shared_prefix(tmp_path, answer): + check = symbols_check(["mkl_"], [str(_mkl_header(tmp_path))], []) + result = grade_check(check, "q", answer, None) + assert result.passed is False + assert result.detail == "not in source: mkl_bogus_fn" + + +def test_a_shared_library_stem_with_the_prefix_is_a_known_name_not_a_symbol(tmp_path): + if shutil.which("nm") is None or shutil.which("cc") is None: + pytest.skip("needs nm and cc to build a fixture library") + source = tmp_path / "fixture.c" + source.write_text("int mkl_set_num_threads(int n) { return n; }\n", encoding="utf-8") + library = tmp_path / "libmkl_rt.so" + subprocess.run(["cc", "-shared", "-fPIC", "-o", str(library), str(source)], check=True) + check = symbols_check(["mkl_"], [], [str(library)]) + assert "mkl_rt" in check["known"] # the stem of libmkl_rt.so, from its file name + result = grade_check(check, "q", "Load mkl_rt before the solve.", None) + assert result.passed is True, result.detail + + +def _mkl_header(tmp_path): + include = tmp_path / "include" + include.mkdir(exist_ok=True) + (include / "mkl.h").write_text(MKL_H, encoding="utf-8") + return include / "mkl.h" From 294023daa61cd60cc7cea580ca2d9ec1c55aeb57 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 17:39:37 -0800 Subject: [PATCH 14/14] fix(novelty): a refresh keeps the link exemption of the check it merges into _merged built the refreshed check from the older one and dropped link_context, so a check written before the exemption vetoed the link names again after a refresh. The merged check now takes the flag from the new symbol checks. Also says in the changelog that an undeclared name outside a link context is still vetoed. Signed-off-by: Nikolay Petrov --- ...17_170000_napetrov_novelty-ground-link-context.md | 4 ++-- dokimi/dokimi/novelty/grounding.py | 3 ++- dokimi/tests/test_novelty_ground_symbols.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md b/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md index 1a5d58bc7..6cc76d70f 100644 --- a/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md +++ b/dokimi/changelog.d/20261117_170000_napetrov_novelty-ground-link-context.md @@ -9,5 +9,5 @@ SPDX-License-Identifier: Apache-2.0 mkl_intel_lp64, mkl_sequential and mkl_core.`, `Use the mkl_rt single dynamic library.` and `target_link_libraries(app MKL::mkl_core)` were refused as "not in source". A name after `-l` or `::`, or in a sentence about linking or a library, is not vetoed, and a library file's stem (`mkl_rt` for - `libmkl_rt.so`) is a known name. A called name (`mkl_x(`) and a name no header or library declares - elsewhere are still vetoed. + `libmkl_rt.so`) is a known name. A called name (`mkl_x(`) is still vetoed, and so is an undeclared name + outside a link context (`Call mkl_x to set threads`). diff --git a/dokimi/dokimi/novelty/grounding.py b/dokimi/dokimi/novelty/grounding.py index 9104ee99b..bd118b90d 100644 --- a/dokimi/dokimi/novelty/grounding.py +++ b/dokimi/dokimi/novelty/grounding.py @@ -237,7 +237,8 @@ def _merged(old: dict, mine: list[dict]) -> dict | None: for m in symbol: inputs.update({(i["kind"], i["path"], i.get("sha256")): i for i in m["source"].get("inputs") or []}) source["inputs"] = list(inputs.values()) - return dict(old, known=sorted(kept | fresh), source=source) + return dict(old, known=sorted(kept | fresh), source=source, + link_context=any(m.get("link_context") for m in symbol)) def _absorbed(new: dict, kept: list[dict]) -> bool: diff --git a/dokimi/tests/test_novelty_ground_symbols.py b/dokimi/tests/test_novelty_ground_symbols.py index b9e12c81b..159338162 100644 --- a/dokimi/tests/test_novelty_ground_symbols.py +++ b/dokimi/tests/test_novelty_ground_symbols.py @@ -508,3 +508,15 @@ def _mkl_header(tmp_path): include.mkdir(exist_ok=True) (include / "mkl.h").write_text(MKL_H, encoding="utf-8") return include / "mkl.h" + + +def test_a_refresh_that_merges_into_an_older_check_keeps_the_link_exemption(tmp_path): + from dokimi.novelty.grounding import _merged + include = _headers(tmp_path) + (include / "mkl.h").write_text(MKL_H, encoding="utf-8") + old = symbols_check(["cblas_", "mkl_"], [str(include / "*.h")], []) + del old["link_context"] # a check written before the exemption existed + merged = _merged(old, [symbols_check(["mkl_"], [str(include / "mkl.h")], [])]) + assert merged is not None and merged["link_context"] is True + result = grade_check(merged, "q", "Use the mkl_rt single dynamic library.", None) + assert result.passed is True, result.detail