From 6a85f12ba48de16e1e9775552a0d8637f71792de Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 20:14:34 +0000 Subject: [PATCH 1/4] fix(subjects): scorecard digests frame each field as a fixed-width digest (#878) `_digest_ref` hashed each file's relative path, a NUL, the raw file bytes and a NUL, and `_digest_values` a value, a NUL and the raw bytes of the file it names, so a NUL inside a file ran into the next field: directory `a` = `b`, `c` = empty hashed like `a` = `b\0c\0`, and `("f", "c")` with `f` = `b` like `("f",)` with `f` = `bc\0`. `_digest_arm_spec` joined the spec and each part with a bare NUL the same way. Each variable-length field is now fed as its own sha256 digest, as `tree_digest` and `tree_sha256` do. `_digest_values` feeds the content digest only when the value names a file, so an empty file still differs from none. `_digest_ref` tags its file, directory and no-path branches, so a file whose content equals a missing ref's text no longer shares its digest. Every `ref_digest`, `question_set_hash` and `baseline_digest` changes. The schema descriptions and the skill-novelty doc say how they are computed and that digests of earlier releases differ; no code compares a stored digest with a fresh one. The two tests that pin a literal digest are updated. Signed-off-by: Anton Volkov --- ...0261117_150000_scorecard-digest-framing.md | 8 ++++ dokimi/docs/skill-novelty.md | 4 +- .../dokimi/schemas/subject_scorecard.v2.json | 9 ++-- dokimi/dokimi/subjects/scorecard.py | 32 +++++++------- dokimi/tests/test_subjects_digest.py | 42 ++++++++++++++++++- 5 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md diff --git a/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md new file mode 100644 index 000000000..70588dd2a --- /dev/null +++ b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **Subject scorecard digests no longer share a value across different file sets** — `ref_digest`, `question_set_hash` and `baseline_digest` hashed each file's raw bytes next to NUL-separated names, so a file holding `b\0c\0` hashed like two files `a` and `c`. They now hash the name and the content of each file as separate fixed-width digests. Digests of scorecards written by earlier releases differ from a fresh one for the same inputs; nothing in dokimi compares the two, so rebuild a scorecard to get the new value. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index e4f45a847..0ba4c495c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1393,7 +1393,9 @@ recorded sample) or, with `all_atoms: true`, every one, so a rate over a sample the file's; records written before it have no `sampling`. Every novelty `ref_digest` is the bundle's `measured_digest` (the measured files' paths and text digests); the subject scorecard's `subject.ref_digest` is a different hash, of every file under the -subject path, and the two are not meant to match. `shorten` +subject path (each file's path and content as separate fixed-width digests; a scorecard +written by an earlier release hashed them differently, so its `ref_digest` differs for +the same tree), and the two are not meant to match. `shorten` rewrites one file, so with a bundle-derived atom set it scores only the entry point's spans and says so. diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index b054ac2b8..c3b571afe 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -53,7 +53,8 @@ "type": [ "string", "null" - ] + ], + "description": "sha256 over the ref's content: for a directory, each file's relative path and content as two fixed-width sha256 digests, in sorted path order; for a file, its content digest; for a ref that is no path, its text. Digests written by earlier releases hashed the raw bytes next to NUL separators and differ for the same ref." }, "distribution_digest": { "type": [ @@ -90,7 +91,8 @@ } }, "question_set_hash": { - "type": "string" + "type": "string", + "description": "sha256 over each questions entry: a digest of its text, then a digest of the file it names when it is a file. Digests written by earlier releases differ for the same entries." }, "tasks": { "type": "array", @@ -105,7 +107,8 @@ "type": "string" }, "baseline_digest": { - "type": "string" + "type": "string", + "description": "sha256 over the baseline arm spec and, per part that reads local content, the part and the ref_digest of that content, each as a fixed-width digest. Digests written by earlier releases differ for the same spec." }, "awareness": { "type": "object", diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 0dfe243d9..c7dc108d5 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -132,14 +132,19 @@ def build_subject_scorecard( } +def _field(text: str) -> bytes: + """A fixed-width field for *text*, so a NUL or any byte in it cannot run into the next field.""" + return hashlib.sha256(os.fsencode(text)).digest() + + def _digest_values(values: tuple[str, ...]) -> str: + """Per value: its text's digest, then the digest of the file it names, when it names one.""" h = hashlib.sha256() for value in values: - h.update(os.fsencode(value)) - h.update(b"\0") + h.update(_field(value)) path = Path(value) if path.is_file(): - h.update(path.read_bytes()) + h.update(hashlib.sha256(path.read_bytes()).digest()) return "sha256:" + h.hexdigest() @@ -182,15 +187,14 @@ def _digest_ref(ref: str | None, *, base: Path) -> str | None: path = base / path h = hashlib.sha256() if path.is_file(): - h.update(path.read_bytes()) + h.update(b"f" + hashlib.sha256(path.read_bytes()).digest()) elif path.is_dir(): + h.update(b"d") for child in sorted(p for p in path.rglob("*") if p.is_file()): - h.update(os.fsencode(str(child.relative_to(path)))) - h.update(b"\0") - h.update(child.read_bytes()) - h.update(b"\0") + h.update(_field(str(child.relative_to(path)))) + h.update(hashlib.sha256(child.read_bytes()).digest()) else: - h.update(os.fsencode(ref)) + h.update(b"t" + _field(ref)) return "sha256:" + h.hexdigest() @@ -198,12 +202,8 @@ def _digest_arm_spec(spec: str, *, base: Path) -> str: from dokimi.treatments.specs import content_refs h = hashlib.sha256() - h.update(os.fsencode(spec)) - h.update(b"\0") + h.update(_field(spec)) for part, content_ref in content_refs(spec): - h.update(os.fsencode(part)) - h.update(b"\0") - digest = _digest_ref(content_ref, base=base) - h.update((digest or "").encode("utf-8")) - h.update(b"\0") + h.update(_field(part)) + h.update(_field(_digest_ref(content_ref, base=base) or "")) return "sha256:" + h.hexdigest() diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 99e400e9b..6e6f8c3ea 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -69,7 +69,7 @@ def test_a_directory_ref_of_utf8_names_keeps_its_digest(tmp_path: Path): (tmp_path / "sub").mkdir() (tmp_path / "sub" / "é.md").write_bytes(b"two\n") assert _digest_ref(str(tmp_path), base=tmp_path) == ( - "sha256:b81d3e4952432273d79f200d2d22373cefe695e569138fe2d6feab66fa962a76") + "sha256:8bebaf0c5b5da4acb85facaa866ec8beabd87373df2c2f38f0fc86a655e1303f") # gitleaks:allow @pytest.mark.skipif(sys.platform == "win32", reason="a name that is not UTF-8 cannot be created on Windows") @@ -89,7 +89,7 @@ def test_a_directory_ref_hashes_the_bytes_of_a_file_name_that_is_not_utf8(tmp_pa def test_a_ref_that_is_no_path_keeps_its_digest(tmp_path: Path): assert _digest_ref("no-such-é", base=tmp_path) == ( - "sha256:63661dbcb1a932eeb0f6555a4df5f16b59516d27e1d56d1ce19777691bf1b78a") + "sha256:3124a1d714c608077eca68f5b382e3db438cc8826e0523d3e82768e517f08f9b") # gitleaks:allow @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") @@ -121,6 +121,44 @@ def test_each_part_of_a_combined_spec_with_a_lone_surrogate_is_digested(tmp_path assert digest.startswith("sha256:") and digest != _digest_arm_spec("skill:./a+skill:./n\udcfe", base=tmp_path) +def _tree(root: Path, files: dict[str, bytes]) -> str: + root.mkdir() + for name, content in files.items(): + (root / name).write_bytes(content) + digest = _digest_ref(str(root), base=root) + assert digest is not None + return digest + + +def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_file_name(tmp_path: Path): + # #878: a = "b" and c = "" hashed the same bytes as a = "b\0c\0" alone. + two = _tree(tmp_path / "s1", {"a": b"b", "c": b""}) + one = _tree(tmp_path / "s2", {"a": b"b\0c\0"}) + assert two != one + + +def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_value(tmp_path: Path): + # #878: ("f", "c") with f = "b" hashed the same bytes as ("f",) with f = "bc\0". + f = tmp_path / "f" + f.write_bytes(b"b") + pair = _digest_values((str(f), "c")) + f.write_bytes(b"bc\0") + assert _digest_values((str(f),)) != pair + + +def test_an_arm_spec_digest_frames_the_spec_and_each_part_as_fixed_width_fields(tmp_path: Path): + # #878: the spec and each part were joined with a bare NUL, so this literal moved. + assert _digest_arm_spec("skill:./a+skill:./b", base=tmp_path) == ( + "sha256:8c142870b5eab2dd85f0380146b5b58e1de2576a17f49bc6312bbfce6b00d7cc") # gitleaks:allow + assert _digest_arm_spec("baseline", base=tmp_path) == ( + "sha256:f22917cace49ee841b942ac46ec9a946203ba2f89f8c816143557337cbcdf72c") # gitleaks:allow + + +def test_a_file_ref_differs_from_a_missing_ref_spelled_as_its_content(tmp_path: Path): + (tmp_path / "doc.md").write_bytes(b"missing.md") + assert _digest_ref("doc.md", base=tmp_path) != _digest_ref("missing.md", base=tmp_path) + + def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypatch): from dokimi.artifacts import validate_artifact from dokimi.subjects import build_subject_scorecard, load_subject From 18a7d0a60d7f20a961132965d12fcbdccc6b1d77 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 20:14:47 +0000 Subject: [PATCH 2/4] fix(subjects): scorecard digests encode a lone high surrogate instead of raising (#795) `_digest_values`, `_digest_ref` and `_digest_arm_spec` encoded text with `os.fsencode`, which on POSIX is `surrogateescape` and cannot map a high surrogate, so a spec or ref with a JSON `\ud800` escape raised `UnicodeEncodeError`. The shared field encoder now uses `utils.key_bytes`, the encoding of the cache keys fixed for the same input: a high surrogate is tagged and kept with `surrogatepass`, so `n\ud800` and `n\ud801` get different digests and `n\udcff` still gets one. It is used on every OS, so the same text digests the same everywhere; on Windows the digest of a high-surrogate ref differs from before, when `os.fsencode` encoded it untagged. The digest of an encodable text is unchanged by this commit. Signed-off-by: Anton Volkov --- .../20261117_150100_scorecard-digest-surrogate.md | 8 ++++++++ dokimi/dokimi/subjects/scorecard.py | 4 ++-- dokimi/tests/test_subjects_digest.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md diff --git a/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md b/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md new file mode 100644 index 000000000..087561ac0 --- /dev/null +++ b/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A subject scorecard digest no longer raises for a spec or ref with a JSON high surrogate** — `ref_digest`, `question_set_hash` and `baseline_digest` encoded text with `os.fsencode`, which cannot map `\ud800`, so a descriptor or arm spec read from a file with that escape raised `UnicodeEncodeError`. They now encode with the cache keys' `key_bytes`, on every OS. On Windows, where `os.fsencode` already encoded a lone surrogate, the digest of such a ref differs from before. diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index c7dc108d5..c59afada7 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -6,13 +6,13 @@ from __future__ import annotations import hashlib -import os import re from datetime import datetime, timezone from pathlib import Path from typing import Any from dokimi import __version__ +from dokimi.utils import key_bytes from .loader import awareness_arm_specs, work_arm_specs from .models import SubjectDescriptor @@ -134,7 +134,7 @@ def build_subject_scorecard( def _field(text: str) -> bytes: """A fixed-width field for *text*, so a NUL or any byte in it cannot run into the next field.""" - return hashlib.sha256(os.fsencode(text)).digest() + return hashlib.sha256(key_bytes(text)).digest() def _digest_values(values: tuple[str, ...]) -> str: diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 6e6f8c3ea..d3969a3a5 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -159,6 +159,19 @@ def test_a_file_ref_differs_from_a_missing_ref_spelled_as_its_content(tmp_path: assert _digest_ref("doc.md", base=tmp_path) != _digest_ref("missing.md", base=tmp_path) +@pytest.mark.parametrize("digest", [ + lambda text, base: _digest_ref(text, base=base), + lambda text, base: _digest_values((text,)), + lambda text, base: _digest_arm_spec(f"skill:./{text}", base=base), +], ids=["ref", "values", "arm_spec"]) +def test_a_high_surrogate_is_digested_and_told_apart(digest, tmp_path: Path): + # #795: a JSON "\ud800" escape has no surrogateescape byte form, so os.fsencode raised UnicodeEncodeError. + first = digest("n\ud800", tmp_path) + assert first.startswith("sha256:") and first == digest("n\ud800", tmp_path) + assert first != digest("n\ud801", tmp_path) + assert digest("n\udcff", tmp_path).startswith("sha256:") + + def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypatch): from dokimi.artifacts import validate_artifact from dokimi.subjects import build_subject_scorecard, load_subject From 76c6739aff45303b34bb352f835ee44dd2cc67d1 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 20:25:51 +0000 Subject: [PATCH 3/4] test(subjects): pin that an empty file digests apart from no file in scorecard digests (#878) `_digest_values` feeds a file's content digest only when the value names a file, and `_digest_ref` tags a file ref apart from a missing one, but no test covered an empty file. The new tests pin that a value naming an empty file differs from the same value naming no file, and that an empty file ref differs from a missing ref spelled as the same path. Signed-off-by: Anton Volkov --- dokimi/tests/test_subjects_digest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index d3969a3a5..0c9ff8fd0 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -159,6 +159,21 @@ def test_a_file_ref_differs_from_a_missing_ref_spelled_as_its_content(tmp_path: assert _digest_ref("doc.md", base=tmp_path) != _digest_ref("missing.md", base=tmp_path) +def test_a_value_naming_an_empty_file_differs_from_a_value_naming_no_file(tmp_path: Path): + f = tmp_path / "f" + f.write_bytes(b"") + empty = _digest_values((str(f),)) + f.unlink() + assert _digest_values((str(f),)) != empty + + +def test_an_empty_file_ref_differs_from_a_missing_ref_spelled_as_the_same_path(tmp_path: Path): + (tmp_path / "present").mkdir() + (tmp_path / "present" / "e.md").write_bytes(b"") + (tmp_path / "absent").mkdir() + assert _digest_ref("e.md", base=tmp_path / "present") != _digest_ref("e.md", base=tmp_path / "absent") + + @pytest.mark.parametrize("digest", [ lambda text, base: _digest_ref(text, base=base), lambda text, base: _digest_values((text,)), From d5657b9937a75404a3e0b5bb51c2d044737abdfe Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 20:45:29 +0000 Subject: [PATCH 4/4] fix(subjects): question_set_hash tags each question as naming a file or not (#878) Settles the exai-bot review comment on #892: a question file whose content was b"missing-question" hashed the same as the file question followed by a question spelled "missing-question", because the file's content digest was the same bytes as the next value's text digest. _digest_values now writes a one-byte tag (file or no file) after each value's text digest. Test: test_a_file_value_whose_content_spells_the_next_value_differs_from_the_two_values. Signed-off-by: Anton Volkov --- .../20261117_150000_scorecard-digest-framing.md | 2 +- dokimi/dokimi/subjects/scorecard.py | 6 ++++-- dokimi/tests/test_subjects_digest.py | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md index 70588dd2a..ad6ce5345 100644 --- a/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md +++ b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **Subject scorecard digests no longer share a value across different file sets** — `ref_digest`, `question_set_hash` and `baseline_digest` hashed each file's raw bytes next to NUL-separated names, so a file holding `b\0c\0` hashed like two files `a` and `c`. They now hash the name and the content of each file as separate fixed-width digests. Digests of scorecards written by earlier releases differ from a fresh one for the same inputs; nothing in dokimi compares the two, so rebuild a scorecard to get the new value. +- **Subject scorecard digests no longer share a value across different file sets** — `ref_digest`, `question_set_hash` and `baseline_digest` hashed each file's raw bytes next to NUL-separated names, so a file holding `b\0c\0` hashed like two files `a` and `c`. They now hash the name and the content of each file as separate fixed-width digests, and `question_set_hash` tags each question as naming a file or not, so a question file whose content spells the next question no longer hashes like the two questions. Digests of scorecards written by earlier releases differ from a fresh one for the same inputs; nothing in dokimi compares the two, so rebuild a scorecard to get the new value. diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index c59afada7..c80049a5b 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -138,13 +138,15 @@ def _field(text: str) -> bytes: def _digest_values(values: tuple[str, ...]) -> str: - """Per value: its text's digest, then the digest of the file it names, when it names one.""" + """Per value: its text's digest, then a tag for file or no file, then the file's digest when it names one.""" h = hashlib.sha256() for value in values: h.update(_field(value)) path = Path(value) if path.is_file(): - h.update(hashlib.sha256(path.read_bytes()).digest()) + h.update(b"f" + hashlib.sha256(path.read_bytes()).digest()) + else: + h.update(b"t") return "sha256:" + h.hexdigest() diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 0c9ff8fd0..fec81dc71 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -146,6 +146,15 @@ def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_value(tmp_path: Path): assert _digest_values((str(f),)) != pair +def test_a_file_value_whose_content_spells_the_next_value_differs_from_the_two_values(tmp_path: Path): + # #892 review: the file's content digest was the same bytes as the digest of a following value spelled as that content. + f = tmp_path / "f" + f.write_bytes(b"missing-question") + single = _digest_values((str(f),)) + f.unlink() + assert _digest_values((str(f), "missing-question")) != single + + def test_an_arm_spec_digest_frames_the_spec_and_each_part_as_fixed_width_fields(tmp_path: Path): # #878: the spec and each part were joined with a bare NUL, so this literal moved. assert _digest_arm_spec("skill:./a+skill:./b", base=tmp_path) == (