diff --git a/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md new file mode 100644 index 000000000..ad6ce5345 --- /dev/null +++ b/dokimi/changelog.d/20261117_150000_scorecard-digest-framing.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **Subject scorecard digests no longer share a value across different file sets** — `ref_digest`, `question_set_hash` and `baseline_digest` hashed each file's raw bytes next to NUL-separated names, so a file holding `b\0c\0` hashed like two files `a` and `c`. They now hash the name and the content of each file as separate fixed-width digests, and `question_set_hash` tags each question as naming a file or not, so a question file whose content spells the next question no longer hashes like the two questions. Digests of scorecards written by earlier releases differ from a fresh one for the same inputs; nothing in dokimi compares the two, so rebuild a scorecard to get the new value. diff --git a/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md b/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md new file mode 100644 index 000000000..087561ac0 --- /dev/null +++ b/dokimi/changelog.d/20261117_150100_scorecard-digest-surrogate.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A subject scorecard digest no longer raises for a spec or ref with a JSON high surrogate** — `ref_digest`, `question_set_hash` and `baseline_digest` encoded text with `os.fsencode`, which cannot map `\ud800`, so a descriptor or arm spec read from a file with that escape raised `UnicodeEncodeError`. They now encode with the cache keys' `key_bytes`, on every OS. On Windows, where `os.fsencode` already encoded a lone surrogate, the digest of such a ref differs from before. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index e4f45a847..0ba4c495c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1393,7 +1393,9 @@ recorded sample) or, with `all_atoms: true`, every one, so a rate over a sample the file's; records written before it have no `sampling`. Every novelty `ref_digest` is the bundle's `measured_digest` (the measured files' paths and text digests); the subject scorecard's `subject.ref_digest` is a different hash, of every file under the -subject path, and the two are not meant to match. `shorten` +subject path (each file's path and content as separate fixed-width digests; a scorecard +written by an earlier release hashed them differently, so its `ref_digest` differs for +the same tree), and the two are not meant to match. `shorten` rewrites one file, so with a bundle-derived atom set it scores only the entry point's spans and says so. diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index b054ac2b8..c3b571afe 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -53,7 +53,8 @@ "type": [ "string", "null" - ] + ], + "description": "sha256 over the ref's content: for a directory, each file's relative path and content as two fixed-width sha256 digests, in sorted path order; for a file, its content digest; for a ref that is no path, its text. Digests written by earlier releases hashed the raw bytes next to NUL separators and differ for the same ref." }, "distribution_digest": { "type": [ @@ -90,7 +91,8 @@ } }, "question_set_hash": { - "type": "string" + "type": "string", + "description": "sha256 over each questions entry: a digest of its text, then a digest of the file it names when it is a file. Digests written by earlier releases differ for the same entries." }, "tasks": { "type": "array", @@ -105,7 +107,8 @@ "type": "string" }, "baseline_digest": { - "type": "string" + "type": "string", + "description": "sha256 over the baseline arm spec and, per part that reads local content, the part and the ref_digest of that content, each as a fixed-width digest. Digests written by earlier releases differ for the same spec." }, "awareness": { "type": "object", diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 0dfe243d9..c80049a5b 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -6,13 +6,13 @@ from __future__ import annotations import hashlib -import os import re from datetime import datetime, timezone from pathlib import Path from typing import Any from dokimi import __version__ +from dokimi.utils import key_bytes from .loader import awareness_arm_specs, work_arm_specs from .models import SubjectDescriptor @@ -132,14 +132,21 @@ def build_subject_scorecard( } +def _field(text: str) -> bytes: + """A fixed-width field for *text*, so a NUL or any byte in it cannot run into the next field.""" + return hashlib.sha256(key_bytes(text)).digest() + + def _digest_values(values: tuple[str, ...]) -> str: + """Per value: its text's digest, then a tag for file or no file, then the file's digest when it names one.""" h = hashlib.sha256() for value in values: - h.update(os.fsencode(value)) - h.update(b"\0") + h.update(_field(value)) path = Path(value) if path.is_file(): - h.update(path.read_bytes()) + h.update(b"f" + hashlib.sha256(path.read_bytes()).digest()) + else: + h.update(b"t") return "sha256:" + h.hexdigest() @@ -182,15 +189,14 @@ def _digest_ref(ref: str | None, *, base: Path) -> str | None: path = base / path h = hashlib.sha256() if path.is_file(): - h.update(path.read_bytes()) + h.update(b"f" + hashlib.sha256(path.read_bytes()).digest()) elif path.is_dir(): + h.update(b"d") for child in sorted(p for p in path.rglob("*") if p.is_file()): - h.update(os.fsencode(str(child.relative_to(path)))) - h.update(b"\0") - h.update(child.read_bytes()) - h.update(b"\0") + h.update(_field(str(child.relative_to(path)))) + h.update(hashlib.sha256(child.read_bytes()).digest()) else: - h.update(os.fsencode(ref)) + h.update(b"t" + _field(ref)) return "sha256:" + h.hexdigest() @@ -198,12 +204,8 @@ def _digest_arm_spec(spec: str, *, base: Path) -> str: from dokimi.treatments.specs import content_refs h = hashlib.sha256() - h.update(os.fsencode(spec)) - h.update(b"\0") + h.update(_field(spec)) for part, content_ref in content_refs(spec): - h.update(os.fsencode(part)) - h.update(b"\0") - digest = _digest_ref(content_ref, base=base) - h.update((digest or "").encode("utf-8")) - h.update(b"\0") + h.update(_field(part)) + h.update(_field(_digest_ref(content_ref, base=base) or "")) return "sha256:" + h.hexdigest() diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 99e400e9b..fec81dc71 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -69,7 +69,7 @@ def test_a_directory_ref_of_utf8_names_keeps_its_digest(tmp_path: Path): (tmp_path / "sub").mkdir() (tmp_path / "sub" / "é.md").write_bytes(b"two\n") assert _digest_ref(str(tmp_path), base=tmp_path) == ( - "sha256:b81d3e4952432273d79f200d2d22373cefe695e569138fe2d6feab66fa962a76") + "sha256:8bebaf0c5b5da4acb85facaa866ec8beabd87373df2c2f38f0fc86a655e1303f") # gitleaks:allow @pytest.mark.skipif(sys.platform == "win32", reason="a name that is not UTF-8 cannot be created on Windows") @@ -89,7 +89,7 @@ def test_a_directory_ref_hashes_the_bytes_of_a_file_name_that_is_not_utf8(tmp_pa def test_a_ref_that_is_no_path_keeps_its_digest(tmp_path: Path): assert _digest_ref("no-such-é", base=tmp_path) == ( - "sha256:63661dbcb1a932eeb0f6555a4df5f16b59516d27e1d56d1ce19777691bf1b78a") + "sha256:3124a1d714c608077eca68f5b382e3db438cc8826e0523d3e82768e517f08f9b") # gitleaks:allow @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") @@ -121,6 +121,81 @@ def test_each_part_of_a_combined_spec_with_a_lone_surrogate_is_digested(tmp_path assert digest.startswith("sha256:") and digest != _digest_arm_spec("skill:./a+skill:./n\udcfe", base=tmp_path) +def _tree(root: Path, files: dict[str, bytes]) -> str: + root.mkdir() + for name, content in files.items(): + (root / name).write_bytes(content) + digest = _digest_ref(str(root), base=root) + assert digest is not None + return digest + + +def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_file_name(tmp_path: Path): + # #878: a = "b" and c = "" hashed the same bytes as a = "b\0c\0" alone. + two = _tree(tmp_path / "s1", {"a": b"b", "c": b""}) + one = _tree(tmp_path / "s2", {"a": b"b\0c\0"}) + assert two != one + + +def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_value(tmp_path: Path): + # #878: ("f", "c") with f = "b" hashed the same bytes as ("f",) with f = "bc\0". + f = tmp_path / "f" + f.write_bytes(b"b") + pair = _digest_values((str(f), "c")) + f.write_bytes(b"bc\0") + assert _digest_values((str(f),)) != pair + + +def test_a_file_value_whose_content_spells_the_next_value_differs_from_the_two_values(tmp_path: Path): + # #892 review: the file's content digest was the same bytes as the digest of a following value spelled as that content. + f = tmp_path / "f" + f.write_bytes(b"missing-question") + single = _digest_values((str(f),)) + f.unlink() + assert _digest_values((str(f), "missing-question")) != single + + +def test_an_arm_spec_digest_frames_the_spec_and_each_part_as_fixed_width_fields(tmp_path: Path): + # #878: the spec and each part were joined with a bare NUL, so this literal moved. + assert _digest_arm_spec("skill:./a+skill:./b", base=tmp_path) == ( + "sha256:8c142870b5eab2dd85f0380146b5b58e1de2576a17f49bc6312bbfce6b00d7cc") # gitleaks:allow + assert _digest_arm_spec("baseline", base=tmp_path) == ( + "sha256:f22917cace49ee841b942ac46ec9a946203ba2f89f8c816143557337cbcdf72c") # gitleaks:allow + + +def test_a_file_ref_differs_from_a_missing_ref_spelled_as_its_content(tmp_path: Path): + (tmp_path / "doc.md").write_bytes(b"missing.md") + assert _digest_ref("doc.md", base=tmp_path) != _digest_ref("missing.md", base=tmp_path) + + +def test_a_value_naming_an_empty_file_differs_from_a_value_naming_no_file(tmp_path: Path): + f = tmp_path / "f" + f.write_bytes(b"") + empty = _digest_values((str(f),)) + f.unlink() + assert _digest_values((str(f),)) != empty + + +def test_an_empty_file_ref_differs_from_a_missing_ref_spelled_as_the_same_path(tmp_path: Path): + (tmp_path / "present").mkdir() + (tmp_path / "present" / "e.md").write_bytes(b"") + (tmp_path / "absent").mkdir() + assert _digest_ref("e.md", base=tmp_path / "present") != _digest_ref("e.md", base=tmp_path / "absent") + + +@pytest.mark.parametrize("digest", [ + lambda text, base: _digest_ref(text, base=base), + lambda text, base: _digest_values((text,)), + lambda text, base: _digest_arm_spec(f"skill:./{text}", base=base), +], ids=["ref", "values", "arm_spec"]) +def test_a_high_surrogate_is_digested_and_told_apart(digest, tmp_path: Path): + # #795: a JSON "\ud800" escape has no surrogateescape byte form, so os.fsencode raised UnicodeEncodeError. + first = digest("n\ud800", tmp_path) + assert first.startswith("sha256:") and first == digest("n\ud800", tmp_path) + assert first != digest("n\ud801", tmp_path) + assert digest("n\udcff", tmp_path).startswith("sha256:") + + def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypatch): from dokimi.artifacts import validate_artifact from dokimi.subjects import build_subject_scorecard, load_subject