Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **Subject scorecard digests no longer share a value across different file sets** — `ref_digest`, `question_set_hash` and `baseline_digest` hashed each file's raw bytes next to NUL-separated names, so a file holding `b\0c\0` hashed like two files `a` and `c`. They now hash the name and the content of each file as separate fixed-width digests, and `question_set_hash` tags each question as naming a file or not, so a question file whose content spells the next question no longer hashes like the two questions. Digests of scorecards written by earlier releases differ from a fresh one for the same inputs; nothing in dokimi compares the two, so rebuild a scorecard to get the new value.
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **A subject scorecard digest no longer raises for a spec or ref with a JSON high surrogate** — `ref_digest`, `question_set_hash` and `baseline_digest` encoded text with `os.fsencode`, which cannot map `\ud800`, so a descriptor or arm spec read from a file with that escape raised `UnicodeEncodeError`. They now encode with the cache keys' `key_bytes`, on every OS. On Windows, where `os.fsencode` already encoded a lone surrogate, the digest of such a ref differs from before.
4 changes: 3 additions & 1 deletion dokimi/docs/skill-novelty.md
Original file line number Diff line number Diff line change
Expand Up @@ -1393,7 +1393,9 @@ recorded sample) or, with `all_atoms: true`, every one, so a rate over a sample
the file's; records written before it have no `sampling`. Every novelty
`ref_digest` is the bundle's `measured_digest` (the measured files' paths and text digests);
the subject scorecard's `subject.ref_digest` is a different hash, of every file under the
subject path, and the two are not meant to match. `shorten`
subject path (each file's path and content as separate fixed-width digests; a scorecard
written by an earlier release hashed them differently, so its `ref_digest` differs for
the same tree), and the two are not meant to match. `shorten`
rewrites one file, so with a bundle-derived atom set it scores only the entry point's spans
and says so.

Expand Down
9 changes: 6 additions & 3 deletions dokimi/dokimi/schemas/subject_scorecard.v2.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@
"type": [
"string",
"null"
]
],
"description": "sha256 over the ref's content: for a directory, each file's relative path and content as two fixed-width sha256 digests, in sorted path order; for a file, its content digest; for a ref that is no path, its text. Digests written by earlier releases hashed the raw bytes next to NUL separators and differ for the same ref."
},
"distribution_digest": {
"type": [
Expand Down Expand Up @@ -90,7 +91,8 @@
}
},
"question_set_hash": {
"type": "string"
"type": "string",
"description": "sha256 over each questions entry: a digest of its text, then a digest of the file it names when it is a file. Digests written by earlier releases differ for the same entries."
},
"tasks": {
"type": "array",
Expand All @@ -105,7 +107,8 @@
"type": "string"
},
"baseline_digest": {
"type": "string"
"type": "string",
"description": "sha256 over the baseline arm spec and, per part that reads local content, the part and the ref_digest of that content, each as a fixed-width digest. Digests written by earlier releases differ for the same spec."
},
"awareness": {
"type": "object",
Expand Down
36 changes: 19 additions & 17 deletions dokimi/dokimi/subjects/scorecard.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
from __future__ import annotations

import hashlib
import os
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from dokimi import __version__
from dokimi.utils import key_bytes

from .loader import awareness_arm_specs, work_arm_specs
from .models import SubjectDescriptor
Expand Down Expand Up @@ -132,14 +132,21 @@ def build_subject_scorecard(
}


def _field(text: str) -> bytes:
"""A fixed-width field for *text*, so a NUL or any byte in it cannot run into the next field."""
return hashlib.sha256(key_bytes(text)).digest()


def _digest_values(values: tuple[str, ...]) -> str:
"""Per value: its text's digest, then a tag for file or no file, then the file's digest when it names one."""
h = hashlib.sha256()
for value in values:
h.update(os.fsencode(value))
h.update(b"\0")
h.update(_field(value))
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
path = Path(value)
if path.is_file():
h.update(path.read_bytes())
h.update(b"f" + hashlib.sha256(path.read_bytes()).digest())
else:
h.update(b"t")
return "sha256:" + h.hexdigest()


Expand Down Expand Up @@ -182,28 +189,23 @@ def _digest_ref(ref: str | None, *, base: Path) -> str | None:
path = base / path
h = hashlib.sha256()
if path.is_file():
h.update(path.read_bytes())
h.update(b"f" + hashlib.sha256(path.read_bytes()).digest())
elif path.is_dir():
h.update(b"d")
for child in sorted(p for p in path.rglob("*") if p.is_file()):
h.update(os.fsencode(str(child.relative_to(path))))
h.update(b"\0")
h.update(child.read_bytes())
h.update(b"\0")
h.update(_field(str(child.relative_to(path))))
h.update(hashlib.sha256(child.read_bytes()).digest())
else:
h.update(os.fsencode(ref))
h.update(b"t" + _field(ref))
return "sha256:" + h.hexdigest()


def _digest_arm_spec(spec: str, *, base: Path) -> str:
from dokimi.treatments.specs import content_refs

h = hashlib.sha256()
h.update(os.fsencode(spec))
h.update(b"\0")
h.update(_field(spec))
for part, content_ref in content_refs(spec):
h.update(os.fsencode(part))
h.update(b"\0")
digest = _digest_ref(content_ref, base=base)
h.update((digest or "").encode("utf-8"))
h.update(b"\0")
h.update(_field(part))
h.update(_field(_digest_ref(content_ref, base=base) or ""))
return "sha256:" + h.hexdigest()
79 changes: 77 additions & 2 deletions dokimi/tests/test_subjects_digest.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def test_a_directory_ref_of_utf8_names_keeps_its_digest(tmp_path: Path):
(tmp_path / "sub").mkdir()
(tmp_path / "sub" / "é.md").write_bytes(b"two\n")
assert _digest_ref(str(tmp_path), base=tmp_path) == (
"sha256:b81d3e4952432273d79f200d2d22373cefe695e569138fe2d6feab66fa962a76")
"sha256:8bebaf0c5b5da4acb85facaa866ec8beabd87373df2c2f38f0fc86a655e1303f") # gitleaks:allow


@pytest.mark.skipif(sys.platform == "win32", reason="a name that is not UTF-8 cannot be created on Windows")
Expand All @@ -89,7 +89,7 @@ def test_a_directory_ref_hashes_the_bytes_of_a_file_name_that_is_not_utf8(tmp_pa

def test_a_ref_that_is_no_path_keeps_its_digest(tmp_path: Path):
assert _digest_ref("no-such-é", base=tmp_path) == (
"sha256:63661dbcb1a932eeb0f6555a4df5f16b59516d27e1d56d1ce19777691bf1b78a")
"sha256:3124a1d714c608077eca68f5b382e3db438cc8826e0523d3e82768e517f08f9b") # gitleaks:allow


@pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form")
Expand Down Expand Up @@ -121,6 +121,81 @@ def test_each_part_of_a_combined_spec_with_a_lone_surrogate_is_digested(tmp_path
assert digest.startswith("sha256:") and digest != _digest_arm_spec("skill:./a+skill:./n\udcfe", base=tmp_path)


def _tree(root: Path, files: dict[str, bytes]) -> str:
root.mkdir()
for name, content in files.items():
(root / name).write_bytes(content)
digest = _digest_ref(str(root), base=root)
assert digest is not None
return digest


def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_file_name(tmp_path: Path):
# #878: a = "b" and c = "" hashed the same bytes as a = "b\0c\0" alone.
two = _tree(tmp_path / "s1", {"a": b"b", "c": b""})
one = _tree(tmp_path / "s2", {"a": b"b\0c\0"})
assert two != one


def test_a_nul_in_a_file_cannot_move_bytes_into_the_next_value(tmp_path: Path):
# #878: ("f", "c") with f = "b" hashed the same bytes as ("f",) with f = "bc\0".
f = tmp_path / "f"
f.write_bytes(b"b")
pair = _digest_values((str(f), "c"))
f.write_bytes(b"bc\0")
assert _digest_values((str(f),)) != pair


def test_a_file_value_whose_content_spells_the_next_value_differs_from_the_two_values(tmp_path: Path):
# #892 review: the file's content digest was the same bytes as the digest of a following value spelled as that content.
f = tmp_path / "f"
f.write_bytes(b"missing-question")
single = _digest_values((str(f),))
f.unlink()
assert _digest_values((str(f), "missing-question")) != single


def test_an_arm_spec_digest_frames_the_spec_and_each_part_as_fixed_width_fields(tmp_path: Path):
# #878: the spec and each part were joined with a bare NUL, so this literal moved.
assert _digest_arm_spec("skill:./a+skill:./b", base=tmp_path) == (
"sha256:8c142870b5eab2dd85f0380146b5b58e1de2576a17f49bc6312bbfce6b00d7cc") # gitleaks:allow
assert _digest_arm_spec("baseline", base=tmp_path) == (
"sha256:f22917cace49ee841b942ac46ec9a946203ba2f89f8c816143557337cbcdf72c") # gitleaks:allow


def test_a_file_ref_differs_from_a_missing_ref_spelled_as_its_content(tmp_path: Path):
(tmp_path / "doc.md").write_bytes(b"missing.md")
assert _digest_ref("doc.md", base=tmp_path) != _digest_ref("missing.md", base=tmp_path)


def test_a_value_naming_an_empty_file_differs_from_a_value_naming_no_file(tmp_path: Path):
f = tmp_path / "f"
f.write_bytes(b"")
empty = _digest_values((str(f),))
f.unlink()
assert _digest_values((str(f),)) != empty


def test_an_empty_file_ref_differs_from_a_missing_ref_spelled_as_the_same_path(tmp_path: Path):
(tmp_path / "present").mkdir()
(tmp_path / "present" / "e.md").write_bytes(b"")
(tmp_path / "absent").mkdir()
assert _digest_ref("e.md", base=tmp_path / "present") != _digest_ref("e.md", base=tmp_path / "absent")


@pytest.mark.parametrize("digest", [
lambda text, base: _digest_ref(text, base=base),
lambda text, base: _digest_values((text,)),
lambda text, base: _digest_arm_spec(f"skill:./{text}", base=base),
], ids=["ref", "values", "arm_spec"])
def test_a_high_surrogate_is_digested_and_told_apart(digest, tmp_path: Path):
# #795: a JSON "\ud800" escape has no surrogateescape byte form, so os.fsencode raised UnicodeEncodeError.
first = digest("n\ud800", tmp_path)
assert first.startswith("sha256:") and first == digest("n\ud800", tmp_path)
assert first != digest("n\ud801", tmp_path)
assert digest("n\udcff", tmp_path).startswith("sha256:")


def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypatch):
from dokimi.artifacts import validate_artifact
from dokimi.subjects import build_subject_scorecard, load_subject
Expand Down
Loading