diff --git a/dokimi/changelog.d/20261117_140000_retention-key-nul.md b/dokimi/changelog.d/20261117_140000_retention-key-nul.md new file mode 100644 index 000000000..0d6e145c4 --- /dev/null +++ b/dokimi/changelog.d/20261117_140000_retention-key-nul.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A novelty retention cache entry is no longer replayed for a different atom when a NUL separates them** — the key joined the statement and the text with a bare NUL, so a statement ending in NUL and a text starting with NUL hashed the same bytes. The key now hashes the statement's length before the statement. The cache version moves to `retention-cache-v6`, so entries already in `--cache-dir` are not read; the old files can be deleted, and the first run after upgrading pays the adjudication calls again. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index ddefde067..e4f45a847 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1401,7 +1401,7 @@ And the retention oracle runs retrieval → NLI locally, escalating to an LLM on ambiguous band, with a disk cache (`--cache-dir`) keyed on (atom, text). Full sweeps stay opt-in (`--all-atoms`). -The cache key carries a version (`retention-cache-v5`). Entries written before failed +The cache key carries a version (`retention-cache-v6`). Entries written before failed adjudications were kept out of the cache can hold a failure stored as `absent`, entries written before a hedged verdict (`retained?`) counted as no verdict can hold it stored as `retained`, and entries written before a reply cut at the output ceiling was an error can hold a verdict parsed @@ -1410,6 +1410,13 @@ first run after upgrading therefore pays one adjudication call per pair the old `DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty calls now send 16384 where they sent none, so those entries miss too. +Entries written under `retention-cache-v5` are not read either: that key used to join the +statement and the text with a bare NUL byte, so a statement ending in NUL and a text starting with +NUL hashed the same bytes and one atom could replay another atom's verdict. The key now hashes the +statement's byte length as 8 bytes (big-endian) before the statement, which no statement and text +can collide on. The old files stay in `--cache-dir` and can be deleted, and the first run after +upgrading pays the adjudication calls again. + **Pass `--max-tokens` when a reply can outgrow the output ceiling.** Novelty calls ask for 16384 output tokens unless `--max-tokens` names a ceiling. litellm sends 4096 to Bedrock-hosted Claude when none is given, and on a dense skill a 25-span extraction batch can need more than diff --git a/dokimi/dokimi/novelty/retention.py b/dokimi/dokimi/novelty/retention.py index e5a200493..2bcf3404e 100644 --- a/dokimi/dokimi/novelty/retention.py +++ b/dokimi/dokimi/novelty/retention.py @@ -32,7 +32,7 @@ ABSENT = "absent" CONTRADICTED = "contradicted" _VERDICTS = (RETAINED, ABSENT, CONTRADICTED) -_CACHE_VERSION = b"retention-cache-v5\0" +_CACHE_VERSION = b"retention-cache-v6\0" _STOP = frozenset(""" a an the and or but if then than that this these those is are was were be been being @@ -265,7 +265,8 @@ def _key(atom: Atom, text: str) -> str: # Bumped when what an entry means changes: every older entry is then not read. # The reason for each bump is in docs/skill-novelty.md, under the cache key. digest.update(_CACHE_VERSION) - digest.update(atom.statement.strip().encode("utf-8", "surrogatepass")) # model output and files can hold a lone surrogate - digest.update(b"\0") + statement = atom.statement.strip().encode("utf-8", "surrogatepass") # model output and files can hold a lone surrogate + digest.update(len(statement).to_bytes(8, "big")) + digest.update(statement) digest.update(text.encode("utf-8", "surrogatepass")) return digest.hexdigest()[:32] diff --git a/dokimi/tests/test_lone_surrogate_cache_keys.py b/dokimi/tests/test_lone_surrogate_cache_keys.py index 9c24a1807..5ae504730 100644 --- a/dokimi/tests/test_lone_surrogate_cache_keys.py +++ b/dokimi/tests/test_lone_surrogate_cache_keys.py @@ -206,25 +206,39 @@ def _retention_key(statement: str, text: str) -> str: return _key(Atom(statement=statement, question="q", answer="a", id="a1"), text) -def _old_retention_key(statement: bytes, text: bytes) -> str: +def _v6_retention_key(statement: bytes, text: bytes) -> str: from dokimi.novelty.retention import _CACHE_VERSION - return _sha(_CACHE_VERSION + statement + b"\0" + text)[:32] + return _sha(_CACHE_VERSION + len(statement).to_bytes(8, "big") + statement + text)[:32] -def test_retention_key_of_a_utf8_statement_and_text_is_unchanged(): - old = _old_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8")) - assert _retention_key(" Use café? ", "tëxt") == old +def test_retention_key_of_a_utf8_statement_and_text_is_the_v6_layout(): + # Not the v5 key on purpose: the layout changed and _CACHE_VERSION moved to v6 (#847). + expected = _v6_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8")) + assert _retention_key(" Use café? ", "tëxt") == expected def test_retention_key_of_a_statement_with_a_lone_surrogate_is_a_digest(): - assert _retention_key(HIGH, "t") == _old_retention_key(b"n\xed\xa0\x80", b"t") - assert _retention_key(LOW, "t") == _old_retention_key(b"n\xed\xb3\xbf", b"t") + assert _retention_key(HIGH, "t") == _v6_retention_key(b"n\xed\xa0\x80", b"t") + assert _retention_key(LOW, "t") == _v6_retention_key(b"n\xed\xb3\xbf", b"t") def test_retention_key_of_a_text_with_a_lone_surrogate_is_a_digest(): - assert _retention_key("s", HIGH) == _old_retention_key(b"s", b"n\xed\xa0\x80") - assert _retention_key("s", LOW) == _old_retention_key(b"s", b"n\xed\xb3\xbf") + assert _retention_key("s", HIGH) == _v6_retention_key(b"s", b"n\xed\xa0\x80") + assert _retention_key("s", LOW) == _v6_retention_key(b"s", b"n\xed\xb3\xbf") + + +def test_retention_key_of_a_statement_ending_in_nul_is_apart_from_a_text_starting_with_nul(): + assert _retention_key("a\0", "") != _retention_key("a", "\0") + + +def test_retention_key_of_a_nul_inside_the_text_is_apart_from_one_inside_the_statement(): + assert _retention_key("a", "\0b") != _retention_key("a\0", "b") + + +def test_retention_key_is_stable_and_strips_the_statement(): + assert _retention_key(" a \n", "t") == _retention_key("a", "t") + assert _retention_key("a", "t") != _retention_key("a", " t") def test_url_client_cache_key_of_a_utf8_source_is_unchanged():