From d42d056830d8208ee949edca5a603c03a894e097 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 17:13:07 +0000 Subject: [PATCH 1/3] fix(novelty): retention cache key length-prefixes the statement (#847) The retention cache key joined the stripped statement and the text with a bare NUL byte. A statement ending in NUL and a text starting with NUL hashed the same bytes (`Atom(statement="a\0")` with text "" and `Atom(statement="a")` with text "\0"), so a cached verdict could be replayed for a different atom. The key now hashes the statement's byte length as 8 bytes big-endian, then the statement, then the text, both still encoded with surrogatepass. No statement and text can collide on that layout. Every key changes, so `_CACHE_VERSION` moves to `retention-cache-v6` and v5 entries are not read; docs/skill-novelty.md gives the reason. The three tests in test_lone_surrogate_cache_keys.py that pinned the v5 byte layout now pin the v6 one; new tests cover the NUL pair, a NUL in the middle, and stripping. Test-narrowed: test_retention_key_of_a_utf8_statement_and_text_is_unchanged is renamed to test_retention_key_of_a_utf8_statement_and_text_is_the_v6_layout; the UTF-8 key is no longer the v5 one on purpose, since the layout changed and the cache version moved to v6. Signed-off-by: Anton Volkov --- .../20261117_140000_retention-key-nul.md | 8 +++++ dokimi/docs/skill-novelty.md | 4 ++- dokimi/dokimi/novelty/retention.py | 7 ++-- .../tests/test_lone_surrogate_cache_keys.py | 33 ++++++++++++++----- 4 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 dokimi/changelog.d/20261117_140000_retention-key-nul.md diff --git a/dokimi/changelog.d/20261117_140000_retention-key-nul.md b/dokimi/changelog.d/20261117_140000_retention-key-nul.md new file mode 100644 index 000000000..0d6e145c4 --- /dev/null +++ b/dokimi/changelog.d/20261117_140000_retention-key-nul.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A novelty retention cache entry is no longer replayed for a different atom when a NUL separates them** — the key joined the statement and the text with a bare NUL, so a statement ending in NUL and a text starting with NUL hashed the same bytes. The key now hashes the statement's length before the statement. The cache version moves to `retention-cache-v6`, so entries already in `--cache-dir` are not read; the old files can be deleted, and the first run after upgrading pays the adjudication calls again. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index ddefde067..357f1304c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1401,7 +1401,7 @@ And the retention oracle runs retrieval → NLI locally, escalating to an LLM on ambiguous band, with a disk cache (`--cache-dir`) keyed on (atom, text). Full sweeps stay opt-in (`--all-atoms`). -The cache key carries a version (`retention-cache-v5`). Entries written before failed +The cache key carries a version (`retention-cache-v6`). Entries written before failed adjudications were kept out of the cache can hold a failure stored as `absent`, entries written before a hedged verdict (`retained?`) counted as no verdict can hold it stored as `retained`, and entries written before a reply cut at the output ceiling was an error can hold a verdict parsed @@ -1410,6 +1410,8 @@ first run after upgrading therefore pays one adjudication call per pair the old `DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty calls now send 16384 where they sent none, so those entries miss too. +Entries written under `retention-cache-v5` are not read either: that key used to join the statement and the text with a bare NUL byte, so a statement ending in NUL and a text starting with NUL hashed the same bytes and one atom could replay another atom's verdict. The key now hashes the statement's byte length as 8 bytes (big-endian) before the statement, which no statement and text can collide on. Every key changed, so entries written under `retention-cache-v5` are not read. The old files stay in `--cache-dir` and can be deleted, and the first run after upgrading pays the adjudication calls again. + **Pass `--max-tokens` when a reply can outgrow the output ceiling.** Novelty calls ask for 16384 output tokens unless `--max-tokens` names a ceiling. litellm sends 4096 to Bedrock-hosted Claude when none is given, and on a dense skill a 25-span extraction batch can need more than diff --git a/dokimi/dokimi/novelty/retention.py b/dokimi/dokimi/novelty/retention.py index e5a200493..2bcf3404e 100644 --- a/dokimi/dokimi/novelty/retention.py +++ b/dokimi/dokimi/novelty/retention.py @@ -32,7 +32,7 @@ ABSENT = "absent" CONTRADICTED = "contradicted" _VERDICTS = (RETAINED, ABSENT, CONTRADICTED) -_CACHE_VERSION = b"retention-cache-v5\0" +_CACHE_VERSION = b"retention-cache-v6\0" _STOP = frozenset(""" a an the and or but if then than that this these those is are was were be been being @@ -265,7 +265,8 @@ def _key(atom: Atom, text: str) -> str: # Bumped when what an entry means changes: every older entry is then not read. # The reason for each bump is in docs/skill-novelty.md, under the cache key. digest.update(_CACHE_VERSION) - digest.update(atom.statement.strip().encode("utf-8", "surrogatepass")) # model output and files can hold a lone surrogate - digest.update(b"\0") + statement = atom.statement.strip().encode("utf-8", "surrogatepass") # model output and files can hold a lone surrogate + digest.update(len(statement).to_bytes(8, "big")) + digest.update(statement) digest.update(text.encode("utf-8", "surrogatepass")) return digest.hexdigest()[:32] diff --git a/dokimi/tests/test_lone_surrogate_cache_keys.py b/dokimi/tests/test_lone_surrogate_cache_keys.py index 9c24a1807..301530882 100644 --- a/dokimi/tests/test_lone_surrogate_cache_keys.py +++ b/dokimi/tests/test_lone_surrogate_cache_keys.py @@ -206,25 +206,40 @@ def _retention_key(statement: str, text: str) -> str: return _key(Atom(statement=statement, question="q", answer="a", id="a1"), text) -def _old_retention_key(statement: bytes, text: bytes) -> str: +def _v6_retention_key(statement: bytes, text: bytes) -> str: from dokimi.novelty.retention import _CACHE_VERSION - return _sha(_CACHE_VERSION + statement + b"\0" + text)[:32] + return _sha(_CACHE_VERSION + len(statement).to_bytes(8, "big") + statement + text)[:32] -def test_retention_key_of_a_utf8_statement_and_text_is_unchanged(): - old = _old_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8")) - assert _retention_key(" Use café? ", "tëxt") == old +def test_retention_key_of_a_utf8_statement_and_text_is_the_v6_layout(): + # Not the v5 key on purpose: the layout changed and _CACHE_VERSION moved to v6 (#847). + expected = _v6_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8")) + assert _retention_key(" Use café? ", "tëxt") == expected def test_retention_key_of_a_statement_with_a_lone_surrogate_is_a_digest(): - assert _retention_key(HIGH, "t") == _old_retention_key(b"n\xed\xa0\x80", b"t") - assert _retention_key(LOW, "t") == _old_retention_key(b"n\xed\xb3\xbf", b"t") + assert _retention_key(HIGH, "t") == _v6_retention_key(b"n\xed\xa0\x80", b"t") + assert _retention_key(LOW, "t") == _v6_retention_key(b"n\xed\xb3\xbf", b"t") def test_retention_key_of_a_text_with_a_lone_surrogate_is_a_digest(): - assert _retention_key("s", HIGH) == _old_retention_key(b"s", b"n\xed\xa0\x80") - assert _retention_key("s", LOW) == _old_retention_key(b"s", b"n\xed\xb3\xbf") + assert _retention_key("s", HIGH) == _v6_retention_key(b"s", b"n\xed\xa0\x80") + assert _retention_key("s", LOW) == _v6_retention_key(b"s", b"n\xed\xb3\xbf") + + +def test_retention_key_of_a_statement_ending_in_nul_is_apart_from_a_text_starting_with_nul(): + assert _retention_key("a\0", "") != _retention_key("a", "\0") + + +def test_retention_key_of_a_nul_inside_the_text_is_apart_from_one_inside_the_statement(): + assert _retention_key("a", "\0b") != _retention_key("a\0", "b") + + +def test_retention_key_is_stable_and_strips_the_statement(): + assert _retention_key("a", "t") == _retention_key("a", "t") + assert _retention_key(" a \n", "t") == _retention_key("a", "t") + assert _retention_key("a", "t") != _retention_key("a", " t") def test_url_client_cache_key_of_a_utf8_source_is_unchanged(): From 29b52ceeebf29440062c46b9d5fd067d4a217741 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:03:44 -0700 Subject: [PATCH 2/3] docs(novelty): drop the repeated v5 cache sentence and wrap the paragraph Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/skill-novelty.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 357f1304c..e4f45a847 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1410,7 +1410,12 @@ first run after upgrading therefore pays one adjudication call per pair the old `DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty calls now send 16384 where they sent none, so those entries miss too. -Entries written under `retention-cache-v5` are not read either: that key used to join the statement and the text with a bare NUL byte, so a statement ending in NUL and a text starting with NUL hashed the same bytes and one atom could replay another atom's verdict. The key now hashes the statement's byte length as 8 bytes (big-endian) before the statement, which no statement and text can collide on. Every key changed, so entries written under `retention-cache-v5` are not read. The old files stay in `--cache-dir` and can be deleted, and the first run after upgrading pays the adjudication calls again. +Entries written under `retention-cache-v5` are not read either: that key used to join the +statement and the text with a bare NUL byte, so a statement ending in NUL and a text starting with +NUL hashed the same bytes and one atom could replay another atom's verdict. The key now hashes the +statement's byte length as 8 bytes (big-endian) before the statement, which no statement and text +can collide on. The old files stay in `--cache-dir` and can be deleted, and the first run after +upgrading pays the adjudication calls again. **Pass `--max-tokens` when a reply can outgrow the output ceiling.** Novelty calls ask for 16384 output tokens unless `--max-tokens` names a ceiling. litellm sends 4096 to Bedrock-hosted From 940a6ce050c95625f770fe7156934cf6c0c78cde Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:03:47 -0700 Subject: [PATCH 3/3] test(novelty): drop the retention key assert that compares a call with itself Signed-off-by: Vladislav Perevezentsev --- dokimi/tests/test_lone_surrogate_cache_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dokimi/tests/test_lone_surrogate_cache_keys.py b/dokimi/tests/test_lone_surrogate_cache_keys.py index 301530882..5ae504730 100644 --- a/dokimi/tests/test_lone_surrogate_cache_keys.py +++ b/dokimi/tests/test_lone_surrogate_cache_keys.py @@ -237,7 +237,6 @@ def test_retention_key_of_a_nul_inside_the_text_is_apart_from_one_inside_the_sta def test_retention_key_is_stable_and_strips_the_statement(): - assert _retention_key("a", "t") == _retention_key("a", "t") assert _retention_key(" a \n", "t") == _retention_key("a", "t") assert _retention_key("a", "t") != _retention_key("a", " t")