Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions dokimi/changelog.d/20261117_140000_retention-key-nul.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **A novelty retention cache entry is no longer replayed for a different atom when a NUL separates them** — the key joined the statement and the text with a bare NUL, so a statement ending in NUL and a text starting with NUL hashed the same bytes. The key now hashes the statement's length before the statement. The cache version moves to `retention-cache-v6`, so entries already in `--cache-dir` are not read; the old files can be deleted, and the first run after upgrading pays the adjudication calls again.
9 changes: 8 additions & 1 deletion dokimi/docs/skill-novelty.md
Original file line number Diff line number Diff line change
Expand Up @@ -1401,7 +1401,7 @@ And the retention oracle runs retrieval → NLI locally, escalating to an LLM on
ambiguous band, with a disk cache (`--cache-dir`) keyed on (atom, text). Full sweeps stay opt-in
(`--all-atoms`).

The cache key carries a version (`retention-cache-v5`). Entries written before failed
The cache key carries a version (`retention-cache-v6`). Entries written before failed
adjudications were kept out of the cache can hold a failure stored as `absent`, entries written
before a hedged verdict (`retained?`) counted as no verdict can hold it stored as `retained`, and
entries written before a reply cut at the output ceiling was an error can hold a verdict parsed
Expand All @@ -1410,6 +1410,13 @@ first run after upgrading therefore pays one adjudication call per pair the old
`DOKIMI_RESPONSE_CACHE` replays it, but the response-cache key carries `max_tokens`, and novelty
calls now send 16384 where they sent none, so those entries miss too.

Entries written under `retention-cache-v5` are not read either: that key used to join the
statement and the text with a bare NUL byte, so a statement ending in NUL and a text starting with
NUL hashed the same bytes and one atom could replay another atom's verdict. The key now hashes the
statement's byte length as 8 bytes (big-endian) before the statement, which no statement and text
can collide on. The old files stay in `--cache-dir` and can be deleted, and the first run after
upgrading pays the adjudication calls again.

**Pass `--max-tokens` when a reply can outgrow the output ceiling.** Novelty calls ask for
16384 output tokens unless `--max-tokens` names a ceiling. litellm sends 4096 to Bedrock-hosted
Claude when none is given, and on a dense skill a 25-span extraction batch can need more than
Expand Down
7 changes: 4 additions & 3 deletions dokimi/dokimi/novelty/retention.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
ABSENT = "absent"
CONTRADICTED = "contradicted"
_VERDICTS = (RETAINED, ABSENT, CONTRADICTED)
_CACHE_VERSION = b"retention-cache-v5\0"
_CACHE_VERSION = b"retention-cache-v6\0"

_STOP = frozenset("""
a an the and or but if then than that this these those is are was were be been being
Expand Down Expand Up @@ -265,7 +265,8 @@ def _key(atom: Atom, text: str) -> str:
# Bumped when what an entry means changes: every older entry is then not read.
# The reason for each bump is in docs/skill-novelty.md, under the cache key.
digest.update(_CACHE_VERSION)
digest.update(atom.statement.strip().encode("utf-8", "surrogatepass")) # model output and files can hold a lone surrogate
digest.update(b"\0")
statement = atom.statement.strip().encode("utf-8", "surrogatepass") # model output and files can hold a lone surrogate
digest.update(len(statement).to_bytes(8, "big"))
digest.update(statement)
digest.update(text.encode("utf-8", "surrogatepass"))
return digest.hexdigest()[:32]
32 changes: 23 additions & 9 deletions dokimi/tests/test_lone_surrogate_cache_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,25 +206,39 @@ def _retention_key(statement: str, text: str) -> str:
return _key(Atom(statement=statement, question="q", answer="a", id="a1"), text)


def _old_retention_key(statement: bytes, text: bytes) -> str:
def _v6_retention_key(statement: bytes, text: bytes) -> str:
from dokimi.novelty.retention import _CACHE_VERSION

return _sha(_CACHE_VERSION + statement + b"\0" + text)[:32]
return _sha(_CACHE_VERSION + len(statement).to_bytes(8, "big") + statement + text)[:32]


def test_retention_key_of_a_utf8_statement_and_text_is_unchanged():
old = _old_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8"))
assert _retention_key(" Use café? ", "tëxt") == old
def test_retention_key_of_a_utf8_statement_and_text_is_the_v6_layout():
# Not the v5 key on purpose: the layout changed and _CACHE_VERSION moved to v6 (#847).
expected = _v6_retention_key("Use café?".encode("utf-8"), "tëxt".encode("utf-8"))
assert _retention_key(" Use café? ", "tëxt") == expected


def test_retention_key_of_a_statement_with_a_lone_surrogate_is_a_digest():
assert _retention_key(HIGH, "t") == _old_retention_key(b"n\xed\xa0\x80", b"t")
assert _retention_key(LOW, "t") == _old_retention_key(b"n\xed\xb3\xbf", b"t")
assert _retention_key(HIGH, "t") == _v6_retention_key(b"n\xed\xa0\x80", b"t")
assert _retention_key(LOW, "t") == _v6_retention_key(b"n\xed\xb3\xbf", b"t")


def test_retention_key_of_a_text_with_a_lone_surrogate_is_a_digest():
assert _retention_key("s", HIGH) == _old_retention_key(b"s", b"n\xed\xa0\x80")
assert _retention_key("s", LOW) == _old_retention_key(b"s", b"n\xed\xb3\xbf")
assert _retention_key("s", HIGH) == _v6_retention_key(b"s", b"n\xed\xa0\x80")
assert _retention_key("s", LOW) == _v6_retention_key(b"s", b"n\xed\xb3\xbf")


def test_retention_key_of_a_statement_ending_in_nul_is_apart_from_a_text_starting_with_nul():
assert _retention_key("a\0", "") != _retention_key("a", "\0")


def test_retention_key_of_a_nul_inside_the_text_is_apart_from_one_inside_the_statement():
assert _retention_key("a", "\0b") != _retention_key("a\0", "b")


def test_retention_key_is_stable_and_strips_the_statement():
assert _retention_key(" a \n", "t") == _retention_key("a", "t")
assert _retention_key("a", "t") != _retention_key("a", " t")


def test_url_client_cache_key_of_a_utf8_source_is_unchanged():
Expand Down
Loading