From 1b1d7eeec4580dd0eb2fec125e16206e724e0d66 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 19:49:10 +0000 Subject: [PATCH 1/3] test(repo): the code-span stripping test bounds CPU growth, not wall-clock time (#882) test_code_span_stripping_is_linear_on_backtick_runs bounded each call by 5 s of wall-clock time, so a stalled runner failed it although _strip_code_spans is linear (about 15 ms on its largest input). It now takes the CPU-time growth from n to 4n (tests/growth_timing.py, the fix of #505) on each of the three input shapes and asserts it stays under GROWTH_LIMIT. The three equality asserts and the CodeQL py/redos reason are unchanged. Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_budget.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dokimi/tests/test_repo_budget.py b/dokimi/tests/test_repo_budget.py index ad11ef7d4..4e59edcf0 100644 --- a/dokimi/tests/test_repo_budget.py +++ b/dokimi/tests/test_repo_budget.py @@ -15,6 +15,7 @@ from dokimi.repos.budget import budget as _budget from dokimi.repos.source import GitTree, SourceError, WorkTree from tests.cli_refusal import refusal +from tests.growth_timing import GROWTH_LIMIT, growth NAMES = ("CLAUDE.md", "CLAUDE.local.md", "AGENTS.md") budget = partial(_budget, claude_displaces_agents=True) # as `repo budget --harness claude-code` calls it @@ -692,15 +693,12 @@ def test_a_code_span_wrapped_onto_the_next_line_hides_its_import(tmp_path): def test_code_span_stripping_is_linear_on_backtick_runs(): """Long backtick runs once made the code-span regex backtrack exponentially (CodeQL py/redos).""" - import time - from dokimi.repos.budget import _strip_code_spans - for text in ("`" * 50_000, "`" + "`a" * 50_000, "``" + "` " * 50_000 + "\n\n@x.md"): - started = time.monotonic() - _strip_code_spans(text) - # Permissive timing check to avoid flakiness on slower/contended runners - assert time.monotonic() - started < 5 + shapes = (lambda k: "`" * k, lambda k: "`" + "`a" * k, lambda k: "``" + "` " * k + "\n\n@x.md") + for shape in shapes: + ratio, _ = growth(lambda k, shape=shape: partial(_strip_code_spans, shape(k)), 1_000) + assert ratio < GROWTH_LIMIT assert _strip_code_spans("a `b` c ``d`e`` f") == "a c f" assert _strip_code_spans("`a\nb` @x.md") == " @x.md" assert _strip_code_spans("`a\n\nb` @x.md") == "`a\n\nb` @x.md" From 23feb9b136263e8f80255918a890d437a89e55c9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 19:49:14 +0000 Subject: [PATCH 2/3] fix(llm): the JSON extractors scan a reply of unclosed openers in linear time (#883) extract_json_array("[" * 100000) and extract_json_object("{" * 100000) took about 10 s each: the last fallback, re.search(r"\[[\s\S]*\]", text), starts a greedy scan to the end at every opener and backtracks, so a run of openers with no closer is quadratic (CPU time ratio from n to 4n about 16). With the 60 s test timeout a slow runner failed test_a_bare_deep_reply_is_a_parse_error_not_a_crash. The block is now text[first opener : last closer + 1], which is the span the greedy regex matched, through a helper shared by both functions. The fence regex was quadratic too on "```json" followed by n spaces and no closing fence (ratio about 15.6): \s* gave back one space at a time and rescanned the lazy body each time. The body is stripped by the caller, so the \s* is dropped and the result is unchanged. New tests pin the growth of both extractors on a run of "[", of "{" and of an unclosed fence plus spaces, and that the fallback still finds the same blocks. Signed-off-by: Anton Volkov --- .../20261117_150000_json-extract-linear.md | 8 ++++ dokimi/dokimi/llm_json.py | 25 +++++++---- dokimi/tests/test_fuzz_targets.py | 43 +++++++++++++++++++ 3 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 dokimi/changelog.d/20261117_150000_json-extract-linear.md diff --git a/dokimi/changelog.d/20261117_150000_json-extract-linear.md b/dokimi/changelog.d/20261117_150000_json-extract-linear.md new file mode 100644 index 000000000..f51cd582c --- /dev/null +++ b/dokimi/changelog.d/20261117_150000_json-extract-linear.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A model reply that opens many `[` or `{` and never closes them is rejected at once** — `extract_json_array` and `extract_json_object` took their last-resort block with a regex that backtracked quadratically (about 10 s for 100000 openers), and the fence regex did the same on a fence followed by a long run of whitespace. Both now find the block with `str.find` and `str.rfind`, and the fence regex no longer retries per space. The text each function returns or rejects is unchanged. diff --git a/dokimi/dokimi/llm_json.py b/dokimi/dokimi/llm_json.py index 582fffee5..c528c7360 100644 --- a/dokimi/dokimi/llm_json.py +++ b/dokimi/dokimi/llm_json.py @@ -8,6 +8,15 @@ import json import re +# No \s* before the body: callers strip it, and \s* retries the lazy body once per space (#883). +_FENCE = re.compile(r"```(?:json)?([\s\S]*?)```") + + +def _outer_block(text: str, opener: str, closer: str) -> str | None: + """From the first ``opener`` to the last ``closer`` after it; a regex backtracks quadratically on a run of openers.""" + i, j = text.find(opener), text.rfind(closer) + return text[i : j + 1] if i != -1 and j > i else None + def extract_json_object(text: str) -> dict: """Extract a JSON object from LLM output, handling markdown fences and leading text. @@ -28,7 +37,7 @@ def extract_json_object(text: str) -> dict: except (ValueError, RecursionError): # not JSON here: try the next form pass # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + fence = _FENCE.search(text) if fence: try: result = json.loads(fence.group(1).strip()) @@ -37,10 +46,10 @@ def extract_json_object(text: str) -> dict: except (ValueError, RecursionError): pass # First { ... } block - m = re.search(r"\{[\s\S]*\}", text) - if m: + block = _outer_block(text, "{", "}") + if block is not None: try: - result = json.loads(m.group(0)) + result = json.loads(block) except RecursionError as exc: raise ValueError(f"JSON in LLM response is nested too deeply: {text[:200]!r}") from exc if isinstance(result, dict): @@ -61,7 +70,7 @@ def extract_json_array(text: str) -> list: except (ValueError, RecursionError): # not JSON here: try the next form pass # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + fence = _FENCE.search(text) if fence: try: result = json.loads(fence.group(1).strip()) @@ -70,10 +79,10 @@ def extract_json_array(text: str) -> list: except (ValueError, RecursionError): pass # First [ ... ] block - m = re.search(r"\[[\s\S]*\]", text) - if m: + block = _outer_block(text, "[", "]") + if block is not None: try: - result = json.loads(m.group(0)) + result = json.loads(block) except RecursionError as exc: raise ValueError(f"JSON in LLM response is nested too deeply: {text[:200]!r}") from exc if isinstance(result, list): diff --git a/dokimi/tests/test_fuzz_targets.py b/dokimi/tests/test_fuzz_targets.py index a1e1e3626..ae343b874 100644 --- a/dokimi/tests/test_fuzz_targets.py +++ b/dokimi/tests/test_fuzz_targets.py @@ -84,6 +84,49 @@ def test_a_bare_deep_reply_is_a_parse_error_not_a_crash(extract_name: str, text: getattr(llm_json, extract_name)(text) +def _outcome(name: str, text: str) -> str: + from dokimi import llm_json + + try: + return repr(getattr(llm_json, name)(text)) + except ValueError: + return "ValueError" + + +@pytest.mark.parametrize("name", ["extract_json_object", "extract_json_array"]) +@pytest.mark.parametrize( + "shape", + [ + lambda k: "[" * k, + lambda k: "{" * k, + lambda k: "```json" + " " * k + "x", + ], + ids=["brackets", "braces", "unclosed-fence-spaces"], +) +def test_extractors_scan_a_reply_without_a_closer_in_linear_time(name: str, shape) -> None: + # CPU time at 4n over n, not a wall-clock bound: the last-block regexes were quadratic (#883). + from tests.growth_timing import GROWTH_LIMIT, growth + + ratio, outcome = growth(lambda k: lambda: _outcome(name, shape(k)), 4000) + assert outcome == "ValueError" + assert ratio < GROWTH_LIMIT + + +@pytest.mark.parametrize( + ("name", "text", "want"), + [ + ("extract_json_array", "x [1, [2]] y", "[1, [2]]"), + ("extract_json_array", "a ] [1] b", "[1]"), + ("extract_json_array", "x [1]z", "[1]"), + ("extract_json_object", '[{"a": 1}]', "{'a': 1}"), + ("extract_json_object", 'x {"a": {"b": 2}} y', "{'a': {'b': 2}}"), + ("extract_json_object", '{"a": 1} then }', "ValueError"), + ], +) +def test_the_last_fallback_still_takes_the_outermost_block(name: str, text: str, want: str) -> None: + assert _outcome(name, text) == want + + def test_a_date_that_is_not_real_is_invalid_frontmatter() -> None: # PyYAML raises a plain ValueError for 2026-02-30: the lenient reader reads no metadata, # and the strict one raises the YAML error its docstring names. From b1f6fa1b8680da06870b127d2a3201123ca21bbc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 22:10:39 +0000 Subject: [PATCH 3/3] test(llm): pin the message of a reply with a closer and no opener Settles the review thread on test_fuzz_targets.py: `_outcome` keeps only the exception type, so `i != -1 or j > i` in `_outer_block` changed the message of `extract_json_array("a]")` from "No JSON array found" to a decode error and no test failed. The new test asserts the ValueError message for `a]`. Signed-off-by: Anton Volkov --- dokimi/tests/test_fuzz_targets.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dokimi/tests/test_fuzz_targets.py b/dokimi/tests/test_fuzz_targets.py index ae343b874..18623843b 100644 --- a/dokimi/tests/test_fuzz_targets.py +++ b/dokimi/tests/test_fuzz_targets.py @@ -127,6 +127,13 @@ def test_the_last_fallback_still_takes_the_outermost_block(name: str, text: str, assert _outcome(name, text) == want +def test_a_reply_with_a_closer_and_no_opener_keeps_the_head_message() -> None: + from dokimi.llm_json import extract_json_array + + with pytest.raises(ValueError, match="No JSON array found"): + extract_json_array("a]") + + def test_a_date_that_is_not_real_is_invalid_frontmatter() -> None: # PyYAML raises a plain ValueError for 2026-02-30: the lenient reader reads no metadata, # and the strict one raises the YAML error its docstring names.