From b401c1d388d73c683a6cb2f5a9f8907edc5585b8 Mon Sep 17 00:00:00 2001 From: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:25:32 +0530 Subject: [PATCH 1/3] test: preserve interval contracts across numerical backends Signed-off-by: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> --- dokimi/tests/test_repo_compare_interval.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 12c4c0fb6..6e98396be 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -71,7 +71,12 @@ def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(monkeypatch, summe for v in (1e16, 1e150): # finite measures go up to SCORE_BOUND, 1e150 assert sign_flip_interval([v] * 6, sign_flip_p([v] * 6)) == (v, v) assert sign_flip_interval([-v] * 6, sign_flip_p([-v] * 6)) == (-v, -v) - assert sign_flip_interval([1e16] * 5 + [2e16], sign_flip_p([1e16] * 5 + [2e16])) == (1e16, 2e16) + d = [1e16] * 5 + [2e16] + lo, hi = sign_flip_interval(d, sign_flip_p(d)) + assert lo == 1e16 + assert hi is not None and 2e16 <= hi <= math.nextafter(2e16, math.inf) + test = ShiftTest(d) + assert test.p(hi) >= sign_flip.ALPHA > test.p(math.nextafter(hi, math.inf)) def test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right(monkeypatch): @@ -124,15 +129,24 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None -@pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], +@pytest.mark.parametrize(("summed", "p"), [(math.fsum, 0.0625), (_left_to_right, 0.046875)], ids=["compensated", "left_to_right"]) -def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p, lo): +def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p): # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625; the # printed p itself is sum()'s, 0.0625 compensated (3.12 on) and 0.047 left to right (3.11): 0 follows either monkeypatch.setattr(sign_flip, "sum", summed, raising=False) a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2 row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b])) - assert row["p"] == p and row["delta"]["lo"] == lo and row["interval_p_disagree"] is False + assert row["p"] == p and _includes_zero(row["delta"]) == (p >= 0.05) + assert row["interval_p_disagree"] is False + + +@pytest.mark.parametrize(("p_zero", "backend_boundary"), [(0.0625, 0.25), (0.046875, -0.25), (None, 0.25)]) +def test_the_callers_p_places_zero_when_the_backend_disagrees(monkeypatch, p_zero, backend_boundary): + monkeypatch.setattr(ShiftTest, "p", lambda self, theta: 0.5 if backend_boundary <= theta <= 1.5 else 0.01) + lo, hi = sign_flip_interval([1.0] * 6, p_zero) + assert (lo <= 0 <= hi) == (p_zero is None or p_zero >= 0.05) + assert hi == pytest.approx(1.5, abs=sign_flip.TOL) def test_the_interval_excludes_zero_exactly_when_p_is_below_005(): From 790da668ef17507a7621bdaf11ec49fa076b4e28 Mon Sep 17 00:00:00 2001 From: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:25:33 +0530 Subject: [PATCH 2/3] fix(repo): count physical Markdown source lines Signed-off-by: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> --- ...13_100500_mangeshraut712_markdown-lines.md | 3 ++ dokimi/dokimi/repos/l0/markdown.py | 5 ++- dokimi/tests/test_repo_l0_markdown_lines.py | 44 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md create mode 100644 dokimi/tests/test_repo_l0_markdown_lines.py diff --git a/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md b/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md new file mode 100644 index 000000000..69adc8c3c --- /dev/null +++ b/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md @@ -0,0 +1,3 @@ +### Fixed + +- Keep Markdown link, inline-code and fenced-code claim line numbers aligned with source files containing Unicode line separators. diff --git a/dokimi/dokimi/repos/l0/markdown.py b/dokimi/dokimi/repos/l0/markdown.py index b2768cfcf..d39e9fa72 100644 --- a/dokimi/dokimi/repos/l0/markdown.py +++ b/dokimi/dokimi/repos/l0/markdown.py @@ -47,7 +47,10 @@ def split(text: str) -> Pieces: out = Pieces() fence: Fence | None = None marker = "" - for n, line in enumerate(text.splitlines(), 1): + lines = re.split(r"\r\n?|\n", text) + if lines[-1] == "": + lines.pop() + for n, line in enumerate(lines, 1): m = _FENCE_RE.match(line) if m and m.group(1)[0] == "`" and "`" in line[m.end(1):]: m = None # CommonMark: a backtick fence's info string has no backticks, so this is inline code diff --git a/dokimi/tests/test_repo_l0_markdown_lines.py b/dokimi/tests/test_repo_l0_markdown_lines.py new file mode 100644 index 000000000..fdbd7710c --- /dev/null +++ b/dokimi/tests/test_repo_l0_markdown_lines.py @@ -0,0 +1,44 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Markdown claim locations use source-file lines, not Unicode separators.""" + +import pytest + +from dokimi.repos.l0.markdown import split + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +def test_unicode_separators_do_not_shift_markdown_claims(separator): + text = f"Intro{separator}continued\n[link](missing.md) and `src/missing.py`\n```bash\nmake missing\n```\n" + pieces = split(text) + assert [(n, target) for n, target, _ in pieces.links] == [(2, "missing.md")] + assert [(n, span) for n, span, _ in pieces.spans] == [(2, "src/missing.py")] + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(3, "bash", [(4, "make missing")])] + + +def test_unicode_separator_does_not_create_a_fence_boundary(): + pieces = split("prose\u2028```bash\n[link](missing.md)\n") + assert pieces.fences == [] + assert [(n, target) for n, target, _ in pieces.links] == [(2, "missing.md")] + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_scan_keeps_physical_line_numbers_after_unicode_separator(tmp_path, newline): + from dokimi.repos import parse_manifest + from dokimi.repos.l0 import scan_repo + from dokimi.repos.source import WorkTree + from tests.test_repo_l0 import MANIFEST, _repo + + guidance = newline.join(["Intro\u2028continued", "[link](missing.md)", "Paths: `src/missing.py`", ""]) + _repo(tmp_path, {"AGENTS.md": guidance, "src/keep.py": "", "CONTRIBUTING.md": ""}) + result = scan_repo(WorkTree(tmp_path), parse_manifest(MANIFEST)) + claims = {(c["kind"], c["claim"]): c["line"] for c in result["claims"]} + assert claims[("link", "missing.md")] == 2 + assert claims[("path", "src/missing.py")] == 3 + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_fences_keep_physical_newlines_and_trailing_blank_line(newline): + pieces = split(newline.join(["intro", "```bash", "make all", "", ""])) + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "bash", [(3, "make all"), (4, "")])] From 3b1cb2b4ba796b09bc454eee3f8b9f305eabc719 Mon Sep 17 00:00:00 2001 From: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:22:10 +0530 Subject: [PATCH 3/3] fix(repo): preserve Markdown recognition while counting source lines Signed-off-by: Mangesh Raut <95149253+mangeshraut712@users.noreply.github.com> --- dokimi/dokimi/repos/l0/markdown.py | 9 ++++--- dokimi/tests/test_repo_compare_interval.py | 1 + dokimi/tests/test_repo_l0_markdown_lines.py | 27 ++++++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/repos/l0/markdown.py b/dokimi/dokimi/repos/l0/markdown.py index d39e9fa72..afed1acab 100644 --- a/dokimi/dokimi/repos/l0/markdown.py +++ b/dokimi/dokimi/repos/l0/markdown.py @@ -47,10 +47,11 @@ def split(text: str) -> Pieces: out = Pieces() fence: Fence | None = None marker = "" - lines = re.split(r"\r\n?|\n", text) - if lines[-1] == "": - lines.pop() - for n, line in enumerate(lines, 1): + next_line = 1 + for chunk in text.splitlines(keepends=True): + n = next_line + next_line += chunk.endswith(("\r", "\n")) + line = chunk.splitlines()[0] m = _FENCE_RE.match(line) if m and m.group(1)[0] == "`" and "`" in line[m.end(1):]: m = None # CommonMark: a backtick fence's info string has no backticks, so this is inline code diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 6e98396be..333d4a770 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -146,6 +146,7 @@ def test_the_callers_p_places_zero_when_the_backend_disagrees(monkeypatch, p_zer monkeypatch.setattr(ShiftTest, "p", lambda self, theta: 0.5 if backend_boundary <= theta <= 1.5 else 0.01) lo, hi = sign_flip_interval([1.0] * 6, p_zero) assert (lo <= 0 <= hi) == (p_zero is None or p_zero >= 0.05) + assert lo == (1e-4 if p_zero is not None and p_zero < 0.05 else 0.0) assert hi == pytest.approx(1.5, abs=sign_flip.TOL) diff --git a/dokimi/tests/test_repo_l0_markdown_lines.py b/dokimi/tests/test_repo_l0_markdown_lines.py index fdbd7710c..c22d02332 100644 --- a/dokimi/tests/test_repo_l0_markdown_lines.py +++ b/dokimi/tests/test_repo_l0_markdown_lines.py @@ -17,10 +17,10 @@ def test_unicode_separators_do_not_shift_markdown_claims(separator): assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(3, "bash", [(4, "make missing")])] -def test_unicode_separator_does_not_create_a_fence_boundary(): +def test_unicode_separator_preserves_fence_matching_with_physical_locations(): pieces = split("prose\u2028```bash\n[link](missing.md)\n") - assert pieces.fences == [] - assert [(n, target) for n, target, _ in pieces.links] == [(2, "missing.md")] + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "[link](missing.md)")])] + assert pieces.links == [] @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) @@ -42,3 +42,24 @@ def test_scan_keeps_physical_line_numbers_after_unicode_separator(tmp_path, newl def test_fences_keep_physical_newlines_and_trailing_blank_line(newline): pieces = split(newline.join(["intro", "```bash", "make all", "", ""])) assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "bash", [(3, "make all"), (4, "")])] + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +def test_unicode_separator_preserves_closing_fences_and_reference_definitions(separator): + pieces = split(f"```bash\nmake all\n```{separator}Then run this\nSee [docs](missing.md).\n" + f"[x]: other.md{separator}more text\n") + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all")])] + assert [(n, target) for n, target, _ in pieces.links] == [(4, "missing.md"), (5, "other.md")] + + +def test_unicode_separator_preserves_span_and_info_string_matching(): + pieces = split("`src/keep.py\u2028and src/missing.py`\n```\u2028bash\nmake all\n```\n") + assert pieces.spans == [] + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "", [(2, "bash"), (3, "make all")])] + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_unicode_separator_preserves_blank_fenced_segments(separator, newline): + pieces = split(newline.join(["```bash", f"make all{separator}", "```", ""])) + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all"), (2, "")])]