diff --git a/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md b/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md new file mode 100644 index 000000000..69adc8c3c --- /dev/null +++ b/dokimi/changelog.d/20261113_100500_mangeshraut712_markdown-lines.md @@ -0,0 +1,3 @@ +### Fixed + +- Keep Markdown link, inline-code and fenced-code claim line numbers aligned with source files containing Unicode line separators. diff --git a/dokimi/dokimi/repos/l0/markdown.py b/dokimi/dokimi/repos/l0/markdown.py index b2768cfcf..afed1acab 100644 --- a/dokimi/dokimi/repos/l0/markdown.py +++ b/dokimi/dokimi/repos/l0/markdown.py @@ -47,7 +47,11 @@ def split(text: str) -> Pieces: out = Pieces() fence: Fence | None = None marker = "" - for n, line in enumerate(text.splitlines(), 1): + next_line = 1 + for chunk in text.splitlines(keepends=True): + n = next_line + next_line += chunk.endswith(("\r", "\n")) + line = chunk.splitlines()[0] m = _FENCE_RE.match(line) if m and m.group(1)[0] == "`" and "`" in line[m.end(1):]: m = None # CommonMark: a backtick fence's info string has no backticks, so this is inline code diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 0fa4f2c86..78cbb81f3 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -71,7 +71,12 @@ def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(monkeypatch, summe for v in (1e16, 1e150): # finite measures go up to SCORE_BOUND, 1e150 assert sign_flip_interval([v] * 6, sign_flip_p([v] * 6)) == (v, v) assert sign_flip_interval([-v] * 6, sign_flip_p([-v] * 6)) == (-v, -v) - assert sign_flip_interval([1e16] * 5 + [2e16], sign_flip_p([1e16] * 5 + [2e16])) == (1e16, 2e16) + d = [1e16] * 5 + [2e16] + lo, hi = sign_flip_interval(d, sign_flip_p(d)) + assert lo == 1e16 + assert hi is not None and 2e16 <= hi <= math.nextafter(2e16, math.inf) + test = ShiftTest(d) + assert test.p(hi) >= sign_flip.ALPHA > test.p(math.nextafter(hi, math.inf)) def test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right(monkeypatch): @@ -124,15 +129,25 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None -@pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], +@pytest.mark.parametrize(("summed", "p"), [(math.fsum, 0.0625), (_left_to_right, 0.046875)], ids=["compensated", "left_to_right"]) -def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p, lo): +def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p): # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625; the # printed p itself is sum()'s, 0.0625 compensated (3.12 on) and 0.047 left to right (3.11): 0 follows either monkeypatch.setattr(sign_flip, "sum", summed, raising=False) a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2 row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b])) - assert row["p"] == p and row["delta"]["lo"] == lo and row["interval_p_disagree"] is False + assert row["p"] == p and _includes_zero(row["delta"]) == (p >= 0.05) + assert row["interval_p_disagree"] is False + + +@pytest.mark.parametrize(("p_zero", "backend_boundary"), [(0.0625, 0.25), (0.046875, -0.25), (None, 0.25)]) +def test_the_callers_p_places_zero_when_the_backend_disagrees(monkeypatch, p_zero, backend_boundary): + monkeypatch.setattr(ShiftTest, "p", lambda self, theta: 0.5 if backend_boundary <= theta <= 1.5 else 0.01) + lo, hi = sign_flip_interval([1.0] * 6, p_zero) + assert (lo <= 0 <= hi) == (p_zero is None or p_zero >= 0.05) + assert lo == (1e-4 if p_zero is not None and p_zero < 0.05 else 0.0) + assert hi == pytest.approx(1.5, abs=sign_flip.TOL) def test_the_interval_excludes_zero_exactly_when_p_is_below_005(): diff --git a/dokimi/tests/test_repo_l0_markdown_lines.py b/dokimi/tests/test_repo_l0_markdown_lines.py new file mode 100644 index 000000000..c22d02332 --- /dev/null +++ b/dokimi/tests/test_repo_l0_markdown_lines.py @@ -0,0 +1,65 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Markdown claim locations use source-file lines, not Unicode separators.""" + +import pytest + +from dokimi.repos.l0.markdown import split + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +def test_unicode_separators_do_not_shift_markdown_claims(separator): + text = f"Intro{separator}continued\n[link](missing.md) and `src/missing.py`\n```bash\nmake missing\n```\n" + pieces = split(text) + assert [(n, target) for n, target, _ in pieces.links] == [(2, "missing.md")] + assert [(n, span) for n, span, _ in pieces.spans] == [(2, "src/missing.py")] + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(3, "bash", [(4, "make missing")])] + + +def test_unicode_separator_preserves_fence_matching_with_physical_locations(): + pieces = split("prose\u2028```bash\n[link](missing.md)\n") + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "[link](missing.md)")])] + assert pieces.links == [] + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_scan_keeps_physical_line_numbers_after_unicode_separator(tmp_path, newline): + from dokimi.repos import parse_manifest + from dokimi.repos.l0 import scan_repo + from dokimi.repos.source import WorkTree + from tests.test_repo_l0 import MANIFEST, _repo + + guidance = newline.join(["Intro\u2028continued", "[link](missing.md)", "Paths: `src/missing.py`", ""]) + _repo(tmp_path, {"AGENTS.md": guidance, "src/keep.py": "", "CONTRIBUTING.md": ""}) + result = scan_repo(WorkTree(tmp_path), parse_manifest(MANIFEST)) + claims = {(c["kind"], c["claim"]): c["line"] for c in result["claims"]} + assert claims[("link", "missing.md")] == 2 + assert claims[("path", "src/missing.py")] == 3 + + +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_fences_keep_physical_newlines_and_trailing_blank_line(newline): + pieces = split(newline.join(["intro", "```bash", "make all", "", ""])) + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "bash", [(3, "make all"), (4, "")])] + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +def test_unicode_separator_preserves_closing_fences_and_reference_definitions(separator): + pieces = split(f"```bash\nmake all\n```{separator}Then run this\nSee [docs](missing.md).\n" + f"[x]: other.md{separator}more text\n") + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all")])] + assert [(n, target) for n, target, _ in pieces.links] == [(4, "missing.md"), (5, "other.md")] + + +def test_unicode_separator_preserves_span_and_info_string_matching(): + pieces = split("`src/keep.py\u2028and src/missing.py`\n```\u2028bash\nmake all\n```\n") + assert pieces.spans == [] + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "", [(2, "bash"), (3, "make all")])] + + +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"]) +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +def test_unicode_separator_preserves_blank_fenced_segments(separator, newline): + pieces = split(newline.join(["```bash", f"make all{separator}", "```", ""])) + assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all"), (2, "")])]