Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- Keep Markdown link, inline-code and fenced-code claim line numbers aligned with source files containing Unicode line separators.
6 changes: 5 additions & 1 deletion dokimi/dokimi/repos/l0/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,11 @@ def split(text: str) -> Pieces:
out = Pieces()
fence: Fence | None = None
marker = ""
for n, line in enumerate(text.splitlines(), 1):
next_line = 1
for chunk in text.splitlines(keepends=True):
n = next_line
next_line += chunk.endswith(("\r", "\n"))
line = chunk.splitlines()[0]
m = _FENCE_RE.match(line)
if m and m.group(1)[0] == "`" and "`" in line[m.end(1):]:
m = None # CommonMark: a backtick fence's info string has no backticks, so this is inline code
Expand Down
23 changes: 19 additions & 4 deletions dokimi/tests/test_repo_compare_interval.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,12 @@ def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(monkeypatch, summe
for v in (1e16, 1e150): # finite measures go up to SCORE_BOUND, 1e150
assert sign_flip_interval([v] * 6, sign_flip_p([v] * 6)) == (v, v)
assert sign_flip_interval([-v] * 6, sign_flip_p([-v] * 6)) == (-v, -v)
assert sign_flip_interval([1e16] * 5 + [2e16], sign_flip_p([1e16] * 5 + [2e16])) == (1e16, 2e16)
d = [1e16] * 5 + [2e16]
lo, hi = sign_flip_interval(d, sign_flip_p(d))
assert lo == 1e16
assert hi is not None and 2e16 <= hi <= math.nextafter(2e16, math.inf)
test = ShiftTest(d)
assert test.p(hi) >= sign_flip.ALPHA > test.p(math.nextafter(hi, math.inf))


def test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right(monkeypatch):
Expand Down Expand Up @@ -124,15 +129,25 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo():
assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None


@pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)],
@pytest.mark.parametrize(("summed", "p"), [(math.fsum, 0.0625), (_left_to_right, 0.046875)],
ids=["compensated", "left_to_right"])
def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p, lo):
def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: the exact lo pins (0.0 compensated, 1e-4 left to right) are gone, and hi is now a one-ULP window. The body mentions the one-ULP allowance, not the dropped lo pins. Is the new boundary check (test.p(hi) >= ALPHA > test.p(nextafter(hi))) enough to pin the ends? I could not run these 7 tests here (scipy stub fails on both trees).

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In test_the_callers_p_places_zero_when_the_backend_disagrees, lo is pinned to 0.0 when the backend disagrees. For real delta intervals where floating-point rounding varies across runtime math libraries, test.p(hi) >= ALPHA > test.p(nextafter(hi)) pins hi to the exact single step of p across ALPHA, while lo maintains the sign constraint without relying on non-portable machine-precision constants. The PR description has been updated to clarify what the tests stub and verify.

# at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625; the
# printed p itself is sum()'s, 0.0625 compensated (3.12 on) and 0.047 left to right (3.11): 0 follows either
monkeypatch.setattr(sign_flip, "sum", summed, raising=False)
a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2
row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b]))
assert row["p"] == p and row["delta"]["lo"] == lo and row["interval_p_disagree"] is False
assert row["p"] == p and _includes_zero(row["delta"]) == (p >= 0.05)
assert row["interval_p_disagree"] is False


@pytest.mark.parametrize(("p_zero", "backend_boundary"), [(0.0625, 0.25), (0.046875, -0.25), (None, 0.25)])
def test_the_callers_p_places_zero_when_the_backend_disagrees(monkeypatch, p_zero, backend_boundary):
monkeypatch.setattr(ShiftTest, "p", lambda self, theta: 0.5 if backend_boundary <= theta <= 1.5 else 0.01)
lo, hi = sign_flip_interval([1.0] * 6, p_zero)
assert (lo <= 0 <= hi) == (p_zero is None or p_zero >= 0.05)
assert lo == (1e-4 if p_zero is not None and p_zero < 0.05 else 0.0)
assert hi == pytest.approx(1.5, abs=sign_flip.TOL)


def test_the_interval_excludes_zero_exactly_when_p_is_below_005():
Expand Down
65 changes: 65 additions & 0 deletions dokimi/tests/test_repo_l0_markdown_lines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""Markdown claim locations use source-file lines, not Unicode separators."""

import pytest

from dokimi.repos.l0.markdown import split


@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"])
def test_unicode_separators_do_not_shift_markdown_claims(separator):
text = f"Intro{separator}continued\n[link](missing.md) and `src/missing.py`\n```bash\nmake missing\n```\n"
pieces = split(text)
assert [(n, target) for n, target, _ in pieces.links] == [(2, "missing.md")]
assert [(n, span) for n, span, _ in pieces.spans] == [(2, "src/missing.py")]
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(3, "bash", [(4, "make missing")])]


def test_unicode_separator_preserves_fence_matching_with_physical_locations():
pieces = split("prose\u2028```bash\n[link](missing.md)\n")
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "[link](missing.md)")])]
assert pieces.links == []


@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
def test_scan_keeps_physical_line_numbers_after_unicode_separator(tmp_path, newline):
from dokimi.repos import parse_manifest
from dokimi.repos.l0 import scan_repo
from dokimi.repos.source import WorkTree
from tests.test_repo_l0 import MANIFEST, _repo

guidance = newline.join(["Intro\u2028continued", "[link](missing.md)", "Paths: `src/missing.py`", ""])
_repo(tmp_path, {"AGENTS.md": guidance, "src/keep.py": "", "CONTRIBUTING.md": ""})
result = scan_repo(WorkTree(tmp_path), parse_manifest(MANIFEST))
claims = {(c["kind"], c["claim"]): c["line"] for c in result["claims"]}
assert claims[("link", "missing.md")] == 2
assert claims[("path", "src/missing.py")] == 3


@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
def test_fences_keep_physical_newlines_and_trailing_blank_line(newline):
pieces = split(newline.join(["intro", "```bash", "make all", "", ""]))
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "bash", [(3, "make all"), (4, "")])]


@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"])
def test_unicode_separator_preserves_closing_fences_and_reference_definitions(separator):
pieces = split(f"```bash\nmake all\n```{separator}Then run this\nSee [docs](missing.md).\n"
f"[x]: other.md{separator}more text\n")
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all")])]
assert [(n, target) for n, target, _ in pieces.links] == [(4, "missing.md"), (5, "other.md")]


def test_unicode_separator_preserves_span_and_info_string_matching():
pieces = split("`src/keep.py\u2028and src/missing.py`\n```\u2028bash\nmake all\n```\n")
assert pieces.spans == []
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(2, "", [(2, "bash"), (3, "make all")])]


@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85", "\v", "\f", "\x1c", "\x1d", "\x1e"])
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
def test_unicode_separator_preserves_blank_fenced_segments(separator, newline):
pieces = split(newline.join(["```bash", f"make all{separator}", "```", ""]))
assert [(f.line, f.lang, f.lines) for f in pieces.fences] == [(1, "bash", [(2, "make all"), (2, "")])]
Loading