From d4904c814604aa625447c5650a51323528033093 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:42:53 +0000 Subject: [PATCH 1/2] Translation checks avoid repeated row and interval scans Index normalized table rows while preserving first-match offsets and unmatched row order. Use prefix maximum ends for protected-line lookups. Signed-off-by: zlareb1 --- scripts/i18n/chunks.py | 29 ++++++--- tests/unit/test_i18n_request_bounds.py | 88 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/scripts/i18n/chunks.py b/scripts/i18n/chunks.py index 7f6d139ebd..a2586e1896 100644 --- a/scripts/i18n/chunks.py +++ b/scripts/i18n/chunks.py @@ -3,9 +3,11 @@ from __future__ import annotations import re +from bisect import bisect_right +from collections import defaultdict, deque from dataclasses import replace from html.parser import HTMLParser -from itertools import pairwise +from itertools import accumulate, pairwise from xml.etree.ElementTree import Element from markdown import Markdown @@ -129,14 +131,19 @@ def _site_line(line: str) -> tuple[int, str]: def _site_lists(text: str, protected: list[tuple[int, int]]) -> list[tuple[int, int]]: # The site's two-space list extension has no source maps; protect its continuations # in addition to CommonMark's marker-width-dependent ranges. + ordered = sorted(protected) + starts = [a for a, _ in ordered] + ends = list(accumulate((b for _, b in ordered), max)) + exact_starts = set(starts) spans = [] active, after_blank, offset = None, False, 0 for line in text.splitlines(keepends=True): - inside = any(a <= offset < b for a, b in protected) + index = bisect_right(starts, offset) + inside = bool(index and offset < ends[index - 1]) depth, logical = _site_line(line) item = None if inside else re.match(r"^( *)(?:[-+*]|\d+[.)])\s", logical) indent = len(logical) - len(logical.lstrip(" ")) - new_block = not inside or any(a == offset for a, _ in protected) + new_block = not inside or offset in exact_starts if ( active is not None and new_block @@ -438,17 +445,21 @@ def table_labels(text: str) -> dict[tuple[str, int, int], Label]: """Cell text from both existing dialects; source maps survive inline-code pipes.""" github = _github_rows(text) site = _site_tables(text).rows - remaining = list(github) + by_label: dict[tuple[Label, ...], deque[int]] = defaultdict(deque) + for index, row in enumerate(github): + by_label[tuple(replace(c, start=0, end=0) for c in row)].append(index) + consumed = set() rows = [] for row in site: - match = next((r for r in remaining if [replace(c, start=0, end=0) for c in r] == row), None) - if match is not None: - remaining.remove(match) - rows.append(match) + matches = by_label.get(tuple(row)) + if matches: + index = matches.popleft() + consumed.add(index) + rows.append(github[index]) else: # The site parser has no source maps. These obligations remain page-wide. rows.append([replace(c, end=len(text)) for c in row]) - rows.extend(remaining) + rows.extend(row for index, row in enumerate(github) if index not in consumed) labels = {} for index, row in enumerate(rows): field = row[0].code[0] if row and not row[0].text and len(row[0].code) == 1 else "" diff --git a/tests/unit/test_i18n_request_bounds.py b/tests/unit/test_i18n_request_bounds.py index b17180b3d4..8311a2f27f 100644 --- a/tests/unit/test_i18n_request_bounds.py +++ b/tests/unit/test_i18n_request_bounds.py @@ -1,10 +1,14 @@ """Translation requests bound source bytes without breaking Markdown units.""" from collections import Counter +from dataclasses import replace +from types import SimpleNamespace import pytest +from scripts.i18n import chunks from scripts.i18n import translate as t +from scripts.i18n.context import Label MAXIMUM = 16_384 @@ -295,3 +299,87 @@ def test_valid_table_reflow_is_written_in_each_supported_container( ).read_text() ) assert not (tmp_path / t.target_path(t.LANGUAGES[0].code, "docs/example.md")).exists() + + +@pytest.mark.parametrize( + ("text", "protected", "expected"), + [ + ( + "- one\n continued\n\noutside\n- two\n", + [(10, 11), (7, 27), (0, 0), (23, 22)], + [(0, 27), (27, 33)], + ), + ( + "- one\n continued\n\n> outside\n- two\n", + [(19, 19), (10, 11), (7, 29)], + [(0, 19), (29, 35)], + ), + ( + "- one\n\noutside\n- two\n", + [(7, 15)], + [(0, 7), (15, 21)], + ), + ], +) +def test_site_list_boundaries_keep_overlaps_and_exact_starts(text, protected, expected): + assert chunks._site_lists(text, protected) == expected + + +def test_site_lists_do_not_revisit_every_interval_for_each_line(): + class Intervals(list): + visits = 0 + + def __iter__(self): + for interval in super().__iter__(): + self.visits += 1 + yield interval + + protected = Intervals((i * 6 + 1, i * 6 + 2) for i in range(64)) + assert chunks._site_lists("plain\n" * 64, protected) == [] + assert protected.visits <= 4 * len(protected) + + +def test_table_labels_keep_first_duplicate_offsets_and_unmatched_dialect_rows(monkeypatch): + def row(text, start=0, end=0): + return [Label(text, minimum=2, start=start, end=end)] + + github = [ + row("one", 1, 2), + row("repeat", 3, 4), + row("unmatched", 5, 6), + row("repeat", 7, 8), + row("last", 9, 10), + ] + site = [row("repeat"), row("site"), row("one"), row("repeat"), row("repeat")] + monkeypatch.setattr(chunks, "_github_rows", lambda text: github) + monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site)) + assert list(chunks.table_labels("x" * 20).values()) == [ + Label("repeat", minimum=2, start=3, end=4), + Label("site", minimum=2, end=20), + Label("one", minimum=2, start=1, end=2), + Label("repeat", minimum=2, start=7, end=8), + Label("repeat", minimum=2, end=20), + Label("unmatched", minimum=2, start=5, end=6), + Label("last", minimum=2, start=9, end=10), + ] + + +def test_table_labels_copy_cells_in_proportion_to_rows(monkeypatch): + github = [[Label(f"github {i}", start=i, end=i + 1)] for i in range(64)] + site = [[Label(f"site {i}")] for i in range(64)] + copies = 0 + + def counted_replace(*args, **kwargs): + nonlocal copies + copies += 1 + return replace(*args, **kwargs) + + monkeypatch.setattr(chunks, "_github_rows", lambda text: github) + monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site)) + monkeypatch.setattr(chunks, "replace", counted_replace) + labels = chunks.table_labels("example") + assert list(labels.values()) == [ + *[Label(f"site {i}", end=7) for i in range(64)], + *[row[0] for row in github], + ] + assert copies <= 4 * (len(github) + len(site)) From 0c30d3c8e626215cce873662489820394f74a13e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:20:36 +0000 Subject: [PATCH 2/2] English fallback checks reuse destination bodies Parse each anchored destination once per validation or repair pass. Check fragmentless destinations without reading their bodies. Signed-off-by: zlareb1 --- scripts/i18n/lifecycle.py | 17 ++++-- tests/unit/test_i18n_lifecycle.py | 88 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/scripts/i18n/lifecycle.py b/scripts/i18n/lifecycle.py index 86d47595d3..ec7325c97d 100644 --- a/scripts/i18n/lifecycle.py +++ b/scripts/i18n/lifecycle.py @@ -149,11 +149,16 @@ def english_fallback(link: str) -> tuple[str, str] | None: return (source, url.fragment) if source_path(source) else None -def english_live(source: str, anchor: str, inventory: frozenset[str]) -> bool: +def english_live( + source: str, anchor: str, inventory: frozenset[str], bodies: dict[str, str] +) -> bool: if source not in inventory or not regular(source): return False - body = "\n".join(t.english_requests((t.ROOT / source).read_text())) - return not anchor or f'' in body + if not anchor: + return True + if source not in bodies: + bodies[source] = "\n".join(t.english_requests((t.ROOT / source).read_text())) + return f'' in bodies[source] def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, str]]: @@ -162,6 +167,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s } pages.update(extra or {}) inventory = frozenset(t.sources()) + english_bodies: dict[str, str] = {} problems = [] for path, text in pages.items(): for match in t.prose_links(text[body_start(path, text) :]): @@ -169,7 +175,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s fallback = english_fallback(match[2]) if fallback: source, fragment = fallback - if not english_live(source, fragment, inventory): + if not english_live(source, fragment, inventory, english_bodies): problems.append( ( path, @@ -200,6 +206,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s def repair_english_fallbacks(codes: list[str] | None = None) -> None: """Keep deleted English body links on an owned published target that retains the anchor.""" inventory = frozenset(t.sources()) + english_bodies: dict[str, str] = {} for path in output_files(): if not regular(path): continue @@ -212,7 +219,7 @@ def repair_english_fallbacks(codes: list[str] | None = None) -> None: def fix(prose: str, path: str = path, language: str = owned[1].code) -> str: def replace(match: re.Match[str]) -> str: fallback = english_fallback(match[2]) - if not fallback or english_live(*fallback, inventory): + if not fallback or english_live(*fallback, inventory, english_bodies): return match[0] source, anchor = fallback target = t.target_path(language, source) diff --git a/tests/unit/test_i18n_lifecycle.py b/tests/unit/test_i18n_lifecycle.py index 82f0b9b2fa..aa17c8af50 100644 --- a/tests/unit/test_i18n_lifecycle.py +++ b/tests/unit/test_i18n_lifecycle.py @@ -917,3 +917,91 @@ def test_composed_lifecycle_stages_native_table_rows_without_changing_body( assert lc.candidate_problems(path, candidate) == [] lc.stage(path, candidate) assert (repo / lc.checkpoint_path(path)).read_text() == candidate + + +def test_english_fallbacks_reuse_body_and_refresh_between_checks(repo, monkeypatch): + put(repo, "docs/a.md", "# Alpha\n\n## Second\n") + publish(repo, "docs/b.md") + page = "i18n/ja/docs/b.md" + path = repo / page + url = t.REPO_URL + "/blob/main/docs/a.md" + anchors = ("alpha", "%73econd", "missing", "alpha") + path.write_text(path.read_text() + "\n".join(f"[guide]({url}#{a})" for a in anchors)) + original = t.english_requests + calls = [] + + def requests(text): + calls.append(text) + return original(text) + + monkeypatch.setattr(t, "english_requests", requests) + + def message(anchor): + return ( + page, + "docs/a.md", + f"{page} links to {url}#{anchor}, whose English destination or anchor is missing", + ) + + assert lc.link_failures() == [message("missing")] + assert calls == ["# Alpha\n\n## Second\n"] + put(repo, "docs/a.md", "# New\n\n## Second\n") + assert lc.link_failures() == [message("alpha"), message("missing"), message("alpha")] + assert calls == ["# Alpha\n\n## Second\n", "# New\n\n## Second\n"] + + +def test_fragmentless_fallback_checks_existence_without_reading_english(repo, monkeypatch): + publish(repo, "docs/a.md") + page = "i18n/ja/docs/a.md" + path = repo / page + names = ("a", "b", "link", "directory", "unlisted") + urls = [t.REPO_URL + f"/blob/main/docs/{name}.md" for name in names] + path.write_text(path.read_text() + "\n".join(f"[guide]({url})" for url in urls)) + (repo / "docs/b.md").unlink() + (repo / "docs/link.md").symlink_to("a.md") + (repo / "docs/directory.md").mkdir() + put(repo, "docs/unlisted.md", "# Alpha\n") + monkeypatch.setattr(t, "sources", lambda: tuple(f"docs/{name}.md" for name in names[:-1])) + original = pathlib.Path.read_text + + def read_text(path, *args, **kwargs): + assert path.parent != repo / "docs", "English destination body was read" + return original(path, *args, **kwargs) + + def requests(text): + pytest.fail("a fragmentless destination needs no request layout") + + monkeypatch.setattr(pathlib.Path, "read_text", read_text) + monkeypatch.setattr(t, "english_requests", requests) + assert lc.link_failures() == [ + ( + page, + f"docs/{name}.md", + f"{page} links to {url}, whose English destination or anchor is missing", + ) + for name, url in zip(names[1:], urls[1:], strict=True) + ] + + +def test_fallback_repair_reuses_body_and_keeps_other_bytes(repo, monkeypatch): + publish(repo, "docs/a.md") + publish(repo, "docs/b.md") + put(repo, "docs/a.md", "# New\n") + path = repo / "i18n/ja/docs/b.md" + url = t.REPO_URL + "/blob/main/docs/a.md" + before = path.read_text() + ( + f"\n[guide]({url}#alpha) [guide]({url}#new) [guide]({url}#alpha)\n" + "[guide](https://example.test/blob/main/docs/a.md#alpha)\n" + ) + path.write_text(before) + original = t.english_requests + calls = [] + + def requests(text): + calls.append(text) + return original(text) + + monkeypatch.setattr(t, "english_requests", requests) + lc.repair_english_fallbacks(["ja"]) + assert path.read_text() == before.replace(url + "#alpha", "a.md#alpha") + assert calls == ["# New\n"]