From d4904c814604aa625447c5650a51323528033093 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:42:53 +0000 Subject: [PATCH] Translation checks avoid repeated row and interval scans Index normalized table rows while preserving first-match offsets and unmatched row order. Use prefix maximum ends for protected-line lookups. Signed-off-by: zlareb1 --- scripts/i18n/chunks.py | 29 ++++++--- tests/unit/test_i18n_request_bounds.py | 88 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/scripts/i18n/chunks.py b/scripts/i18n/chunks.py index 7f6d139ebd..a2586e1896 100644 --- a/scripts/i18n/chunks.py +++ b/scripts/i18n/chunks.py @@ -3,9 +3,11 @@ from __future__ import annotations import re +from bisect import bisect_right +from collections import defaultdict, deque from dataclasses import replace from html.parser import HTMLParser -from itertools import pairwise +from itertools import accumulate, pairwise from xml.etree.ElementTree import Element from markdown import Markdown @@ -129,14 +131,19 @@ def _site_line(line: str) -> tuple[int, str]: def _site_lists(text: str, protected: list[tuple[int, int]]) -> list[tuple[int, int]]: # The site's two-space list extension has no source maps; protect its continuations # in addition to CommonMark's marker-width-dependent ranges. + ordered = sorted(protected) + starts = [a for a, _ in ordered] + ends = list(accumulate((b for _, b in ordered), max)) + exact_starts = set(starts) spans = [] active, after_blank, offset = None, False, 0 for line in text.splitlines(keepends=True): - inside = any(a <= offset < b for a, b in protected) + index = bisect_right(starts, offset) + inside = bool(index and offset < ends[index - 1]) depth, logical = _site_line(line) item = None if inside else re.match(r"^( *)(?:[-+*]|\d+[.)])\s", logical) indent = len(logical) - len(logical.lstrip(" ")) - new_block = not inside or any(a == offset for a, _ in protected) + new_block = not inside or offset in exact_starts if ( active is not None and new_block @@ -438,17 +445,21 @@ def table_labels(text: str) -> dict[tuple[str, int, int], Label]: """Cell text from both existing dialects; source maps survive inline-code pipes.""" github = _github_rows(text) site = _site_tables(text).rows - remaining = list(github) + by_label: dict[tuple[Label, ...], deque[int]] = defaultdict(deque) + for index, row in enumerate(github): + by_label[tuple(replace(c, start=0, end=0) for c in row)].append(index) + consumed = set() rows = [] for row in site: - match = next((r for r in remaining if [replace(c, start=0, end=0) for c in r] == row), None) - if match is not None: - remaining.remove(match) - rows.append(match) + matches = by_label.get(tuple(row)) + if matches: + index = matches.popleft() + consumed.add(index) + rows.append(github[index]) else: # The site parser has no source maps. These obligations remain page-wide. rows.append([replace(c, end=len(text)) for c in row]) - rows.extend(remaining) + rows.extend(row for index, row in enumerate(github) if index not in consumed) labels = {} for index, row in enumerate(rows): field = row[0].code[0] if row and not row[0].text and len(row[0].code) == 1 else "" diff --git a/tests/unit/test_i18n_request_bounds.py b/tests/unit/test_i18n_request_bounds.py index b17180b3d4..8311a2f27f 100644 --- a/tests/unit/test_i18n_request_bounds.py +++ b/tests/unit/test_i18n_request_bounds.py @@ -1,10 +1,14 @@ """Translation requests bound source bytes without breaking Markdown units.""" from collections import Counter +from dataclasses import replace +from types import SimpleNamespace import pytest +from scripts.i18n import chunks from scripts.i18n import translate as t +from scripts.i18n.context import Label MAXIMUM = 16_384 @@ -295,3 +299,87 @@ def test_valid_table_reflow_is_written_in_each_supported_container( ).read_text() ) assert not (tmp_path / t.target_path(t.LANGUAGES[0].code, "docs/example.md")).exists() + + +@pytest.mark.parametrize( + ("text", "protected", "expected"), + [ + ( + "- one\n continued\n\noutside\n- two\n", + [(10, 11), (7, 27), (0, 0), (23, 22)], + [(0, 27), (27, 33)], + ), + ( + "- one\n continued\n\n> outside\n- two\n", + [(19, 19), (10, 11), (7, 29)], + [(0, 19), (29, 35)], + ), + ( + "- one\n\noutside\n- two\n", + [(7, 15)], + [(0, 7), (15, 21)], + ), + ], +) +def test_site_list_boundaries_keep_overlaps_and_exact_starts(text, protected, expected): + assert chunks._site_lists(text, protected) == expected + + +def test_site_lists_do_not_revisit_every_interval_for_each_line(): + class Intervals(list): + visits = 0 + + def __iter__(self): + for interval in super().__iter__(): + self.visits += 1 + yield interval + + protected = Intervals((i * 6 + 1, i * 6 + 2) for i in range(64)) + assert chunks._site_lists("plain\n" * 64, protected) == [] + assert protected.visits <= 4 * len(protected) + + +def test_table_labels_keep_first_duplicate_offsets_and_unmatched_dialect_rows(monkeypatch): + def row(text, start=0, end=0): + return [Label(text, minimum=2, start=start, end=end)] + + github = [ + row("one", 1, 2), + row("repeat", 3, 4), + row("unmatched", 5, 6), + row("repeat", 7, 8), + row("last", 9, 10), + ] + site = [row("repeat"), row("site"), row("one"), row("repeat"), row("repeat")] + monkeypatch.setattr(chunks, "_github_rows", lambda text: github) + monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site)) + assert list(chunks.table_labels("x" * 20).values()) == [ + Label("repeat", minimum=2, start=3, end=4), + Label("site", minimum=2, end=20), + Label("one", minimum=2, start=1, end=2), + Label("repeat", minimum=2, start=7, end=8), + Label("repeat", minimum=2, end=20), + Label("unmatched", minimum=2, start=5, end=6), + Label("last", minimum=2, start=9, end=10), + ] + + +def test_table_labels_copy_cells_in_proportion_to_rows(monkeypatch): + github = [[Label(f"github {i}", start=i, end=i + 1)] for i in range(64)] + site = [[Label(f"site {i}")] for i in range(64)] + copies = 0 + + def counted_replace(*args, **kwargs): + nonlocal copies + copies += 1 + return replace(*args, **kwargs) + + monkeypatch.setattr(chunks, "_github_rows", lambda text: github) + monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site)) + monkeypatch.setattr(chunks, "replace", counted_replace) + labels = chunks.table_labels("example") + assert list(labels.values()) == [ + *[Label(f"site {i}", end=7) for i in range(64)], + *[row[0] for row in github], + ] + assert copies <= 4 * (len(github) + len(site))