Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions scripts/i18n/chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
from __future__ import annotations

import re
from bisect import bisect_right
from collections import defaultdict, deque
from dataclasses import replace
from html.parser import HTMLParser
from itertools import pairwise
from itertools import accumulate, pairwise
from xml.etree.ElementTree import Element

from markdown import Markdown
Expand Down Expand Up @@ -129,14 +131,19 @@ def _site_line(line: str) -> tuple[int, str]:
def _site_lists(text: str, protected: list[tuple[int, int]]) -> list[tuple[int, int]]:
# The site's two-space list extension has no source maps; protect its continuations
# in addition to CommonMark's marker-width-dependent ranges.
ordered = sorted(protected)
starts = [a for a, _ in ordered]
ends = list(accumulate((b for _, b in ordered), max))
exact_starts = set(starts)
spans = []
active, after_blank, offset = None, False, 0
for line in text.splitlines(keepends=True):
inside = any(a <= offset < b for a, b in protected)
index = bisect_right(starts, offset)
inside = bool(index and offset < ends[index - 1])
depth, logical = _site_line(line)
item = None if inside else re.match(r"^( *)(?:[-+*]|\d+[.)])\s", logical)
indent = len(logical) - len(logical.lstrip(" "))
new_block = not inside or any(a == offset for a, _ in protected)
new_block = not inside or offset in exact_starts
if (
active is not None
and new_block
Expand Down Expand Up @@ -438,17 +445,21 @@ def table_labels(text: str) -> dict[tuple[str, int, int], Label]:
"""Cell text from both existing dialects; source maps survive inline-code pipes."""
github = _github_rows(text)
site = _site_tables(text).rows
remaining = list(github)
by_label: dict[tuple[Label, ...], deque[int]] = defaultdict(deque)
for index, row in enumerate(github):
by_label[tuple(replace(c, start=0, end=0) for c in row)].append(index)
consumed = set()
rows = []
for row in site:
match = next((r for r in remaining if [replace(c, start=0, end=0) for c in r] == row), None)
if match is not None:
remaining.remove(match)
rows.append(match)
matches = by_label.get(tuple(row))
if matches:
index = matches.popleft()
consumed.add(index)
rows.append(github[index])
else:
# The site parser has no source maps. These obligations remain page-wide.
rows.append([replace(c, end=len(text)) for c in row])
rows.extend(remaining)
rows.extend(row for index, row in enumerate(github) if index not in consumed)
labels = {}
for index, row in enumerate(rows):
field = row[0].code[0] if row and not row[0].text and len(row[0].code) == 1 else ""
Expand Down
88 changes: 88 additions & 0 deletions tests/unit/test_i18n_request_bounds.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
"""Translation requests bound source bytes without breaking Markdown units."""

from collections import Counter
from dataclasses import replace
from types import SimpleNamespace

import pytest

from scripts.i18n import chunks
from scripts.i18n import translate as t
from scripts.i18n.context import Label

MAXIMUM = 16_384

Expand Down Expand Up @@ -295,3 +299,87 @@ def test_valid_table_reflow_is_written_in_each_supported_container(
).read_text()
)
assert not (tmp_path / t.target_path(t.LANGUAGES[0].code, "docs/example.md")).exists()


@pytest.mark.parametrize(
("text", "protected", "expected"),
[
(
"- one\n continued\n\noutside\n- two\n",
[(10, 11), (7, 27), (0, 0), (23, 22)],
[(0, 27), (27, 33)],
),
(
"- one\n continued\n\n> outside\n- two\n",
[(19, 19), (10, 11), (7, 29)],
[(0, 19), (29, 35)],
),
(
"- one\n\noutside\n- two\n",
[(7, 15)],
[(0, 7), (15, 21)],
),
],
)
def test_site_list_boundaries_keep_overlaps_and_exact_starts(text, protected, expected):
assert chunks._site_lists(text, protected) == expected


def test_site_lists_do_not_revisit_every_interval_for_each_line():
class Intervals(list):
visits = 0

def __iter__(self):
for interval in super().__iter__():
self.visits += 1
yield interval

protected = Intervals((i * 6 + 1, i * 6 + 2) for i in range(64))
assert chunks._site_lists("plain\n" * 64, protected) == []
assert protected.visits <= 4 * len(protected)


def test_table_labels_keep_first_duplicate_offsets_and_unmatched_dialect_rows(monkeypatch):
def row(text, start=0, end=0):
return [Label(text, minimum=2, start=start, end=end)]

github = [
row("one", 1, 2),
row("repeat", 3, 4),
row("unmatched", 5, 6),
row("repeat", 7, 8),
row("last", 9, 10),
]
site = [row("repeat"), row("site"), row("one"), row("repeat"), row("repeat")]
monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
assert list(chunks.table_labels("x" * 20).values()) == [
Label("repeat", minimum=2, start=3, end=4),
Label("site", minimum=2, end=20),
Label("one", minimum=2, start=1, end=2),
Label("repeat", minimum=2, start=7, end=8),
Label("repeat", minimum=2, end=20),
Label("unmatched", minimum=2, start=5, end=6),
Label("last", minimum=2, start=9, end=10),
]


def test_table_labels_copy_cells_in_proportion_to_rows(monkeypatch):
github = [[Label(f"github {i}", start=i, end=i + 1)] for i in range(64)]
site = [[Label(f"site {i}")] for i in range(64)]
copies = 0

def counted_replace(*args, **kwargs):
nonlocal copies
copies += 1
return replace(*args, **kwargs)

monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
monkeypatch.setattr(chunks, "replace", counted_replace)
labels = chunks.table_labels("example")
assert list(labels.values()) == [
*[Label(f"site {i}", end=7) for i in range(64)],
*[row[0] for row in github],
]
assert copies <= 4 * (len(github) + len(site))
Loading