Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions scripts/i18n/chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
from __future__ import annotations

import re
from bisect import bisect_right
from collections import defaultdict, deque
from dataclasses import replace
from html.parser import HTMLParser
from itertools import pairwise
from itertools import accumulate, pairwise
from xml.etree.ElementTree import Element

from markdown import Markdown
Expand Down Expand Up @@ -129,14 +131,19 @@ def _site_line(line: str) -> tuple[int, str]:
def _site_lists(text: str, protected: list[tuple[int, int]]) -> list[tuple[int, int]]:
# The site's two-space list extension has no source maps; protect its continuations
# in addition to CommonMark's marker-width-dependent ranges.
ordered = sorted(protected)
starts = [a for a, _ in ordered]
ends = list(accumulate((b for _, b in ordered), max))
exact_starts = set(starts)
spans = []
active, after_blank, offset = None, False, 0
for line in text.splitlines(keepends=True):
inside = any(a <= offset < b for a, b in protected)
index = bisect_right(starts, offset)
inside = bool(index and offset < ends[index - 1])
depth, logical = _site_line(line)
item = None if inside else re.match(r"^( *)(?:[-+*]|\d+[.)])\s", logical)
indent = len(logical) - len(logical.lstrip(" "))
new_block = not inside or any(a == offset for a, _ in protected)
new_block = not inside or offset in exact_starts
if (
active is not None
and new_block
Expand Down Expand Up @@ -438,17 +445,21 @@ def table_labels(text: str) -> dict[tuple[str, int, int], Label]:
"""Cell text from both existing dialects; source maps survive inline-code pipes."""
github = _github_rows(text)
site = _site_tables(text).rows
remaining = list(github)
by_label: dict[tuple[Label, ...], deque[int]] = defaultdict(deque)
for index, row in enumerate(github):
by_label[tuple(replace(c, start=0, end=0) for c in row)].append(index)
consumed = set()
rows = []
for row in site:
match = next((r for r in remaining if [replace(c, start=0, end=0) for c in r] == row), None)
if match is not None:
remaining.remove(match)
rows.append(match)
matches = by_label.get(tuple(row))
if matches:
index = matches.popleft()
consumed.add(index)
rows.append(github[index])
else:
# The site parser has no source maps. These obligations remain page-wide.
rows.append([replace(c, end=len(text)) for c in row])
rows.extend(remaining)
rows.extend(row for index, row in enumerate(github) if index not in consumed)
labels = {}
for index, row in enumerate(rows):
field = row[0].code[0] if row and not row[0].text and len(row[0].code) == 1 else ""
Expand Down
17 changes: 12 additions & 5 deletions scripts/i18n/lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,16 @@ def english_fallback(link: str) -> tuple[str, str] | None:
return (source, url.fragment) if source_path(source) else None


def english_live(source: str, anchor: str, inventory: frozenset[str]) -> bool:
def english_live(
source: str, anchor: str, inventory: frozenset[str], bodies: dict[str, str]
) -> bool:
if source not in inventory or not regular(source):
return False
body = "\n".join(t.english_requests((t.ROOT / source).read_text()))
return not anchor or f'<a id="{unquote(anchor)}"></a>' in body
if not anchor:
return True
if source not in bodies:
bodies[source] = "\n".join(t.english_requests((t.ROOT / source).read_text()))
return f'<a id="{unquote(anchor)}"></a>' in bodies[source]


def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, str]]:
Expand All @@ -162,14 +167,15 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s
}
pages.update(extra or {})
inventory = frozenset(t.sources())
english_bodies: dict[str, str] = {}
problems = []
for path, text in pages.items():
for match in t.prose_links(text[body_start(path, text) :]):
link, _, anchor = match[2].partition("#")
fallback = english_fallback(match[2])
if fallback:
source, fragment = fallback
if not english_live(source, fragment, inventory):
if not english_live(source, fragment, inventory, english_bodies):
problems.append(
(
path,
Expand Down Expand Up @@ -200,6 +206,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s
def repair_english_fallbacks(codes: list[str] | None = None) -> None:
"""Keep deleted English body links on an owned published target that retains the anchor."""
inventory = frozenset(t.sources())
english_bodies: dict[str, str] = {}
for path in output_files():
if not regular(path):
continue
Expand All @@ -212,7 +219,7 @@ def repair_english_fallbacks(codes: list[str] | None = None) -> None:
def fix(prose: str, path: str = path, language: str = owned[1].code) -> str:
def replace(match: re.Match[str]) -> str:
fallback = english_fallback(match[2])
if not fallback or english_live(*fallback, inventory):
if not fallback or english_live(*fallback, inventory, english_bodies):
return match[0]
source, anchor = fallback
target = t.target_path(language, source)
Expand Down
88 changes: 88 additions & 0 deletions tests/unit/test_i18n_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -917,3 +917,91 @@ def test_composed_lifecycle_stages_native_table_rows_without_changing_body(
assert lc.candidate_problems(path, candidate) == []
lc.stage(path, candidate)
assert (repo / lc.checkpoint_path(path)).read_text() == candidate


def test_english_fallbacks_reuse_body_and_refresh_between_checks(repo, monkeypatch):
put(repo, "docs/a.md", "# Alpha\n\n## Second\n")
publish(repo, "docs/b.md")
page = "i18n/ja/docs/b.md"
path = repo / page
url = t.REPO_URL + "/blob/main/docs/a.md"
anchors = ("alpha", "%73econd", "missing", "alpha")
path.write_text(path.read_text() + "\n".join(f"[guide]({url}#{a})" for a in anchors))
original = t.english_requests
calls = []

def requests(text):
calls.append(text)
return original(text)

monkeypatch.setattr(t, "english_requests", requests)

def message(anchor):
return (
page,
"docs/a.md",
f"{page} links to {url}#{anchor}, whose English destination or anchor is missing",
)

assert lc.link_failures() == [message("missing")]
assert calls == ["# Alpha\n\n## Second\n"]
put(repo, "docs/a.md", "# New\n\n## Second\n")
assert lc.link_failures() == [message("alpha"), message("missing"), message("alpha")]
assert calls == ["# Alpha\n\n## Second\n", "# New\n\n## Second\n"]


def test_fragmentless_fallback_checks_existence_without_reading_english(repo, monkeypatch):
publish(repo, "docs/a.md")
page = "i18n/ja/docs/a.md"
path = repo / page
names = ("a", "b", "link", "directory", "unlisted")
urls = [t.REPO_URL + f"/blob/main/docs/{name}.md" for name in names]
path.write_text(path.read_text() + "\n".join(f"[guide]({url})" for url in urls))
(repo / "docs/b.md").unlink()
(repo / "docs/link.md").symlink_to("a.md")
(repo / "docs/directory.md").mkdir()
put(repo, "docs/unlisted.md", "# Alpha\n")
monkeypatch.setattr(t, "sources", lambda: tuple(f"docs/{name}.md" for name in names[:-1]))
original = pathlib.Path.read_text

def read_text(path, *args, **kwargs):
assert path.parent != repo / "docs", "English destination body was read"
return original(path, *args, **kwargs)

def requests(text):
pytest.fail("a fragmentless destination needs no request layout")

monkeypatch.setattr(pathlib.Path, "read_text", read_text)
monkeypatch.setattr(t, "english_requests", requests)
assert lc.link_failures() == [
(
page,
f"docs/{name}.md",
f"{page} links to {url}, whose English destination or anchor is missing",
)
for name, url in zip(names[1:], urls[1:], strict=True)
]


def test_fallback_repair_reuses_body_and_keeps_other_bytes(repo, monkeypatch):
publish(repo, "docs/a.md")
publish(repo, "docs/b.md")
put(repo, "docs/a.md", "# New\n")
path = repo / "i18n/ja/docs/b.md"
url = t.REPO_URL + "/blob/main/docs/a.md"
before = path.read_text() + (
f"\n[guide]({url}#alpha) [guide]({url}#new) [guide]({url}#alpha)\n"
"[guide](https://example.test/blob/main/docs/a.md#alpha)\n"
)
path.write_text(before)
original = t.english_requests
calls = []

def requests(text):
calls.append(text)
return original(text)

monkeypatch.setattr(t, "english_requests", requests)
lc.repair_english_fallbacks(["ja"])
assert path.read_text() == before.replace(url + "#alpha", "a.md#alpha")
assert calls == ["# New\n"]
88 changes: 88 additions & 0 deletions tests/unit/test_i18n_request_bounds.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
"""Translation requests bound source bytes without breaking Markdown units."""

from collections import Counter
from dataclasses import replace
from types import SimpleNamespace

import pytest

from scripts.i18n import chunks
from scripts.i18n import translate as t
from scripts.i18n.context import Label

MAXIMUM = 16_384

Expand Down Expand Up @@ -295,3 +299,87 @@ def test_valid_table_reflow_is_written_in_each_supported_container(
).read_text()
)
assert not (tmp_path / t.target_path(t.LANGUAGES[0].code, "docs/example.md")).exists()


@pytest.mark.parametrize(
("text", "protected", "expected"),
[
(
"- one\n continued\n\noutside\n- two\n",
[(10, 11), (7, 27), (0, 0), (23, 22)],
[(0, 27), (27, 33)],
),
(
"- one\n continued\n\n> outside\n- two\n",
[(19, 19), (10, 11), (7, 29)],
[(0, 19), (29, 35)],
),
(
"- one\n\noutside\n- two\n",
[(7, 15)],
[(0, 7), (15, 21)],
),
],
)
def test_site_list_boundaries_keep_overlaps_and_exact_starts(text, protected, expected):
assert chunks._site_lists(text, protected) == expected


def test_site_lists_do_not_revisit_every_interval_for_each_line():
class Intervals(list):
visits = 0

def __iter__(self):
for interval in super().__iter__():
self.visits += 1
yield interval

protected = Intervals((i * 6 + 1, i * 6 + 2) for i in range(64))
assert chunks._site_lists("plain\n" * 64, protected) == []
assert protected.visits <= 4 * len(protected)


def test_table_labels_keep_first_duplicate_offsets_and_unmatched_dialect_rows(monkeypatch):
def row(text, start=0, end=0):
return [Label(text, minimum=2, start=start, end=end)]

github = [
row("one", 1, 2),
row("repeat", 3, 4),
row("unmatched", 5, 6),
row("repeat", 7, 8),
row("last", 9, 10),
]
site = [row("repeat"), row("site"), row("one"), row("repeat"), row("repeat")]
monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
assert list(chunks.table_labels("x" * 20).values()) == [
Label("repeat", minimum=2, start=3, end=4),
Label("site", minimum=2, end=20),
Label("one", minimum=2, start=1, end=2),
Label("repeat", minimum=2, start=7, end=8),
Label("repeat", minimum=2, end=20),
Label("unmatched", minimum=2, start=5, end=6),
Label("last", minimum=2, start=9, end=10),
]


def test_table_labels_copy_cells_in_proportion_to_rows(monkeypatch):
github = [[Label(f"github {i}", start=i, end=i + 1)] for i in range(64)]
site = [[Label(f"site {i}")] for i in range(64)]
copies = 0

def counted_replace(*args, **kwargs):
nonlocal copies
copies += 1
return replace(*args, **kwargs)

monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
monkeypatch.setattr(chunks, "replace", counted_replace)
labels = chunks.table_labels("example")
assert list(labels.values()) == [
*[Label(f"site {i}", end=7) for i in range(64)],
*[row[0] for row in github],
]
assert copies <= 4 * (len(github) + len(site))
Loading