diff --git a/scripts/i18n/chunks.py b/scripts/i18n/chunks.py
index 7f6d139ebd..a2586e1896 100644
--- a/scripts/i18n/chunks.py
+++ b/scripts/i18n/chunks.py
@@ -3,9 +3,11 @@
from __future__ import annotations
import re
+from bisect import bisect_right
+from collections import defaultdict, deque
from dataclasses import replace
from html.parser import HTMLParser
-from itertools import pairwise
+from itertools import accumulate, pairwise
from xml.etree.ElementTree import Element
from markdown import Markdown
@@ -129,14 +131,19 @@ def _site_line(line: str) -> tuple[int, str]:
def _site_lists(text: str, protected: list[tuple[int, int]]) -> list[tuple[int, int]]:
# The site's two-space list extension has no source maps; protect its continuations
# in addition to CommonMark's marker-width-dependent ranges.
+ ordered = sorted(protected)
+ starts = [a for a, _ in ordered]
+ ends = list(accumulate((b for _, b in ordered), max))
+ exact_starts = set(starts)
spans = []
active, after_blank, offset = None, False, 0
for line in text.splitlines(keepends=True):
- inside = any(a <= offset < b for a, b in protected)
+ index = bisect_right(starts, offset)
+ inside = bool(index and offset < ends[index - 1])
depth, logical = _site_line(line)
item = None if inside else re.match(r"^( *)(?:[-+*]|\d+[.)])\s", logical)
indent = len(logical) - len(logical.lstrip(" "))
- new_block = not inside or any(a == offset for a, _ in protected)
+ new_block = not inside or offset in exact_starts
if (
active is not None
and new_block
@@ -438,17 +445,21 @@ def table_labels(text: str) -> dict[tuple[str, int, int], Label]:
"""Cell text from both existing dialects; source maps survive inline-code pipes."""
github = _github_rows(text)
site = _site_tables(text).rows
- remaining = list(github)
+ by_label: dict[tuple[Label, ...], deque[int]] = defaultdict(deque)
+ for index, row in enumerate(github):
+ by_label[tuple(replace(c, start=0, end=0) for c in row)].append(index)
+ consumed = set()
rows = []
for row in site:
- match = next((r for r in remaining if [replace(c, start=0, end=0) for c in r] == row), None)
- if match is not None:
- remaining.remove(match)
- rows.append(match)
+ matches = by_label.get(tuple(row))
+ if matches:
+ index = matches.popleft()
+ consumed.add(index)
+ rows.append(github[index])
else:
# The site parser has no source maps. These obligations remain page-wide.
rows.append([replace(c, end=len(text)) for c in row])
- rows.extend(remaining)
+ rows.extend(row for index, row in enumerate(github) if index not in consumed)
labels = {}
for index, row in enumerate(rows):
field = row[0].code[0] if row and not row[0].text and len(row[0].code) == 1 else ""
diff --git a/scripts/i18n/lifecycle.py b/scripts/i18n/lifecycle.py
index 86d47595d3..ec7325c97d 100644
--- a/scripts/i18n/lifecycle.py
+++ b/scripts/i18n/lifecycle.py
@@ -149,11 +149,16 @@ def english_fallback(link: str) -> tuple[str, str] | None:
return (source, url.fragment) if source_path(source) else None
-def english_live(source: str, anchor: str, inventory: frozenset[str]) -> bool:
+def english_live(
+ source: str, anchor: str, inventory: frozenset[str], bodies: dict[str, str]
+) -> bool:
if source not in inventory or not regular(source):
return False
- body = "\n".join(t.english_requests((t.ROOT / source).read_text()))
- return not anchor or f'' in body
+ if not anchor:
+ return True
+ if source not in bodies:
+ bodies[source] = "\n".join(t.english_requests((t.ROOT / source).read_text()))
+ return f'' in bodies[source]
def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, str]]:
@@ -162,6 +167,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s
}
pages.update(extra or {})
inventory = frozenset(t.sources())
+ english_bodies: dict[str, str] = {}
problems = []
for path, text in pages.items():
for match in t.prose_links(text[body_start(path, text) :]):
@@ -169,7 +175,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s
fallback = english_fallback(match[2])
if fallback:
source, fragment = fallback
- if not english_live(source, fragment, inventory):
+ if not english_live(source, fragment, inventory, english_bodies):
problems.append(
(
path,
@@ -200,6 +206,7 @@ def link_failures(extra: dict[str, str] | None = None) -> list[tuple[str, str, s
def repair_english_fallbacks(codes: list[str] | None = None) -> None:
"""Keep deleted English body links on an owned published target that retains the anchor."""
inventory = frozenset(t.sources())
+ english_bodies: dict[str, str] = {}
for path in output_files():
if not regular(path):
continue
@@ -212,7 +219,7 @@ def repair_english_fallbacks(codes: list[str] | None = None) -> None:
def fix(prose: str, path: str = path, language: str = owned[1].code) -> str:
def replace(match: re.Match[str]) -> str:
fallback = english_fallback(match[2])
- if not fallback or english_live(*fallback, inventory):
+ if not fallback or english_live(*fallback, inventory, english_bodies):
return match[0]
source, anchor = fallback
target = t.target_path(language, source)
diff --git a/tests/unit/test_i18n_lifecycle.py b/tests/unit/test_i18n_lifecycle.py
index 82f0b9b2fa..aa17c8af50 100644
--- a/tests/unit/test_i18n_lifecycle.py
+++ b/tests/unit/test_i18n_lifecycle.py
@@ -917,3 +917,91 @@ def test_composed_lifecycle_stages_native_table_rows_without_changing_body(
assert lc.candidate_problems(path, candidate) == []
lc.stage(path, candidate)
assert (repo / lc.checkpoint_path(path)).read_text() == candidate
+
+
+def test_english_fallbacks_reuse_body_and_refresh_between_checks(repo, monkeypatch):
+ put(repo, "docs/a.md", "# Alpha\n\n## Second\n")
+ publish(repo, "docs/b.md")
+ page = "i18n/ja/docs/b.md"
+ path = repo / page
+ url = t.REPO_URL + "/blob/main/docs/a.md"
+ anchors = ("alpha", "%73econd", "missing", "alpha")
+ path.write_text(path.read_text() + "\n".join(f"[guide]({url}#{a})" for a in anchors))
+ original = t.english_requests
+ calls = []
+
+ def requests(text):
+ calls.append(text)
+ return original(text)
+
+ monkeypatch.setattr(t, "english_requests", requests)
+
+ def message(anchor):
+ return (
+ page,
+ "docs/a.md",
+ f"{page} links to {url}#{anchor}, whose English destination or anchor is missing",
+ )
+
+ assert lc.link_failures() == [message("missing")]
+ assert calls == ["# Alpha\n\n## Second\n"]
+ put(repo, "docs/a.md", "# New\n\n## Second\n")
+ assert lc.link_failures() == [message("alpha"), message("missing"), message("alpha")]
+ assert calls == ["# Alpha\n\n## Second\n", "# New\n\n## Second\n"]
+
+
+def test_fragmentless_fallback_checks_existence_without_reading_english(repo, monkeypatch):
+ publish(repo, "docs/a.md")
+ page = "i18n/ja/docs/a.md"
+ path = repo / page
+ names = ("a", "b", "link", "directory", "unlisted")
+ urls = [t.REPO_URL + f"/blob/main/docs/{name}.md" for name in names]
+ path.write_text(path.read_text() + "\n".join(f"[guide]({url})" for url in urls))
+ (repo / "docs/b.md").unlink()
+ (repo / "docs/link.md").symlink_to("a.md")
+ (repo / "docs/directory.md").mkdir()
+ put(repo, "docs/unlisted.md", "# Alpha\n")
+ monkeypatch.setattr(t, "sources", lambda: tuple(f"docs/{name}.md" for name in names[:-1]))
+ original = pathlib.Path.read_text
+
+ def read_text(path, *args, **kwargs):
+ assert path.parent != repo / "docs", "English destination body was read"
+ return original(path, *args, **kwargs)
+
+ def requests(text):
+ pytest.fail("a fragmentless destination needs no request layout")
+
+ monkeypatch.setattr(pathlib.Path, "read_text", read_text)
+ monkeypatch.setattr(t, "english_requests", requests)
+ assert lc.link_failures() == [
+ (
+ page,
+ f"docs/{name}.md",
+ f"{page} links to {url}, whose English destination or anchor is missing",
+ )
+ for name, url in zip(names[1:], urls[1:], strict=True)
+ ]
+
+
+def test_fallback_repair_reuses_body_and_keeps_other_bytes(repo, monkeypatch):
+ publish(repo, "docs/a.md")
+ publish(repo, "docs/b.md")
+ put(repo, "docs/a.md", "# New\n")
+ path = repo / "i18n/ja/docs/b.md"
+ url = t.REPO_URL + "/blob/main/docs/a.md"
+ before = path.read_text() + (
+ f"\n[guide]({url}#alpha) [guide]({url}#new) [guide]({url}#alpha)\n"
+ "[guide](https://example.test/blob/main/docs/a.md#alpha)\n"
+ )
+ path.write_text(before)
+ original = t.english_requests
+ calls = []
+
+ def requests(text):
+ calls.append(text)
+ return original(text)
+
+ monkeypatch.setattr(t, "english_requests", requests)
+ lc.repair_english_fallbacks(["ja"])
+ assert path.read_text() == before.replace(url + "#alpha", "a.md#alpha")
+ assert calls == ["# New\n"]
diff --git a/tests/unit/test_i18n_request_bounds.py b/tests/unit/test_i18n_request_bounds.py
index b17180b3d4..8311a2f27f 100644
--- a/tests/unit/test_i18n_request_bounds.py
+++ b/tests/unit/test_i18n_request_bounds.py
@@ -1,10 +1,14 @@
"""Translation requests bound source bytes without breaking Markdown units."""
from collections import Counter
+from dataclasses import replace
+from types import SimpleNamespace
import pytest
+from scripts.i18n import chunks
from scripts.i18n import translate as t
+from scripts.i18n.context import Label
MAXIMUM = 16_384
@@ -295,3 +299,87 @@ def test_valid_table_reflow_is_written_in_each_supported_container(
).read_text()
)
assert not (tmp_path / t.target_path(t.LANGUAGES[0].code, "docs/example.md")).exists()
+
+
+@pytest.mark.parametrize(
+ ("text", "protected", "expected"),
+ [
+ (
+ "- one\n continued\n\noutside\n- two\n",
+ [(10, 11), (7, 27), (0, 0), (23, 22)],
+ [(0, 27), (27, 33)],
+ ),
+ (
+ "- one\n continued\n\n> outside\n- two\n",
+ [(19, 19), (10, 11), (7, 29)],
+ [(0, 19), (29, 35)],
+ ),
+ (
+ "- one\n\noutside\n- two\n",
+ [(7, 15)],
+ [(0, 7), (15, 21)],
+ ),
+ ],
+)
+def test_site_list_boundaries_keep_overlaps_and_exact_starts(text, protected, expected):
+ assert chunks._site_lists(text, protected) == expected
+
+
+def test_site_lists_do_not_revisit_every_interval_for_each_line():
+ class Intervals(list):
+ visits = 0
+
+ def __iter__(self):
+ for interval in super().__iter__():
+ self.visits += 1
+ yield interval
+
+ protected = Intervals((i * 6 + 1, i * 6 + 2) for i in range(64))
+ assert chunks._site_lists("plain\n" * 64, protected) == []
+ assert protected.visits <= 4 * len(protected)
+
+
+def test_table_labels_keep_first_duplicate_offsets_and_unmatched_dialect_rows(monkeypatch):
+ def row(text, start=0, end=0):
+ return [Label(text, minimum=2, start=start, end=end)]
+
+ github = [
+ row("one", 1, 2),
+ row("repeat", 3, 4),
+ row("unmatched", 5, 6),
+ row("repeat", 7, 8),
+ row("last", 9, 10),
+ ]
+ site = [row("repeat"), row("site"), row("one"), row("repeat"), row("repeat")]
+ monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
+ monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
+ assert list(chunks.table_labels("x" * 20).values()) == [
+ Label("repeat", minimum=2, start=3, end=4),
+ Label("site", minimum=2, end=20),
+ Label("one", minimum=2, start=1, end=2),
+ Label("repeat", minimum=2, start=7, end=8),
+ Label("repeat", minimum=2, end=20),
+ Label("unmatched", minimum=2, start=5, end=6),
+ Label("last", minimum=2, start=9, end=10),
+ ]
+
+
+def test_table_labels_copy_cells_in_proportion_to_rows(monkeypatch):
+ github = [[Label(f"github {i}", start=i, end=i + 1)] for i in range(64)]
+ site = [[Label(f"site {i}")] for i in range(64)]
+ copies = 0
+
+ def counted_replace(*args, **kwargs):
+ nonlocal copies
+ copies += 1
+ return replace(*args, **kwargs)
+
+ monkeypatch.setattr(chunks, "_github_rows", lambda text: github)
+ monkeypatch.setattr(chunks, "_site_tables", lambda text: SimpleNamespace(rows=site))
+ monkeypatch.setattr(chunks, "replace", counted_replace)
+ labels = chunks.table_labels("example")
+ assert list(labels.values()) == [
+ *[Label(f"site {i}", end=7) for i in range(64)],
+ *[row[0] for row in github],
+ ]
+ assert copies <= 4 * (len(github) + len(site))