From e5c6a3fd395eb2e2790093c11bbb807c9a3c5b29 Mon Sep 17 00:00:00 2001 From: Sakina Roufid Date: Thu, 29 Oct 2026 03:50:40 +0000 Subject: [PATCH] ci: fail the docs build on schema tables that rendered wrong The field tables on the spec pages are generated from the schemas and OpenAPI documents. When the macro cannot name a type, the build still succeeds and the cell reads "any", "OneOf[]", "Array[any]" or nothing. That has reached the published site five times (#352, #465, #549, #552, #894), each found by someone reading the page, and the Location REST binding shipped without its headers table the same way (#784). Every fix added a test for the case it found; nothing checked the next one. Add scripts/check_rendered_tables.py and run it after the build in both verify jobs, next to check_links.py. Over the built HTML it asserts that every Name / Type / Requirement / Description table has a named type in each row, that every REST binding page renders its HTTP Headers table, and that every response_*_schema definition in ucp.json has a rendered section. Hand-written tables with other headers are left alone. Unit tests run from pre-commit like the other script tests. The first run against main found a live case: amenity_type.json is a $ref to reverse_domain_name.json, a string with a pattern. The macro emitted the table header and then the type's prose, which Markdown read as a one-cell row with an empty Type, on the location REST and MCP pages and the reference page. A $ref that resolves to prose is now returned without the header, so Amenity Type renders like Reverse Domain Name does. The macro tests pin it. Verified: full draft build, check_links and the new check pass; test_schema_tables 18 passed; test_check_rendered_tables 16 passed. --- .github/workflows/docs.yml | 2 + .pre-commit-config.yaml | 7 + AGENTS.md | 1 + main.py | 12 +- scripts/check_rendered_tables.py | 214 ++++++++++++++++ scripts/test_check_rendered_tables.py | 336 ++++++++++++++++++++++++++ scripts/test_schema_tables.py | 16 ++ 7 files changed, 584 insertions(+), 4 deletions(-) create mode 100755 scripts/check_rendered_tables.py create mode 100755 scripts/test_check_rendered_tables.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7f8beb0fe..468cff492 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -113,6 +113,7 @@ jobs: run: | bash scripts/build_local.sh --draft-only uv run python scripts/check_links.py local_preview + uv run python scripts/check_rendered_tables.py local_preview build_and_verify_release: needs: lint @@ -144,6 +145,7 @@ jobs: export DOCS_MODE=spec uv run mkdocs build --strict uv run python scripts/check_links.py site + uv run python scripts/check_rendered_tables.py site deploy_main: needs: build_and_verify_main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d89a35c11..219a8e11b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,13 @@ repos: pass_filenames: false files: ^(main\.py|scripts/test_schema_tables\.py|source/schemas/ucp\.json|docs/specification/reference\.md)$ stages: [pre-commit, pre-push] + - id: check-rendered-tables-tests + name: Rendered table checker unit tests + entry: uv run --frozen python scripts/test_check_rendered_tables.py + language: system + pass_filenames: false + files: ^scripts/(check_rendered_tables|test_check_rendered_tables)\.py$ + stages: [pre-commit, pre-push] - id: check-links-tests name: Link checker unit tests entry: uv run --frozen python scripts/test_check_links.py diff --git a/AGENTS.md b/AGENTS.md index 3886dacab..f9cb3e967 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,7 @@ These commands should be run from the ucp root directory. * **Build Full Multi-Version Site**: `./scripts/build_local.sh`. The resulting build will be placed in the local_preview/ directory. Add `[--draft-only]` to avoid building every version of the specification. * **Serve the site locally after build**: `python3 -m http.server 8000 -d local_preview/` * **Check Broken Links after build:** `uv run ./scripts/check_links.py local_preview/` +* **Check Rendered Schema Tables after build:** `uv run ./scripts/check_rendered_tables.py local_preview/` #### Optional: build and serve specification by version(s) diff --git a/main.py b/main.py index 33a2230ad..cf37fc5b8 100644 --- a/main.py +++ b/main.py @@ -1739,11 +1739,15 @@ def _format_schema_type( ) ) elif "$ref" in schema_data: - md.append( - _render_table_from_ref( - schema_data.get("$ref"), required_list, spec_file_name, context - ) + body = _render_table_from_ref( + schema_data.get("$ref"), required_list, spec_file_name, context ) + # A $ref to a scalar type (amenity_type -> reverse_domain_name) renders + # as prose. Under a table header, Markdown reads that prose as a row + # with an empty Type cell, so return it on its own. + if need_header and not body.lstrip().startswith("|"): + return body + md.append(body) else: for field_name, details in properties.items(): if field_name == "$ref": diff --git a/scripts/check_rendered_tables.py b/scripts/check_rendered_tables.py new file mode 100755 index 000000000..dd49207cf --- /dev/null +++ b/scripts/check_rendered_tables.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Check the built site for schema tables that rendered wrong. + +The field tables on the spec pages are generated from the JSON schemas and +OpenAPI documents by the macros in main.py. When a macro cannot name a +type, the build still succeeds: the Type cell just reads "any", "OneOf[]", +"Array[any]", or is empty. Each of those has reached the published site +before and was found by someone reading the page. + +This runs after the MkDocs build and fails on the rendered output, so a +regression is caught in the PR that introduces it: + +- Every schema field table (Name / Type / Requirement / Description) has a + non-empty Name and a Type that names something: no placeholder, no empty + composite such as "OneOf[]", no "any" at any nesting level. +- Every REST binding page renders its HTTP Headers table. +- Every `response_*_schema` definition in ucp.json has a rendered section + with the `ucp-response--schema` anchor somewhere on the site. + +Usage: python3 scripts/check_rendered_tables.py [SITE_DIR] [--schema-base DIR] +SITE_DIR defaults to local_preview; --schema-base to source/schemas. +Exit: 0 when clean, 1 when anything is reported. +""" + +from __future__ import annotations + +import argparse +from html.parser import HTMLParser +import json +from pathlib import Path +import re +import sys + +SCHEMA_TABLE_HEADER = ("Name", "Type", "Requirement", "Description") +HEADER_TABLE_HEADER = ("Header", "Required", "Description") + +# A Type cell that reads like one of these did not resolve. +_BAD_TYPE_PATTERNS = ( + re.compile(r"^any$"), + re.compile(r"\[any\]"), + re.compile(r"\[\]"), + re.compile(r"[{}]"), +) + + +class _TableParser(HTMLParser): + """Collect every table on a page as a list of rows of cell text.""" + + def __init__(self) -> None: + super().__init__() + self.title = "" + self.ids: set[str] = set() + self.tables: list[list[list[str]]] = [] + self._table: list[list[str]] | None = None + self._row: list[str] | None = None + self._cell: list[str] | None = None + self._in_title = False + + def handle_starttag(self, tag, attrs): + attrs_dict = dict(attrs) + if "id" in attrs_dict: + self.ids.add(attrs_dict["id"]) + if tag == "title": + self._in_title = True + elif tag == "table": + self._table = [] + elif tag == "tr" and self._table is not None: + self._row = [] + elif tag in ("td", "th") and self._row is not None: + self._cell = [] + + def handle_endtag(self, tag): + if tag == "title": + self._in_title = False + elif tag in ("td", "th") and self._cell is not None: + self._row.append(" ".join("".join(self._cell).split())) + self._cell = None + elif tag == "tr" and self._row is not None: + self._table.append(self._row) + self._row = None + elif tag == "table" and self._table is not None: + self.tables.append(self._table) + self._table = None + + def handle_data(self, data): + if self._in_title: + self.title += data + if self._cell is not None: + self._cell.append(data) + + +def _parse(path: Path) -> _TableParser: + parser = _TableParser() + parser.feed(path.read_text(encoding="utf-8")) + return parser + + +def _is_redirect_stub(page: _TableParser) -> bool: + return page.title.strip() == "Redirecting..." + + +def _bad_type(cell: str) -> bool: + return not cell or any(p.search(cell) for p in _BAD_TYPE_PATTERNS) + + +def check_schema_tables(path: Path, page: _TableParser) -> list[str]: + """Report rows whose Name or Type cell did not render.""" + problems = [] + for table in page.tables: + if not table or tuple(table[0]) != SCHEMA_TABLE_HEADER: + continue + for row in table[1:]: + name = row[0] if row else "" + type_cell = row[1] if len(row) > 1 else "" + if not name: + problems.append( + f"{path}: schema table row has no Name (type {type_cell!r})" + ) + elif _bad_type(type_cell): + shown = name if len(name) <= 60 else name[:57] + "..." + problems.append( + f"{path}: field {shown!r} rendered with type {type_cell!r}" + ) + return problems + + +def check_rest_headers(path: Path, page: _TableParser) -> list[str]: + """Report a REST binding page that renders no HTTP Headers table.""" + if not any( + table and tuple(table[0]) == HEADER_TABLE_HEADER for table in page.tables + ): + return [f"{path}: REST binding page has no HTTP Headers table"] + return [] + + +def response_metadata_anchors(schema_base: Path) -> dict[str, str]: + """Map each ucp.json `response_*_schema` def to its reference anchor.""" + ucp = json.loads((schema_base / "ucp.json").read_text(encoding="utf-8")) + anchors = {} + for name in ucp.get("$defs", {}): + match = re.fullmatch(r"response_(\w+)_schema", name) + if match: + anchors[name] = f"ucp-response-{match.group(1)}-schema" + return anchors + + +def _is_rest_binding(path: Path, site_dir: Path) -> bool: + parts = path.relative_to(site_dir).parts + return "specification" in parts and parts[-2:] == ("rest", "index.html") + + +def check_site(site_dir: Path, schema_base: Path) -> list[str]: + """Run every check over the built site and return the problems found.""" + problems: list[str] = [] + seen_ids: set[str] = set() + pages = 0 + + for path in sorted(site_dir.rglob("*.html")): + page = _parse(path) + if _is_redirect_stub(page): + continue + pages += 1 + seen_ids |= page.ids + problems += check_schema_tables(path, page) + if _is_rest_binding(path, site_dir): + problems += check_rest_headers(path, page) + + if pages == 0: + return [f"{site_dir}: no HTML pages found; was the site built?"] + + for def_name, anchor in sorted( + response_metadata_anchors(schema_base).items() + ): + if anchor not in seen_ids: + problems.append( + f"ucp.json $defs/{def_name} has no rendered section (#{anchor})" + ) + + return problems + + +def main(argv: list[str] | None = None) -> int: + """Parse arguments, run the checks and return the exit code.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "site_dir", + nargs="?", + default="local_preview", + help="built site directory (default: local_preview)", + ) + parser.add_argument( + "--schema-base", + default="source/schemas", + help="schema directory holding ucp.json (default: source/schemas)", + ) + args = parser.parse_args(argv) + + site_dir = Path(args.site_dir) + if not site_dir.is_dir(): + print(f"Error: {site_dir} is not a directory", file=sys.stderr) + return 1 + + problems = check_site(site_dir, Path(args.schema_base)) + for problem in problems: + print(problem) + if problems: + print(f"\n{len(problems)} rendering problem(s) found") + return 1 + print("Rendered tables OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_check_rendered_tables.py b/scripts/test_check_rendered_tables.py new file mode 100755 index 000000000..c70329277 --- /dev/null +++ b/scripts/test_check_rendered_tables.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Tests for scripts/check_rendered_tables.py. + +Each test writes a small built site into a temp directory and runs the +checks over it, so the script is exercised on the same HTML shapes +mkdocs-material emits, without a full docs build. + +Run: python3 scripts/test_check_rendered_tables.py +Exit: 0 on all pass, 1 on any failure. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_rendered_tables as crt # noqa: E402 + +_RESULTS: list[tuple[str, bool, str]] = [] + + +def _check(name: str, condition: bool, detail: str = "") -> None: + _RESULTS.append((name, condition, detail)) + + +def _report() -> int: + passed = sum(1 for _, ok, _ in _RESULTS if ok) + failed = [(n, d) for n, ok, d in _RESULTS if not ok] + for name, ok, detail in _RESULTS: + status = "PASS" if ok else "FAIL" + suffix = f" — {detail}" if detail and not ok else "" + print(f" {status} {name}{suffix}") + print(f"\n{passed} passed, {len(failed)} failed") + return 0 if not failed else 1 + + +# ----------------------------------------------------------- +# Fixtures +# ----------------------------------------------------------- + + +def _table(header: tuple[str, ...], rows: list[tuple[str, ...]]) -> str: + def tr(cells, tag): + inner = "".join( + f'<{tag} style="text-align: left;">{c}' for c in cells + ) + return f"{inner}" + + body = "".join(tr(r, "td") for r in rows) + return ( + f"{tr(header, 'th')}{body}
" + ) + + +def _page(title: str, body: str) -> str: + return ( + '' + f"{title}{body}" + ) + + +SCHEMA = crt.SCHEMA_TABLE_HEADER +HEADERS = crt.HEADER_TABLE_HEADER + +GOOD_SCHEMA_TABLE = _table( + SCHEMA, + [ + ("id", "string", "Required", "Identifier."), + ("totals", "Array[Total]", "Optional", "Totals."), + ("ucp", 'UCP Checkout Response Schema', "Required", ""), + ("extends", "OneOf[string, array]", "Optional", "Extends."), + ], +) +GOOD_HEADER_TABLE = _table( + HEADERS, [("UCP-Agent", "Yes", "Agent.")] +) +METADATA_SECTION = '

Checkout

' + + +class _Site: + """A temp site dir plus a schema dir with one ucp.json.""" + + def __init__(self, defs: list[str] = ("response_checkout_schema",)): + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + self.site = root / "site" + self.schemas = root / "schemas" + self.schemas.mkdir() + (self.schemas / "ucp.json").write_text( + json.dumps({"$defs": {name: {} for name in defs}}), encoding="utf-8" + ) + + def write(self, rel: str, html: str) -> None: + path = self.site / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(html, encoding="utf-8") + + def problems(self) -> list[str]: + return crt.check_site(self.site, self.schemas) + + def close(self) -> None: + self._tmp.cleanup() + + +def _clean_site() -> _Site: + site = _Site() + site.write( + "specification/reference/index.html", + _page("Reference", METADATA_SECTION + GOOD_SCHEMA_TABLE), + ) + site.write( + "specification/shopping/checkout/rest/index.html", + _page("Checkout REST", GOOD_HEADER_TABLE + GOOD_SCHEMA_TABLE), + ) + return site + + +# ----------------------------------------------------------- +# Tests +# ----------------------------------------------------------- + + +def test_clean_site_passes() -> None: + """A site with good tables and a metadata section reports nothing.""" + site = _clean_site() + problems = site.problems() + site.close() + _check("clean_site_passes", problems == [], f"got {problems!r}") + + +def test_bad_type_cells_are_reported() -> None: + """Each placeholder or empty Type cell is reported once, naming the page.""" + cases = { + "any": "any", + "array_any": "Array[any]", + "empty_one_of": "OneOf[]", + "empty_all_of": "AllOf[]", + "placeholder": "{type}", + "empty": "", + } + for label, cell in cases.items(): + site = _clean_site() + site.write( + "specification/shopping/cart/index.html", + _page("Cart", _table(SCHEMA, [("field", cell, "Optional", "d")])), + ) + problems = site.problems() + site.close() + _check( + f"bad_type_cell_reported_{label}", + len(problems) == 1 + and "cart/index.html" in problems[0] + and repr(cell) in problems[0], + f"got {problems!r}", + ) + + +def test_good_type_cells_pass() -> None: + """Named, nested and union types are never flagged; only placeholders.""" + cells = [ + "string", + "Array[string]", + "Map[string, Array[object]]", + "OneOf[string, null]", + "AllOf[Ap2 With Merchant Authorization, Ap2 With Checkout Mandate]", + 'Geo', + "Map[string, OneOf[Constraint Expression, object]]", + ] + site = _clean_site() + site.write( + "specification/shopping/cart/index.html", + _page( + "Cart", + _table( + SCHEMA, [(f"f{i}", c, "Optional", "d") for i, c in enumerate(cells)] + ), + ), + ) + problems = site.problems() + site.close() + _check("good_type_cells_pass", problems == [], f"got {problems!r}") + + +def test_scalar_prose_under_header_is_reported() -> None: + """The amenity_type shape: description as a one-cell row, Type empty.""" + site = _clean_site() + site.write( + "specification/common/location/index.html", + _page( + "Location", + _table(SCHEMA, [("Reverse-domain identifier used for ...", "", "", "")]), + ), + ) + problems = site.problems() + site.close() + _check( + "scalar_prose_under_header_is_reported", + len(problems) == 1 and "rendered with type ''" in problems[0], + f"got {problems!r}", + ) + + +def test_hand_written_tables_are_ignored() -> None: + """Only macro-shaped tables are checked; other Type columns are prose.""" + site = _clean_site() + site.write( + "specification/payment/template/index.html", + _page( + "Template", + _table( + ("Field", "Type", "Required", "Description"), + [("{field}", "{type}", "{Yes/No}", "{description}")], + ) + + _table(("Type", "Description"), [("info", "Informational.")]), + ), + ) + problems = site.problems() + site.close() + _check("hand_written_tables_are_ignored", problems == [], f"got {problems!r}") + + +def test_rest_page_without_headers_table_is_reported() -> None: + """A REST binding page missing its HTTP Headers table is reported.""" + site = _clean_site() + site.write( + "specification/common/location/rest/index.html", + _page("Location REST", GOOD_SCHEMA_TABLE), + ) + problems = site.problems() + site.close() + _check( + "rest_page_without_headers_table_is_reported", + len(problems) == 1 + and "location/rest/index.html" in problems[0] + and "HTTP Headers" in problems[0], + f"got {problems!r}", + ) + + +def test_redirect_stubs_are_skipped() -> None: + """Redirect stubs mike writes at old paths are not REST bindings.""" + site = _clean_site() + site.write( + "specification/catalog/rest/index.html", + _page("Redirecting...", "You're being redirected"), + ) + problems = site.problems() + site.close() + _check("redirect_stubs_are_skipped", problems == [], f"got {problems!r}") + + +def test_missing_response_metadata_section_is_reported() -> None: + """A ucp.json response metadata def with no rendered anchor is reported.""" + site = _Site(defs=["response_checkout_schema", "response_ask_schema"]) + site.write( + "specification/reference/index.html", + _page("Reference", METADATA_SECTION + GOOD_SCHEMA_TABLE), + ) + problems = site.problems() + site.close() + _check( + "missing_response_metadata_section_is_reported", + len(problems) == 1 + and "response_ask_schema" in problems[0] + and "ucp-response-ask-schema" in problems[0], + f"got {problems!r}", + ) + + +def test_metadata_anchor_anywhere_on_site_counts() -> None: + """The section may live on a binding page rather than reference.md.""" + site = _Site(defs=["response_ask_schema"]) + site.write( + "specification/common/ask/rest/index.html", + _page( + "Ask REST", + '

Ask

' + GOOD_HEADER_TABLE, + ), + ) + problems = site.problems() + site.close() + _check( + "metadata_anchor_anywhere_on_site_counts", + problems == [], + f"got {problems!r}", + ) + + +def test_empty_site_is_an_error() -> None: + """A directory with no HTML is reported rather than passing vacuously.""" + site = _Site() + site.site.mkdir() + problems = site.problems() + site.close() + _check( + "empty_site_is_an_error", + len(problems) == 1 and "no HTML pages" in problems[0], + f"got {problems!r}", + ) + + +def test_cli_exit_codes() -> None: + """The CLI exits 0 on a clean site and 1 when anything is reported.""" + site = _clean_site() + ok = crt.main([str(site.site), "--schema-base", str(site.schemas)]) + site.write( + "specification/shopping/cart/index.html", + _page("Cart", _table(SCHEMA, [("f", "any", "Optional", "d")])), + ) + bad = crt.main([str(site.site), "--schema-base", str(site.schemas)]) + site.close() + _check("cli_exit_codes", (ok, bad) == (0, 1), f"got {(ok, bad)!r}") + + +def main() -> int: + """Run every test and report.""" + test_clean_site_passes() + test_bad_type_cells_are_reported() + test_good_type_cells_pass() + test_scalar_prose_under_header_is_reported() + test_hand_written_tables_are_ignored() + test_rest_page_without_headers_table_is_reported() + test_redirect_stubs_are_skipped() + test_missing_response_metadata_section_is_reported() + test_metadata_anchor_anywhere_on_site_counts() + test_empty_site_is_an_error() + test_cli_exit_codes() + return _report() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_schema_tables.py b/scripts/test_schema_tables.py index ccf2f8d4f..57178cf39 100755 --- a/scripts/test_schema_tables.py +++ b/scripts/test_schema_tables.py @@ -1373,6 +1373,21 @@ def test_declared_header_parameter_keeps_single_row(macros: dict) -> None: # ----------------------------------------------------------- +def test_scalar_ref_schema_renders_prose_without_header(macros: dict) -> None: + """A schema that is only a $ref to a scalar type renders as prose. + + amenity_type.json is a $ref to reverse_domain_name.json, a string with a + pattern. Emitting the table header above that prose made Markdown read + the description as a row with an empty Type cell. + """ + out = macros["schema_fields"]("types/amenity_type", "common/location/rest") + _check( + "scalar_ref_schema_renders_prose_without_header", + not out.lstrip().startswith("|") and "**Pattern:**" in out, + f"got {out[:120]!r}", + ) + + def main() -> int: """Run all tests and report. Exit 0 on pass, 1 on failure.""" print("Running schema and header table tests...\n") @@ -1401,6 +1416,7 @@ def main() -> int: test_security_header_rows_follow_requirements() test_content_header_rows_follow_media_types() test_declared_header_parameter_keeps_single_row(macros) + test_scalar_ref_schema_renders_prose_without_header(macros) return _report()