From d239e2eca4886232112cdd05293a6a1018c7dcf8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 13 Oct 2026 13:26:15 +0900 Subject: [PATCH] require exact fast-mlsirm GPU JUnit evidence --- .github/workflows/fast-mlsirm-ci.yml | 51 ++++++++++++++++--- .../test_fast_mlsirm_self_hosted_workflows.py | 41 +++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/.github/workflows/fast-mlsirm-ci.yml b/.github/workflows/fast-mlsirm-ci.yml index 61c46c60f8..6a851d9833 100644 --- a/.github/workflows/fast-mlsirm-ci.yml +++ b/.github/workflows/fast-mlsirm-ci.yml @@ -173,14 +173,53 @@ jobs: from pathlib import Path from xml.etree import ElementTree + required = { + ("tests.test_marginal_parity", "test_marginal_gpu_agrees_with_cpu_loosely"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q121"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q241"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q481"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q241_wide_items_metal_workgroups"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_cpu_q121_vs_q241_agree"), + ("tests.test_bifactor_bootstrap_benchmark", "test_joint_bootstrap_cpu_vs_gpu_wall_time_q121"), + } + + def count(node, field): + value = node.attrib.get(field, "") + if not value.isascii() or not value.isdecimal(): + raise SystemExit(f"GPU evidence has invalid {field} count") + return int(value) + root = ElementTree.parse(Path("gpu-junit.xml")).getroot() - suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) - tests = sum(int(suite.attrib.get("tests", "0")) for suite in suites) - skipped = sum(int(suite.attrib.get("skipped", "0")) for suite in suites) - if tests < 1: + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites" and all(child.tag == "testsuite" for child in root): + suites = list(root) + else: + raise SystemExit("GPU evidence has unsupported JUnit layout") + if not suites: raise SystemExit("GPU evidence did not execute a test") - if skipped != 0: - raise SystemExit(f"GPU evidence contained {skipped} skipped test(s)") + identities = [] + for suite in suites: + if any(child.tag not in {"testcase", "properties", "system-out", "system-err"} for child in suite): + raise SystemExit("GPU evidence has unsupported suite children") + cases = list(suite.findall("testcase")) + if list(suite.iter("testcase")) != cases: + raise SystemExit("GPU evidence has nested testcase rows") + if count(suite, "tests") != len(cases): + raise SystemExit("GPU evidence header/testcase count mismatch") + for field, tag in (("failures", "failure"), ("errors", "error"), ("skipped", "skipped")): + actual = sum(len(list(case.iter(tag))) for case in cases) + if count(suite, field) != actual or actual != 0: + raise SystemExit(f"GPU evidence contained invalid or nonzero {field}") + identities.extend((case.get("classname"), case.get("name")) for case in cases) + if len(identities) != len(required) or set(identities) != required: + raise SystemExit("GPU evidence requires each exact selector once, without extras") + if root.tag == "testsuites": + for field in ("tests", "failures", "errors", "skipped"): + if field in root.attrib: + expected = len(identities) if field == "tests" else 0 + if count(root, field) != expected: + raise SystemExit("GPU evidence aggregate header count mismatch") PY fuzz: diff --git a/tests/test_fast_mlsirm_self_hosted_workflows.py b/tests/test_fast_mlsirm_self_hosted_workflows.py index 050b4811b3..16905cce94 100644 --- a/tests/test_fast_mlsirm_self_hosted_workflows.py +++ b/tests/test_fast_mlsirm_self_hosted_workflows.py @@ -1,4 +1,8 @@ from pathlib import Path +import textwrap +from xml.etree import ElementTree as ET + +import pytest ROOT = Path(__file__).resolve().parents[1] @@ -10,6 +14,43 @@ def workflow_text(name: str) -> str: return (WORKFLOWS / name).read_text(encoding="utf-8") +GPU_IDENTITIES = ( + ("tests.test_marginal_parity", "test_marginal_gpu_agrees_with_cpu_loosely"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q121"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q241"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q481"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_gpu_parity_q241_wide_items_metal_workgroups"), + ("tests.test_bifactor_gpu_high_q", "test_bifactor_cpu_q121_vs_q241_agree"), + ("tests.test_bifactor_bootstrap_benchmark", "test_joint_bootstrap_cpu_vs_gpu_wall_time_q121"), +) + + +def _gpu_suite(identities=GPU_IDENTITIES) -> ET.Element: + suite = ET.Element("testsuite", tests=str(len(identities)), failures="0", errors="0", skipped="0") + for classname, name in identities: + ET.SubElement(suite, "testcase", classname=classname, name=name) + return suite + + +def _run_gpu_guard(xml: ET.Element, tmp_path, monkeypatch) -> None: + step = workflow_text("fast-mlsirm-ci.yml").split(" - name: Reject skipped GPU evidence\n", 1)[1] + script = textwrap.dedent(step.split("python - <<'PY'\n", 1)[1].split(" PY\n", 1)[0]) + (tmp_path / "gpu-junit.xml").write_bytes(ET.tostring(xml)) + monkeypatch.chdir(tmp_path) + exec(compile(script, "fast-mlsirm-ci.yml::gpu-junit-guard", "exec"), {}) + + +def test_gpu_guard_accepts_only_the_exact_successful_seven(tmp_path, monkeypatch): + """Execute the producer guard against complete and false-admission evidence.""" + _run_gpu_guard(_gpu_suite(), tmp_path, monkeypatch) + partial = _gpu_suite(GPU_IDENTITIES[:1]) + skipped = _gpu_suite() + ET.SubElement(skipped[0], "skipped") + for xml in (partial, skipped): + with pytest.raises(SystemExit): + _run_gpu_guard(xml, tmp_path, monkeypatch) + + def test_fast_mlsirm_reusable_workflows_keep_isolated_runner_and_gate_contracts(): """Require every reusable fast-mlsirm job to use the isolated runner boundary.""" for name in ("ci", "codeql", "statistical-studies", "hourly-pr-governance", "pypi-gap-guard"):