From 890c304c57bcafc4dc6346f60d39533f363e4c12 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 19:09:16 +0000 Subject: [PATCH 1/2] Add reproducibility tests; clamp Wilson CI bounds; regenerate stale analysis.json Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DHVivZEPU29QtcvUurZSXX --- .github/workflows/ci.yml | 14 ++++ REPORT.md | 2 +- data/analysis.json | 40 +++++++++- scripts/analyze.py | 3 +- tests/test_reproducibility.py | 135 ++++++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 tests/test_reproducibility.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13727dc..64489a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,17 @@ jobs: python-version: "3.12" - name: Compile research scripts run: python -m compileall -q scripts + + reproducibility: + name: Results reproduce from data + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + # 0.7.0 is the released form of patches/mcpaudit-0.7.patch, which eval_v07.py evaluates. + - run: python -m pip install pytest "pyhroff-mcpaudit==0.7.0" + - run: python -m pytest -q tests diff --git a/REPORT.md b/REPORT.md index c648e36..08b7a6f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -37,7 +37,7 @@ Raw counts: mcpaudit 1,018 findings (765 high, 224 medium, 29 low) across 39 rep - Dynamic (LLM) checks were not run. ## Follow-up: mcpaudit v0.7 patch (held-out evaluation) -The failure modes above were turned into a patch (`patches/mcpaudit-0.7.patch`, applied copy in `patches/mcpaudit-0.7/`; the original `mcpaudit` folder was not modified). All 49 existing tests still pass. +The failure modes above were turned into a patch (`patches/mcpaudit-0.7.patch`, applied copy in `patches/mcpaudit-0.7/`; the original `mcpaudit` folder was not modified). The applied copy is not committed; to rerun `scripts/eval_v07.py`, `pip install pyhroff-mcpaudit==0.7.0`, which is this patch as released and reproduces `data/eval_v07.json` exactly. All 49 existing tests still pass. Method: a seeded shuffle of the 761 HIGH permission_scope findings gave a dev sample (first 40, used to find failure modes) and a held-out sample (next 40, labelled before the patch was written). Labels: `data/labels.json`; evaluation: `scripts/eval_v07.py`, `data/eval_v07.json`. diff --git a/data/analysis.json b/data/analysis.json index c3a798d..1cf9008 100644 --- a/data/analysis.json +++ b/data/analysis.json @@ -27,13 +27,49 @@ }, "strict_precision_pct": 0.0, "strict_ci95": [ - -0.0, + 0.0, 16.1 ], "lenient_precision_pct": 0.0, "lenient_ci95": [ - -0.0, + 0.0, 16.1 ] + }, + "permission_scope_high_heldout40": { + "n": 40, + "counts": { + "weak": 9, + "accurate": 15, + "false_positive": 16 + }, + "strict_precision_pct": 37.5, + "strict_ci95": [ + 24.2, + 53.0 + ], + "lenient_precision_pct": 60.0, + "lenient_ci95": [ + 44.6, + 73.7 + ] + }, + "permission_scope_high_fresh40": { + "n": 40, + "counts": { + "false_positive": 14, + "accurate": 16, + "weak": 10 + }, + "strict_precision_pct": 40.0, + "strict_ci95": [ + 26.3, + 55.4 + ], + "lenient_precision_pct": 65.0, + "lenient_ci95": [ + 49.5, + 77.9 + ] } } \ No newline at end of file diff --git a/scripts/analyze.py b/scripts/analyze.py index 6bc0467..483d184 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -5,7 +5,8 @@ def wilson(k, n, z=1.96): if n == 0: return (0, 0) p = k / n; d = 1 + z * z / n; c = p + z * z / (2 * n); h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) - return round((c - h) / d * 100, 1), round((c + h) / d * 100, 1) + # Clamp: at k=0 or k=n float error can push a bound fractionally outside [0, 100] (e.g. -0.0). + return max(0.0, round((c - h) / d * 100, 1)), min(100.0, round((c + h) / d * 100, 1)) NOISE = re.compile(r"(^|/)(tests?|__tests__|e2e|bench|benchmarks?|fixtures?|eslint[_-]?rules?|lint|scripts|examples?|mocks?|stories|plugin-community-nodes)(/|$)|\.(spec|test)\.", re.I) manifests = D("manifests.json") tools = [(r, t) for r, v in manifests.items() for t in v["tools"]] diff --git a/tests/test_reproducibility.py b/tests/test_reproducibility.py new file mode 100644 index 0000000..1a38298 --- /dev/null +++ b/tests/test_reproducibility.py @@ -0,0 +1,135 @@ +"""The study's published numbers must be reproducible from the checked-in data. + +Each analysis script is run on a temporary copy of the repository (the scripts overwrite their +output files) and its output compared to the committed result, so a stale or hand-edited data file, +or a script change that silently shifts a headline number, fails CI. +""" +import ast +import json +import math +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" + + +def _load_wilson(): + source = (ROOT / "scripts" / "analyze.py").read_text(encoding="utf-8") + fn = next(n for n in ast.parse(source).body if isinstance(n, ast.FunctionDef) and n.name == "wilson") + namespace = {"math": math} + exec(compile(ast.Module(body=[fn], type_ignores=[]), "analyze.py", "exec"), namespace) + return namespace["wilson"] + + +wilson = _load_wilson() + + +def _run_on_copy(tmp_path: Path, script: str, output: str) -> dict: + shutil.copytree(ROOT / "scripts", tmp_path / "scripts") + shutil.copytree(DATA, tmp_path / "data") + subprocess.run( + [sys.executable, str(tmp_path / "scripts" / script)], + cwd=tmp_path, + check=True, + capture_output=True, + timeout=300, + ) + return json.loads((tmp_path / "data" / output).read_text(encoding="utf-8")) + + +# --- Wilson interval --------------------------------------------------------- + + +@pytest.mark.parametrize( + ("k", "n", "expected"), + [ + (15, 40, (24.2, 53.0)), # strict precision, permission_scope sample + (27, 40, (52.0, 79.9)), # lenient precision + (0, 20, (0.0, 16.1)), # poisoning class: lower bound must clamp to 0, not -0.0 + (20, 20, (83.9, 100.0)), + ], +) +def test_wilson_interval(k, n, expected): + assert wilson(k, n) == expected + + +def test_wilson_bounds_never_leave_0_100(): + for n in range(1, 60): + for k in range(n + 1): + lo, hi = wilson(k, n) + assert 0.0 <= lo <= hi <= 100.0, (k, n) + assert math.copysign(1, lo) == 1, f"negative zero for k={k}, n={n}" + + +def test_wilson_empty_sample(): + assert wilson(0, 0) == (0, 0) + + +# --- reproducibility --------------------------------------------------------- + + +def test_analysis_json_reproduces_from_data(tmp_path): + committed = json.loads((DATA / "analysis.json").read_text(encoding="utf-8")) + assert _run_on_copy(tmp_path, "analyze.py", "analysis.json") == committed + + +def test_every_label_set_is_analysed(): + labels = json.loads((DATA / "labels.json").read_text(encoding="utf-8")) + analysis = json.loads((DATA / "analysis.json").read_text(encoding="utf-8")) + assert set(labels) <= set(analysis) + + +def test_eval_v07_reproduces_with_mcpaudit_0_7(tmp_path): + # eval_v07.py imports the patched scanner; pyhroff-mcpaudit==0.7.0 on PyPI is that patch released. + mcpaudit = pytest.importorskip("mcpaudit") + from importlib.metadata import version + + if version("pyhroff-mcpaudit") != "0.7.0": + pytest.skip("needs pyhroff-mcpaudit==0.7.0") + committed = json.loads((DATA / "eval_v07.json").read_text(encoding="utf-8")) + assert _run_on_copy(tmp_path, "eval_v07.py", "eval_v07.json") == committed + + +# --- prose matches data ------------------------------------------------------ + + +def _ci(pair) -> str: + return f"{round(pair[0])}-{round(pair[1])}%" + + +def test_report_headline_numbers_match_analysis(): + analysis = json.loads((DATA / "analysis.json").read_text(encoding="utf-8")) + report = (ROOT / "REPORT.md").read_text(encoding="utf-8") + sample = analysis["permission_scope_high_sample40"] + poisoning = analysis["poisoning_class_all20"] + + assert f"{sample['counts']['accurate']} / {sample['n']} = {sample['strict_precision_pct']}% (95% CI {_ci(sample['strict_ci95'])})" in report + lenient_k = sample["counts"]["accurate"] + sample["counts"]["weak"] + assert f"{lenient_k} / {sample['n']} = {sample['lenient_precision_pct']}% (95% CI {_ci(sample['lenient_ci95'])})" in report + assert f"0 / {poisoning['n']} (95% CI {_ci(poisoning['strict_ci95'])})" in report + assert f"({analysis['pct_nonproduction']}% of tools come from tests/fixtures/lint rules)" in report + + +def test_readme_summary_matches_analysis(): + analysis = json.loads((DATA / "analysis.json").read_text(encoding="utf-8")) + readme = (ROOT / "README.md").read_text(encoding="utf-8") + sample = analysis["permission_scope_high_sample40"] + + assert f"{analysis['tools_extracted']:,} tools" in readme + assert f"{sample['strict_precision_pct']}% strict precision (95% CI {_ci(sample['strict_ci95'])[:-1]}%)" in readme + + +def test_report_v07_numbers_match_eval(): + ev = json.loads((DATA / "eval_v07.json").read_text(encoding="utf-8"))["permission_scope_high_heldout40"] + report = (ROOT / "REPORT.md").read_text(encoding="utf-8") + ci = ev["strict_ci95_after"] + + assert f"{ev['strict_precision_after_pct']}% (95% CI {_ci(ci)}, n={ev['n_after']})" in report + assert re.search(rf"\| Findings still raised \| 40 \| {ev['n_after']} \|", report) + assert f"{ev['accurate_retained_pct']:g}% ({ev['after']['accurate']} of {ev['before']['accurate']})" in report From 1b3bddb8b3cc7a6bd292866c96429409240d15ce Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 5 Nov 2026 04:22:01 +0000 Subject: [PATCH 2/2] docs: add CI badge and a section on verifying the published numbers Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DHVivZEPU29QtcvUurZSXX --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5dcd3a8..9e5d069 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # mcp-scan-study +[![CI](https://github.com/Pyhroff/mcp-scan-study/actions/workflows/ci.yml/badge.svg)](https://github.com/Pyhroff/mcp-scan-study/actions/workflows/ci.yml) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + An empirical evaluation of two static AI-security scanners, mcpaudit and memsentry, on public MCP-server repositories, with hand-labelled precision measurements and the scanner patches that came out of them. Full write-up: **[REPORT.md](REPORT.md)**. Scan date: 2026-09-28. Single reviewer. @@ -22,7 +25,18 @@ patches/ mcpaudit-0.7.patch and memsentry-1.1.patch (the scanner changes) scripts/ collect, extract, scan, analyze, and evaluation scripts ``` -## Reproducing +## Verify the published numbers + +The headline figures can be checked without re-running the scan: the tests recompute `data/analysis.json` and `data/eval_v07.json` from the committed labels and manifests and compare the numbers quoted in this README and REPORT.md with the data. + +```bash +pip install pytest "pyhroff-mcpaudit==0.7.0" # 0.7.0 is patches/mcpaudit-0.7.patch as released +pytest -q tests +``` + +The labelled findings and corpus SHAs here also feed [aisec-suite](https://github.com/Pyhroff/aisec-suite)'s benchmark workflow, which re-derives its detector and reachability numbers from this repository every week. + +## Reproducing the scan The scripts read the scanner repositories from sibling directories and clone the corpus repos into `~/mcp-study-work`. In order: