From f66bd5ce857d197d01ad5e4cefaf82505da35b34 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 25 Oct 2026 23:06:53 +0900 Subject: [PATCH 01/11] fix(noema): separate paid auto authority from retention attestation --- .github/workflows/noema-review.yml | 2 + .../noema-paid-auto-authorization.md | 20 ++++++++++ scripts/ci/personal_review_route.py | 11 +++-- tests/test_noema_paid_auto_authorization.py | 40 +++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 docs/automation/noema-paid-auto-authorization.md create mode 100644 tests/test_noema_paid_auto_authorization.py diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index cb9d671694..7ac11815e4 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -890,6 +890,7 @@ jobs: if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' env: LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }} + NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }} PERSONAL_REVIEW_ZERO_COST_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZERO_COST_ATTESTED || 'false' }} PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED: ${{ vars.PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED || 'false' }} PERSONAL_REVIEW_ZDR_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZDR_ATTESTED || 'false' }} @@ -901,6 +902,7 @@ jobs: exit 1 fi python3 -m scripts.ci.personal_review_route \ + --paid-auto-authorized "$NOEMA_REVIEW_PAID_AUTO_AUTHORIZED" \ --zero-cost "$PERSONAL_REVIEW_ZERO_COST_ATTESTED" \ --public-auto-authorized "$PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED" \ --zero-retention "$PERSONAL_REVIEW_ZDR_ATTESTED" \ diff --git a/docs/automation/noema-paid-auto-authorization.md b/docs/automation/noema-paid-auto-authorization.md new file mode 100644 index 0000000000..086aeebc03 --- /dev/null +++ b/docs/automation/noema-paid-auto-authorization.md @@ -0,0 +1,20 @@ +# Noema paid auto cost authorization (source proposal) + +The October 25 CI request authorizes paid inference without a monetary cap on +`https://litellm.poinnetworks.net`, model `auto`. It does not require zero cost. +`NOEMA_REVIEW_PAID_AUTO_AUTHORIZED` is a separate, default-false cost authorization +for the existing Noema route. It does not attest privacy, select another key, +change endpoint/model, or relax private/internal visibility admission. Existing +OpenCode/Strix callers omit the new argument and keep their behavior. + +Private/internal requests still require the existing real retention evidence and +`PERSONAL_REVIEW_ZDR_ATTESTED`. Do not set that value merely because a key has no +expiry, unlimited budget, no configured logging integration, or successful HTTP +responses. The variable's legacy PERSONAL name does not identify the key's owner. + +Operational adoption requires normal independent source review and protected +shipment, purpose-bound existing central Secret custody, then truthful supplier +retention evidence for the gateway and every eligible downstream/fallback of its +literal auto route. No settings or Secrets are changed by this proposal. If the +supplier retains content, obtain an explicit privacy-policy decision instead of +calling it ZDR or silently falling back to another provider. diff --git a/scripts/ci/personal_review_route.py b/scripts/ci/personal_review_route.py index 01c53bc505..d48f279f34 100644 --- a/scripts/ci/personal_review_route.py +++ b/scripts/ci/personal_review_route.py @@ -10,10 +10,11 @@ def require_admission(*, zero_cost: bool, zero_retention: bool, private_target: bool, - public_auto_authorized: bool = False) -> None: - """Admit owner-authorized public routing; private supplier gates stay intact.""" + public_auto_authorized: bool = False, + paid_auto_authorized: bool = False) -> None: + """Admit explicit cost authority without relaxing private retention.""" authorized_public = private_target is False and public_auto_authorized is True - if zero_cost is not True and not authorized_public: + if zero_cost is not True and not authorized_public and paid_auto_authorized is not True: raise RuntimeError('STOP: personal auto zero-cost supplier is not attested') if private_target and zero_retention is not True: raise RuntimeError('STOP: private personal auto zero-retention supplier is not attested') @@ -38,6 +39,7 @@ def configure_opencode(path: Path) -> None: def main(argv: list[str] | None = None) -> int: """Validate trusted nonsecret admission inputs and optionally reset config.""" parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--paid-auto-authorized', choices=('true', 'false'), default='false') parser.add_argument('--zero-cost', choices=('true', 'false'), required=True) parser.add_argument('--zero-retention', choices=('true', 'false'), required=True) parser.add_argument('--private-target', choices=('true', 'false'), required=True) @@ -47,7 +49,8 @@ def main(argv: list[str] | None = None) -> int: require_admission(zero_cost=args.zero_cost == 'true', zero_retention=args.zero_retention == 'true', private_target=args.private_target == 'true', - public_auto_authorized=args.public_auto_authorized == 'true') + public_auto_authorized=args.public_auto_authorized == 'true', + paid_auto_authorized=args.paid_auto_authorized == 'true') if args.opencode_config is not None: configure_opencode(args.opencode_config) return 0 diff --git a/tests/test_noema_paid_auto_authorization.py b/tests/test_noema_paid_auto_authorization.py new file mode 100644 index 0000000000..8cb11d6eb5 --- /dev/null +++ b/tests/test_noema_paid_auto_authorization.py @@ -0,0 +1,40 @@ +"""Noema cost authorization is independent of private retention admission.""" +import pathlib +import unittest +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +class PaidAutoWorkflowTests(unittest.TestCase): + def test_noema_passes_explicit_default_closed_paid_authorization(self): + source = (ROOT / '.github/workflows/noema-review.yml').read_text() + self.assertIn("NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }}", source) + self.assertIn('--paid-auto-authorized "$NOEMA_REVIEW_PAID_AUTO_AUTHORIZED"', source) + self.assertIn('--zero-retention "$PERSONAL_REVIEW_ZDR_ATTESTED"', source) + self.assertIn('--private-target "$PRIVATE_TARGET"', source) + + def test_paid_authorized_private_route_with_retention_is_admitted(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--paid-auto-authorized', 'true', '--zero-cost', 'false', + '--zero-retention', 'true', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_paid_authority_never_substitutes_private_retention(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--paid-auto-authorized', 'true', '--zero-cost', 'false', + '--zero-retention', 'false', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('zero-retention supplier is not attested', result.stderr) + + def test_paid_authority_defaults_closed(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--zero-cost', 'false', '--zero-retention', 'true', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('zero-cost supplier is not attested', result.stderr) + +if __name__ == '__main__': + unittest.main() From b7b275179e71fcf03863595c4a03378c9a45b132 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 25 Oct 2026 23:08:15 +0900 Subject: [PATCH 02/11] test(noema): exercise actual admission shell with paid authority --- tests/test_noema_paid_auto_authorization.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_noema_paid_auto_authorization.py b/tests/test_noema_paid_auto_authorization.py index 8cb11d6eb5..97af73996b 100644 --- a/tests/test_noema_paid_auto_authorization.py +++ b/tests/test_noema_paid_auto_authorization.py @@ -3,6 +3,8 @@ import unittest import subprocess import sys +import os +import tempfile ROOT = pathlib.Path(__file__).resolve().parents[1] @@ -36,5 +38,29 @@ def test_paid_authority_defaults_closed(self): self.assertNotEqual(result.returncode, 0) self.assertIn('zero-cost supplier is not attested', result.stderr) + def test_real_workflow_shell_preserves_privacy_boundary(self): + source = (ROOT / '.github/workflows/noema-review.yml').read_text() + section = source.split(' - name: Admit existing personal LiteLLM auto route\n', 1)[1] + block = section.split(' run: |\n', 1)[1].split('\n - name:', 1)[0] + shell = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + env = dict(os.environ, LLM_GATEWAY_API_KEY='synthetic-not-a-key', + NOEMA_REVIEW_PAID_AUTO_AUTHORIZED='true', + PERSONAL_REVIEW_ZERO_COST_ATTESTED='false', + PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED='false', + PERSONAL_REVIEW_ZDR_ATTESTED='false', PRIVATE_TARGET='true', + GITHUB_OUTPUT=str(output)) + denied = subprocess.run(['bash', '-c', shell], cwd=ROOT, env=env, + capture_output=True, text=True) + self.assertNotEqual(denied.returncode, 0) + self.assertIn('zero-retention supplier is not attested', denied.stderr) + self.assertFalse(output.exists()) + env['PERSONAL_REVIEW_ZDR_ATTESTED'] = 'true' # synthetic positive only + admitted = subprocess.run(['bash', '-c', shell], cwd=ROOT, env=env, + capture_output=True, text=True) + self.assertEqual(admitted.returncode, 0, admitted.stderr) + self.assertEqual(output.read_text(), 'selected=true\n') + if __name__ == '__main__': unittest.main() From cbf46c1081a0254bb6d260099f725c15f34e4738 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 00:30:23 +0900 Subject: [PATCH 03/11] fix(ci): enroll paid Noema route regressions in runtime quality --- .../agent-review-runtime-quality-ci.yml | 4 ++++ tests/test_noema_paid_auto_authorization.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 38f617e7d6..935ca6361c 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -74,6 +74,7 @@ on: - "tests/test_contextual_orchestrator_review_launcher.py" - "tests/test_review_auto_route.py" - "tests/test_personal_review_route.py" + - "tests/test_noema_paid_auto_authorization.py" - "tests/test_tokyo_review_consumer.py" - "tests/test_review_auto_cost_admission.py" - "tests/test_review_ai_app_independence.py" @@ -304,6 +305,7 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py|\ tests/test_review_auto_route.py|\ tests/test_personal_review_route.py|\ + tests/test_noema_paid_auto_authorization.py|\ tests/test_tokyo_review_consumer.py|\ tests/test_review_auto_cost_admission.py|\ tests/test_review_ai_app_independence.py|\ @@ -504,6 +506,7 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py \ tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ + tests/test_noema_paid_auto_authorization.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ @@ -550,6 +553,7 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py \ tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ + tests/test_noema_paid_auto_authorization.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ diff --git a/tests/test_noema_paid_auto_authorization.py b/tests/test_noema_paid_auto_authorization.py index 97af73996b..684348f01a 100644 --- a/tests/test_noema_paid_auto_authorization.py +++ b/tests/test_noema_paid_auto_authorization.py @@ -9,6 +9,26 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] class PaidAutoWorkflowTests(unittest.TestCase): + def test_paid_regressions_enrolled_in_runtime_quality(self): + source = (ROOT / '.github/workflows/agent-review-runtime-quality-ci.yml').read_text() + path = 'tests/test_noema_paid_auto_authorization.py' + trigger = source.split('jobs:', 1)[0] + self.assertIn('"' + path + '"', trigger) + section = source.split(' - name: Select affected contract suites\n', 1)[1] + block = section.split(' run: |\n', 1)[1].split('\n - name:', 1)[0] + shell = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + stub = 'git() { if [ "$1" = rev-parse ]; then printf "%s\\n" "$HEAD_SHA"; else printf "%s\\n" "$CHANGED_PATH"; fi; };\n' + env = dict(os.environ, BASE_SHA='base', HEAD_SHA='head', + CHANGED_PATH=path, GITHUB_OUTPUT=str(output)) + result = subprocess.run(['bash', '-c', stub + shell], env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('review_repair=true', output.read_text()) + # Existing enrolled sibling appears in trigger, selector, pytest and compile. + self.assertEqual(source.count(path), source.count('tests/test_personal_review_route.py')) + def test_noema_passes_explicit_default_closed_paid_authorization(self): source = (ROOT / '.github/workflows/noema-review.yml').read_text() self.assertIn("NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }}", source) From 22d6dacea70709c58e57b637b210f3c18b267a6f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 00:45:33 +0900 Subject: [PATCH 04/11] feat(noema): scope non-ZDR canary payload to approved immutable YAML --- .../agent-review-runtime-quality-ci.yml | 6 ++ .github/workflows/noema-review.yml | 18 ++++ scripts/ci/noema_exact_canary.py | 87 +++++++++++++++++++ scripts/ci/noema_review_gate.py | 25 +++++- tests/test_noema_exact_canary.py | 84 ++++++++++++++++++ 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/noema_exact_canary.py create mode 100644 tests/test_noema_exact_canary.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 935ca6361c..15d93b1f99 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -75,6 +75,8 @@ on: - "tests/test_review_auto_route.py" - "tests/test_personal_review_route.py" - "tests/test_noema_paid_auto_authorization.py" + - "tests/test_noema_exact_canary.py" + - "scripts/ci/noema_exact_canary.py" - "tests/test_tokyo_review_consumer.py" - "tests/test_review_auto_cost_admission.py" - "tests/test_review_ai_app_independence.py" @@ -306,6 +308,8 @@ jobs: tests/test_review_auto_route.py|\ tests/test_personal_review_route.py|\ tests/test_noema_paid_auto_authorization.py|\ + tests/test_noema_exact_canary.py|\ + scripts/ci/noema_exact_canary.py|\ tests/test_tokyo_review_consumer.py|\ tests/test_review_auto_cost_admission.py|\ tests/test_review_ai_app_independence.py|\ @@ -507,6 +511,7 @@ jobs: tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ + tests/test_noema_exact_canary.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ @@ -554,6 +559,7 @@ jobs: tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ + tests/test_noema_exact_canary.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 7ac11815e4..7305bcd5a0 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -891,12 +891,24 @@ jobs: env: LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }} NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }} + NOEMA_EXACT_CANARY_API_KEY: ${{ secrets.NOEMA_EXACT_CANARY_API_KEY }} PERSONAL_REVIEW_ZERO_COST_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZERO_COST_ATTESTED || 'false' }} PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED: ${{ vars.PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED || 'false' }} PERSONAL_REVIEW_ZDR_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZDR_ATTESTED || 'false' }} PRIVATE_TARGET: ${{ steps.target_visibility.outputs.require_zdr }} run: | set -euo pipefail + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'true' ]; then + test -n "${NOEMA_EXACT_CANARY_API_KEY:-}" + python3 -m scripts.ci.noema_exact_canary \ + --repo "$TARGET_REPOSITORY" --pr-number "$PR_NUMBER" \ + --expected-head "$EXPECTED_HEAD_SHA" + echo 'selected=true' >>"$GITHUB_OUTPUT" + echo 'exact_canary=true' >>"$GITHUB_OUTPUT" + exit 0 + fi + test "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'false' if [ -z "${LLM_GATEWAY_API_KEY:-}" ]; then echo '::error::STOP: existing personal review Secret is unavailable.' exit 1 @@ -1018,6 +1030,8 @@ jobs: NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }} PERSONAL_ROUTE_SELECTED: ${{ steps.personal_route.outputs.selected }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ steps.personal_route.outputs.exact_canary || 'false' }} + NOEMA_EXACT_CANARY_API_KEY: ${{ secrets.NOEMA_EXACT_CANARY_API_KEY }} NOEMA_GITHUB_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} run: | @@ -1031,6 +1045,10 @@ jobs: echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot prepare a verdict." exit 1 fi + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'true' ]; then + test -n "${NOEMA_EXACT_CANARY_API_KEY:-}" + LLM_GATEWAY_API_KEY="$NOEMA_EXACT_CANARY_API_KEY" + fi if [ "${PERSONAL_ROUTE_SELECTED:-}" = 'true' ]; then test -n "${LLM_GATEWAY_API_KEY:-}" export NOEMA_LLM_API_URL="https://litellm.poinnetworks.net/v1/chat/completions" diff --git a/scripts/ci/noema_exact_canary.py b/scripts/ci/noema_exact_canary.py new file mode 100644 index 0000000000..64e7ca453e --- /dev/null +++ b/scripts/ci/noema_exact_canary.py @@ -0,0 +1,87 @@ +"""Explicit non-ZDR consent for one immutable YAML, never ZDR attestation. + +Consent includes disclosed gateway storage (no established finite deletion period) +and external classifier/provider/fallback recipients. No other private input is +covered. This module is inert unless the trusted Noema caller selects it. +""" +import hashlib +import json + +REPOSITORY = 'ContextualWisdomLab/traveling-agent' +NUMBER = 4 +HEAD = 'bf85ddde01f871f0e713db54aae5a16883777d9f' +PATH = '.github/workflows/docker-ephemeral-canary.yml' +DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d' +SYSTEM = 'Return strict JSON only. Do not include markdown.' +INSTRUCTION = ('Independently review this exact YAML for correctness and security. ' + 'Return the declared response schema. Cite changed RIGHT-side lines. ' + 'For approval falsify at least one concrete regression hypothesis; ' + 'for request_changes confirm a concrete finding. This is a complete ' + 'new file; no other repository context is authorized.\n' + f'Repository: {REPOSITORY}\nPR: {NUMBER}\nHead: {HEAD}\nPath: {PATH}\n') + + +def require_target(repo, number, head): + if repo != REPOSITORY or type(number) is not int or number != NUMBER or head != HEAD: + raise RuntimeError('Non-ZDR canary scope mismatch') + + +def require_content(content): + if type(content) is not str or hashlib.sha256(content.encode('utf-8')).hexdigest() != DIGEST: + raise RuntimeError('Non-ZDR canary content scope mismatch') + + +def messages(content): + require_content(content) + return [{'role': 'system', 'content': SYSTEM}, + {'role': 'user', 'content': INSTRUCTION + content}] + + +def validate_payload(payload): + """Verify complete actual JSON payload; no title, threads or extra context.""" + if type(payload) is not dict or set(payload) != {'model', 'response_format', 'messages'}: + raise RuntimeError('Non-ZDR canary payload scope mismatch') + from scripts.ci.noema_review_gate import _noema_verdict_response_format + if payload['response_format'] != _noema_verdict_response_format(1): + raise RuntimeError('Non-ZDR canary schema scope mismatch') + if payload['model'] != 'auto': + raise RuntimeError('Non-ZDR canary model scope mismatch') + rows = payload['messages'] + if type(rows) is not list or len(rows) != 2 or rows[0] != {'role': 'system', 'content': SYSTEM}: + raise RuntimeError('Non-ZDR canary messages scope mismatch') + row = rows[1] + if type(row) is not dict or set(row) != {'role', 'content'} or row['role'] != 'user': + raise RuntimeError('Non-ZDR canary message scope mismatch') + text = row['content'] + if type(text) is not str or not text.startswith(INSTRUCTION): + raise RuntimeError('Non-ZDR canary instruction scope mismatch') + require_content(text[len(INSTRUCTION):]) + + +def serialize_payload(content, response_format): + """Serialize only approved input and fixed caller-owned schema.""" + payload = {'model': 'auto', 'response_format': response_format, 'messages': messages(content)} + raw = json.dumps(payload).encode('utf-8') + validate_payload(json.loads(raw)) + return raw + + +def main(): + """Authenticate exact target content through the existing GitHub reader.""" + import argparse + from scripts.ci import noema_review_gate as gate + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo', required=True) + parser.add_argument('--pr-number', required=True, type=int) + parser.add_argument('--expected-head', required=True) + args = parser.parse_args() + require_target(args.repo, args.pr_number, args.expected_head) + pr = gate.fetch_pr(args.repo, args.pr_number) + gate.require_expected_head(pr, HEAD) + if gate.fetch_changed_files(args.repo, args.pr_number) != [(PATH, 'added')]: + raise RuntimeError('Non-ZDR canary changed-file scope mismatch') + require_content(gate.fetch_file_content_at_ref(args.repo, PATH, HEAD)) + + +if __name__ == '__main__': + main() diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 156f5aafa3..839900e30d 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1668,9 +1668,32 @@ def call_llm( prompt, ], } + exact_canary = os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") + if exact_canary not in {"true", "false"}: + raise RuntimeError("Non-ZDR canary mode is invalid") + if exact_canary == "true": + from scripts.ci import noema_exact_canary as canary + canary.require_target(repo, number, expected_head) + if pr.get("headRefOid") != expected_head or truncated: + raise RuntimeError("Non-ZDR canary head scope mismatch") + if api_url != 'https://litellm.poinnetworks.net/v1/chat/completions' or model != 'auto': + raise RuntimeError("Non-ZDR canary route scope mismatch") + if tuple(changed_paths) != (canary.PATH,) or fetch_changed_files(repo, number) != [(canary.PATH, 'added')]: + raise RuntimeError("Non-ZDR canary changed-file scope mismatch") + content = fetch_file_content_at_ref(repo, canary.PATH, expected_head) + canary.require_content(content) + # Reconstruct the complete new-file diff from admitted bytes. Never send + # caller diff, title, threads, base source or other repository context. + diff = (f'diff --git a/{canary.PATH} b/{canary.PATH}\n' + f'new file mode 100644\n--- /dev/null\n+++ b/{canary.PATH}\n' + f'@@ -0,0 +1,{len(content.splitlines())} @@\n' + + ''.join('+' + line for line in content.splitlines(keepends=True))) + payload_bytes = canary.serialize_payload(content, _noema_verdict_response_format(1)) + else: + payload_bytes = json.dumps(payload).encode("utf-8") request = urllib.request.Request( api_url, - data=json.dumps(payload).encode("utf-8"), + data=payload_bytes, headers={ "authorization": f"Bearer {api_key}", "content-type": "application/json", diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py new file mode 100644 index 0000000000..efc65fb738 --- /dev/null +++ b/tests/test_noema_exact_canary.py @@ -0,0 +1,84 @@ +"""Exact content exception must not disclose PR prose or sibling source.""" +import unittest +from scripts.ci import noema_exact_canary as policy + +class ExactCanaryTests(unittest.TestCase): + def test_actual_serialization_contains_only_approved_yaml(self): + import json + from unittest.mock import patch + # Synthetic approved-content fixture; the real digest remains immutable. + import hashlib + content = 'name: synthetic\n' + with patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()): + from scripts.ci.noema_review_gate import _noema_verdict_response_format + raw = policy.serialize_payload(content, _noema_verdict_response_format(1)) + decoded = json.loads(raw) + self.assertEqual(decoded['messages'], policy.messages(content)) + self.assertNotIn('title', decoded) + for extra in ('\nforeign source', '\nreview thread'): + decoded['messages'][1]['content'] = policy.INSTRUCTION + content + extra + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.validate_payload(decoded) + + def test_foreign_schema_is_not_an_input_channel(self): + import hashlib + from unittest.mock import patch + content = 'name: synthetic\n' + with patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()): + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.serialize_payload(content, {'description': 'foreign private source'}) + + def test_call_llm_uses_only_exact_serialized_input(self): + import os + import json + import hashlib + from unittest.mock import patch + from scripts.ci import noema_review_gate as gate + content = 'name: synthetic\n' + requests = [] + class StopProbe(BaseException): + pass + class Opener: + def open(self, request): + requests.append(json.loads(request.data)) + raise StopProbe() + env = {'NOEMA_EXACT_CANARY_NON_ZDR': 'true', + 'NOEMA_LLM_API_URL': 'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_API_KEY': 'synthetic', 'NOEMA_LLM_MODEL': 'auto'} + with patch.dict(os.environ, env, clear=True), \ + patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()), \ + patch.object(gate, 'reject_private_llm_url'), \ + patch.object(gate, 'fetch_changed_files', return_value=[(policy.PATH, 'added')]), \ + patch.object(gate, 'fetch_file_content_at_ref', return_value=content), \ + patch.object(gate.urllib.request, 'build_opener', return_value=Opener()): + with self.assertRaises(StopProbe): + gate.call_llm(policy.REPOSITORY, 4, + {'headRefOid': policy.HEAD, 'title': 'PRIVATE TITLE'}, + 'PRIVATE DIFF', False, policy.HEAD, 'PRIVATE THREAD', [policy.PATH]) + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0]['messages'], [ + {'role': 'system', 'content': policy.SYSTEM}, + {'role': 'user', 'content': policy.INSTRUCTION + content}]) + + def test_workflow_uses_dedicated_default_closed_exception(self): + from pathlib import Path + source = (Path(__file__).resolve().parents[1] / '.github/workflows/noema-review.yml').read_text() + self.assertIn("NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }}", source) + self.assertIn('secrets.NOEMA_EXACT_CANARY_API_KEY', source) + self.assertIn('python3 -m scripts.ci.noema_exact_canary', source) + + def test_scope_is_exact(self): + policy.require_target(policy.REPOSITORY, policy.NUMBER, policy.HEAD) + for args in [('foreign/repo', 4, policy.HEAD), + (policy.REPOSITORY, True, policy.HEAD), + (policy.REPOSITORY, 5, policy.HEAD), + (policy.REPOSITORY, 4, 'a' * 40)]: + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.require_target(*args) + + def test_payload_refuses_unapproved_context(self): + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.validate_payload({'messages': [{'role': 'user', 'content': 'foreign private source'}]}) + +if __name__ == '__main__': + unittest.main() From 1aaeae245af2c6e15f72103a6cc40312d2e793e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 00:47:45 +0900 Subject: [PATCH 05/11] docs(noema): bind canary consent and App reader requirements --- .github/workflows/noema-review.yml | 1 + docs/automation/noema-exact-canary-non-zdr.md | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docs/automation/noema-exact-canary-non-zdr.md diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 7305bcd5a0..ba23a85ff2 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -893,6 +893,7 @@ jobs: NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }} NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }} NOEMA_EXACT_CANARY_API_KEY: ${{ secrets.NOEMA_EXACT_CANARY_API_KEY }} + GH_TOKEN: ${{ steps.noema_github_app_token.outputs.token }} PERSONAL_REVIEW_ZERO_COST_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZERO_COST_ATTESTED || 'false' }} PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED: ${{ vars.PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED || 'false' }} PERSONAL_REVIEW_ZDR_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZDR_ATTESTED || 'false' }} diff --git a/docs/automation/noema-exact-canary-non-zdr.md b/docs/automation/noema-exact-canary-non-zdr.md new file mode 100644 index 0000000000..a8acd4085e --- /dev/null +++ b/docs/automation/noema-exact-canary-non-zdr.md @@ -0,0 +1,25 @@ +# Exact non-ZDR canary (default disabled) + +Direct user consent covers ONLY ContextualWisdomLab/traveling-agent PR4, +head bf85ddde01f871f0e713db54aae5a16883777d9f, added file +`.github/workflows/docker-ephemeral-canary.yml`, SHA256 +5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d. + +Consent includes disclosed gateway prompt/response storage without an established +finite deletion period and the external classifier/provider/fallback chain. +It is NOT ZDR, an upstream no-training claim, or organization-wide consent. +Never enable PERSONAL_REVIEW_ZDR_ATTESTED on this basis. + +Trusted variable NOEMA_EXACT_CANARY_NON_ZDR defaults false. Its true mode requires +a separate NOEMA_EXACT_CANARY_API_KEY Secret: no fallback to or overwrite of the +unresolved central Secret. Dedicated issuance/install must have authorized issuer +custody receipts before activation. No dedicated Secret has been installed by +this patch. The existing App publisher and exact-head verification are unchanged. + +Admission authenticates current head, one added path and exact content digest. +The HTTP producer repeats these checks and serializes only the approved YAML, +fixed instructions/identities and fixed verdict schema; PR title, review threads, +base source and caller diff do not enter the outgoing canary payload. The local +verdict validator receives a reconstructed new-file diff for line validation. +All other targets in this mode fail closed. Normal mode keeps existing private +retention admission unchanged. This patch does not authorize deployment/merge. From 27c319c9ab95bf1ac083e235938d55ced152135e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 01:42:45 +0900 Subject: [PATCH 06/11] fix(noema): align exact canary schema and regression enrollment --- .../agent-review-runtime-quality-ci.yml | 3 ++ .github/workflows/noema-review.yml | 10 ++++++ docs/automation/noema-exact-canary-non-zdr.md | 9 +++++ scripts/ci/noema_exact_canary.py | 6 ++-- scripts/ci/noema_review_gate.py | 3 +- tests/test_noema_exact_canary.py | 33 ++++++++++++++++++- tests/test_noema_paid_auto_authorization.py | 33 +++++++++++++++++++ 7 files changed, 92 insertions(+), 5 deletions(-) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 15d93b1f99..cf949ebee4 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -77,6 +77,7 @@ on: - "tests/test_noema_paid_auto_authorization.py" - "tests/test_noema_exact_canary.py" - "scripts/ci/noema_exact_canary.py" + - "scripts/ci/noema_review_gate.py" - "tests/test_tokyo_review_consumer.py" - "tests/test_review_auto_cost_admission.py" - "tests/test_review_ai_app_independence.py" @@ -225,6 +226,7 @@ jobs: review_repair_suite=true ;; .github/workflows/noema-review.yml|\ + scripts/ci/noema_review_gate.py|\ .github/actions/noema-review/two_phase.py|\ tests/test_noema_reviewer_token_lifetime.py|\ tests/test_noema_two_phase_handoff.py|\ @@ -237,6 +239,7 @@ jobs: tests/test_noema_document_review_context.py|\ docs/doctoring/noema-review-token-lifetime.md) noema_suite=true + review_repair_suite=true ;; .github/workflows/opencode-review-dispatch.yml|\ scripts/ci/ensure_rust_llvm19.sh|\ diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index ba23a85ff2..dea5d39945 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -663,6 +663,7 @@ jobs: NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} TOKEN_EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} NOEMA_REVIEW_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || '' }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }} run: | set -euo pipefail @@ -673,6 +674,15 @@ jobs: repository_name="${TARGET_REPOSITORY#*/}" echo "repository=$repository_name" >>"$GITHUB_OUTPUT" + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = "true" ]; then + if [ -n "${NOEMA_REVIEW_TOKEN:-}" ] || [ -z "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] || [ -z "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then + echo "::error::Non-ZDR canary requires native GitHub App selection; PAT precedence and OIDC fallback are unsupported. No model call is permitted." + exit 1 + fi + echo "source=github-app" >>"$GITHUB_OUTPUT" + exit 0 + fi + if [ "$TARGET_REPOSITORY" = "ContextualWisdomLab/k-csap-skills" ]; then if [ -z "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] || [ -z "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then echo "::error::k-csap-skills Noema reviewer requires the native GitHub App client ID and private key; PAT/OIDC fallback is forbidden." diff --git a/docs/automation/noema-exact-canary-non-zdr.md b/docs/automation/noema-exact-canary-non-zdr.md index a8acd4085e..7b2cc9e647 100644 --- a/docs/automation/noema-exact-canary-non-zdr.md +++ b/docs/automation/noema-exact-canary-non-zdr.md @@ -15,6 +15,15 @@ a separate NOEMA_EXACT_CANARY_API_KEY Secret: no fallback to or overwrite of the unresolved central Secret. Dedicated issuance/install must have authorized issuer custody receipts before activation. No dedicated Secret has been installed by this patch. The existing App publisher and exact-head verification are unchanged. +The canary requires native GitHub App selection. If NOEMA_REVIEW_TOKEN would +preempt the App, or only OIDC is available, credential selection explicitly rejects +before model work; ambient GH_TOKEN is never borrowed. Normal mode retains its +existing credential precedence. Activation must resolve this prerequisite without +silently replacing any existing Secret. + +Workflow changes require two distinct probes under the existing substantive +validator. The emitted schema and fixed approval instruction require two as well; +one-probe responses remain rejected rather than weakening that validator. Admission authenticates current head, one added path and exact content digest. The HTTP producer repeats these checks and serializes only the approved YAML, diff --git a/scripts/ci/noema_exact_canary.py b/scripts/ci/noema_exact_canary.py index 64e7ca453e..467c39ce5b 100644 --- a/scripts/ci/noema_exact_canary.py +++ b/scripts/ci/noema_exact_canary.py @@ -15,7 +15,7 @@ SYSTEM = 'Return strict JSON only. Do not include markdown.' INSTRUCTION = ('Independently review this exact YAML for correctness and security. ' 'Return the declared response schema. Cite changed RIGHT-side lines. ' - 'For approval falsify at least one concrete regression hypothesis; ' + 'For approval falsify at least two distinct concrete regression hypotheses; ' 'for request_changes confirm a concrete finding. This is a complete ' 'new file; no other repository context is authorized.\n' f'Repository: {REPOSITORY}\nPR: {NUMBER}\nHead: {HEAD}\nPath: {PATH}\n') @@ -41,8 +41,8 @@ def validate_payload(payload): """Verify complete actual JSON payload; no title, threads or extra context.""" if type(payload) is not dict or set(payload) != {'model', 'response_format', 'messages'}: raise RuntimeError('Non-ZDR canary payload scope mismatch') - from scripts.ci.noema_review_gate import _noema_verdict_response_format - if payload['response_format'] != _noema_verdict_response_format(1): + from scripts.ci.noema_review_gate import _noema_verdict_response_format, _required_probe_count + if payload['response_format'] != _noema_verdict_response_format(_required_probe_count('', (PATH,))): raise RuntimeError('Non-ZDR canary schema scope mismatch') if payload['model'] != 'auto': raise RuntimeError('Non-ZDR canary model scope mismatch') diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 839900e30d..5bf0e8bc12 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1688,7 +1688,8 @@ def call_llm( f'new file mode 100644\n--- /dev/null\n+++ b/{canary.PATH}\n' f'@@ -0,0 +1,{len(content.splitlines())} @@\n' + ''.join('+' + line for line in content.splitlines(keepends=True))) - payload_bytes = canary.serialize_payload(content, _noema_verdict_response_format(1)) + payload_bytes = canary.serialize_payload(content, _noema_verdict_response_format( + _required_probe_count(diff, (canary.PATH,)))) else: payload_bytes = json.dumps(payload).encode("utf-8") request = urllib.request.Request( diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py index efc65fb738..842adcfb63 100644 --- a/tests/test_noema_exact_canary.py +++ b/tests/test_noema_exact_canary.py @@ -11,7 +11,7 @@ def test_actual_serialization_contains_only_approved_yaml(self): content = 'name: synthetic\n' with patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()): from scripts.ci.noema_review_gate import _noema_verdict_response_format - raw = policy.serialize_payload(content, _noema_verdict_response_format(1)) + raw = policy.serialize_payload(content, _noema_verdict_response_format(2)) decoded = json.loads(raw) self.assertEqual(decoded['messages'], policy.messages(content)) self.assertNotIn('title', decoded) @@ -56,6 +56,9 @@ def open(self, request): {'headRefOid': policy.HEAD, 'title': 'PRIVATE TITLE'}, 'PRIVATE DIFF', False, policy.HEAD, 'PRIVATE THREAD', [policy.PATH]) self.assertEqual(len(requests), 1) + schema = requests[0]['response_format']['json_schema']['schema'] + self.assertEqual(schema['properties']['adversarial_validation']['properties']['probes']['minItems'], + gate._required_probe_count('', [policy.PATH])) self.assertEqual(requests[0]['messages'], [ {'role': 'system', 'content': policy.SYSTEM}, {'role': 'user', 'content': policy.INSTRUCTION + content}]) @@ -67,6 +70,34 @@ def test_workflow_uses_dedicated_default_closed_exception(self): self.assertIn('secrets.NOEMA_EXACT_CANARY_API_KEY', source) self.assertIn('python3 -m scripts.ci.noema_exact_canary', source) + def test_canary_credential_selection_rejects_pat_oidc_before_model(self): + import os + import subprocess + import tempfile + from pathlib import Path + import yaml + root = Path(__file__).resolve().parents[1] + workflow = yaml.load((root / '.github/workflows/noema-review.yml').read_text(), Loader=yaml.BaseLoader) + step = next(s for s in workflow['jobs']['noema-review']['steps'] if s['name'] == 'Select fail-closed Noema reviewer credential') + for mode, pat, app, expected in [('true', True, True, None), ('true', False, False, None), + ('true', False, True, 'github-app'), ('false', True, True, 'pat')]: + with self.subTest(mode=mode, pat=pat, app=app), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'out' + env = dict(PATH=os.environ['PATH'], TARGET_REPOSITORY=policy.REPOSITORY, + NOEMA_EXACT_CANARY_NON_ZDR=mode, GITHUB_OUTPUT=str(output), + NOEMA_REVIEW_TOKEN='synthetic-pat' if pat else '', + NOEMA_GITHUB_APP_CLIENT_ID='synthetic-id' if app else '', + NOEMA_GITHUB_APP_PRIVATE_KEY='synthetic-key' if app else '', + TOKEN_EXCHANGE_URL='https://inert.invalid', GH_TOKEN='ambient-must-not-borrow') + result = subprocess.run(['bash', '-c', step['run']], env=env, capture_output=True, text=True) + if expected is None: + self.assertNotEqual(result.returncode, 0) + self.assertIn('Non-ZDR canary requires native GitHub App selection', result.stdout) + self.assertNotIn('source=', output.read_text()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('source=' + expected, output.read_text()) + def test_scope_is_exact(self): policy.require_target(policy.REPOSITORY, policy.NUMBER, policy.HEAD) for args in [('foreign/repo', 4, policy.HEAD), diff --git a/tests/test_noema_paid_auto_authorization.py b/tests/test_noema_paid_auto_authorization.py index 684348f01a..6a103a91ad 100644 --- a/tests/test_noema_paid_auto_authorization.py +++ b/tests/test_noema_paid_auto_authorization.py @@ -29,6 +29,39 @@ def test_paid_regressions_enrolled_in_runtime_quality(self): # Existing enrolled sibling appears in trigger, selector, pytest and compile. self.assertEqual(source.count(path), source.count('tests/test_personal_review_route.py')) + def test_producer_paths_select_and_execute_regressions(self): + import yaml + workflow = yaml.load((ROOT / '.github/workflows/agent-review-runtime-quality-ci.yml').read_text(), Loader=yaml.BaseLoader) + steps = {s['name']: s for s in workflow['jobs']['agent_review_runtime_quality']['steps']} + shell = steps['Select affected contract suites']['run'] + paths = ['.github/workflows/noema-review.yml', 'scripts/ci/noema_review_gate.py', + 'scripts/ci/noema_exact_canary.py', 'scripts/ci/personal_review_route.py', + 'tests/test_noema_exact_canary.py', 'tests/test_noema_paid_auto_authorization.py', + '.github/workflows/agent-review-runtime-quality-ci.yml'] + for path in paths + ['docs/unrelated.md']: + with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + stub = 'git() { if [ "$1" = rev-parse ]; then printf "%s\\n" "$HEAD_SHA"; else printf "%s\\n" "$CHANGED_PATH"; fi; };\n' + env = dict(PATH=os.environ['PATH'], BASE_SHA='base', HEAD_SHA='head', + CHANGED_PATH=path, GITHUB_OUTPUT=str(output)) + result = subprocess.run(['bash', '-c', stub + shell], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + selected = dict(line.split('=', 1) for line in output.read_text().splitlines()) + if path == 'docs/unrelated.md': + self.assertEqual(selected['review_repair'], 'false') + continue + self.assertIn(path, workflow['on']['pull_request']['paths']) + self.assertEqual(selected['review_repair'], 'true') + command = steps['Verify scheduler and contextual-orchestrator review-repair contracts']['run'] + # Execute actual command bodies, recording only the Python boundary. + recorder = 'python() { printf "%s\\n" "$*"; };\n' + run = subprocess.run(['bash', '-c', recorder + command], env=env, cwd=ROOT, capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stderr) + pytest_calls = [line for line in run.stdout.splitlines() if line.startswith('-m pytest ')] + self.assertTrue(pytest_calls) + for test in ('tests/test_noema_exact_canary.py', 'tests/test_noema_paid_auto_authorization.py'): + self.assertTrue(any(test in line.split() for line in pytest_calls), test) + def test_noema_passes_explicit_default_closed_paid_authorization(self): source = (ROOT / '.github/workflows/noema-review.yml').read_text() self.assertIn("NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }}", source) From ca8ce6284bfd2ced42ce7fbd941ce3c860817e3a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 01:45:55 +0900 Subject: [PATCH 07/11] test(noema): connect advertised minimum to actual verdict consumer --- tests/test_noema_exact_canary.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py index 842adcfb63..4445202047 100644 --- a/tests/test_noema_exact_canary.py +++ b/tests/test_noema_exact_canary.py @@ -98,6 +98,46 @@ def test_canary_credential_selection_rejects_pat_oidc_before_model(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn('source=' + expected, output.read_text()) + def test_advertised_minimum_survives_actual_validator(self): + import os + import json + import hashlib + from unittest.mock import patch + from scripts.ci import noema_review_gate as gate + content = 'name: synthetic\non: workflow_dispatch\n' + requests = [] + class Response: + def __enter__(self): return self + def __exit__(self, *args): return False + def read(self): + schema = requests[-1]['response_format']['json_schema']['schema'] + count = schema['properties']['adversarial_validation']['properties']['probes']['minItems'] + verdict = {'decision': 'approve', 'summary': 'Synthetic review.', 'findings': [], + 'reviewed_lines': [{'path': policy.PATH, 'line': 1, 'side': 'RIGHT', 'analysis': 'Examined exact added workflow.'}], + 'adversarial_validation': {'status': 'passed', 'residual_risk': 'Offline fixture only.', + 'probes': [{'path': policy.PATH, 'line': i + 1, 'side': 'RIGHT', + 'hypothesis': f'Regression {i}', 'attack_or_counterexample': f'Counterexample {i}', + 'evidence': f'Exact line {i + 1}', 'outcome': 'falsified'} for i in range(count)]}} + return json.dumps({'choices': [{'message': {'content': json.dumps(verdict)}}]}).encode() + class Opener: + def open(self, request): + requests.append(json.loads(request.data)) + return Response() + env = {'NOEMA_EXACT_CANARY_NON_ZDR': 'true', + 'NOEMA_LLM_API_URL': 'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_API_KEY': 'synthetic', 'NOEMA_LLM_MODEL': 'auto'} + with patch.dict(os.environ, env, clear=True), \ + patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()), \ + patch.object(gate, 'reject_private_llm_url'), \ + patch.object(gate, 'fetch_changed_files', return_value=[(policy.PATH, 'added')]), \ + patch.object(gate, 'fetch_file_content_at_ref', return_value=content), \ + patch.object(gate.urllib.request, 'build_opener', return_value=Opener()): + verdict = gate.call_llm(policy.REPOSITORY, 4, {'headRefOid': policy.HEAD}, + 'PRIVATE DIFF', False, policy.HEAD, 'PRIVATE THREAD', [policy.PATH]) + self.assertEqual(verdict['decision'], 'approve') + self.assertEqual(len(verdict['adversarial_validation']['probes']), 2) + self.assertEqual(len(requests), 1) + def test_scope_is_exact(self): policy.require_target(policy.REPOSITORY, policy.NUMBER, policy.HEAD) for args in [('foreign/repo', 4, policy.HEAD), From 4c2c4b0734b0af252c563ef314d9c6aa829b7dab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 02:47:20 +0900 Subject: [PATCH 08/11] fix(ci): document exact canary policy helpers --- scripts/ci/noema_exact_canary.py | 3 +++ tests/test_noema_exact_canary.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/ci/noema_exact_canary.py b/scripts/ci/noema_exact_canary.py index 467c39ce5b..0ab669e77d 100644 --- a/scripts/ci/noema_exact_canary.py +++ b/scripts/ci/noema_exact_canary.py @@ -22,16 +22,19 @@ def require_target(repo, number, head): + """Reject any repository, pull request or head outside exact consent.""" if repo != REPOSITORY or type(number) is not int or number != NUMBER or head != HEAD: raise RuntimeError('Non-ZDR canary scope mismatch') def require_content(content): + """Admit only UTF-8 text with the explicitly consented content digest.""" if type(content) is not str or hashlib.sha256(content.encode('utf-8')).hexdigest() != DIGEST: raise RuntimeError('Non-ZDR canary content scope mismatch') def messages(content): + """Build fixed instructions plus the digest-admitted file and nothing else.""" require_content(content) return [{'role': 'system', 'content': SYSTEM}, {'role': 'user', 'content': INSTRUCTION + content}] diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py index 4445202047..f33e709e28 100644 --- a/tests/test_noema_exact_canary.py +++ b/tests/test_noema_exact_canary.py @@ -3,6 +3,16 @@ from scripts.ci import noema_exact_canary as policy class ExactCanaryTests(unittest.TestCase): + def test_public_canary_helpers_have_docstrings(self): + """The canary module remains eligible for the 100% docstring gate.""" + import ast + from pathlib import Path + tree = ast.parse(Path(policy.__file__).read_text()) + helpers = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)} + for name in ('require_target', 'require_content', 'messages'): + with self.subTest(helper=name): + self.assertTrue(ast.get_docstring(helpers[name]), name) + def test_actual_serialization_contains_only_approved_yaml(self): import json from unittest.mock import patch From 3b56c6c110b3dace75f1f0d51b85c11a9fb46ce5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 05:21:15 +0900 Subject: [PATCH 09/11] fix(noema): close exact-canary privacy boundaries and enroll publication regressions --- .github/actions/noema-review/two_phase.py | 15 +- .../agent-review-runtime-quality-ci.yml | 7 + .github/workflows/noema-review.yml | 1 + scripts/ci/noema_exact_canary.py | 17 +- scripts/ci/noema_review_gate.py | 39 ++- tests/fixtures/noema_canary_cli_fixture.py | 153 ++++++++++++ tests/fixtures/noema_canary_github_fixture.py | 39 +++ .../noema_canary_publication_fixture.py | 197 +++++++++++++++ ...nt_review_runtime_quality_consolidation.py | 51 ++++ tests/test_noema_canary_publication.py | 232 ++++++++++++++++++ tests/test_noema_exact_canary.py | 106 ++++++++ 11 files changed, 849 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/noema_canary_cli_fixture.py create mode 100644 tests/fixtures/noema_canary_github_fixture.py create mode 100644 tests/fixtures/noema_canary_publication_fixture.py create mode 100644 tests/test_noema_canary_publication.py diff --git a/.github/actions/noema-review/two_phase.py b/.github/actions/noema-review/two_phase.py index d5da7b8270..9f2e79a23f 100755 --- a/.github/actions/noema-review/two_phase.py +++ b/.github/actions/noema-review/two_phase.py @@ -164,10 +164,16 @@ def prepare_verdict(repo: str, number: int, expected_head: str, path: Path) -> i print("Current head already has a Noema review; verdict preparation skipped.") return 0 - diff, truncated = gate.fetch_diff(repo, number) - changed_files = gate.fetch_changed_files(repo, number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci import noema_exact_canary as canary + canary.require_target(repo, number, expected) + changed_files = gate.fetch_changed_files(repo, number) + diff, truncated, review_context = "", False, "" + else: + diff, truncated = gate.fetch_diff(repo, number) + changed_files = gate.fetch_changed_files(repo, number) + review_context = gate.build_review_context(repo, number, pull_request, changed_files) changed_paths = tuple(file_path for file_path, _status in changed_files) - review_context = gate.build_review_context(repo, number, pull_request, changed_files) try: verdict = gate.call_llm( repo, @@ -325,6 +331,9 @@ def main(argv: list[str]) -> int: if __name__ == "__main__": + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + from scripts.ci.noema_exact_canary import run_private_cli + raise SystemExit(run_private_cli(lambda: main(sys.argv[1:]))) try: raise SystemExit(main(sys.argv[1:])) except RuntimeError as exc: diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index cf949ebee4..ee5c2df50e 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -76,6 +76,8 @@ on: - "tests/test_personal_review_route.py" - "tests/test_noema_paid_auto_authorization.py" - "tests/test_noema_exact_canary.py" + - "tests/test_noema_canary_publication.py" + - "tests/fixtures/noema_canary_publication_fixture.py" - "scripts/ci/noema_exact_canary.py" - "scripts/ci/noema_review_gate.py" - "tests/test_tokyo_review_consumer.py" @@ -312,6 +314,8 @@ jobs: tests/test_personal_review_route.py|\ tests/test_noema_paid_auto_authorization.py|\ tests/test_noema_exact_canary.py|\ + tests/test_noema_canary_publication.py|\ + tests/fixtures/noema_canary_publication_fixture.py|\ scripts/ci/noema_exact_canary.py|\ tests/test_tokyo_review_consumer.py|\ tests/test_review_auto_cost_admission.py|\ @@ -515,6 +519,7 @@ jobs: tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ tests/test_noema_exact_canary.py \ + tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ @@ -550,6 +555,7 @@ jobs: scripts/ci/contextual_orchestrator_review_policy.py \ scripts/ci/contextual_orchestrator_review_launcher.py python -m compileall -q \ + tests/fixtures/noema_canary_publication_fixture.py \ scripts/ci/pr_review_conflict_scope.py \ scripts/ci/pr_review_autofix_context.py \ tests/test_pr_review_conflict_scope.py \ @@ -563,6 +569,7 @@ jobs: tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ tests/test_noema_exact_canary.py \ + tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index dea5d39945..2006254413 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -1132,6 +1132,7 @@ jobs: - name: Publish prepared Noema verdict on the exact live head if: env.PR_NUMBER != '' && steps.noema_prepare.outputs.prepared == 'true' env: + NOEMA_EXACT_CANARY_NON_ZDR: ${{ steps.personal_route.outputs.exact_canary || 'false' }} GH_TOKEN: ${{ env.TARGET_REPOSITORY == 'ContextualWisdomLab/k-csap-skills' && steps.noema_github_app_publication_token.outputs.token || (env.TARGET_REPOSITORY != 'ContextualWisdomLab/k-csap-skills' && (steps.noema_credential.outputs.source == 'pat' && secrets.NOEMA_REVIEW_TOKEN || steps.noema_credential.outputs.source == 'github-app' && steps.noema_github_app_publication_token.outputs.token || steps.noema_credential.outputs.source == 'oidc' && steps.noema_oidc_token.outputs.token || '')) || '' }} NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app-refresh' || steps.noema_credential.outputs.source == 'oidc' && 'noema-review-app-oidc' || '' }} NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_publication_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_publication_token.outputs['app-slug']) || '' }} diff --git a/scripts/ci/noema_exact_canary.py b/scripts/ci/noema_exact_canary.py index 0ab669e77d..97b818be29 100644 --- a/scripts/ci/noema_exact_canary.py +++ b/scripts/ci/noema_exact_canary.py @@ -86,5 +86,20 @@ def main(): require_content(gate.fetch_file_content_at_ref(args.repo, PATH, HEAD)) +def run_private_cli(callback): + """Keep canary CLI failures finite without rendering private exceptions.""" + import sys + try: + return callback() + except BaseException as exc: + if isinstance(exc, SystemExit) and (exc.code is None or type(exc.code) is int): + raise SystemExit(exc.code) from None + if isinstance(exc, KeyboardInterrupt): + print('::error::Non-ZDR canary interrupted; private diagnostics withheld.', file=sys.stderr) + raise SystemExit(130) from None + print('::error::Non-ZDR canary failed closed; private diagnostics withheld.', file=sys.stderr) + raise SystemExit(1) from None + + if __name__ == '__main__': - main() + run_private_cli(main) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5bf0e8bc12..b48f63e59e 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -265,6 +265,8 @@ def parse_http_retry_after_seconds(headers: Any) -> int | None: def _stable_failure_diagnostic(exc: BaseException) -> str: """Return actionable trusted diagnostics without reflecting model values.""" + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + return "canary-operation-failed" message = scrub_sensitive_data(str(exc)) or type(exc).__name__ if not isinstance(exc, NoemaModelOutputError): return message @@ -415,7 +417,18 @@ def graphql(query: str, **fields: str | int) -> dict[str, Any]: def fetch_pr(repo: str, number: int) -> dict[str, Any]: """Fetch the pull request data required for Noema review gating.""" owner, name = split_repo(repo) - data = graphql(PR_QUERY, owner=owner, name=name, number=number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci.noema_exact_canary import HEAD, require_target + require_target(repo, number, HEAD) + query = """query($owner:String!,$name:String!,$number:Int!) { + repository(owner:$owner,name:$name) { pullRequest(number:$number) { + number author { login } isDraft state headRefOid baseRefOid + reviews(last:100) { nodes { state body author { login } commit { oid } } } + } } + }""" + else: + query = PR_QUERY + data = graphql(query, owner=owner, name=name, number=number) pr = data.get("data", {}).get("repository", {}).get("pullRequest") if not pr: raise RuntimeError(f"PR #{number} was not found in {repo}") @@ -1765,6 +1778,13 @@ def call_llm( served_model = model_value if isinstance(model_value, str) else None finally: exc.close() + if exact_canary == "true": + served_model = None + gateway_telemetry = { + key: value for key, value in gateway_telemetry.items() + if key in {"provider_attempt_count", "attempt_number", "upstream_status"} + and type(value) is int + } elapsed = time.monotonic() - attempt_started current_failure = _stable_failure_diagnostic(exc) model_note = served_model or "unknown" @@ -1812,6 +1832,8 @@ def call_llm( f"Noema review failed closed: {current_failure}{suffix}" ) from exc elapsed = time.monotonic() - attempt_started + if exact_canary == "true": + served_model = None print( f"::notice::Noema gateway attempt outcome=success phase={active_phase} " f"duration={elapsed:.1f}s served_model={served_model or 'unknown'}; " @@ -1931,10 +1953,16 @@ def inspect_and_review(repo: str, number: int, expected_head: str) -> int: if existing_noema_review(pr, actor): print("Current head already has a Noema review; nothing to do.") return 0 - diff, truncated = fetch_diff(repo, number) - changed_files = fetch_changed_files(repo, number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci.noema_exact_canary import require_target + require_target(repo, number, expected_head) + changed_files = fetch_changed_files(repo, number) + diff, truncated, review_context = "", False, "" + else: + diff, truncated = fetch_diff(repo, number) + changed_files = fetch_changed_files(repo, number) + review_context = build_review_context(repo, number, pr, changed_files) changed_paths = tuple(path for path, _status in changed_files) - review_context = build_review_context(repo, number, pr, changed_files) verdict = call_llm(repo, number, pr, diff, truncated, expected_head, review_context, changed_paths) current_pr = fetch_pr(repo, number) try: @@ -1969,6 +1997,9 @@ def main(argv: list[str]) -> int: if __name__ == "__main__": # pragma: no cover + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + from scripts.ci.noema_exact_canary import run_private_cli + raise SystemExit(run_private_cli(lambda: main(sys.argv[1:]))) try: raise SystemExit(main(sys.argv[1:])) except RuntimeError as exc: diff --git a/tests/fixtures/noema_canary_cli_fixture.py b/tests/fixtures/noema_canary_cli_fixture.py new file mode 100644 index 0000000000..af5575d306 --- /dev/null +++ b/tests/fixtures/noema_canary_cli_fixture.py @@ -0,0 +1,153 @@ +"""Finite offline CLI fixture; importing it performs no IO.""" + + +def main(): + """Run one private synthetic CLI scenario.""" + """Execute unchanged Noema public CLI bodies with finite external IO.""" + import base64, hashlib, importlib.util, json, os, pathlib, runpy, socket, subprocess, sys, urllib.error, urllib.request + root=pathlib.Path(os.environ['FIXTURE_SOURCE']) + fixture=json.loads(pathlib.Path(os.environ['FIXTURE_INPUT']).read_text()) + state_path=pathlib.Path(os.environ['FIXTURE_STATE']) + cli_argv=sys.argv[1:] + sys.path.insert(0,str(root)) + from scripts.ci import noema_review_gate as gate, noema_exact_canary as policy + marker=fixture['marker']; case=fixture['case']; entry=fixture['entry'] + content='name: synthetic\non: workflow_dispatch\n' + policy.DIGEST=hashlib.sha256(content.encode()).hexdigest() + pr={'number':4,'state':'OPEN','isDraft':False,'headRefOid':policy.HEAD,'baseRefOid':'b'*40, + 'title':marker,'body':marker,'author':{'login':'synthetic-author'}, + 'reviews':{'nodes':[]},'reviewThreads':{'nodes':[{'path':marker,'line':1,'comments':{'nodes':[{'body':marker,'author':{'login':'synthetic'}}]}}]}} + state={'external_calls':[],'http_requests':[],'publications':0,'content_reads':0,'pr_reads':0,'fixture_digest_substitution':True,'stage_faults':[]} + REAL_OPEN=os.open; REAL_FSYNC=os.fsync; REAL_UNLINK=pathlib.Path.unlink + def faulted_open(path,*args,**kwargs): + if str(path).endswith('.envelope') and boundary=='envelope': + state['stage_faults'].append('envelope');save();fail(kind) + return REAL_OPEN(path,*args,**kwargs) + def faulted_fsync(fd): + if boundary=='fsync': + state['stage_faults'].append('fsync');save();fail(kind) + return REAL_FSYNC(fd) + def faulted_unlink(path,*args,**kwargs): + if str(path).endswith('.envelope') and boundary=='cleanup': + state['stage_faults'].append('cleanup');save();fail(kind) + return REAL_UNLINK(path,*args,**kwargs) + os.open=faulted_open;os.fsync=faulted_fsync;pathlib.Path.unlink=faulted_unlink + def save(): state_path.write_text(json.dumps(state,indent=2)) + def fail(kind): + if kind=='nonzero':return subprocess.CompletedProcess(['gh'],1,marker,marker) + if kind=='malformed':return subprocess.CompletedProcess(['gh'],0,marker,'') + if kind=='decode':raise UnicodeDecodeError('utf-8',marker.encode()+b'\xff',len(marker),len(marker)+1,marker) + cls={'runtime':RuntimeError,'os':OSError,'value':ValueError,'interrupt':KeyboardInterrupt,'exit':SystemExit,'type':TypeError}[kind] + error=cls(marker);error.add_note(marker) + raise error + boundary,_,kind=case.partition(':') + def fake_run(args,**kwargs): + if args[:3]==['gh','api','graphql']: + state['pr_reads']+=1 + stage='readback' if boundary=='readback' and state['pr_reads']>1 else 'pr' + state['context_skipped']='reviewThreads' not in kwargs.get('input','') + elif args[:2]==['gh','api'] and '/files' in args[2]:stage='files' + elif args[:2]==['gh','api'] and '/contents/' in args[2]: + state['content_reads']+=1 + stage='context' if state['content_reads']==1 and entry!='policy' and not state.get('context_skipped',False) else 'content' + if state.get('context_skipped',False):stage='content' + elif args[:2]==['gh','api'] and 'Accept: application/vnd.github.v3.diff' in args:stage='diff' + else:raise AssertionError('Unexpected external operation') + state['external_calls'].append(stage);save() + if boundary==stage: + result=fail(kind) + if result is not None:return result + if stage in {'pr','readback'}: value=json.dumps({'data':{'repository':{'pullRequest':pr}}}) + elif stage=='files': value=json.dumps([policy.PATH,'added'])+'\n' + elif stage=='diff': value='diff --git a/'+policy.PATH+' b/'+policy.PATH+'\n--- /dev/null\n+++ b/'+policy.PATH+'\n@@ -0,0 +1,1 @@\n+'+marker+'\n' + else: + data=content.encode() + value=json.dumps({'encoding':'base64','size':len(data),'content':base64.b64encode(data).decode()}) + return subprocess.CompletedProcess(args,0,value,'') + native=fixture.get('native',False) + if not native: + subprocess.run=fake_run + socket.getaddrinfo=lambda host,*args,**kwargs:[(socket.AF_INET,socket.SOCK_STREAM,6,'',('93.184.216.34',443))] + def verdict(): + decision=marker if boundary=='verdict' else 'approve' + return {'decision':decision,'summary':'Synthetic exact-file review.','findings':[], + 'reviewed_lines':[{'path':policy.PATH,'line':1,'side':'RIGHT','analysis':'Reviewed the added workflow.'}], + 'adversarial_validation':{'status':'passed','residual_risk':'Offline fixture only.','probes':[ + {'path':policy.PATH,'line':i+1,'side':'RIGHT','hypothesis':f'Concrete regression {i}', + 'attack_or_counterexample':f'Concrete counterexample {i}','evidence':f'Exact line {i+1}','outcome':'falsified'} for i in range(2)]}} + class Response: + def __enter__(self): + if boundary=='enter':fail(kind) + return self + def __exit__(self,*args): + if boundary=='close':fail(kind) + return False + def read(self): + if boundary=='read':fail(kind) + if boundary=='model_decode':return marker.encode()+b'\xff' + model=marker if boundary=='telemetry' else 'auto' + return json.dumps({'model':model,'choices':[{'message':{'content':json.dumps(verdict())}}]}).encode() + class ErrorBody: + def read(self,*args): + if boundary=='error_read':fail(kind) + return json.dumps({'error':{'detail':{'model':marker,'terminal_reason':marker,'attempts':[{'provider_name':marker,'phase':marker,'attempt_number':1,'provider_status':503}]}}}).encode() + def close(self): + if boundary=='error_close':fail(kind) + class Opener: + def open(self,request): + if native:state.update(json.loads(state_path.read_text())) + state['http_requests'].append({'url':request.full_url,'data':request.data.decode(),'method':request.method});save() + if boundary=='http': + if kind=='url':raise urllib.error.URLError(marker) + if kind=='http':raise urllib.error.HTTPError(request.full_url,503,marker,{},ErrorBody()) + fail(kind) + if boundary in {'error_read','error_close'}:raise urllib.error.HTTPError(request.full_url,503,marker,{},ErrorBody()) + return Response() + urllib.request.build_opener=lambda *args,**kwargs:Opener() + def no_publish(*args,**kwargs):state['publications']+=1;save();raise AssertionError('Publication forbidden') + gate.submit_review=no_publish + # Defense in depth: no native process/socket operation may escape adapters. + def audit(event,args): + if event.startswith('socket.') or event in {'os.system','os.posix_spawn'}: + raise AssertionError('Offline IO fence') + if event=='subprocess.Popen': + if not native or pathlib.Path(args[0]).name!='gh':raise AssertionError('Offline process fence') + sys.addaudithook(audit) + save() + args=['--repo',policy.REPOSITORY,'--pr-number','4','--expected-head',policy.HEAD] + if entry=='prepare': + invocation=[str(root/'.github/actions/noema-review/two_phase.py'),*args,'--prepare-verdict-file',str(state_path.with_suffix('.envelope'))] + if cli_argv: + assert cli_argv[0]==invocation[0] + invocation=cli_argv + sys.argv=invocation + runpy.run_path(sys.argv[0],run_name='__main__') + elif entry=='gate': + sys.argv=[str(root/'scripts/ci/noema_review_gate.py'),*args] + runpy.run_path(sys.argv[0],run_name='__main__') + elif entry=='policy': + sys.argv=[str(root/'scripts/ci/noema_exact_canary.py'),*args] + if cli_argv: + assert cli_argv[:2]==['-m','scripts.ci.noema_exact_canary'] + sys.argv=[sys.argv[0],*cli_argv[2:]] + # Compile the complete actual CLI; only the explicit fixture digest literal + # is substituted, matching the imported policy fixture used by prepare. + text=pathlib.Path(sys.argv[0]).read_text() + old="DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d'" + assert text.count(old)==1 + text=text.replace(old,'DIGEST = '+repr(policy.DIGEST)) + exec(compile(text,sys.argv[0],'exec'),{'__name__':'__main__','__file__':sys.argv[0]}) + elif entry=='publish': + envelope=state_path.with_suffix('.envelope') + payload={'schema_version':1,'repository':policy.REPOSITORY,'pull_request_number':4,'expected_head':policy.HEAD, + 'expected_base':'b'*40,'reviewer_actor':'cwl-noema-review[bot]','pull_request_author':'synthetic-author','verdict':verdict()} + envelope.write_text(json.dumps(payload));envelope.chmod(0o600) + if boundary=='parse_envelope':envelope.write_bytes(marker.encode()+b'\xff') + if boundary=='cleanup':pr['isDraft']=True + sys.argv=[str(root/'.github/actions/noema-review/two_phase.py'),*args,'--publish-verdict-file',str(envelope)] + runpy.run_path(sys.argv[0],run_name='__main__') + else:raise AssertionError('Unknown fixture entry') + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/noema_canary_github_fixture.py b/tests/fixtures/noema_canary_github_fixture.py new file mode 100644 index 0000000000..714a6cbd40 --- /dev/null +++ b/tests/fixtures/noema_canary_github_fixture.py @@ -0,0 +1,39 @@ +"""Inert native gh fixture emitting synthetic API bytes offline.""" + +def main(): + """Answer one finite native process acquisition or fail on selected stage.""" + import base64 + import json + import os + import pathlib + import sys + fixture=json.loads(pathlib.Path(os.environ['FIXTURE_INPUT']).read_text()) + state_file=pathlib.Path(os.environ['FIXTURE_STATE']) + state=json.loads(state_file.read_text()) + marker=fixture['marker'];case=fixture['case'];args=sys.argv[1:] + if args[:2]==['api','graphql']: + stage='pr';query=sys.stdin.read();state['context_skipped']='reviewThreads' not in query + elif args[:1]==['api'] and '/files' in args[1]:stage='files' + elif args[:1]==['api'] and '/contents/' in args[1]: + state['content_reads']+=1 + stage='content' if state['context_skipped'] else 'context' if state['content_reads']==1 else 'content' + elif args[:1]==['api'] and 'Accept: application/vnd.github.v3.diff' in args:stage='diff' + else:raise SystemExit(2) + state['external_calls'].append(stage);state_file.write_text(json.dumps(state)) + if case==stage+':nonzero': + print(marker,file=sys.stderr) + raise SystemExit(1) + path='.github/workflows/docker-ephemeral-canary.yml';head='bf85ddde01f871f0e713db54aae5a16883777d9f' + pr={'number':4,'state':'OPEN','isDraft':False,'headRefOid':head,'baseRefOid':'b'*40,'title':marker,'body':marker, + 'author':{'login':'synthetic-author'},'reviews':{'nodes':[]},'reviewThreads':{'nodes':[{'path':marker,'comments':{'nodes':[{'body':marker,'author':{'login':'synthetic'}}]}}]}} + if stage=='pr':value=json.dumps({'data':{'repository':{'pullRequest':pr}}}) + elif stage=='files':value=json.dumps([path,'added']) + elif stage=='diff':value='diff --git a/'+path+' b/'+path+'\n--- /dev/null\n+++ b/'+path+'\n@@ -0,0 +1,1 @@\n+'+marker+'\n' + else: + data=b'name: synthetic\non: workflow_dispatch\n' + value=json.dumps({'encoding':'base64','size':len(data),'content':base64.b64encode(data).decode()}) + print(value) + + +if __name__=='__main__': + main() diff --git a/tests/fixtures/noema_canary_publication_fixture.py b/tests/fixtures/noema_canary_publication_fixture.py new file mode 100644 index 0000000000..e5deb299b6 --- /dev/null +++ b/tests/fixtures/noema_canary_publication_fixture.py @@ -0,0 +1,197 @@ +"""Offline external-IO adapters for the actual workflow-to-CLI composition.""" + + +def main(): + """Run a full trusted CLI with only synthetic gh, HTTP and IO faults.""" + import base64 + import hashlib + import json + import os + from pathlib import Path + import runpy + import socket + import subprocess + import sys + import urllib.error + import urllib.request + + root = Path(os.environ['FIXTURE_SOURCE']) + settings = json.loads(Path(os.environ['FIXTURE_INPUT']).read_text()) + state_path = Path(os.environ['FIXTURE_STATE']) + sys.path.insert(0, str(root)) + from scripts.ci import noema_exact_canary as policy, noema_review_gate as gate + + phase, case, marker = settings['phase'], settings['case'], settings['marker'] + content = 'name: synthetic\non: workflow_dispatch\n' + # The only policy substitution is explicitly synthetic content provenance. + policy.DIGEST = hashlib.sha256(content.encode()).hexdigest() + state = {'phase': phase, 'case': case, 'external_calls': [], 'queries': [], + 'requests': [], 'posts': [], 'faults': [], + 'canary_mode': os.environ.get('NOEMA_EXACT_CANARY_NON_ZDR'), + 'argv': sys.argv[1:], 'fixture_digest_substitution': True} + + def save(): + """Persist external observations outside public CLI streams.""" + state_path.write_text(json.dumps(state, indent=2)) + + def fault(): + """Raise private diagnostics only at the selected external boundary.""" + error = OSError(marker) + error.add_note(marker) + raise error + + def verdict(): + """Return substantive, line-cited evidence admitted by the real gate.""" + return {'decision': 'approve', 'summary': 'Synthetic exact-file review.', + 'findings': [], 'reviewed_lines': [ + {'path': policy.PATH, 'line': 1, 'side': 'RIGHT', + 'analysis': 'Reviewed the new workflow.'}], + 'adversarial_validation': {'status': 'passed', + 'residual_risk': 'Offline fixture only.', 'probes': [ + {'path': policy.PATH, 'line': i + 1, 'side': 'RIGHT', + 'hypothesis': f'Concrete regression {i}', + 'attack_or_counterexample': f'Concrete counterexample {i}', + 'evidence': f'Exact line {i + 1}', 'outcome': 'falsified'} + for i in range(2)]}} + + def fake_run(args, **kwargs): + """Replace only external gh IO, retaining all real gate consumers.""" + state['external_calls'].append(list(args)) + save() + if args[:3] == ['gh', 'api', 'graphql']: + query = kwargs.get('input', '') + state['queries'].append(query) + save() + if phase == 'publish' and case == 'pr_nonzero': + state['faults'].append('gh_stderr') + save() + return subprocess.CompletedProcess(args, 1, '', marker) + pr = {'number': policy.NUMBER, 'state': 'OPEN', 'isDraft': False, + 'headRefOid': policy.HEAD, 'baseRefOid': 'b' * 40, + 'title': marker, 'body': marker, + 'author': {'login': 'synthetic-author'}, + 'reviews': {'nodes': []}, 'reviewThreads': {'nodes': []}} + if phase == 'publish': + if case in {'cleanup_os', 'draft'}: + pr['isDraft'] = True + elif case == 'head_drift': + pr['headRefOid'] = 'c' * 40 + elif case == 'base_drift': + pr['baseRefOid'] = 'c' * 40 + elif case == 'author_drift': + pr['author']['login'] = 'different-author' + elif case == 'self_author': + pr['author']['login'] = 'cwl-noema-review[bot]' + value = json.dumps({'data': {'repository': {'pullRequest': pr}}}) + elif args[:4] == ['gh', 'api', '-X', 'POST']: + state['posts'].append(json.loads(kwargs['input'])) + save() + value = '{}' + elif args[:2] == ['gh', 'api'] and '/files' in args[2]: + value = json.dumps([policy.PATH, 'added']) + '\n' + elif args[:2] == ['gh', 'api'] and '/contents/' in args[2]: + data = content.encode() + value = json.dumps({'encoding': 'base64', 'size': len(data), + 'content': base64.b64encode(data).decode()}) + elif args[:2] == ['gh', 'api'] and 'Accept: application/vnd.github.v3.diff' in args: + value = (f'diff --git a/{policy.PATH} b/{policy.PATH}\n' + f'--- /dev/null\n+++ b/{policy.PATH}\n' + '@@ -0,0 +1,2 @@\n+name: synthetic\n+on: workflow_dispatch\n') + else: + raise AssertionError('Unexpected external operation') + return subprocess.CompletedProcess(args, 0, value, '') + + subprocess.run = fake_run + socket.getaddrinfo = lambda *_a, **_kw: [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('93.184.216.34', 443))] + + class Response: + """Synthetic successful model reply with untrusted model metadata.""" + def __enter__(self): + """Enter the actual HTTP consumer's context.""" + return self + + def __exit__(self, *_args): + """Preserve normal response settlement.""" + return False + + def read(self): + """Return valid substantive evidence and a private metadata marker.""" + return json.dumps({'model': marker, 'choices': [ + {'message': {'content': json.dumps(verdict())}}]}).encode() + + class ErrorBody: + """Synthetic capacity failure with private provider metadata.""" + def read(self, *_args): + """Return typed and free-text telemetry through the real parser.""" + return json.dumps({'error': {'detail': {'model': marker, + 'terminal_reason': marker, 'attempts': [{'provider_name': marker, + 'phase': marker, 'attempt_number': 1, + 'provider_status': 503}]}}}).encode() + + def close(self): + """Settle the inert error body.""" + return None + + class Opener: + """Record real Request bytes without network dispatch.""" + def open(self, request): + """Serve one finite synthetic HTTP outcome.""" + state['requests'].append({'url': request.full_url, + 'data': request.data.decode()}) + save() + if case == 'prepare_http': + raise urllib.error.HTTPError(request.full_url, 503, marker, {}, ErrorBody()) + return Response() + + urllib.request.build_opener = lambda *_a, **_kw: Opener() + real_unlink, real_fsync = Path.unlink, os.fsync + + def unlink(path, *args, **kwargs): + """Inject a reached cleanup fault without replacing its caller.""" + if phase == 'publish' and case == 'cleanup_os' and path.name == 'noema-verdict-envelope.json': + state['faults'].append('cleanup_os') + save() + fault() + return real_unlink(path, *args, **kwargs) + + def fsync(fd): + """Inject preparation persistence failure through a real file owner.""" + if phase == 'prepare' and case == 'prepare_fsync_os': + state['faults'].append('prepare_fsync_os') + save() + fault() + return real_fsync(fd) + + Path.unlink, os.fsync = unlink, fsync + + def audit(event, _args): + """Forbid every unadapted process or socket operation.""" + if event.startswith('socket.') or event in {'subprocess.Popen', 'os.system', 'os.posix_spawn'}: + raise AssertionError('Offline fixture IO fence') + + sys.addaudithook(audit) + save() + argv = sys.argv[1:] + if argv[:1] == ['-m']: + module = argv[1] + sys.argv = [module, *argv[2:]] + if module == 'scripts.ci.noema_exact_canary': + # Execute its complete real CLI, substituting only the fixture digest. + path = root / 'scripts/ci/noema_exact_canary.py' + text = path.read_text() + old = "DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d'" + assert text.count(old) == 1 + text = text.replace(old, 'DIGEST = ' + repr(policy.DIGEST)) + exec(compile(text, str(path), 'exec'), {'__name__': '__main__', '__file__': str(path)}) + else: + assert module == 'scripts.ci.personal_review_route' + runpy.run_module(module, run_name='__main__') + else: + assert argv[0] == str(root / '.github/actions/noema-review/two_phase.py') + sys.argv = argv + runpy.run_path(argv[0], run_name='__main__') + + +if __name__ == '__main__': + main() diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index 4fecc93656..05fbd6b903 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -240,6 +240,57 @@ def test_ghas_runtime_step_runs_exact_tests_and_fails_closed( assert result.stdout == result.stderr == "" +@pytest.mark.parametrize( + "changed_path", + ("tests/test_noema_canary_publication.py", + "tests/fixtures/noema_canary_publication_fixture.py"), +) +def test_publication_regressions_start_and_select_quality( + changed_path: str, tmp_path: Path +) -> None: + """Enroll each publication input in the trigger and actual selector.""" + trigger = _workflow_text().split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger + test_ghas_runtime_selector_executes_the_actual_shell( + changed_path, {"review_repair"}, tmp_path + ) + + +@pytest.mark.parametrize("pytest_exit", (0, 42)) +def test_publication_regressions_run_and_compile( + pytest_exit: int, tmp_path: Path +) -> None: + """Observe real step argv and prevent compile after failed pytest.""" + step = _workflow_text().split( + "- name: Verify scheduler and contextual-orchestrator review-repair contracts", 1 + )[1].split(" - name:", 1)[0] + assert "if: steps.affected_suites.outputs.review_repair == 'true'" in step + script = textwrap.dedent(step.split(" run: |\n", 1)[1]) + calls = tmp_path / "calls.txt" + recorder = ('python() { printf "%s\\n" "$*" >> "$CALLS"; ' + 'if [[ "$*" == *pytest* ]]; then return "$PYTEST_EXIT"; fi; }\n') + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", + recorder + script], cwd=REPOSITORY_ROOT, + env={**os.environ, "CALLS": str(calls), "PYTEST_EXIT": str(pytest_exit)}, + text=True, capture_output=True, check=False, + ) + rows = [line.split() for line in calls.read_text().splitlines()] + assert rows[0][:3] == ["-m", "pytest", "-q"] + assert rows[0].count("tests/test_noema_canary_publication.py") == 1 + assert "tests/fixtures/noema_canary_publication_fixture.py" not in rows[0] + assert result.returncode == pytest_exit + assert result.stdout == result.stderr == "" + if pytest_exit: + assert len(rows) == 1 + else: + assert len(rows) == 3 + assert rows[2][:3] == ["-m", "compileall", "-q"] + for path in ("tests/test_noema_canary_publication.py", + "tests/fixtures/noema_canary_publication_fixture.py"): + assert rows[2].count(path) == 1 + + def test_exact_head_is_verified_before_selected_suites_run() -> None: """Reject a checkout that differs from the pull request's current head.""" diff --git a/tests/test_noema_canary_publication.py b/tests/test_noema_canary_publication.py new file mode 100644 index 0000000000..6eadc7d9a3 --- /dev/null +++ b/tests/test_noema_canary_publication.py @@ -0,0 +1,232 @@ +"""Exercise parsed admission/prepare/publish shells and the full real CLI.""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import uuid + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = 'ContextualWisdomLab/traveling-agent' +HEAD = 'bf85ddde01f871f0e713db54aae5a16883777d9f' +ADMIT = 'Admit existing personal LiteLLM auto route' +PREPARE = 'Prepare Noema model verdict' +PUBLISH = 'Publish prepared Noema verdict on the exact live head' +MODE = 'NOEMA_EXACT_CANARY_NON_ZDR' + + +def _steps(): + """Parse actual YAML instead of supplying a synthetic workflow mapping.""" + document = yaml.load((ROOT / '.github/workflows/noema-review.yml').read_text(), Loader=yaml.BaseLoader) + return {step['name']: step for step in document['jobs']['noema-review']['steps']} + + +def _output(path): + """Read simple runner output records from the actual successful shell.""" + return dict(line.split('=', 1) for line in path.read_text().splitlines()) if path.exists() else {} + + +def _mode_from_declaration(expression, *, variables, outputs): + """Evaluate only the workflow's finite mode references, never invent one.""" + match = re.fullmatch(r"\$\{\{ (vars\.[A-Z_]+|steps\.[a-z_]+\.outputs\.[a-z_]+) \|\| 'false' \}\}", expression) + assert match, 'Unsupported actual canary mode declaration' + reference = match[1].split('.') + if reference[0] == 'vars': + return variables.get(reference[1]) or 'false' + return outputs.get(reference[1], {}).get(reference[3]) or 'false' + + +def _run_step(tmp_path, name, phase, case, marker, *, variables, outputs): + """Run unchanged shell bytes; derive mode exclusively from its declaration.""" + tmp_path.mkdir() + step = _steps()[name] + input_path, state_path = tmp_path / 'input.json', tmp_path / 'state.json' + input_path.write_text(json.dumps({'phase': phase, 'case': case, 'marker': marker})) + bin_path = tmp_path / 'bin' + bin_path.mkdir() + launcher = bin_path / 'python3' + launcher.write_text('#!/bin/sh\nexec ' + shlex.join([ + sys.executable, '-I', '-B', str(ROOT / 'tests/fixtures/noema_canary_publication_fixture.py')]) + ' "$@"\n') + launcher.chmod(0o700) + output = tmp_path / 'outputs' + # All credentials below are inert external-IO inputs, not real Secrets. + environment = {'PATH': str(bin_path) + os.pathsep + os.environ['PATH'], + 'HOME': str(tmp_path), 'TMPDIR': str(tmp_path), + 'TARGET_REPOSITORY': REPOSITORY, 'PR_NUMBER': '4', 'EXPECTED_HEAD_SHA': HEAD, + 'GITHUB_WORKSPACE': str(ROOT), 'RUNNER_TEMP': str(tmp_path.parent), + 'GITHUB_OUTPUT': str(output), 'GH_TOKEN': 'synthetic', + 'NOEMA_REVIEW_TOKEN_SOURCE': 'noema-review-github-app-refresh' if phase == 'publish' else 'noema-review-github-app', + 'NOEMA_REVIEW_ACTOR': 'cwl-noema-review[bot]', 'NOEMA_REVIEW_INSTALLATION_ID': '1', + 'NOEMA_EXACT_CANARY_API_KEY': 'synthetic', 'LLM_GATEWAY_API_KEY': 'synthetic', + 'NOEMA_REVIEW_PAID_AUTO_AUTHORIZED': 'false', 'PERSONAL_REVIEW_ZERO_COST_ATTESTED': 'true', + 'PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED': 'false', 'PERSONAL_REVIEW_ZDR_ATTESTED': 'true', + 'PRIVATE_TARGET': 'true', 'PERSONAL_ROUTE_SELECTED': outputs.get('personal_route', {}).get('selected', ''), + 'FIXTURE_SOURCE': str(ROOT), 'FIXTURE_INPUT': str(input_path), 'FIXTURE_STATE': str(state_path)} + # Step-local Actions env does not persist. Missing Publish env stays absent. + assert MODE not in environment + if MODE in step.get('env', {}): + environment[MODE] = _mode_from_declaration(step['env'][MODE], variables=variables, outputs=outputs) + command = ['bash', '-c', step['run']] + if sys.platform == 'darwin': + profile = ('(version 1)(allow default)(deny network*)(deny file-write*)' + '(allow file-write* (subpath ' + json.dumps(str(tmp_path.parent)) + '))') + command = ['/usr/bin/sandbox-exec', '-p', profile, *command] + result = subprocess.run(command, cwd=ROOT, env=environment, capture_output=True, timeout=20) + (tmp_path / 'stdout.bin').write_bytes(result.stdout) + (tmp_path / 'stderr.bin').write_bytes(result.stderr) + state = json.loads(state_path.read_text()) if state_path.exists() else None + row = {'phase': phase, 'case': case, 'exit': result.returncode, + 'mode_declaration': step.get('env', {}).get(MODE), + 'mode_value': environment.get(MODE), 'outputs': _output(output), + 'stdout_sha256': hashlib.sha256(result.stdout).hexdigest(), + 'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(), + 'marker_in_logs': marker.encode() in result.stdout + result.stderr, + 'state': state} + (tmp_path / 'result.json').write_text(json.dumps(row, indent=2)) + return row, result + + +def _admit_and_prepare(tmp_path, marker, *, mode='true', prepare_case='success'): + """Connect real policy admission output to real preparation's environment.""" + outputs, variables = {}, {MODE: mode} + admitted, _ = _run_step(tmp_path / 'admit', ADMIT, 'admit', 'success', marker, + variables=variables, outputs=outputs) + assert admitted['exit'] == 0, 'Actual admission did not succeed' + outputs['personal_route'] = admitted['outputs'] + assert outputs['personal_route'].get('selected') == 'true' + assert (outputs['personal_route'].get('exact_canary') == 'true') == (mode == 'true') + prepared, _ = _run_step(tmp_path / 'prepare', PREPARE, 'prepare', prepare_case, marker, + variables=variables, outputs=outputs) + outputs['noema_prepare'] = prepared['outputs'] + if prepare_case == 'success': + assert prepared['exit'] == 0 and prepared['outputs'].get('prepared') == 'true' + envelope = tmp_path / 'noema-verdict-envelope.json' + assert envelope.is_file() and envelope.stat().st_mode & 0o077 == 0 + return variables, outputs, admitted, prepared + + +@pytest.mark.parametrize('case', ['pr_nonzero', 'cleanup_os']) +def test_actual_publish_shell_contains_external_failures(tmp_path, case): + """SEC1: no manual missing-env injection can hide the real composition.""" + marker = 'PRIVATE_PUBLISH_' + uuid.uuid4().hex + variables, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker) + published, result = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables=variables, outputs=outputs) + assert result.returncode == 1 + state = published['state'] + assert state['faults'] == ['gh_stderr' if case == 'pr_nonzero' else 'cleanup_os'] + assert state['posts'] == [] and state['requests'] == [] + assert not published['marker_in_logs'], 'Private marker reached actual Publish shell->CLI output' + assert state['canary_mode'] == 'true' + assert all('reviewThreads' not in query for query in state['queries']) + assert not admitted['marker_in_logs'] and not prepared['marker_in_logs'] + + +@pytest.mark.parametrize('mode', ['true', 'false']) +@pytest.mark.parametrize('case,expected_exit,expected_posts', [ + ('success', 0, 1), ('head_drift', 0, 0), ('base_drift', 0, 0), + ('draft', 0, 0), ('author_drift', 1, 0), ('self_author', 1, 0), +]) +def test_connected_publication_preserves_privacy_and_exact_identity( + tmp_path, mode, case, expected_exit, expected_posts, +): + """Real admission->prepare->publish preserves normal behavior and guards.""" + marker = 'PRIVATE_PHASE_' + uuid.uuid4().hex + variables, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker, mode=mode) + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables=variables, outputs=outputs) + assert published['exit'] == expected_exit + state = published['state'] + assert state['canary_mode'] == mode + assert len(state['posts']) == expected_posts + assert state['requests'] == [] + assert not (tmp_path / 'noema-verdict-envelope.json').exists() + assert all(('reviewThreads' not in query) == (mode == 'true') for query in state['queries']) + assert not any(marker in json.dumps(post) for post in state['posts']) + if state['posts']: + assert state['posts'][0]['commit_id'] == HEAD + assert state['posts'][0]['event'] == 'APPROVE' + assert 'noema-review-github-app-refresh' in state['posts'][0]['body'] + if mode == 'true': + assert not any(row['marker_in_logs'] for row in (admitted, prepared, published)) + requests = prepared['state']['requests'] + assert len(requests) == 1 and marker not in requests[0]['data'] + payload = json.loads(requests[0]['data']) + assert set(payload) == {'model', 'messages', 'response_format'} + from scripts.ci import noema_exact_canary as policy + assert payload['model'] == 'auto' + assert payload['messages'] == [ + {'role': 'system', 'content': policy.SYSTEM}, + {'role': 'user', 'content': policy.INSTRUCTION + 'name: synthetic\non: workflow_dispatch\n'}] + else: + # Normal mode intentionally retains its broader context and telemetry. + assert prepared['marker_in_logs'] + assert marker in prepared['state']['requests'][0]['data'] + + +@pytest.mark.parametrize('case', ['prepare_fsync_os', 'prepare_http']) +def test_connected_preparation_cleanup_and_telemetry_stay_private(tmp_path, case): + """Real prep failure sinks remain contained with admission-derived mode.""" + marker = 'PRIVATE_PREP_' + uuid.uuid4().hex + _, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker, prepare_case=case) + assert prepared['exit'] == 1 + assert not admitted['marker_in_logs'] and not prepared['marker_in_logs'] + assert not (tmp_path / 'noema-verdict-envelope.json').exists() + assert outputs['noema_prepare'].get('prepared') != 'true' + state = prepared['state'] + assert state['canary_mode'] == 'true' and state['posts'] == [] + assert len(state['requests']) == 1 + assert marker not in state['requests'][0]['data'] + assert marker not in json.dumps(outputs) + if case == 'prepare_fsync_os': + assert state['faults'] == ['prepare_fsync_os'] + else: + assert outputs['noema_prepare']['transport_http_status'] == '503' + assert outputs['noema_prepare']['provider_attempt_count'] == '1' + + +@pytest.mark.parametrize('case', ['pr_nonzero', 'cleanup_os']) +def test_user_variable_alone_cannot_select_publication_privacy(tmp_path, case): + """Without the executable admission output, even vars=true stays off.""" + marker = 'NORMAL_PUBLISH_' + uuid.uuid4().hex + _, outputs, _, _ = _admit_and_prepare(tmp_path, marker, mode='false') + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables={MODE: 'true'}, outputs=outputs) + assert published['exit'] == 1 + assert published['state']['canary_mode'] == 'false' + assert published['marker_in_logs'] # unchanged normal-mode diagnostic control + assert published['state']['posts'] == [] + + +@pytest.mark.parametrize('mode', ['false', 'true']) +def test_admitted_publication_mode_does_not_follow_later_user_variable(tmp_path, mode): + """The successful admission output, not a later variable, selects mode.""" + marker = 'ADMISSION_BINDING_' + uuid.uuid4().hex + _, outputs, _, _ = _admit_and_prepare(tmp_path, marker, mode=mode) + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', 'success', marker, + variables={MODE: 'false' if mode == 'true' else 'true'}, outputs=outputs) + assert published['exit'] == 0 + assert published['state']['canary_mode'] == mode + assert len(published['state']['posts']) == 1 + + +@pytest.mark.parametrize('mode', ['', 'invalid']) +def test_invalid_admission_cannot_emit_canary_identity(tmp_path, mode): + """The trusted shell emits no identity when its executable condition fails.""" + marker = 'DENIED_ADMISSION_' + uuid.uuid4().hex + admitted, _ = _run_step(tmp_path / 'admit', ADMIT, 'admit', 'success', marker, + variables={MODE: mode}, outputs={}) + if mode == '': + assert admitted['exit'] == 0 + assert 'exact_canary' not in admitted['outputs'] + else: + assert admitted['exit'] != 0 and admitted['outputs'] == {} + assert not admitted['marker_in_logs'] diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py index f33e709e28..829dbf0e52 100644 --- a/tests/test_noema_exact_canary.py +++ b/tests/test_noema_exact_canary.py @@ -163,3 +163,109 @@ def test_payload_refuses_unapproved_context(self): if __name__ == '__main__': unittest.main() + + +"""Public CLI privacy regression using real consumers and finite IO adapters.""" +import hashlib, json, os, pathlib, subprocess, sys, uuid +import pytest + +ROOT=pathlib.Path(__file__).resolve().parents[1] + +MATRIX = [(entry, stage+':'+kind) + for entry,stages in [('prepare',('pr','files','content','http','enter','read','close','envelope','fsync')), + ('policy',('pr','files','content')), + ('gate',('pr','files','content','http','enter','read','close','readback')), + ('publish',('pr','cleanup'))] + for stage in stages for kind in ('nonzero','decode','runtime','os','value','interrupt','exit') + if kind!='nonzero' or stage in {'pr','files','content','readback'}] +MATRIX += [('prepare',stage+':'+kind) for stage in ('error_read','error_close') for kind in ('runtime','os','value','interrupt','exit')] +MATRIX += [(entry,case) for entry in ('prepare','gate') for case in ('http:url','http:http','verdict:invalid','model_decode:invalid')] +MATRIX += [('prepare','telemetry:success'),('prepare','context:runtime'),('prepare','context:os'),('prepare','context:interrupt'),('prepare','context:exit'),('prepare','diff:nonzero'),('prepare','success:ok'),('policy','success:ok'),('publish','parse_envelope:decode')] +MATRIX += [(entry,stage+':'+kind) for entry,stages in ( + ('workflow_admit',('pr','files','content')),('workflow_prepare',('pr','files','content','read','close','fsync'))) + for stage in stages for kind in ('nonzero','os','interrupt','exit') + if kind!='nonzero' or stage in {'pr','files','content'}] +MATRIX += [('workflow_admit','success:ok'),('workflow_prepare','success:ok')] +MATRIX += [('native_'+entry,case) for entry,case in ( + ('prepare','pr:nonzero'),('prepare','files:nonzero'),('prepare','content:nonzero'), + ('prepare','success:ok'),('policy','content:nonzero'),('policy','success:ok'))] + +@pytest.mark.parametrize('entry,case', MATRIX) +def test_public_cli_does_not_expose_private_context(entry,case,tmp_path): + marker='PRIVATE_CANARY_'+uuid.uuid4().hex + input_file=tmp_path/'input.json';state_file=tmp_path/'state.json' + input_file.write_text(json.dumps({'entry':entry,'case':case,'marker':marker})) + source=str(ROOT) + workflow=entry.startswith('workflow_') + native=entry.startswith('native_') + actual_entry={'workflow_admit':'policy','workflow_prepare':'prepare'}.get(entry,entry.removeprefix('native_')) + input_file.write_text(json.dumps({'entry':actual_entry,'case':case,'marker':marker,'native':native})) + environment={'PATH':os.environ['PATH'],'HOME':str(tmp_path),'TMPDIR':str(tmp_path), + 'NOEMA_EXACT_CANARY_NON_ZDR':'true','NOEMA_LLM_API_URL':'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_MODEL':'auto','NOEMA_LLM_API_KEY':'synthetic', + 'NOEMA_REVIEW_TOKEN_SOURCE':'noema-review-github-app','NOEMA_REVIEW_ACTOR':'cwl-noema-review[bot]', + 'NOEMA_REVIEW_INSTALLATION_ID':'1','FIXTURE_SOURCE':source,'FIXTURE_INPUT':str(input_file),'FIXTURE_STATE':str(state_file)} + command=[sys.executable,'-I','-B',str(ROOT/'tests/fixtures/noema_canary_cli_fixture.py')] + if native: + import shlex + launcher=tmp_path/'native-bin';launcher.mkdir() + script=launcher/'gh' + script.write_text('#!/bin/sh\nexec '+shlex.join([sys.executable,'-I','-B',str(ROOT/'tests/fixtures/noema_canary_github_fixture.py')])+' "$@"\n') + script.chmod(0o700) + environment['PATH']=str(launcher)+os.pathsep+environment['PATH'] + if workflow: + import shlex + import yaml + launcher=tmp_path/'bin';launcher.mkdir() + script=launcher/'python3' + script.write_text('#!/bin/sh\nexec '+shlex.join(command)+' "$@"\n');script.chmod(0o700) + environment.update(PATH=str(launcher)+os.pathsep+environment['PATH'],TARGET_REPOSITORY=policy.REPOSITORY, + PR_NUMBER='4',EXPECTED_HEAD_SHA=policy.HEAD,NOEMA_EXACT_CANARY_API_KEY='synthetic',GH_TOKEN='synthetic', + PERSONAL_ROUTE_SELECTED='true',GITHUB_WORKSPACE=str(ROOT),RUNNER_TEMP=str(tmp_path),GITHUB_OUTPUT=str(tmp_path/'out')) + workflow_doc=yaml.load((ROOT/'.github/workflows/noema-review.yml').read_text(),Loader=yaml.BaseLoader) + step_name='Admit existing personal LiteLLM auto route' if entry=='workflow_admit' else 'Prepare Noema model verdict' + step=next(s for s in workflow_doc['jobs']['noema-review']['steps'] if s.get('name')==step_name) + command=['bash','-c',step['run']] + if sys.platform=='darwin': + profile='(version 1)(allow default)(deny network*)(deny file-write*)(allow file-write* (subpath '+json.dumps(str(tmp_path))+'))' + command=['/usr/bin/sandbox-exec','-p',profile,*command] + result=subprocess.run(command,env=environment,cwd=source,capture_output=True,timeout=20) + (tmp_path/'stdout.bin').write_bytes(result.stdout);(tmp_path/'stderr.bin').write_bytes(result.stderr) + record={'entry':entry,'case':case,'exit':result.returncode,'stdout_sha256':hashlib.sha256(result.stdout).hexdigest(), + 'stderr_sha256':hashlib.sha256(result.stderr).hexdigest(),'marker_in_logs':marker.encode() in result.stdout+result.stderr} + state=json.loads(state_file.read_text()) + record['external_calls']=state['external_calls'];record['requests']=len(state['http_requests']);record['publications']=state['publications'] + record['stage_faults']=state['stage_faults'];record['context_skipped']=state.get('context_skipped',False) + record['marker_in_requests']=any(marker in r['data'] for r in state['http_requests']) + if state['http_requests']: + payload=json.loads(state['http_requests'][0]['data']) + assert set(payload)=={'model','messages','response_format'} + assert payload['model']=='auto' + assert payload['messages']==[{'role':'system','content':'Return strict JSON only. Do not include markdown.'}, + {'role':'user','content':'Independently review this exact YAML for correctness and security. Return the declared response schema. Cite changed RIGHT-side lines. For approval falsify at least two distinct concrete regression hypotheses; for request_changes confirm a concrete finding. This is a complete new file; no other repository context is authorized.\nRepository: ContextualWisdomLab/traveling-agent\nPR: 4\nHead: bf85ddde01f871f0e713db54aae5a16883777d9f\nPath: .github/workflows/docker-ephemeral-canary.yml\nname: synthetic\non: workflow_dispatch\n'}] + (tmp_path/'result.json').write_text(json.dumps(record,indent=2)) + if workflow: + assert not any(stage in state['external_calls'] for stage in ('diff','context')) + if case=='success:ok': + assert result.returncode==0 + outputs=(tmp_path/'out').read_text() + assert ('selected=true' if entry=='workflow_admit' else 'prepared=true') in outputs + else: + assert result.returncode!=0 + assert not record['marker_in_logs'], 'Private sentinel reached the actual public CLI output' + assert not record['marker_in_requests'], 'Private sentinel reached the actual HTTP request payload' + assert record['publications']==0 + stage,kind=case.split(':',1) + skipped=entry=='prepare' and state.get('context_skipped',False) and stage in {'context','diff'} + success=stage in {'success','telemetry'} or skipped or (stage=='context' and kind=='runtime') + expected=0 if success else 130 if kind=='interrupt' else 1 + if not workflow: + assert result.returncode==expected + if not success and stage in {'pr','files','content','readback'}: + assert stage in state['external_calls'], 'Failure boundary was not reached' + if stage in {'envelope','fsync','cleanup'}: + assert stage in state['stage_faults'], 'File lifecycle boundary was not reached' + if stage in {'http','enter','read','close','error_read','error_close','verdict','model_decode','telemetry','envelope','fsync'}: + assert record['requests']==1 + if stage in {'pr','files','content','parse_envelope','cleanup'}: + assert record['requests']==0 From 657acdcda8af80bb7ccacaf3d9fe1673e3a64138 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 07:35:30 +0900 Subject: [PATCH 10/11] fix(noema): preserve bounded complete auto input and enroll runtime tests --- .../agent-review-runtime-quality-ci.yml | 6 +++ scripts/ci/noema_review_gate.py | 29 +++++++++++- ...nt_review_runtime_quality_consolidation.py | 11 ++++- tests/test_noema_complete_auto_input.py | 44 +++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/test_noema_complete_auto_input.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index ee5c2df50e..32c1f04ef9 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -76,6 +76,7 @@ on: - "tests/test_personal_review_route.py" - "tests/test_noema_paid_auto_authorization.py" - "tests/test_noema_exact_canary.py" + - "tests/test_noema_complete_auto_input.py" - "tests/test_noema_canary_publication.py" - "tests/fixtures/noema_canary_publication_fixture.py" - "scripts/ci/noema_exact_canary.py" @@ -314,6 +315,7 @@ jobs: tests/test_personal_review_route.py|\ tests/test_noema_paid_auto_authorization.py|\ tests/test_noema_exact_canary.py|\ + tests/test_noema_complete_auto_input.py|\ tests/test_noema_canary_publication.py|\ tests/fixtures/noema_canary_publication_fixture.py|\ scripts/ci/noema_exact_canary.py|\ @@ -519,6 +521,7 @@ jobs: tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ tests/test_noema_exact_canary.py \ + tests/test_noema_complete_auto_input.py \ tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ @@ -555,6 +558,8 @@ jobs: scripts/ci/contextual_orchestrator_review_policy.py \ scripts/ci/contextual_orchestrator_review_launcher.py python -m compileall -q \ + scripts/ci/noema_review_gate.py \ + scripts/ci/noema_exact_canary.py \ tests/fixtures/noema_canary_publication_fixture.py \ scripts/ci/pr_review_conflict_scope.py \ scripts/ci/pr_review_autofix_context.py \ @@ -569,6 +574,7 @@ jobs: tests/test_personal_review_route.py \ tests/test_noema_paid_auto_authorization.py \ tests/test_noema_exact_canary.py \ + tests/test_noema_complete_auto_input.py \ tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index b48f63e59e..3c58900f17 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -66,6 +66,20 @@ # noema_review_handoff.py. NOEMA_REVIEW_BODY_HEAD_RE = re.compile(r"^- Head SHA:\s*`([0-9a-fA-F]{40})`$", re.MULTILINE) MAX_DIFF_CHARS = 60000 +# Personal auto advertises 500,000 input tokens. Bound the complete JSON +# request conservatively by UTF-8 bytes (including schema/envelope), reserving +# 8,192 for chat framing. Do not apply this route's capacity to other models. +MAX_AUTO_REVIEW_REQUEST_BYTES = 500000 - 8192 + + +def complete_auto_input() -> bool: + """Select only the existing personal-auto endpoint, not arbitrary aliases.""" + return ( + os.environ.get("NOEMA_LLM_MODEL", "").strip() == "auto" + and os.environ.get("NOEMA_LLM_API_URL", "").strip() + == "https://litellm.poinnetworks.net/v1/chat/completions" + ) + MAX_CONTEXT_FILES = 12 MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 @@ -553,8 +567,12 @@ def require_distinct_pr_author(pr: dict[str, Any], actor: str) -> str: def fetch_diff(repo: str, number: int) -> tuple[str, bool]: - """Fetch the PR diff and truncate it to the bounded LLM prompt size.""" + """Keep complete personal-auto diffs; retain the legacy other-route bound.""" diff = run(["gh", "api", f"repos/{repo}/pulls/{number}", "-H", "Accept: application/vnd.github.v3.diff"]) + if complete_auto_input(): + if len(diff.encode("utf-8")) > MAX_AUTO_REVIEW_REQUEST_BYTES: + raise RuntimeError("Noema complete review input exceeds personal auto budget") + return diff, False truncated = len(diff) > MAX_DIFF_CHARS if truncated: marker = "[overlong changed line content omitted]" @@ -1638,6 +1656,8 @@ def call_llm( "Noema LLM review unavailable: NOEMA_LLM_API_URL or NOEMA_LLM_API_KEY is not configured." ) reject_private_llm_url(api_url) + if complete_auto_input() and truncated: + raise RuntimeError("Noema complete review input requires an untruncated diff") allowed_locations = [ {"path": path, "line": line, "side": side} @@ -1646,7 +1666,10 @@ def call_llm( location_example = allowed_locations[0] if allowed_locations else { "path": "path", "line": 0, "side": "RIGHT" } - allowed_locations_json = _bounded_allowed_locations_json(allowed_locations) + allowed_locations_json = ( + json.dumps({"locations": allowed_locations, "truncated": False}, separators=(",", ":")) + if complete_auto_input() else _bounded_allowed_locations_json(allowed_locations) + ) prompt = { "role": "user", "content": "\n".join( @@ -1705,6 +1728,8 @@ def call_llm( _required_probe_count(diff, (canary.PATH,)))) else: payload_bytes = json.dumps(payload).encode("utf-8") + if complete_auto_input() and len(payload_bytes) > MAX_AUTO_REVIEW_REQUEST_BYTES: + raise RuntimeError("Noema complete review input exceeds personal auto budget") request = urllib.request.Request( api_url, data=payload_bytes, diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index 05fbd6b903..47a7ff1c19 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -148,6 +148,9 @@ def test_ghas_runtime_changes_start_the_quality_runner(changed_path: str) -> Non @pytest.mark.parametrize( ("changed_path", "expected_suites"), ( + ("tests/test_noema_complete_auto_input.py", {"review_repair"}), + ("scripts/ci/noema_review_gate.py", {"noema", "review_repair"}), + ("scripts/ci/noema_exact_canary.py", {"review_repair"}), ("scripts/ci/codeql_ghas_configuration_identity.py", {"ghas"}), ("tests/test_codeql_ghas_configuration_identity.py", {"ghas"}), ("tests/test_codeql_ghas_analyses_pagination.py", {"ghas"}), @@ -243,7 +246,9 @@ def test_ghas_runtime_step_runs_exact_tests_and_fails_closed( @pytest.mark.parametrize( "changed_path", ("tests/test_noema_canary_publication.py", - "tests/fixtures/noema_canary_publication_fixture.py"), + "tests/fixtures/noema_canary_publication_fixture.py", + "tests/test_noema_complete_auto_input.py", + "scripts/ci/noema_exact_canary.py"), ) def test_publication_regressions_start_and_select_quality( changed_path: str, tmp_path: Path @@ -278,6 +283,7 @@ def test_publication_regressions_run_and_compile( rows = [line.split() for line in calls.read_text().splitlines()] assert rows[0][:3] == ["-m", "pytest", "-q"] assert rows[0].count("tests/test_noema_canary_publication.py") == 1 + assert rows[0].count("tests/test_noema_complete_auto_input.py") == 1 assert "tests/fixtures/noema_canary_publication_fixture.py" not in rows[0] assert result.returncode == pytest_exit assert result.stdout == result.stderr == "" @@ -287,6 +293,9 @@ def test_publication_regressions_run_and_compile( assert len(rows) == 3 assert rows[2][:3] == ["-m", "compileall", "-q"] for path in ("tests/test_noema_canary_publication.py", + "tests/test_noema_complete_auto_input.py", + "scripts/ci/noema_review_gate.py", + "scripts/ci/noema_exact_canary.py", "tests/fixtures/noema_canary_publication_fixture.py"): assert rows[2].count(path) == 1 diff --git a/tests/test_noema_complete_auto_input.py b/tests/test_noema_complete_auto_input.py new file mode 100644 index 0000000000..6d0945d80e --- /dev/null +++ b/tests/test_noema_complete_auto_input.py @@ -0,0 +1,44 @@ +"""Complete source admission through the real Noema serializer (no network).""" +import json +from unittest.mock import patch +import pytest +from scripts.ci import noema_review_gate as gate + + +def test_auto_fetch_preserves_complete_diff(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + diff = 'diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -0,0 +1,2 @@\n+' + 'a'*61000 + '\n+final_source_line\n' + monkeypatch.setattr(gate, 'run', lambda _: diff) + assert gate.fetch_diff('o/r', 1) == (diff, False) + + +def test_real_serializer_rejects_oversized_auto_before_network(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + monkeypatch.setenv('NOEMA_LLM_API_KEY', 'inert-test-value') + monkeypatch.setattr(gate, 'reject_private_llm_url', lambda _: None) + with patch.object(gate.urllib.request, 'build_opener') as opener: + with pytest.raises(RuntimeError, match='complete review input exceeds'): + gate.call_llm('o/r', 1, {'headRefOid':'a'*40}, '+'+'x'*500000, False, 'a'*40) + opener.assert_not_called() + + +@pytest.mark.parametrize('model,url', [('auto','https://example.com/v1/chat/completions'), ('orchestrator/auto','https://litellm.poinnetworks.net/v1/chat/completions')]) +def test_other_routes_keep_existing_bound(monkeypatch, model, url): + monkeypatch.setenv('NOEMA_LLM_MODEL', model) + monkeypatch.setenv('NOEMA_LLM_API_URL', url) + monkeypatch.setattr(gate, 'run', lambda _: 'x'*61000) + text, truncated = gate.fetch_diff('o/r', 1) + assert truncated and len(text) <= gate.MAX_DIFF_CHARS + + +def test_auto_rejects_partial_input(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + monkeypatch.setenv('NOEMA_LLM_API_KEY', 'inert-test-value') + monkeypatch.setattr(gate, 'reject_private_llm_url', lambda _: None) + with patch.object(gate.urllib.request, 'build_opener') as opener: + with pytest.raises(RuntimeError, match='untruncated'): + gate.call_llm('o/r', 1, {}, 'partial', True, 'a'*40) + opener.assert_not_called() From f47ab13d3b54b661dba8cd63999c010a3eefde6c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 26 Oct 2026 09:18:44 +0900 Subject: [PATCH 11/11] test(noema): enroll exact canary CLI fixtures --- .../agent-review-runtime-quality-ci.yml | 6 ++ ...nt_review_runtime_quality_consolidation.py | 79 ++++++++++++++++++- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 32c1f04ef9..463b2aa02b 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -77,6 +77,8 @@ on: - "tests/test_noema_paid_auto_authorization.py" - "tests/test_noema_exact_canary.py" - "tests/test_noema_complete_auto_input.py" + - "tests/fixtures/noema_canary_cli_fixture.py" + - "tests/fixtures/noema_canary_github_fixture.py" - "tests/test_noema_canary_publication.py" - "tests/fixtures/noema_canary_publication_fixture.py" - "scripts/ci/noema_exact_canary.py" @@ -316,6 +318,8 @@ jobs: tests/test_noema_paid_auto_authorization.py|\ tests/test_noema_exact_canary.py|\ tests/test_noema_complete_auto_input.py|\ + tests/fixtures/noema_canary_cli_fixture.py|\ + tests/fixtures/noema_canary_github_fixture.py|\ tests/test_noema_canary_publication.py|\ tests/fixtures/noema_canary_publication_fixture.py|\ scripts/ci/noema_exact_canary.py|\ @@ -560,6 +564,8 @@ jobs: python -m compileall -q \ scripts/ci/noema_review_gate.py \ scripts/ci/noema_exact_canary.py \ + tests/fixtures/noema_canary_cli_fixture.py \ + tests/fixtures/noema_canary_github_fixture.py \ tests/fixtures/noema_canary_publication_fixture.py \ scripts/ci/pr_review_conflict_scope.py \ scripts/ci/pr_review_autofix_context.py \ diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index 47a7ff1c19..a079efc7a8 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -261,6 +261,78 @@ def test_publication_regressions_start_and_select_quality( ) +CANARY_CLI_FIXTURES = ( + "tests/fixtures/noema_canary_cli_fixture.py", + "tests/fixtures/noema_canary_github_fixture.py", +) + + +@pytest.mark.parametrize("changed_path", CANARY_CLI_FIXTURES) +def test_canary_cli_fixture_starts_quality_runner(changed_path: str) -> None: + """A fixture-only PR must start the workflow, not just a sibling test edit.""" + trigger = _workflow_text().split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger + + +@pytest.mark.parametrize("changed_path", CANARY_CLI_FIXTURES) +def test_canary_cli_fixture_selects_review_repair_with_real_git_diff( + changed_path: str, tmp_path: Path +) -> None: + """Run the unchanged selector against a real one-file, two-commit PR.""" + repo = tmp_path / "source" + repo.mkdir() + env = {key: value for key, value in os.environ.items() + if not key.startswith("GIT_")} + env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_SYSTEM=os.devnull, + GIT_TERMINAL_PROMPT="0") + + def git(*args: str) -> str: + """Execute only real Git with fixture-local author and no hooks.""" + return subprocess.check_output( + ["git", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", + "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + *args], cwd=repo, env=env, text=True, + ).strip() + + git("init", "--quiet") + fixture = repo / changed_path + fixture.parent.mkdir(parents=True) + fixture.write_bytes((REPOSITORY_ROOT / changed_path).read_bytes()) + git("add", "--", changed_path) + git("commit", "--quiet", "-m", "base") + base = git("rev-parse", "HEAD") + with fixture.open("a") as stream: + stream.write("\n# Fixture-only change.\n") + git("add", "--", changed_path) + git("commit", "--quiet", "-m", "fixture change") + head = git("rev-parse", "HEAD") + assert git("diff", "--name-only", f"{base}...{head}") == changed_path + step = _workflow_text().split("- name: Select affected contract suites", 1)[1] + script = textwrap.dedent(step.split(" - name:", 1)[0].split(" run: |\n", 1)[1]) + output = tmp_path / "suites" + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=repo, env={**env, "BASE_SHA": base, "HEAD_SHA": head, + "GITHUB_OUTPUT": str(output)}, + text=True, capture_output=True, check=True, + ) + suites = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert suites == {name: str(name == "review_repair").lower() for name in + ("noema", "opencode", "strix", "queue", "review_repair", + "commercial_readiness", "exact_artifact", "ghas")} + assert result.stdout == result.stderr == "" + + +def test_canary_cli_fixture_selects_hashed_review_dependencies() -> None: + """The selected suite must install its actual CLI/YAML test dependencies.""" + step = _workflow_text().split("- name: Install exact review dependencies", 1)[1] + step = step.split(" - name:", 1)[0] + assert "steps.affected_suites.outputs.review_repair == 'true'" in step + assert "--require-hashes" in step + assert "-r requirements-opencode-review-ci-hashes.txt" in step + assert "-r requirements-noema-document-ci-hashes.txt" in step + + @pytest.mark.parametrize("pytest_exit", (0, 42)) def test_publication_regressions_run_and_compile( pytest_exit: int, tmp_path: Path @@ -282,9 +354,12 @@ def test_publication_regressions_run_and_compile( ) rows = [line.split() for line in calls.read_text().splitlines()] assert rows[0][:3] == ["-m", "pytest", "-q"] + assert rows[0].count("tests/test_noema_exact_canary.py") == 1 assert rows[0].count("tests/test_noema_canary_publication.py") == 1 assert rows[0].count("tests/test_noema_complete_auto_input.py") == 1 - assert "tests/fixtures/noema_canary_publication_fixture.py" not in rows[0] + for fixture in (*CANARY_CLI_FIXTURES, + "tests/fixtures/noema_canary_publication_fixture.py"): + assert fixture not in rows[0] assert result.returncode == pytest_exit assert result.stdout == result.stderr == "" if pytest_exit: @@ -293,6 +368,8 @@ def test_publication_regressions_run_and_compile( assert len(rows) == 3 assert rows[2][:3] == ["-m", "compileall", "-q"] for path in ("tests/test_noema_canary_publication.py", + "tests/test_noema_exact_canary.py", + *CANARY_CLI_FIXTURES, "tests/test_noema_complete_auto_input.py", "scripts/ci/noema_review_gate.py", "scripts/ci/noema_exact_canary.py",