diff --git a/.github/actions/noema-review/two_phase.py b/.github/actions/noema-review/two_phase.py index d5da7b8270..9f2e79a23f 100755 --- a/.github/actions/noema-review/two_phase.py +++ b/.github/actions/noema-review/two_phase.py @@ -164,10 +164,16 @@ def prepare_verdict(repo: str, number: int, expected_head: str, path: Path) -> i print("Current head already has a Noema review; verdict preparation skipped.") return 0 - diff, truncated = gate.fetch_diff(repo, number) - changed_files = gate.fetch_changed_files(repo, number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci import noema_exact_canary as canary + canary.require_target(repo, number, expected) + changed_files = gate.fetch_changed_files(repo, number) + diff, truncated, review_context = "", False, "" + else: + diff, truncated = gate.fetch_diff(repo, number) + changed_files = gate.fetch_changed_files(repo, number) + review_context = gate.build_review_context(repo, number, pull_request, changed_files) changed_paths = tuple(file_path for file_path, _status in changed_files) - review_context = gate.build_review_context(repo, number, pull_request, changed_files) try: verdict = gate.call_llm( repo, @@ -325,6 +331,9 @@ def main(argv: list[str]) -> int: if __name__ == "__main__": + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + from scripts.ci.noema_exact_canary import run_private_cli + raise SystemExit(run_private_cli(lambda: main(sys.argv[1:]))) try: raise SystemExit(main(sys.argv[1:])) except RuntimeError as exc: diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 38f617e7d6..463b2aa02b 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -74,6 +74,15 @@ on: - "tests/test_contextual_orchestrator_review_launcher.py" - "tests/test_review_auto_route.py" - "tests/test_personal_review_route.py" + - "tests/test_noema_paid_auto_authorization.py" + - "tests/test_noema_exact_canary.py" + - "tests/test_noema_complete_auto_input.py" + - "tests/fixtures/noema_canary_cli_fixture.py" + - "tests/fixtures/noema_canary_github_fixture.py" + - "tests/test_noema_canary_publication.py" + - "tests/fixtures/noema_canary_publication_fixture.py" + - "scripts/ci/noema_exact_canary.py" + - "scripts/ci/noema_review_gate.py" - "tests/test_tokyo_review_consumer.py" - "tests/test_review_auto_cost_admission.py" - "tests/test_review_ai_app_independence.py" @@ -222,6 +231,7 @@ jobs: review_repair_suite=true ;; .github/workflows/noema-review.yml|\ + scripts/ci/noema_review_gate.py|\ .github/actions/noema-review/two_phase.py|\ tests/test_noema_reviewer_token_lifetime.py|\ tests/test_noema_two_phase_handoff.py|\ @@ -234,6 +244,7 @@ jobs: tests/test_noema_document_review_context.py|\ docs/doctoring/noema-review-token-lifetime.md) noema_suite=true + review_repair_suite=true ;; .github/workflows/opencode-review-dispatch.yml|\ scripts/ci/ensure_rust_llvm19.sh|\ @@ -304,6 +315,14 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py|\ tests/test_review_auto_route.py|\ tests/test_personal_review_route.py|\ + tests/test_noema_paid_auto_authorization.py|\ + tests/test_noema_exact_canary.py|\ + tests/test_noema_complete_auto_input.py|\ + tests/fixtures/noema_canary_cli_fixture.py|\ + tests/fixtures/noema_canary_github_fixture.py|\ + tests/test_noema_canary_publication.py|\ + tests/fixtures/noema_canary_publication_fixture.py|\ + scripts/ci/noema_exact_canary.py|\ tests/test_tokyo_review_consumer.py|\ tests/test_review_auto_cost_admission.py|\ tests/test_review_ai_app_independence.py|\ @@ -504,6 +523,10 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py \ tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ + tests/test_noema_paid_auto_authorization.py \ + tests/test_noema_exact_canary.py \ + tests/test_noema_complete_auto_input.py \ + tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ @@ -539,6 +562,11 @@ jobs: scripts/ci/contextual_orchestrator_review_policy.py \ scripts/ci/contextual_orchestrator_review_launcher.py python -m compileall -q \ + scripts/ci/noema_review_gate.py \ + scripts/ci/noema_exact_canary.py \ + tests/fixtures/noema_canary_cli_fixture.py \ + tests/fixtures/noema_canary_github_fixture.py \ + tests/fixtures/noema_canary_publication_fixture.py \ scripts/ci/pr_review_conflict_scope.py \ scripts/ci/pr_review_autofix_context.py \ tests/test_pr_review_conflict_scope.py \ @@ -550,6 +578,10 @@ jobs: tests/test_contextual_orchestrator_review_launcher.py \ tests/test_review_auto_route.py \ tests/test_personal_review_route.py \ + tests/test_noema_paid_auto_authorization.py \ + tests/test_noema_exact_canary.py \ + tests/test_noema_complete_auto_input.py \ + tests/test_noema_canary_publication.py \ tests/test_tokyo_review_consumer.py \ tests/test_review_auto_cost_admission.py \ tests/test_review_ai_app_independence.py \ diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index cb9d671694..2006254413 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -663,6 +663,7 @@ jobs: NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} TOKEN_EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} NOEMA_REVIEW_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || '' }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }} run: | set -euo pipefail @@ -673,6 +674,15 @@ jobs: repository_name="${TARGET_REPOSITORY#*/}" echo "repository=$repository_name" >>"$GITHUB_OUTPUT" + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = "true" ]; then + if [ -n "${NOEMA_REVIEW_TOKEN:-}" ] || [ -z "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] || [ -z "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then + echo "::error::Non-ZDR canary requires native GitHub App selection; PAT precedence and OIDC fallback are unsupported. No model call is permitted." + exit 1 + fi + echo "source=github-app" >>"$GITHUB_OUTPUT" + exit 0 + fi + if [ "$TARGET_REPOSITORY" = "ContextualWisdomLab/k-csap-skills" ]; then if [ -z "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] || [ -z "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then echo "::error::k-csap-skills Noema reviewer requires the native GitHub App client ID and private key; PAT/OIDC fallback is forbidden." @@ -890,17 +900,32 @@ jobs: if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' env: LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }} + NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }} + NOEMA_EXACT_CANARY_API_KEY: ${{ secrets.NOEMA_EXACT_CANARY_API_KEY }} + GH_TOKEN: ${{ steps.noema_github_app_token.outputs.token }} PERSONAL_REVIEW_ZERO_COST_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZERO_COST_ATTESTED || 'false' }} PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED: ${{ vars.PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED || 'false' }} PERSONAL_REVIEW_ZDR_ATTESTED: ${{ vars.PERSONAL_REVIEW_ZDR_ATTESTED || 'false' }} PRIVATE_TARGET: ${{ steps.target_visibility.outputs.require_zdr }} run: | set -euo pipefail + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'true' ]; then + test -n "${NOEMA_EXACT_CANARY_API_KEY:-}" + python3 -m scripts.ci.noema_exact_canary \ + --repo "$TARGET_REPOSITORY" --pr-number "$PR_NUMBER" \ + --expected-head "$EXPECTED_HEAD_SHA" + echo 'selected=true' >>"$GITHUB_OUTPUT" + echo 'exact_canary=true' >>"$GITHUB_OUTPUT" + exit 0 + fi + test "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'false' if [ -z "${LLM_GATEWAY_API_KEY:-}" ]; then echo '::error::STOP: existing personal review Secret is unavailable.' exit 1 fi python3 -m scripts.ci.personal_review_route \ + --paid-auto-authorized "$NOEMA_REVIEW_PAID_AUTO_AUTHORIZED" \ --zero-cost "$PERSONAL_REVIEW_ZERO_COST_ATTESTED" \ --public-auto-authorized "$PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED" \ --zero-retention "$PERSONAL_REVIEW_ZDR_ATTESTED" \ @@ -1016,6 +1041,8 @@ jobs: NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }} PERSONAL_ROUTE_SELECTED: ${{ steps.personal_route.outputs.selected }} + NOEMA_EXACT_CANARY_NON_ZDR: ${{ steps.personal_route.outputs.exact_canary || 'false' }} + NOEMA_EXACT_CANARY_API_KEY: ${{ secrets.NOEMA_EXACT_CANARY_API_KEY }} NOEMA_GITHUB_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} run: | @@ -1029,6 +1056,10 @@ jobs: echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot prepare a verdict." exit 1 fi + if [ "${NOEMA_EXACT_CANARY_NON_ZDR:-false}" = 'true' ]; then + test -n "${NOEMA_EXACT_CANARY_API_KEY:-}" + LLM_GATEWAY_API_KEY="$NOEMA_EXACT_CANARY_API_KEY" + fi if [ "${PERSONAL_ROUTE_SELECTED:-}" = 'true' ]; then test -n "${LLM_GATEWAY_API_KEY:-}" export NOEMA_LLM_API_URL="https://litellm.poinnetworks.net/v1/chat/completions" @@ -1101,6 +1132,7 @@ jobs: - name: Publish prepared Noema verdict on the exact live head if: env.PR_NUMBER != '' && steps.noema_prepare.outputs.prepared == 'true' env: + NOEMA_EXACT_CANARY_NON_ZDR: ${{ steps.personal_route.outputs.exact_canary || 'false' }} GH_TOKEN: ${{ env.TARGET_REPOSITORY == 'ContextualWisdomLab/k-csap-skills' && steps.noema_github_app_publication_token.outputs.token || (env.TARGET_REPOSITORY != 'ContextualWisdomLab/k-csap-skills' && (steps.noema_credential.outputs.source == 'pat' && secrets.NOEMA_REVIEW_TOKEN || steps.noema_credential.outputs.source == 'github-app' && steps.noema_github_app_publication_token.outputs.token || steps.noema_credential.outputs.source == 'oidc' && steps.noema_oidc_token.outputs.token || '')) || '' }} NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app-refresh' || steps.noema_credential.outputs.source == 'oidc' && 'noema-review-app-oidc' || '' }} NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_publication_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_publication_token.outputs['app-slug']) || '' }} diff --git a/docs/automation/noema-exact-canary-non-zdr.md b/docs/automation/noema-exact-canary-non-zdr.md new file mode 100644 index 0000000000..7b2cc9e647 --- /dev/null +++ b/docs/automation/noema-exact-canary-non-zdr.md @@ -0,0 +1,34 @@ +# Exact non-ZDR canary (default disabled) + +Direct user consent covers ONLY ContextualWisdomLab/traveling-agent PR4, +head bf85ddde01f871f0e713db54aae5a16883777d9f, added file +`.github/workflows/docker-ephemeral-canary.yml`, SHA256 +5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d. + +Consent includes disclosed gateway prompt/response storage without an established +finite deletion period and the external classifier/provider/fallback chain. +It is NOT ZDR, an upstream no-training claim, or organization-wide consent. +Never enable PERSONAL_REVIEW_ZDR_ATTESTED on this basis. + +Trusted variable NOEMA_EXACT_CANARY_NON_ZDR defaults false. Its true mode requires +a separate NOEMA_EXACT_CANARY_API_KEY Secret: no fallback to or overwrite of the +unresolved central Secret. Dedicated issuance/install must have authorized issuer +custody receipts before activation. No dedicated Secret has been installed by +this patch. The existing App publisher and exact-head verification are unchanged. +The canary requires native GitHub App selection. If NOEMA_REVIEW_TOKEN would +preempt the App, or only OIDC is available, credential selection explicitly rejects +before model work; ambient GH_TOKEN is never borrowed. Normal mode retains its +existing credential precedence. Activation must resolve this prerequisite without +silently replacing any existing Secret. + +Workflow changes require two distinct probes under the existing substantive +validator. The emitted schema and fixed approval instruction require two as well; +one-probe responses remain rejected rather than weakening that validator. + +Admission authenticates current head, one added path and exact content digest. +The HTTP producer repeats these checks and serializes only the approved YAML, +fixed instructions/identities and fixed verdict schema; PR title, review threads, +base source and caller diff do not enter the outgoing canary payload. The local +verdict validator receives a reconstructed new-file diff for line validation. +All other targets in this mode fail closed. Normal mode keeps existing private +retention admission unchanged. This patch does not authorize deployment/merge. diff --git a/docs/automation/noema-paid-auto-authorization.md b/docs/automation/noema-paid-auto-authorization.md new file mode 100644 index 0000000000..086aeebc03 --- /dev/null +++ b/docs/automation/noema-paid-auto-authorization.md @@ -0,0 +1,20 @@ +# Noema paid auto cost authorization (source proposal) + +The October 25 CI request authorizes paid inference without a monetary cap on +`https://litellm.poinnetworks.net`, model `auto`. It does not require zero cost. +`NOEMA_REVIEW_PAID_AUTO_AUTHORIZED` is a separate, default-false cost authorization +for the existing Noema route. It does not attest privacy, select another key, +change endpoint/model, or relax private/internal visibility admission. Existing +OpenCode/Strix callers omit the new argument and keep their behavior. + +Private/internal requests still require the existing real retention evidence and +`PERSONAL_REVIEW_ZDR_ATTESTED`. Do not set that value merely because a key has no +expiry, unlimited budget, no configured logging integration, or successful HTTP +responses. The variable's legacy PERSONAL name does not identify the key's owner. + +Operational adoption requires normal independent source review and protected +shipment, purpose-bound existing central Secret custody, then truthful supplier +retention evidence for the gateway and every eligible downstream/fallback of its +literal auto route. No settings or Secrets are changed by this proposal. If the +supplier retains content, obtain an explicit privacy-policy decision instead of +calling it ZDR or silently falling back to another provider. diff --git a/scripts/ci/noema_exact_canary.py b/scripts/ci/noema_exact_canary.py new file mode 100644 index 0000000000..97b818be29 --- /dev/null +++ b/scripts/ci/noema_exact_canary.py @@ -0,0 +1,105 @@ +"""Explicit non-ZDR consent for one immutable YAML, never ZDR attestation. + +Consent includes disclosed gateway storage (no established finite deletion period) +and external classifier/provider/fallback recipients. No other private input is +covered. This module is inert unless the trusted Noema caller selects it. +""" +import hashlib +import json + +REPOSITORY = 'ContextualWisdomLab/traveling-agent' +NUMBER = 4 +HEAD = 'bf85ddde01f871f0e713db54aae5a16883777d9f' +PATH = '.github/workflows/docker-ephemeral-canary.yml' +DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d' +SYSTEM = 'Return strict JSON only. Do not include markdown.' +INSTRUCTION = ('Independently review this exact YAML for correctness and security. ' + 'Return the declared response schema. Cite changed RIGHT-side lines. ' + 'For approval falsify at least two distinct concrete regression hypotheses; ' + 'for request_changes confirm a concrete finding. This is a complete ' + 'new file; no other repository context is authorized.\n' + f'Repository: {REPOSITORY}\nPR: {NUMBER}\nHead: {HEAD}\nPath: {PATH}\n') + + +def require_target(repo, number, head): + """Reject any repository, pull request or head outside exact consent.""" + if repo != REPOSITORY or type(number) is not int or number != NUMBER or head != HEAD: + raise RuntimeError('Non-ZDR canary scope mismatch') + + +def require_content(content): + """Admit only UTF-8 text with the explicitly consented content digest.""" + if type(content) is not str or hashlib.sha256(content.encode('utf-8')).hexdigest() != DIGEST: + raise RuntimeError('Non-ZDR canary content scope mismatch') + + +def messages(content): + """Build fixed instructions plus the digest-admitted file and nothing else.""" + require_content(content) + return [{'role': 'system', 'content': SYSTEM}, + {'role': 'user', 'content': INSTRUCTION + content}] + + +def validate_payload(payload): + """Verify complete actual JSON payload; no title, threads or extra context.""" + if type(payload) is not dict or set(payload) != {'model', 'response_format', 'messages'}: + raise RuntimeError('Non-ZDR canary payload scope mismatch') + from scripts.ci.noema_review_gate import _noema_verdict_response_format, _required_probe_count + if payload['response_format'] != _noema_verdict_response_format(_required_probe_count('', (PATH,))): + raise RuntimeError('Non-ZDR canary schema scope mismatch') + if payload['model'] != 'auto': + raise RuntimeError('Non-ZDR canary model scope mismatch') + rows = payload['messages'] + if type(rows) is not list or len(rows) != 2 or rows[0] != {'role': 'system', 'content': SYSTEM}: + raise RuntimeError('Non-ZDR canary messages scope mismatch') + row = rows[1] + if type(row) is not dict or set(row) != {'role', 'content'} or row['role'] != 'user': + raise RuntimeError('Non-ZDR canary message scope mismatch') + text = row['content'] + if type(text) is not str or not text.startswith(INSTRUCTION): + raise RuntimeError('Non-ZDR canary instruction scope mismatch') + require_content(text[len(INSTRUCTION):]) + + +def serialize_payload(content, response_format): + """Serialize only approved input and fixed caller-owned schema.""" + payload = {'model': 'auto', 'response_format': response_format, 'messages': messages(content)} + raw = json.dumps(payload).encode('utf-8') + validate_payload(json.loads(raw)) + return raw + + +def main(): + """Authenticate exact target content through the existing GitHub reader.""" + import argparse + from scripts.ci import noema_review_gate as gate + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo', required=True) + parser.add_argument('--pr-number', required=True, type=int) + parser.add_argument('--expected-head', required=True) + args = parser.parse_args() + require_target(args.repo, args.pr_number, args.expected_head) + pr = gate.fetch_pr(args.repo, args.pr_number) + gate.require_expected_head(pr, HEAD) + if gate.fetch_changed_files(args.repo, args.pr_number) != [(PATH, 'added')]: + raise RuntimeError('Non-ZDR canary changed-file scope mismatch') + require_content(gate.fetch_file_content_at_ref(args.repo, PATH, HEAD)) + + +def run_private_cli(callback): + """Keep canary CLI failures finite without rendering private exceptions.""" + import sys + try: + return callback() + except BaseException as exc: + if isinstance(exc, SystemExit) and (exc.code is None or type(exc.code) is int): + raise SystemExit(exc.code) from None + if isinstance(exc, KeyboardInterrupt): + print('::error::Non-ZDR canary interrupted; private diagnostics withheld.', file=sys.stderr) + raise SystemExit(130) from None + print('::error::Non-ZDR canary failed closed; private diagnostics withheld.', file=sys.stderr) + raise SystemExit(1) from None + + +if __name__ == '__main__': + run_private_cli(main) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 156f5aafa3..3c58900f17 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -66,6 +66,20 @@ # noema_review_handoff.py. NOEMA_REVIEW_BODY_HEAD_RE = re.compile(r"^- Head SHA:\s*`([0-9a-fA-F]{40})`$", re.MULTILINE) MAX_DIFF_CHARS = 60000 +# Personal auto advertises 500,000 input tokens. Bound the complete JSON +# request conservatively by UTF-8 bytes (including schema/envelope), reserving +# 8,192 for chat framing. Do not apply this route's capacity to other models. +MAX_AUTO_REVIEW_REQUEST_BYTES = 500000 - 8192 + + +def complete_auto_input() -> bool: + """Select only the existing personal-auto endpoint, not arbitrary aliases.""" + return ( + os.environ.get("NOEMA_LLM_MODEL", "").strip() == "auto" + and os.environ.get("NOEMA_LLM_API_URL", "").strip() + == "https://litellm.poinnetworks.net/v1/chat/completions" + ) + MAX_CONTEXT_FILES = 12 MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 @@ -265,6 +279,8 @@ def parse_http_retry_after_seconds(headers: Any) -> int | None: def _stable_failure_diagnostic(exc: BaseException) -> str: """Return actionable trusted diagnostics without reflecting model values.""" + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + return "canary-operation-failed" message = scrub_sensitive_data(str(exc)) or type(exc).__name__ if not isinstance(exc, NoemaModelOutputError): return message @@ -415,7 +431,18 @@ def graphql(query: str, **fields: str | int) -> dict[str, Any]: def fetch_pr(repo: str, number: int) -> dict[str, Any]: """Fetch the pull request data required for Noema review gating.""" owner, name = split_repo(repo) - data = graphql(PR_QUERY, owner=owner, name=name, number=number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci.noema_exact_canary import HEAD, require_target + require_target(repo, number, HEAD) + query = """query($owner:String!,$name:String!,$number:Int!) { + repository(owner:$owner,name:$name) { pullRequest(number:$number) { + number author { login } isDraft state headRefOid baseRefOid + reviews(last:100) { nodes { state body author { login } commit { oid } } } + } } + }""" + else: + query = PR_QUERY + data = graphql(query, owner=owner, name=name, number=number) pr = data.get("data", {}).get("repository", {}).get("pullRequest") if not pr: raise RuntimeError(f"PR #{number} was not found in {repo}") @@ -540,8 +567,12 @@ def require_distinct_pr_author(pr: dict[str, Any], actor: str) -> str: def fetch_diff(repo: str, number: int) -> tuple[str, bool]: - """Fetch the PR diff and truncate it to the bounded LLM prompt size.""" + """Keep complete personal-auto diffs; retain the legacy other-route bound.""" diff = run(["gh", "api", f"repos/{repo}/pulls/{number}", "-H", "Accept: application/vnd.github.v3.diff"]) + if complete_auto_input(): + if len(diff.encode("utf-8")) > MAX_AUTO_REVIEW_REQUEST_BYTES: + raise RuntimeError("Noema complete review input exceeds personal auto budget") + return diff, False truncated = len(diff) > MAX_DIFF_CHARS if truncated: marker = "[overlong changed line content omitted]" @@ -1625,6 +1656,8 @@ def call_llm( "Noema LLM review unavailable: NOEMA_LLM_API_URL or NOEMA_LLM_API_KEY is not configured." ) reject_private_llm_url(api_url) + if complete_auto_input() and truncated: + raise RuntimeError("Noema complete review input requires an untruncated diff") allowed_locations = [ {"path": path, "line": line, "side": side} @@ -1633,7 +1666,10 @@ def call_llm( location_example = allowed_locations[0] if allowed_locations else { "path": "path", "line": 0, "side": "RIGHT" } - allowed_locations_json = _bounded_allowed_locations_json(allowed_locations) + allowed_locations_json = ( + json.dumps({"locations": allowed_locations, "truncated": False}, separators=(",", ":")) + if complete_auto_input() else _bounded_allowed_locations_json(allowed_locations) + ) prompt = { "role": "user", "content": "\n".join( @@ -1668,9 +1704,35 @@ def call_llm( prompt, ], } + exact_canary = os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") + if exact_canary not in {"true", "false"}: + raise RuntimeError("Non-ZDR canary mode is invalid") + if exact_canary == "true": + from scripts.ci import noema_exact_canary as canary + canary.require_target(repo, number, expected_head) + if pr.get("headRefOid") != expected_head or truncated: + raise RuntimeError("Non-ZDR canary head scope mismatch") + if api_url != 'https://litellm.poinnetworks.net/v1/chat/completions' or model != 'auto': + raise RuntimeError("Non-ZDR canary route scope mismatch") + if tuple(changed_paths) != (canary.PATH,) or fetch_changed_files(repo, number) != [(canary.PATH, 'added')]: + raise RuntimeError("Non-ZDR canary changed-file scope mismatch") + content = fetch_file_content_at_ref(repo, canary.PATH, expected_head) + canary.require_content(content) + # Reconstruct the complete new-file diff from admitted bytes. Never send + # caller diff, title, threads, base source or other repository context. + diff = (f'diff --git a/{canary.PATH} b/{canary.PATH}\n' + f'new file mode 100644\n--- /dev/null\n+++ b/{canary.PATH}\n' + f'@@ -0,0 +1,{len(content.splitlines())} @@\n' + + ''.join('+' + line for line in content.splitlines(keepends=True))) + payload_bytes = canary.serialize_payload(content, _noema_verdict_response_format( + _required_probe_count(diff, (canary.PATH,)))) + else: + payload_bytes = json.dumps(payload).encode("utf-8") + if complete_auto_input() and len(payload_bytes) > MAX_AUTO_REVIEW_REQUEST_BYTES: + raise RuntimeError("Noema complete review input exceeds personal auto budget") request = urllib.request.Request( api_url, - data=json.dumps(payload).encode("utf-8"), + data=payload_bytes, headers={ "authorization": f"Bearer {api_key}", "content-type": "application/json", @@ -1741,6 +1803,13 @@ def call_llm( served_model = model_value if isinstance(model_value, str) else None finally: exc.close() + if exact_canary == "true": + served_model = None + gateway_telemetry = { + key: value for key, value in gateway_telemetry.items() + if key in {"provider_attempt_count", "attempt_number", "upstream_status"} + and type(value) is int + } elapsed = time.monotonic() - attempt_started current_failure = _stable_failure_diagnostic(exc) model_note = served_model or "unknown" @@ -1788,6 +1857,8 @@ def call_llm( f"Noema review failed closed: {current_failure}{suffix}" ) from exc elapsed = time.monotonic() - attempt_started + if exact_canary == "true": + served_model = None print( f"::notice::Noema gateway attempt outcome=success phase={active_phase} " f"duration={elapsed:.1f}s served_model={served_model or 'unknown'}; " @@ -1907,10 +1978,16 @@ def inspect_and_review(repo: str, number: int, expected_head: str) -> int: if existing_noema_review(pr, actor): print("Current head already has a Noema review; nothing to do.") return 0 - diff, truncated = fetch_diff(repo, number) - changed_files = fetch_changed_files(repo, number) + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") == "true": + from scripts.ci.noema_exact_canary import require_target + require_target(repo, number, expected_head) + changed_files = fetch_changed_files(repo, number) + diff, truncated, review_context = "", False, "" + else: + diff, truncated = fetch_diff(repo, number) + changed_files = fetch_changed_files(repo, number) + review_context = build_review_context(repo, number, pr, changed_files) changed_paths = tuple(path for path, _status in changed_files) - review_context = build_review_context(repo, number, pr, changed_files) verdict = call_llm(repo, number, pr, diff, truncated, expected_head, review_context, changed_paths) current_pr = fetch_pr(repo, number) try: @@ -1945,6 +2022,9 @@ def main(argv: list[str]) -> int: if __name__ == "__main__": # pragma: no cover + if os.environ.get("NOEMA_EXACT_CANARY_NON_ZDR", "false") != "false": + from scripts.ci.noema_exact_canary import run_private_cli + raise SystemExit(run_private_cli(lambda: main(sys.argv[1:]))) try: raise SystemExit(main(sys.argv[1:])) except RuntimeError as exc: diff --git a/scripts/ci/personal_review_route.py b/scripts/ci/personal_review_route.py index 01c53bc505..d48f279f34 100644 --- a/scripts/ci/personal_review_route.py +++ b/scripts/ci/personal_review_route.py @@ -10,10 +10,11 @@ def require_admission(*, zero_cost: bool, zero_retention: bool, private_target: bool, - public_auto_authorized: bool = False) -> None: - """Admit owner-authorized public routing; private supplier gates stay intact.""" + public_auto_authorized: bool = False, + paid_auto_authorized: bool = False) -> None: + """Admit explicit cost authority without relaxing private retention.""" authorized_public = private_target is False and public_auto_authorized is True - if zero_cost is not True and not authorized_public: + if zero_cost is not True and not authorized_public and paid_auto_authorized is not True: raise RuntimeError('STOP: personal auto zero-cost supplier is not attested') if private_target and zero_retention is not True: raise RuntimeError('STOP: private personal auto zero-retention supplier is not attested') @@ -38,6 +39,7 @@ def configure_opencode(path: Path) -> None: def main(argv: list[str] | None = None) -> int: """Validate trusted nonsecret admission inputs and optionally reset config.""" parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--paid-auto-authorized', choices=('true', 'false'), default='false') parser.add_argument('--zero-cost', choices=('true', 'false'), required=True) parser.add_argument('--zero-retention', choices=('true', 'false'), required=True) parser.add_argument('--private-target', choices=('true', 'false'), required=True) @@ -47,7 +49,8 @@ def main(argv: list[str] | None = None) -> int: require_admission(zero_cost=args.zero_cost == 'true', zero_retention=args.zero_retention == 'true', private_target=args.private_target == 'true', - public_auto_authorized=args.public_auto_authorized == 'true') + public_auto_authorized=args.public_auto_authorized == 'true', + paid_auto_authorized=args.paid_auto_authorized == 'true') if args.opencode_config is not None: configure_opencode(args.opencode_config) return 0 diff --git a/tests/fixtures/noema_canary_cli_fixture.py b/tests/fixtures/noema_canary_cli_fixture.py new file mode 100644 index 0000000000..af5575d306 --- /dev/null +++ b/tests/fixtures/noema_canary_cli_fixture.py @@ -0,0 +1,153 @@ +"""Finite offline CLI fixture; importing it performs no IO.""" + + +def main(): + """Run one private synthetic CLI scenario.""" + """Execute unchanged Noema public CLI bodies with finite external IO.""" + import base64, hashlib, importlib.util, json, os, pathlib, runpy, socket, subprocess, sys, urllib.error, urllib.request + root=pathlib.Path(os.environ['FIXTURE_SOURCE']) + fixture=json.loads(pathlib.Path(os.environ['FIXTURE_INPUT']).read_text()) + state_path=pathlib.Path(os.environ['FIXTURE_STATE']) + cli_argv=sys.argv[1:] + sys.path.insert(0,str(root)) + from scripts.ci import noema_review_gate as gate, noema_exact_canary as policy + marker=fixture['marker']; case=fixture['case']; entry=fixture['entry'] + content='name: synthetic\non: workflow_dispatch\n' + policy.DIGEST=hashlib.sha256(content.encode()).hexdigest() + pr={'number':4,'state':'OPEN','isDraft':False,'headRefOid':policy.HEAD,'baseRefOid':'b'*40, + 'title':marker,'body':marker,'author':{'login':'synthetic-author'}, + 'reviews':{'nodes':[]},'reviewThreads':{'nodes':[{'path':marker,'line':1,'comments':{'nodes':[{'body':marker,'author':{'login':'synthetic'}}]}}]}} + state={'external_calls':[],'http_requests':[],'publications':0,'content_reads':0,'pr_reads':0,'fixture_digest_substitution':True,'stage_faults':[]} + REAL_OPEN=os.open; REAL_FSYNC=os.fsync; REAL_UNLINK=pathlib.Path.unlink + def faulted_open(path,*args,**kwargs): + if str(path).endswith('.envelope') and boundary=='envelope': + state['stage_faults'].append('envelope');save();fail(kind) + return REAL_OPEN(path,*args,**kwargs) + def faulted_fsync(fd): + if boundary=='fsync': + state['stage_faults'].append('fsync');save();fail(kind) + return REAL_FSYNC(fd) + def faulted_unlink(path,*args,**kwargs): + if str(path).endswith('.envelope') and boundary=='cleanup': + state['stage_faults'].append('cleanup');save();fail(kind) + return REAL_UNLINK(path,*args,**kwargs) + os.open=faulted_open;os.fsync=faulted_fsync;pathlib.Path.unlink=faulted_unlink + def save(): state_path.write_text(json.dumps(state,indent=2)) + def fail(kind): + if kind=='nonzero':return subprocess.CompletedProcess(['gh'],1,marker,marker) + if kind=='malformed':return subprocess.CompletedProcess(['gh'],0,marker,'') + if kind=='decode':raise UnicodeDecodeError('utf-8',marker.encode()+b'\xff',len(marker),len(marker)+1,marker) + cls={'runtime':RuntimeError,'os':OSError,'value':ValueError,'interrupt':KeyboardInterrupt,'exit':SystemExit,'type':TypeError}[kind] + error=cls(marker);error.add_note(marker) + raise error + boundary,_,kind=case.partition(':') + def fake_run(args,**kwargs): + if args[:3]==['gh','api','graphql']: + state['pr_reads']+=1 + stage='readback' if boundary=='readback' and state['pr_reads']>1 else 'pr' + state['context_skipped']='reviewThreads' not in kwargs.get('input','') + elif args[:2]==['gh','api'] and '/files' in args[2]:stage='files' + elif args[:2]==['gh','api'] and '/contents/' in args[2]: + state['content_reads']+=1 + stage='context' if state['content_reads']==1 and entry!='policy' and not state.get('context_skipped',False) else 'content' + if state.get('context_skipped',False):stage='content' + elif args[:2]==['gh','api'] and 'Accept: application/vnd.github.v3.diff' in args:stage='diff' + else:raise AssertionError('Unexpected external operation') + state['external_calls'].append(stage);save() + if boundary==stage: + result=fail(kind) + if result is not None:return result + if stage in {'pr','readback'}: value=json.dumps({'data':{'repository':{'pullRequest':pr}}}) + elif stage=='files': value=json.dumps([policy.PATH,'added'])+'\n' + elif stage=='diff': value='diff --git a/'+policy.PATH+' b/'+policy.PATH+'\n--- /dev/null\n+++ b/'+policy.PATH+'\n@@ -0,0 +1,1 @@\n+'+marker+'\n' + else: + data=content.encode() + value=json.dumps({'encoding':'base64','size':len(data),'content':base64.b64encode(data).decode()}) + return subprocess.CompletedProcess(args,0,value,'') + native=fixture.get('native',False) + if not native: + subprocess.run=fake_run + socket.getaddrinfo=lambda host,*args,**kwargs:[(socket.AF_INET,socket.SOCK_STREAM,6,'',('93.184.216.34',443))] + def verdict(): + decision=marker if boundary=='verdict' else 'approve' + return {'decision':decision,'summary':'Synthetic exact-file review.','findings':[], + 'reviewed_lines':[{'path':policy.PATH,'line':1,'side':'RIGHT','analysis':'Reviewed the added workflow.'}], + 'adversarial_validation':{'status':'passed','residual_risk':'Offline fixture only.','probes':[ + {'path':policy.PATH,'line':i+1,'side':'RIGHT','hypothesis':f'Concrete regression {i}', + 'attack_or_counterexample':f'Concrete counterexample {i}','evidence':f'Exact line {i+1}','outcome':'falsified'} for i in range(2)]}} + class Response: + def __enter__(self): + if boundary=='enter':fail(kind) + return self + def __exit__(self,*args): + if boundary=='close':fail(kind) + return False + def read(self): + if boundary=='read':fail(kind) + if boundary=='model_decode':return marker.encode()+b'\xff' + model=marker if boundary=='telemetry' else 'auto' + return json.dumps({'model':model,'choices':[{'message':{'content':json.dumps(verdict())}}]}).encode() + class ErrorBody: + def read(self,*args): + if boundary=='error_read':fail(kind) + return json.dumps({'error':{'detail':{'model':marker,'terminal_reason':marker,'attempts':[{'provider_name':marker,'phase':marker,'attempt_number':1,'provider_status':503}]}}}).encode() + def close(self): + if boundary=='error_close':fail(kind) + class Opener: + def open(self,request): + if native:state.update(json.loads(state_path.read_text())) + state['http_requests'].append({'url':request.full_url,'data':request.data.decode(),'method':request.method});save() + if boundary=='http': + if kind=='url':raise urllib.error.URLError(marker) + if kind=='http':raise urllib.error.HTTPError(request.full_url,503,marker,{},ErrorBody()) + fail(kind) + if boundary in {'error_read','error_close'}:raise urllib.error.HTTPError(request.full_url,503,marker,{},ErrorBody()) + return Response() + urllib.request.build_opener=lambda *args,**kwargs:Opener() + def no_publish(*args,**kwargs):state['publications']+=1;save();raise AssertionError('Publication forbidden') + gate.submit_review=no_publish + # Defense in depth: no native process/socket operation may escape adapters. + def audit(event,args): + if event.startswith('socket.') or event in {'os.system','os.posix_spawn'}: + raise AssertionError('Offline IO fence') + if event=='subprocess.Popen': + if not native or pathlib.Path(args[0]).name!='gh':raise AssertionError('Offline process fence') + sys.addaudithook(audit) + save() + args=['--repo',policy.REPOSITORY,'--pr-number','4','--expected-head',policy.HEAD] + if entry=='prepare': + invocation=[str(root/'.github/actions/noema-review/two_phase.py'),*args,'--prepare-verdict-file',str(state_path.with_suffix('.envelope'))] + if cli_argv: + assert cli_argv[0]==invocation[0] + invocation=cli_argv + sys.argv=invocation + runpy.run_path(sys.argv[0],run_name='__main__') + elif entry=='gate': + sys.argv=[str(root/'scripts/ci/noema_review_gate.py'),*args] + runpy.run_path(sys.argv[0],run_name='__main__') + elif entry=='policy': + sys.argv=[str(root/'scripts/ci/noema_exact_canary.py'),*args] + if cli_argv: + assert cli_argv[:2]==['-m','scripts.ci.noema_exact_canary'] + sys.argv=[sys.argv[0],*cli_argv[2:]] + # Compile the complete actual CLI; only the explicit fixture digest literal + # is substituted, matching the imported policy fixture used by prepare. + text=pathlib.Path(sys.argv[0]).read_text() + old="DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d'" + assert text.count(old)==1 + text=text.replace(old,'DIGEST = '+repr(policy.DIGEST)) + exec(compile(text,sys.argv[0],'exec'),{'__name__':'__main__','__file__':sys.argv[0]}) + elif entry=='publish': + envelope=state_path.with_suffix('.envelope') + payload={'schema_version':1,'repository':policy.REPOSITORY,'pull_request_number':4,'expected_head':policy.HEAD, + 'expected_base':'b'*40,'reviewer_actor':'cwl-noema-review[bot]','pull_request_author':'synthetic-author','verdict':verdict()} + envelope.write_text(json.dumps(payload));envelope.chmod(0o600) + if boundary=='parse_envelope':envelope.write_bytes(marker.encode()+b'\xff') + if boundary=='cleanup':pr['isDraft']=True + sys.argv=[str(root/'.github/actions/noema-review/two_phase.py'),*args,'--publish-verdict-file',str(envelope)] + runpy.run_path(sys.argv[0],run_name='__main__') + else:raise AssertionError('Unknown fixture entry') + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/noema_canary_github_fixture.py b/tests/fixtures/noema_canary_github_fixture.py new file mode 100644 index 0000000000..714a6cbd40 --- /dev/null +++ b/tests/fixtures/noema_canary_github_fixture.py @@ -0,0 +1,39 @@ +"""Inert native gh fixture emitting synthetic API bytes offline.""" + +def main(): + """Answer one finite native process acquisition or fail on selected stage.""" + import base64 + import json + import os + import pathlib + import sys + fixture=json.loads(pathlib.Path(os.environ['FIXTURE_INPUT']).read_text()) + state_file=pathlib.Path(os.environ['FIXTURE_STATE']) + state=json.loads(state_file.read_text()) + marker=fixture['marker'];case=fixture['case'];args=sys.argv[1:] + if args[:2]==['api','graphql']: + stage='pr';query=sys.stdin.read();state['context_skipped']='reviewThreads' not in query + elif args[:1]==['api'] and '/files' in args[1]:stage='files' + elif args[:1]==['api'] and '/contents/' in args[1]: + state['content_reads']+=1 + stage='content' if state['context_skipped'] else 'context' if state['content_reads']==1 else 'content' + elif args[:1]==['api'] and 'Accept: application/vnd.github.v3.diff' in args:stage='diff' + else:raise SystemExit(2) + state['external_calls'].append(stage);state_file.write_text(json.dumps(state)) + if case==stage+':nonzero': + print(marker,file=sys.stderr) + raise SystemExit(1) + path='.github/workflows/docker-ephemeral-canary.yml';head='bf85ddde01f871f0e713db54aae5a16883777d9f' + pr={'number':4,'state':'OPEN','isDraft':False,'headRefOid':head,'baseRefOid':'b'*40,'title':marker,'body':marker, + 'author':{'login':'synthetic-author'},'reviews':{'nodes':[]},'reviewThreads':{'nodes':[{'path':marker,'comments':{'nodes':[{'body':marker,'author':{'login':'synthetic'}}]}}]}} + if stage=='pr':value=json.dumps({'data':{'repository':{'pullRequest':pr}}}) + elif stage=='files':value=json.dumps([path,'added']) + elif stage=='diff':value='diff --git a/'+path+' b/'+path+'\n--- /dev/null\n+++ b/'+path+'\n@@ -0,0 +1,1 @@\n+'+marker+'\n' + else: + data=b'name: synthetic\non: workflow_dispatch\n' + value=json.dumps({'encoding':'base64','size':len(data),'content':base64.b64encode(data).decode()}) + print(value) + + +if __name__=='__main__': + main() diff --git a/tests/fixtures/noema_canary_publication_fixture.py b/tests/fixtures/noema_canary_publication_fixture.py new file mode 100644 index 0000000000..e5deb299b6 --- /dev/null +++ b/tests/fixtures/noema_canary_publication_fixture.py @@ -0,0 +1,197 @@ +"""Offline external-IO adapters for the actual workflow-to-CLI composition.""" + + +def main(): + """Run a full trusted CLI with only synthetic gh, HTTP and IO faults.""" + import base64 + import hashlib + import json + import os + from pathlib import Path + import runpy + import socket + import subprocess + import sys + import urllib.error + import urllib.request + + root = Path(os.environ['FIXTURE_SOURCE']) + settings = json.loads(Path(os.environ['FIXTURE_INPUT']).read_text()) + state_path = Path(os.environ['FIXTURE_STATE']) + sys.path.insert(0, str(root)) + from scripts.ci import noema_exact_canary as policy, noema_review_gate as gate + + phase, case, marker = settings['phase'], settings['case'], settings['marker'] + content = 'name: synthetic\non: workflow_dispatch\n' + # The only policy substitution is explicitly synthetic content provenance. + policy.DIGEST = hashlib.sha256(content.encode()).hexdigest() + state = {'phase': phase, 'case': case, 'external_calls': [], 'queries': [], + 'requests': [], 'posts': [], 'faults': [], + 'canary_mode': os.environ.get('NOEMA_EXACT_CANARY_NON_ZDR'), + 'argv': sys.argv[1:], 'fixture_digest_substitution': True} + + def save(): + """Persist external observations outside public CLI streams.""" + state_path.write_text(json.dumps(state, indent=2)) + + def fault(): + """Raise private diagnostics only at the selected external boundary.""" + error = OSError(marker) + error.add_note(marker) + raise error + + def verdict(): + """Return substantive, line-cited evidence admitted by the real gate.""" + return {'decision': 'approve', 'summary': 'Synthetic exact-file review.', + 'findings': [], 'reviewed_lines': [ + {'path': policy.PATH, 'line': 1, 'side': 'RIGHT', + 'analysis': 'Reviewed the new workflow.'}], + 'adversarial_validation': {'status': 'passed', + 'residual_risk': 'Offline fixture only.', 'probes': [ + {'path': policy.PATH, 'line': i + 1, 'side': 'RIGHT', + 'hypothesis': f'Concrete regression {i}', + 'attack_or_counterexample': f'Concrete counterexample {i}', + 'evidence': f'Exact line {i + 1}', 'outcome': 'falsified'} + for i in range(2)]}} + + def fake_run(args, **kwargs): + """Replace only external gh IO, retaining all real gate consumers.""" + state['external_calls'].append(list(args)) + save() + if args[:3] == ['gh', 'api', 'graphql']: + query = kwargs.get('input', '') + state['queries'].append(query) + save() + if phase == 'publish' and case == 'pr_nonzero': + state['faults'].append('gh_stderr') + save() + return subprocess.CompletedProcess(args, 1, '', marker) + pr = {'number': policy.NUMBER, 'state': 'OPEN', 'isDraft': False, + 'headRefOid': policy.HEAD, 'baseRefOid': 'b' * 40, + 'title': marker, 'body': marker, + 'author': {'login': 'synthetic-author'}, + 'reviews': {'nodes': []}, 'reviewThreads': {'nodes': []}} + if phase == 'publish': + if case in {'cleanup_os', 'draft'}: + pr['isDraft'] = True + elif case == 'head_drift': + pr['headRefOid'] = 'c' * 40 + elif case == 'base_drift': + pr['baseRefOid'] = 'c' * 40 + elif case == 'author_drift': + pr['author']['login'] = 'different-author' + elif case == 'self_author': + pr['author']['login'] = 'cwl-noema-review[bot]' + value = json.dumps({'data': {'repository': {'pullRequest': pr}}}) + elif args[:4] == ['gh', 'api', '-X', 'POST']: + state['posts'].append(json.loads(kwargs['input'])) + save() + value = '{}' + elif args[:2] == ['gh', 'api'] and '/files' in args[2]: + value = json.dumps([policy.PATH, 'added']) + '\n' + elif args[:2] == ['gh', 'api'] and '/contents/' in args[2]: + data = content.encode() + value = json.dumps({'encoding': 'base64', 'size': len(data), + 'content': base64.b64encode(data).decode()}) + elif args[:2] == ['gh', 'api'] and 'Accept: application/vnd.github.v3.diff' in args: + value = (f'diff --git a/{policy.PATH} b/{policy.PATH}\n' + f'--- /dev/null\n+++ b/{policy.PATH}\n' + '@@ -0,0 +1,2 @@\n+name: synthetic\n+on: workflow_dispatch\n') + else: + raise AssertionError('Unexpected external operation') + return subprocess.CompletedProcess(args, 0, value, '') + + subprocess.run = fake_run + socket.getaddrinfo = lambda *_a, **_kw: [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('93.184.216.34', 443))] + + class Response: + """Synthetic successful model reply with untrusted model metadata.""" + def __enter__(self): + """Enter the actual HTTP consumer's context.""" + return self + + def __exit__(self, *_args): + """Preserve normal response settlement.""" + return False + + def read(self): + """Return valid substantive evidence and a private metadata marker.""" + return json.dumps({'model': marker, 'choices': [ + {'message': {'content': json.dumps(verdict())}}]}).encode() + + class ErrorBody: + """Synthetic capacity failure with private provider metadata.""" + def read(self, *_args): + """Return typed and free-text telemetry through the real parser.""" + return json.dumps({'error': {'detail': {'model': marker, + 'terminal_reason': marker, 'attempts': [{'provider_name': marker, + 'phase': marker, 'attempt_number': 1, + 'provider_status': 503}]}}}).encode() + + def close(self): + """Settle the inert error body.""" + return None + + class Opener: + """Record real Request bytes without network dispatch.""" + def open(self, request): + """Serve one finite synthetic HTTP outcome.""" + state['requests'].append({'url': request.full_url, + 'data': request.data.decode()}) + save() + if case == 'prepare_http': + raise urllib.error.HTTPError(request.full_url, 503, marker, {}, ErrorBody()) + return Response() + + urllib.request.build_opener = lambda *_a, **_kw: Opener() + real_unlink, real_fsync = Path.unlink, os.fsync + + def unlink(path, *args, **kwargs): + """Inject a reached cleanup fault without replacing its caller.""" + if phase == 'publish' and case == 'cleanup_os' and path.name == 'noema-verdict-envelope.json': + state['faults'].append('cleanup_os') + save() + fault() + return real_unlink(path, *args, **kwargs) + + def fsync(fd): + """Inject preparation persistence failure through a real file owner.""" + if phase == 'prepare' and case == 'prepare_fsync_os': + state['faults'].append('prepare_fsync_os') + save() + fault() + return real_fsync(fd) + + Path.unlink, os.fsync = unlink, fsync + + def audit(event, _args): + """Forbid every unadapted process or socket operation.""" + if event.startswith('socket.') or event in {'subprocess.Popen', 'os.system', 'os.posix_spawn'}: + raise AssertionError('Offline fixture IO fence') + + sys.addaudithook(audit) + save() + argv = sys.argv[1:] + if argv[:1] == ['-m']: + module = argv[1] + sys.argv = [module, *argv[2:]] + if module == 'scripts.ci.noema_exact_canary': + # Execute its complete real CLI, substituting only the fixture digest. + path = root / 'scripts/ci/noema_exact_canary.py' + text = path.read_text() + old = "DIGEST = '5427ca0c17d818d49f9af91b5cd495ab4599c969b894cb2467b794d674a96d3d'" + assert text.count(old) == 1 + text = text.replace(old, 'DIGEST = ' + repr(policy.DIGEST)) + exec(compile(text, str(path), 'exec'), {'__name__': '__main__', '__file__': str(path)}) + else: + assert module == 'scripts.ci.personal_review_route' + runpy.run_module(module, run_name='__main__') + else: + assert argv[0] == str(root / '.github/actions/noema-review/two_phase.py') + sys.argv = argv + runpy.run_path(argv[0], run_name='__main__') + + +if __name__ == '__main__': + main() diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index 4fecc93656..a079efc7a8 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -148,6 +148,9 @@ def test_ghas_runtime_changes_start_the_quality_runner(changed_path: str) -> Non @pytest.mark.parametrize( ("changed_path", "expected_suites"), ( + ("tests/test_noema_complete_auto_input.py", {"review_repair"}), + ("scripts/ci/noema_review_gate.py", {"noema", "review_repair"}), + ("scripts/ci/noema_exact_canary.py", {"review_repair"}), ("scripts/ci/codeql_ghas_configuration_identity.py", {"ghas"}), ("tests/test_codeql_ghas_configuration_identity.py", {"ghas"}), ("tests/test_codeql_ghas_analyses_pagination.py", {"ghas"}), @@ -240,6 +243,140 @@ def test_ghas_runtime_step_runs_exact_tests_and_fails_closed( assert result.stdout == result.stderr == "" +@pytest.mark.parametrize( + "changed_path", + ("tests/test_noema_canary_publication.py", + "tests/fixtures/noema_canary_publication_fixture.py", + "tests/test_noema_complete_auto_input.py", + "scripts/ci/noema_exact_canary.py"), +) +def test_publication_regressions_start_and_select_quality( + changed_path: str, tmp_path: Path +) -> None: + """Enroll each publication input in the trigger and actual selector.""" + trigger = _workflow_text().split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger + test_ghas_runtime_selector_executes_the_actual_shell( + changed_path, {"review_repair"}, tmp_path + ) + + +CANARY_CLI_FIXTURES = ( + "tests/fixtures/noema_canary_cli_fixture.py", + "tests/fixtures/noema_canary_github_fixture.py", +) + + +@pytest.mark.parametrize("changed_path", CANARY_CLI_FIXTURES) +def test_canary_cli_fixture_starts_quality_runner(changed_path: str) -> None: + """A fixture-only PR must start the workflow, not just a sibling test edit.""" + trigger = _workflow_text().split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger + + +@pytest.mark.parametrize("changed_path", CANARY_CLI_FIXTURES) +def test_canary_cli_fixture_selects_review_repair_with_real_git_diff( + changed_path: str, tmp_path: Path +) -> None: + """Run the unchanged selector against a real one-file, two-commit PR.""" + repo = tmp_path / "source" + repo.mkdir() + env = {key: value for key, value in os.environ.items() + if not key.startswith("GIT_")} + env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_SYSTEM=os.devnull, + GIT_TERMINAL_PROMPT="0") + + def git(*args: str) -> str: + """Execute only real Git with fixture-local author and no hooks.""" + return subprocess.check_output( + ["git", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", + "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + *args], cwd=repo, env=env, text=True, + ).strip() + + git("init", "--quiet") + fixture = repo / changed_path + fixture.parent.mkdir(parents=True) + fixture.write_bytes((REPOSITORY_ROOT / changed_path).read_bytes()) + git("add", "--", changed_path) + git("commit", "--quiet", "-m", "base") + base = git("rev-parse", "HEAD") + with fixture.open("a") as stream: + stream.write("\n# Fixture-only change.\n") + git("add", "--", changed_path) + git("commit", "--quiet", "-m", "fixture change") + head = git("rev-parse", "HEAD") + assert git("diff", "--name-only", f"{base}...{head}") == changed_path + step = _workflow_text().split("- name: Select affected contract suites", 1)[1] + script = textwrap.dedent(step.split(" - name:", 1)[0].split(" run: |\n", 1)[1]) + output = tmp_path / "suites" + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=repo, env={**env, "BASE_SHA": base, "HEAD_SHA": head, + "GITHUB_OUTPUT": str(output)}, + text=True, capture_output=True, check=True, + ) + suites = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert suites == {name: str(name == "review_repair").lower() for name in + ("noema", "opencode", "strix", "queue", "review_repair", + "commercial_readiness", "exact_artifact", "ghas")} + assert result.stdout == result.stderr == "" + + +def test_canary_cli_fixture_selects_hashed_review_dependencies() -> None: + """The selected suite must install its actual CLI/YAML test dependencies.""" + step = _workflow_text().split("- name: Install exact review dependencies", 1)[1] + step = step.split(" - name:", 1)[0] + assert "steps.affected_suites.outputs.review_repair == 'true'" in step + assert "--require-hashes" in step + assert "-r requirements-opencode-review-ci-hashes.txt" in step + assert "-r requirements-noema-document-ci-hashes.txt" in step + + +@pytest.mark.parametrize("pytest_exit", (0, 42)) +def test_publication_regressions_run_and_compile( + pytest_exit: int, tmp_path: Path +) -> None: + """Observe real step argv and prevent compile after failed pytest.""" + step = _workflow_text().split( + "- name: Verify scheduler and contextual-orchestrator review-repair contracts", 1 + )[1].split(" - name:", 1)[0] + assert "if: steps.affected_suites.outputs.review_repair == 'true'" in step + script = textwrap.dedent(step.split(" run: |\n", 1)[1]) + calls = tmp_path / "calls.txt" + recorder = ('python() { printf "%s\\n" "$*" >> "$CALLS"; ' + 'if [[ "$*" == *pytest* ]]; then return "$PYTEST_EXIT"; fi; }\n') + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", + recorder + script], cwd=REPOSITORY_ROOT, + env={**os.environ, "CALLS": str(calls), "PYTEST_EXIT": str(pytest_exit)}, + text=True, capture_output=True, check=False, + ) + rows = [line.split() for line in calls.read_text().splitlines()] + assert rows[0][:3] == ["-m", "pytest", "-q"] + assert rows[0].count("tests/test_noema_exact_canary.py") == 1 + assert rows[0].count("tests/test_noema_canary_publication.py") == 1 + assert rows[0].count("tests/test_noema_complete_auto_input.py") == 1 + for fixture in (*CANARY_CLI_FIXTURES, + "tests/fixtures/noema_canary_publication_fixture.py"): + assert fixture not in rows[0] + assert result.returncode == pytest_exit + assert result.stdout == result.stderr == "" + if pytest_exit: + assert len(rows) == 1 + else: + assert len(rows) == 3 + assert rows[2][:3] == ["-m", "compileall", "-q"] + for path in ("tests/test_noema_canary_publication.py", + "tests/test_noema_exact_canary.py", + *CANARY_CLI_FIXTURES, + "tests/test_noema_complete_auto_input.py", + "scripts/ci/noema_review_gate.py", + "scripts/ci/noema_exact_canary.py", + "tests/fixtures/noema_canary_publication_fixture.py"): + assert rows[2].count(path) == 1 + + def test_exact_head_is_verified_before_selected_suites_run() -> None: """Reject a checkout that differs from the pull request's current head.""" diff --git a/tests/test_noema_canary_publication.py b/tests/test_noema_canary_publication.py new file mode 100644 index 0000000000..6eadc7d9a3 --- /dev/null +++ b/tests/test_noema_canary_publication.py @@ -0,0 +1,232 @@ +"""Exercise parsed admission/prepare/publish shells and the full real CLI.""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import uuid + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = 'ContextualWisdomLab/traveling-agent' +HEAD = 'bf85ddde01f871f0e713db54aae5a16883777d9f' +ADMIT = 'Admit existing personal LiteLLM auto route' +PREPARE = 'Prepare Noema model verdict' +PUBLISH = 'Publish prepared Noema verdict on the exact live head' +MODE = 'NOEMA_EXACT_CANARY_NON_ZDR' + + +def _steps(): + """Parse actual YAML instead of supplying a synthetic workflow mapping.""" + document = yaml.load((ROOT / '.github/workflows/noema-review.yml').read_text(), Loader=yaml.BaseLoader) + return {step['name']: step for step in document['jobs']['noema-review']['steps']} + + +def _output(path): + """Read simple runner output records from the actual successful shell.""" + return dict(line.split('=', 1) for line in path.read_text().splitlines()) if path.exists() else {} + + +def _mode_from_declaration(expression, *, variables, outputs): + """Evaluate only the workflow's finite mode references, never invent one.""" + match = re.fullmatch(r"\$\{\{ (vars\.[A-Z_]+|steps\.[a-z_]+\.outputs\.[a-z_]+) \|\| 'false' \}\}", expression) + assert match, 'Unsupported actual canary mode declaration' + reference = match[1].split('.') + if reference[0] == 'vars': + return variables.get(reference[1]) or 'false' + return outputs.get(reference[1], {}).get(reference[3]) or 'false' + + +def _run_step(tmp_path, name, phase, case, marker, *, variables, outputs): + """Run unchanged shell bytes; derive mode exclusively from its declaration.""" + tmp_path.mkdir() + step = _steps()[name] + input_path, state_path = tmp_path / 'input.json', tmp_path / 'state.json' + input_path.write_text(json.dumps({'phase': phase, 'case': case, 'marker': marker})) + bin_path = tmp_path / 'bin' + bin_path.mkdir() + launcher = bin_path / 'python3' + launcher.write_text('#!/bin/sh\nexec ' + shlex.join([ + sys.executable, '-I', '-B', str(ROOT / 'tests/fixtures/noema_canary_publication_fixture.py')]) + ' "$@"\n') + launcher.chmod(0o700) + output = tmp_path / 'outputs' + # All credentials below are inert external-IO inputs, not real Secrets. + environment = {'PATH': str(bin_path) + os.pathsep + os.environ['PATH'], + 'HOME': str(tmp_path), 'TMPDIR': str(tmp_path), + 'TARGET_REPOSITORY': REPOSITORY, 'PR_NUMBER': '4', 'EXPECTED_HEAD_SHA': HEAD, + 'GITHUB_WORKSPACE': str(ROOT), 'RUNNER_TEMP': str(tmp_path.parent), + 'GITHUB_OUTPUT': str(output), 'GH_TOKEN': 'synthetic', + 'NOEMA_REVIEW_TOKEN_SOURCE': 'noema-review-github-app-refresh' if phase == 'publish' else 'noema-review-github-app', + 'NOEMA_REVIEW_ACTOR': 'cwl-noema-review[bot]', 'NOEMA_REVIEW_INSTALLATION_ID': '1', + 'NOEMA_EXACT_CANARY_API_KEY': 'synthetic', 'LLM_GATEWAY_API_KEY': 'synthetic', + 'NOEMA_REVIEW_PAID_AUTO_AUTHORIZED': 'false', 'PERSONAL_REVIEW_ZERO_COST_ATTESTED': 'true', + 'PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED': 'false', 'PERSONAL_REVIEW_ZDR_ATTESTED': 'true', + 'PRIVATE_TARGET': 'true', 'PERSONAL_ROUTE_SELECTED': outputs.get('personal_route', {}).get('selected', ''), + 'FIXTURE_SOURCE': str(ROOT), 'FIXTURE_INPUT': str(input_path), 'FIXTURE_STATE': str(state_path)} + # Step-local Actions env does not persist. Missing Publish env stays absent. + assert MODE not in environment + if MODE in step.get('env', {}): + environment[MODE] = _mode_from_declaration(step['env'][MODE], variables=variables, outputs=outputs) + command = ['bash', '-c', step['run']] + if sys.platform == 'darwin': + profile = ('(version 1)(allow default)(deny network*)(deny file-write*)' + '(allow file-write* (subpath ' + json.dumps(str(tmp_path.parent)) + '))') + command = ['/usr/bin/sandbox-exec', '-p', profile, *command] + result = subprocess.run(command, cwd=ROOT, env=environment, capture_output=True, timeout=20) + (tmp_path / 'stdout.bin').write_bytes(result.stdout) + (tmp_path / 'stderr.bin').write_bytes(result.stderr) + state = json.loads(state_path.read_text()) if state_path.exists() else None + row = {'phase': phase, 'case': case, 'exit': result.returncode, + 'mode_declaration': step.get('env', {}).get(MODE), + 'mode_value': environment.get(MODE), 'outputs': _output(output), + 'stdout_sha256': hashlib.sha256(result.stdout).hexdigest(), + 'stderr_sha256': hashlib.sha256(result.stderr).hexdigest(), + 'marker_in_logs': marker.encode() in result.stdout + result.stderr, + 'state': state} + (tmp_path / 'result.json').write_text(json.dumps(row, indent=2)) + return row, result + + +def _admit_and_prepare(tmp_path, marker, *, mode='true', prepare_case='success'): + """Connect real policy admission output to real preparation's environment.""" + outputs, variables = {}, {MODE: mode} + admitted, _ = _run_step(tmp_path / 'admit', ADMIT, 'admit', 'success', marker, + variables=variables, outputs=outputs) + assert admitted['exit'] == 0, 'Actual admission did not succeed' + outputs['personal_route'] = admitted['outputs'] + assert outputs['personal_route'].get('selected') == 'true' + assert (outputs['personal_route'].get('exact_canary') == 'true') == (mode == 'true') + prepared, _ = _run_step(tmp_path / 'prepare', PREPARE, 'prepare', prepare_case, marker, + variables=variables, outputs=outputs) + outputs['noema_prepare'] = prepared['outputs'] + if prepare_case == 'success': + assert prepared['exit'] == 0 and prepared['outputs'].get('prepared') == 'true' + envelope = tmp_path / 'noema-verdict-envelope.json' + assert envelope.is_file() and envelope.stat().st_mode & 0o077 == 0 + return variables, outputs, admitted, prepared + + +@pytest.mark.parametrize('case', ['pr_nonzero', 'cleanup_os']) +def test_actual_publish_shell_contains_external_failures(tmp_path, case): + """SEC1: no manual missing-env injection can hide the real composition.""" + marker = 'PRIVATE_PUBLISH_' + uuid.uuid4().hex + variables, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker) + published, result = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables=variables, outputs=outputs) + assert result.returncode == 1 + state = published['state'] + assert state['faults'] == ['gh_stderr' if case == 'pr_nonzero' else 'cleanup_os'] + assert state['posts'] == [] and state['requests'] == [] + assert not published['marker_in_logs'], 'Private marker reached actual Publish shell->CLI output' + assert state['canary_mode'] == 'true' + assert all('reviewThreads' not in query for query in state['queries']) + assert not admitted['marker_in_logs'] and not prepared['marker_in_logs'] + + +@pytest.mark.parametrize('mode', ['true', 'false']) +@pytest.mark.parametrize('case,expected_exit,expected_posts', [ + ('success', 0, 1), ('head_drift', 0, 0), ('base_drift', 0, 0), + ('draft', 0, 0), ('author_drift', 1, 0), ('self_author', 1, 0), +]) +def test_connected_publication_preserves_privacy_and_exact_identity( + tmp_path, mode, case, expected_exit, expected_posts, +): + """Real admission->prepare->publish preserves normal behavior and guards.""" + marker = 'PRIVATE_PHASE_' + uuid.uuid4().hex + variables, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker, mode=mode) + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables=variables, outputs=outputs) + assert published['exit'] == expected_exit + state = published['state'] + assert state['canary_mode'] == mode + assert len(state['posts']) == expected_posts + assert state['requests'] == [] + assert not (tmp_path / 'noema-verdict-envelope.json').exists() + assert all(('reviewThreads' not in query) == (mode == 'true') for query in state['queries']) + assert not any(marker in json.dumps(post) for post in state['posts']) + if state['posts']: + assert state['posts'][0]['commit_id'] == HEAD + assert state['posts'][0]['event'] == 'APPROVE' + assert 'noema-review-github-app-refresh' in state['posts'][0]['body'] + if mode == 'true': + assert not any(row['marker_in_logs'] for row in (admitted, prepared, published)) + requests = prepared['state']['requests'] + assert len(requests) == 1 and marker not in requests[0]['data'] + payload = json.loads(requests[0]['data']) + assert set(payload) == {'model', 'messages', 'response_format'} + from scripts.ci import noema_exact_canary as policy + assert payload['model'] == 'auto' + assert payload['messages'] == [ + {'role': 'system', 'content': policy.SYSTEM}, + {'role': 'user', 'content': policy.INSTRUCTION + 'name: synthetic\non: workflow_dispatch\n'}] + else: + # Normal mode intentionally retains its broader context and telemetry. + assert prepared['marker_in_logs'] + assert marker in prepared['state']['requests'][0]['data'] + + +@pytest.mark.parametrize('case', ['prepare_fsync_os', 'prepare_http']) +def test_connected_preparation_cleanup_and_telemetry_stay_private(tmp_path, case): + """Real prep failure sinks remain contained with admission-derived mode.""" + marker = 'PRIVATE_PREP_' + uuid.uuid4().hex + _, outputs, admitted, prepared = _admit_and_prepare(tmp_path, marker, prepare_case=case) + assert prepared['exit'] == 1 + assert not admitted['marker_in_logs'] and not prepared['marker_in_logs'] + assert not (tmp_path / 'noema-verdict-envelope.json').exists() + assert outputs['noema_prepare'].get('prepared') != 'true' + state = prepared['state'] + assert state['canary_mode'] == 'true' and state['posts'] == [] + assert len(state['requests']) == 1 + assert marker not in state['requests'][0]['data'] + assert marker not in json.dumps(outputs) + if case == 'prepare_fsync_os': + assert state['faults'] == ['prepare_fsync_os'] + else: + assert outputs['noema_prepare']['transport_http_status'] == '503' + assert outputs['noema_prepare']['provider_attempt_count'] == '1' + + +@pytest.mark.parametrize('case', ['pr_nonzero', 'cleanup_os']) +def test_user_variable_alone_cannot_select_publication_privacy(tmp_path, case): + """Without the executable admission output, even vars=true stays off.""" + marker = 'NORMAL_PUBLISH_' + uuid.uuid4().hex + _, outputs, _, _ = _admit_and_prepare(tmp_path, marker, mode='false') + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', case, marker, + variables={MODE: 'true'}, outputs=outputs) + assert published['exit'] == 1 + assert published['state']['canary_mode'] == 'false' + assert published['marker_in_logs'] # unchanged normal-mode diagnostic control + assert published['state']['posts'] == [] + + +@pytest.mark.parametrize('mode', ['false', 'true']) +def test_admitted_publication_mode_does_not_follow_later_user_variable(tmp_path, mode): + """The successful admission output, not a later variable, selects mode.""" + marker = 'ADMISSION_BINDING_' + uuid.uuid4().hex + _, outputs, _, _ = _admit_and_prepare(tmp_path, marker, mode=mode) + published, _ = _run_step(tmp_path / 'publish', PUBLISH, 'publish', 'success', marker, + variables={MODE: 'false' if mode == 'true' else 'true'}, outputs=outputs) + assert published['exit'] == 0 + assert published['state']['canary_mode'] == mode + assert len(published['state']['posts']) == 1 + + +@pytest.mark.parametrize('mode', ['', 'invalid']) +def test_invalid_admission_cannot_emit_canary_identity(tmp_path, mode): + """The trusted shell emits no identity when its executable condition fails.""" + marker = 'DENIED_ADMISSION_' + uuid.uuid4().hex + admitted, _ = _run_step(tmp_path / 'admit', ADMIT, 'admit', 'success', marker, + variables={MODE: mode}, outputs={}) + if mode == '': + assert admitted['exit'] == 0 + assert 'exact_canary' not in admitted['outputs'] + else: + assert admitted['exit'] != 0 and admitted['outputs'] == {} + assert not admitted['marker_in_logs'] diff --git a/tests/test_noema_complete_auto_input.py b/tests/test_noema_complete_auto_input.py new file mode 100644 index 0000000000..6d0945d80e --- /dev/null +++ b/tests/test_noema_complete_auto_input.py @@ -0,0 +1,44 @@ +"""Complete source admission through the real Noema serializer (no network).""" +import json +from unittest.mock import patch +import pytest +from scripts.ci import noema_review_gate as gate + + +def test_auto_fetch_preserves_complete_diff(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + diff = 'diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -0,0 +1,2 @@\n+' + 'a'*61000 + '\n+final_source_line\n' + monkeypatch.setattr(gate, 'run', lambda _: diff) + assert gate.fetch_diff('o/r', 1) == (diff, False) + + +def test_real_serializer_rejects_oversized_auto_before_network(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + monkeypatch.setenv('NOEMA_LLM_API_KEY', 'inert-test-value') + monkeypatch.setattr(gate, 'reject_private_llm_url', lambda _: None) + with patch.object(gate.urllib.request, 'build_opener') as opener: + with pytest.raises(RuntimeError, match='complete review input exceeds'): + gate.call_llm('o/r', 1, {'headRefOid':'a'*40}, '+'+'x'*500000, False, 'a'*40) + opener.assert_not_called() + + +@pytest.mark.parametrize('model,url', [('auto','https://example.com/v1/chat/completions'), ('orchestrator/auto','https://litellm.poinnetworks.net/v1/chat/completions')]) +def test_other_routes_keep_existing_bound(monkeypatch, model, url): + monkeypatch.setenv('NOEMA_LLM_MODEL', model) + monkeypatch.setenv('NOEMA_LLM_API_URL', url) + monkeypatch.setattr(gate, 'run', lambda _: 'x'*61000) + text, truncated = gate.fetch_diff('o/r', 1) + assert truncated and len(text) <= gate.MAX_DIFF_CHARS + + +def test_auto_rejects_partial_input(monkeypatch): + monkeypatch.setenv('NOEMA_LLM_MODEL', 'auto') + monkeypatch.setenv('NOEMA_LLM_API_URL', 'https://litellm.poinnetworks.net/v1/chat/completions') + monkeypatch.setenv('NOEMA_LLM_API_KEY', 'inert-test-value') + monkeypatch.setattr(gate, 'reject_private_llm_url', lambda _: None) + with patch.object(gate.urllib.request, 'build_opener') as opener: + with pytest.raises(RuntimeError, match='untruncated'): + gate.call_llm('o/r', 1, {}, 'partial', True, 'a'*40) + opener.assert_not_called() diff --git a/tests/test_noema_exact_canary.py b/tests/test_noema_exact_canary.py new file mode 100644 index 0000000000..829dbf0e52 --- /dev/null +++ b/tests/test_noema_exact_canary.py @@ -0,0 +1,271 @@ +"""Exact content exception must not disclose PR prose or sibling source.""" +import unittest +from scripts.ci import noema_exact_canary as policy + +class ExactCanaryTests(unittest.TestCase): + def test_public_canary_helpers_have_docstrings(self): + """The canary module remains eligible for the 100% docstring gate.""" + import ast + from pathlib import Path + tree = ast.parse(Path(policy.__file__).read_text()) + helpers = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)} + for name in ('require_target', 'require_content', 'messages'): + with self.subTest(helper=name): + self.assertTrue(ast.get_docstring(helpers[name]), name) + + def test_actual_serialization_contains_only_approved_yaml(self): + import json + from unittest.mock import patch + # Synthetic approved-content fixture; the real digest remains immutable. + import hashlib + content = 'name: synthetic\n' + with patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()): + from scripts.ci.noema_review_gate import _noema_verdict_response_format + raw = policy.serialize_payload(content, _noema_verdict_response_format(2)) + decoded = json.loads(raw) + self.assertEqual(decoded['messages'], policy.messages(content)) + self.assertNotIn('title', decoded) + for extra in ('\nforeign source', '\nreview thread'): + decoded['messages'][1]['content'] = policy.INSTRUCTION + content + extra + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.validate_payload(decoded) + + def test_foreign_schema_is_not_an_input_channel(self): + import hashlib + from unittest.mock import patch + content = 'name: synthetic\n' + with patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()): + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.serialize_payload(content, {'description': 'foreign private source'}) + + def test_call_llm_uses_only_exact_serialized_input(self): + import os + import json + import hashlib + from unittest.mock import patch + from scripts.ci import noema_review_gate as gate + content = 'name: synthetic\n' + requests = [] + class StopProbe(BaseException): + pass + class Opener: + def open(self, request): + requests.append(json.loads(request.data)) + raise StopProbe() + env = {'NOEMA_EXACT_CANARY_NON_ZDR': 'true', + 'NOEMA_LLM_API_URL': 'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_API_KEY': 'synthetic', 'NOEMA_LLM_MODEL': 'auto'} + with patch.dict(os.environ, env, clear=True), \ + patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()), \ + patch.object(gate, 'reject_private_llm_url'), \ + patch.object(gate, 'fetch_changed_files', return_value=[(policy.PATH, 'added')]), \ + patch.object(gate, 'fetch_file_content_at_ref', return_value=content), \ + patch.object(gate.urllib.request, 'build_opener', return_value=Opener()): + with self.assertRaises(StopProbe): + gate.call_llm(policy.REPOSITORY, 4, + {'headRefOid': policy.HEAD, 'title': 'PRIVATE TITLE'}, + 'PRIVATE DIFF', False, policy.HEAD, 'PRIVATE THREAD', [policy.PATH]) + self.assertEqual(len(requests), 1) + schema = requests[0]['response_format']['json_schema']['schema'] + self.assertEqual(schema['properties']['adversarial_validation']['properties']['probes']['minItems'], + gate._required_probe_count('', [policy.PATH])) + self.assertEqual(requests[0]['messages'], [ + {'role': 'system', 'content': policy.SYSTEM}, + {'role': 'user', 'content': policy.INSTRUCTION + content}]) + + def test_workflow_uses_dedicated_default_closed_exception(self): + from pathlib import Path + source = (Path(__file__).resolve().parents[1] / '.github/workflows/noema-review.yml').read_text() + self.assertIn("NOEMA_EXACT_CANARY_NON_ZDR: ${{ vars.NOEMA_EXACT_CANARY_NON_ZDR || 'false' }}", source) + self.assertIn('secrets.NOEMA_EXACT_CANARY_API_KEY', source) + self.assertIn('python3 -m scripts.ci.noema_exact_canary', source) + + def test_canary_credential_selection_rejects_pat_oidc_before_model(self): + import os + import subprocess + import tempfile + from pathlib import Path + import yaml + root = Path(__file__).resolve().parents[1] + workflow = yaml.load((root / '.github/workflows/noema-review.yml').read_text(), Loader=yaml.BaseLoader) + step = next(s for s in workflow['jobs']['noema-review']['steps'] if s['name'] == 'Select fail-closed Noema reviewer credential') + for mode, pat, app, expected in [('true', True, True, None), ('true', False, False, None), + ('true', False, True, 'github-app'), ('false', True, True, 'pat')]: + with self.subTest(mode=mode, pat=pat, app=app), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'out' + env = dict(PATH=os.environ['PATH'], TARGET_REPOSITORY=policy.REPOSITORY, + NOEMA_EXACT_CANARY_NON_ZDR=mode, GITHUB_OUTPUT=str(output), + NOEMA_REVIEW_TOKEN='synthetic-pat' if pat else '', + NOEMA_GITHUB_APP_CLIENT_ID='synthetic-id' if app else '', + NOEMA_GITHUB_APP_PRIVATE_KEY='synthetic-key' if app else '', + TOKEN_EXCHANGE_URL='https://inert.invalid', GH_TOKEN='ambient-must-not-borrow') + result = subprocess.run(['bash', '-c', step['run']], env=env, capture_output=True, text=True) + if expected is None: + self.assertNotEqual(result.returncode, 0) + self.assertIn('Non-ZDR canary requires native GitHub App selection', result.stdout) + self.assertNotIn('source=', output.read_text()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('source=' + expected, output.read_text()) + + def test_advertised_minimum_survives_actual_validator(self): + import os + import json + import hashlib + from unittest.mock import patch + from scripts.ci import noema_review_gate as gate + content = 'name: synthetic\non: workflow_dispatch\n' + requests = [] + class Response: + def __enter__(self): return self + def __exit__(self, *args): return False + def read(self): + schema = requests[-1]['response_format']['json_schema']['schema'] + count = schema['properties']['adversarial_validation']['properties']['probes']['minItems'] + verdict = {'decision': 'approve', 'summary': 'Synthetic review.', 'findings': [], + 'reviewed_lines': [{'path': policy.PATH, 'line': 1, 'side': 'RIGHT', 'analysis': 'Examined exact added workflow.'}], + 'adversarial_validation': {'status': 'passed', 'residual_risk': 'Offline fixture only.', + 'probes': [{'path': policy.PATH, 'line': i + 1, 'side': 'RIGHT', + 'hypothesis': f'Regression {i}', 'attack_or_counterexample': f'Counterexample {i}', + 'evidence': f'Exact line {i + 1}', 'outcome': 'falsified'} for i in range(count)]}} + return json.dumps({'choices': [{'message': {'content': json.dumps(verdict)}}]}).encode() + class Opener: + def open(self, request): + requests.append(json.loads(request.data)) + return Response() + env = {'NOEMA_EXACT_CANARY_NON_ZDR': 'true', + 'NOEMA_LLM_API_URL': 'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_API_KEY': 'synthetic', 'NOEMA_LLM_MODEL': 'auto'} + with patch.dict(os.environ, env, clear=True), \ + patch.object(policy, 'DIGEST', hashlib.sha256(content.encode()).hexdigest()), \ + patch.object(gate, 'reject_private_llm_url'), \ + patch.object(gate, 'fetch_changed_files', return_value=[(policy.PATH, 'added')]), \ + patch.object(gate, 'fetch_file_content_at_ref', return_value=content), \ + patch.object(gate.urllib.request, 'build_opener', return_value=Opener()): + verdict = gate.call_llm(policy.REPOSITORY, 4, {'headRefOid': policy.HEAD}, + 'PRIVATE DIFF', False, policy.HEAD, 'PRIVATE THREAD', [policy.PATH]) + self.assertEqual(verdict['decision'], 'approve') + self.assertEqual(len(verdict['adversarial_validation']['probes']), 2) + self.assertEqual(len(requests), 1) + + def test_scope_is_exact(self): + policy.require_target(policy.REPOSITORY, policy.NUMBER, policy.HEAD) + for args in [('foreign/repo', 4, policy.HEAD), + (policy.REPOSITORY, True, policy.HEAD), + (policy.REPOSITORY, 5, policy.HEAD), + (policy.REPOSITORY, 4, 'a' * 40)]: + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.require_target(*args) + + def test_payload_refuses_unapproved_context(self): + with self.assertRaisesRegex(RuntimeError, 'scope'): + policy.validate_payload({'messages': [{'role': 'user', 'content': 'foreign private source'}]}) + +if __name__ == '__main__': + unittest.main() + + +"""Public CLI privacy regression using real consumers and finite IO adapters.""" +import hashlib, json, os, pathlib, subprocess, sys, uuid +import pytest + +ROOT=pathlib.Path(__file__).resolve().parents[1] + +MATRIX = [(entry, stage+':'+kind) + for entry,stages in [('prepare',('pr','files','content','http','enter','read','close','envelope','fsync')), + ('policy',('pr','files','content')), + ('gate',('pr','files','content','http','enter','read','close','readback')), + ('publish',('pr','cleanup'))] + for stage in stages for kind in ('nonzero','decode','runtime','os','value','interrupt','exit') + if kind!='nonzero' or stage in {'pr','files','content','readback'}] +MATRIX += [('prepare',stage+':'+kind) for stage in ('error_read','error_close') for kind in ('runtime','os','value','interrupt','exit')] +MATRIX += [(entry,case) for entry in ('prepare','gate') for case in ('http:url','http:http','verdict:invalid','model_decode:invalid')] +MATRIX += [('prepare','telemetry:success'),('prepare','context:runtime'),('prepare','context:os'),('prepare','context:interrupt'),('prepare','context:exit'),('prepare','diff:nonzero'),('prepare','success:ok'),('policy','success:ok'),('publish','parse_envelope:decode')] +MATRIX += [(entry,stage+':'+kind) for entry,stages in ( + ('workflow_admit',('pr','files','content')),('workflow_prepare',('pr','files','content','read','close','fsync'))) + for stage in stages for kind in ('nonzero','os','interrupt','exit') + if kind!='nonzero' or stage in {'pr','files','content'}] +MATRIX += [('workflow_admit','success:ok'),('workflow_prepare','success:ok')] +MATRIX += [('native_'+entry,case) for entry,case in ( + ('prepare','pr:nonzero'),('prepare','files:nonzero'),('prepare','content:nonzero'), + ('prepare','success:ok'),('policy','content:nonzero'),('policy','success:ok'))] + +@pytest.mark.parametrize('entry,case', MATRIX) +def test_public_cli_does_not_expose_private_context(entry,case,tmp_path): + marker='PRIVATE_CANARY_'+uuid.uuid4().hex + input_file=tmp_path/'input.json';state_file=tmp_path/'state.json' + input_file.write_text(json.dumps({'entry':entry,'case':case,'marker':marker})) + source=str(ROOT) + workflow=entry.startswith('workflow_') + native=entry.startswith('native_') + actual_entry={'workflow_admit':'policy','workflow_prepare':'prepare'}.get(entry,entry.removeprefix('native_')) + input_file.write_text(json.dumps({'entry':actual_entry,'case':case,'marker':marker,'native':native})) + environment={'PATH':os.environ['PATH'],'HOME':str(tmp_path),'TMPDIR':str(tmp_path), + 'NOEMA_EXACT_CANARY_NON_ZDR':'true','NOEMA_LLM_API_URL':'https://litellm.poinnetworks.net/v1/chat/completions', + 'NOEMA_LLM_MODEL':'auto','NOEMA_LLM_API_KEY':'synthetic', + 'NOEMA_REVIEW_TOKEN_SOURCE':'noema-review-github-app','NOEMA_REVIEW_ACTOR':'cwl-noema-review[bot]', + 'NOEMA_REVIEW_INSTALLATION_ID':'1','FIXTURE_SOURCE':source,'FIXTURE_INPUT':str(input_file),'FIXTURE_STATE':str(state_file)} + command=[sys.executable,'-I','-B',str(ROOT/'tests/fixtures/noema_canary_cli_fixture.py')] + if native: + import shlex + launcher=tmp_path/'native-bin';launcher.mkdir() + script=launcher/'gh' + script.write_text('#!/bin/sh\nexec '+shlex.join([sys.executable,'-I','-B',str(ROOT/'tests/fixtures/noema_canary_github_fixture.py')])+' "$@"\n') + script.chmod(0o700) + environment['PATH']=str(launcher)+os.pathsep+environment['PATH'] + if workflow: + import shlex + import yaml + launcher=tmp_path/'bin';launcher.mkdir() + script=launcher/'python3' + script.write_text('#!/bin/sh\nexec '+shlex.join(command)+' "$@"\n');script.chmod(0o700) + environment.update(PATH=str(launcher)+os.pathsep+environment['PATH'],TARGET_REPOSITORY=policy.REPOSITORY, + PR_NUMBER='4',EXPECTED_HEAD_SHA=policy.HEAD,NOEMA_EXACT_CANARY_API_KEY='synthetic',GH_TOKEN='synthetic', + PERSONAL_ROUTE_SELECTED='true',GITHUB_WORKSPACE=str(ROOT),RUNNER_TEMP=str(tmp_path),GITHUB_OUTPUT=str(tmp_path/'out')) + workflow_doc=yaml.load((ROOT/'.github/workflows/noema-review.yml').read_text(),Loader=yaml.BaseLoader) + step_name='Admit existing personal LiteLLM auto route' if entry=='workflow_admit' else 'Prepare Noema model verdict' + step=next(s for s in workflow_doc['jobs']['noema-review']['steps'] if s.get('name')==step_name) + command=['bash','-c',step['run']] + if sys.platform=='darwin': + profile='(version 1)(allow default)(deny network*)(deny file-write*)(allow file-write* (subpath '+json.dumps(str(tmp_path))+'))' + command=['/usr/bin/sandbox-exec','-p',profile,*command] + result=subprocess.run(command,env=environment,cwd=source,capture_output=True,timeout=20) + (tmp_path/'stdout.bin').write_bytes(result.stdout);(tmp_path/'stderr.bin').write_bytes(result.stderr) + record={'entry':entry,'case':case,'exit':result.returncode,'stdout_sha256':hashlib.sha256(result.stdout).hexdigest(), + 'stderr_sha256':hashlib.sha256(result.stderr).hexdigest(),'marker_in_logs':marker.encode() in result.stdout+result.stderr} + state=json.loads(state_file.read_text()) + record['external_calls']=state['external_calls'];record['requests']=len(state['http_requests']);record['publications']=state['publications'] + record['stage_faults']=state['stage_faults'];record['context_skipped']=state.get('context_skipped',False) + record['marker_in_requests']=any(marker in r['data'] for r in state['http_requests']) + if state['http_requests']: + payload=json.loads(state['http_requests'][0]['data']) + assert set(payload)=={'model','messages','response_format'} + assert payload['model']=='auto' + assert payload['messages']==[{'role':'system','content':'Return strict JSON only. Do not include markdown.'}, + {'role':'user','content':'Independently review this exact YAML for correctness and security. Return the declared response schema. Cite changed RIGHT-side lines. For approval falsify at least two distinct concrete regression hypotheses; for request_changes confirm a concrete finding. This is a complete new file; no other repository context is authorized.\nRepository: ContextualWisdomLab/traveling-agent\nPR: 4\nHead: bf85ddde01f871f0e713db54aae5a16883777d9f\nPath: .github/workflows/docker-ephemeral-canary.yml\nname: synthetic\non: workflow_dispatch\n'}] + (tmp_path/'result.json').write_text(json.dumps(record,indent=2)) + if workflow: + assert not any(stage in state['external_calls'] for stage in ('diff','context')) + if case=='success:ok': + assert result.returncode==0 + outputs=(tmp_path/'out').read_text() + assert ('selected=true' if entry=='workflow_admit' else 'prepared=true') in outputs + else: + assert result.returncode!=0 + assert not record['marker_in_logs'], 'Private sentinel reached the actual public CLI output' + assert not record['marker_in_requests'], 'Private sentinel reached the actual HTTP request payload' + assert record['publications']==0 + stage,kind=case.split(':',1) + skipped=entry=='prepare' and state.get('context_skipped',False) and stage in {'context','diff'} + success=stage in {'success','telemetry'} or skipped or (stage=='context' and kind=='runtime') + expected=0 if success else 130 if kind=='interrupt' else 1 + if not workflow: + assert result.returncode==expected + if not success and stage in {'pr','files','content','readback'}: + assert stage in state['external_calls'], 'Failure boundary was not reached' + if stage in {'envelope','fsync','cleanup'}: + assert stage in state['stage_faults'], 'File lifecycle boundary was not reached' + if stage in {'http','enter','read','close','error_read','error_close','verdict','model_decode','telemetry','envelope','fsync'}: + assert record['requests']==1 + if stage in {'pr','files','content','parse_envelope','cleanup'}: + assert record['requests']==0 diff --git a/tests/test_noema_paid_auto_authorization.py b/tests/test_noema_paid_auto_authorization.py new file mode 100644 index 0000000000..6a103a91ad --- /dev/null +++ b/tests/test_noema_paid_auto_authorization.py @@ -0,0 +1,119 @@ +"""Noema cost authorization is independent of private retention admission.""" +import pathlib +import unittest +import subprocess +import sys +import os +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +class PaidAutoWorkflowTests(unittest.TestCase): + def test_paid_regressions_enrolled_in_runtime_quality(self): + source = (ROOT / '.github/workflows/agent-review-runtime-quality-ci.yml').read_text() + path = 'tests/test_noema_paid_auto_authorization.py' + trigger = source.split('jobs:', 1)[0] + self.assertIn('"' + path + '"', trigger) + section = source.split(' - name: Select affected contract suites\n', 1)[1] + block = section.split(' run: |\n', 1)[1].split('\n - name:', 1)[0] + shell = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + stub = 'git() { if [ "$1" = rev-parse ]; then printf "%s\\n" "$HEAD_SHA"; else printf "%s\\n" "$CHANGED_PATH"; fi; };\n' + env = dict(os.environ, BASE_SHA='base', HEAD_SHA='head', + CHANGED_PATH=path, GITHUB_OUTPUT=str(output)) + result = subprocess.run(['bash', '-c', stub + shell], env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('review_repair=true', output.read_text()) + # Existing enrolled sibling appears in trigger, selector, pytest and compile. + self.assertEqual(source.count(path), source.count('tests/test_personal_review_route.py')) + + def test_producer_paths_select_and_execute_regressions(self): + import yaml + workflow = yaml.load((ROOT / '.github/workflows/agent-review-runtime-quality-ci.yml').read_text(), Loader=yaml.BaseLoader) + steps = {s['name']: s for s in workflow['jobs']['agent_review_runtime_quality']['steps']} + shell = steps['Select affected contract suites']['run'] + paths = ['.github/workflows/noema-review.yml', 'scripts/ci/noema_review_gate.py', + 'scripts/ci/noema_exact_canary.py', 'scripts/ci/personal_review_route.py', + 'tests/test_noema_exact_canary.py', 'tests/test_noema_paid_auto_authorization.py', + '.github/workflows/agent-review-runtime-quality-ci.yml'] + for path in paths + ['docs/unrelated.md']: + with self.subTest(path=path), tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + stub = 'git() { if [ "$1" = rev-parse ]; then printf "%s\\n" "$HEAD_SHA"; else printf "%s\\n" "$CHANGED_PATH"; fi; };\n' + env = dict(PATH=os.environ['PATH'], BASE_SHA='base', HEAD_SHA='head', + CHANGED_PATH=path, GITHUB_OUTPUT=str(output)) + result = subprocess.run(['bash', '-c', stub + shell], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + selected = dict(line.split('=', 1) for line in output.read_text().splitlines()) + if path == 'docs/unrelated.md': + self.assertEqual(selected['review_repair'], 'false') + continue + self.assertIn(path, workflow['on']['pull_request']['paths']) + self.assertEqual(selected['review_repair'], 'true') + command = steps['Verify scheduler and contextual-orchestrator review-repair contracts']['run'] + # Execute actual command bodies, recording only the Python boundary. + recorder = 'python() { printf "%s\\n" "$*"; };\n' + run = subprocess.run(['bash', '-c', recorder + command], env=env, cwd=ROOT, capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stderr) + pytest_calls = [line for line in run.stdout.splitlines() if line.startswith('-m pytest ')] + self.assertTrue(pytest_calls) + for test in ('tests/test_noema_exact_canary.py', 'tests/test_noema_paid_auto_authorization.py'): + self.assertTrue(any(test in line.split() for line in pytest_calls), test) + + def test_noema_passes_explicit_default_closed_paid_authorization(self): + source = (ROOT / '.github/workflows/noema-review.yml').read_text() + self.assertIn("NOEMA_REVIEW_PAID_AUTO_AUTHORIZED: ${{ vars.NOEMA_REVIEW_PAID_AUTO_AUTHORIZED || 'false' }}", source) + self.assertIn('--paid-auto-authorized "$NOEMA_REVIEW_PAID_AUTO_AUTHORIZED"', source) + self.assertIn('--zero-retention "$PERSONAL_REVIEW_ZDR_ATTESTED"', source) + self.assertIn('--private-target "$PRIVATE_TARGET"', source) + + def test_paid_authorized_private_route_with_retention_is_admitted(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--paid-auto-authorized', 'true', '--zero-cost', 'false', + '--zero-retention', 'true', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_paid_authority_never_substitutes_private_retention(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--paid-auto-authorized', 'true', '--zero-cost', 'false', + '--zero-retention', 'false', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('zero-retention supplier is not attested', result.stderr) + + def test_paid_authority_defaults_closed(self): + result = subprocess.run([sys.executable, '-m', 'scripts.ci.personal_review_route', + '--zero-cost', 'false', '--zero-retention', 'true', '--private-target', 'true'], + cwd=ROOT, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('zero-cost supplier is not attested', result.stderr) + + def test_real_workflow_shell_preserves_privacy_boundary(self): + source = (ROOT / '.github/workflows/noema-review.yml').read_text() + section = source.split(' - name: Admit existing personal LiteLLM auto route\n', 1)[1] + block = section.split(' run: |\n', 1)[1].split('\n - name:', 1)[0] + shell = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + env = dict(os.environ, LLM_GATEWAY_API_KEY='synthetic-not-a-key', + NOEMA_REVIEW_PAID_AUTO_AUTHORIZED='true', + PERSONAL_REVIEW_ZERO_COST_ATTESTED='false', + PERSONAL_REVIEW_PUBLIC_AUTO_AUTHORIZED='false', + PERSONAL_REVIEW_ZDR_ATTESTED='false', PRIVATE_TARGET='true', + GITHUB_OUTPUT=str(output)) + denied = subprocess.run(['bash', '-c', shell], cwd=ROOT, env=env, + capture_output=True, text=True) + self.assertNotEqual(denied.returncode, 0) + self.assertIn('zero-retention supplier is not attested', denied.stderr) + self.assertFalse(output.exists()) + env['PERSONAL_REVIEW_ZDR_ATTESTED'] = 'true' # synthetic positive only + admitted = subprocess.run(['bash', '-c', shell], cwd=ROOT, env=env, + capture_output=True, text=True) + self.assertEqual(admitted.returncode, 0, admitted.stderr) + self.assertEqual(output.read_text(), 'selected=true\n') + +if __name__ == '__main__': + unittest.main()