diff --git a/CHANGELOG.md b/CHANGELOG.md index 5439bd4c..b1e722ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to `github-delivery` are documented here. +## [1.9.1] - 2026-11-08 + +### Fixed + +- `ship-gate-snapshot` now accepts the documented GitHub plan/feature-unavailable 403 for private repositories without protected branches or rulesets only when the exact message, per-endpoint documentation URLs, private repository metadata, unprotected branch metadata, and empty classic rules all agree. Permission-denied, ambiguous, server, malformed, spoofed, and drifted evidence stays fail-closed, existing rules and required checks keep being enforced, and the snapshot remains complete and boundary-checked. This unblocks the merge gate for Wibias/streamer-content without changing repository plan, visibility, permissions, protection, or thresholds. + ## [1.9.0] - 2026-11-08 ### Added diff --git a/package.json b/package.json index cf826ac8..6b849c1b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "github-delivery", - "version": "1.9.0", + "version": "1.9.1", "description": "Guided installer and safety tooling for GitHub Delivery workflows", "type": "module", "engines": { diff --git a/scripts/lib/github-pagination.mjs b/scripts/lib/github-pagination.mjs index 55323f77..2c01b2c0 100644 --- a/scripts/lib/github-pagination.mjs +++ b/scripts/lib/github-pagination.mjs @@ -19,6 +19,7 @@ export function collectPaginated({ pages: page - 1, rows, error: response?.error || `${label} request failed`, + failure: response || null, }; } diff --git a/scripts/lib/rules-boundary.mjs b/scripts/lib/rules-boundary.mjs new file mode 100644 index 00000000..0a985f75 --- /dev/null +++ b/scripts/lib/rules-boundary.mjs @@ -0,0 +1,309 @@ +import { createHash } from "node:crypto"; + +/** + * GitHub returns this documented plan/feature gate for repository rules and + * classic branch protection when a private repository is on a plan without + * those features. The exact message and the per-endpoint documentation URL are + * part of the evidence contract: anything else stays fail-closed. + */ +export const RULES_PLAN_MESSAGE = + "Upgrade to GitHub Pro or make this repository public to enable this feature."; + +export const RULES_BOUNDARY_EVIDENCE_KIND = + "github-delivery/rules-boundary-evidence"; +export const RULES_BOUNDARY_MODE = "plan_feature_unavailable"; + +const ENDPOINTS = Object.freeze({ + rulesets: Object.freeze({ + documentationUrl: + "https://docs.github.com/rest/repos/rules#get-all-repository-rulesets", + }), + branchRules: Object.freeze({ + documentationUrl: + "https://docs.github.com/rest/repos/rules#get-rules-for-a-branch", + }), + classicProtection: Object.freeze({ + documentationUrl: + "https://docs.github.com/rest/branches/branch-protection#get-branch-protection", + }), +}); + +const ENDPOINT_KEYS = Object.freeze(Object.keys(ENDPOINTS)); + +function stripAnsi(value) { + return String(value || "").replace(/\u001b\[[0-9;]*m/g, ""); +} + +function compact(value) { + return stripAnsi(value).replace(/\s+/g, " ").trim(); +} + +function canonicalJson(value) { + if (value === null || typeof value === "string" || typeof value === "boolean") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("rules_boundary_number_invalid"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJson(item)).join(",")}]`; + } + if (value && typeof value === "object") { + const entries = Object.entries(value) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + return `{${entries + .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) + .join(",")}}`; + } + throw new Error("rules_boundary_value_invalid"); +} + +export function rulesBoundaryFingerprint(value) { + return createHash("sha256").update(canonicalJson(value), "utf8").digest("hex"); +} + +/** + * Parse a failed gh API response into a structured HTTP failure. Both the gh + * stderr status line and the JSON error payload must agree; a missing, + * duplicated, or mismatched status is ambiguous evidence. + */ +export function parseGitHubApiFailure(response = {}) { + if (response?.ok === true) return { failure: null, reason: "probe_succeeded" }; + const errorText = compact(response?.error); + if (!errorText) return { failure: null, reason: "error_text_missing" }; + + const statusMatches = [...errorText.matchAll(/\(HTTP\s+(\d{3})\)/g)]; + if (statusMatches.length !== 1) { + return { failure: null, reason: "http_status_ambiguous" }; + } + const status = Number(statusMatches[0][1]); + const errorMessage = errorText + .slice(0, statusMatches[0].index) + .replace(/^gh:\s*/i, "") + .trim(); + + let body = null; + try { + body = JSON.parse(String(response?.body || "")); + } catch { + return { failure: null, reason: "error_payload_invalid_json" }; + } + if (!body || typeof body !== "object" || Array.isArray(body)) { + return { failure: null, reason: "error_payload_missing" }; + } + + const bodyStatus = Number(body.status); + if (!Number.isInteger(bodyStatus)) { + return { failure: null, reason: "error_payload_status_missing" }; + } + if (bodyStatus !== status) { + return { failure: null, reason: "error_payload_status_mismatch" }; + } + + const message = typeof body.message === "string" ? body.message.trim() : ""; + if (!message) { + return { failure: null, reason: "error_payload_message_missing" }; + } + if (errorMessage && errorMessage !== message) { + return { failure: null, reason: "error_text_message_mismatch" }; + } + const documentationUrl = + typeof body.documentation_url === "string" + ? body.documentation_url.trim() + : ""; + if (!documentationUrl) { + return { failure: null, reason: "error_payload_documentation_url_missing" }; + } + + return { + failure: { status, message, documentationUrl }, + reason: null, + }; +} + +function classifyEndpointProbe(response, endpointKey) { + const endpoint = ENDPOINTS[endpointKey]; + if (!endpoint) throw new Error("rules_boundary_endpoint_unknown"); + const parsed = parseGitHubApiFailure(response); + if (!parsed.failure) { + return { + planGated: false, + reason: `${endpointKey}_${parsed.reason || "probe_unreadable"}`, + }; + } + const failure = parsed.failure; + if (failure.status !== 403) { + return { + planGated: false, + reason: `${endpointKey}_http_${failure.status}`, + }; + } + if (failure.message !== RULES_PLAN_MESSAGE) { + return { + planGated: false, + reason: `${endpointKey}_message_not_plan_gate`, + }; + } + if (failure.documentationUrl !== endpoint.documentationUrl) { + return { + planGated: false, + reason: `${endpointKey}_documentation_url_mismatch`, + }; + } + return { + planGated: true, + evidence: { status: 403, documentationUrl: endpoint.documentationUrl }, + }; +} + +function emptyPolicyEvidence(policy) { + const rules = policy?.branchProtectionRules; + return ( + policy?.readable === true && + policy?.complete === true && + rules && + Array.isArray(rules.nodes) && + rules.nodes.length === 0 && + rules.pageInfo?.hasNextPage !== true + ); +} + +function unprotectedBranchEvidence(branch) { + const protection = branch?.protection; + const required = protection?.required_status_checks; + return ( + branch?.protected === false && + protection?.enabled === false && + required?.enforcement_level === "off" && + Array.isArray(required?.checks) && + required.checks.length === 0 && + Array.isArray(required?.contexts) && + required.contexts.length === 0 + ); +} + +/** + * Accept a plan/feature-unavailable rules boundary only when every plan-gated + * endpoint returns the exact documented response and independent structured + * reads confirm the repository cannot have active rules. Any other combination + * (permission denial, transient/server error, missing data, message variant, + * conflicting evidence) stays unresolved and therefore fail-closed. + */ +export function classifyRulesPlanFeatureUnavailable({ + repo = null, + branch = null, + policy = null, + probes = null, +} = {}) { + const reasons = []; + const endpoints = {}; + + for (const key of ENDPOINT_KEYS) { + const result = classifyEndpointProbe(probes?.[key], key); + if (!result.planGated) reasons.push(result.reason); + endpoints[key] = result.evidence || null; + } + + if ( + !repo || + typeof repo !== "object" || + repo.private !== true || + repo.visibility !== "private" + ) { + reasons.push("repository_not_confirmed_private"); + } + if (!unprotectedBranchEvidence(branch)) { + reasons.push("branch_not_confirmed_unprotected"); + } + if (!emptyPolicyEvidence(policy)) { + reasons.push("classic_rules_not_confirmed_empty"); + } + + if (reasons.length) { + return { + determined: false, + unavailable: false, + reason: reasons[0], + reasons, + evidence: null, + }; + } + + const evidence = { + kind: RULES_BOUNDARY_EVIDENCE_KIND, + schemaVersion: 1, + mode: RULES_BOUNDARY_MODE, + message: RULES_PLAN_MESSAGE, + repo: { private: true, visibility: "private" }, + branch: { + protected: false, + protectionEnabled: false, + requiredStatusChecksEnforcement: "off", + requiredStatusCheckCount: 0, + requiredStatusContextCount: 0, + }, + policy: { branchProtectionRuleCount: 0, pageComplete: true }, + endpoints, + }; + return { + determined: true, + unavailable: true, + reason: null, + reasons: [], + evidence, + }; +} + +/** + * Revalidate a boundary evidence object that was carried through a snapshot. + * Snapshot replay is diagnostic-only, but internally inconsistent evidence must + * never be read as an authoritative absence of rules. + */ +export function validateRulesBoundaryEvidence(evidence = null) { + const reasons = []; + if (!evidence || typeof evidence !== "object" || Array.isArray(evidence)) { + return { valid: false, reasons: ["rules_boundary_evidence_missing"] }; + } + if (evidence.kind !== RULES_BOUNDARY_EVIDENCE_KIND) { + reasons.push("rules_boundary_evidence_kind_invalid"); + } + if (evidence.schemaVersion !== 1) { + reasons.push("rules_boundary_evidence_schema_invalid"); + } + if (evidence.mode !== RULES_BOUNDARY_MODE) { + reasons.push("rules_boundary_mode_invalid"); + } + if (evidence.message !== RULES_PLAN_MESSAGE) { + reasons.push("rules_boundary_message_invalid"); + } + if (evidence.repo?.private !== true || evidence.repo?.visibility !== "private") { + reasons.push("rules_boundary_repository_invalid"); + } + if ( + evidence.branch?.protected !== false || + evidence.branch?.protectionEnabled !== false || + evidence.branch?.requiredStatusChecksEnforcement !== "off" || + evidence.branch?.requiredStatusCheckCount !== 0 || + evidence.branch?.requiredStatusContextCount !== 0 + ) { + reasons.push("rules_boundary_branch_invalid"); + } + if ( + evidence.policy?.branchProtectionRuleCount !== 0 || + evidence.policy?.pageComplete !== true + ) { + reasons.push("rules_boundary_policy_invalid"); + } + for (const key of ENDPOINT_KEYS) { + if ( + evidence.endpoints?.[key]?.status !== 403 || + evidence.endpoints?.[key]?.documentationUrl !== + ENDPOINTS[key].documentationUrl + ) { + reasons.push(`rules_boundary_endpoint_invalid:${key}`); + } + } + return { valid: reasons.length === 0, reasons }; +} diff --git a/scripts/lib/snapshot-capture-payload.mjs b/scripts/lib/snapshot-capture-payload.mjs index 4d49c074..37fcd49a 100644 --- a/scripts/lib/snapshot-capture-payload.mjs +++ b/scripts/lib/snapshot-capture-payload.mjs @@ -1,5 +1,10 @@ import { createHash } from "node:crypto"; +import { + rulesBoundaryFingerprint, + validateRulesBoundaryEvidence, +} from "./rules-boundary.mjs"; + const MAX_BOUNDARY_FAILURE_CHARS = 2_000; function collectionSource(collection, required = true) { @@ -9,6 +14,9 @@ function collectionSource(collection, required = true) { complete: collection?.complete === true, pages: collection?.pages ?? null, error: collection?.error || null, + ...(collection?.featureUnavailable === true + ? { featureUnavailable: true } + : {}), }; } @@ -103,6 +111,8 @@ export function verifySnapshotBoundary( finalBaseOid = null, initialRules = null, finalRules = null, + initialRulesBoundary = null, + finalRulesBoundary = null, initialTestMergeOid = undefined, finalTestMergeOid = undefined, } = {}, @@ -154,6 +164,7 @@ export function verifySnapshotBoundary( } let rulesFingerprint = null; + let rulesBoundaryFingerprintValue = null; if (initialRules !== null || finalRules !== null) { if (initialRules?.complete !== true || finalRules?.complete !== true) { const causeMessage = collectionFailureMessage(initialRules, finalRules); @@ -174,12 +185,43 @@ export function verifySnapshotBoundary( rulesFingerprint = before; } + if (initialRulesBoundary !== null) { + const initialEvidence = initialRulesBoundary?.evidence || null; + const finalEvidence = finalRulesBoundary?.evidence || null; + const initialValid = + initialRulesBoundary?.unavailable === true && + validateRulesBoundaryEvidence(initialEvidence).valid; + const finalValid = + finalRulesBoundary?.unavailable === true && + validateRulesBoundaryEvidence(finalEvidence).valid; + if (!initialValid || !finalValid) { + const causeMessage = "rules_plan_feature_state_unproven"; + const error = new Error( + `snapshot_rules_boundary_incomplete: ${causeMessage}`, + ); + error.code = "snapshot_rules_boundary_incomplete"; + error.causeMessage = causeMessage; + throw error; + } + const before = rulesBoundaryFingerprint(initialEvidence); + const after = rulesBoundaryFingerprint(finalEvidence); + if (before !== after) { + throw new Error( + `snapshot_rules_moved:rules_boundary_evidence: expected ${before}, observed ${after}`, + ); + } + rulesBoundaryFingerprintValue = before; + } + return { headOid: initialHead, baseRefName: initialBase, ...(baseOid ? { baseOid } : {}), ...(testMergeOid ? { testMergeOid } : {}), ...(rulesFingerprint ? { rulesFingerprint } : {}), + ...(rulesBoundaryFingerprintValue + ? { rulesBoundaryFingerprint: rulesBoundaryFingerprintValue } + : {}), }; } @@ -200,6 +242,7 @@ export function assembleSnapshotCapture({ viewer, boundary = null, checkEvidence = null, + rulesBoundary = null, } = {}) { const sources = { pr: { required: true, readable: true, complete: true, error: null }, @@ -216,6 +259,9 @@ export function assembleSnapshotCapture({ readable: branchProtection?.readable === true, complete: branchProtection?.complete === true, error: branchProtection?.error || null, + ...(branchProtection?.featureUnavailable === true + ? { featureUnavailable: true } + : {}), }, codeowners: { required: false, @@ -254,6 +300,7 @@ export function assembleSnapshotCapture({ evidence: { pullRequest: prEvidence, captureBoundary: boundary, + rulesBoundary: rulesBoundary || null, changedFiles: changedFiles?.rows || [], branchProtection: branchProtection?.payload ?? null, activeRules: activeRules?.rows || [], diff --git a/scripts/lib/snapshot-evaluators.mjs b/scripts/lib/snapshot-evaluators.mjs index 4f187838..d1559e04 100644 --- a/scripts/lib/snapshot-evaluators.mjs +++ b/scripts/lib/snapshot-evaluators.mjs @@ -1,5 +1,6 @@ import { ownersForPath, parseCodeowners } from "./codeowners.mjs"; import { protectionRefName } from "./native-stack-policy.mjs"; +import { validateRulesBoundaryEvidence } from "./rules-boundary.mjs"; import { evaluateRequiredCheckCompleteness, evaluateRequiredChecks, @@ -31,6 +32,27 @@ function sourceReadable(snapshot, name) { return source(snapshot, name).readable === true; } +function rulesBoundaryEvidenceConsistent(snapshot) { + const active = source(snapshot, "activeRules"); + const classic = source(snapshot, "branchProtection"); + if ( + active?.featureUnavailable !== true && + classic?.featureUnavailable !== true + ) { + return true; + } + const boundary = snapshot?.evidence?.rulesBoundary; + return ( + boundary?.mode === "plan_feature_unavailable" && + validateRulesBoundaryEvidence(boundary?.evidence).valid + ); +} + +function activeRulesEvidenceComplete(snapshot) { + return sourceComplete(snapshot, "activeRules") && + rulesBoundaryEvidenceConsistent(snapshot); +} + function escapeRegexCharacter(value) { return /[\\^$.*+?()[\]{}|]/.test(value) ? `\\${value}` : value; } @@ -161,6 +183,7 @@ function matchingClassicRules(snapshot) { function classicProtectionReadable(snapshot, matchingRules) { if (!sourceReadable(snapshot, "branchProtection")) return false; + if (!rulesBoundaryEvidenceConsistent(snapshot)) return false; if (unsupportedClassicPatterns(snapshot).length > 0) return false; if (!matchingRules.length) return true; return snapshot?.evidence?.branchProtection !== null && @@ -227,7 +250,7 @@ export function evaluateRequiredChecksSnapshot(snapshot) { branchRules.pageInfo?.hasNextPage !== true, matchingClassicRuleCount: matchingRules.length, classicProtectionReadable: classicProtectionReadable(snapshot, matchingRules), - activeRulesComplete: sourceComplete(snapshot, "activeRules"), + activeRulesComplete: activeRulesEvidenceComplete(snapshot), checkRunsComplete: sourceComplete(snapshot, "checkRuns"), statusesComplete: sourceComplete(snapshot, "statuses"), }); @@ -323,7 +346,7 @@ export function evaluateReviewPolicySnapshot(snapshot) { branchRules.pageInfo?.hasNextPage !== true, matchingClassicRuleCount: matchingRules.length, classicProtectionReadable: classicProtectionReadable(snapshot, matchingRules), - activeRulesComplete: sourceComplete(snapshot, "activeRules"), + activeRulesComplete: activeRulesEvidenceComplete(snapshot), }); const blockers = [...evaluation.blockers]; diff --git a/scripts/ship-gate-snapshot.mjs b/scripts/ship-gate-snapshot.mjs index 10726412..99ec99b0 100755 --- a/scripts/ship-gate-snapshot.mjs +++ b/scripts/ship-gate-snapshot.mjs @@ -29,6 +29,7 @@ import { runGitHubCommandWithRetry } from "./lib/github-retry.mjs"; import { boundedSpawnSync } from "./lib/subprocess-policy.mjs"; import { graphqlCliField } from "./lib/graphql-cli-fields.mjs"; import { normalizeNativeStack } from "./lib/native-stack-policy.mjs"; +import { classifyRulesPlanFeatureUnavailable } from "./lib/rules-boundary.mjs"; function parseArgs(argv) { const positionals = []; @@ -505,14 +506,70 @@ function fetchPolicy(owner, name, pr) { } function fetchBranchProtection(owner, name, base) { - return classifyBranchProtectionResponse( + const response = ghOk([ + "api", + `repos/${owner}/${name}/branches/${encodeURIComponent(base)}/protection`, + ]); + return { ...classifyBranchProtectionResponse(response), response }; +} + +function parseJsonResponse(response, label) { + if (!response?.ok) { + return { + readable: false, + complete: false, + payload: null, + error: response?.error || `${label} request failed`, + }; + } + let payload; + try { + payload = JSON.parse(response.body || "null"); + } catch { + return { + readable: false, + complete: false, + payload: null, + error: `${label} returned invalid JSON`, + }; + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + return { + readable: false, + complete: false, + payload: null, + error: `${label} returned an unexpected payload`, + }; + } + return { readable: true, complete: true, payload, error: null }; +} + +function fetchRepoMetadata(owner, name) { + return parseJsonResponse( + ghOk(["api", `repos/${owner}/${name}`]), + "repository metadata", + ); +} + +function fetchBranchMetadata(owner, name, base) { + return parseJsonResponse( ghOk([ "api", - `repos/${owner}/${name}/branches/${encodeURIComponent(base)}/protection`, + `repos/${owner}/${name}/branches/${encodeURIComponent(base)}`, ]), + "branch metadata", ); } +function unprovenRulesCollection(collection, rulesBoundary) { + if (!collection || collection.complete === true) return collection; + const reason = `rules_plan_feature_unavailable_unproven:${rulesBoundary?.reason || "unknown"}`; + return { + ...collection, + error: collection.error ? `${collection.error}; ${reason}` : reason, + }; +} + function fetchBranchOid(owner, name, base) { const response = ghOk([ "api", @@ -886,11 +943,48 @@ try { ); const threads = reviewThreads(owner, name, pr); const branchProtection = fetchBranchProtection(owner, name, protectionBase); + let activeRulesEvidence = activeRules; + let branchProtectionEvidence = branchProtection; + let rulesBoundary = null; + if (activeRules.complete !== true) { + const repoMetadata = fetchRepoMetadata(owner, name); + const branchMetadata = fetchBranchMetadata(owner, name, protectionBase); + rulesBoundary = classifyRulesPlanFeatureUnavailable({ + repo: repoMetadata.payload, + branch: branchMetadata.payload, + policy, + probes: { + rulesets: ghOk(["api", `repos/${owner}/${name}/rulesets`]), + branchRules: activeRules.failure || null, + classicProtection: branchProtection.response || null, + }, + }); + if (rulesBoundary.unavailable === true) { + activeRulesEvidence = { + readable: true, + complete: true, + pages: 0, + rows: [], + error: null, + featureUnavailable: true, + }; + branchProtectionEvidence = { + required: false, + readable: true, + complete: true, + payload: null, + error: null, + featureUnavailable: true, + }; + } else { + activeRulesEvidence = unprovenRulesCollection(activeRules, rulesBoundary); + } + } const codeowners = fetchCodeowners(owner, name, protectionBase); const requiredChecks = normalizeRequiredChecks({ classicRequiredStatusChecks: - branchProtection?.payload?.required_status_checks || null, - activeRules: activeRules.rows || [], + branchProtectionEvidence?.payload?.required_status_checks || null, + activeRules: activeRulesEvidence.rows || [], }).descriptors; const workflowCoverage = scanTargetWorkflows( owner, @@ -904,10 +998,43 @@ try { ); const viewer = fetchViewer(); - const finalActiveRules = restCollection( + const finalActiveRulesRaw = restCollection( `repos/${owner}/${name}/rules/branches/${encodeURIComponent(protectionBase)}`, "final active rules", ); + let finalActiveRules = finalActiveRulesRaw; + let finalRulesBoundary = null; + if (rulesBoundary?.unavailable === true) { + const finalRepoMetadata = fetchRepoMetadata(owner, name); + const finalBranchMetadata = fetchBranchMetadata( + owner, + name, + protectionBase, + ); + finalRulesBoundary = classifyRulesPlanFeatureUnavailable({ + repo: finalRepoMetadata.payload, + branch: finalBranchMetadata.payload, + policy, + probes: { + rulesets: ghOk(["api", `repos/${owner}/${name}/rulesets`]), + branchRules: finalActiveRulesRaw.failure || null, + classicProtection: ghOk([ + "api", + `repos/${owner}/${name}/branches/${encodeURIComponent(protectionBase)}/protection`, + ]), + }, + }); + if (finalRulesBoundary.unavailable === true) { + finalActiveRules = { + readable: true, + complete: true, + pages: 0, + rows: [], + error: null, + featureUnavailable: true, + }; + } + } const finalBaseOid = fetchBranchOid(owner, name, base); const finalTestMergeOid = fetchTestMergeOid(owner, name, pr); const finalPrEvidence = ghJson([ @@ -922,8 +1049,12 @@ try { const boundary = verifySnapshotBoundary(prEvidence, finalPrEvidence, { initialBaseOid: baseOid, finalBaseOid, - initialRules: activeRules, + initialRules: activeRulesEvidence, finalRules: finalActiveRules, + initialRulesBoundary: + rulesBoundary?.unavailable === true ? rulesBoundary : null, + finalRulesBoundary: + rulesBoundary?.unavailable === true ? finalRulesBoundary : null, initialTestMergeOid: testMergeOid, finalTestMergeOid, }); @@ -931,19 +1062,30 @@ try { const capture = assembleSnapshotCapture({ prEvidence, changedFiles, - activeRules, + activeRules: activeRulesEvidence, checkRuns, statuses, issueComments, reviewComments, reviews, threads, - branchProtection, + branchProtection: branchProtectionEvidence, codeowners, policy, workflowCoverage, viewer, boundary, + rulesBoundary: rulesBoundary + ? { + mode: + rulesBoundary.unavailable === true + ? "plan_feature_unavailable" + : "unproven", + reason: + rulesBoundary.unavailable === true ? null : rulesBoundary.reason, + evidence: rulesBoundary.evidence, + } + : { mode: "rules_api_available", reason: null, evidence: null }, checkEvidence: { authoritative: { sha: selectedChecks.sha, diff --git a/tests/unit/rules-boundary.test.mjs b/tests/unit/rules-boundary.test.mjs new file mode 100644 index 00000000..3e0a0152 --- /dev/null +++ b/tests/unit/rules-boundary.test.mjs @@ -0,0 +1,609 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + RULES_BOUNDARY_EVIDENCE_KIND, + RULES_PLAN_MESSAGE, + classifyRulesPlanFeatureUnavailable, + parseGitHubApiFailure, + rulesBoundaryFingerprint, + validateRulesBoundaryEvidence, +} from "../../scripts/lib/rules-boundary.mjs"; +import { + snapshotBoundaryFingerprint, + verifySnapshotBoundary, +} from "../../scripts/lib/snapshot-capture-payload.mjs"; +import { evaluateRequiredChecksSnapshot } from "../../scripts/lib/snapshot-evaluators.mjs"; + +const DOCUMENTATION_URLS = { + rulesets: "https://docs.github.com/rest/repos/rules#get-all-repository-rulesets", + branchRules: "https://docs.github.com/rest/repos/rules#get-rules-for-a-branch", + classicProtection: + "https://docs.github.com/rest/branches/branch-protection#get-branch-protection", +}; + +function apiFailure({ + message = RULES_PLAN_MESSAGE, + documentationUrl = DOCUMENTATION_URLS.rulesets, + bodyStatus = 403, + httpStatus = 403, + errorMessage = message, + body = undefined, +} = {}) { + const payload = + body === undefined + ? { + message, + documentation_url: documentationUrl, + status: bodyStatus, + } + : body; + return { + ok: false, + body: typeof payload === "string" ? payload : JSON.stringify(payload), + error: `gh: ${errorMessage} (HTTP ${httpStatus})`, + }; +} + +function planGated(endpointKey) { + return apiFailure({ documentationUrl: DOCUMENTATION_URLS[endpointKey] }); +} + +function goodRepo(overrides = {}) { + return { private: true, visibility: "private", ...overrides }; +} + +function goodBranch(overrides = {}) { + return { + protected: false, + protection: { + enabled: false, + required_status_checks: { + checks: [], + contexts: [], + enforcement_level: "off", + }, + }, + ...overrides, + }; +} + +function goodPolicy(overrides = {}) { + return { + readable: true, + complete: true, + branchProtectionRules: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + ...overrides, + }; +} + +function classify({ + repo = goodRepo(), + branch = goodBranch(), + policy = goodPolicy(), + probes = {}, +} = {}) { + return classifyRulesPlanFeatureUnavailable({ + repo, + branch, + policy, + probes: { + rulesets: planGated("rulesets"), + branchRules: planGated("branchRules"), + classicProtection: planGated("classicProtection"), + ...probes, + }, + }); +} + +function collection(rows = []) { + return { readable: true, complete: true, pages: 1, rows, error: null }; +} + +const PR = { headRefOid: "head-a", baseRefName: "main" }; +const BASE = { initialBaseOid: "base-a", finalBaseOid: "base-a" }; + +test("accepts the documented plan gate only with complete corroborating evidence", () => { + const result = classify(); + assert.equal(result.determined, true); + assert.equal(result.unavailable, true); + assert.equal(result.reason, null); + assert.equal(result.evidence.kind, RULES_BOUNDARY_EVIDENCE_KIND); + assert.equal(result.evidence.message, RULES_PLAN_MESSAGE); + assert.deepEqual(validateRulesBoundaryEvidence(result.evidence), { + valid: true, + reasons: [], + }); + assert.equal( + rulesBoundaryFingerprint(result.evidence), + rulesBoundaryFingerprint(structuredClone(result.evidence)), + ); +}); + +test("rejects a permission-denied 403 even when another probe is plan-gated", () => { + const result = classify({ + probes: { + branchRules: apiFailure({ + message: "Resource not accessible by integration", + documentationUrl: DOCUMENTATION_URLS.branchRules, + }), + }, + }); + assert.equal(result.unavailable, false); + assert.ok(result.reasons.includes("branchRules_message_not_plan_gate")); +}); + +test("rejects transient and server failures", () => { + for (const probe of [ + apiFailure({ + message: "Server Error", + documentationUrl: DOCUMENTATION_URLS.rulesets, + bodyStatus: 500, + httpStatus: 500, + }), + apiFailure({ + message: "Service unavailable", + documentationUrl: DOCUMENTATION_URLS.rulesets, + bodyStatus: 503, + httpStatus: 503, + }), + ]) { + const result = classify({ probes: { rulesets: probe } }); + assert.equal(result.unavailable, false); + assert.match(result.reasons[0], /rulesets_http_5/); + } +}); + +test("rejects ambiguous mixed probe outcomes", () => { + for (const probe of [ + { ok: true, body: "[]", error: null }, + apiFailure({ + message: "Not Found", + documentationUrl: DOCUMENTATION_URLS.rulesets, + bodyStatus: 404, + httpStatus: 404, + }), + ]) { + const result = classify({ probes: { rulesets: probe } }); + assert.equal(result.unavailable, false); + assert.ok( + result.reasons.includes("rulesets_probe_succeeded") || + result.reasons.includes("rulesets_http_404"), + ); + } +}); + +test("rejects message-spoof variants", () => { + const variants = [ + { message: `${RULES_PLAN_MESSAGE} (see docs)` }, + { message: `Warning: ${RULES_PLAN_MESSAGE}` }, + { message: RULES_PLAN_MESSAGE.toLowerCase() }, + { message: RULES_PLAN_MESSAGE.replace(/\.$/, "") }, + { message: RULES_PLAN_MESSAGE.replace("repository", "repo") }, + ]; + for (const variant of variants) { + const result = classify({ + probes: { + rulesets: apiFailure({ + ...variant, + documentationUrl: DOCUMENTATION_URLS.rulesets, + }), + }, + }); + assert.equal(result.unavailable, false, JSON.stringify(variant)); + assert.ok(result.reasons.includes("rulesets_message_not_plan_gate")); + } + + const documentationVariants = [ + `${DOCUMENTATION_URLS.rulesets}?mine=1`, + "https://docs.github.com/rest/repos/rules#get-rules-for-a-branch", + "https://docs.github.com/rest/repos/rules", + ]; + for (const documentationUrl of documentationVariants) { + const result = classify({ + probes: { rulesets: apiFailure({ documentationUrl }) }, + }); + assert.equal(result.unavailable, false, documentationUrl); + assert.ok(result.reasons.includes("rulesets_documentation_url_mismatch")); + } +}); + +test("rejects inconsistent or malformed HTTP failure payloads", () => { + const bodyStatusMismatch = apiFailure({ + documentationUrl: DOCUMENTATION_URLS.rulesets, + bodyStatus: 500, + httpStatus: 403, + }); + assert.equal( + parseGitHubApiFailure(bodyStatusMismatch).reason, + "error_payload_status_mismatch", + ); + assert.equal( + classify({ probes: { rulesets: bodyStatusMismatch } }).unavailable, + false, + ); + + const duplicateStatus = { + ok: false, + body: JSON.stringify({ + message: RULES_PLAN_MESSAGE, + documentation_url: DOCUMENTATION_URLS.rulesets, + status: 403, + }), + error: `gh: ${RULES_PLAN_MESSAGE} (HTTP 403) (HTTP 500)`, + }; + assert.equal( + parseGitHubApiFailure(duplicateStatus).reason, + "http_status_ambiguous", + ); + + const textMessageMismatch = { + ok: false, + body: JSON.stringify({ + message: RULES_PLAN_MESSAGE, + documentation_url: DOCUMENTATION_URLS.rulesets, + status: 403, + }), + error: "gh: Upgrade to GitHub Pro to use repository rules (HTTP 403)", + }; + assert.equal( + parseGitHubApiFailure(textMessageMismatch).reason, + "error_text_message_mismatch", + ); + + for (const body of ["not-json", "[]", "null"]) { + const response = { + ok: false, + body, + error: `gh: ${RULES_PLAN_MESSAGE} (HTTP 403)`, + }; + assert.equal( + classify({ probes: { rulesets: response } }).unavailable, + false, + ); + } +}); + +test("rejects a plan-gated response when the repository is not confirmed private", () => { + for (const repo of [ + goodRepo({ private: false, visibility: "public" }), + goodRepo({ visibility: "internal" }), + goodRepo({ private: "true" }), + null, + ]) { + const result = classify({ repo }); + assert.equal(result.unavailable, false); + assert.ok(result.reasons.includes("repository_not_confirmed_private")); + } +}); + +test("rejects any branch state that could still enforce classic protection", () => { + const variants = [ + goodBranch({ protected: true }), + { + protected: false, + protection: { + enabled: true, + required_status_checks: { + checks: [], + contexts: [], + enforcement_level: "off", + }, + }, + }, + { + protected: false, + protection: { + enabled: false, + required_status_checks: { + checks: [], + contexts: [], + enforcement_level: "all", + }, + }, + }, + { + protected: false, + protection: { + enabled: false, + required_status_checks: { + checks: ["CI"], + contexts: [], + enforcement_level: "off", + }, + }, + }, + { + protected: false, + protection: { + enabled: false, + required_status_checks: { + checks: [], + contexts: ["CI"], + enforcement_level: "off", + }, + }, + }, + { protected: false }, + ]; + for (const branch of variants) { + const result = classify({ branch }); + assert.equal(result.unavailable, false, JSON.stringify(branch)); + assert.ok(result.reasons.includes("branch_not_confirmed_unprotected")); + } +}); + +test("rejects policy evidence that still contains a classic rule", () => { + const variants = [ + goodPolicy({ + branchProtectionRules: { + pageInfo: { hasNextPage: false }, + nodes: [{ pattern: "main" }], + }, + }), + goodPolicy({ complete: false }), + goodPolicy({ readable: false }), + goodPolicy({ + branchProtectionRules: { + pageInfo: { hasNextPage: true }, + nodes: [], + }, + }), + ]; + for (const policy of variants) { + const result = classify({ policy }); + assert.equal(result.unavailable, false); + assert.ok(result.reasons.includes("classic_rules_not_confirmed_empty")); + } +}); + +test("boundary evidence validation rejects tampered fields", () => { + const evidence = classify().evidence; + const tampered = [ + { ...evidence, message: `${RULES_PLAN_MESSAGE} extra` }, + { ...evidence, repo: { private: true, visibility: "public" } }, + { + ...evidence, + branch: { ...evidence.branch, requiredStatusCheckCount: 1 }, + }, + { ...evidence, policy: { ...evidence.policy, branchProtectionRuleCount: 1 } }, + { + ...evidence, + endpoints: { + ...evidence.endpoints, + rulesets: { status: 403, documentationUrl: "https://example.test/rules" }, + }, + }, + ]; + for (const value of tampered) { + assert.equal(validateRulesBoundaryEvidence(value).valid, false); + } + assert.equal(validateRulesBoundaryEvidence(null).valid, false); +}); + +test("snapshot boundary accepts one unchanged plan-feature boundary", () => { + const boundary = classify(); + const result = verifySnapshotBoundary(PR, PR, { + ...BASE, + initialRules: collection(), + finalRules: collection(), + initialRulesBoundary: boundary, + finalRulesBoundary: boundary, + }); + assert.equal(result.rulesFingerprint, snapshotBoundaryFingerprint([])); + assert.equal( + result.rulesBoundaryFingerprint, + rulesBoundaryFingerprint(boundary.evidence), + ); +}); + +test("snapshot boundary refuses a boundary that stopped being proven", () => { + const boundary = classify(); + const degraded = classify({ repo: goodRepo({ private: false, visibility: "public" }) }); + assert.throws( + () => + verifySnapshotBoundary(PR, PR, { + ...BASE, + initialRules: collection(), + finalRules: collection(), + initialRulesBoundary: boundary, + finalRulesBoundary: degraded, + }), + /snapshot_rules_boundary_incomplete/, + ); + assert.throws( + () => + verifySnapshotBoundary(PR, PR, { + ...BASE, + initialRules: collection(), + finalRules: collection(), + initialRulesBoundary: boundary, + finalRulesBoundary: null, + }), + /snapshot_rules_boundary_incomplete/, + ); +}); + +test("snapshot boundary rejects changed boundary evidence", () => { + const boundary = classify(); + const changed = { + ...boundary, + evidence: { ...boundary.evidence, observerNote: "changed" }, + }; + assert.equal(validateRulesBoundaryEvidence(changed.evidence).valid, true); + assert.throws( + () => + verifySnapshotBoundary(PR, PR, { + ...BASE, + initialRules: collection(), + finalRules: collection(), + initialRulesBoundary: boundary, + finalRulesBoundary: changed, + }), + /snapshot_rules_moved:rules_boundary_evidence/, + ); +}); + +function source(required = true) { + return { required, readable: true, complete: true, error: null }; +} + +function evaluatorSnapshot() { + return { + schemaVersion: 1, + kind: "github-delivery/evidence-snapshot", + snapshotId: "snap", + capturedAt: "2026-11-08T03:00:00.000Z", + repo: "Wibias/streamer-content", + pr: 1, + headOid: "head-a", + complete: true, + incompleteReasons: [], + sources: { + pr: source(), + changedFiles: source(), + activeRules: source(), + checkRuns: source(), + statuses: source(), + issueComments: source(), + reviewComments: source(), + reviews: source(), + reviewThreads: source(), + branchProtection: source(false), + codeowners: source(false), + codeownersErrors: source(false), + policyGraphql: source(), + workflowCoverage: source(), + viewer: source(), + }, + evidence: { + pullRequest: { + number: 1, + url: "https://github.com/Wibias/streamer-content/pull/1", + baseRefName: "main", + headRefOid: "head-a", + mergeStateStatus: "CLEAN", + mergeable: "MERGEABLE", + reviewDecision: "APPROVED", + isDraft: false, + reviewRequests: [], + commits: [], + }, + changedFiles: [], + branchProtection: null, + activeRules: [], + checks: { + checkRuns: [], + statuses: [], + authoritative: { + sha: "head-a", + reason: "head_only", + complete: true, + incompleteReasons: [], + }, + }, + feedback: { + issueComments: [], + reviewComments: [], + reviews: [], + reviewThreads: [], + }, + codeowners: { path: null, text: null, errors: [] }, + policy: { + branchProtectionRules: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + latestOpinionatedReviews: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + mergeQueue: { enabled: false, inQueue: false, entry: null }, + }, + workflowCoverage: { + complete: true, + scannedRef: "main", + workflowFiles: 0, + hasPullRequestTrigger: null, + hasMergeGroupTrigger: null, + warning: null, + }, + viewer: { login: "Wibias" }, + }, + }; +} + +function featureUnavailableSnapshot() { + const boundary = classify(); + const value = evaluatorSnapshot(); + value.sources.activeRules = { + ...source(), + featureUnavailable: true, + }; + value.sources.branchProtection = { + ...source(false), + featureUnavailable: true, + }; + value.evidence.rulesBoundary = { + mode: "plan_feature_unavailable", + reason: null, + evidence: boundary.evidence, + }; + return value; +} + +test("required checks stay ready with valid plan-feature evidence", () => { + const result = evaluateRequiredChecksSnapshot(featureUnavailableSnapshot()); + assert.equal(result.complete, true); + assert.equal(result.decision, "ready"); +}); + +test("feature-unavailable evidence without a valid boundary stays unknown", () => { + for (const evidence of [ + null, + { mode: "plan_feature_unavailable", evidence: null }, + { + mode: "plan_feature_unavailable", + evidence: { + ...classify().evidence, + endpoints: { + ...classify().evidence.endpoints, + branchRules: { + status: 403, + documentationUrl: "https://docs.github.com/rest/repos/rules#wrong", + }, + }, + }, + }, + ]) { + const value = featureUnavailableSnapshot(); + value.evidence.rulesBoundary = evidence; + const result = evaluateRequiredChecksSnapshot(value); + assert.equal(result.decision, "unknown"); + assert.ok(result.unknowns.includes("active_rules_incomplete")); + } +}); + +test("existing rules and required checks are still enforced", () => { + const value = evaluatorSnapshot(); + value.evidence.activeRules = [ + { + type: "required_status_checks", + parameters: { + required_status_checks: [{ context: "CI", integration_id: 7 }], + }, + }, + ]; + const result = evaluateRequiredChecksSnapshot(value); + assert.equal(result.decision, "blocked"); + assert.ok(result.blockers.includes("missing:CI@7")); + + const inconsistent = featureUnavailableSnapshot(); + inconsistent.evidence.activeRules = value.evidence.activeRules; + const enforced = evaluateRequiredChecksSnapshot(inconsistent); + assert.equal(enforced.decision, "blocked"); + assert.ok(enforced.blockers.includes("missing:CI@7")); +});