diff --git a/LifeOS/install/LIFEOS/ERRATA/Errata.ts b/LifeOS/install/LIFEOS/ERRATA/Errata.ts index a577e99f32..e9ff5cca82 100644 --- a/LifeOS/install/LIFEOS/ERRATA/Errata.ts +++ b/LifeOS/install/LIFEOS/ERRATA/Errata.ts @@ -23,6 +23,7 @@ import { main as evalMain } from "./Eval"; import { act, dailyV4 } from "./Act"; import { formatLanded, landedImpacts } from "./Landed"; import { errataReport, summarize } from "./Report"; +import { isV4Outage } from "./V4Outage"; import { complaintRows, falsePasses, fileAssayMissUpgrades, formatImpact, formatMisses, impact, loadSources, retireCandidates, skillRows, skillRowsMarkdown } from "./OutputJoin"; type DailyDependencies = { @@ -30,7 +31,7 @@ type DailyDependencies = { enrich?: (store: ErrataStore) => Promise; label?: (store: ErrataStore, opts: LabelOptions) => ReturnType; triage?: typeof triage; - /** Errata v4 detection over new exchanges (ISC-51); shadow, so it never changes the exit code. */ + /** Errata v4 detection over new exchanges (ISC-51); shadow, so single errors are non-fatal, but engine outages (>50% errors) or thrown exceptions fail the stage (#2789). */ v4?: () => Promise; /** Errata's daily state: roll-up event, landed-fix check, state file (Report.ts). */ report?: (store: ErrataStore) => unknown; @@ -123,11 +124,19 @@ export async function daily( failures++; failed.push(`report threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("report", stageError("daily report", error)); } - // v4 runs last and in shadow: it reads the ledger, never writes it, and its failure never - // changes the exit code or delays the v3 stages above (ISC-49, ISC-51). + // v4 runs last and in shadow: single errors are non-fatal, but a thrown error or an engine outage + // (>= 50% errors on detect or diagnose candidates) fails the stage so Vigil raises an alert (public issue #2789). try { - say("v4", await (deps.v4 ?? (() => dailyV4()))()); + const result = await (deps.v4 ?? (() => dailyV4()))(); + say("v4", result); + const outage = isV4Outage(result); + if (outage.failed) { + failures++; + failed.push(`v4: ${outage.reason}`); + } } catch (error) { + failures++; + failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("v4", stageError("daily v4", error)); } if (failed.length) console.error(`[errata daily] failed stages: ${failed.join("; ")}`); @@ -156,8 +165,16 @@ export async function weekly(store: ErrataStore, deps: WeeklyDependencies = {}, } // Same isolation as daily: v4 is shadow and never files to Upgrades from the schedule. try { - say("v4", await (deps.v4 ?? (() => act({ file: false })))()); + const result = await (deps.v4 ?? (() => act({ file: false })))(); + say("v4", result); + const outage = isV4Outage(result); + if (outage.failed) { + failures++; + failed.push(`v4: ${outage.reason}`); + } } catch (error) { + failures++; + failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("v4", stageError("weekly v4", error)); } if (failed.length) console.error(`[errata weekly] failed stages: ${failed.join("; ")}`); diff --git a/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts b/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts new file mode 100644 index 0000000000..7c5192f4f4 --- /dev/null +++ b/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts @@ -0,0 +1,47 @@ +/** + * Tests for V4Outage error detection. + * Validates that single non-fatal errors are tolerated while engine outages + * (>= 50% errors on candidates) and fatal errors fail the stage. + * + * Issue #2789 + */ +import { describe, expect, test } from "bun:test"; +import { isV4Outage } from "./V4Outage"; + +describe("isV4Outage error detection (#2789)", () => { + test("clean run with no errors is not an outage", () => { + const res = { detect: { candidates: 30, written: 30, errors: 0 }, diagnose: { candidates: 20, errors: 0 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); + + test("single non-fatal errors (< 50%) are tolerated", () => { + const res = { detect: { candidates: 30, errors: 2 }, diagnose: { candidates: 20, errors: 1 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); + + test("detect engine total outage (e.g. 30/30 errors) triggers failure", () => { + const res = { detect: { candidates: 30, written: 30, errors: 30 }, diagnose: { candidates: 0, errors: 0 }, errors: [] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toContain("detect outage: 30/30 candidate errors"); + }); + + test("diagnose engine outage (>= 50% errors) triggers failure", () => { + const res = { detect: { candidates: 30, errors: 0 }, diagnose: { candidates: 10, errors: 5 }, errors: [] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toContain("diagnose outage: 5/10 candidate errors"); + }); + + test("explicit error entries trigger failure", () => { + const res = { errors: ["api.cloudflare.com timeout"] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toBe("api.cloudflare.com timeout"); + }); + + test("zero candidates (nothing to process) is not an outage", () => { + const res = { detect: { candidates: 0, errors: 0 }, diagnose: { candidates: 0, errors: 0 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); +}); diff --git a/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts b/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts new file mode 100644 index 0000000000..db13daf9c4 --- /dev/null +++ b/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts @@ -0,0 +1,61 @@ +/** + * V4Outage — evaluates whether an Errata v4 pass suffered an engine outage or failure. + * + * Single errors remain non-fatal for shadow stages, but when detect or diagnose fails + * on at least half of its candidates, or when unhandled errors are returned, the stage + * is flagged as failed so that scheduled jobs exit non-zero and raise a Vigil alert. + * + * Public issue #2789. + */ + +export interface V4StageResult { + errors?: string[]; + detect?: { candidates?: number; written?: number; errors?: number }; + diagnose?: { candidates?: number; written?: number; errors?: number }; +} + +export interface V4OutageCheck { + failed: boolean; + reason?: string; +} + +/** + * Checks whether the v4 stage result represents an engine outage or fatal error. + * Tolerates single non-fatal errors; flags outages when errors >= 50% of candidates. + */ +export function isV4Outage(result: unknown): V4OutageCheck { + if (!result || typeof result !== "object") return { failed: false }; + const r = result as V4StageResult; + + if (Array.isArray(r.errors) && r.errors.length > 0) { + return { failed: true, reason: r.errors.join("; ") }; + } + + const detect = r.detect; + if ( + typeof detect?.candidates === "number" && + detect.candidates > 0 && + typeof detect?.errors === "number" && + detect.errors >= Math.ceil(detect.candidates / 2) + ) { + return { + failed: true, + reason: `detect outage: ${detect.errors}/${detect.candidates} candidate errors`, + }; + } + + const diagnose = r.diagnose; + if ( + typeof diagnose?.candidates === "number" && + diagnose.candidates > 0 && + typeof diagnose?.errors === "number" && + diagnose.errors >= Math.ceil(diagnose.candidates / 2) + ) { + return { + failed: true, + reason: `diagnose outage: ${diagnose.errors}/${diagnose.candidates} candidate errors`, + }; + } + + return { failed: false }; +}