From db765fb451051b68952b0c6a9745e75133fcbbe3 Mon Sep 17 00:00:00 2001 From: isihussain <35667459+isihussain@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:27:19 +0300 Subject: [PATCH] fix(errata): fail daily and weekly runs when v4 engine experiences an outage (#2789) Errata daily previously caught v4 errors without incrementing failures, meaning that if every call to the v4 engine timed out or failed (such as an api.cloudflare.com network outage), errata daily still exited with status 0. This caused the scheduled run to be recorded as successful and silenced the normal Vigil alert. Single occasional errors remain non-fatal for shadow stages, but when detect or diagnose errors on at least 50% of candidates or when the stage throws, mark the stage as failed and exit non-zero so Vigil raises an alert. Add V4Outage.ts and V4Outage.test.ts covering non-fatal vs outage thresholds. Closes #2789 Signed-off-by: isihussain --- LifeOS/install/LIFEOS/ERRATA/Errata.ts | 27 ++++++-- LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts | 47 ++++++++++++++ LifeOS/install/LIFEOS/ERRATA/V4Outage.ts | 61 +++++++++++++++++++ 3 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts create mode 100644 LifeOS/install/LIFEOS/ERRATA/V4Outage.ts diff --git a/LifeOS/install/LIFEOS/ERRATA/Errata.ts b/LifeOS/install/LIFEOS/ERRATA/Errata.ts index a577e99f32..e9ff5cca82 100644 --- a/LifeOS/install/LIFEOS/ERRATA/Errata.ts +++ b/LifeOS/install/LIFEOS/ERRATA/Errata.ts @@ -23,6 +23,7 @@ import { main as evalMain } from "./Eval"; import { act, dailyV4 } from "./Act"; import { formatLanded, landedImpacts } from "./Landed"; import { errataReport, summarize } from "./Report"; +import { isV4Outage } from "./V4Outage"; import { complaintRows, falsePasses, fileAssayMissUpgrades, formatImpact, formatMisses, impact, loadSources, retireCandidates, skillRows, skillRowsMarkdown } from "./OutputJoin"; type DailyDependencies = { @@ -30,7 +31,7 @@ type DailyDependencies = { enrich?: (store: ErrataStore) => Promise; label?: (store: ErrataStore, opts: LabelOptions) => ReturnType; triage?: typeof triage; - /** Errata v4 detection over new exchanges (ISC-51); shadow, so it never changes the exit code. */ + /** Errata v4 detection over new exchanges (ISC-51); shadow, so single errors are non-fatal, but engine outages (>50% errors) or thrown exceptions fail the stage (#2789). */ v4?: () => Promise; /** Errata's daily state: roll-up event, landed-fix check, state file (Report.ts). */ report?: (store: ErrataStore) => unknown; @@ -123,11 +124,19 @@ export async function daily( failures++; failed.push(`report threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("report", stageError("daily report", error)); } - // v4 runs last and in shadow: it reads the ledger, never writes it, and its failure never - // changes the exit code or delays the v3 stages above (ISC-49, ISC-51). + // v4 runs last and in shadow: single errors are non-fatal, but a thrown error or an engine outage + // (>= 50% errors on detect or diagnose candidates) fails the stage so Vigil raises an alert (public issue #2789). try { - say("v4", await (deps.v4 ?? (() => dailyV4()))()); + const result = await (deps.v4 ?? (() => dailyV4()))(); + say("v4", result); + const outage = isV4Outage(result); + if (outage.failed) { + failures++; + failed.push(`v4: ${outage.reason}`); + } } catch (error) { + failures++; + failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("v4", stageError("daily v4", error)); } if (failed.length) console.error(`[errata daily] failed stages: ${failed.join("; ")}`); @@ -156,8 +165,16 @@ export async function weekly(store: ErrataStore, deps: WeeklyDependencies = {}, } // Same isolation as daily: v4 is shadow and never files to Upgrades from the schedule. try { - say("v4", await (deps.v4 ?? (() => act({ file: false })))()); + const result = await (deps.v4 ?? (() => act({ file: false })))(); + say("v4", result); + const outage = isV4Outage(result); + if (outage.failed) { + failures++; + failed.push(`v4: ${outage.reason}`); + } } catch (error) { + failures++; + failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`); say("v4", stageError("weekly v4", error)); } if (failed.length) console.error(`[errata weekly] failed stages: ${failed.join("; ")}`); diff --git a/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts b/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts new file mode 100644 index 0000000000..7c5192f4f4 --- /dev/null +++ b/LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts @@ -0,0 +1,47 @@ +/** + * Tests for V4Outage error detection. + * Validates that single non-fatal errors are tolerated while engine outages + * (>= 50% errors on candidates) and fatal errors fail the stage. + * + * Issue #2789 + */ +import { describe, expect, test } from "bun:test"; +import { isV4Outage } from "./V4Outage"; + +describe("isV4Outage error detection (#2789)", () => { + test("clean run with no errors is not an outage", () => { + const res = { detect: { candidates: 30, written: 30, errors: 0 }, diagnose: { candidates: 20, errors: 0 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); + + test("single non-fatal errors (< 50%) are tolerated", () => { + const res = { detect: { candidates: 30, errors: 2 }, diagnose: { candidates: 20, errors: 1 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); + + test("detect engine total outage (e.g. 30/30 errors) triggers failure", () => { + const res = { detect: { candidates: 30, written: 30, errors: 30 }, diagnose: { candidates: 0, errors: 0 }, errors: [] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toContain("detect outage: 30/30 candidate errors"); + }); + + test("diagnose engine outage (>= 50% errors) triggers failure", () => { + const res = { detect: { candidates: 30, errors: 0 }, diagnose: { candidates: 10, errors: 5 }, errors: [] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toContain("diagnose outage: 5/10 candidate errors"); + }); + + test("explicit error entries trigger failure", () => { + const res = { errors: ["api.cloudflare.com timeout"] }; + const outage = isV4Outage(res); + expect(outage.failed).toBe(true); + expect(outage.reason).toBe("api.cloudflare.com timeout"); + }); + + test("zero candidates (nothing to process) is not an outage", () => { + const res = { detect: { candidates: 0, errors: 0 }, diagnose: { candidates: 0, errors: 0 }, errors: [] }; + expect(isV4Outage(res).failed).toBe(false); + }); +}); diff --git a/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts b/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts new file mode 100644 index 0000000000..db13daf9c4 --- /dev/null +++ b/LifeOS/install/LIFEOS/ERRATA/V4Outage.ts @@ -0,0 +1,61 @@ +/** + * V4Outage — evaluates whether an Errata v4 pass suffered an engine outage or failure. + * + * Single errors remain non-fatal for shadow stages, but when detect or diagnose fails + * on at least half of its candidates, or when unhandled errors are returned, the stage + * is flagged as failed so that scheduled jobs exit non-zero and raise a Vigil alert. + * + * Public issue #2789. + */ + +export interface V4StageResult { + errors?: string[]; + detect?: { candidates?: number; written?: number; errors?: number }; + diagnose?: { candidates?: number; written?: number; errors?: number }; +} + +export interface V4OutageCheck { + failed: boolean; + reason?: string; +} + +/** + * Checks whether the v4 stage result represents an engine outage or fatal error. + * Tolerates single non-fatal errors; flags outages when errors >= 50% of candidates. + */ +export function isV4Outage(result: unknown): V4OutageCheck { + if (!result || typeof result !== "object") return { failed: false }; + const r = result as V4StageResult; + + if (Array.isArray(r.errors) && r.errors.length > 0) { + return { failed: true, reason: r.errors.join("; ") }; + } + + const detect = r.detect; + if ( + typeof detect?.candidates === "number" && + detect.candidates > 0 && + typeof detect?.errors === "number" && + detect.errors >= Math.ceil(detect.candidates / 2) + ) { + return { + failed: true, + reason: `detect outage: ${detect.errors}/${detect.candidates} candidate errors`, + }; + } + + const diagnose = r.diagnose; + if ( + typeof diagnose?.candidates === "number" && + diagnose.candidates > 0 && + typeof diagnose?.errors === "number" && + diagnose.errors >= Math.ceil(diagnose.candidates / 2) + ) { + return { + failed: true, + reason: `diagnose outage: ${diagnose.errors}/${diagnose.candidates} candidate errors`, + }; + } + + return { failed: false }; +}