Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions LifeOS/install/LIFEOS/ERRATA/Errata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@ import { main as evalMain } from "./Eval";
import { act, dailyV4 } from "./Act";
import { formatLanded, landedImpacts } from "./Landed";
import { errataReport, summarize } from "./Report";
import { isV4Outage } from "./V4Outage";
import { complaintRows, falsePasses, fileAssayMissUpgrades, formatImpact, formatMisses, impact, loadSources, retireCandidates, skillRows, skillRowsMarkdown } from "./OutputJoin";

type DailyDependencies = {
mine?: (store: ErrataStore, opts: { limitFiles?: number }) => ReturnType<typeof mine>;
enrich?: (store: ErrataStore) => Promise<EnrichResult>;
label?: (store: ErrataStore, opts: LabelOptions) => ReturnType<typeof label>;
triage?: typeof triage;
/** Errata v4 detection over new exchanges (ISC-51); shadow, so it never changes the exit code. */
/** Errata v4 detection over new exchanges (ISC-51); shadow, so single errors are non-fatal, but engine outages (>50% errors) or thrown exceptions fail the stage (#2789). */
v4?: () => Promise<unknown>;
/** Errata's daily state: roll-up event, landed-fix check, state file (Report.ts). */
report?: (store: ErrataStore) => unknown;
Expand Down Expand Up @@ -123,11 +124,19 @@ export async function daily(
failures++; failed.push(`report threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`);
say("report", stageError("daily report", error));
}
// v4 runs last and in shadow: it reads the ledger, never writes it, and its failure never
// changes the exit code or delays the v3 stages above (ISC-49, ISC-51).
// v4 runs last and in shadow: single errors are non-fatal, but a thrown error or an engine outage
// (>= 50% errors on detect or diagnose candidates) fails the stage so Vigil raises an alert (public issue #2789).
try {
say("v4", await (deps.v4 ?? (() => dailyV4()))());
const result = await (deps.v4 ?? (() => dailyV4()))();
say("v4", result);
const outage = isV4Outage(result);
if (outage.failed) {
failures++;
failed.push(`v4: ${outage.reason}`);
}
} catch (error) {
failures++;
failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`);
say("v4", stageError("daily v4", error));
}
if (failed.length) console.error(`[errata daily] failed stages: ${failed.join("; ")}`);
Expand Down Expand Up @@ -156,8 +165,16 @@ export async function weekly(store: ErrataStore, deps: WeeklyDependencies = {},
}
// Same isolation as daily: v4 is shadow and never files to Upgrades from the schedule.
try {
say("v4", await (deps.v4 ?? (() => act({ file: false })))());
const result = await (deps.v4 ?? (() => act({ file: false })))();
say("v4", result);
const outage = isV4Outage(result);
if (outage.failed) {
failures++;
failed.push(`v4: ${outage.reason}`);
}
} catch (error) {
failures++;
failed.push(`v4 threw: ${String((error as Error)?.message ?? error).slice(0, 160)}`);
say("v4", stageError("weekly v4", error));
}
if (failed.length) console.error(`[errata weekly] failed stages: ${failed.join("; ")}`);
Expand Down
47 changes: 47 additions & 0 deletions LifeOS/install/LIFEOS/ERRATA/V4Outage.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/**
* Tests for V4Outage error detection.
* Validates that single non-fatal errors are tolerated while engine outages
* (>= 50% errors on candidates) and fatal errors fail the stage.
*
* Issue #2789
*/
import { describe, expect, test } from "bun:test";
import { isV4Outage } from "./V4Outage";

describe("isV4Outage error detection (#2789)", () => {
test("clean run with no errors is not an outage", () => {
const res = { detect: { candidates: 30, written: 30, errors: 0 }, diagnose: { candidates: 20, errors: 0 }, errors: [] };
expect(isV4Outage(res).failed).toBe(false);
});

test("single non-fatal errors (< 50%) are tolerated", () => {
const res = { detect: { candidates: 30, errors: 2 }, diagnose: { candidates: 20, errors: 1 }, errors: [] };
expect(isV4Outage(res).failed).toBe(false);
});

test("detect engine total outage (e.g. 30/30 errors) triggers failure", () => {
const res = { detect: { candidates: 30, written: 30, errors: 30 }, diagnose: { candidates: 0, errors: 0 }, errors: [] };
const outage = isV4Outage(res);
expect(outage.failed).toBe(true);
expect(outage.reason).toContain("detect outage: 30/30 candidate errors");
});

test("diagnose engine outage (>= 50% errors) triggers failure", () => {
const res = { detect: { candidates: 30, errors: 0 }, diagnose: { candidates: 10, errors: 5 }, errors: [] };
const outage = isV4Outage(res);
expect(outage.failed).toBe(true);
expect(outage.reason).toContain("diagnose outage: 5/10 candidate errors");
});

test("explicit error entries trigger failure", () => {
const res = { errors: ["api.cloudflare.com timeout"] };
const outage = isV4Outage(res);
expect(outage.failed).toBe(true);
expect(outage.reason).toBe("api.cloudflare.com timeout");
});

test("zero candidates (nothing to process) is not an outage", () => {
const res = { detect: { candidates: 0, errors: 0 }, diagnose: { candidates: 0, errors: 0 }, errors: [] };
expect(isV4Outage(res).failed).toBe(false);
});
});
61 changes: 61 additions & 0 deletions LifeOS/install/LIFEOS/ERRATA/V4Outage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* V4Outage — evaluates whether an Errata v4 pass suffered an engine outage or failure.
*
* Single errors remain non-fatal for shadow stages, but when detect or diagnose fails
* on at least half of its candidates, or when unhandled errors are returned, the stage
* is flagged as failed so that scheduled jobs exit non-zero and raise a Vigil alert.
*
* Public issue #2789.
*/

export interface V4StageResult {
errors?: string[];
detect?: { candidates?: number; written?: number; errors?: number };
diagnose?: { candidates?: number; written?: number; errors?: number };
}

export interface V4OutageCheck {
failed: boolean;
reason?: string;
}

/**
* Checks whether the v4 stage result represents an engine outage or fatal error.
* Tolerates single non-fatal errors; flags outages when errors >= 50% of candidates.
*/
export function isV4Outage(result: unknown): V4OutageCheck {
if (!result || typeof result !== "object") return { failed: false };
const r = result as V4StageResult;

if (Array.isArray(r.errors) && r.errors.length > 0) {
return { failed: true, reason: r.errors.join("; ") };
}

const detect = r.detect;
if (
typeof detect?.candidates === "number" &&
detect.candidates > 0 &&
typeof detect?.errors === "number" &&
detect.errors >= Math.ceil(detect.candidates / 2)
) {
return {
failed: true,
reason: `detect outage: ${detect.errors}/${detect.candidates} candidate errors`,
};
}

const diagnose = r.diagnose;
if (
typeof diagnose?.candidates === "number" &&
diagnose.candidates > 0 &&
typeof diagnose?.errors === "number" &&
diagnose.errors >= Math.ceil(diagnose.candidates / 2)
) {
return {
failed: true,
reason: `diagnose outage: ${diagnose.errors}/${diagnose.candidates} candidate errors`,
};
}

return { failed: false };
}