From 0585bf1042e2e441d56c7e2a67543b4717af9a0b Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:46:53 -0800 Subject: [PATCH 1/7] perf(api): bound check-alerts history lookup by alert interval getPreviousAlertHistories scanned a fixed 7-day window of AlertHistory per alert on every tick and reduced it with a $sort + $group aggregation that MongoDB does not short-circuit per group, so cost grew with history volume. - Look up each alert's latest history row first and skip the history scan for alerts already evaluated in the current window. - Seed non-grouped alerts' previous state from that row; they never scan. - Grouped alerts scan a window of 52 intervals clamped to [1d, 7d] with a streamed find on {alert, createdAt}, reducing to the latest row per group in code. - getConsecutiveWindowHistories uses find + in-app sort for the same reason. - A failing evaluator lookup during the skip check marks the alert as due instead of failing the tick. Ported from the enterprise repo. Fixes #2434 --- .changeset/alert-history-lookback.md | 10 + packages/api/src/models/alertHistory.ts | 6 +- .../__tests__/checkAlerts.int.test.ts | 271 +++++++++++++++--- .../checkAlerts/__tests__/evaluators.test.ts | 16 +- packages/api/src/tasks/checkAlerts/index.ts | 265 +++++++++++------ .../providers/__tests__/default.int.test.ts | 183 +++++++++++- .../tasks/checkAlerts/providers/default.ts | 169 ++++++++--- packages/api/src/tasks/util.ts | 2 +- 8 files changed, 755 insertions(+), 167 deletions(-) create mode 100644 .changeset/alert-history-lookback.md diff --git a/.changeset/alert-history-lookback.md b/.changeset/alert-history-lookback.md new file mode 100644 index 000000000..7127a6158 --- /dev/null +++ b/.changeset/alert-history-lookback.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': patch +--- + +perf: bound the check-alerts previous-history lookup by the alert interval + +The check-alerts task no longer scans 7 days of alert history per alert on +every tick. A one-row lookup per alert skips alerts already evaluated in the +current window. Non-grouped alerts take their previous state from that row, +and grouped alerts scan a window sized to their interval (1 to 7 days). diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 53712c30a..d9330f970 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -169,10 +169,12 @@ AlertHistorySchema.index( { expireAfterSeconds: ms('30d') / 1000 }, ); -// Used by getRecentAlertHistories (matches on alert, sorts by createdAt) +// Serves every alert-history read: each matches on alert with a createdAt +// range or sort. AlertHistorySchema.index({ alert: 1, createdAt: -1 }); -// Used by getPreviousAlertHistories (groups by {alert, group}, sorts by createdAt) +// No longer read by any query. Kept for one release so task pods still running +// the previous query keep their index during a rolling deploy. AlertHistorySchema.index({ alert: 1, group: 1, createdAt: -1 }); export default mongoose.model( diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 30badac6d..c92edbfc6 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -39,7 +39,9 @@ import * as checkAlert from '@/tasks/checkAlerts'; import { alertHasGroupBy, doesExceedThreshold, + getAlertHistoryLookbackMs, getConsecutiveWindowHistories, + getLatestAlertHistories, getPreviousAlertHistories, getScheduledWindowStart, parseAlertData, @@ -2152,7 +2154,7 @@ describe('checkAlerts', () => { teamWebhooksById: Map, ) => { const [previousMap, recentHistoryMap] = await Promise.all([ - getPreviousAlertHistories([details.alert.id], now), + getPreviousAlertHistories([details.alert], now), getConsecutiveWindowHistories([details.alert], now), ]); await processAlert( @@ -6986,7 +6988,7 @@ describe('checkAlerts', () => { teamWebhooksById, }: Awaited>) => { const shared = await getPreviousAlertHistories( - [grouped.alert.id, firing.alert.id], + [grouped.alert, firing.alert], now, ); expect(shared.get(firing.alert.id)?.state).toBe(AlertState.ALERT); @@ -9084,7 +9086,7 @@ describe('checkAlerts', () => { // Should NOT rescan 22:00-22:05 where service-b had data but was already checked const nextRun = new Date('2023-11-16T22:23:00.000Z'); const previousMapNextRun = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextRun, ); @@ -12058,10 +12060,7 @@ describe('checkAlerts', () => { ]); // Act - Run alerts twice to cover two periods - let previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); + let previousMap = await getPreviousAlertHistories([details.alert], now); await processAlert( now, { @@ -12076,7 +12075,7 @@ describe('checkAlerts', () => { const nextWindow = new Date('2023-11-16T22:15:00.000Z'); previousMap = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextWindow, ); await processAlert( @@ -12202,10 +12201,7 @@ describe('checkAlerts', () => { ]); // Act - Run alerts twice to cover two periods - let previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); + let previousMap = await getPreviousAlertHistories([details.alert], now); await processAlert( now, { @@ -12220,7 +12216,7 @@ describe('checkAlerts', () => { const nextWindow = new Date('2023-11-16T22:15:00.000Z'); previousMap = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextWindow, ); await processAlert( @@ -12267,21 +12263,33 @@ describe('checkAlerts', () => { afterEach(async () => { await server.clearDBs(); - jest.clearAllMocks(); + jest.restoreAllMocks(); }); afterAll(async () => { await server.stop(); }); - const saveAlert = (id: mongoose.Types.ObjectId, createdAt: Date) => { + const saveAlert = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + group?: string, + state: AlertState = AlertState.OK, + ) => { return new AlertHistory({ alert: id, createdAt, - state: AlertState.OK, + state, + group, }).save(); }; + // getPreviousAlertHistories only reads id + interval off the alert. + const asAlert = ( + id: mongoose.Types.ObjectId, + interval: IAlert['interval'] = '5m', + ) => ({ id: id.toString(), interval }); + it('should return the latest alert history for each alert', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12292,7 +12300,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12305,6 +12313,51 @@ describe('checkAlerts', () => { ); }); + it('should return the latest history per group for grouped alerts', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-a'); + await saveAlert(alertId, new Date('2025-01-01T00:05:00Z'), 'group-a'); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-b'); + // A row without a group keys off the bare alert ID + await saveAlert(alertId, new Date('2025-01-01T00:05:00Z')); + + const result = await getPreviousAlertHistories( + [asAlert(alertId)], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.size).toBe(3); + expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + expect(result.get(`${alertId.toString()}||group-b`)!.createdAt).toEqual( + new Date('2025-01-01T00:00:00Z'), + ); + expect(result.get(alertId.toString())!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + }); + + it('should ignore ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:05:00Z'), 'group-a'); + await saveAlert( + alertId, + new Date('2025-01-01T00:10:00Z'), + 'group-a', + AlertState.ERROR, + ); + + const result = await getPreviousAlertHistories( + [asAlert(alertId)], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + }); + it('should not return alert histories from the future', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12315,7 +12368,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); // This one is in the future const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:14:00Z'), ); @@ -12328,6 +12381,29 @@ describe('checkAlerts', () => { ); }); + it('should not return alert histories older than the interval-derived lookback', async () => { + const now = new Date('2025-01-10T00:00:00Z'); + const alertId = new mongoose.Types.ObjectId(); + // 1m interval: 1d lookback floor, so a row from 2 days ago is out of + // range even though the old fixed 7d window included it. + await saveAlert(alertId, new Date('2025-01-08T00:00:00Z')); + const result = await getPreviousAlertHistories( + [asAlert(alertId, '1m')], + now, + ); + expect(result.size).toBe(0); + + // 1h interval: 52 windows (52h) reaches the same row. + const resultHourly = await getPreviousAlertHistories( + [asAlert(alertId, '1h')], + now, + ); + expect(resultHourly.size).toBe(1); + expect(resultHourly.get(alertId.toString())!.createdAt).toEqual( + new Date('2025-01-08T00:00:00Z'), + ); + }); + it('should not return a history if there are no histories for the given alert', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12336,7 +12412,7 @@ describe('checkAlerts', () => { const alert2Id = new mongoose.Types.ObjectId(); const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12357,7 +12433,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); const result = await getPreviousAlertHistories( - [alert1Id.toString()], + [asAlert(alert1Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12367,7 +12443,7 @@ describe('checkAlerts', () => { ); }); - it('should issue one aggregation per alert ID (per-alert queries)', async () => { + it('should issue one streamed range find per alert and no aggregation', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); await saveAlert(alert1Id, new Date('2025-01-01T00:05:00Z')); @@ -12376,24 +12452,23 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:10:00Z')); await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); + const findSpy = jest.spyOn(AlertHistory, 'find'); const aggregateSpy = jest.spyOn(AlertHistory, 'aggregate'); + const cursorSpy = jest.spyOn(mongoose.Query.prototype, 'cursor'); - const fakeAlertIds = Array(150) + const fakeAlerts = Array(150) .fill(null) - .map(() => new mongoose.Types.ObjectId().toString()); - const allIds = [ - alert1Id.toString(), - ...fakeAlertIds, - alert2Id.toString(), - ]; + .map(() => asAlert(new mongoose.Types.ObjectId())); + const allAlerts = [asAlert(alert1Id), ...fakeAlerts, asAlert(alert2Id)]; const result = await getPreviousAlertHistories( - allIds, + allAlerts, new Date('2025-01-01T00:20:00Z'), ); - // One aggregation per alert ID (no chunking) - expect(aggregateSpy).toHaveBeenCalledTimes(allIds.length); + expect(findSpy).toHaveBeenCalledTimes(allAlerts.length); + expect(aggregateSpy).not.toHaveBeenCalled(); + expect(cursorSpy).toHaveBeenCalledTimes(allAlerts.length); expect(result.size).toBe(2); expect(result.get(alert1Id.toString())!.createdAt).toEqual( new Date('2025-01-01T00:05:00Z'), @@ -12402,6 +12477,136 @@ describe('checkAlerts', () => { new Date('2025-01-01T00:15:00Z'), ); }); + + it('should scan the {alert, createdAt} index with tight createdAt bounds', async () => { + const alertId = new mongoose.Types.ObjectId(); + const now = new Date('2025-01-10T00:00:00Z'); + for (let i = 0; i < 10; i++) { + await saveAlert(alertId, new Date(now.getTime() - i * ms('1d'))); + } + await AlertHistory.syncIndexes(); + + const explain = (await AlertHistory.find({ + alert: alertId, + createdAt: { + $lte: now, + $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), + }, + state: { $ne: AlertState.ERROR }, + }).explain('executionStats')) as any; + + const stages = JSON.stringify(explain.queryPlanner.winningPlan); + expect(stages).toContain('alert_1_createdAt_-1'); + // 1d lookback: only today's and yesterday's rows are touched. + expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(2); + }); + }); + + describe('getAlertHistoryLookbackMs', () => { + it('applies the 1d floor for short intervals', () => { + expect(getAlertHistoryLookbackMs('1m')).toBe(ms('1d')); + expect(getAlertHistoryLookbackMs('5m')).toBe(ms('1d')); + expect(getAlertHistoryLookbackMs('15m')).toBe(ms('1d')); + }); + + it('derives 52 windows for mid-size intervals', () => { + expect(getAlertHistoryLookbackMs('30m')).toBe(52 * ms('30m')); + expect(getAlertHistoryLookbackMs('1h')).toBe(52 * ms('1h')); + }); + + it('caps at 7d for long intervals', () => { + expect(getAlertHistoryLookbackMs('6h')).toBe(ms('7d')); + expect(getAlertHistoryLookbackMs('12h')).toBe(ms('7d')); + expect(getAlertHistoryLookbackMs('1d')).toBe(ms('7d')); + }); + }); + + describe('getLatestAlertHistories', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + jest.clearAllMocks(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const saveAlert = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + state: AlertState = AlertState.OK, + ) => { + return new AlertHistory({ + alert: id, + createdAt, + state, + }).save(); + }; + + it('returns the latest row per alert and null when no history exists', async () => { + const alert1Id = new mongoose.Types.ObjectId(); + await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); + await saveAlert( + alert1Id, + new Date('2025-01-01T00:05:00Z'), + AlertState.ALERT, + ); + + const alert2Id = new mongoose.Types.ObjectId(); + + const result = await getLatestAlertHistories( + [alert1Id.toString(), alert2Id.toString()], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(alert1Id.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:05:00Z'), + state: AlertState.ALERT, + }); + expect(result.get(alert2Id.toString())).toBeNull(); + }); + + it('ignores histories created after now', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z')); + await saveAlert(alertId, new Date('2025-01-01T00:10:00Z')); + + const result = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-01T00:05:00Z'), + ); + + expect(result.get(alertId.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:00:00Z'), + state: AlertState.OK, + }); + }); + + it('ignores ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z')); + await saveAlert( + alertId, + new Date('2025-01-01T00:05:00Z'), + AlertState.ERROR, + ); + + const result = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(alertId.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:00:00Z'), + state: AlertState.OK, + }); + }); }); describe('getConsecutiveWindowHistories', () => { @@ -12451,14 +12656,14 @@ describe('checkAlerts', () => { const alertId = new mongoose.Types.ObjectId(); await saveHistory(alertId, new Date('2025-01-01T00:10:00Z')); - const aggregateSpy = jest.spyOn(AlertHistory, 'aggregate'); + const findSpy = jest.spyOn(AlertHistory, 'find'); const result = await getConsecutiveWindowHistories( [makeAlert(alertId, 1), makeAlert(new mongoose.Types.ObjectId())], new Date('2025-01-01T00:17:00Z'), ); - expect(aggregateSpy).not.toHaveBeenCalled(); + expect(findSpy).not.toHaveBeenCalled(); expect(result.size).toBe(0); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts index 77b317248..e9415f5e2 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts @@ -316,11 +316,13 @@ describe('alert evaluators', () => { }); const healthy = makeAlert({ numConsecutiveWindows: 2 }); const broken = makeAlert({ numConsecutiveWindows: 2, threshold: 42 }); - const aggregate = jest - .spyOn(AlertHistory, 'aggregate') - .mockResolvedValue([ - { createdAt: new Date(), state: AlertState.ALERT, group: '' }, - ]); + const find = jest.spyOn(AlertHistory, 'find').mockReturnValue({ + lean: jest + .fn() + .mockResolvedValue([ + { createdAt: new Date(), state: AlertState.ALERT, group: '' }, + ]), + } as any); try { const map = await getConsecutiveWindowHistories( @@ -328,11 +330,11 @@ describe('alert evaluators', () => { new Date('2025-01-01T00:07:00Z'), ); - expect(aggregate).toHaveBeenCalledTimes(1); + expect(find).toHaveBeenCalledTimes(1); expect(map.has(healthy.id)).toBe(true); expect(map.has(broken.id)).toBe(false); } finally { - aggregate.mockRestore(); + find.mockRestore(); } }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index b4bc6b5bf..a1b08bce2 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -39,6 +39,7 @@ import { import { ALERT_NOTIFICATION_TARGETS_LIMIT, AlertErrorType, + AlertInterval, AlertNotificationTargetTiming, AlertThresholdType, BuilderChartConfigWithOptDateRange, @@ -51,7 +52,6 @@ import { } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; -import { ObjectId } from 'mongoose'; import mongoose from 'mongoose'; import ms from 'ms'; import { performance } from 'perf_hooks'; @@ -103,9 +103,10 @@ import { } from '@/tasks/checkAlerts/template'; import { handleSendGenericWebhook } from '@/tasks/checkAlerts/transports'; import { tasksTracer } from '@/tasks/tracer'; -import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types'; +import { CheckAlertsTaskArgs, HdxTask, MappedOmit } from '@/tasks/types'; import { calcAlertDateRange, + MAX_NUM_WINDOWS, roundDownToXMinutes, unflattenObject, } from '@/tasks/util'; @@ -148,7 +149,9 @@ const alertBatchFailuresCounter = getCounter('hyperdx.alerts.batch_failures', { * For saved search alerts, groupBy is on alert.groupBy. * For tile and inline alerts, groupBy is on the chart config. */ -export const alertHasGroupBy = (details: AlertDetails): boolean => { +export const alertHasGroupBy = ( + details: MappedOmit, +): boolean => { // AlertChartConfig members are the tile config types minus the embedded // alert field, so they're assignable to SavedChartConfig (which the // chart-config guards narrow on). @@ -614,7 +617,10 @@ const getAlertKeyPrefix = (alertId: string): string => { * Uses || as delimiter since it's unlikely to appear in alert IDs (MongoDB ObjectIds) * or in typical group key values. */ -const computeHistoryMapKey = (alertId: string, groupKey: string): string => { +export const computeHistoryMapKey = ( + alertId: string, + groupKey: string, +): string => { return groupKey ? `${getAlertKeyPrefix(alertId)}${groupKey}` : alertId; }; @@ -1947,7 +1953,8 @@ export const processAlert = async ( export { handleSendGenericWebhook }; export interface AggregatedAlertHistory { - _id: ObjectId; + /** The alert's ObjectId (not the history document's own _id). */ + _id: mongoose.Types.ObjectId; createdAt: Date; state: AlertState; group?: string; @@ -1955,96 +1962,178 @@ export interface AggregatedAlertHistory { groupValues?: GroupValue[] | null; } +// The previous-history lookback must cover the catch-up horizon +// (MAX_NUM_WINDOWS windows, see calcAlertDateRange) plus slack, and is floored +// at 1d so groups whose last row predates a task outage still auto-resolve. +const ALERT_HISTORY_MIN_LOOKBACK_MS = ms('1d'); +const ALERT_HISTORY_MAX_LOOKBACK_MS = ms('7d'); +// +2 windows so a boundary window is never clipped by the lookback. +const ALERT_HISTORY_LOOKBACK_WINDOWS = MAX_NUM_WINDOWS + 2; + /** - * Fetch the most recent AlertHistory value for each of the given alert IDs. + * How far back to scan AlertHistory when loading an alert's previous + * per-group state. The scan is a range on {alert: 1, createdAt: -1}, so this + * directly bounds the index keys and documents examined per alert. + */ +export const getAlertHistoryLookbackMs = (interval: AlertInterval): number => { + const windowMs = ms(interval); + return Math.min( + ALERT_HISTORY_MAX_LOOKBACK_MS, + Math.max( + ALERT_HISTORY_MIN_LOOKBACK_MS, + ALERT_HISTORY_LOOKBACK_WINDOWS * windowMs, + ), + ); +}; + +/** Latest AlertHistory row per alert, minus the large `lastValues` array. */ +export type LatestAlertHistory = Pick< + AggregatedAlertHistory, + 'createdAt' | 'state' | 'group' | 'fired' +>; + +/** + * Fetch the single most recent AlertHistory row for each of the given alert + * IDs. + * + * Each lookup is a one-key index seek on {alert: 1, createdAt: -1}, so it is + * cheap enough to run for every alert on every tick. The provider uses it to + * tell whether an alert was already evaluated in the current window, which + * limits the heavier per-group fetch (getPreviousAlertHistories) to grouped + * alerts that are due. For non-grouped alerts this row is the previous state. + * + * @returns A map from alert ID to its latest history row, or null when the + * alert has no history. + */ +export const getLatestAlertHistories = async ( + alertIds: string[], + now: Date, + sharedQueue?: PQueue, +): Promise> => { + const queue = + sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); + + const results = await Promise.all( + alertIds.map(alertId => + queue.add(async () => { + const latest = await AlertHistory.findOne( + { + alert: new mongoose.Types.ObjectId(alertId), + createdAt: { $lte: now }, + // ERROR rows record failed evaluations; they must not count as + // "window evaluated" or the failed window would never be + // retried/backfilled. + state: { $ne: AlertState.ERROR }, + }, + { createdAt: 1, state: 1, group: 1, fired: 1 }, + ) + .sort({ createdAt: -1 }) + .lean(); + return [alertId, latest] as const; + }), + ), + ); + + return new Map( + results + .filter((r): r is NonNullable => r !== undefined) + .map(([alertId, latest]) => [ + alertId, + latest + ? { + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + } + : null, + ]), + ); +}; + +/** + * Fetch the most recent AlertHistory value for each of the given alerts. * For group-by alerts, returns the latest history for each group within each alert. * - * Uses per-alert queries instead of batched $in to leverage the compound index - * {alert: 1, group: 1, createdAt: -1} for index-backed sorting. With a single - * alert value, the index delivers results already sorted by {group, createdAt desc}, - * so the $sort is a no-op and $group + $first can short-circuit per group. + * Runs one range `find()` per alert on {alert: 1, createdAt: -1} and reduces + * to the latest row per group in application code. A $sort {alert, group, + * createdAt} + $group/$first aggregation is avoided on purpose: MongoDB does + * not short-circuit it per group, so it reads every row in the window, and no + * single index serves both a tight createdAt bound and a group-major sort. + * + * The window is derived from each alert's interval (see + * getAlertHistoryLookbackMs) and `lastValues` is not fetched. * - * @param alertIds The list of alert IDs to query the latest history for. + * @param alerts The alerts (id + interval) to query the latest history for. * @param now The current date and time. AlertHistory documents that have a createdAt > now are ignored. * @returns A map from Alert IDs (or Alert ID + group) to their most recent AlertHistory. * For non-grouped alerts, the key is just the alert ID. * For grouped alerts, the key is "alertId||group" to track per-group state. */ export const getPreviousAlertHistories = async ( - alertIds: string[], + alerts: Array>, now: Date, sharedQueue?: PQueue, ) => { - const lookbackDate = new Date(now.getTime() - ms('7d')); - // Concurrency-limited per-alert queries to avoid overwhelming the connection // pool when there are many alerts (e.g., 200+ alert IDs). const queue = sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); const results = await Promise.all( - alertIds.map(alertId => + alerts.map(alert => queue.add(async () => { - const id = new mongoose.Types.ObjectId(alertId); - return AlertHistory.aggregate([ - { - $match: { - alert: id, - createdAt: { $lte: now, $gte: lookbackDate }, - // ERROR rows record failed evaluations; they must not count as - // "window evaluated" or the failed window would never be - // retried/backfilled. - state: { $ne: AlertState.ERROR }, - }, - }, - // With a single alert value, the compound index {alert: 1, group: 1, createdAt: -1} - // delivers results already in this sort order — this is an index-backed no-op sort. - { - $sort: { alert: 1, group: 1, createdAt: -1 }, - }, - // Group by {alert, group}, taking the first (latest) document's fields. - // Using $first on individual fields instead of $first: '$$ROOT' allows - // DocumentDB to avoid fetching full documents when not needed. - { - $group: { - _id: { - alert: '$alert', - group: '$group', - }, - createdAt: { $first: '$createdAt' }, - state: { $first: '$state' }, - fired: { $first: '$fired' }, - groupValues: { $first: '$groupValues' }, - }, - }, + const alertObjectId = new mongoose.Types.ObjectId(alert.id); + const lookbackDate = new Date( + now.getTime() - getAlertHistoryLookbackMs(alert.interval), + ); + // Streamed so memory stays at O(batch + groups) rather than + // O(groups × windows) while the reduce keeps only the latest row. + const cursor = AlertHistory.find( { - $project: { - _id: '$_id.alert', - createdAt: 1, - state: 1, - group: '$_id.group', - fired: 1, - groupValues: 1, - }, + alert: alertObjectId, + createdAt: { $lte: now, $gte: lookbackDate }, + // ERROR rows record failed evaluations; they must not count as + // "window evaluated" or the failed window would never be + // retried/backfilled. + state: { $ne: AlertState.ERROR }, }, - ]); + { group: 1, state: 1, createdAt: 1, fired: 1, groupValues: 1 }, + ) + .lean() + .cursor(); + + const latestByKey = new Map(); + for await (const row of cursor) { + const key = computeHistoryMapKey(alert.id, row.group || ''); + const existing = latestByKey.get(key); + if (!existing || row.createdAt > existing.createdAt) { + latestByKey.set(key, { + _id: alertObjectId, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + groupValues: row.groupValues, + }); + } + } + return latestByKey; }), ), ); - // Create a map with composite keys for grouped alerts (alertId||group) or simple keys for non-grouped alerts - return new Map( - results - .flat() - .filter((h): h is AggregatedAlertHistory => h !== undefined) - .map(history => { - const key = computeHistoryMapKey( - history._id.toString(), - history.group || '', - ); - return [key, history]; - }), - ); + // Keys are namespaced by alert ID, so merging cannot collide. + const merged = new Map(); + for (const result of results) { + if (!result) { + continue; + } + for (const [key, history] of result) { + merged.set(key, history); + } + } + return merged; }; /** @@ -2103,27 +2192,27 @@ export const getConsecutiveWindowHistories = async ( (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, ); const id = new mongoose.Types.ObjectId(alert.id); - const histories = await AlertHistory.aggregate([ + // Range scan on {alert: 1, createdAt: -1}; the newest-first order + // the consumer needs is applied below instead of a DB-side sort. + const rows = await AlertHistory.find( { - $match: { - alert: id, - createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, - // Failed evaluations (ERROR rows) are not evaluated windows and - // must not affect consecutive-window counting. - state: { $ne: AlertState.ERROR }, - }, - }, - { $sort: { alert: 1, group: 1, createdAt: -1 } }, - { - $project: { - _id: '$alert', - createdAt: 1, - state: 1, - group: 1, - fired: 1, - }, + alert: id, + createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, + // Failed evaluations (ERROR rows) are not evaluated windows and + // must not affect consecutive-window counting. + state: { $ne: AlertState.ERROR }, }, - ]); + { group: 1, state: 1, createdAt: 1, fired: 1 }, + ).lean(); + const histories: AggregatedAlertHistory[] = rows + .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()) + .map(row => ({ + _id: id, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + })); return { alertId: alert.id, histories }; }), ), diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts index 27ef9da7c..e58acf563 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts @@ -1,5 +1,6 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; +import ms from 'ms'; import { createAlert } from '@/controllers/alerts'; import { createTeam } from '@/controllers/team'; @@ -9,11 +10,21 @@ import { makeTile, RAW_SQL_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; +import Alert, { + AlertSource, + AlertState, + AlertThresholdType, +} from '@/models/alert'; +import AlertHistory from '@/models/alertHistory'; import Connection from '@/models/connection'; import Dashboard from '@/models/dashboard'; import { SavedSearch } from '@/models/savedSearch'; import { Source } from '@/models/source'; +import { getScheduledWindowStart } from '@/tasks/checkAlerts'; +import { + clearAlertEvaluators, + registerAlertEvaluator, +} from '@/tasks/checkAlerts/evaluators'; import { AlertProvider, AlertTaskType, @@ -940,6 +951,176 @@ describe('DefaultAlertProvider', () => { }); }); + describe('history fetch gating', () => { + afterEach(() => { + jest.restoreAllMocks(); + clearAlertEvaluators(); + }); + + const createSavedSearchAlert = async ({ + groupBy, + interval = '5m', + }: { groupBy?: string; interval?: '5m' | '1d' } = {}) => { + const team = await createTeam({ name: 'Test Team' }); + const connection = await Connection.create({ + team: team._id, + name: 'Test Connection', + host: 'http://localhost:8123', + username: 'test', + password: 'test', + }); + const source = await Source.create({ + team: team._id, + name: 'Test Source', + kind: 'log', + from: { databaseName: 'default', tableName: 'logs' }, + timestampValueExpression: 'timestamp', + connection: connection._id, + }); + const savedSearch = await SavedSearch.create({ + team: team._id, + name: 'Test Search', + select: 'message', + where: 'level: error', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: source._id, + tags: [], + }); + return createAlert( + team._id, + { + source: AlertSource.SAVED_SEARCH, + savedSearchId: savedSearch._id.toString(), + threshold: 10, + thresholdType: AlertThresholdType.ABOVE, + interval, + groupBy, + channel: { + type: 'webhook', + webhookId: new mongoose.Types.ObjectId().toString(), + }, + }, + new mongoose.Types.ObjectId(), + ); + }; + + const previousWindowStart = () => + new Date(getScheduledWindowStart(new Date(), 5, 0).getTime() - ms('5m')); + + it('skips the history fetch and seeds a sentinel for alerts already evaluated in the current window', async () => { + // 1d interval so the current window cannot roll over mid-test. + const alert = await createSavedSearchAlert({ interval: '1d' }); + const windowStart = getScheduledWindowStart(new Date(), 1440, 0); + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ALERT, + group: 'group-a', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).not.toHaveBeenCalled(); + expect(result).toHaveLength(1); + const sentinel = result[0].alerts[0].previousMap.get(alert.id); + expect(sentinel?.createdAt).toEqual(windowStart); + expect(sentinel?.state).toBe(AlertState.ALERT); + }); + + it('fetches per-group history for grouped alerts that are due', async () => { + const alert = await createSavedSearchAlert({ groupBy: 'ServiceName' }); + const createdAt = previousWindowStart(); + await AlertHistory.create({ + alert: alert._id, + createdAt, + state: AlertState.ALERT, + group: 'group-a', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt, + state: AlertState.OK, + group: 'group-b', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).toHaveBeenCalledTimes(1); + const { previousMap } = result[0].alerts[0]; + expect(previousMap.get(`${alert.id}||group-a`)?.state).toBe( + AlertState.ALERT, + ); + expect(previousMap.get(`${alert.id}||group-a`)?.createdAt).toEqual( + createdAt, + ); + expect(previousMap.get(`${alert.id}||group-b`)?.state).toBe( + AlertState.OK, + ); + }); + + it('seeds non-grouped due alerts from the latest row without a range scan', async () => { + const alert = await createSavedSearchAlert(); + const base = previousWindowStart().getTime(); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - ms('10m')), + state: AlertState.OK, + }); + const newest = await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base), + state: AlertState.ALERT, + fired: true, + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).not.toHaveBeenCalled(); + const previous = result[0].alerts[0].previousMap.get(alert.id); + expect(previous?.createdAt).toEqual(newest.createdAt); + expect(previous?.state).toBe(AlertState.ALERT); + expect(previous?.fired).toBe(true); + }); + + it('skips the history fetch for alerts with no history', async () => { + const alert = await createSavedSearchAlert({ groupBy: 'ServiceName' }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).not.toHaveBeenCalled(); + expect(result[0].alerts[0].alert.id).toBe(alert.id); + expect(result[0].alerts[0].previousMap.size).toBe(0); + }); + + it('treats an alert as due when its evaluator lookup fails', async () => { + const alert = await createSavedSearchAlert(); + await AlertHistory.create({ + alert: alert._id, + createdAt: previousWindowStart(), + state: AlertState.ALERT, + }); + registerAlertEvaluator({ + name: 'broken', + appliesTo: () => { + throw new Error('lookup failed'); + }, + evaluate: jest.fn(), + }); + + const result = await provider.getAlertTasks(); + + expect(result).toHaveLength(1); + expect(result[0].alerts[0].previousMap.get(alert.id)?.state).toBe( + AlertState.ALERT, + ); + }); + }); + describe('buildLogSearchLink', () => { it('should build correct URL with basic parameters', () => { const result = provider.buildLogSearchLink({ diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index b409fafe9..23143f2c6 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -8,6 +8,7 @@ import { } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; +import { serializeError } from 'serialize-error'; import { URLSearchParams } from 'url'; import { ClickhouseClient } from '@/clickhouse'; @@ -31,9 +32,13 @@ import { type ISavedSearch, SavedSearch } from '@/models/savedSearch'; import { type ISource, Source } from '@/models/source'; import Webhook, { IWebhook } from '@/models/webhook'; import { - AggregatedAlertHistory, + alertHasGroupBy, + computeHistoryMapKey, + getAlertWindowStart, getConsecutiveWindowHistories, + getLatestAlertHistories, getPreviousAlertHistories, + type LatestAlertHistory, } from '@/tasks/checkAlerts'; import { type AlertDetails, @@ -349,13 +354,9 @@ async function getInlineAlertDetails( ]; } -async function loadAlert( +async function resolveAlertDetails( alert: IAlert, - groupedTasks: Map, - previousAlerts: Map, - recentHistoryMap: Map, - now: Date, -) { +): Promise<[IConnection, PartialAlertDetails]> { if (!alert.source) { throw new Error('alert does not have a source'); } @@ -393,18 +394,30 @@ async function loadAlert( throw new Error('failed to fetch alert connection'); } - if (!groupedTasks.has(conn.id)) { - groupedTasks.set(conn.id, { alerts: [], conn, now }); - } - const v = groupedTasks.get(conn.id); - if (!v) { - throw new Error(`provider did not set key ${conn.id} before appending`); + return [conn, details]; +} + +/** + * Whether the alert's latest history row is from the window processAlert would + * evaluate now, i.e. processAlert will skip it. A failing evaluator lookup + * counts as due so processAlert can record the error on that alert. + */ +function wasEvaluatedInCurrentWindow( + alert: IAlert, + latest: LatestAlertHistory, + now: Date, +): boolean { + try { + return ( + latest.createdAt.getTime() === getAlertWindowStart(alert, now).getTime() + ); + } catch (e) { + logger.error( + { alertId: alert.id, error: serializeError(e) }, + 'Failed to compute alert window start for history gating', + ); + return false; } - v.alerts.push({ - ...details, - previousMap: previousAlerts, - recentHistoryMap, - }); } export default class DefaultAlertProvider implements AlertProvider { @@ -426,26 +439,16 @@ export default class DefaultAlertProvider implements AlertProvider { const alerts = await Alert.find({}); const now = new Date(); - const alertIds = alerts.map(({ id }) => id); - // Share a single queue across both history fetches so their combined - // in-flight per-alert queries stay within one global cap. - const historyQueryQueue = new PQueue({ - concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, - }); - const [previousAlerts, recentHistoryMap] = await Promise.all([ - getPreviousAlertHistories(alertIds, now, historyQueryQueue), - getConsecutiveWindowHistories(alerts, now, historyQueryQueue), - ]); + const resolved: { + alert: IAlert; + conn: IConnection; + details: PartialAlertDetails; + }[] = []; for (const alert of alerts) { try { - await loadAlert( - alert, - groupedTasks, - previousAlerts, - recentHistoryMap, - now, - ); + const [conn, details] = await resolveAlertDetails(alert); + resolved.push({ alert, conn, details }); } catch (e) { logger.error({ message: `failed to load alert: ${e}`, @@ -457,6 +460,102 @@ export default class DefaultAlertProvider implements AlertProvider { } } + // Share a single queue across all history fetches so their combined + // in-flight per-alert queries stay within one global cap. + const historyQueryQueue = new PQueue({ + concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, + }); + + // One index seek per alert decides whether it is due this window, so + // alerts on longer intervals don't re-scan their history every tick. + const latestHistories = await getLatestAlertHistories( + resolved.map(({ alert }) => alert.id), + now, + historyQueryQueue, + ); + + const dueAlertsWithHistory: IAlert[] = []; + const dueGroupedAlerts: IAlert[] = []; + const dueNonGroupedAlerts: { + alert: IAlert; + latest: LatestAlertHistory; + }[] = []; + const gatedAlerts: { alert: IAlert; latest: LatestAlertHistory }[] = []; + let noHistoryCount = 0; + for (const { alert, details } of resolved) { + const latest = latestHistories.get(alert.id); + if (!latest) { + noHistoryCount += 1; + continue; + } + if (wasEvaluatedInCurrentWindow(alert, latest, now)) { + gatedAlerts.push({ alert, latest }); + continue; + } + dueAlertsWithHistory.push(alert); + // Non-grouped alerts only need the latest row, which the gate already + // fetched. Grouped alerts need the latest row of every group. + if (alertHasGroupBy(details)) { + dueGroupedAlerts.push(alert); + } else { + dueNonGroupedAlerts.push({ alert, latest }); + } + } + + const [previousAlerts, recentHistoryMap] = await Promise.all([ + getPreviousAlertHistories(dueGroupedAlerts, now, historyQueryQueue), + getConsecutiveWindowHistories( + dueAlertsWithHistory, + now, + historyQueryQueue, + ), + ]); + + for (const { alert, latest } of dueNonGroupedAlerts) { + previousAlerts.set(computeHistoryMapKey(alert.id, latest.group || ''), { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + }); + } + + // A row in the current window under the bare alert ID makes + // processAlert's shouldSkipAlertCheck skip the alert, as it did before. + for (const { alert, latest } of gatedAlerts) { + previousAlerts.set(alert.id, { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + }); + } + + logger.info( + { + alertCount: alerts.length, + resolvedCount: resolved.length, + dueCount: dueAlertsWithHistory.length, + dueGroupedCount: dueGroupedAlerts.length, + gatedCount: gatedAlerts.length, + noHistoryCount, + }, + 'Loaded alert histories', + ); + + for (const { conn, details } of resolved) { + let task = groupedTasks.get(conn.id); + if (!task) { + task = { alerts: [], conn, now }; + groupedTasks.set(conn.id, task); + } + task.alerts.push({ + ...details, + previousMap: previousAlerts, + recentHistoryMap, + }); + } + // Flatten out our groupings for execution return Array.from(groupedTasks.values()); } diff --git a/packages/api/src/tasks/util.ts b/packages/api/src/tasks/util.ts index ce6cbc5db..5f199d3e9 100644 --- a/packages/api/src/tasks/util.ts +++ b/packages/api/src/tasks/util.ts @@ -41,7 +41,7 @@ export const escapeJsonString = (str: string) => { return JSON.stringify(str).slice(1, -1); }; -const MAX_NUM_WINDOWS = 50; +export const MAX_NUM_WINDOWS = 50; const maxLookbackTime = (windowSizeInMins: number) => 3600_000 * (windowSizeInMins < 15 ? 6 : 24); export function calcAlertDateRange( From d0b48fff6e6f1fb4105cc1c516f93ef6255bb21b Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:04:42 -0800 Subject: [PATCH 2/7] refactor(api): mirror the enterprise alert-history code to cut sync conflicts Use the enterprise build's text for the alert-history lookback, the latest- and previous-history fetches, the provider's due-ness gate (now a loadAlerts() helper), and their tests, so the upstream sync into the enterprise repo conflicts only where that repo has its own code. - Drop the unused { alert, group, createdAt } index declaration, as the enterprise build did. Existing deployments keep the index; it is no longer created on new ones. - Keep the guard that treats an alert as due when its window start cannot be computed, now as a helper at the end of providers/default.ts, and test it by stubbing getAlertWindowStart instead of clearing the evaluators. - Move the extra ERROR-row and index-plan tests into their own block. Refs #2434 --- packages/api/src/models/alertHistory.ts | 11 +- .../__tests__/checkAlerts.int.test.ts | 185 ++++---- .../checkAlerts/__tests__/evaluators.test.ts | 3 + packages/api/src/tasks/checkAlerts/index.ts | 100 +++-- .../providers/__tests__/default.int.test.ts | 401 ++++++++++-------- .../tasks/checkAlerts/providers/default.ts | 298 ++++++------- 6 files changed, 574 insertions(+), 424 deletions(-) diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index d9330f970..203fbe074 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -169,14 +169,13 @@ AlertHistorySchema.index( { expireAfterSeconds: ms('30d') / 1000 }, ); -// Serves every alert-history read: each matches on alert with a createdAt -// range or sort. +// Used by all alert/task reads: getRecentAlertHistories, +// getAlertTransitionsInRange, getLatestAlertHistories, +// getPreviousAlertHistories, and getConsecutiveWindowHistories — all match on +// alert equality with a createdAt range/sort, which this index serves with +// tight bounds. AlertHistorySchema.index({ alert: 1, createdAt: -1 }); -// No longer read by any query. Kept for one release so task pods still running -// the previous query keep their index during a rolling deploy. -AlertHistorySchema.index({ alert: 1, group: 1, createdAt: -1 }); - export default mongoose.model( 'AlertHistory', AlertHistorySchema, diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index c92edbfc6..34ba95cc1 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -12263,7 +12263,7 @@ describe('checkAlerts', () => { afterEach(async () => { await server.clearDBs(); - jest.restoreAllMocks(); + jest.clearAllMocks(); }); afterAll(async () => { @@ -12274,12 +12274,11 @@ describe('checkAlerts', () => { id: mongoose.Types.ObjectId, createdAt: Date, group?: string, - state: AlertState = AlertState.OK, ) => { return new AlertHistory({ alert: id, createdAt, - state, + state: AlertState.OK, group, }).save(); }; @@ -12318,7 +12317,7 @@ describe('checkAlerts', () => { await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-a'); await saveAlert(alertId, new Date('2025-01-01T00:05:00Z'), 'group-a'); await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-b'); - // A row without a group keys off the bare alert ID + // A row without a group should key off the bare alert ID await saveAlert(alertId, new Date('2025-01-01T00:05:00Z')); const result = await getPreviousAlertHistories( @@ -12338,26 +12337,6 @@ describe('checkAlerts', () => { ); }); - it('should ignore ERROR rows', async () => { - const alertId = new mongoose.Types.ObjectId(); - await saveAlert(alertId, new Date('2025-01-01T00:05:00Z'), 'group-a'); - await saveAlert( - alertId, - new Date('2025-01-01T00:10:00Z'), - 'group-a', - AlertState.ERROR, - ); - - const result = await getPreviousAlertHistories( - [asAlert(alertId)], - new Date('2025-01-01T00:20:00Z'), - ); - - expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( - new Date('2025-01-01T00:05:00Z'), - ); - }); - it('should not return alert histories from the future', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12384,8 +12363,8 @@ describe('checkAlerts', () => { it('should not return alert histories older than the interval-derived lookback', async () => { const now = new Date('2025-01-10T00:00:00Z'); const alertId = new mongoose.Types.ObjectId(); - // 1m interval: 1d lookback floor, so a row from 2 days ago is out of - // range even though the old fixed 7d window included it. + // 1m interval → lookback floor of 1 day. A row from 2 days ago is out of + // range even though a fixed 7d window would have included it. await saveAlert(alertId, new Date('2025-01-08T00:00:00Z')); const result = await getPreviousAlertHistories( [asAlert(alertId, '1m')], @@ -12393,7 +12372,7 @@ describe('checkAlerts', () => { ); expect(result.size).toBe(0); - // 1h interval: 52 windows (52h) reaches the same row. + // 1h interval → lookback of 52 windows (52h) picks the same row up. const resultHourly = await getPreviousAlertHistories( [asAlert(alertId, '1h')], now, @@ -12443,7 +12422,7 @@ describe('checkAlerts', () => { ); }); - it('should issue one streamed range find per alert and no aggregation', async () => { + it('should issue one range find per alert (no aggregation, no chunking)', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); await saveAlert(alert1Id, new Date('2025-01-01T00:05:00Z')); @@ -12466,8 +12445,13 @@ describe('checkAlerts', () => { new Date('2025-01-01T00:20:00Z'), ); + // One tight-bounds find per alert; no aggregation pipeline is used + // (DocumentDB cannot serve the old $sort+$group shape from an index). expect(findSpy).toHaveBeenCalledTimes(allAlerts.length); expect(aggregateSpy).not.toHaveBeenCalled(); + // Each find is consumed through a streaming cursor rather than + // materialized as an array — the reduce only keeps the latest row per + // group, so buffering O(groups × windows) rows would waste heap. expect(cursorSpy).toHaveBeenCalledTimes(allAlerts.length); expect(result.size).toBe(2); expect(result.get(alert1Id.toString())!.createdAt).toEqual( @@ -12477,29 +12461,6 @@ describe('checkAlerts', () => { new Date('2025-01-01T00:15:00Z'), ); }); - - it('should scan the {alert, createdAt} index with tight createdAt bounds', async () => { - const alertId = new mongoose.Types.ObjectId(); - const now = new Date('2025-01-10T00:00:00Z'); - for (let i = 0; i < 10; i++) { - await saveAlert(alertId, new Date(now.getTime() - i * ms('1d'))); - } - await AlertHistory.syncIndexes(); - - const explain = (await AlertHistory.find({ - alert: alertId, - createdAt: { - $lte: now, - $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), - }, - state: { $ne: AlertState.ERROR }, - }).explain('executionStats')) as any; - - const stages = JSON.stringify(explain.queryPlanner.winningPlan); - expect(stages).toContain('alert_1_createdAt_-1'); - // 1d lookback: only today's and yesterday's rows are touched. - expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(2); - }); }); describe('getAlertHistoryLookbackMs', () => { @@ -12515,6 +12476,7 @@ describe('checkAlerts', () => { }); it('caps at 7d for long intervals', () => { + // 52 windows of 6h/12h/1d all exceed 7 days expect(getAlertHistoryLookbackMs('6h')).toBe(ms('7d')); expect(getAlertHistoryLookbackMs('12h')).toBe(ms('7d')); expect(getAlertHistoryLookbackMs('1d')).toBe(ms('7d')); @@ -12549,7 +12511,7 @@ describe('checkAlerts', () => { }).save(); }; - it('returns the latest row per alert and null when no history exists', async () => { + it('returns the latest createdAt/state per alert and null when no history exists', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); await saveAlert( @@ -12587,26 +12549,6 @@ describe('checkAlerts', () => { state: AlertState.OK, }); }); - - it('ignores ERROR rows', async () => { - const alertId = new mongoose.Types.ObjectId(); - await saveAlert(alertId, new Date('2025-01-01T00:00:00Z')); - await saveAlert( - alertId, - new Date('2025-01-01T00:05:00Z'), - AlertState.ERROR, - ); - - const result = await getLatestAlertHistories( - [alertId.toString()], - new Date('2025-01-01T00:20:00Z'), - ); - - expect(result.get(alertId.toString())).toEqual({ - createdAt: new Date('2025-01-01T00:00:00Z'), - state: AlertState.OK, - }); - }); }); describe('getConsecutiveWindowHistories', () => { @@ -12719,4 +12661,103 @@ describe('checkAlerts', () => { ]); }); }); + describe('alert history query shape', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const saveHistory = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + state: AlertState, + group?: string, + ) => new AlertHistory({ alert: id, createdAt, state, group }).save(); + + it('getPreviousAlertHistories ignores ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-01T00:05:00Z'), + AlertState.OK, + 'group-a', + ); + await saveHistory( + alertId, + new Date('2025-01-01T00:10:00Z'), + AlertState.ERROR, + 'group-a', + ); + + const result = await getPreviousAlertHistories( + [{ id: alertId.toString(), interval: '5m' }], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + }); + + it('getLatestAlertHistories ignores ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-01T00:00:00Z'), + AlertState.OK, + ); + await saveHistory( + alertId, + new Date('2025-01-01T00:05:00Z'), + AlertState.ERROR, + ); + + const result = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(alertId.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:00:00Z'), + state: AlertState.OK, + }); + }); + + it('scans the {alert, createdAt} index with tight createdAt bounds', async () => { + const alertId = new mongoose.Types.ObjectId(); + const now = new Date('2025-01-10T00:00:00Z'); + for (let i = 0; i < 10; i++) { + await saveHistory( + alertId, + new Date(now.getTime() - i * ms('1d')), + AlertState.OK, + ); + } + await AlertHistory.syncIndexes(); + + const explain = (await AlertHistory.find({ + alert: alertId, + createdAt: { + $lte: now, + $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), + }, + state: { $ne: AlertState.ERROR }, + }).explain('executionStats')) as any; + + expect(JSON.stringify(explain.queryPlanner.winningPlan)).toContain( + 'alert_1_createdAt_-1', + ); + // 1d lookback: only today's and yesterday's rows are touched. + expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(2); + }); + }); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts index e9415f5e2..c8e75ec69 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts @@ -316,6 +316,9 @@ describe('alert evaluators', () => { }); const healthy = makeAlert({ numConsecutiveWindows: 2 }); const broken = makeAlert({ numConsecutiveWindows: 2, threshold: 42 }); + // EE reads consecutive-window history with find().lean() rather than + // aggregate (HDX-4791). + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test double only needs lean() const find = jest.spyOn(AlertHistory, 'find').mockReturnValue({ lean: jest .fn() diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index a1b08bce2..4f95a0658 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1962,18 +1962,26 @@ export interface AggregatedAlertHistory { groupValues?: GroupValue[] | null; } -// The previous-history lookback must cover the catch-up horizon -// (MAX_NUM_WINDOWS windows, see calcAlertDateRange) plus slack, and is floored -// at 1d so groups whose last row predates a task outage still auto-resolve. +// Floor/cap for the previous-history lookback window. The lookback only needs +// to cover (a) the catch-up horizon (capped at MAX_NUM_WINDOWS windows — see +// calcAlertDateRange) and (b) enough slack to auto-resolve groups whose last +// history row predates a task outage. History rows are written every window +// for every group present in the data, so anything beyond a couple of windows +// exists purely for outage/stale-group coverage. const ALERT_HISTORY_MIN_LOOKBACK_MS = ms('1d'); const ALERT_HISTORY_MAX_LOOKBACK_MS = ms('7d'); -// +2 windows so a boundary window is never clipped by the lookback. +// +2 windows of slack over the catch-up cap so a boundary window is never +// clipped by the history lookback. const ALERT_HISTORY_LOOKBACK_WINDOWS = MAX_NUM_WINDOWS + 2; /** - * How far back to scan AlertHistory when loading an alert's previous - * per-group state. The scan is a range on {alert: 1, createdAt: -1}, so this - * directly bounds the index keys and documents examined per alert. + * Compute how far back to scan AlertHistory when loading previous per-group + * state for an alert, derived from the alert's evaluation interval. + * + * Bounding the scan matters on DocumentDB: the previous-history query is a + * range scan on {alert: 1, createdAt: -1}, so the lookback directly bounds the + * index entries examined and documents fetched on the (read-preferring) + * secondary instances. */ export const getAlertHistoryLookbackMs = (interval: AlertInterval): number => { const windowMs = ms(interval); @@ -1986,7 +1994,7 @@ export const getAlertHistoryLookbackMs = (interval: AlertInterval): number => { ); }; -/** Latest AlertHistory row per alert, minus the large `lastValues` array. */ +/** Latest AlertHistory row per alert, minus the fat `lastValues` array. */ export type LatestAlertHistory = Pick< AggregatedAlertHistory, 'createdAt' | 'state' | 'group' | 'fired' @@ -1996,14 +2004,15 @@ export type LatestAlertHistory = Pick< * Fetch the single most recent AlertHistory row for each of the given alert * IDs. * - * Each lookup is a one-key index seek on {alert: 1, createdAt: -1}, so it is - * cheap enough to run for every alert on every tick. The provider uses it to - * tell whether an alert was already evaluated in the current window, which - * limits the heavier per-group fetch (getPreviousAlertHistories) to grouped - * alerts that are due. For non-grouped alerts this row is the previous state. + * Each lookup is a one-key index seek on {alert: 1, createdAt: -1}, so this is + * cheap enough to run for every alert on every task tick. loadAlerts uses the + * result to decide whether an alert was already evaluated in the current + * window — gating the much heavier per-group fetch (getPreviousAlertHistories) + * to grouped alerts that are due; for non-grouped alerts this row is the + * previous state itself. * * @returns A map from alert ID to its latest history row, or null when the - * alert has no history. + * alert has no history at all. */ export const getLatestAlertHistories = async ( alertIds: string[], @@ -2025,7 +2034,12 @@ export const getLatestAlertHistories = async ( // retried/backfilled. state: { $ne: AlertState.ERROR }, }, - { createdAt: 1, state: 1, group: 1, fired: 1 }, + { + createdAt: 1, + state: 1, + group: 1, + fired: 1, + }, ) .sort({ createdAt: -1 }) .lean(); @@ -2055,14 +2069,20 @@ export const getLatestAlertHistories = async ( * Fetch the most recent AlertHistory value for each of the given alerts. * For group-by alerts, returns the latest history for each group within each alert. * - * Runs one range `find()` per alert on {alert: 1, createdAt: -1} and reduces - * to the latest row per group in application code. A $sort {alert, group, - * createdAt} + $group/$first aggregation is avoided on purpose: MongoDB does - * not short-circuit it per group, so it reads every row in the window, and no - * single index serves both a tight createdAt bound and a group-major sort. + * Implemented as a per-alert range `find()` on the {alert: 1, createdAt: -1} + * index with the latest-per-group reduce done in application code. An + * aggregation with $sort {alert, group, createdAt} + $group/$first is NOT used + * on purpose: DocumentDB has no DISTINCT_SCAN optimization, and no single + * B-tree index can serve both a tight createdAt range bound and a group-major + * sort — the aggregation either walks every index entry for the alert across + * the full TTL retention (loose createdAt bounds behind the unbounded `group` + * key) or performs a blocking in-memory sort. A plain tight-bounds range scan + * examines exactly the lookback window and keeps long-lived sort/group cursors + * off the DocumentDB secondaries (HDX-4791). * - * The window is derived from each alert's interval (see - * getAlertHistoryLookbackMs) and `lastValues` is not fetched. + * The lookback window is derived from each alert's interval (see + * getAlertHistoryLookbackMs); the projection excludes the fat `lastValues` + * array so only the per-group state fields are transferred. * * @param alerts The alerts (id + interval) to query the latest history for. * @param now The current date and time. AlertHistory documents that have a createdAt > now are ignored. @@ -2087,8 +2107,14 @@ export const getPreviousAlertHistories = async ( const lookbackDate = new Date( now.getTime() - getAlertHistoryLookbackMs(alert.interval), ); - // Streamed so memory stays at O(batch + groups) rather than - // O(groups × windows) while the reduce keeps only the latest row. + // Stream the range scan instead of materializing it: the row count is + // O(groups × windows-in-lookback) — up to ~1,440 windows for a + // 1m-interval alert at the 1d lookback floor — while the reduce below + // only ever needs the latest row per group. Iterating a cursor bounds + // app memory at O(driver batch + groups) per alert instead of + // O(groups × windows), which matters with up to + // ALERT_HISTORY_QUERY_CONCURRENCY of these in flight when alerts + // sharing a window boundary all come due at once. const cursor = AlertHistory.find( { alert: alertObjectId, @@ -2098,11 +2124,20 @@ export const getPreviousAlertHistories = async ( // retried/backfilled. state: { $ne: AlertState.ERROR }, }, - { group: 1, state: 1, createdAt: 1, fired: 1, groupValues: 1 }, + { + group: 1, + state: 1, + createdAt: 1, + fired: 1, + groupValues: 1, + }, ) .lean() .cursor(); + // Reduce to the latest row per group. Ties on createdAt are resolved + // arbitrarily (first row wins), matching the previous aggregation's + // unstable $sort + $first behavior. const latestByKey = new Map(); for await (const row of cursor) { const key = computeHistoryMapKey(alert.id, row.group || ''); @@ -2123,7 +2158,8 @@ export const getPreviousAlertHistories = async ( ), ); - // Keys are namespaced by alert ID, so merging cannot collide. + // Merge the per-alert maps. Keys are namespaced by alert ID, so there are no + // collisions across alerts. const merged = new Map(); for (const result of results) { if (!result) { @@ -2192,8 +2228,16 @@ export const getConsecutiveWindowHistories = async ( (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, ); const id = new mongoose.Types.ObjectId(alert.id); - // Range scan on {alert: 1, createdAt: -1}; the newest-first order - // the consumer needs is applied below instead of a DB-side sort. + // Tight-bounds range scan on {alert: 1, createdAt: -1}; the + // newest-first ordering the consumer needs is applied in app code + // below. A DB-side $sort including `group` would force DocumentDB + // to walk the alert's full index range instead (see + // getPreviousAlertHistories). + // + // Unlike getPreviousAlertHistories, this deliberately materializes + // the result array: the consumer needs every row (sorted), so + // streaming saves nothing, and the row count is bounded by the small + // numConsecutiveWindows lookback rather than the full history window. const rows = await AlertHistory.find( { alert: id, diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts index e58acf563..42bc716ba 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts @@ -1,6 +1,5 @@ import { DisplayType } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; -import ms from 'ms'; import { createAlert } from '@/controllers/alerts'; import { createTeam } from '@/controllers/team'; @@ -20,11 +19,8 @@ import Connection from '@/models/connection'; import Dashboard from '@/models/dashboard'; import { SavedSearch } from '@/models/savedSearch'; import { Source } from '@/models/source'; +import * as checkAlertTasks from '@/tasks/checkAlerts'; import { getScheduledWindowStart } from '@/tasks/checkAlerts'; -import { - clearAlertEvaluators, - registerAlertEvaluator, -} from '@/tasks/checkAlerts/evaluators'; import { AlertProvider, AlertTaskType, @@ -746,6 +742,231 @@ describe('DefaultAlertProvider', () => { expect(result[0].conn.host).toBe('http://localhost:8124'); }); + describe('history fetch gating (HDX-4791)', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + // Alerts created by the helpers use interval '5m' with no schedule + // offset, matching processAlert's window computation. + const currentWindowStart = () => + getScheduledWindowStart(new Date(), 5, 0); + + it('skips the per-group history fetch and seeds a sentinel for alerts already evaluated in the current window', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + // Use a 1d interval so the current window (start of the UTC day) + // cannot roll over mid-test. + await Alert.updateOne({ _id: alert._id }, { $set: { interval: '1d' } }); + const windowStart = getScheduledWindowStart(new Date(), 1440, 0); + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ALERT, + group: 'group-a', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // The gate (a findOne per alert) decided the alert was already + // evaluated this window, so the per-group range find never runs. + expect(findSpy).not.toHaveBeenCalled(); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + const sentinel = details.previousMap.get(alert.id); + expect(sentinel).toBeDefined(); + expect(sentinel!.createdAt).toEqual(windowStart); + expect(sentinel!.state).toBe(AlertState.ALERT); + }); + + it('fetches full per-group history for grouped alerts that are due', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + { groupBy: 'ServiceName' }, + ); + // Latest history is from a previous window -> the alert is due. + const previousWindowStart = new Date( + currentWindowStart().getTime() - 5 * 60_000, + ); + await AlertHistory.create({ + alert: alert._id, + createdAt: previousWindowStart, + state: AlertState.ALERT, + group: 'group-a', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: previousWindowStart, + state: AlertState.OK, + group: 'group-b', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).toHaveBeenCalledTimes(1); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + const groupA = details.previousMap.get(`${alert.id}||group-a`); + expect(groupA).toBeDefined(); + expect(groupA!.state).toBe(AlertState.ALERT); + expect(groupA!.createdAt).toEqual(previousWindowStart); + expect(details.previousMap.get(`${alert.id}||group-b`)!.state).toBe( + AlertState.OK, + ); + }); + + it('seeds non-grouped due alerts from the gate row without a range scan (HDX-4980)', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + // Several previous windows of history; the newest row carries the + // state the previousMap must reflect. + const windowMs = 5 * 60_000; + const base = currentWindowStart().getTime(); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - 3 * windowMs), + state: AlertState.OK, + group: '', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - 2 * windowMs), + state: AlertState.OK, + group: '', + }); + const newest = await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - windowMs), + state: AlertState.ALERT, + group: '', + fired: true, + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // The gate's findOne already fetched the latest row; a non-grouped + // alert must not pay for the per-group range scan. + expect(findSpy).not.toHaveBeenCalled(); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + // Seeded entry is exactly what the legacy range scan would have + // reduced to: the newest row, including state and fired. + const previous = details.previousMap.get(alert.id); + expect(previous).toBeDefined(); + expect(previous!.createdAt).toEqual(newest.createdAt); + expect(previous!.state).toBe(AlertState.ALERT); + expect(previous!.fired).toBe(true); + }); + + it('skips the per-group history fetch for alerts with no history at all', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // No history exists, so there is nothing to range-scan. + expect(findSpy).not.toHaveBeenCalled(); + + // The alert is still due and loaded, just with an empty previousMap. + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + expect(details.alert.id).toBe(alert.id); + expect(details.previousMap.get(alert.id)).toBeUndefined(); + }); + + it('treats an alert as due when its window start cannot be computed', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(currentWindowStart().getTime() - 5 * 60_000), + state: AlertState.ALERT, + }); + const windowStartSpy = jest + .spyOn(checkAlertTasks, 'getAlertWindowStart') + .mockImplementation(() => { + throw new Error('lookup failed'); + }); + + const result = await provider.getAlertTasks(); + + expect(windowStartSpy).toHaveBeenCalled(); + expect(result).toHaveLength(1); + expect(result[0].alerts[0].previousMap.get(alert.id)?.state).toBe( + AlertState.ALERT, + ); + }); + + let team: Awaited>; + + beforeEach(async () => { + team = await createTeam({ name: 'Test Team' }); + }); + + const createConnectionAndSource = async () => { + const connection = await Connection.create({ + team: team._id, + name: 'Test Connection', + host: 'http://localhost:8123', + username: 'test', + password: 'test', + }); + const source = await Source.create({ + team: team._id, + name: 'Test Source', + kind: 'log', + from: { databaseName: 'default', tableName: 'logs' }, + timestampValueExpression: 'timestamp', + connection: connection._id, + }); + return { connection, source }; + }; + + const createValidSavedSearchAlert = async ( + { source }: Awaited>, + { groupBy }: { groupBy?: string } = {}, + ) => { + const savedSearch = await SavedSearch.create({ + team: team._id, + name: 'Test Search', + select: 'message', + where: 'level: error', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: source._id, + tags: [], + }); + const alert = await createAlert( + team._id, + { + source: AlertSource.SAVED_SEARCH, + savedSearchId: savedSearch._id.toString(), + threshold: 10, + thresholdType: AlertThresholdType.ABOVE, + interval: '5m', + ...(groupBy ? { groupBy } : {}), + channel: { + type: 'webhook', + webhookId: new mongoose.Types.ObjectId().toString(), + }, + }, + new mongoose.Types.ObjectId(), + ); + return { alert }; + }; + }); + it('should process a single inline alert', async () => { const team = await createTeam({ name: 'Test Team' }); @@ -951,176 +1172,6 @@ describe('DefaultAlertProvider', () => { }); }); - describe('history fetch gating', () => { - afterEach(() => { - jest.restoreAllMocks(); - clearAlertEvaluators(); - }); - - const createSavedSearchAlert = async ({ - groupBy, - interval = '5m', - }: { groupBy?: string; interval?: '5m' | '1d' } = {}) => { - const team = await createTeam({ name: 'Test Team' }); - const connection = await Connection.create({ - team: team._id, - name: 'Test Connection', - host: 'http://localhost:8123', - username: 'test', - password: 'test', - }); - const source = await Source.create({ - team: team._id, - name: 'Test Source', - kind: 'log', - from: { databaseName: 'default', tableName: 'logs' }, - timestampValueExpression: 'timestamp', - connection: connection._id, - }); - const savedSearch = await SavedSearch.create({ - team: team._id, - name: 'Test Search', - select: 'message', - where: 'level: error', - whereLanguage: 'lucene', - orderBy: 'timestamp', - source: source._id, - tags: [], - }); - return createAlert( - team._id, - { - source: AlertSource.SAVED_SEARCH, - savedSearchId: savedSearch._id.toString(), - threshold: 10, - thresholdType: AlertThresholdType.ABOVE, - interval, - groupBy, - channel: { - type: 'webhook', - webhookId: new mongoose.Types.ObjectId().toString(), - }, - }, - new mongoose.Types.ObjectId(), - ); - }; - - const previousWindowStart = () => - new Date(getScheduledWindowStart(new Date(), 5, 0).getTime() - ms('5m')); - - it('skips the history fetch and seeds a sentinel for alerts already evaluated in the current window', async () => { - // 1d interval so the current window cannot roll over mid-test. - const alert = await createSavedSearchAlert({ interval: '1d' }); - const windowStart = getScheduledWindowStart(new Date(), 1440, 0); - await AlertHistory.create({ - alert: alert._id, - createdAt: windowStart, - state: AlertState.ALERT, - group: 'group-a', - }); - - const findSpy = jest.spyOn(AlertHistory, 'find'); - const result = await provider.getAlertTasks(); - - expect(findSpy).not.toHaveBeenCalled(); - expect(result).toHaveLength(1); - const sentinel = result[0].alerts[0].previousMap.get(alert.id); - expect(sentinel?.createdAt).toEqual(windowStart); - expect(sentinel?.state).toBe(AlertState.ALERT); - }); - - it('fetches per-group history for grouped alerts that are due', async () => { - const alert = await createSavedSearchAlert({ groupBy: 'ServiceName' }); - const createdAt = previousWindowStart(); - await AlertHistory.create({ - alert: alert._id, - createdAt, - state: AlertState.ALERT, - group: 'group-a', - }); - await AlertHistory.create({ - alert: alert._id, - createdAt, - state: AlertState.OK, - group: 'group-b', - }); - - const findSpy = jest.spyOn(AlertHistory, 'find'); - const result = await provider.getAlertTasks(); - - expect(findSpy).toHaveBeenCalledTimes(1); - const { previousMap } = result[0].alerts[0]; - expect(previousMap.get(`${alert.id}||group-a`)?.state).toBe( - AlertState.ALERT, - ); - expect(previousMap.get(`${alert.id}||group-a`)?.createdAt).toEqual( - createdAt, - ); - expect(previousMap.get(`${alert.id}||group-b`)?.state).toBe( - AlertState.OK, - ); - }); - - it('seeds non-grouped due alerts from the latest row without a range scan', async () => { - const alert = await createSavedSearchAlert(); - const base = previousWindowStart().getTime(); - await AlertHistory.create({ - alert: alert._id, - createdAt: new Date(base - ms('10m')), - state: AlertState.OK, - }); - const newest = await AlertHistory.create({ - alert: alert._id, - createdAt: new Date(base), - state: AlertState.ALERT, - fired: true, - }); - - const findSpy = jest.spyOn(AlertHistory, 'find'); - const result = await provider.getAlertTasks(); - - expect(findSpy).not.toHaveBeenCalled(); - const previous = result[0].alerts[0].previousMap.get(alert.id); - expect(previous?.createdAt).toEqual(newest.createdAt); - expect(previous?.state).toBe(AlertState.ALERT); - expect(previous?.fired).toBe(true); - }); - - it('skips the history fetch for alerts with no history', async () => { - const alert = await createSavedSearchAlert({ groupBy: 'ServiceName' }); - - const findSpy = jest.spyOn(AlertHistory, 'find'); - const result = await provider.getAlertTasks(); - - expect(findSpy).not.toHaveBeenCalled(); - expect(result[0].alerts[0].alert.id).toBe(alert.id); - expect(result[0].alerts[0].previousMap.size).toBe(0); - }); - - it('treats an alert as due when its evaluator lookup fails', async () => { - const alert = await createSavedSearchAlert(); - await AlertHistory.create({ - alert: alert._id, - createdAt: previousWindowStart(), - state: AlertState.ALERT, - }); - registerAlertEvaluator({ - name: 'broken', - appliesTo: () => { - throw new Error('lookup failed'); - }, - evaluate: jest.fn(), - }); - - const result = await provider.getAlertTasks(); - - expect(result).toHaveLength(1); - expect(result[0].alerts[0].previousMap.get(alert.id)?.state).toBe( - AlertState.ALERT, - ); - }); - }); - describe('buildLogSearchLink', () => { it('should build correct URL with basic parameters', () => { const result = provider.buildLogSearchLink({ diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 23143f2c6..e4d52554b 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -397,27 +397,138 @@ async function resolveAlertDetails( return [conn, details]; } -/** - * Whether the alert's latest history row is from the window processAlert would - * evaluate now, i.e. processAlert will skip it. A failing evaluator lookup - * counts as due so processAlert can record the error on that alert. - */ -function wasEvaluatedInCurrentWindow( - alert: IAlert, - latest: LatestAlertHistory, - now: Date, -): boolean { - try { - return ( - latest.createdAt.getTime() === getAlertWindowStart(alert, now).getTime() - ); - } catch (e) { - logger.error( - { alertId: alert.id, error: serializeError(e) }, - 'Failed to compute alert window start for history gating', - ); - return false; +async function loadAlerts(): Promise { + const groupedTasks = new Map(); + + const alerts = await Alert.find({}); + + const now = new Date(); + + const resolved: { + alert: IAlert; + conn: IConnection; + details: PartialAlertDetails; + }[] = []; + for (const alert of alerts) { + try { + const [conn, details] = await resolveAlertDetails(alert); + resolved.push({ alert, conn, details }); + } catch (e) { + logger.error({ + message: `failed to load alert: ${e}`, + alertId: alert.id, + team: alert.team, + channel: alert.channel, + provider: 'default', + }); + } + } + + // Share a single queue across all history fetches so their combined + // in-flight per-alert queries stay within one global cap. + const historyQueryQueue = new PQueue({ + concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, + }); + + // Due-ness gate (HDX-4791): a cheap one-key lookup per (in-scope) alert + // decides whether the alert was already evaluated in the current window. + // Only alerts that are actually due pay for the per-group history range + // scan — without this, every task tick (once a minute) re-scans the + // history window for every alert regardless of the alert's interval, which + // was the main driver of DocDB secondary CPU / cursor timeouts. + const latestHistories = await getLatestAlertHistories( + resolved.map(({ alert }) => alert.id), + now, + historyQueryQueue, + ); + + const dueAlertsWithHistory: IAlert[] = []; + const dueGroupedAlerts: IAlert[] = []; + const dueNonGroupedAlerts: { + alert: IAlert; + latest: NonNullable>; + }[] = []; + const gatedAlerts: { + alert: IAlert; + latest: NonNullable>; + }[] = []; + let noHistoryCount = 0; + for (const { alert, details } of resolved) { + const latest = latestHistories.get(alert.id); + if (!latest) { + // Never evaluated (or history expired): the alert is due, but there is + // no previous state to fetch. + noHistoryCount += 1; + continue; + } + if (wasEvaluatedInCurrentWindow(alert, latest, now)) { + // Already evaluated in the current window — processAlert will skip it. + gatedAlerts.push({ alert, latest }); + continue; + } + dueAlertsWithHistory.push(alert); + // Non-grouped alerts only need the latest row, which the gate already + // fetched; only grouped alerts pay for the per-group range scan (its + // lookback also serves stale-group auto-resolve). + if (alertHasGroupBy(details)) { + dueGroupedAlerts.push(alert); + } else { + dueNonGroupedAlerts.push({ alert, latest }); + } + } + + const [previousAlerts, recentHistoryMap] = await Promise.all([ + getPreviousAlertHistories(dueGroupedAlerts, now, historyQueryQueue), + getConsecutiveWindowHistories(dueAlertsWithHistory, now, historyQueryQueue), + ]); + + // Seed non-grouped previous state from the gate row — identical to what + // the range scan would reduce to. + for (const { alert, latest } of dueNonGroupedAlerts) { + previousAlerts.set(computeHistoryMapKey(alert.id, latest.group || ''), { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + }); + } + + // Seed a sentinel entry for each gated alert so processAlert's + // shouldSkipAlertCheck sees a history row in the current window and skips + // the evaluation exactly as before (including the skipped_window metric). + for (const { alert, latest } of gatedAlerts) { + previousAlerts.set(alert.id, { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + }); + } + + logger.info( + { + alertCount: alerts.length, + dueCount: dueAlertsWithHistory.length, + dueGroupedCount: dueGroupedAlerts.length, + gatedCount: gatedAlerts.length, + noHistoryCount, + }, + 'Loaded alert histories with due-ness gating', + ); + + for (const { conn, details } of resolved) { + if (!groupedTasks.has(conn.id)) { + groupedTasks.set(conn.id, { alerts: [], conn, now }); + } + groupedTasks.get(conn.id)!.alerts.push({ + ...details, + previousMap: previousAlerts, + recentHistoryMap, + }); } + + // Flatten out our groupings for execution + return Array.from(groupedTasks.values()); } export default class DefaultAlertProvider implements AlertProvider { @@ -435,129 +546,7 @@ export default class DefaultAlertProvider implements AlertProvider { } async getAlertTasks(): Promise { - const groupedTasks = new Map(); - const alerts = await Alert.find({}); - - const now = new Date(); - - const resolved: { - alert: IAlert; - conn: IConnection; - details: PartialAlertDetails; - }[] = []; - for (const alert of alerts) { - try { - const [conn, details] = await resolveAlertDetails(alert); - resolved.push({ alert, conn, details }); - } catch (e) { - logger.error({ - message: `failed to load alert: ${e}`, - alertId: alert.id, - team: alert.team, - channel: alert.channel, - provider: 'default', - }); - } - } - - // Share a single queue across all history fetches so their combined - // in-flight per-alert queries stay within one global cap. - const historyQueryQueue = new PQueue({ - concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, - }); - - // One index seek per alert decides whether it is due this window, so - // alerts on longer intervals don't re-scan their history every tick. - const latestHistories = await getLatestAlertHistories( - resolved.map(({ alert }) => alert.id), - now, - historyQueryQueue, - ); - - const dueAlertsWithHistory: IAlert[] = []; - const dueGroupedAlerts: IAlert[] = []; - const dueNonGroupedAlerts: { - alert: IAlert; - latest: LatestAlertHistory; - }[] = []; - const gatedAlerts: { alert: IAlert; latest: LatestAlertHistory }[] = []; - let noHistoryCount = 0; - for (const { alert, details } of resolved) { - const latest = latestHistories.get(alert.id); - if (!latest) { - noHistoryCount += 1; - continue; - } - if (wasEvaluatedInCurrentWindow(alert, latest, now)) { - gatedAlerts.push({ alert, latest }); - continue; - } - dueAlertsWithHistory.push(alert); - // Non-grouped alerts only need the latest row, which the gate already - // fetched. Grouped alerts need the latest row of every group. - if (alertHasGroupBy(details)) { - dueGroupedAlerts.push(alert); - } else { - dueNonGroupedAlerts.push({ alert, latest }); - } - } - - const [previousAlerts, recentHistoryMap] = await Promise.all([ - getPreviousAlertHistories(dueGroupedAlerts, now, historyQueryQueue), - getConsecutiveWindowHistories( - dueAlertsWithHistory, - now, - historyQueryQueue, - ), - ]); - - for (const { alert, latest } of dueNonGroupedAlerts) { - previousAlerts.set(computeHistoryMapKey(alert.id, latest.group || ''), { - _id: new mongoose.Types.ObjectId(alert.id), - createdAt: latest.createdAt, - state: latest.state, - group: latest.group, - fired: latest.fired, - }); - } - - // A row in the current window under the bare alert ID makes - // processAlert's shouldSkipAlertCheck skip the alert, as it did before. - for (const { alert, latest } of gatedAlerts) { - previousAlerts.set(alert.id, { - _id: new mongoose.Types.ObjectId(alert.id), - createdAt: latest.createdAt, - state: latest.state, - }); - } - - logger.info( - { - alertCount: alerts.length, - resolvedCount: resolved.length, - dueCount: dueAlertsWithHistory.length, - dueGroupedCount: dueGroupedAlerts.length, - gatedCount: gatedAlerts.length, - noHistoryCount, - }, - 'Loaded alert histories', - ); - - for (const { conn, details } of resolved) { - let task = groupedTasks.get(conn.id); - if (!task) { - task = { alerts: [], conn, now }; - groupedTasks.set(conn.id, task); - } - task.alerts.push({ - ...details, - previousMap: previousAlerts, - recentHistoryMap, - }); - } - - // Flatten out our groupings for execution - return Array.from(groupedTasks.values()); + return loadAlerts(); } buildLogSearchLink({ @@ -801,3 +790,26 @@ export default class DefaultAlertProvider implements AlertProvider { }); } } + +/** + * Whether the alert's latest history row is from the window processAlert would + * evaluate now, i.e. processAlert will skip it. A failing evaluator lookup + * counts as due so processAlert can record the error on that alert. + */ +function wasEvaluatedInCurrentWindow( + alert: IAlert, + latest: LatestAlertHistory, + now: Date, +): boolean { + try { + return ( + latest.createdAt.getTime() === getAlertWindowStart(alert, now).getTime() + ); + } catch (e) { + logger.error( + { alertId: alert.id, error: serializeError(e) }, + 'Failed to compute alert window start for history gating', + ); + return false; + } +} From d5e66b9b88d0382e447de58d16cb8acc7046352f Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:20:42 -0800 Subject: [PATCH 3/7] fix(api): bound the latest-history lookup by the alert's lookback window getLatestAlertHistories had no lower createdAt bound, and its state != ERROR filter is not in the {alert, createdAt} index. For an alert that keeps failing, every tick walked back through every ERROR row until it found a non-ERROR one, up to the full 30-day retention. It also let a non-grouped alert pick up a weeks-old row as its previous state. Bound it by getAlertHistoryLookbackMs(interval), the same window getPreviousAlertHistories uses, so the skip check, the grouped scan and the non-grouped path agree. The function now takes alerts instead of IDs. Also drop an `as any` in a test that pushed the API package over its lint warning cap. Refs #2434 --- .../__tests__/checkAlerts.int.test.ts | 61 +++++++++++++++---- packages/api/src/tasks/checkAlerts/index.ts | 18 ++++-- .../tasks/checkAlerts/providers/default.ts | 2 +- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 34ba95cc1..5cebb88fe 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -10,6 +10,7 @@ import { } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; +import { z } from 'zod'; import * as config from '@/config'; import { createAlert } from '@/controllers/alerts'; @@ -12523,7 +12524,7 @@ describe('checkAlerts', () => { const alert2Id = new mongoose.Types.ObjectId(); const result = await getLatestAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [alert1Id, alert2Id].map(id => ({ id: id.toString(), interval: '5m' })), new Date('2025-01-01T00:20:00Z'), ); @@ -12540,7 +12541,7 @@ describe('checkAlerts', () => { await saveAlert(alertId, new Date('2025-01-01T00:10:00Z')); const result = await getLatestAlertHistories( - [alertId.toString()], + [{ id: alertId.toString(), interval: '5m' }], new Date('2025-01-01T00:05:00Z'), ); @@ -12722,7 +12723,7 @@ describe('checkAlerts', () => { ); const result = await getLatestAlertHistories( - [alertId.toString()], + [{ id: alertId.toString(), interval: '5m' }], new Date('2025-01-01T00:20:00Z'), ); @@ -12732,6 +12733,37 @@ describe('checkAlerts', () => { }); }); + it('getLatestAlertHistories ignores rows older than the interval-derived lookback', async () => { + const now = new Date('2025-01-10T00:00:00Z'); + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-08T00:00:00Z'), + AlertState.OK, + ); + await saveHistory( + alertId, + new Date('2025-01-09T12:00:00Z'), + AlertState.ERROR, + ); + + // 1m interval: 1d lookback, so only the ERROR row is in range. + const result = await getLatestAlertHistories( + [{ id: alertId.toString(), interval: '1m' }], + now, + ); + expect(result.get(alertId.toString())).toBeNull(); + + // 1h interval: 52h lookback reaches the OK row. + const resultHourly = await getLatestAlertHistories( + [{ id: alertId.toString(), interval: '1h' }], + now, + ); + expect(resultHourly.get(alertId.toString())?.createdAt).toEqual( + new Date('2025-01-08T00:00:00Z'), + ); + }); + it('scans the {alert, createdAt} index with tight createdAt bounds', async () => { const alertId = new mongoose.Types.ObjectId(); const now = new Date('2025-01-10T00:00:00Z'); @@ -12744,14 +12776,21 @@ describe('checkAlerts', () => { } await AlertHistory.syncIndexes(); - const explain = (await AlertHistory.find({ - alert: alertId, - createdAt: { - $lte: now, - $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), - }, - state: { $ne: AlertState.ERROR }, - }).explain('executionStats')) as any; + const explain = z + .object({ + queryPlanner: z.object({ winningPlan: z.unknown() }), + executionStats: z.object({ totalKeysExamined: z.number() }), + }) + .parse( + await AlertHistory.find({ + alert: alertId, + createdAt: { + $lte: now, + $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), + }, + state: { $ne: AlertState.ERROR }, + }).explain('executionStats'), + ); expect(JSON.stringify(explain.queryPlanner.winningPlan)).toContain( 'alert_1_createdAt_-1', diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 4f95a0658..5ecdf547c 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -2012,10 +2012,11 @@ export type LatestAlertHistory = Pick< * previous state itself. * * @returns A map from alert ID to its latest history row, or null when the - * alert has no history at all. + * alert has no history within its lookback window (see + * getAlertHistoryLookbackMs). */ export const getLatestAlertHistories = async ( - alertIds: string[], + alerts: Array>, now: Date, sharedQueue?: PQueue, ): Promise> => { @@ -2023,12 +2024,21 @@ export const getLatestAlertHistories = async ( sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); const results = await Promise.all( - alertIds.map(alertId => + alerts.map(alert => queue.add(async () => { + const alertId = alert.id; const latest = await AlertHistory.findOne( { alert: new mongoose.Types.ObjectId(alertId), - createdAt: { $lte: now }, + // The lower bound caps the walk past consecutive ERROR rows (the + // state filter is not in the index) for an alert that keeps + // failing, and matches getPreviousAlertHistories' window. + createdAt: { + $lte: now, + $gte: new Date( + now.getTime() - getAlertHistoryLookbackMs(alert.interval), + ), + }, // ERROR rows record failed evaluations; they must not count as // "window evaluated" or the failed window would never be // retried/backfilled. diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index e4d52554b..79fae67b8 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -437,7 +437,7 @@ async function loadAlerts(): Promise { // history window for every alert regardless of the alert's interval, which // was the main driver of DocDB secondary CPU / cursor timeouts. const latestHistories = await getLatestAlertHistories( - resolved.map(({ alert }) => alert.id), + resolved.map(({ alert }) => alert), now, historyQueryQueue, ); From 6a781df2bc3a8af64ef099e1243fe7dcbfe3a640 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:29:23 -0800 Subject: [PATCH 4/7] test(api): type the consecutive-window find() stub instead of casting to any The escape-hatch ratchet caps 'as any' in the api package; the stub added one over the baseline. Refs #2434 --- packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts index c8e75ec69..e06dd5688 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts @@ -325,7 +325,7 @@ describe('alert evaluators', () => { .mockResolvedValue([ { createdAt: new Date(), state: AlertState.ALERT, group: '' }, ]), - } as any); + } as unknown as ReturnType); try { const map = await getConsecutiveWindowHistories( From 8bba968f2cd75be1ca49c7ca4f083103852ea2ca Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:38:28 -0800 Subject: [PATCH 5/7] fix(api): bound the latest-history lookup at 7 days, not the alert interval The interval-derived bound (1 day for 1m to 15m alerts) made non-grouped alerts forget their previous state after 1 day. An alert in ALERT whose evaluations only wrote ERROR rows for longer than that would recover without sending its resolve notification. main kept that state for 7 days. Use ALERT_HISTORY_MAX_LOOKBACK_MS instead. It still caps the walk past ERROR rows, at the 7 days main already scanned, and restores the enterprise build's signature, so the call site and tests match it again. Refs #2434 --- .../__tests__/checkAlerts.int.test.ts | 31 +++++++++---------- packages/api/src/tasks/checkAlerts/index.ts | 16 +++------- .../tasks/checkAlerts/providers/default.ts | 2 +- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 5cebb88fe..1b56108ac 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -12524,7 +12524,7 @@ describe('checkAlerts', () => { const alert2Id = new mongoose.Types.ObjectId(); const result = await getLatestAlertHistories( - [alert1Id, alert2Id].map(id => ({ id: id.toString(), interval: '5m' })), + [alert1Id.toString(), alert2Id.toString()], new Date('2025-01-01T00:20:00Z'), ); @@ -12541,7 +12541,7 @@ describe('checkAlerts', () => { await saveAlert(alertId, new Date('2025-01-01T00:10:00Z')); const result = await getLatestAlertHistories( - [{ id: alertId.toString(), interval: '5m' }], + [alertId.toString()], new Date('2025-01-01T00:05:00Z'), ); @@ -12723,7 +12723,7 @@ describe('checkAlerts', () => { ); const result = await getLatestAlertHistories( - [{ id: alertId.toString(), interval: '5m' }], + [alertId.toString()], new Date('2025-01-01T00:20:00Z'), ); @@ -12733,13 +12733,13 @@ describe('checkAlerts', () => { }); }); - it('getLatestAlertHistories ignores rows older than the interval-derived lookback', async () => { + it('getLatestAlertHistories ignores rows older than 7 days', async () => { const now = new Date('2025-01-10T00:00:00Z'); const alertId = new mongoose.Types.ObjectId(); await saveHistory( alertId, - new Date('2025-01-08T00:00:00Z'), - AlertState.OK, + new Date('2025-01-02T00:00:00Z'), + AlertState.ALERT, ); await saveHistory( alertId, @@ -12747,21 +12747,18 @@ describe('checkAlerts', () => { AlertState.ERROR, ); - // 1m interval: 1d lookback, so only the ERROR row is in range. - const result = await getLatestAlertHistories( - [{ id: alertId.toString(), interval: '1m' }], - now, - ); + // The ALERT row is 8 days old, so only the ERROR row is in range. + const result = await getLatestAlertHistories([alertId.toString()], now); expect(result.get(alertId.toString())).toBeNull(); - // 1h interval: 52h lookback reaches the OK row. - const resultHourly = await getLatestAlertHistories( - [{ id: alertId.toString(), interval: '1h' }], - now, - ); - expect(resultHourly.get(alertId.toString())?.createdAt).toEqual( + // Six days after the ALERT row, it is still found. + const sixDaysLater = await getLatestAlertHistories( + [alertId.toString()], new Date('2025-01-08T00:00:00Z'), ); + expect(sixDaysLater.get(alertId.toString())?.state).toBe( + AlertState.ALERT, + ); }); it('scans the {alert, createdAt} index with tight createdAt bounds', async () => { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 5ecdf547c..404108a7f 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -2012,11 +2012,10 @@ export type LatestAlertHistory = Pick< * previous state itself. * * @returns A map from alert ID to its latest history row, or null when the - * alert has no history within its lookback window (see - * getAlertHistoryLookbackMs). + * alert has no history at all. */ export const getLatestAlertHistories = async ( - alerts: Array>, + alertIds: string[], now: Date, sharedQueue?: PQueue, ): Promise> => { @@ -2024,20 +2023,15 @@ export const getLatestAlertHistories = async ( sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); const results = await Promise.all( - alerts.map(alert => + alertIds.map(alertId => queue.add(async () => { - const alertId = alert.id; const latest = await AlertHistory.findOne( { alert: new mongoose.Types.ObjectId(alertId), - // The lower bound caps the walk past consecutive ERROR rows (the - // state filter is not in the index) for an alert that keeps - // failing, and matches getPreviousAlertHistories' window. + // Caps the walk past ERROR rows, which the index can't skip. createdAt: { $lte: now, - $gte: new Date( - now.getTime() - getAlertHistoryLookbackMs(alert.interval), - ), + $gte: new Date(now.getTime() - ALERT_HISTORY_MAX_LOOKBACK_MS), }, // ERROR rows record failed evaluations; they must not count as // "window evaluated" or the failed window would never be diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 79fae67b8..e4d52554b 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -437,7 +437,7 @@ async function loadAlerts(): Promise { // history window for every alert regardless of the alert's interval, which // was the main driver of DocDB secondary CPU / cursor timeouts. const latestHistories = await getLatestAlertHistories( - resolved.map(({ alert }) => alert), + resolved.map(({ alert }) => alert.id), now, historyQueryQueue, ); From 9a5cfed3b161cbf82e05a8f1332bedceb0a15dff Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:47:15 -0800 Subject: [PATCH 6/7] fix(api): keep a grouped alert's catch-up when its last row predates the scan The latest-row lookup covers 7 days, but the per-group scan covers only the alert's interval-derived window (1 day for fast alerts). After an outage longer than that window the scan returned nothing, so the evaluation started from now - 1 window and skipped the missed windows that main would have caught up on (up to 50). When the scan has no row for a due grouped alert, seed the row from the latest-row lookup. The evaluation starts from it again, and that group's previous state is restored. Refs #2434 --- .../providers/__tests__/default.int.test.ts | 29 +++++++++++++++++++ .../tasks/checkAlerts/providers/default.ts | 21 ++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts index 42bc716ba..d7ee75e97 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts @@ -884,6 +884,35 @@ describe('DefaultAlertProvider', () => { expect(details.previousMap.get(alert.id)).toBeUndefined(); }); + it('seeds a grouped alert from its latest row when that row is older than the scan window', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + { groupBy: 'ServiceName' }, + ); + // 5m interval: the per-group scan covers 1 day, so a 2-day-old row is + // outside it but inside the 7-day latest-row lookup. + const twoDaysAgo = new Date( + currentWindowStart().getTime() - 2 * 24 * 60 * 60_000, + ); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.ALERT, + group: 'group-a', + fired: true, + }); + + const result = await provider.getAlertTasks(); + + expect(result).toHaveLength(1); + const { previousMap } = result[0].alerts[0]; + expect(previousMap.size).toBe(1); + const groupA = previousMap.get(`${alert.id}||group-a`); + expect(groupA?.createdAt).toEqual(twoDaysAgo); + expect(groupA?.state).toBe(AlertState.ALERT); + expect(groupA?.fired).toBe(true); + }); + it('treats an alert as due when its window start cannot be computed', async () => { const { alert } = await createValidSavedSearchAlert( await createConnectionAndSource(), diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index e4d52554b..6f8e3972d 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -494,6 +494,27 @@ async function loadAlerts(): Promise { }); } + // A grouped alert whose latest row is older than its scan window gets no + // rows back. Seed that row so the evaluation still starts from it and + // catches up on the missed windows. When the scan did return rows, the + // latest one is among them and its key is already set. + for (const alert of dueGroupedAlerts) { + const latest = latestHistories.get(alert.id); + if (!latest) { + continue; + } + const key = computeHistoryMapKey(alert.id, latest.group || ''); + if (!previousAlerts.has(key)) { + previousAlerts.set(key, { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + }); + } + } + // Seed a sentinel entry for each gated alert so processAlert's // shouldSkipAlertCheck sees a history row in the current window and skips // the evaluation exactly as before (including the skipped_window metric). From 3511be90f433ef328587c99a3e55a2fb5c93eff3 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:24:47 -0800 Subject: [PATCH 7/7] fix(api): restore every group from the last evaluation after a long outage When a due grouped alert's latest row is older than its scan window, the provider seeded only that one row, without groupValues. A group that then resolved was notified with empty group attributes, and the other groups from that evaluation lost their previous state. Read all rows that share the latest row's createdAt instead: one per group, with groupValues. It is one extra query per affected alert, on the first tick after such an outage. Tests: the last evaluation is restored per group with its values; a grouped alert already evaluated this window is skipped from the loaded sentinel alone; the window-start failure test now tells "due" from "skipped". Refs #2434 --- .../__tests__/checkAlerts.int.test.ts | 59 +++++++++++++++++++ .../providers/__tests__/default.int.test.ts | 53 +++++++++++++---- .../tasks/checkAlerts/providers/default.ts | 52 ++++++++++------ 3 files changed, 136 insertions(+), 28 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 1b56108ac..f44af5385 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -7063,6 +7063,65 @@ describe('checkAlerts', () => { }); }); + it('skips a grouped alert already evaluated this window, from the loaded sentinel alone', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + // 1d interval so the current window cannot roll over mid-test. + const { alert } = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '1d', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + const windowStart = getScheduledWindowStart(new Date(), 1440, 0); + for (const group of ['ServiceName:a', 'ServiceName:b']) { + await AlertHistory.create({ + alert: alert.id, + createdAt: windowStart, + state: AlertState.ALERT, + group, + fired: true, + counts: 1, + lastValues: [], + }); + } + + const [task] = partitionPreviousMapsByAlert( + await alertProvider.getAlertTasks(), + ); + const loaded = task.alerts[0]; + // No per-group rows were loaded, only the sentinel. + expect([...loaded.previousMap.keys()]).toEqual([alert.id]); + + const querySpy = jest.spyOn(clickhouseClient, 'queryChartConfig'); + await processAlert( + task.now, + loaded, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + expect(querySpy).not.toHaveBeenCalled(); + expect(await AlertHistory.countDocuments({ alert: alert.id })).toBe(2); + }); + // The sample rows quoted in the message body carry no group predicate, so // every group's notification used to re-run the identical query. it('fetches the message body sample rows once for all groups in a window', async () => { diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts index d7ee75e97..516685e08 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts @@ -884,13 +884,13 @@ describe('DefaultAlertProvider', () => { expect(details.previousMap.get(alert.id)).toBeUndefined(); }); - it('seeds a grouped alert from its latest row when that row is older than the scan window', async () => { + it('reads the last evaluation for a grouped alert whose latest rows are older than the scan window', async () => { const { alert } = await createValidSavedSearchAlert( await createConnectionAndSource(), { groupBy: 'ServiceName' }, ); - // 5m interval: the per-group scan covers 1 day, so a 2-day-old row is - // outside it but inside the 7-day latest-row lookup. + // 5m interval: the per-group scan covers 1 day, so a 2-day-old + // evaluation is outside it but inside the 7-day latest-row lookup. const twoDaysAgo = new Date( currentWindowStart().getTime() - 2 * 24 * 60 * 60_000, ); @@ -898,29 +898,61 @@ describe('DefaultAlertProvider', () => { alert: alert._id, createdAt: twoDaysAgo, state: AlertState.ALERT, - group: 'group-a', + group: 'ServiceName:a', fired: true, + groupValues: [{ key: 'ServiceName', value: 'a' }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.OK, + group: 'ServiceName:b', + }); + // An earlier evaluation and a failed one must not be picked up. + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(twoDaysAgo.getTime() - 5 * 60_000), + state: AlertState.ALERT, + group: 'ServiceName:c', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.ERROR, }); const result = await provider.getAlertTasks(); expect(result).toHaveLength(1); const { previousMap } = result[0].alerts[0]; - expect(previousMap.size).toBe(1); - const groupA = previousMap.get(`${alert.id}||group-a`); + expect([...previousMap.keys()].sort()).toEqual([ + `${alert.id}||ServiceName:a`, + `${alert.id}||ServiceName:b`, + ]); + const groupA = previousMap.get(`${alert.id}||ServiceName:a`); expect(groupA?.createdAt).toEqual(twoDaysAgo); expect(groupA?.state).toBe(AlertState.ALERT); expect(groupA?.fired).toBe(true); + expect(groupA?.groupValues).toEqual([ + { key: 'ServiceName', value: 'a' }, + ]); + expect(previousMap.get(`${alert.id}||ServiceName:b`)?.state).toBe( + AlertState.OK, + ); }); it('treats an alert as due when its window start cannot be computed', async () => { const { alert } = await createValidSavedSearchAlert( await createConnectionAndSource(), ); + // 1d interval and a row at the current window start: without the + // failure this alert would be skipped, with only a sentinel entry. + await Alert.updateOne({ _id: alert._id }, { $set: { interval: '1d' } }); await AlertHistory.create({ alert: alert._id, - createdAt: new Date(currentWindowStart().getTime() - 5 * 60_000), + createdAt: getScheduledWindowStart(new Date(), 1440, 0), state: AlertState.ALERT, + fired: true, }); const windowStartSpy = jest .spyOn(checkAlertTasks, 'getAlertWindowStart') @@ -932,9 +964,10 @@ describe('DefaultAlertProvider', () => { expect(windowStartSpy).toHaveBeenCalled(); expect(result).toHaveLength(1); - expect(result[0].alerts[0].previousMap.get(alert.id)?.state).toBe( - AlertState.ALERT, - ); + // The sentinel carries no `fired`; the real previous row does. + const previous = result[0].alerts[0].previousMap.get(alert.id); + expect(previous?.state).toBe(AlertState.ALERT); + expect(previous?.fired).toBe(true); }); let team: Awaited>; diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 6f8e3972d..bed6d5210 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -495,25 +495,41 @@ async function loadAlerts(): Promise { } // A grouped alert whose latest row is older than its scan window gets no - // rows back. Seed that row so the evaluation still starts from it and - // catches up on the missed windows. When the scan did return rows, the - // latest one is among them and its key is already set. - for (const alert of dueGroupedAlerts) { - const latest = latestHistories.get(alert.id); - if (!latest) { - continue; - } - const key = computeHistoryMapKey(alert.id, latest.group || ''); - if (!previousAlerts.has(key)) { - previousAlerts.set(key, { - _id: new mongoose.Types.ObjectId(alert.id), - createdAt: latest.createdAt, - state: latest.state, - group: latest.group, - fired: latest.fired, + // rows back. Read the rows of that last evaluation instead (one per group, + // all sharing its createdAt), so the evaluation still starts from it and + // every group keeps its previous state. + await Promise.all( + dueGroupedAlerts.map(alert => { + const latest = latestHistories.get(alert.id); + if ( + !latest || + previousAlerts.has(computeHistoryMapKey(alert.id, latest.group || '')) + ) { + return undefined; + } + return historyQueryQueue.add(async () => { + const alertObjectId = new mongoose.Types.ObjectId(alert.id); + const rows = await AlertHistory.find( + { + alert: alertObjectId, + createdAt: latest.createdAt, + state: { $ne: AlertState.ERROR }, + }, + { group: 1, state: 1, createdAt: 1, fired: 1, groupValues: 1 }, + ).lean(); + for (const row of rows) { + previousAlerts.set(computeHistoryMapKey(alert.id, row.group || ''), { + _id: alertObjectId, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + groupValues: row.groupValues, + }); + } }); - } - } + }), + ); // Seed a sentinel entry for each gated alert so processAlert's // shouldSkipAlertCheck sees a history row in the current window and skips