diff --git a/.changeset/alert-history-lookback.md b/.changeset/alert-history-lookback.md new file mode 100644 index 000000000..7127a6158 --- /dev/null +++ b/.changeset/alert-history-lookback.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/api': patch +--- + +perf: bound the check-alerts previous-history lookup by the alert interval + +The check-alerts task no longer scans 7 days of alert history per alert on +every tick. A one-row lookup per alert skips alerts already evaluated in the +current window. Non-grouped alerts take their previous state from that row, +and grouped alerts scan a window sized to their interval (1 to 7 days). diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 53712c30a..203fbe074 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -169,12 +169,13 @@ AlertHistorySchema.index( { expireAfterSeconds: ms('30d') / 1000 }, ); -// Used by getRecentAlertHistories (matches on alert, sorts by createdAt) +// Used by all alert/task reads: getRecentAlertHistories, +// getAlertTransitionsInRange, getLatestAlertHistories, +// getPreviousAlertHistories, and getConsecutiveWindowHistories — all match on +// alert equality with a createdAt range/sort, which this index serves with +// tight bounds. AlertHistorySchema.index({ alert: 1, createdAt: -1 }); -// Used by getPreviousAlertHistories (groups by {alert, group}, sorts by createdAt) -AlertHistorySchema.index({ alert: 1, group: 1, createdAt: -1 }); - export default mongoose.model( 'AlertHistory', AlertHistorySchema, diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 30badac6d..f44af5385 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -10,6 +10,7 @@ import { } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; +import { z } from 'zod'; import * as config from '@/config'; import { createAlert } from '@/controllers/alerts'; @@ -39,7 +40,9 @@ import * as checkAlert from '@/tasks/checkAlerts'; import { alertHasGroupBy, doesExceedThreshold, + getAlertHistoryLookbackMs, getConsecutiveWindowHistories, + getLatestAlertHistories, getPreviousAlertHistories, getScheduledWindowStart, parseAlertData, @@ -2152,7 +2155,7 @@ describe('checkAlerts', () => { teamWebhooksById: Map, ) => { const [previousMap, recentHistoryMap] = await Promise.all([ - getPreviousAlertHistories([details.alert.id], now), + getPreviousAlertHistories([details.alert], now), getConsecutiveWindowHistories([details.alert], now), ]); await processAlert( @@ -6986,7 +6989,7 @@ describe('checkAlerts', () => { teamWebhooksById, }: Awaited>) => { const shared = await getPreviousAlertHistories( - [grouped.alert.id, firing.alert.id], + [grouped.alert, firing.alert], now, ); expect(shared.get(firing.alert.id)?.state).toBe(AlertState.ALERT); @@ -7060,6 +7063,65 @@ describe('checkAlerts', () => { }); }); + it('skips a grouped alert already evaluated this window, from the loaded sentinel alone', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + // 1d interval so the current window cannot roll over mid-test. + const { alert } = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '1d', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + const windowStart = getScheduledWindowStart(new Date(), 1440, 0); + for (const group of ['ServiceName:a', 'ServiceName:b']) { + await AlertHistory.create({ + alert: alert.id, + createdAt: windowStart, + state: AlertState.ALERT, + group, + fired: true, + counts: 1, + lastValues: [], + }); + } + + const [task] = partitionPreviousMapsByAlert( + await alertProvider.getAlertTasks(), + ); + const loaded = task.alerts[0]; + // No per-group rows were loaded, only the sentinel. + expect([...loaded.previousMap.keys()]).toEqual([alert.id]); + + const querySpy = jest.spyOn(clickhouseClient, 'queryChartConfig'); + await processAlert( + task.now, + loaded, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + expect(querySpy).not.toHaveBeenCalled(); + expect(await AlertHistory.countDocuments({ alert: alert.id })).toBe(2); + }); + // The sample rows quoted in the message body carry no group predicate, so // every group's notification used to re-run the identical query. it('fetches the message body sample rows once for all groups in a window', async () => { @@ -9084,7 +9146,7 @@ describe('checkAlerts', () => { // Should NOT rescan 22:00-22:05 where service-b had data but was already checked const nextRun = new Date('2023-11-16T22:23:00.000Z'); const previousMapNextRun = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextRun, ); @@ -12058,10 +12120,7 @@ describe('checkAlerts', () => { ]); // Act - Run alerts twice to cover two periods - let previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); + let previousMap = await getPreviousAlertHistories([details.alert], now); await processAlert( now, { @@ -12076,7 +12135,7 @@ describe('checkAlerts', () => { const nextWindow = new Date('2023-11-16T22:15:00.000Z'); previousMap = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextWindow, ); await processAlert( @@ -12202,10 +12261,7 @@ describe('checkAlerts', () => { ]); // Act - Run alerts twice to cover two periods - let previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); + let previousMap = await getPreviousAlertHistories([details.alert], now); await processAlert( now, { @@ -12220,7 +12276,7 @@ describe('checkAlerts', () => { const nextWindow = new Date('2023-11-16T22:15:00.000Z'); previousMap = await getPreviousAlertHistories( - [details.alert.id], + [details.alert], nextWindow, ); await processAlert( @@ -12274,14 +12330,25 @@ describe('checkAlerts', () => { await server.stop(); }); - const saveAlert = (id: mongoose.Types.ObjectId, createdAt: Date) => { + const saveAlert = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + group?: string, + ) => { return new AlertHistory({ alert: id, createdAt, state: AlertState.OK, + group, }).save(); }; + // getPreviousAlertHistories only reads id + interval off the alert. + const asAlert = ( + id: mongoose.Types.ObjectId, + interval: IAlert['interval'] = '5m', + ) => ({ id: id.toString(), interval }); + it('should return the latest alert history for each alert', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12292,7 +12359,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12305,6 +12372,31 @@ describe('checkAlerts', () => { ); }); + it('should return the latest history per group for grouped alerts', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-a'); + await saveAlert(alertId, new Date('2025-01-01T00:05:00Z'), 'group-a'); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z'), 'group-b'); + // A row without a group should key off the bare alert ID + await saveAlert(alertId, new Date('2025-01-01T00:05:00Z')); + + const result = await getPreviousAlertHistories( + [asAlert(alertId)], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.size).toBe(3); + expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + expect(result.get(`${alertId.toString()}||group-b`)!.createdAt).toEqual( + new Date('2025-01-01T00:00:00Z'), + ); + expect(result.get(alertId.toString())!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + }); + it('should not return alert histories from the future', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12315,7 +12407,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); // This one is in the future const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:14:00Z'), ); @@ -12328,6 +12420,29 @@ describe('checkAlerts', () => { ); }); + it('should not return alert histories older than the interval-derived lookback', async () => { + const now = new Date('2025-01-10T00:00:00Z'); + const alertId = new mongoose.Types.ObjectId(); + // 1m interval → lookback floor of 1 day. A row from 2 days ago is out of + // range even though a fixed 7d window would have included it. + await saveAlert(alertId, new Date('2025-01-08T00:00:00Z')); + const result = await getPreviousAlertHistories( + [asAlert(alertId, '1m')], + now, + ); + expect(result.size).toBe(0); + + // 1h interval → lookback of 52 windows (52h) picks the same row up. + const resultHourly = await getPreviousAlertHistories( + [asAlert(alertId, '1h')], + now, + ); + expect(resultHourly.size).toBe(1); + expect(resultHourly.get(alertId.toString())!.createdAt).toEqual( + new Date('2025-01-08T00:00:00Z'), + ); + }); + it('should not return a history if there are no histories for the given alert', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); @@ -12336,7 +12451,7 @@ describe('checkAlerts', () => { const alert2Id = new mongoose.Types.ObjectId(); const result = await getPreviousAlertHistories( - [alert1Id.toString(), alert2Id.toString()], + [asAlert(alert1Id), asAlert(alert2Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12357,7 +12472,7 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); const result = await getPreviousAlertHistories( - [alert1Id.toString()], + [asAlert(alert1Id)], new Date('2025-01-01T00:20:00Z'), ); @@ -12367,7 +12482,7 @@ describe('checkAlerts', () => { ); }); - it('should issue one aggregation per alert ID (per-alert queries)', async () => { + it('should issue one range find per alert (no aggregation, no chunking)', async () => { const alert1Id = new mongoose.Types.ObjectId(); await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); await saveAlert(alert1Id, new Date('2025-01-01T00:05:00Z')); @@ -12376,24 +12491,28 @@ describe('checkAlerts', () => { await saveAlert(alert2Id, new Date('2025-01-01T00:10:00Z')); await saveAlert(alert2Id, new Date('2025-01-01T00:15:00Z')); + const findSpy = jest.spyOn(AlertHistory, 'find'); const aggregateSpy = jest.spyOn(AlertHistory, 'aggregate'); + const cursorSpy = jest.spyOn(mongoose.Query.prototype, 'cursor'); - const fakeAlertIds = Array(150) + const fakeAlerts = Array(150) .fill(null) - .map(() => new mongoose.Types.ObjectId().toString()); - const allIds = [ - alert1Id.toString(), - ...fakeAlertIds, - alert2Id.toString(), - ]; + .map(() => asAlert(new mongoose.Types.ObjectId())); + const allAlerts = [asAlert(alert1Id), ...fakeAlerts, asAlert(alert2Id)]; const result = await getPreviousAlertHistories( - allIds, + allAlerts, new Date('2025-01-01T00:20:00Z'), ); - // One aggregation per alert ID (no chunking) - expect(aggregateSpy).toHaveBeenCalledTimes(allIds.length); + // One tight-bounds find per alert; no aggregation pipeline is used + // (DocumentDB cannot serve the old $sort+$group shape from an index). + expect(findSpy).toHaveBeenCalledTimes(allAlerts.length); + expect(aggregateSpy).not.toHaveBeenCalled(); + // Each find is consumed through a streaming cursor rather than + // materialized as an array — the reduce only keeps the latest row per + // group, so buffering O(groups × windows) rows would waste heap. + expect(cursorSpy).toHaveBeenCalledTimes(allAlerts.length); expect(result.size).toBe(2); expect(result.get(alert1Id.toString())!.createdAt).toEqual( new Date('2025-01-01T00:05:00Z'), @@ -12404,6 +12523,94 @@ describe('checkAlerts', () => { }); }); + describe('getAlertHistoryLookbackMs', () => { + it('applies the 1d floor for short intervals', () => { + expect(getAlertHistoryLookbackMs('1m')).toBe(ms('1d')); + expect(getAlertHistoryLookbackMs('5m')).toBe(ms('1d')); + expect(getAlertHistoryLookbackMs('15m')).toBe(ms('1d')); + }); + + it('derives 52 windows for mid-size intervals', () => { + expect(getAlertHistoryLookbackMs('30m')).toBe(52 * ms('30m')); + expect(getAlertHistoryLookbackMs('1h')).toBe(52 * ms('1h')); + }); + + it('caps at 7d for long intervals', () => { + // 52 windows of 6h/12h/1d all exceed 7 days + expect(getAlertHistoryLookbackMs('6h')).toBe(ms('7d')); + expect(getAlertHistoryLookbackMs('12h')).toBe(ms('7d')); + expect(getAlertHistoryLookbackMs('1d')).toBe(ms('7d')); + }); + }); + + describe('getLatestAlertHistories', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + jest.clearAllMocks(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const saveAlert = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + state: AlertState = AlertState.OK, + ) => { + return new AlertHistory({ + alert: id, + createdAt, + state, + }).save(); + }; + + it('returns the latest createdAt/state per alert and null when no history exists', async () => { + const alert1Id = new mongoose.Types.ObjectId(); + await saveAlert(alert1Id, new Date('2025-01-01T00:00:00Z')); + await saveAlert( + alert1Id, + new Date('2025-01-01T00:05:00Z'), + AlertState.ALERT, + ); + + const alert2Id = new mongoose.Types.ObjectId(); + + const result = await getLatestAlertHistories( + [alert1Id.toString(), alert2Id.toString()], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(alert1Id.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:05:00Z'), + state: AlertState.ALERT, + }); + expect(result.get(alert2Id.toString())).toBeNull(); + }); + + it('ignores histories created after now', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveAlert(alertId, new Date('2025-01-01T00:00:00Z')); + await saveAlert(alertId, new Date('2025-01-01T00:10:00Z')); + + const result = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-01T00:05:00Z'), + ); + + expect(result.get(alertId.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:00:00Z'), + state: AlertState.OK, + }); + }); + }); + describe('getConsecutiveWindowHistories', () => { const server = getServer(); @@ -12451,14 +12658,14 @@ describe('checkAlerts', () => { const alertId = new mongoose.Types.ObjectId(); await saveHistory(alertId, new Date('2025-01-01T00:10:00Z')); - const aggregateSpy = jest.spyOn(AlertHistory, 'aggregate'); + const findSpy = jest.spyOn(AlertHistory, 'find'); const result = await getConsecutiveWindowHistories( [makeAlert(alertId, 1), makeAlert(new mongoose.Types.ObjectId())], new Date('2025-01-01T00:17:00Z'), ); - expect(aggregateSpy).not.toHaveBeenCalled(); + expect(findSpy).not.toHaveBeenCalled(); expect(result.size).toBe(0); }); @@ -12514,4 +12721,138 @@ describe('checkAlerts', () => { ]); }); }); + describe('alert history query shape', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const saveHistory = ( + id: mongoose.Types.ObjectId, + createdAt: Date, + state: AlertState, + group?: string, + ) => new AlertHistory({ alert: id, createdAt, state, group }).save(); + + it('getPreviousAlertHistories ignores ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-01T00:05:00Z'), + AlertState.OK, + 'group-a', + ); + await saveHistory( + alertId, + new Date('2025-01-01T00:10:00Z'), + AlertState.ERROR, + 'group-a', + ); + + const result = await getPreviousAlertHistories( + [{ id: alertId.toString(), interval: '5m' }], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(`${alertId.toString()}||group-a`)!.createdAt).toEqual( + new Date('2025-01-01T00:05:00Z'), + ); + }); + + it('getLatestAlertHistories ignores ERROR rows', async () => { + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-01T00:00:00Z'), + AlertState.OK, + ); + await saveHistory( + alertId, + new Date('2025-01-01T00:05:00Z'), + AlertState.ERROR, + ); + + const result = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-01T00:20:00Z'), + ); + + expect(result.get(alertId.toString())).toEqual({ + createdAt: new Date('2025-01-01T00:00:00Z'), + state: AlertState.OK, + }); + }); + + it('getLatestAlertHistories ignores rows older than 7 days', async () => { + const now = new Date('2025-01-10T00:00:00Z'); + const alertId = new mongoose.Types.ObjectId(); + await saveHistory( + alertId, + new Date('2025-01-02T00:00:00Z'), + AlertState.ALERT, + ); + await saveHistory( + alertId, + new Date('2025-01-09T12:00:00Z'), + AlertState.ERROR, + ); + + // The ALERT row is 8 days old, so only the ERROR row is in range. + const result = await getLatestAlertHistories([alertId.toString()], now); + expect(result.get(alertId.toString())).toBeNull(); + + // Six days after the ALERT row, it is still found. + const sixDaysLater = await getLatestAlertHistories( + [alertId.toString()], + new Date('2025-01-08T00:00:00Z'), + ); + expect(sixDaysLater.get(alertId.toString())?.state).toBe( + AlertState.ALERT, + ); + }); + + it('scans the {alert, createdAt} index with tight createdAt bounds', async () => { + const alertId = new mongoose.Types.ObjectId(); + const now = new Date('2025-01-10T00:00:00Z'); + for (let i = 0; i < 10; i++) { + await saveHistory( + alertId, + new Date(now.getTime() - i * ms('1d')), + AlertState.OK, + ); + } + await AlertHistory.syncIndexes(); + + const explain = z + .object({ + queryPlanner: z.object({ winningPlan: z.unknown() }), + executionStats: z.object({ totalKeysExamined: z.number() }), + }) + .parse( + await AlertHistory.find({ + alert: alertId, + createdAt: { + $lte: now, + $gte: new Date(now.getTime() - getAlertHistoryLookbackMs('5m')), + }, + state: { $ne: AlertState.ERROR }, + }).explain('executionStats'), + ); + + expect(JSON.stringify(explain.queryPlanner.winningPlan)).toContain( + 'alert_1_createdAt_-1', + ); + // 1d lookback: only today's and yesterday's rows are touched. + expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(2); + }); + }); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts index 77b317248..e06dd5688 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluators.test.ts @@ -316,11 +316,16 @@ describe('alert evaluators', () => { }); const healthy = makeAlert({ numConsecutiveWindows: 2 }); const broken = makeAlert({ numConsecutiveWindows: 2, threshold: 42 }); - const aggregate = jest - .spyOn(AlertHistory, 'aggregate') - .mockResolvedValue([ - { createdAt: new Date(), state: AlertState.ALERT, group: '' }, - ]); + // EE reads consecutive-window history with find().lean() rather than + // aggregate (HDX-4791). + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test double only needs lean() + const find = jest.spyOn(AlertHistory, 'find').mockReturnValue({ + lean: jest + .fn() + .mockResolvedValue([ + { createdAt: new Date(), state: AlertState.ALERT, group: '' }, + ]), + } as unknown as ReturnType); try { const map = await getConsecutiveWindowHistories( @@ -328,11 +333,11 @@ describe('alert evaluators', () => { new Date('2025-01-01T00:07:00Z'), ); - expect(aggregate).toHaveBeenCalledTimes(1); + expect(find).toHaveBeenCalledTimes(1); expect(map.has(healthy.id)).toBe(true); expect(map.has(broken.id)).toBe(false); } finally { - aggregate.mockRestore(); + find.mockRestore(); } }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index b4bc6b5bf..404108a7f 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -39,6 +39,7 @@ import { import { ALERT_NOTIFICATION_TARGETS_LIMIT, AlertErrorType, + AlertInterval, AlertNotificationTargetTiming, AlertThresholdType, BuilderChartConfigWithOptDateRange, @@ -51,7 +52,6 @@ import { } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; -import { ObjectId } from 'mongoose'; import mongoose from 'mongoose'; import ms from 'ms'; import { performance } from 'perf_hooks'; @@ -103,9 +103,10 @@ import { } from '@/tasks/checkAlerts/template'; import { handleSendGenericWebhook } from '@/tasks/checkAlerts/transports'; import { tasksTracer } from '@/tasks/tracer'; -import { CheckAlertsTaskArgs, HdxTask } from '@/tasks/types'; +import { CheckAlertsTaskArgs, HdxTask, MappedOmit } from '@/tasks/types'; import { calcAlertDateRange, + MAX_NUM_WINDOWS, roundDownToXMinutes, unflattenObject, } from '@/tasks/util'; @@ -148,7 +149,9 @@ const alertBatchFailuresCounter = getCounter('hyperdx.alerts.batch_failures', { * For saved search alerts, groupBy is on alert.groupBy. * For tile and inline alerts, groupBy is on the chart config. */ -export const alertHasGroupBy = (details: AlertDetails): boolean => { +export const alertHasGroupBy = ( + details: MappedOmit, +): boolean => { // AlertChartConfig members are the tile config types minus the embedded // alert field, so they're assignable to SavedChartConfig (which the // chart-config guards narrow on). @@ -614,7 +617,10 @@ const getAlertKeyPrefix = (alertId: string): string => { * Uses || as delimiter since it's unlikely to appear in alert IDs (MongoDB ObjectIds) * or in typical group key values. */ -const computeHistoryMapKey = (alertId: string, groupKey: string): string => { +export const computeHistoryMapKey = ( + alertId: string, + groupKey: string, +): string => { return groupKey ? `${getAlertKeyPrefix(alertId)}${groupKey}` : alertId; }; @@ -1947,7 +1953,8 @@ export const processAlert = async ( export { handleSendGenericWebhook }; export interface AggregatedAlertHistory { - _id: ObjectId; + /** The alert's ObjectId (not the history document's own _id). */ + _id: mongoose.Types.ObjectId; createdAt: Date; state: AlertState; group?: string; @@ -1955,96 +1962,218 @@ export interface AggregatedAlertHistory { groupValues?: GroupValue[] | null; } +// Floor/cap for the previous-history lookback window. The lookback only needs +// to cover (a) the catch-up horizon (capped at MAX_NUM_WINDOWS windows — see +// calcAlertDateRange) and (b) enough slack to auto-resolve groups whose last +// history row predates a task outage. History rows are written every window +// for every group present in the data, so anything beyond a couple of windows +// exists purely for outage/stale-group coverage. +const ALERT_HISTORY_MIN_LOOKBACK_MS = ms('1d'); +const ALERT_HISTORY_MAX_LOOKBACK_MS = ms('7d'); +// +2 windows of slack over the catch-up cap so a boundary window is never +// clipped by the history lookback. +const ALERT_HISTORY_LOOKBACK_WINDOWS = MAX_NUM_WINDOWS + 2; + +/** + * Compute how far back to scan AlertHistory when loading previous per-group + * state for an alert, derived from the alert's evaluation interval. + * + * Bounding the scan matters on DocumentDB: the previous-history query is a + * range scan on {alert: 1, createdAt: -1}, so the lookback directly bounds the + * index entries examined and documents fetched on the (read-preferring) + * secondary instances. + */ +export const getAlertHistoryLookbackMs = (interval: AlertInterval): number => { + const windowMs = ms(interval); + return Math.min( + ALERT_HISTORY_MAX_LOOKBACK_MS, + Math.max( + ALERT_HISTORY_MIN_LOOKBACK_MS, + ALERT_HISTORY_LOOKBACK_WINDOWS * windowMs, + ), + ); +}; + +/** Latest AlertHistory row per alert, minus the fat `lastValues` array. */ +export type LatestAlertHistory = Pick< + AggregatedAlertHistory, + 'createdAt' | 'state' | 'group' | 'fired' +>; + /** - * Fetch the most recent AlertHistory value for each of the given alert IDs. + * Fetch the single most recent AlertHistory row for each of the given alert + * IDs. + * + * Each lookup is a one-key index seek on {alert: 1, createdAt: -1}, so this is + * cheap enough to run for every alert on every task tick. loadAlerts uses the + * result to decide whether an alert was already evaluated in the current + * window — gating the much heavier per-group fetch (getPreviousAlertHistories) + * to grouped alerts that are due; for non-grouped alerts this row is the + * previous state itself. + * + * @returns A map from alert ID to its latest history row, or null when the + * alert has no history at all. + */ +export const getLatestAlertHistories = async ( + alertIds: string[], + now: Date, + sharedQueue?: PQueue, +): Promise> => { + const queue = + sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); + + const results = await Promise.all( + alertIds.map(alertId => + queue.add(async () => { + const latest = await AlertHistory.findOne( + { + alert: new mongoose.Types.ObjectId(alertId), + // Caps the walk past ERROR rows, which the index can't skip. + createdAt: { + $lte: now, + $gte: new Date(now.getTime() - ALERT_HISTORY_MAX_LOOKBACK_MS), + }, + // ERROR rows record failed evaluations; they must not count as + // "window evaluated" or the failed window would never be + // retried/backfilled. + state: { $ne: AlertState.ERROR }, + }, + { + createdAt: 1, + state: 1, + group: 1, + fired: 1, + }, + ) + .sort({ createdAt: -1 }) + .lean(); + return [alertId, latest] as const; + }), + ), + ); + + return new Map( + results + .filter((r): r is NonNullable => r !== undefined) + .map(([alertId, latest]) => [ + alertId, + latest + ? { + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + } + : null, + ]), + ); +}; + +/** + * Fetch the most recent AlertHistory value for each of the given alerts. * For group-by alerts, returns the latest history for each group within each alert. * - * Uses per-alert queries instead of batched $in to leverage the compound index - * {alert: 1, group: 1, createdAt: -1} for index-backed sorting. With a single - * alert value, the index delivers results already sorted by {group, createdAt desc}, - * so the $sort is a no-op and $group + $first can short-circuit per group. + * Implemented as a per-alert range `find()` on the {alert: 1, createdAt: -1} + * index with the latest-per-group reduce done in application code. An + * aggregation with $sort {alert, group, createdAt} + $group/$first is NOT used + * on purpose: DocumentDB has no DISTINCT_SCAN optimization, and no single + * B-tree index can serve both a tight createdAt range bound and a group-major + * sort — the aggregation either walks every index entry for the alert across + * the full TTL retention (loose createdAt bounds behind the unbounded `group` + * key) or performs a blocking in-memory sort. A plain tight-bounds range scan + * examines exactly the lookback window and keeps long-lived sort/group cursors + * off the DocumentDB secondaries (HDX-4791). + * + * The lookback window is derived from each alert's interval (see + * getAlertHistoryLookbackMs); the projection excludes the fat `lastValues` + * array so only the per-group state fields are transferred. * - * @param alertIds The list of alert IDs to query the latest history for. + * @param alerts The alerts (id + interval) to query the latest history for. * @param now The current date and time. AlertHistory documents that have a createdAt > now are ignored. * @returns A map from Alert IDs (or Alert ID + group) to their most recent AlertHistory. * For non-grouped alerts, the key is just the alert ID. * For grouped alerts, the key is "alertId||group" to track per-group state. */ export const getPreviousAlertHistories = async ( - alertIds: string[], + alerts: Array>, now: Date, sharedQueue?: PQueue, ) => { - const lookbackDate = new Date(now.getTime() - ms('7d')); - // Concurrency-limited per-alert queries to avoid overwhelming the connection // pool when there are many alerts (e.g., 200+ alert IDs). const queue = sharedQueue ?? new PQueue({ concurrency: ALERT_HISTORY_QUERY_CONCURRENCY }); const results = await Promise.all( - alertIds.map(alertId => + alerts.map(alert => queue.add(async () => { - const id = new mongoose.Types.ObjectId(alertId); - return AlertHistory.aggregate([ - { - $match: { - alert: id, - createdAt: { $lte: now, $gte: lookbackDate }, - // ERROR rows record failed evaluations; they must not count as - // "window evaluated" or the failed window would never be - // retried/backfilled. - state: { $ne: AlertState.ERROR }, - }, - }, - // With a single alert value, the compound index {alert: 1, group: 1, createdAt: -1} - // delivers results already in this sort order — this is an index-backed no-op sort. - { - $sort: { alert: 1, group: 1, createdAt: -1 }, - }, - // Group by {alert, group}, taking the first (latest) document's fields. - // Using $first on individual fields instead of $first: '$$ROOT' allows - // DocumentDB to avoid fetching full documents when not needed. + const alertObjectId = new mongoose.Types.ObjectId(alert.id); + const lookbackDate = new Date( + now.getTime() - getAlertHistoryLookbackMs(alert.interval), + ); + // Stream the range scan instead of materializing it: the row count is + // O(groups × windows-in-lookback) — up to ~1,440 windows for a + // 1m-interval alert at the 1d lookback floor — while the reduce below + // only ever needs the latest row per group. Iterating a cursor bounds + // app memory at O(driver batch + groups) per alert instead of + // O(groups × windows), which matters with up to + // ALERT_HISTORY_QUERY_CONCURRENCY of these in flight when alerts + // sharing a window boundary all come due at once. + const cursor = AlertHistory.find( { - $group: { - _id: { - alert: '$alert', - group: '$group', - }, - createdAt: { $first: '$createdAt' }, - state: { $first: '$state' }, - fired: { $first: '$fired' }, - groupValues: { $first: '$groupValues' }, - }, + alert: alertObjectId, + createdAt: { $lte: now, $gte: lookbackDate }, + // ERROR rows record failed evaluations; they must not count as + // "window evaluated" or the failed window would never be + // retried/backfilled. + state: { $ne: AlertState.ERROR }, }, { - $project: { - _id: '$_id.alert', - createdAt: 1, - state: 1, - group: '$_id.group', - fired: 1, - groupValues: 1, - }, + group: 1, + state: 1, + createdAt: 1, + fired: 1, + groupValues: 1, }, - ]); + ) + .lean() + .cursor(); + + // Reduce to the latest row per group. Ties on createdAt are resolved + // arbitrarily (first row wins), matching the previous aggregation's + // unstable $sort + $first behavior. + const latestByKey = new Map(); + for await (const row of cursor) { + const key = computeHistoryMapKey(alert.id, row.group || ''); + const existing = latestByKey.get(key); + if (!existing || row.createdAt > existing.createdAt) { + latestByKey.set(key, { + _id: alertObjectId, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + groupValues: row.groupValues, + }); + } + } + return latestByKey; }), ), ); - // Create a map with composite keys for grouped alerts (alertId||group) or simple keys for non-grouped alerts - return new Map( - results - .flat() - .filter((h): h is AggregatedAlertHistory => h !== undefined) - .map(history => { - const key = computeHistoryMapKey( - history._id.toString(), - history.group || '', - ); - return [key, history]; - }), - ); + // Merge the per-alert maps. Keys are namespaced by alert ID, so there are no + // collisions across alerts. + const merged = new Map(); + for (const result of results) { + if (!result) { + continue; + } + for (const [key, history] of result) { + merged.set(key, history); + } + } + return merged; }; /** @@ -2103,27 +2232,35 @@ export const getConsecutiveWindowHistories = async ( (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, ); const id = new mongoose.Types.ObjectId(alert.id); - const histories = await AlertHistory.aggregate([ + // Tight-bounds range scan on {alert: 1, createdAt: -1}; the + // newest-first ordering the consumer needs is applied in app code + // below. A DB-side $sort including `group` would force DocumentDB + // to walk the alert's full index range instead (see + // getPreviousAlertHistories). + // + // Unlike getPreviousAlertHistories, this deliberately materializes + // the result array: the consumer needs every row (sorted), so + // streaming saves nothing, and the row count is bounded by the small + // numConsecutiveWindows lookback rather than the full history window. + const rows = await AlertHistory.find( { - $match: { - alert: id, - createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, - // Failed evaluations (ERROR rows) are not evaluated windows and - // must not affect consecutive-window counting. - state: { $ne: AlertState.ERROR }, - }, - }, - { $sort: { alert: 1, group: 1, createdAt: -1 } }, - { - $project: { - _id: '$alert', - createdAt: 1, - state: 1, - group: 1, - fired: 1, - }, + alert: id, + createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, + // Failed evaluations (ERROR rows) are not evaluated windows and + // must not affect consecutive-window counting. + state: { $ne: AlertState.ERROR }, }, - ]); + { group: 1, state: 1, createdAt: 1, fired: 1 }, + ).lean(); + const histories: AggregatedAlertHistory[] = rows + .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()) + .map(row => ({ + _id: id, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + })); return { alertId: alert.id, histories }; }), ), diff --git a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts index 27ef9da7c..516685e08 100644 --- a/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/providers/__tests__/default.int.test.ts @@ -9,11 +9,18 @@ import { makeTile, RAW_SQL_ALERT_TEMPLATE, } from '@/fixtures'; -import Alert, { AlertSource, AlertThresholdType } from '@/models/alert'; +import Alert, { + AlertSource, + AlertState, + AlertThresholdType, +} from '@/models/alert'; +import AlertHistory from '@/models/alertHistory'; import Connection from '@/models/connection'; import Dashboard from '@/models/dashboard'; import { SavedSearch } from '@/models/savedSearch'; import { Source } from '@/models/source'; +import * as checkAlertTasks from '@/tasks/checkAlerts'; +import { getScheduledWindowStart } from '@/tasks/checkAlerts'; import { AlertProvider, AlertTaskType, @@ -735,6 +742,293 @@ describe('DefaultAlertProvider', () => { expect(result[0].conn.host).toBe('http://localhost:8124'); }); + describe('history fetch gating (HDX-4791)', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + // Alerts created by the helpers use interval '5m' with no schedule + // offset, matching processAlert's window computation. + const currentWindowStart = () => + getScheduledWindowStart(new Date(), 5, 0); + + it('skips the per-group history fetch and seeds a sentinel for alerts already evaluated in the current window', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + // Use a 1d interval so the current window (start of the UTC day) + // cannot roll over mid-test. + await Alert.updateOne({ _id: alert._id }, { $set: { interval: '1d' } }); + const windowStart = getScheduledWindowStart(new Date(), 1440, 0); + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ALERT, + group: 'group-a', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // The gate (a findOne per alert) decided the alert was already + // evaluated this window, so the per-group range find never runs. + expect(findSpy).not.toHaveBeenCalled(); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + const sentinel = details.previousMap.get(alert.id); + expect(sentinel).toBeDefined(); + expect(sentinel!.createdAt).toEqual(windowStart); + expect(sentinel!.state).toBe(AlertState.ALERT); + }); + + it('fetches full per-group history for grouped alerts that are due', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + { groupBy: 'ServiceName' }, + ); + // Latest history is from a previous window -> the alert is due. + const previousWindowStart = new Date( + currentWindowStart().getTime() - 5 * 60_000, + ); + await AlertHistory.create({ + alert: alert._id, + createdAt: previousWindowStart, + state: AlertState.ALERT, + group: 'group-a', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: previousWindowStart, + state: AlertState.OK, + group: 'group-b', + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + expect(findSpy).toHaveBeenCalledTimes(1); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + const groupA = details.previousMap.get(`${alert.id}||group-a`); + expect(groupA).toBeDefined(); + expect(groupA!.state).toBe(AlertState.ALERT); + expect(groupA!.createdAt).toEqual(previousWindowStart); + expect(details.previousMap.get(`${alert.id}||group-b`)!.state).toBe( + AlertState.OK, + ); + }); + + it('seeds non-grouped due alerts from the gate row without a range scan (HDX-4980)', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + // Several previous windows of history; the newest row carries the + // state the previousMap must reflect. + const windowMs = 5 * 60_000; + const base = currentWindowStart().getTime(); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - 3 * windowMs), + state: AlertState.OK, + group: '', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - 2 * windowMs), + state: AlertState.OK, + group: '', + }); + const newest = await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(base - windowMs), + state: AlertState.ALERT, + group: '', + fired: true, + }); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // The gate's findOne already fetched the latest row; a non-grouped + // alert must not pay for the per-group range scan. + expect(findSpy).not.toHaveBeenCalled(); + + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + // Seeded entry is exactly what the legacy range scan would have + // reduced to: the newest row, including state and fired. + const previous = details.previousMap.get(alert.id); + expect(previous).toBeDefined(); + expect(previous!.createdAt).toEqual(newest.createdAt); + expect(previous!.state).toBe(AlertState.ALERT); + expect(previous!.fired).toBe(true); + }); + + it('skips the per-group history fetch for alerts with no history at all', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + + const findSpy = jest.spyOn(AlertHistory, 'find'); + const result = await provider.getAlertTasks(); + + // No history exists, so there is nothing to range-scan. + expect(findSpy).not.toHaveBeenCalled(); + + // The alert is still due and loaded, just with an empty previousMap. + expect(result).toHaveLength(1); + const details = result[0].alerts[0]; + expect(details.alert.id).toBe(alert.id); + expect(details.previousMap.get(alert.id)).toBeUndefined(); + }); + + it('reads the last evaluation for a grouped alert whose latest rows are older than the scan window', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + { groupBy: 'ServiceName' }, + ); + // 5m interval: the per-group scan covers 1 day, so a 2-day-old + // evaluation is outside it but inside the 7-day latest-row lookup. + const twoDaysAgo = new Date( + currentWindowStart().getTime() - 2 * 24 * 60 * 60_000, + ); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.ALERT, + group: 'ServiceName:a', + fired: true, + groupValues: [{ key: 'ServiceName', value: 'a' }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.OK, + group: 'ServiceName:b', + }); + // An earlier evaluation and a failed one must not be picked up. + await AlertHistory.create({ + alert: alert._id, + createdAt: new Date(twoDaysAgo.getTime() - 5 * 60_000), + state: AlertState.ALERT, + group: 'ServiceName:c', + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: twoDaysAgo, + state: AlertState.ERROR, + }); + + const result = await provider.getAlertTasks(); + + expect(result).toHaveLength(1); + const { previousMap } = result[0].alerts[0]; + expect([...previousMap.keys()].sort()).toEqual([ + `${alert.id}||ServiceName:a`, + `${alert.id}||ServiceName:b`, + ]); + const groupA = previousMap.get(`${alert.id}||ServiceName:a`); + expect(groupA?.createdAt).toEqual(twoDaysAgo); + expect(groupA?.state).toBe(AlertState.ALERT); + expect(groupA?.fired).toBe(true); + expect(groupA?.groupValues).toEqual([ + { key: 'ServiceName', value: 'a' }, + ]); + expect(previousMap.get(`${alert.id}||ServiceName:b`)?.state).toBe( + AlertState.OK, + ); + }); + + it('treats an alert as due when its window start cannot be computed', async () => { + const { alert } = await createValidSavedSearchAlert( + await createConnectionAndSource(), + ); + // 1d interval and a row at the current window start: without the + // failure this alert would be skipped, with only a sentinel entry. + await Alert.updateOne({ _id: alert._id }, { $set: { interval: '1d' } }); + await AlertHistory.create({ + alert: alert._id, + createdAt: getScheduledWindowStart(new Date(), 1440, 0), + state: AlertState.ALERT, + fired: true, + }); + const windowStartSpy = jest + .spyOn(checkAlertTasks, 'getAlertWindowStart') + .mockImplementation(() => { + throw new Error('lookup failed'); + }); + + const result = await provider.getAlertTasks(); + + expect(windowStartSpy).toHaveBeenCalled(); + expect(result).toHaveLength(1); + // The sentinel carries no `fired`; the real previous row does. + const previous = result[0].alerts[0].previousMap.get(alert.id); + expect(previous?.state).toBe(AlertState.ALERT); + expect(previous?.fired).toBe(true); + }); + + let team: Awaited>; + + beforeEach(async () => { + team = await createTeam({ name: 'Test Team' }); + }); + + const createConnectionAndSource = async () => { + const connection = await Connection.create({ + team: team._id, + name: 'Test Connection', + host: 'http://localhost:8123', + username: 'test', + password: 'test', + }); + const source = await Source.create({ + team: team._id, + name: 'Test Source', + kind: 'log', + from: { databaseName: 'default', tableName: 'logs' }, + timestampValueExpression: 'timestamp', + connection: connection._id, + }); + return { connection, source }; + }; + + const createValidSavedSearchAlert = async ( + { source }: Awaited>, + { groupBy }: { groupBy?: string } = {}, + ) => { + const savedSearch = await SavedSearch.create({ + team: team._id, + name: 'Test Search', + select: 'message', + where: 'level: error', + whereLanguage: 'lucene', + orderBy: 'timestamp', + source: source._id, + tags: [], + }); + const alert = await createAlert( + team._id, + { + source: AlertSource.SAVED_SEARCH, + savedSearchId: savedSearch._id.toString(), + threshold: 10, + thresholdType: AlertThresholdType.ABOVE, + interval: '5m', + ...(groupBy ? { groupBy } : {}), + channel: { + type: 'webhook', + webhookId: new mongoose.Types.ObjectId().toString(), + }, + }, + new mongoose.Types.ObjectId(), + ); + return { alert }; + }; + }); + it('should process a single inline alert', async () => { const team = await createTeam({ name: 'Test Team' }); diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index b409fafe9..bed6d5210 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -8,6 +8,7 @@ import { } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; import ms from 'ms'; +import { serializeError } from 'serialize-error'; import { URLSearchParams } from 'url'; import { ClickhouseClient } from '@/clickhouse'; @@ -31,9 +32,13 @@ import { type ISavedSearch, SavedSearch } from '@/models/savedSearch'; import { type ISource, Source } from '@/models/source'; import Webhook, { IWebhook } from '@/models/webhook'; import { - AggregatedAlertHistory, + alertHasGroupBy, + computeHistoryMapKey, + getAlertWindowStart, getConsecutiveWindowHistories, + getLatestAlertHistories, getPreviousAlertHistories, + type LatestAlertHistory, } from '@/tasks/checkAlerts'; import { type AlertDetails, @@ -349,13 +354,9 @@ async function getInlineAlertDetails( ]; } -async function loadAlert( +async function resolveAlertDetails( alert: IAlert, - groupedTasks: Map, - previousAlerts: Map, - recentHistoryMap: Map, - now: Date, -) { +): Promise<[IConnection, PartialAlertDetails]> { if (!alert.source) { throw new Error('alert does not have a source'); } @@ -393,18 +394,178 @@ async function loadAlert( throw new Error('failed to fetch alert connection'); } - if (!groupedTasks.has(conn.id)) { - groupedTasks.set(conn.id, { alerts: [], conn, now }); - } - const v = groupedTasks.get(conn.id); - if (!v) { - throw new Error(`provider did not set key ${conn.id} before appending`); + return [conn, details]; +} + +async function loadAlerts(): Promise { + const groupedTasks = new Map(); + + const alerts = await Alert.find({}); + + const now = new Date(); + + const resolved: { + alert: IAlert; + conn: IConnection; + details: PartialAlertDetails; + }[] = []; + for (const alert of alerts) { + try { + const [conn, details] = await resolveAlertDetails(alert); + resolved.push({ alert, conn, details }); + } catch (e) { + logger.error({ + message: `failed to load alert: ${e}`, + alertId: alert.id, + team: alert.team, + channel: alert.channel, + provider: 'default', + }); + } } - v.alerts.push({ - ...details, - previousMap: previousAlerts, - recentHistoryMap, + + // Share a single queue across all history fetches so their combined + // in-flight per-alert queries stay within one global cap. + const historyQueryQueue = new PQueue({ + concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, }); + + // Due-ness gate (HDX-4791): a cheap one-key lookup per (in-scope) alert + // decides whether the alert was already evaluated in the current window. + // Only alerts that are actually due pay for the per-group history range + // scan — without this, every task tick (once a minute) re-scans the + // history window for every alert regardless of the alert's interval, which + // was the main driver of DocDB secondary CPU / cursor timeouts. + const latestHistories = await getLatestAlertHistories( + resolved.map(({ alert }) => alert.id), + now, + historyQueryQueue, + ); + + const dueAlertsWithHistory: IAlert[] = []; + const dueGroupedAlerts: IAlert[] = []; + const dueNonGroupedAlerts: { + alert: IAlert; + latest: NonNullable>; + }[] = []; + const gatedAlerts: { + alert: IAlert; + latest: NonNullable>; + }[] = []; + let noHistoryCount = 0; + for (const { alert, details } of resolved) { + const latest = latestHistories.get(alert.id); + if (!latest) { + // Never evaluated (or history expired): the alert is due, but there is + // no previous state to fetch. + noHistoryCount += 1; + continue; + } + if (wasEvaluatedInCurrentWindow(alert, latest, now)) { + // Already evaluated in the current window — processAlert will skip it. + gatedAlerts.push({ alert, latest }); + continue; + } + dueAlertsWithHistory.push(alert); + // Non-grouped alerts only need the latest row, which the gate already + // fetched; only grouped alerts pay for the per-group range scan (its + // lookback also serves stale-group auto-resolve). + if (alertHasGroupBy(details)) { + dueGroupedAlerts.push(alert); + } else { + dueNonGroupedAlerts.push({ alert, latest }); + } + } + + const [previousAlerts, recentHistoryMap] = await Promise.all([ + getPreviousAlertHistories(dueGroupedAlerts, now, historyQueryQueue), + getConsecutiveWindowHistories(dueAlertsWithHistory, now, historyQueryQueue), + ]); + + // Seed non-grouped previous state from the gate row — identical to what + // the range scan would reduce to. + for (const { alert, latest } of dueNonGroupedAlerts) { + previousAlerts.set(computeHistoryMapKey(alert.id, latest.group || ''), { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + group: latest.group, + fired: latest.fired, + }); + } + + // A grouped alert whose latest row is older than its scan window gets no + // rows back. Read the rows of that last evaluation instead (one per group, + // all sharing its createdAt), so the evaluation still starts from it and + // every group keeps its previous state. + await Promise.all( + dueGroupedAlerts.map(alert => { + const latest = latestHistories.get(alert.id); + if ( + !latest || + previousAlerts.has(computeHistoryMapKey(alert.id, latest.group || '')) + ) { + return undefined; + } + return historyQueryQueue.add(async () => { + const alertObjectId = new mongoose.Types.ObjectId(alert.id); + const rows = await AlertHistory.find( + { + alert: alertObjectId, + createdAt: latest.createdAt, + state: { $ne: AlertState.ERROR }, + }, + { group: 1, state: 1, createdAt: 1, fired: 1, groupValues: 1 }, + ).lean(); + for (const row of rows) { + previousAlerts.set(computeHistoryMapKey(alert.id, row.group || ''), { + _id: alertObjectId, + createdAt: row.createdAt, + state: row.state, + group: row.group, + fired: row.fired, + groupValues: row.groupValues, + }); + } + }); + }), + ); + + // Seed a sentinel entry for each gated alert so processAlert's + // shouldSkipAlertCheck sees a history row in the current window and skips + // the evaluation exactly as before (including the skipped_window metric). + for (const { alert, latest } of gatedAlerts) { + previousAlerts.set(alert.id, { + _id: new mongoose.Types.ObjectId(alert.id), + createdAt: latest.createdAt, + state: latest.state, + }); + } + + logger.info( + { + alertCount: alerts.length, + dueCount: dueAlertsWithHistory.length, + dueGroupedCount: dueGroupedAlerts.length, + gatedCount: gatedAlerts.length, + noHistoryCount, + }, + 'Loaded alert histories with due-ness gating', + ); + + for (const { conn, details } of resolved) { + if (!groupedTasks.has(conn.id)) { + groupedTasks.set(conn.id, { alerts: [], conn, now }); + } + groupedTasks.get(conn.id)!.alerts.push({ + ...details, + previousMap: previousAlerts, + recentHistoryMap, + }); + } + + // Flatten out our groupings for execution + return Array.from(groupedTasks.values()); } export default class DefaultAlertProvider implements AlertProvider { @@ -422,43 +583,7 @@ export default class DefaultAlertProvider implements AlertProvider { } async getAlertTasks(): Promise { - const groupedTasks = new Map(); - const alerts = await Alert.find({}); - - const now = new Date(); - const alertIds = alerts.map(({ id }) => id); - // Share a single queue across both history fetches so their combined - // in-flight per-alert queries stay within one global cap. - const historyQueryQueue = new PQueue({ - concurrency: ALERT_HISTORY_QUERY_CONCURRENCY, - }); - const [previousAlerts, recentHistoryMap] = await Promise.all([ - getPreviousAlertHistories(alertIds, now, historyQueryQueue), - getConsecutiveWindowHistories(alerts, now, historyQueryQueue), - ]); - - for (const alert of alerts) { - try { - await loadAlert( - alert, - groupedTasks, - previousAlerts, - recentHistoryMap, - now, - ); - } catch (e) { - logger.error({ - message: `failed to load alert: ${e}`, - alertId: alert.id, - team: alert.team, - channel: alert.channel, - provider: 'default', - }); - } - } - - // Flatten out our groupings for execution - return Array.from(groupedTasks.values()); + return loadAlerts(); } buildLogSearchLink({ @@ -702,3 +827,26 @@ export default class DefaultAlertProvider implements AlertProvider { }); } } + +/** + * Whether the alert's latest history row is from the window processAlert would + * evaluate now, i.e. processAlert will skip it. A failing evaluator lookup + * counts as due so processAlert can record the error on that alert. + */ +function wasEvaluatedInCurrentWindow( + alert: IAlert, + latest: LatestAlertHistory, + now: Date, +): boolean { + try { + return ( + latest.createdAt.getTime() === getAlertWindowStart(alert, now).getTime() + ); + } catch (e) { + logger.error( + { alertId: alert.id, error: serializeError(e) }, + 'Failed to compute alert window start for history gating', + ); + return false; + } +} diff --git a/packages/api/src/tasks/util.ts b/packages/api/src/tasks/util.ts index ce6cbc5db..5f199d3e9 100644 --- a/packages/api/src/tasks/util.ts +++ b/packages/api/src/tasks/util.ts @@ -41,7 +41,7 @@ export const escapeJsonString = (str: string) => { return JSON.stringify(str).slice(1, -1); }; -const MAX_NUM_WINDOWS = 50; +export const MAX_NUM_WINDOWS = 50; const maxLookbackTime = (windowSizeInMins: number) => 3600_000 * (windowSizeInMins < 15 ? 6 : 24); export function calcAlertDateRange(