diff --git a/backend/app/dlq/__init__.py b/backend/app/dlq/__init__.py index e93735c8..d6a5035d 100644 --- a/backend/app/dlq/__init__.py +++ b/backend/app/dlq/__init__.py @@ -12,14 +12,12 @@ DLQMessageStatus, DLQTopicSummary, RetryPolicy, - RetryStrategy, ) __all__ = [ "DLQ_GROUP_HEADER", "DLQ_RETRY_HEADER", "DLQMessageStatus", - "RetryStrategy", "DLQMessageChanges", "DLQMessageFilter", "RetryPolicy", diff --git a/backend/app/dlq/manager.py b/backend/app/dlq/manager.py index 97345e75..fe6befb0 100644 --- a/backend/app/dlq/manager.py +++ b/backend/app/dlq/manager.py @@ -10,10 +10,9 @@ from app.dlq.models import ( DLQ_GROUP_HEADER, DLQ_RETRY_HEADER, + RETRY_POLICY, DLQMessageChanges, DLQMessageStatus, - RetryStrategy, - retry_policy_for, ) from app.domain.enums import EventType from app.domain.events import ( @@ -114,22 +113,17 @@ async def handle_message(self, message: DLQMessageDocument) -> None: topic=EventType.DLQ_MESSAGE_RECEIVED, ) - retry_policy = retry_policy_for(message.event.event_type) - - if not retry_policy.should_retry(message.retry_count): + if message.retry_count >= RETRY_POLICY.max_retries: await self.discard_message(message, "max_retries_exceeded") return - next_retry = retry_policy.get_next_retry_time(message.retry_count) + next_retry = RETRY_POLICY.get_next_retry_time(message.retry_count) await self.repository.update_status( message.event.event_id, message.consumer_group, DLQMessageChanges(status=DLQMessageStatus.SCHEDULED, next_retry_at=next_retry), ) - if retry_policy.strategy == RetryStrategy.IMMEDIATE: - await self.retry_message(message) - async def retry_message(self, message: DLQMessageDocument) -> None: """Republish a DLQ message to its original topic for the consumer group that failed on it. diff --git a/backend/app/dlq/models.py b/backend/app/dlq/models.py index cee1d7e1..5f6de0dd 100644 --- a/backend/app/dlq/models.py +++ b/backend/app/dlq/models.py @@ -1,3 +1,4 @@ +import random from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import NotRequired, TypedDict @@ -34,16 +35,6 @@ def is_terminal(self) -> bool: DLQ_TERMINAL = frozenset(s for s in DLQMessageStatus if s.is_terminal) -class RetryStrategy(StringEnum): - """Retry strategies for DLQ messages.""" - - IMMEDIATE = "immediate" - EXPONENTIAL_BACKOFF = "exponential_backoff" - FIXED_INTERVAL = "fixed_interval" - SCHEDULED = "scheduled" - MANUAL = "manual" - - class DLQMessageChanges(TypedDict): """Fields a status change sets on a DLQ row; a missing key keeps the stored value.""" @@ -64,80 +55,26 @@ class DLQMessageFilter: event_type: EventType | None = None -@dataclass +@dataclass(frozen=True) class RetryPolicy: - """Retry policy configuration for DLQ messages.""" + """When a failed event is retried: exponential backoff with jitter, up to max_retries attempts.""" - strategy: RetryStrategy max_retries: int = 5 - base_delay_seconds: float = 60.0 - max_delay_seconds: float = 3600.0 + base_delay_seconds: float = 30.0 + max_delay_seconds: float = 300.0 retry_multiplier: float = 2.0 jitter_factor: float = 0.1 - def should_retry(self, retry_count: int) -> bool: - """Check if a message retried this many times should be retried again.""" - if self.strategy == RetryStrategy.MANUAL: - return False - return retry_count < self.max_retries - def get_next_retry_time(self, retry_count: int) -> datetime: - """Calculate next retry time based on strategy.""" - import random - - if self.strategy == RetryStrategy.IMMEDIATE: - return datetime.now(timezone.utc) - - if self.strategy == RetryStrategy.FIXED_INTERVAL: - delay = self.base_delay_seconds - - elif self.strategy == RetryStrategy.EXPONENTIAL_BACKOFF: - delay = min(self.base_delay_seconds * (self.retry_multiplier**retry_count), self.max_delay_seconds) - # Add jitter to avoid thundering herd - jitter = delay * self.jitter_factor * (2 * random.random() - 1) - delay = max(0, delay + jitter) - - else: # SCHEDULED or custom - delay = self.base_delay_seconds - - return datetime.now(timezone.utc) + timedelta(seconds=delay) - - -AGGRESSIVE_RETRY = RetryPolicy( - strategy=RetryStrategy.EXPONENTIAL_BACKOFF, - max_retries=5, - base_delay_seconds=30, - max_delay_seconds=300, - retry_multiplier=2.0, -) -CAUTIOUS_RETRY = RetryPolicy( - strategy=RetryStrategy.EXPONENTIAL_BACKOFF, - max_retries=3, - base_delay_seconds=60, - max_delay_seconds=600, - retry_multiplier=3.0, -) -IMMEDIATE_RETRY = RetryPolicy(strategy=RetryStrategy.IMMEDIATE, max_retries=3) -DEFAULT_RETRY = RetryPolicy( - strategy=RetryStrategy.EXPONENTIAL_BACKOFF, - max_retries=4, - base_delay_seconds=60, - max_delay_seconds=1800, - retry_multiplier=2.5, -) - - -def retry_policy_for(event_type: EventType) -> RetryPolicy: - """Determine retry policy from event type using category sets.""" - from app.infrastructure.kafka.topics import EXECUTION_TYPES, POD_TYPES, RESULT_TYPES - - if event_type in EXECUTION_TYPES: - return AGGRESSIVE_RETRY - if event_type in POD_TYPES: - return CAUTIOUS_RETRY - if event_type in RESULT_TYPES: - return IMMEDIATE_RETRY - return DEFAULT_RETRY + """The time of the next attempt after retry_count failed ones.""" + delay = min(self.base_delay_seconds * (self.retry_multiplier**retry_count), self.max_delay_seconds) + # Jitter spreads out events that failed together, so their retries don't all land at once + jitter = delay * self.jitter_factor * (2 * random.random() - 1) + return datetime.now(timezone.utc) + timedelta(seconds=max(0, delay + jitter)) + + +# Only consumers of execution events save to the DLQ, so one policy covers every row +RETRY_POLICY = RetryPolicy() @dataclass diff --git a/backend/app/infrastructure/kafka/topics.py b/backend/app/infrastructure/kafka/topics.py index a76fd635..cbd858ca 100644 --- a/backend/app/infrastructure/kafka/topics.py +++ b/backend/app/infrastructure/kafka/topics.py @@ -1,48 +1,10 @@ from app.domain.enums import EventType, ExecutionStatus -EXECUTION_TYPES: set[EventType] = { - EventType.EXECUTION_REQUESTED, - EventType.EXECUTION_ACCEPTED, - EventType.EXECUTION_QUEUED, - EventType.EXECUTION_STARTED, - EventType.EXECUTION_RUNNING, - EventType.EXECUTION_COMPLETED, - EventType.EXECUTION_FAILED, - EventType.EXECUTION_TIMEOUT, - EventType.EXECUTION_CANCELLED, -} - -POD_TYPES: set[EventType] = { - EventType.POD_CREATED, - EventType.POD_SCHEDULED, - EventType.POD_RUNNING, - EventType.POD_SUCCEEDED, - EventType.POD_FAILED, - EventType.POD_TERMINATED, - EventType.POD_DELETED, -} - RESULT_TYPES: set[EventType] = { EventType.RESULT_STORED, EventType.RESULT_FAILED, } -SECURITY_TYPES: set[EventType] = { - EventType.SECURITY_VIOLATION, - EventType.RATE_LIMIT_EXCEEDED, - EventType.AUTH_FAILED, -} - -USER_TYPES: set[EventType] = { - EventType.USER_REGISTERED, - EventType.USER_LOGIN, - EventType.USER_LOGGED_IN, - EventType.USER_LOGGED_OUT, - EventType.USER_UPDATED, - EventType.USER_DELETED, - EventType.USER_SETTINGS_UPDATED, -} - # Events that signal the executor pod has reached a terminal state, so the pod monitor deletes the pod. # A cancel deletes its pod in the execution scheduler instead. EXECUTION_TERMINAL_EVENT_TYPES: frozenset[EventType] = frozenset( diff --git a/backend/tests/unit/dlq/test_dlq_manager.py b/backend/tests/unit/dlq/test_dlq_manager.py index 8680dd2c..aed0389b 100644 --- a/backend/tests/unit/dlq/test_dlq_manager.py +++ b/backend/tests/unit/dlq/test_dlq_manager.py @@ -11,6 +11,7 @@ from app.db import DLQRepository from app.db.docs import DLQMessageDocument from app.dlq.manager import DLQManager +from app.dlq.models import RETRY_POLICY from app.domain.events import EventMetadata, UserRegisteredEvent from app.settings import Settings @@ -79,7 +80,7 @@ async def test_filtered_message_is_not_stored( async def test_message_out_of_retries_is_discarded_inside_the_handle_span( manager: DLQManager, span_exporter: InMemorySpanExporter ) -> None: - await manager.handle_message(_message(retry_count=4)) + await manager.handle_message(_message(retry_count=RETRY_POLICY.max_retries)) discard, handle = span_exporter.get_finished_spans() assert discard.parent is not None @@ -142,3 +143,10 @@ async def test_monitoring_cycle_reports_each_topic_size( ("dlq.queue_size", {"dlq.original_topic": "user_registered", "dlq.size": 1}), ] assert all(s.parent is not None and s.parent.span_id == cycle.context.span_id for s in sizes) + + +@pytest.mark.parametrize(("retry_count", "low", "high"), [(0, 27, 33), (2, 108, 132), (10, 270, 330)]) +def test_retry_delay_grows_and_is_capped(retry_count: int, low: float, high: float) -> None: + """30 s doubles per failed attempt up to 300 s, give or take the 10% jitter.""" + delay = (RETRY_POLICY.get_next_retry_time(retry_count) - datetime.now(timezone.utc)).total_seconds() + assert low - 1 < delay <= high diff --git a/docs/architecture/kafka-topic-architecture.md b/docs/architecture/kafka-topic-architecture.md index f587d58b..c96dfcd9 100644 --- a/docs/architecture/kafka-topic-architecture.md +++ b/docs/architecture/kafka-topic-architecture.md @@ -21,7 +21,7 @@ Each handler gets exactly one typed model. Nobody decodes a message only to drop No code creates topics. The broker has `KAFKA_AUTO_CREATE_TOPICS_ENABLE` on, so a topic appears the first time a client uses it, with the broker defaults from `docker-compose.yaml`: 3 partitions (`KAFKA_NUM_PARTITIONS`) and 7 days of retention (`KAFKA_LOG_RETENTION_HOURS: 168`). Every topic gets the same settings. -`EventType` has 47 members, but the code constructs fewer than half of them, so most of those topics never come into existence. `app/infrastructure/kafka/topics.py` groups event types into sets such as `EXECUTION_TYPES`, `POD_TYPES` and `RESULT_TYPES`. The DLQ uses those sets to choose a retry policy, and they have nothing to do with partitions or retention. +`EventType` has 47 members, but the code constructs fewer than half of them, so most of those topics never come into existence. `app/infrastructure/kafka/topics.py` groups event types into the sets the code needs, such as the terminal events that end an execution's SSE stream. They have nothing to do with partitions or retention. When the producer first publishes them, execution, pod and result events are keyed by execution ID. User events use the username, notifications the user ID and settings events `user_settings_{user_id}`. Within a topic, Kafka sends every message with the same key to the same partition, so they stay in order. Resends don't keep those keys: a DLQ retry keys the message by `event_id`, and a replay by `aggregate_id`, falling back to `event_id`. So a retried or replayed event can land on another partition and arrive out of order with the rest of its execution. Across topics there is no ordering at all, so a group that reads several topics, like the SSE bridge, can see one execution's events out of order. With 3 partitions, up to three consumers in one group can share a topic. diff --git a/docs/components/dead-letter-queue.md b/docs/components/dead-letter-queue.md index 89438e6c..d35feed2 100644 --- a/docs/components/dead-letter-queue.md +++ b/docs/components/dead-letter-queue.md @@ -24,16 +24,7 @@ Every consumer group on that topic receives the retried event, so the publish ca The row is marked before the publish on purpose. A retry can fail and save its new failure within milliseconds, and marking afterwards would overwrite that with `retried`. -Retry policies are hardcoded in `app/dlq/models.py`, chosen by event type: - -| Event types | Strategy | Max retries | Base delay | Max delay | -|-------------|----------|-------------|------------|-----------| -| Execution events and commands | Exponential, x2 | 5 | 30 s | 300 s | -| Pod events | Exponential, x3 | 3 | 60 s | 600 s | -| `result_stored`, `result_failed` | Immediate | 3 | | | -| Everything else | Exponential, x2.5 | 4 | 60 s | 1800 s | - -Exponential delays get 10% jitter. A message whose `retry_count` has reached the limit is discarded with reason `max_retries_exceeded`, and `dlq_message_discarded` goes out. +One retry policy, `RETRY_POLICY` in `app/dlq/models.py`, covers every row, since only consumers of execution events save to the DLQ. The wait starts at 30 seconds and doubles with each failed attempt, up to 300 seconds, with 10% jitter. A message whose `retry_count` has reached 5 is discarded with reason `max_retries_exceeded`, and `dlq_message_discarded` goes out. ## Storage @@ -41,7 +32,7 @@ A row is one event in one consumer group. The unique index is on `event.event_id A database created before consumer groups were added still has the old unique index `idx_dlq_event_id` on `event.event_id` alone, and Beanie never drops indexes. Until someone runs `db.dlq_messages.dropIndex("idx_dlq_event_id")` once, a second group that fails on the same event can't save its row. -The retry policies and the 7 day TTL are set in code, and config.toml has no DLQ keys. There's no HTTP API for the DLQ. +The retry policy and the 7 day TTL are set in code, and config.toml has no DLQ keys. There's no HTTP API for the DLQ. ## Key files @@ -49,6 +40,6 @@ The retry policies and the 7 day TTL are set in code, and config.toml has no DLQ |------|---------| | [`middlewares.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/events/middlewares.py) | `DeadLetterMiddleware` and `RetryTargetMiddleware` | | [`manager.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/manager.py) | Intake, retry, discard | -| [`models.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/models.py) | Statuses, retry policies and the two headers | +| [`models.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/models.py) | Statuses, the retry policy and the two headers | | [`dlq.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/db/docs/dlq.py) | `dlq_messages` document and indexes | | [`worker.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/worker.py) | Worker entry point, schedules the retry job | diff --git a/docs/components/workers/dlq_processor.md b/docs/components/workers/dlq_processor.md index 9530273e..5234550e 100644 --- a/docs/components/workers/dlq_processor.md +++ b/docs/components/workers/dlq_processor.md @@ -12,7 +12,7 @@ graph LR Every 10 seconds `process_monitoring_cycle` does two things. First it loads up to 100 rows with status `scheduled` whose `next_retry_at` has passed. For each one it sets the row to `retried` with `retry_count` plus one, republishes the event to its original topic keyed by event id, with headers naming the consumer group that failed and the retry count, and publishes `dlq_message_retried`. Then it counts the pending and scheduled rows per topic and reports each count on a `dlq.queue_size` span, which the collector keeps as the `dlq_queue_size` gauge. A retry that fails again comes back as a new failure for the same row. -Retry policies are hardcoded in `app/dlq/models.py`. There's no admin API for the DLQ. +The retry policy is set in `app/dlq/models.py`. There's no admin API for the DLQ. ## Key files