Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions backend/app/dlq/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,12 @@
DLQMessageStatus,
DLQTopicSummary,
RetryPolicy,
RetryStrategy,
)

__all__ = [
"DLQ_GROUP_HEADER",
"DLQ_RETRY_HEADER",
"DLQMessageStatus",
"RetryStrategy",
"DLQMessageChanges",
"DLQMessageFilter",
"RetryPolicy",
Expand Down
12 changes: 3 additions & 9 deletions backend/app/dlq/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@
from app.dlq.models import (
DLQ_GROUP_HEADER,
DLQ_RETRY_HEADER,
RETRY_POLICY,
DLQMessageChanges,
DLQMessageStatus,
RetryStrategy,
retry_policy_for,
)
from app.domain.enums import EventType
from app.domain.events import (
Expand Down Expand Up @@ -114,22 +113,17 @@ async def handle_message(self, message: DLQMessageDocument) -> None:
topic=EventType.DLQ_MESSAGE_RECEIVED,
)

retry_policy = retry_policy_for(message.event.event_type)

if not retry_policy.should_retry(message.retry_count):
if message.retry_count >= RETRY_POLICY.max_retries:
await self.discard_message(message, "max_retries_exceeded")
return

next_retry = retry_policy.get_next_retry_time(message.retry_count)
next_retry = RETRY_POLICY.get_next_retry_time(message.retry_count)
await self.repository.update_status(
message.event.event_id,
message.consumer_group,
DLQMessageChanges(status=DLQMessageStatus.SCHEDULED, next_retry_at=next_retry),
)

if retry_policy.strategy == RetryStrategy.IMMEDIATE:
await self.retry_message(message)

async def retry_message(self, message: DLQMessageDocument) -> None:
"""Republish a DLQ message to its original topic for the consumer group that failed on it.

Expand Down
91 changes: 14 additions & 77 deletions backend/app/dlq/models.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import random
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import NotRequired, TypedDict
Expand Down Expand Up @@ -34,16 +35,6 @@ def is_terminal(self) -> bool:
DLQ_TERMINAL = frozenset(s for s in DLQMessageStatus if s.is_terminal)


class RetryStrategy(StringEnum):
"""Retry strategies for DLQ messages."""

IMMEDIATE = "immediate"
EXPONENTIAL_BACKOFF = "exponential_backoff"
FIXED_INTERVAL = "fixed_interval"
SCHEDULED = "scheduled"
MANUAL = "manual"


class DLQMessageChanges(TypedDict):
"""Fields a status change sets on a DLQ row; a missing key keeps the stored value."""

Expand All @@ -64,80 +55,26 @@ class DLQMessageFilter:
event_type: EventType | None = None


@dataclass
@dataclass(frozen=True)
class RetryPolicy:
"""Retry policy configuration for DLQ messages."""
"""When a failed event is retried: exponential backoff with jitter, up to max_retries attempts."""

strategy: RetryStrategy
max_retries: int = 5
base_delay_seconds: float = 60.0
max_delay_seconds: float = 3600.0
base_delay_seconds: float = 30.0
max_delay_seconds: float = 300.0
retry_multiplier: float = 2.0
jitter_factor: float = 0.1

def should_retry(self, retry_count: int) -> bool:
"""Check if a message retried this many times should be retried again."""
if self.strategy == RetryStrategy.MANUAL:
return False
return retry_count < self.max_retries

def get_next_retry_time(self, retry_count: int) -> datetime:
"""Calculate next retry time based on strategy."""
import random

if self.strategy == RetryStrategy.IMMEDIATE:
return datetime.now(timezone.utc)

if self.strategy == RetryStrategy.FIXED_INTERVAL:
delay = self.base_delay_seconds

elif self.strategy == RetryStrategy.EXPONENTIAL_BACKOFF:
delay = min(self.base_delay_seconds * (self.retry_multiplier**retry_count), self.max_delay_seconds)
# Add jitter to avoid thundering herd
jitter = delay * self.jitter_factor * (2 * random.random() - 1)
delay = max(0, delay + jitter)

else: # SCHEDULED or custom
delay = self.base_delay_seconds

return datetime.now(timezone.utc) + timedelta(seconds=delay)


AGGRESSIVE_RETRY = RetryPolicy(
strategy=RetryStrategy.EXPONENTIAL_BACKOFF,
max_retries=5,
base_delay_seconds=30,
max_delay_seconds=300,
retry_multiplier=2.0,
)
CAUTIOUS_RETRY = RetryPolicy(
strategy=RetryStrategy.EXPONENTIAL_BACKOFF,
max_retries=3,
base_delay_seconds=60,
max_delay_seconds=600,
retry_multiplier=3.0,
)
IMMEDIATE_RETRY = RetryPolicy(strategy=RetryStrategy.IMMEDIATE, max_retries=3)
DEFAULT_RETRY = RetryPolicy(
strategy=RetryStrategy.EXPONENTIAL_BACKOFF,
max_retries=4,
base_delay_seconds=60,
max_delay_seconds=1800,
retry_multiplier=2.5,
)


def retry_policy_for(event_type: EventType) -> RetryPolicy:
"""Determine retry policy from event type using category sets."""
from app.infrastructure.kafka.topics import EXECUTION_TYPES, POD_TYPES, RESULT_TYPES

if event_type in EXECUTION_TYPES:
return AGGRESSIVE_RETRY
if event_type in POD_TYPES:
return CAUTIOUS_RETRY
if event_type in RESULT_TYPES:
return IMMEDIATE_RETRY
return DEFAULT_RETRY
"""The time of the next attempt after retry_count failed ones."""
delay = min(self.base_delay_seconds * (self.retry_multiplier**retry_count), self.max_delay_seconds)
# Jitter spreads out events that failed together, so their retries don't all land at once
jitter = delay * self.jitter_factor * (2 * random.random() - 1)
return datetime.now(timezone.utc) + timedelta(seconds=max(0, delay + jitter))


# Only consumers of execution events save to the DLQ, so one policy covers every row
RETRY_POLICY = RetryPolicy()


@dataclass
Expand Down
38 changes: 0 additions & 38 deletions backend/app/infrastructure/kafka/topics.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,10 @@
from app.domain.enums import EventType, ExecutionStatus

EXECUTION_TYPES: set[EventType] = {
EventType.EXECUTION_REQUESTED,
EventType.EXECUTION_ACCEPTED,
EventType.EXECUTION_QUEUED,
EventType.EXECUTION_STARTED,
EventType.EXECUTION_RUNNING,
EventType.EXECUTION_COMPLETED,
EventType.EXECUTION_FAILED,
EventType.EXECUTION_TIMEOUT,
EventType.EXECUTION_CANCELLED,
}

POD_TYPES: set[EventType] = {
EventType.POD_CREATED,
EventType.POD_SCHEDULED,
EventType.POD_RUNNING,
EventType.POD_SUCCEEDED,
EventType.POD_FAILED,
EventType.POD_TERMINATED,
EventType.POD_DELETED,
}

RESULT_TYPES: set[EventType] = {
EventType.RESULT_STORED,
EventType.RESULT_FAILED,
}

SECURITY_TYPES: set[EventType] = {
EventType.SECURITY_VIOLATION,
EventType.RATE_LIMIT_EXCEEDED,
EventType.AUTH_FAILED,
}

USER_TYPES: set[EventType] = {
EventType.USER_REGISTERED,
EventType.USER_LOGIN,
EventType.USER_LOGGED_IN,
EventType.USER_LOGGED_OUT,
EventType.USER_UPDATED,
EventType.USER_DELETED,
EventType.USER_SETTINGS_UPDATED,
}

# Events that signal the executor pod has reached a terminal state, so the pod monitor deletes the pod.
# A cancel deletes its pod in the execution scheduler instead.
EXECUTION_TERMINAL_EVENT_TYPES: frozenset[EventType] = frozenset(
Expand Down
10 changes: 9 additions & 1 deletion backend/tests/unit/dlq/test_dlq_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from app.db import DLQRepository
from app.db.docs import DLQMessageDocument
from app.dlq.manager import DLQManager
from app.dlq.models import RETRY_POLICY
from app.domain.events import EventMetadata, UserRegisteredEvent
from app.settings import Settings

Expand Down Expand Up @@ -79,7 +80,7 @@ async def test_filtered_message_is_not_stored(
async def test_message_out_of_retries_is_discarded_inside_the_handle_span(
manager: DLQManager, span_exporter: InMemorySpanExporter
) -> None:
await manager.handle_message(_message(retry_count=4))
await manager.handle_message(_message(retry_count=RETRY_POLICY.max_retries))

discard, handle = span_exporter.get_finished_spans()
assert discard.parent is not None
Expand Down Expand Up @@ -142,3 +143,10 @@ async def test_monitoring_cycle_reports_each_topic_size(
("dlq.queue_size", {"dlq.original_topic": "user_registered", "dlq.size": 1}),
]
assert all(s.parent is not None and s.parent.span_id == cycle.context.span_id for s in sizes)


@pytest.mark.parametrize(("retry_count", "low", "high"), [(0, 27, 33), (2, 108, 132), (10, 270, 330)])
def test_retry_delay_grows_and_is_capped(retry_count: int, low: float, high: float) -> None:
"""30 s doubles per failed attempt up to 300 s, give or take the 10% jitter."""
delay = (RETRY_POLICY.get_next_retry_time(retry_count) - datetime.now(timezone.utc)).total_seconds()
assert low - 1 < delay <= high
2 changes: 1 addition & 1 deletion docs/architecture/kafka-topic-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Each handler gets exactly one typed model. Nobody decodes a message only to drop

No code creates topics. The broker has `KAFKA_AUTO_CREATE_TOPICS_ENABLE` on, so a topic appears the first time a client uses it, with the broker defaults from `docker-compose.yaml`: 3 partitions (`KAFKA_NUM_PARTITIONS`) and 7 days of retention (`KAFKA_LOG_RETENTION_HOURS: 168`). Every topic gets the same settings.

`EventType` has 47 members, but the code constructs fewer than half of them, so most of those topics never come into existence. `app/infrastructure/kafka/topics.py` groups event types into sets such as `EXECUTION_TYPES`, `POD_TYPES` and `RESULT_TYPES`. The DLQ uses those sets to choose a retry policy, and they have nothing to do with partitions or retention.
`EventType` has 47 members, but the code constructs fewer than half of them, so most of those topics never come into existence. `app/infrastructure/kafka/topics.py` groups event types into the sets the code needs, such as the terminal events that end an execution's SSE stream. They have nothing to do with partitions or retention.

When the producer first publishes them, execution, pod and result events are keyed by execution ID. User events use the username, notifications the user ID and settings events `user_settings_{user_id}`. Within a topic, Kafka sends every message with the same key to the same partition, so they stay in order. Resends don't keep those keys: a DLQ retry keys the message by `event_id`, and a replay by `aggregate_id`, falling back to `event_id`. So a retried or replayed event can land on another partition and arrive out of order with the rest of its execution. Across topics there is no ordering at all, so a group that reads several topics, like the SSE bridge, can see one execution's events out of order. With 3 partitions, up to three consumers in one group can share a topic.

Expand Down
15 changes: 3 additions & 12 deletions docs/components/dead-letter-queue.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,31 +24,22 @@ Every consumer group on that topic receives the retried event, so the publish ca

The row is marked before the publish on purpose. A retry can fail and save its new failure within milliseconds, and marking afterwards would overwrite that with `retried`.

Retry policies are hardcoded in `app/dlq/models.py`, chosen by event type:

| Event types | Strategy | Max retries | Base delay | Max delay |
|-------------|----------|-------------|------------|-----------|
| Execution events and commands | Exponential, x2 | 5 | 30 s | 300 s |
| Pod events | Exponential, x3 | 3 | 60 s | 600 s |
| `result_stored`, `result_failed` | Immediate | 3 | | |
| Everything else | Exponential, x2.5 | 4 | 60 s | 1800 s |

Exponential delays get 10% jitter. A message whose `retry_count` has reached the limit is discarded with reason `max_retries_exceeded`, and `dlq_message_discarded` goes out.
One retry policy, `RETRY_POLICY` in `app/dlq/models.py`, covers every row, since only consumers of execution events save to the DLQ. The wait starts at 30 seconds and doubles with each failed attempt, up to 300 seconds, with 10% jitter. A message whose `retry_count` has reached 5 is discarded with reason `max_retries_exceeded`, and `dlq_message_discarded` goes out.

## Storage

A row is one event in one consumer group. The unique index is on `event.event_id` plus `consumer_group`, so when two groups fail on the same event, each gets its own row and its own retry. A TTL index deletes rows 7 days after `created_at`.

A database created before consumer groups were added still has the old unique index `idx_dlq_event_id` on `event.event_id` alone, and Beanie never drops indexes. Until someone runs `db.dlq_messages.dropIndex("idx_dlq_event_id")` once, a second group that fails on the same event can't save its row.

The retry policies and the 7 day TTL are set in code, and config.toml has no DLQ keys. There's no HTTP API for the DLQ.
The retry policy and the 7 day TTL are set in code, and config.toml has no DLQ keys. There's no HTTP API for the DLQ.

## Key files

| File | Purpose |
|------|---------|
| [`middlewares.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/events/middlewares.py) | `DeadLetterMiddleware` and `RetryTargetMiddleware` |
| [`manager.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/manager.py) | Intake, retry, discard |
| [`models.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/models.py) | Statuses, retry policies and the two headers |
| [`models.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/dlq/models.py) | Statuses, the retry policy and the two headers |
| [`dlq.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/db/docs/dlq.py) | `dlq_messages` document and indexes |
| [`worker.py`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/app/worker.py) | Worker entry point, schedules the retry job |
2 changes: 1 addition & 1 deletion docs/components/workers/dlq_processor.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ graph LR

Every 10 seconds `process_monitoring_cycle` does two things. First it loads up to 100 rows with status `scheduled` whose `next_retry_at` has passed. For each one it sets the row to `retried` with `retry_count` plus one, republishes the event to its original topic keyed by event id, with headers naming the consumer group that failed and the retry count, and publishes `dlq_message_retried`. Then it counts the pending and scheduled rows per topic and reports each count on a `dlq.queue_size` span, which the collector keeps as the `dlq_queue_size` gauge. A retry that fails again comes back as a new failure for the same row.

Retry policies are hardcoded in `app/dlq/models.py`. There's no admin API for the DLQ.
The retry policy is set in `app/dlq/models.py`. There's no admin API for the DLQ.

## Key files

Expand Down
Loading