Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ Rules:
- Use `raise SpecificError(...) from original_exc` to preserve exception chains
- Never catch `Exception` broadly; catch specific exception types (the two allowed cases follow)

A catch-all is allowed in two cases only. The first records or logs, then re-raises: `UnifiedProducer.produce` in `app/events/core/producer.py` marks the event as failed to publish, `IdempotencyMiddleware` and `DeadLetterMiddleware` in `app/events/middlewares.py` release the idempotency key and save the event to the DLQ, `ExecutionService` marks the execution as errored before raising `InfrastructureError`, and `KubernetesWorker.create_pod` marks its `pod.create` span failed. The second is a loop or batch boundary that must survive one bad item: the execution scheduler's queue loop (`schedule_pending`, a pod that can't be created fails its execution and the loop goes on), the pod monitor's watch cycle (`PodMonitor.watch_cycle`) and per-event processing, notification delivery and the scheduled-notification batch, DLQ batch retry and per-event replay. A new catch-all needs the same justification and must log or record the error; never swallow it silently.
A catch-all is allowed in two cases only. The first records or logs, then re-raises: `IdempotencyMiddleware` and `DeadLetterMiddleware` in `app/events/middlewares.py` release the idempotency key and save the event to the DLQ, `ExecutionService` marks the execution as errored before raising `InfrastructureError`, and `KubernetesWorker.create_pod` marks its `pod.create` span failed. The second is a loop or batch boundary that must survive one bad item: the execution scheduler's queue loop (`schedule_pending`, a pod that can't be created fails its execution and the loop goes on), the pod monitor's watch cycle (`PodMonitor.watch_cycle`) and per-event processing, notification delivery and the scheduled-notification batch, DLQ batch retry and per-event replay. A new catch-all needs the same justification and must log or record the error; never swallow it silently.

```python
# Repository: returns None for reads; raises only on write constraints
Expand Down Expand Up @@ -466,7 +466,7 @@ class ExecutionService:
- Location: `app/services/`
- Constructor injection: repo, producer, logger, settings, and `trace.Tracer` when the service makes spans
- New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `backend/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer
- Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`
- Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`. An event sent after its work is done, which no caller waits on (auth and settings events, pod progress, the result processor's announcements), goes through `produce_or_log`: it still raises a MongoDB error but only logs a Kafka one, so a Kafka outage doesn't fail a login or leave a finished pod behind
- Services orchestrate domain logic; no direct DB or HTTP calls
- Services build new documents with the document's constructor and hand them to the repository; they never import `app.schemas_pydantic`
- Always check `None` from repos and raise a domain-specific `NotFoundError` subclass (e.g. `ExecutionNotFoundError`, `SavedScriptNotFoundError`); never return `None` to a route
Expand Down
3 changes: 2 additions & 1 deletion backend/app/core/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,10 @@ def get_unified_producer(
self,
event_repository: EventRepository,
transport: KafkaEventTransport,
logger: structlog.stdlib.BoundLogger,
tracer: trace.Tracer,
) -> UnifiedProducer:
return UnifiedProducer(event_repository, transport, tracer)
return UnifiedProducer(event_repository, transport, logger, tracer)

@provide
def get_idempotency_repository(self, redis_client: redis.Redis) -> RedisIdempotencyRepository:
Expand Down
11 changes: 1 addition & 10 deletions backend/app/db/repositories/event_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from beanie.operators import GTE, LTE, Eq, In, Not, Or, RegEx
from monggregate import Pipeline, S
from opentelemetry import trace
from pymongo.errors import DuplicateKeyError, PyMongoError
from pymongo.errors import DuplicateKeyError

from app.db.docs import EventDocument
from app.domain.enums import EventType
Expand Down Expand Up @@ -58,15 +58,6 @@ async def store_event(self, event: DomainEvent) -> str:
self.logger.debug("Stored event", event_id=event.event_id, event_type=event.event_type)
return event.event_id

async def mark_publish_failed(self, event_id: str) -> None:
"""Best-effort mark of an event as failed to publish. Never raises."""
try:
await EventDocument.find_one(
EventDocument.event_id == event_id,
).update({"$set": {"publish_failed": True, "publish_failed_at": datetime.now(timezone.utc)}})
except PyMongoError as exc:
self.logger.warning("Could not mark event as publish-failed", event_id=event_id, error=str(exc))

async def get_event(self, event_id: str) -> DomainEvent | None:
doc = await EventDocument.find_one(EventDocument.event_id == event_id)
if not doc:
Expand Down
41 changes: 24 additions & 17 deletions backend/app/events/core/producer.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import structlog
from aiokafka.errors import KafkaError
from faststream.exceptions import IncorrectState
from opentelemetry import trace
Expand All @@ -7,41 +8,47 @@
from app.domain.events import DomainEvent
from app.events.core.transport import KafkaEventTransport

PRODUCE_ERRORS: tuple[type[Exception], ...] = (PyMongoError, KafkaError, IncorrectState)
_PUBLISH_ERRORS: tuple[type[Exception], ...] = (KafkaError, IncorrectState)
PRODUCE_ERRORS: tuple[type[Exception], ...] = (PyMongoError, *_PUBLISH_ERRORS)


class UnifiedProducer:
"""Orchestrates the store-then-publish outbox pattern.

Persists the event to MongoDB, then delegates to KafkaEventTransport
for Kafka delivery. On transport failure the event is marked as
failed-to-publish before the exception propagates.
"""
"""Stores each event in MongoDB, then publishes it to Kafka."""

def __init__(
self,
event_repository: EventRepository,
transport: KafkaEventTransport,
logger: structlog.stdlib.BoundLogger,
tracer: trace.Tracer,
):
self._event_repository = event_repository
self._transport = transport
self.logger = logger
self._tracer = tracer

async def produce(self, event_to_produce: DomainEvent, key: str) -> None:
"""Persist event to MongoDB, then publish to Kafka.

On Kafka publish failure, the event is marked as failed-to-publish
in MongoDB before the exception propagates.
"""
"""Store the event, then publish it; a MongoDB or Kafka error goes to the caller."""
event_type = event_to_produce.event_type
with self._tracer.start_as_current_span("publish_event") as span:
span.set_attribute("event.type", event_type)
if event_to_produce.aggregate_id:
span.set_attribute("aggregate.id", event_to_produce.aggregate_id)
await self._event_repository.store_event(event_to_produce)
try:
await self._transport.publish(event_to_produce, event_type, key)
except Exception:
await self._event_repository.mark_publish_failed(event_to_produce.event_id)
raise
await self._transport.publish(event_to_produce, event_type, key)

async def produce_or_log(self, event_to_produce: DomainEvent, key: str) -> None:
"""Store the event, then publish it, logging a Kafka error instead of raising it.

For events sent after their work is done, so a Kafka outage doesn't fail the request or job
that sent them. A MongoDB error still raises, since the stored event is the record.
"""
try:
await self.produce(event_to_produce, key)
except _PUBLISH_ERRORS:
self.logger.error(
"Event stored but not published",
event_id=event_to_produce.event_id,
event_type=event_to_produce.event_type,
exc_info=True,
)
8 changes: 4 additions & 4 deletions backend/app/services/auth_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ async def _fail_login(
user_agent=user_agent,
)
locked = await self._lockout.record_failed_attempt(username)
await self._producer.produce(
await self._producer.produce_or_log(
event_to_produce=AuthFailedEvent(
username=username,
reason=reason,
Expand Down Expand Up @@ -174,7 +174,7 @@ async def login(
)
csrf_token = self.security_service.generate_csrf_token(access_token)

await self._producer.produce(
await self._producer.produce_or_log(
event_to_produce=UserLoggedInEvent(
user_id=user.user_id,
login_method=LoginMethod.PASSWORD,
Expand Down Expand Up @@ -245,7 +245,7 @@ async def register(
user_agent=user_agent,
)

await self._producer.produce(
await self._producer.produce_or_log(
event_to_produce=UserRegisteredEvent(
user_id=created_user.user_id,
username=created_user.username,
Expand All @@ -264,7 +264,7 @@ async def publish_logout_event(self, token: str | None) -> None:
user = await self.user_repo.get_user(username)
if not user:
return
await self._producer.produce(
await self._producer.produce_or_log(
event_to_produce=UserLoggedOutEvent(
user_id=user.user_id,
logout_reason="user_initiated",
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/k8s_worker/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ async def _publish_pod_created(self, event: ExecutionRequestedEvent, pod: k8s_cl
namespace=pod.metadata.namespace,
metadata=event.metadata,
)
await self.producer.produce(event_to_produce=pod_created, key=event.execution_id)
await self.producer.produce_or_log(event_to_produce=pod_created, key=event.execution_id)

async def ensure_namespace_security(self) -> None:
"""Create the executor namespace and apply its security controls at startup.
Expand Down
4 changes: 2 additions & 2 deletions backend/app/services/pod_monitor/monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,10 @@ async def _handle(self, event: PodMonitorEvent) -> None:
case ExecutionTimeoutEvent():
await self._results.handle_execution_timeout(event)
case PodScheduledEvent() | PodRunningEvent():
await self._producer.produce(event_to_produce=event, key=event.execution_id)
await self._producer.produce_or_log(event_to_produce=event, key=event.execution_id)
await self._results.handle_pod_progress(event)
case PodTerminatedEvent():
await self._producer.produce(event_to_produce=event, key=event.execution_id)
await self._producer.produce_or_log(event_to_produce=event, key=event.execution_id)

def _track_pod_phase(self, event_type: WatchEventType, pod: k8s_client.V1Pod, phase: str | None) -> None:
"""Keep the per-phase pod count current for the pods.by.phase gauge."""
Expand Down
42 changes: 23 additions & 19 deletions backend/app/services/result_processor/processor.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import structlog
from opentelemetry import trace
from opentelemetry.util.types import AttributeValue
from pymongo.errors import PyMongoError

from app.db import ExecutionRepository
from app.domain.enums import ExecutionErrorType, ExecutionStatus, StorageType
Expand All @@ -17,7 +18,7 @@
ResultStoredEvent,
)
from app.domain.execution import ExecutionResultDomain
from app.events import PRODUCE_ERRORS, UnifiedProducer
from app.events import UnifiedProducer
from app.infrastructure.kafka.topics import POD_EVENT_STATUS
from app.runtime_registry import parse_quantity
from app.services.runtime_settings import RuntimeSettingsLoader
Expand Down Expand Up @@ -156,27 +157,30 @@ async def _store(
"""Write the terminal result, then publish the announce events and result_stored.

Nothing is published when the execution already ended another way, so the failure of a
cancelled execution's pod neither overwrites the cancel nor notifies anyone.
cancelled execution's pod neither overwrites the cancel nor notifies anyone. Once the result is
written, a Kafka error is only logged, so the pod monitor still deletes the pod and frees the slot.
"""
try:
if not await self._execution_repo.write_terminal_result(result):
self.logger.info(
"Execution already finished, result not stored",
execution_id=result.execution_id,
status=result.status,
)
return
for event in announce:
await self._producer.produce(event_to_produce=event, key=result.execution_id)
await self._publish_result_stored(
result.execution_id,
len(result.stdout or "") + len(result.stderr or ""),
metadata.user_id,
aggregate_id,
)
except PRODUCE_ERRORS as e:
written = await self._execution_repo.write_terminal_result(result)
except PyMongoError as e:
self.logger.error("Failed to store execution result", execution_id=result.execution_id, exc_info=True)
await self._publish_result_failed(result.execution_id, str(e), metadata.user_id, aggregate_id)
return
if not written:
self.logger.info(
"Execution already finished, result not stored",
execution_id=result.execution_id,
status=result.status,
)
return
for event in announce:
await self._producer.produce_or_log(event_to_produce=event, key=result.execution_id)
await self._publish_result_stored(
result.execution_id,
len(result.stdout or "") + len(result.stderr or ""),
metadata.user_id,
aggregate_id,
)

async def _publish_result_stored(
self, execution_id: str, size_bytes: int, user_id: str, aggregate_id: str | None
Expand All @@ -194,7 +198,7 @@ async def _publish_result_stored(
user_id=user_id,
),
)
await self._producer.produce(event_to_produce=event, key=execution_id)
await self._producer.produce_or_log(event_to_produce=event, key=execution_id)

async def _publish_result_failed(
self, execution_id: str, error_message: str, user_id: str, aggregate_id: str | None
Expand Down
4 changes: 2 additions & 2 deletions backend/app/services/user_settings_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ async def _publish_settings_event(
**changes,
}
)
await self._producer.produce(event_to_produce=event, key=f"user_settings_{user_id}")
await self._producer.produce_or_log(event_to_produce=event, key=f"user_settings_{user_id}")
return event

async def update_theme(self, user_id: str, theme: Theme) -> UserSettingsDocument:
Expand Down Expand Up @@ -181,7 +181,7 @@ async def restore_settings_to_point(self, user_id: str, timestamp: datetime) ->
# Rebuilds replay events after updated_at, so the snapshot carries the restore time
settings.updated_at = restore_event.timestamp
await self.repository.create_snapshot(settings)
await self._producer.produce(event_to_produce=restore_event, key=f"user_settings_{user_id}")
await self._producer.produce_or_log(event_to_produce=restore_event, key=f"user_settings_{user_id}")

return settings

Expand Down
32 changes: 28 additions & 4 deletions backend/tests/unit/events/test_producer.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, call

import pytest
import structlog
from aiokafka.errors import KafkaError
from opentelemetry import trace
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import StatusCode
from pymongo.errors import PyMongoError

from app.db import EventRepository
from app.domain.events import EventMetadata, UserRegisteredEvent
Expand All @@ -26,7 +28,7 @@ def _event() -> UserRegisteredEvent:
def _producer(tracer: trace.Tracer) -> tuple[UnifiedProducer, AsyncMock, AsyncMock]:
repo = AsyncMock(spec=EventRepository)
transport = AsyncMock(spec=KafkaEventTransport)
return UnifiedProducer(repo, transport, tracer), repo, transport
return UnifiedProducer(repo, transport, structlog.get_logger("test.producer"), tracer), repo, transport


async def test_produce_stores_publishes_and_reports_a_clean_span(
Expand All @@ -48,7 +50,7 @@ async def test_produce_stores_publishes_and_reports_a_clean_span(
)


async def test_failed_publish_is_marked_and_its_span_is_an_error(
async def test_failed_publish_raises_and_its_span_is_an_error(
tracer: trace.Tracer, span_exporter: InMemorySpanExporter
) -> None:
producer, repo, transport = _producer(tracer)
Expand All @@ -58,6 +60,28 @@ async def test_failed_publish_is_marked_and_its_span_is_an_error(
with pytest.raises(KafkaError):
await producer.produce(event_to_produce=event, key="u1")

repo.mark_publish_failed.assert_awaited_once_with(event.event_id)
assert repo.mock_calls == [call.store_event(event)]
[span] = span_exporter.get_finished_spans()
assert span.status.status_code == StatusCode.ERROR


async def test_produce_or_log_logs_a_kafka_error(tracer: trace.Tracer) -> None:
producer, repo, transport = _producer(tracer)
transport.publish.side_effect = KafkaError("broker down")
event = _event()

with structlog.testing.capture_logs() as logs:
await producer.produce_or_log(event_to_produce=event, key="u1")

repo.store_event.assert_awaited_once_with(event)
assert [(log["event"], log["event_id"]) for log in logs] == [("Event stored but not published", event.event_id)]


async def test_produce_or_log_raises_a_mongo_error(tracer: trace.Tracer) -> None:
producer, repo, transport = _producer(tracer)
repo.store_event.side_effect = PyMongoError("mongo down")

with pytest.raises(PyMongoError):
await producer.produce_or_log(event_to_produce=_event(), key="u1")

transport.publish.assert_not_awaited()
Loading
Loading