From ed07194a551d276dcd05787ce972de579a210063 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Thu, 22 Oct 2026 00:48:10 +0200 Subject: [PATCH] fix: a Kafka outage no longer leaves finished pods behind or fails logins, and the unread publish_failed flag goes The flag was set on every failed publish and nothing read it. Events sent after their work is done now go through produce_or_log, which logs a Kafka error instead of raising. So the result processor no longer throws past the pod monitor's cleanup, and login, register and settings changes succeed while Kafka is down. --- AGENTS.md | 4 +- backend/app/core/providers.py | 3 +- .../app/db/repositories/event_repository.py | 11 +--- backend/app/events/core/producer.py | 41 +++++++----- backend/app/services/auth_service.py | 8 +-- backend/app/services/k8s_worker/worker.py | 2 +- backend/app/services/pod_monitor/monitor.py | 4 +- .../services/result_processor/processor.py | 42 ++++++------ backend/app/services/user_settings_service.py | 4 +- backend/tests/unit/events/test_producer.py | 32 ++++++++-- .../result_processor/test_processor.py | 64 +++++++++++++++++-- .../services/test_auth_service_metrics.py | 38 +++++++++-- .../services/test_user_settings_service.py | 2 +- docs/architecture/event-storage.md | 3 +- docs/architecture/event-system-design.md | 12 ++-- docs/architecture/overview.md | 2 +- docs/components/workers/k8s_worker.md | 2 +- docs/components/workers/result_processor.md | 2 +- docs/testing/kafka-test-stability.md | 2 +- 19 files changed, 192 insertions(+), 86 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4731eefc..9d5ee578 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ Rules: - Use `raise SpecificError(...) from original_exc` to preserve exception chains - Never catch `Exception` broadly; catch specific exception types (the two allowed cases follow) -A catch-all is allowed in two cases only. The first records or logs, then re-raises: `UnifiedProducer.produce` in `app/events/core/producer.py` marks the event as failed to publish, `IdempotencyMiddleware` and `DeadLetterMiddleware` in `app/events/middlewares.py` release the idempotency key and save the event to the DLQ, `ExecutionService` marks the execution as errored before raising `InfrastructureError`, and `KubernetesWorker.create_pod` marks its `pod.create` span failed. The second is a loop or batch boundary that must survive one bad item: the execution scheduler's queue loop (`schedule_pending`, a pod that can't be created fails its execution and the loop goes on), the pod monitor's watch cycle (`PodMonitor.watch_cycle`) and per-event processing, notification delivery and the scheduled-notification batch, DLQ batch retry and per-event replay. A new catch-all needs the same justification and must log or record the error; never swallow it silently. +A catch-all is allowed in two cases only. The first records or logs, then re-raises: `IdempotencyMiddleware` and `DeadLetterMiddleware` in `app/events/middlewares.py` release the idempotency key and save the event to the DLQ, `ExecutionService` marks the execution as errored before raising `InfrastructureError`, and `KubernetesWorker.create_pod` marks its `pod.create` span failed. The second is a loop or batch boundary that must survive one bad item: the execution scheduler's queue loop (`schedule_pending`, a pod that can't be created fails its execution and the loop goes on), the pod monitor's watch cycle (`PodMonitor.watch_cycle`) and per-event processing, notification delivery and the scheduled-notification batch, DLQ batch retry and per-event replay. A new catch-all needs the same justification and must log or record the error; never swallow it silently. ```python # Repository: returns None for reads; raises only on write constraints @@ -466,7 +466,7 @@ class ExecutionService: - Location: `app/services/` - Constructor injection: repo, producer, logger, settings, and `trace.Tracer` when the service makes spans - New metrics come from spans: put the facts on a span as attributes (`queue.priority`, `queue.outcome`) and define the metric under `signaltometrics` in `backend/otel-collector-config.yaml`. There are no metric classes: `app/core/metrics/` only sets up the OTLP `MeterProvider` for the instrumentations ([#389](https://github.com/HardMax71/Integr8sCode/issues/389)). A gauge needs a job that reports it on a timer -- Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)` +- Publish events via `UnifiedProducer`: `await self.producer.produce(event_to_produce=event, key=execution_id)`. An event sent after its work is done, which no caller waits on (auth and settings events, pod progress, the result processor's announcements), goes through `produce_or_log`: it still raises a MongoDB error but only logs a Kafka one, so a Kafka outage doesn't fail a login or leave a finished pod behind - Services orchestrate domain logic; no direct DB or HTTP calls - Services build new documents with the document's constructor and hand them to the repository; they never import `app.schemas_pydantic` - Always check `None` from repos and raise a domain-specific `NotFoundError` subclass (e.g. `ExecutionNotFoundError`, `SavedScriptNotFoundError`); never return `None` to a route diff --git a/backend/app/core/providers.py b/backend/app/core/providers.py index 01330132..b73377ed 100644 --- a/backend/app/core/providers.py +++ b/backend/app/core/providers.py @@ -176,9 +176,10 @@ def get_unified_producer( self, event_repository: EventRepository, transport: KafkaEventTransport, + logger: structlog.stdlib.BoundLogger, tracer: trace.Tracer, ) -> UnifiedProducer: - return UnifiedProducer(event_repository, transport, tracer) + return UnifiedProducer(event_repository, transport, logger, tracer) @provide def get_idempotency_repository(self, redis_client: redis.Redis) -> RedisIdempotencyRepository: diff --git a/backend/app/db/repositories/event_repository.py b/backend/app/db/repositories/event_repository.py index 6b3fcfcf..93e2819c 100644 --- a/backend/app/db/repositories/event_repository.py +++ b/backend/app/db/repositories/event_repository.py @@ -7,7 +7,7 @@ from beanie.operators import GTE, LTE, Eq, In, Not, Or, RegEx from monggregate import Pipeline, S from opentelemetry import trace -from pymongo.errors import DuplicateKeyError, PyMongoError +from pymongo.errors import DuplicateKeyError from app.db.docs import EventDocument from app.domain.enums import EventType @@ -58,15 +58,6 @@ async def store_event(self, event: DomainEvent) -> str: self.logger.debug("Stored event", event_id=event.event_id, event_type=event.event_type) return event.event_id - async def mark_publish_failed(self, event_id: str) -> None: - """Best-effort mark of an event as failed to publish. Never raises.""" - try: - await EventDocument.find_one( - EventDocument.event_id == event_id, - ).update({"$set": {"publish_failed": True, "publish_failed_at": datetime.now(timezone.utc)}}) - except PyMongoError as exc: - self.logger.warning("Could not mark event as publish-failed", event_id=event_id, error=str(exc)) - async def get_event(self, event_id: str) -> DomainEvent | None: doc = await EventDocument.find_one(EventDocument.event_id == event_id) if not doc: diff --git a/backend/app/events/core/producer.py b/backend/app/events/core/producer.py index 96861b44..6f569d4b 100644 --- a/backend/app/events/core/producer.py +++ b/backend/app/events/core/producer.py @@ -1,3 +1,4 @@ +import structlog from aiokafka.errors import KafkaError from faststream.exceptions import IncorrectState from opentelemetry import trace @@ -7,41 +8,47 @@ from app.domain.events import DomainEvent from app.events.core.transport import KafkaEventTransport -PRODUCE_ERRORS: tuple[type[Exception], ...] = (PyMongoError, KafkaError, IncorrectState) +_PUBLISH_ERRORS: tuple[type[Exception], ...] = (KafkaError, IncorrectState) +PRODUCE_ERRORS: tuple[type[Exception], ...] = (PyMongoError, *_PUBLISH_ERRORS) class UnifiedProducer: - """Orchestrates the store-then-publish outbox pattern. - - Persists the event to MongoDB, then delegates to KafkaEventTransport - for Kafka delivery. On transport failure the event is marked as - failed-to-publish before the exception propagates. - """ + """Stores each event in MongoDB, then publishes it to Kafka.""" def __init__( self, event_repository: EventRepository, transport: KafkaEventTransport, + logger: structlog.stdlib.BoundLogger, tracer: trace.Tracer, ): self._event_repository = event_repository self._transport = transport + self.logger = logger self._tracer = tracer async def produce(self, event_to_produce: DomainEvent, key: str) -> None: - """Persist event to MongoDB, then publish to Kafka. - - On Kafka publish failure, the event is marked as failed-to-publish - in MongoDB before the exception propagates. - """ + """Store the event, then publish it; a MongoDB or Kafka error goes to the caller.""" event_type = event_to_produce.event_type with self._tracer.start_as_current_span("publish_event") as span: span.set_attribute("event.type", event_type) if event_to_produce.aggregate_id: span.set_attribute("aggregate.id", event_to_produce.aggregate_id) await self._event_repository.store_event(event_to_produce) - try: - await self._transport.publish(event_to_produce, event_type, key) - except Exception: - await self._event_repository.mark_publish_failed(event_to_produce.event_id) - raise + await self._transport.publish(event_to_produce, event_type, key) + + async def produce_or_log(self, event_to_produce: DomainEvent, key: str) -> None: + """Store the event, then publish it, logging a Kafka error instead of raising it. + + For events sent after their work is done, so a Kafka outage doesn't fail the request or job + that sent them. A MongoDB error still raises, since the stored event is the record. + """ + try: + await self.produce(event_to_produce, key) + except _PUBLISH_ERRORS: + self.logger.error( + "Event stored but not published", + event_id=event_to_produce.event_id, + event_type=event_to_produce.event_type, + exc_info=True, + ) diff --git a/backend/app/services/auth_service.py b/backend/app/services/auth_service.py index c738cce5..f2ebe13b 100644 --- a/backend/app/services/auth_service.py +++ b/backend/app/services/auth_service.py @@ -109,7 +109,7 @@ async def _fail_login( user_agent=user_agent, ) locked = await self._lockout.record_failed_attempt(username) - await self._producer.produce( + await self._producer.produce_or_log( event_to_produce=AuthFailedEvent( username=username, reason=reason, @@ -174,7 +174,7 @@ async def login( ) csrf_token = self.security_service.generate_csrf_token(access_token) - await self._producer.produce( + await self._producer.produce_or_log( event_to_produce=UserLoggedInEvent( user_id=user.user_id, login_method=LoginMethod.PASSWORD, @@ -245,7 +245,7 @@ async def register( user_agent=user_agent, ) - await self._producer.produce( + await self._producer.produce_or_log( event_to_produce=UserRegisteredEvent( user_id=created_user.user_id, username=created_user.username, @@ -264,7 +264,7 @@ async def publish_logout_event(self, token: str | None) -> None: user = await self.user_repo.get_user(username) if not user: return - await self._producer.produce( + await self._producer.produce_or_log( event_to_produce=UserLoggedOutEvent( user_id=user.user_id, logout_reason="user_initiated", diff --git a/backend/app/services/k8s_worker/worker.py b/backend/app/services/k8s_worker/worker.py index cc50127d..3b4b7902 100644 --- a/backend/app/services/k8s_worker/worker.py +++ b/backend/app/services/k8s_worker/worker.py @@ -198,7 +198,7 @@ async def _publish_pod_created(self, event: ExecutionRequestedEvent, pod: k8s_cl namespace=pod.metadata.namespace, metadata=event.metadata, ) - await self.producer.produce(event_to_produce=pod_created, key=event.execution_id) + await self.producer.produce_or_log(event_to_produce=pod_created, key=event.execution_id) async def ensure_namespace_security(self) -> None: """Create the executor namespace and apply its security controls at startup. diff --git a/backend/app/services/pod_monitor/monitor.py b/backend/app/services/pod_monitor/monitor.py index a1e89f31..7da3868f 100644 --- a/backend/app/services/pod_monitor/monitor.py +++ b/backend/app/services/pod_monitor/monitor.py @@ -236,10 +236,10 @@ async def _handle(self, event: PodMonitorEvent) -> None: case ExecutionTimeoutEvent(): await self._results.handle_execution_timeout(event) case PodScheduledEvent() | PodRunningEvent(): - await self._producer.produce(event_to_produce=event, key=event.execution_id) + await self._producer.produce_or_log(event_to_produce=event, key=event.execution_id) await self._results.handle_pod_progress(event) case PodTerminatedEvent(): - await self._producer.produce(event_to_produce=event, key=event.execution_id) + await self._producer.produce_or_log(event_to_produce=event, key=event.execution_id) def _track_pod_phase(self, event_type: WatchEventType, pod: k8s_client.V1Pod, phase: str | None) -> None: """Keep the per-phase pod count current for the pods.by.phase gauge.""" diff --git a/backend/app/services/result_processor/processor.py b/backend/app/services/result_processor/processor.py index ea9757d7..d7a387fb 100644 --- a/backend/app/services/result_processor/processor.py +++ b/backend/app/services/result_processor/processor.py @@ -1,6 +1,7 @@ import structlog from opentelemetry import trace from opentelemetry.util.types import AttributeValue +from pymongo.errors import PyMongoError from app.db import ExecutionRepository from app.domain.enums import ExecutionErrorType, ExecutionStatus, StorageType @@ -17,7 +18,7 @@ ResultStoredEvent, ) from app.domain.execution import ExecutionResultDomain -from app.events import PRODUCE_ERRORS, UnifiedProducer +from app.events import UnifiedProducer from app.infrastructure.kafka.topics import POD_EVENT_STATUS from app.runtime_registry import parse_quantity from app.services.runtime_settings import RuntimeSettingsLoader @@ -156,27 +157,30 @@ async def _store( """Write the terminal result, then publish the announce events and result_stored. Nothing is published when the execution already ended another way, so the failure of a - cancelled execution's pod neither overwrites the cancel nor notifies anyone. + cancelled execution's pod neither overwrites the cancel nor notifies anyone. Once the result is + written, a Kafka error is only logged, so the pod monitor still deletes the pod and frees the slot. """ try: - if not await self._execution_repo.write_terminal_result(result): - self.logger.info( - "Execution already finished, result not stored", - execution_id=result.execution_id, - status=result.status, - ) - return - for event in announce: - await self._producer.produce(event_to_produce=event, key=result.execution_id) - await self._publish_result_stored( - result.execution_id, - len(result.stdout or "") + len(result.stderr or ""), - metadata.user_id, - aggregate_id, - ) - except PRODUCE_ERRORS as e: + written = await self._execution_repo.write_terminal_result(result) + except PyMongoError as e: self.logger.error("Failed to store execution result", execution_id=result.execution_id, exc_info=True) await self._publish_result_failed(result.execution_id, str(e), metadata.user_id, aggregate_id) + return + if not written: + self.logger.info( + "Execution already finished, result not stored", + execution_id=result.execution_id, + status=result.status, + ) + return + for event in announce: + await self._producer.produce_or_log(event_to_produce=event, key=result.execution_id) + await self._publish_result_stored( + result.execution_id, + len(result.stdout or "") + len(result.stderr or ""), + metadata.user_id, + aggregate_id, + ) async def _publish_result_stored( self, execution_id: str, size_bytes: int, user_id: str, aggregate_id: str | None @@ -194,7 +198,7 @@ async def _publish_result_stored( user_id=user_id, ), ) - await self._producer.produce(event_to_produce=event, key=execution_id) + await self._producer.produce_or_log(event_to_produce=event, key=execution_id) async def _publish_result_failed( self, execution_id: str, error_message: str, user_id: str, aggregate_id: str | None diff --git a/backend/app/services/user_settings_service.py b/backend/app/services/user_settings_service.py index 291b9303..7080ff94 100644 --- a/backend/app/services/user_settings_service.py +++ b/backend/app/services/user_settings_service.py @@ -102,7 +102,7 @@ async def _publish_settings_event( **changes, } ) - await self._producer.produce(event_to_produce=event, key=f"user_settings_{user_id}") + await self._producer.produce_or_log(event_to_produce=event, key=f"user_settings_{user_id}") return event async def update_theme(self, user_id: str, theme: Theme) -> UserSettingsDocument: @@ -181,7 +181,7 @@ async def restore_settings_to_point(self, user_id: str, timestamp: datetime) -> # Rebuilds replay events after updated_at, so the snapshot carries the restore time settings.updated_at = restore_event.timestamp await self.repository.create_snapshot(settings) - await self._producer.produce(event_to_produce=restore_event, key=f"user_settings_{user_id}") + await self._producer.produce_or_log(event_to_produce=restore_event, key=f"user_settings_{user_id}") return settings diff --git a/backend/tests/unit/events/test_producer.py b/backend/tests/unit/events/test_producer.py index 5dceae3e..ec7b811b 100644 --- a/backend/tests/unit/events/test_producer.py +++ b/backend/tests/unit/events/test_producer.py @@ -1,10 +1,12 @@ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, call import pytest +import structlog from aiokafka.errors import KafkaError from opentelemetry import trace from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import StatusCode +from pymongo.errors import PyMongoError from app.db import EventRepository from app.domain.events import EventMetadata, UserRegisteredEvent @@ -26,7 +28,7 @@ def _event() -> UserRegisteredEvent: def _producer(tracer: trace.Tracer) -> tuple[UnifiedProducer, AsyncMock, AsyncMock]: repo = AsyncMock(spec=EventRepository) transport = AsyncMock(spec=KafkaEventTransport) - return UnifiedProducer(repo, transport, tracer), repo, transport + return UnifiedProducer(repo, transport, structlog.get_logger("test.producer"), tracer), repo, transport async def test_produce_stores_publishes_and_reports_a_clean_span( @@ -48,7 +50,7 @@ async def test_produce_stores_publishes_and_reports_a_clean_span( ) -async def test_failed_publish_is_marked_and_its_span_is_an_error( +async def test_failed_publish_raises_and_its_span_is_an_error( tracer: trace.Tracer, span_exporter: InMemorySpanExporter ) -> None: producer, repo, transport = _producer(tracer) @@ -58,6 +60,28 @@ async def test_failed_publish_is_marked_and_its_span_is_an_error( with pytest.raises(KafkaError): await producer.produce(event_to_produce=event, key="u1") - repo.mark_publish_failed.assert_awaited_once_with(event.event_id) + assert repo.mock_calls == [call.store_event(event)] [span] = span_exporter.get_finished_spans() assert span.status.status_code == StatusCode.ERROR + + +async def test_produce_or_log_logs_a_kafka_error(tracer: trace.Tracer) -> None: + producer, repo, transport = _producer(tracer) + transport.publish.side_effect = KafkaError("broker down") + event = _event() + + with structlog.testing.capture_logs() as logs: + await producer.produce_or_log(event_to_produce=event, key="u1") + + repo.store_event.assert_awaited_once_with(event) + assert [(log["event"], log["event_id"]) for log in logs] == [("Event stored but not published", event.event_id)] + + +async def test_produce_or_log_raises_a_mongo_error(tracer: trace.Tracer) -> None: + producer, repo, transport = _producer(tracer) + repo.store_event.side_effect = PyMongoError("mongo down") + + with pytest.raises(PyMongoError): + await producer.produce_or_log(event_to_produce=_event(), key="u1") + + transport.publish.assert_not_awaited() diff --git a/backend/tests/unit/services/result_processor/test_processor.py b/backend/tests/unit/services/result_processor/test_processor.py index 93d544f9..61b70b23 100644 --- a/backend/tests/unit/services/result_processor/test_processor.py +++ b/backend/tests/unit/services/result_processor/test_processor.py @@ -2,9 +2,12 @@ import pytest import structlog +from aiokafka.errors import KafkaError from opentelemetry import trace from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from pymongo.errors import PyMongoError +from app.db import EventRepository from app.db.docs import ExecutionDocument from app.domain.admin import SystemSettings from app.domain.enums import EventType, ExecutionErrorType, ExecutionStatus @@ -18,6 +21,8 @@ PodScheduledEvent, ResourceUsageDomain, ) +from app.events import UnifiedProducer +from app.events.core.transport import KafkaEventTransport from app.runtime_registry import POD_RUNTIME_CONFIGS, parse_quantity from app.services.result_processor import ResultProcessor from app.services.runtime_settings import RuntimeSettingsLoader @@ -33,7 +38,7 @@ def _make_processor( tracer: trace.Tracer, exec_repo: AsyncMock | None = None, - producer: AsyncMock | None = None, + producer: UnifiedProducer | None = None, admin_memory_limit: str = "256Mi", ) -> ResultProcessor: settings_repo = AsyncMock() @@ -64,6 +69,7 @@ async def test_skips_a_deleted_execution(self, tracer: trace.Tracer) -> None: repo.write_terminal_result.assert_not_awaited() producer.produce.assert_not_awaited() + producer.produce_or_log.assert_not_awaited() async def test_stores_result_and_publishes(self, tracer: trace.Tracer, span_exporter: InMemorySpanExporter) -> None: repo = AsyncMock() @@ -89,7 +95,7 @@ async def test_stores_result_and_publishes(self, tracer: trace.Tracer, span_expo assert result_arg.execution_id == "e1" assert result_arg.status == ExecutionStatus.COMPLETED assert result_arg.exit_code == 0 - assert [c.kwargs["event_to_produce"].event_type for c in producer.produce.await_args_list] == [ + assert [c.kwargs["event_to_produce"].event_type for c in producer.produce_or_log.await_args_list] == [ EventType.EXECUTION_COMPLETED, EventType.RESULT_STORED, ] @@ -161,7 +167,7 @@ async def test_stores_result_and_publishes(self, tracer: trace.Tracer, span_expo assert result_arg.execution_id == "e2" assert result_arg.status == ExecutionStatus.FAILED assert result_arg.exit_code == 1 - assert [c.kwargs["event_to_produce"].event_type for c in producer.produce.await_args_list] == [ + assert [c.kwargs["event_to_produce"].event_type for c in producer.produce_or_log.await_args_list] == [ EventType.EXECUTION_FAILED, EventType.RESULT_STORED, ] @@ -193,7 +199,7 @@ async def test_stores_result_and_publishes(self, tracer: trace.Tracer, span_expo assert result_arg.execution_id == "e3" assert result_arg.status == ExecutionStatus.TIMEOUT assert result_arg.exit_code == -1 - assert [c.kwargs["event_to_produce"].event_type for c in producer.produce.await_args_list] == [ + assert [c.kwargs["event_to_produce"].event_type for c in producer.produce_or_log.await_args_list] == [ EventType.EXECUTION_TIMEOUT, EventType.RESULT_STORED, ] @@ -225,6 +231,7 @@ async def test_publishes_nothing_when_the_execution_already_ended(self, tracer: repo.write_terminal_result.assert_awaited_once() producer.produce.assert_not_awaited() + producer.produce_or_log.assert_not_awaited() class TestHandleExecutionCancelled: @@ -249,13 +256,60 @@ async def test_stores_cancelled_and_publishes_result_stored( ExecutionStatus.CANCELLED, None, ) - assert [c.kwargs["event_to_produce"].event_type for c in producer.produce.await_args_list] == [ + assert [c.kwargs["event_to_produce"].event_type for c in producer.produce_or_log.await_args_list] == [ EventType.RESULT_STORED ] [span] = span_exporter.get_finished_spans() assert span.attributes == {"execution.status": "cancelled", "execution.lang_and_version": "python-3.11"} +class TestKafkaOutage: + """After the write a Kafka error is only logged, so the pod monitor still deletes the pod and frees the slot.""" + + async def test_publish_failure_after_the_write_is_not_raised(self, tracer: trace.Tracer) -> None: + repo = AsyncMock() + repo.get_execution.return_value = ExecutionDocument( + script="x", execution_id="e6", lang="python", lang_version="3.11" + ) + repo.write_terminal_result.return_value = True + transport = AsyncMock(spec=KafkaEventTransport) + transport.publish.side_effect = KafkaError("broker down") + producer = UnifiedProducer(AsyncMock(spec=EventRepository), transport, _test_logger, tracer) + processor = _make_processor(tracer, exec_repo=repo, producer=producer) + + await processor.handle_execution_completed( + ExecutionCompletedEvent( + execution_id="e6", exit_code=0, resource_usage=ResourceUsageDomain(), metadata=_METADATA + ) + ) + + repo.write_terminal_result.assert_awaited_once() + assert [c.args[1] for c in transport.publish.await_args_list] == [ + EventType.EXECUTION_COMPLETED, + EventType.RESULT_STORED, + ] + + async def test_failed_write_publishes_result_failed(self, tracer: trace.Tracer) -> None: + repo = AsyncMock() + repo.get_execution.return_value = ExecutionDocument( + script="x", execution_id="e7", lang="python", lang_version="3.11" + ) + repo.write_terminal_result.side_effect = PyMongoError("mongo down") + producer = AsyncMock() + processor = _make_processor(tracer, exec_repo=repo, producer=producer) + + await processor.handle_execution_completed( + ExecutionCompletedEvent( + execution_id="e7", exit_code=0, resource_usage=ResourceUsageDomain(), metadata=_METADATA + ) + ) + + assert [c.kwargs["event_to_produce"].event_type for c in producer.produce.await_args_list] == [ + EventType.RESULT_FAILED + ] + producer.produce_or_log.assert_not_awaited() + + class TestPodProgress: """Pod events move the stored execution status forward.""" diff --git a/backend/tests/unit/services/test_auth_service_metrics.py b/backend/tests/unit/services/test_auth_service_metrics.py index 052e36d5..774563c2 100644 --- a/backend/tests/unit/services/test_auth_service_metrics.py +++ b/backend/tests/unit/services/test_auth_service_metrics.py @@ -2,15 +2,19 @@ import pytest import structlog +from aiokafka.errors import KafkaError from opentelemetry import trace from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from pydantic import SecretStr from app.core.security import SecurityService -from app.db import UserRepository +from app.db import EventRepository, UserRepository +from app.db.docs import UserDocument +from app.domain.admin import SystemSettings from app.domain.exceptions import AccountLockedError from app.domain.user import InvalidCredentialsError from app.events import UnifiedProducer +from app.events.core.transport import KafkaEventTransport from app.services.auth_service import AuthService from app.services.login_lockout import LoginLockoutService from app.services.runtime_settings import RuntimeSettingsLoader @@ -19,15 +23,23 @@ pytestmark = pytest.mark.unit -def _service(test_settings: Settings, tracer: trace.Tracer, lockout: AsyncMock, user_repo: AsyncMock) -> AuthService: +def _service( + test_settings: Settings, + tracer: trace.Tracer, + lockout: AsyncMock, + user_repo: AsyncMock, + producer: UnifiedProducer | None = None, +) -> AuthService: + runtime_settings = AsyncMock(spec=RuntimeSettingsLoader) + runtime_settings.get_effective_settings.return_value = SystemSettings() return AuthService( user_repo=user_repo, security_service=MagicMock(spec=SecurityService), tracer=tracer, logger=structlog.get_logger("test.auth_service_metrics"), lockout_service=lockout, - runtime_settings=AsyncMock(spec=RuntimeSettingsLoader), - producer=AsyncMock(spec=UnifiedProducer), + runtime_settings=runtime_settings, + producer=producer or AsyncMock(spec=UnifiedProducer), settings=test_settings, ) @@ -62,3 +74,21 @@ async def test_failed_login_reports_an_unsuccessful_attempt( [span] = span_exporter.get_finished_spans() assert (span.name, span.attributes) == ("auth.login", {"auth.method": "login", "auth.success": False}) + + +async def test_login_succeeds_while_kafka_is_down(test_settings: Settings, tracer: trace.Tracer) -> None: + """Nothing consumes the login event, so a Kafka outage must not lock users out.""" + lockout = AsyncMock(spec=LoginLockoutService) + lockout.check_locked.return_value = False + user_repo = AsyncMock(spec=UserRepository) + user_repo.get_user.return_value = UserDocument(username="alice", email="alice@example.com", hashed_password="h") + event_repo = AsyncMock(spec=EventRepository) + transport = AsyncMock(spec=KafkaEventTransport) + transport.publish.side_effect = KafkaError("broker down") + producer = UnifiedProducer(event_repo, transport, structlog.get_logger("test.auth_service_metrics"), tracer) + service = _service(test_settings, tracer, lockout, user_repo, producer) + + result = await service.login("alice", SecretStr("secret"), ip_address="10.0.0.1", user_agent=None) + + assert result.username == "alice" + event_repo.store_event.assert_awaited_once() diff --git a/backend/tests/unit/services/test_user_settings_service.py b/backend/tests/unit/services/test_user_settings_service.py index 9fe9998b..ec72da72 100644 --- a/backend/tests/unit/services/test_user_settings_service.py +++ b/backend/tests/unit/services/test_user_settings_service.py @@ -73,7 +73,7 @@ def repo() -> _MemoryRepo: @pytest.fixture def service(repo: _MemoryRepo, test_settings: Settings) -> UserSettingsService: producer = AsyncMock(spec=UnifiedProducer) - producer.produce.side_effect = repo.store_produced + producer.produce_or_log.side_effect = repo.store_produced return UserSettingsService(repo, producer, test_settings, structlog.get_logger("test")) diff --git a/docs/architecture/event-storage.md b/docs/architecture/event-storage.md index b5e5669b..dab59672 100644 --- a/docs/architecture/event-storage.md +++ b/docs/architecture/event-storage.md @@ -25,12 +25,11 @@ graph LR UP -->|1. store_event| Events[(events)] UP -->|2. publish| Transport[KafkaEventTransport] Transport --> Kafka[(Kafka topic = event_type)] - UP -.->|on publish error| Mark[mark_publish_failed] ``` `UnifiedProducer.produce()` in `app/events/core/producer.py` does both steps. First `EventRepository.store_event()` dumps the event with `model_dump(exclude_none=True)`, sets `stored_at` and inserts it. A duplicate `event_id` hits the unique index, and the repository logs it and returns, so storing the same event twice is a no-op. Replayed events keep their `event_id`, so a replay doesn't create new rows. -Then `KafkaEventTransport.publish()` sends the event to the topic named after its `event_type`. If that fails, `mark_publish_failed()` sets `publish_failed` and `publish_failed_at` on the stored document and the exception goes back to the caller. Nothing re-publishes flagged events automatically, though a replay session with the Kafka target sends stored events again. +Then `KafkaEventTransport.publish()` sends the event to the topic named after its `event_type`. If that fails, the stored event stays as it is and the error goes back to the caller, or into the log for `produce_or_log`. Nothing re-sends it automatically, though a replay session with the Kafka target sends stored events again. Every service publishes through `UnifiedProducer.produce`, which also opens a trace span and records the publish metrics. diff --git a/docs/architecture/event-system-design.md b/docs/architecture/event-system-design.md index 971c4337..e48a5f60 100644 --- a/docs/architecture/event-system-design.md +++ b/docs/architecture/event-system-design.md @@ -67,19 +67,15 @@ sequenceDiagram ## Publishing -All publishing goes through `UnifiedProducer` in `app/events/core/producer.py`. It's a store-then-publish outbox: it writes the event to MongoDB, then hands it to `KafkaEventTransport`, which calls `broker.publish` with the event type as the topic. +All publishing goes through `UnifiedProducer` in `app/events/core/producer.py`. It writes the event to MongoDB, then hands it to `KafkaEventTransport`, which calls `broker.publish` with the event type as the topic. ```python async def produce(self, event_to_produce: DomainEvent, key: str) -> None: await self._event_repository.store_event(event_to_produce) - try: - await self._transport.publish(event_to_produce, event_to_produce.event_type, key) - except Exception: - await self._event_repository.mark_publish_failed(event_to_produce.event_id) - raise + await self._transport.publish(event_to_produce, event_to_produce.event_type, key) ``` -`store_event` ignores a duplicate `event_id`, so storing the same event twice is a no-op. If Kafka fails, the stored event gets `publish_failed: true` and the exception goes back to the caller. Nothing re-sends those events automatically. A replay session with the Kafka target is the only way they go out again. +`store_event` ignores a duplicate `event_id`, so storing the same event twice is a no-op. If Kafka fails, `produce` raises to the caller, which decides: the API marks a new execution `error` and answers 500, and a cancel answers 500. `produce_or_log` logs the Kafka error instead, for events sent after their work is done: auth and settings events, pod progress and the result processor's terminal event and `result_stored`. Nothing re-sends an event that didn't go out; a replay session with the Kafka target can. ```mermaid graph TB @@ -102,7 +98,7 @@ graph TB Kafka --> APIc ``` -The worker consumes only `execution_requested` and `execution_cancelled`; inside it, the parts call each other directly. The DLQ manager is the one place that skips the outbox: it calls `broker.publish` directly for retries and its `dlq_message_*` status events. See [Kafka Topics](kafka-topic-architecture.md) for the consumer groups. +The worker consumes only `execution_requested` and `execution_cancelled`; inside it, the parts call each other directly. The DLQ manager is the one place that skips the producer: it calls `broker.publish` directly for retries and its `dlq_message_*` status events. See [Kafka Topics](kafka-topic-architecture.md) for the consumer groups. ## Adding an event type diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 4af2b0af..de2dca67 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -114,7 +114,7 @@ The rate limit middleware has already run by the time the route sees the request The execution scheduler in the worker puts the request into the [execution queue](execution-queue.md) and, once a slot is free, has the k8s worker create a ConfigMap with the script and a hardened pod. The k8s worker publishes `pod_created`; if the Kubernetes API refused, the execution is stored as `failed` instead. The pod monitor watches the pod and publishes `pod_scheduled` and `pod_running`. When the pod ends, the monitor reads its output and hands `execution_completed`, `execution_failed` or `execution_timeout` to the result processor. Then it deletes the pod, which frees its share of the ResourceQuota, and Kubernetes garbage-collects the ConfigMap through its owner reference. -The result processor writes the final state and output to MongoDB, then publishes the terminal event and `result_stored`, or `result_failed` if it couldn't. Last, the monitor tells the scheduler the execution finished, so its queue slot frees and the next execution starts. A cancel goes the same way in reverse: the scheduler takes `execution_cancelled`, stores the execution as `cancelled` and deletes its pod. +The result processor writes the final state and output to MongoDB, then publishes the terminal event and `result_stored`, or `result_failed` if the write failed. Last, the monitor tells the scheduler the execution finished, so its queue slot frees and the next execution starts. A cancel goes the same way in reverse: the scheduler takes `execution_cancelled`, stores the execution as `cancelled` and deletes its pod. ## SSE diff --git a/docs/components/workers/k8s_worker.md b/docs/components/workers/k8s_worker.md index f2692ac1..3ffb46e3 100644 --- a/docs/components/workers/k8s_worker.md +++ b/docs/components/workers/k8s_worker.md @@ -11,7 +11,7 @@ graph LR ## Creating a pod -For a new execution the worker creates a ConfigMap `script-{execution_id}` holding the user's script and `entrypoint.sh`, then a pod `executor-{execution_id}`. Once the pod exists, it patches the ConfigMap with an ownerReference to the pod, so Kubernetes deletes the ConfigMap together with the pod. If the Kubernetes API rejects that patch, the worker only logs a warning and the ConfigMap outlives the pod. Then it publishes `pod_created`. If anything else in that sequence fails, the error goes back to the scheduler, which stores the execution as `failed` with error type `system_error` and exit code -1. A pod that already exists (409) counts as created, so a retried request doesn't make a second one. +For a new execution the worker creates a ConfigMap `script-{execution_id}` holding the user's script and `entrypoint.sh`, then a pod `executor-{execution_id}`. Once the pod exists, it patches the ConfigMap with an ownerReference to the pod, so Kubernetes deletes the ConfigMap together with the pod. If the Kubernetes API rejects that patch, the worker only logs a warning and the ConfigMap outlives the pod. Then it publishes `pod_created`; a Kafka error there is only logged, since the pod already exists. If anything else in that sequence fails, the error goes back to the scheduler, which stores the execution as `failed` with error type `system_error` and exit code -1. A pod that already exists (409) counts as created, so a retried request doesn't make a second one. The image and command come from the `execution_requested` event, which the API filled in from the runtime registry (`app/runtime_registry.py`). The registry can also give a language its own pod settings: extra env vars, a bigger `/tmp`, and minimum CPU, memory and timeout. The pod builder takes the larger of the configured value and the language minimum. Only Go uses this today: diff --git a/docs/components/workers/result_processor.md b/docs/components/workers/result_processor.md index 463cf5a0..c3c0f0ce 100644 --- a/docs/components/workers/result_processor.md +++ b/docs/components/workers/result_processor.md @@ -12,7 +12,7 @@ graph LR For each event it loads the execution, records metrics, and writes the status, exit code, stdout, stderr, resource usage and error type onto the execution's document in the `executions` collection. There's no separate results collection. The write is conditional: it only applies while the execution is still in an active status. When it doesn't apply, because the execution already ended another way, nothing is published. That keeps the failure a cancelled pod reports when it's deleted from overwriting the cancel or sending a notification. A timeout is stored with exit code -1, and a cancel with none. -After the write it publishes the terminal event (`execution_completed`, `execution_failed` or `execution_timeout`; a cancel has its event from the API already), then `result_stored`, which carries a storage reference rather than the output. The SSE service reads the output back from MongoDB when it forwards that event. If the MongoDB write or the Kafka publish fails, the processor tries to publish `result_failed` instead. That goes through the same producer, which stores the event in MongoDB first, so it fails too while MongoDB is down. +After the write it publishes the terminal event (`execution_completed`, `execution_failed` or `execution_timeout`; a cancel has its event from the API already), then `result_stored`, which carries a storage reference rather than the output. The SSE service reads the output back from MongoDB when it forwards that event. If the MongoDB write fails, the processor publishes `result_failed` instead. That event is stored in MongoDB first too, so it fails as well while MongoDB is down. A Kafka error after the write is only logged: the result is stored, the pod monitor still deletes the pod and frees the slot, and a client whose stream missed `result_stored` gets the result when it opens the stream again. The result processor doesn't touch Kubernetes. The [pod monitor](pod_monitor.md) deletes finished pods, and their ConfigMaps go with them. diff --git a/docs/testing/kafka-test-stability.md b/docs/testing/kafka-test-stability.md index be01f55f..e9c42748 100644 --- a/docs/testing/kafka-test-stability.md +++ b/docs/testing/kafka-test-stability.md @@ -4,7 +4,7 @@ Most of what used to make Kafka tests flaky is gone. What's left is how long con ## No producer to manage -`UnifiedProducer` is a store-then-publish outbox. It writes the event to MongoDB, then publishes through `KafkaEventTransport`, which calls `broker.publish()`. If the publish fails, it marks the stored event as failed to publish and re-raises. It has no Kafka client of its own and nothing to start or stop. The one `KafkaBroker` per process (aiokafka 0.14 under FastStream 0.7.7) is started by `dishka_lifespan` in the API and by FastStream's app runner in the worker. +`UnifiedProducer` writes the event to MongoDB, then publishes through `KafkaEventTransport`, which calls `broker.publish()`. `produce` raises a failed publish to the caller, and `produce_or_log` logs it. It has no Kafka client of its own and nothing to start or stop. The one `KafkaBroker` per process (aiokafka 0.14 under FastStream 0.7.7) is started by `dishka_lifespan` in the API and by FastStream's app runner in the worker. Unit tests don't touch Kafka. The E2E suite runs inside the backend container, and its session-scoped `app` fixture enters the real lifespan. So each xdist worker starts one broker with the SSE and notification subscribers, and stops it when the session ends.