diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a4a4006f..d5e598efe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BUG FIXES: * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) +* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. ([#4923](https://github.com/microsoft/AzureTRE/issues/4923)) * Protect active validation environments and require successful smoke tests before custom tests. Validate shared-service command arguments and forward explicit Nexus EULA consent. (`devops` 0.6.20, `build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Give E2E cleanup separate deadlines and allow 30 seconds for smoke health requests. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Verify TLS for identity authentication and keep response content and URL parameters out of E2E operation polling logs. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) diff --git a/api_app/_version.py b/api_app/_version.py index 90c08ed81..92f2eb158 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.7" +__version__ = "0.28.8" diff --git a/api_app/main.py b/api_app/main.py index 0100c164b..c5d7e9b79 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -1,4 +1,5 @@ import asyncio +from typing import Any import uvicorn from fastapi import FastAPI @@ -22,8 +23,30 @@ from service_bus.airlock_request_status_update import AirlockStatusUpdater +class BackgroundTaskManager: + def __init__(self) -> None: + self._tasks: set[asyncio.Task[Any]] = set() + self.is_shutting_down: bool = False + + def add(self, task: asyncio.Task[Any]) -> None: + self._tasks.add(task) + + def discard(self, task: asyncio.Task[Any]) -> None: + self._tasks.discard(task) + + def get_tasks(self) -> list[asyncio.Task[Any]]: + return list(self._tasks) + + +SHUTDOWN_TIMEOUT = 10.0 + + @asynccontextmanager async def lifespan(app: FastAPI): + + background_tasks = BackgroundTaskManager() + app.state.background_tasks = background_tasks + while not await bootstrap_database(): await asyncio.sleep(5) logger.warning("Database connection could not be established") @@ -34,9 +57,73 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() - asyncio.create_task(deploymentStatusUpdater.receive_messages()) - asyncio.create_task(airlockStatusUpdater.receive_messages()) - yield + shutdown_inspected_tasks: set[asyncio.Task[Any]] = set() + shutdown_reported_tasks: set[asyncio.Task[Any]] = set() + + def inspect_shutdown_task(task: asyncio.Task[Any]) -> None: + if task.cancelled() or task in shutdown_reported_tasks: + return + + try: + exception = task.exception() + except asyncio.CancelledError: + return + + if exception is not None: + shutdown_reported_tasks.add(task) + logger.warning( + f"Background task {task.get_name()} raised exception during shutdown: {exception}", + exc_info=(type(exception), exception, exception.__traceback__), + ) + + def track(task: asyncio.Task[Any]) -> None: + def _done_callback(task: asyncio.Task[Any]) -> None: + background_tasks.discard(task) + if task in shutdown_inspected_tasks: + inspect_shutdown_task(task) + return + + if task.cancelled(): + return + + try: + exception = task.exception() + except asyncio.CancelledError: + return + + if exception is not None: + logger.error( + f"Background task {task.get_name()} failed", + exc_info=(type(exception), exception, exception.__traceback__), + ) + + background_tasks.add(task) + task.add_done_callback(_done_callback) + + track(asyncio.create_task(deploymentStatusUpdater.receive_messages(), name="deployment-status-updater")) + track(asyncio.create_task(airlockStatusUpdater.receive_messages(), name="airlock-status-updater")) + + try: + yield + finally: + background_tasks.is_shutting_down = True + tasks = background_tasks.get_tasks() + shutdown_inspected_tasks.update(tasks) + logger.info(f"Cancelling {len(tasks)} background tasks") + + for task in tasks: + logger.debug(f"Cancelling task {task.get_name()}") + task.cancel() + + if tasks: + done, pending = await asyncio.wait(tasks, timeout=SHUTDOWN_TIMEOUT) + for task in done: + inspect_shutdown_task(task) + + if pending: + logger.error("Timeout waiting for background tasks to shutdown") + for task in pending: + logger.warning(f"Task {task.get_name()} did not terminate in time during shutdown") def get_application() -> FastAPI: diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py new file mode 100644 index 000000000..facfe13af --- /dev/null +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -0,0 +1,267 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch +import pytest +from fastapi import FastAPI + + +@pytest.mark.asyncio +async def test_background_task_manager(): + from main import BackgroundTaskManager + + manager = BackgroundTaskManager() + assert manager.is_shutting_down is False + + async def dummy(): + pass + + task = asyncio.create_task(dummy()) + manager.add(task) + assert task in manager.get_tasks() + + manager.discard(task) + assert task not in manager.get_tasks() + + await task + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +async def test_lifespan_lifecycle_normal(mock_airlock, mock_deployment, mock_bootstrap): + from main import BackgroundTaskManager, lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.receive_messages = AsyncMock() + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + assert hasattr(app.state, "background_tasks") + assert isinstance(app.state.background_tasks, BackgroundTaskManager) + tasks = app.state.background_tasks.get_tasks() + assert len(tasks) == 2 + names = {t.get_name() for t in tasks} + assert "deployment-status-updater" in names + assert "airlock-status-updater" in names + + assert app.state.background_tasks.is_shutting_down is True + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +async def test_lifespan_task_failure_during_runtime(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + async def fail_immediately(): + raise ValueError("Runtime failure") + + mock_deployment_instance.receive_messages = fail_immediately + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + await asyncio.sleep(0.1) + + # Verify that the runtime failure was logged as an error + error_logs = [ + call + for call in mock_logger.error.call_args_list + if "Background task deployment-status-updater failed" in call[0][0] + ] + assert len(error_logs) == 1 + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +async def test_lifespan_task_failure_during_shutdown(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + async def raise_teardown_exception(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + raise RuntimeError("Database connection closed abruptly") + + mock_deployment_instance.receive_messages = raise_teardown_exception + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + await asyncio.sleep(0.1) + + # The error should NOT be logged as error in _done_callback because it was gated on is_shutting_down + for call in mock_logger.error.call_args_list: + assert "Background task deployment-status-updater failed" not in call[0][0] + + # Instead, it should be caught during shutdown inspection and logged as warning + warning_logs = [ + call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0] + ] + assert len(warning_logs) == 1 + assert "deployment-status-updater" in warning_logs[0][0][0] + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +@patch("main.SHUTDOWN_TIMEOUT", 0.05) +async def test_lifespan_timeout_preserves_completed_task_exceptions( + mock_logger, mock_airlock, mock_deployment, mock_bootstrap +): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + # Deployment updater completes quickly during cancellation by raising an exception + async def raise_teardown_exception(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + raise RuntimeError("Deployment teardown failed") + + mock_deployment_instance.receive_messages = raise_teardown_exception + mock_deployment.return_value = mock_deployment_instance + + hang_event = asyncio.Event() + + # Airlock updater suppresses cancellation and hangs past the timeout + async def hang_on_cancellation(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await hang_event.wait() + raise RuntimeError("Airlock teardown failed late") + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = hang_on_cancellation + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + try: + async with lifespan(app): + await asyncio.sleep(0.01) + + # 1. Completed task exception is still inspected and logged as warning despite timeout + warning_logs = [ + call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0] + ] + assert len(warning_logs) == 1 + assert "deployment-status-updater" in warning_logs[0][0][0] + + # 2. Timeout error is logged + error_logs = [ + call + for call in mock_logger.error.call_args_list + if "Timeout waiting for background tasks to shutdown" in call[0][0] + ] + assert len(error_logs) == 1 + + # 3. Pending task is logged with warning + pending_logs = [ + call + for call in mock_logger.warning.call_args_list + if "did not terminate in time during shutdown" in call[0][0] + ] + assert len(pending_logs) == 1 + assert "airlock-status-updater" in pending_logs[0][0][0] + finally: + hang_event.set() + await asyncio.sleep(0.01) + late_shutdown_logs = [ + call + for call in mock_logger.warning.call_args_list + if "Background task airlock-status-updater raised exception during shutdown" in call[0][0] + ] + assert len(late_shutdown_logs) == 1 + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +@patch("main.SHUTDOWN_TIMEOUT", 0.05) +async def test_lifespan_cancellation_suppression_strictly_bounded( + mock_logger, mock_airlock, mock_deployment, mock_bootstrap +): + import time + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + hang_event = asyncio.Event() + + # Worker suppresses cancellation and ignores cancel signal + async def suppress_cancellation(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await hang_event.wait() + + mock_deployment_instance.receive_messages = suppress_cancellation + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = suppress_cancellation + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + try: + start_time = time.monotonic() + async with lifespan(app): + await asyncio.sleep(0.01) + elapsed = time.monotonic() - start_time + + # Teardown should return promptly near SHUTDOWN_TIMEOUT and NOT hang + assert elapsed < 0.5 + + # Both tasks should be reported as pending + pending_logs = [ + call + for call in mock_logger.warning.call_args_list + if "did not terminate in time during shutdown" in call[0][0] + ] + assert len(pending_logs) == 2 + finally: + hang_event.set() + await asyncio.sleep(0.01)