From 74c89c84f1bd32215e68972f4b6779976241b13e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 5 Jun 2026 10:13:46 +0000 Subject: [PATCH 01/13] implement app-scoped lifecycle management to prevent silent garbage collection of background tasks. --- CHANGELOG.md | 3 +++ api_app/_version.py | 2 +- api_app/main.py | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab4bcb537..75521f08b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ ENHANCEMENTS: * Specify default_outbound_access_enabled = false setting for all subnets ([#4757](https://github.com/microsoft/AzureTRE/pull/4757)) * Pin all GitHub Actions workflow steps to full commit SHAs to prevent supply chain attacks plus update to latest releases ([#4886](https://github.com/microsoft/AzureTRE/pull/4886)) +BUG FIXES: +* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. + ## (0.28.0) (March 2, 2026) **BREAKING CHANGES** * Sonatype Nexus shared service now requires explicit EULA acceptance (`accept_nexus_eula: true`) when deploying. This ensures compliance with Sonatype Nexus Community Edition licensing. ([#4842](https://github.com/microsoft/AzureTRE/issues/4842)) diff --git a/api_app/_version.py b/api_app/_version.py index 2cb28789f2..7923a95d33 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.15" +__version__ = "0.25.16" diff --git a/api_app/main.py b/api_app/main.py index 0bdc769141..12854da8fb 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -24,6 +24,8 @@ @asynccontextmanager async def lifespan(app: FastAPI): + app.state.background_tasks = set() + while not await bootstrap_database(): await asyncio.sleep(5) logger.warning("Database connection could not be established") @@ -34,9 +36,34 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() - asyncio.create_task(deploymentStatusUpdater.receive_messages()) - asyncio.create_task(airlockStatusUpdater.receive_messages()) - yield + def track(task): + app.state.background_tasks.add(task) + task.add_done_callback(app.state.background_tasks.discard) + + track(asyncio.create_task( + deploymentStatusUpdater.receive_messages(), + name="deployment-status-updater" + )) + track(asyncio.create_task( + airlockStatusUpdater.receive_messages(), + name="airlock-status-updater" + )) + + try: + yield + finally: + tasks = list(app.state.background_tasks) + logger.info(f"Cancelling {len(tasks)} background tasks") + + for task in tasks: + logger.debug(f"Cancelling task {task.get_name()}") + task.cancel() + + results = await asyncio.gather(*tasks, return_exceptions=True) + + for task, result in zip(tasks, results): + if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError): + logger.error(f"Task {task.get_name()} failed", exc_info=(type(result), result, result.__traceback__)) def get_application() -> FastAPI: From 7be55fe418a4ab7e15244f11792b3ca8183390c0 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 5 Jun 2026 12:08:34 +0000 Subject: [PATCH 02/13] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75521f08b5..546e0944e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ENHANCEMENTS: * Pin all GitHub Actions workflow steps to full commit SHAs to prevent supply chain attacks plus update to latest releases ([#4886](https://github.com/microsoft/AzureTRE/pull/4886)) BUG FIXES: -* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. +* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. ([#4923](https://github.com/microsoft/AzureTRE/issues/4923)) ## (0.28.0) (March 2, 2026) **BREAKING CHANGES** From eb3bf72b0f7086dab0ee2260febd544049744096 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 5 Jun 2026 12:53:39 +0000 Subject: [PATCH 03/13] bump api --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 7923a95d33..57a322dfc2 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.16" +__version__ = "0.25.17" From c2a4be7e6d283bdc1c4e53a1438f0f95c7e9c43c Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 5 Jun 2026 12:59:06 +0000 Subject: [PATCH 04/13] implement copilot suggestions --- api_app/main.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/api_app/main.py b/api_app/main.py index 12854da8fb..c597b58ac9 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -37,8 +37,24 @@ async def lifespan(app: FastAPI): await airlockStatusUpdater.init_repos() def track(task): + def _done_callback(task): + app.state.background_tasks.discard(task) + if task.cancelled(): + return + + try: + exception = task.exception() + except asyncio.CancelledError: + return + + if exception is not None: + logger.error( + f"Background task {task.get_name()} failed", + exc_info=(type(exception), exception, exception.__traceback__) + ) + app.state.background_tasks.add(task) - task.add_done_callback(app.state.background_tasks.discard) + task.add_done_callback(_done_callback) track(asyncio.create_task( deploymentStatusUpdater.receive_messages(), @@ -59,11 +75,7 @@ def track(task): logger.debug(f"Cancelling task {task.get_name()}") task.cancel() - results = await asyncio.gather(*tasks, return_exceptions=True) - - for task, result in zip(tasks, results): - if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError): - logger.error(f"Task {task.get_name()} failed", exc_info=(type(result), result, result.__traceback__)) + await asyncio.gather(*tasks, return_exceptions=True) def get_application() -> FastAPI: From 2d043f30ca82601268429613f009a37b91339fe2 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 6 Jul 2026 13:29:05 +0000 Subject: [PATCH 05/13] Bump version to 0.25.27 and implement BackgroundTaskManager for improved task management during application lifecycle --- api_app/_version.py | 2 +- api_app/main.py | 46 ++++++++- api_app/tests_ma/test_main_lifecycle.py | 127 ++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 api_app/tests_ma/test_main_lifecycle.py diff --git a/api_app/_version.py b/api_app/_version.py index eda2a21a48..7df7ad28da 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.26" +__version__ = "0.25.27" diff --git a/api_app/main.py b/api_app/main.py index c597b58ac9..d45139341e 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -1,4 +1,5 @@ import asyncio +from typing import Any import uvicorn from fastapi import FastAPI @@ -22,9 +23,24 @@ from service_bus.airlock_request_status_update import AirlockStatusUpdater +class BackgroundTaskManager: + def __init__(self) -> None: + self._tasks: set[asyncio.Task[Any]] = set() + self.is_shutting_down: bool = False + + def add(self, task: asyncio.Task[Any]) -> None: + self._tasks.add(task) + + def discard(self, task: asyncio.Task[Any]) -> None: + self._tasks.discard(task) + + def get_tasks(self) -> list[asyncio.Task[Any]]: + return list(self._tasks) + + @asynccontextmanager async def lifespan(app: FastAPI): - app.state.background_tasks = set() + app.state.background_tasks = BackgroundTaskManager() while not await bootstrap_database(): await asyncio.sleep(5) @@ -36,9 +52,12 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() - def track(task): - def _done_callback(task): + def track(task: asyncio.Task[Any]) -> None: + def _done_callback(task: asyncio.Task[Any]) -> None: app.state.background_tasks.discard(task) + if app.state.background_tasks.is_shutting_down: + return + if task.cancelled(): return @@ -68,14 +87,31 @@ def _done_callback(task): try: yield finally: - tasks = list(app.state.background_tasks) + app.state.background_tasks.is_shutting_down = True + tasks = app.state.background_tasks.get_tasks() logger.info(f"Cancelling {len(tasks)} background tasks") for task in tasks: logger.debug(f"Cancelling task {task.get_name()}") task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) + if tasks: + try: + results = await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), + timeout=10.0 + ) + for task, result in zip(tasks, results): + if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError): + logger.warning( + f"Background task {task.get_name()} raised exception during shutdown: {result}", + exc_info=(type(result), result, result.__traceback__) + ) + except asyncio.TimeoutError: + logger.error("Timeout waiting for background tasks to shutdown") + pending = [t for t in tasks if not t.done()] + for t in pending: + logger.warning(f"Task {t.get_name()} did not terminate in time during shutdown") def get_application() -> FastAPI: diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py new file mode 100644 index 0000000000..8b365522ba --- /dev/null +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -0,0 +1,127 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch +import pytest +from fastapi import FastAPI + + +@pytest.mark.asyncio +async def test_background_task_manager(): + from main import BackgroundTaskManager + + manager = BackgroundTaskManager() + assert manager.is_shutting_down is False + + async def dummy(): + pass + + task = asyncio.create_task(dummy()) + manager.add(task) + assert task in manager.get_tasks() + + manager.discard(task) + assert task not in manager.get_tasks() + + await task + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +async def test_lifespan_lifecycle_normal(mock_airlock, mock_deployment, mock_bootstrap): + from main import BackgroundTaskManager, lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.receive_messages = AsyncMock() + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + assert hasattr(app.state, "background_tasks") + assert isinstance(app.state.background_tasks, BackgroundTaskManager) + tasks = app.state.background_tasks.get_tasks() + assert len(tasks) == 2 + names = {t.get_name() for t in tasks} + assert "deployment-status-updater" in names + assert "airlock-status-updater" in names + + assert app.state.background_tasks.is_shutting_down is True + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +async def test_lifespan_task_failure_during_runtime(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + async def fail_immediately(): + raise ValueError("Runtime failure") + + mock_deployment_instance.receive_messages = fail_immediately + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + await asyncio.sleep(0.1) + + # Verify that the runtime failure was logged as an error + error_logs = [call for call in mock_logger.error.call_args_list if "Background task deployment-status-updater failed" in call[0][0]] + assert len(error_logs) == 1 + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +async def test_lifespan_task_failure_during_shutdown(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + async def raise_teardown_exception(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + raise RuntimeError("Database connection closed abruptly") + + mock_deployment_instance.receive_messages = raise_teardown_exception + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = AsyncMock() + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + async with lifespan(app): + await asyncio.sleep(0.1) + + # The error should NOT be logged as error in _done_callback because it was gated on is_shutting_down + for call in mock_logger.error.call_args_list: + assert "Background task deployment-status-updater failed" not in call[0][0] + + # Instead, it should be caught during shutdown gather and logged as warning + warning_logs = [call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]] + assert len(warning_logs) == 1 + assert "deployment-status-updater" in warning_logs[0][0][0] From 8a403f8a1fe44475b2c75275a1c872a5af044f7b Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 7 Jul 2026 14:42:47 +0000 Subject: [PATCH 06/13] suppress bandit warning --- api_app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/main.py b/api_app/main.py index d45139341e..6b7a3f893f 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -149,4 +149,4 @@ def get_application() -> FastAPI: FastAPIInstrumentor.instrument_app(app) if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000, loop="asyncio") + uvicorn.run(app, host="0.0.0.0", port=8000, loop="asyncio") # nosec B104: intentional bind to all interfaces From 1c7e021a2f147b38561931f7732df86f4ab35c72 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 4 Sep 2026 11:16:07 +0000 Subject: [PATCH 07/13] Refactor shutdown handling in lifespan to improve task exception logging and timeout behavior --- api_app/main.py | 37 ++++---- api_app/tests_ma/test_main_lifecycle.py | 111 +++++++++++++++++++++++- 2 files changed, 132 insertions(+), 16 deletions(-) diff --git a/api_app/main.py b/api_app/main.py index 6b7a3f893f..bd241d27f8 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -38,6 +38,9 @@ def get_tasks(self) -> list[asyncio.Task[Any]]: return list(self._tasks) +SHUTDOWN_TIMEOUT = 10.0 + + @asynccontextmanager async def lifespan(app: FastAPI): app.state.background_tasks = BackgroundTaskManager() @@ -96,22 +99,26 @@ def _done_callback(task: asyncio.Task[Any]) -> None: task.cancel() if tasks: - try: - results = await asyncio.wait_for( - asyncio.gather(*tasks, return_exceptions=True), - timeout=10.0 - ) - for task, result in zip(tasks, results): - if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError): - logger.warning( - f"Background task {task.get_name()} raised exception during shutdown: {result}", - exc_info=(type(result), result, result.__traceback__) - ) - except asyncio.TimeoutError: + done, pending = await asyncio.wait(tasks, timeout=SHUTDOWN_TIMEOUT) + for task in done: + if task.cancelled(): + continue + + try: + exception = task.exception() + except asyncio.CancelledError: + continue + + if exception is not None: + logger.warning( + f"Background task {task.get_name()} raised exception during shutdown: {exception}", + exc_info=(type(exception), exception, exception.__traceback__) + ) + + if pending: logger.error("Timeout waiting for background tasks to shutdown") - pending = [t for t in tasks if not t.done()] - for t in pending: - logger.warning(f"Task {t.get_name()} did not terminate in time during shutdown") + for task in pending: + logger.warning(f"Task {task.get_name()} did not terminate in time during shutdown") def get_application() -> FastAPI: diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py index 8b365522ba..af2ecc539f 100644 --- a/api_app/tests_ma/test_main_lifecycle.py +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -121,7 +121,116 @@ async def raise_teardown_exception(): for call in mock_logger.error.call_args_list: assert "Background task deployment-status-updater failed" not in call[0][0] - # Instead, it should be caught during shutdown gather and logged as warning + # Instead, it should be caught during shutdown inspection and logged as warning warning_logs = [call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]] assert len(warning_logs) == 1 assert "deployment-status-updater" in warning_logs[0][0][0] + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +@patch("main.SHUTDOWN_TIMEOUT", 0.05) +async def test_lifespan_timeout_preserves_completed_task_exceptions(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + # Deployment updater completes quickly during cancellation by raising an exception + async def raise_teardown_exception(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + raise RuntimeError("Deployment teardown failed") + + mock_deployment_instance.receive_messages = raise_teardown_exception + mock_deployment.return_value = mock_deployment_instance + + hang_event = asyncio.Event() + + # Airlock updater suppresses cancellation and hangs past the timeout + async def hang_on_cancellation(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await hang_event.wait() + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = hang_on_cancellation + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + try: + async with lifespan(app): + await asyncio.sleep(0.01) + + # 1. Completed task exception is still inspected and logged as warning despite timeout + warning_logs = [call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]] + assert len(warning_logs) == 1 + assert "deployment-status-updater" in warning_logs[0][0][0] + + # 2. Timeout error is logged + error_logs = [call for call in mock_logger.error.call_args_list if "Timeout waiting for background tasks to shutdown" in call[0][0]] + assert len(error_logs) == 1 + + # 3. Pending task is logged with warning + pending_logs = [call for call in mock_logger.warning.call_args_list if "did not terminate in time during shutdown" in call[0][0]] + assert len(pending_logs) == 1 + assert "airlock-status-updater" in pending_logs[0][0][0] + finally: + hang_event.set() + await asyncio.sleep(0.01) + + +@pytest.mark.asyncio +@patch("main.bootstrap_database", return_value=True) +@patch("main.DeploymentStatusUpdater") +@patch("main.AirlockStatusUpdater") +@patch("main.logger") +@patch("main.SHUTDOWN_TIMEOUT", 0.05) +async def test_lifespan_cancellation_suppression_strictly_bounded(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): + import time + from main import lifespan + + mock_deployment_instance = MagicMock() + mock_deployment_instance.init_repos = AsyncMock() + + hang_event = asyncio.Event() + + # Worker suppresses cancellation and ignores cancel signal + async def suppress_cancellation(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await hang_event.wait() + + mock_deployment_instance.receive_messages = suppress_cancellation + mock_deployment.return_value = mock_deployment_instance + + mock_airlock_instance = MagicMock() + mock_airlock_instance.init_repos = AsyncMock() + mock_airlock_instance.receive_messages = suppress_cancellation + mock_airlock.return_value = mock_airlock_instance + + app = FastAPI() + + try: + start_time = time.monotonic() + async with lifespan(app): + await asyncio.sleep(0.01) + elapsed = time.monotonic() - start_time + + # Teardown should return promptly near SHUTDOWN_TIMEOUT and NOT hang + assert elapsed < 0.5 + + # Both tasks should be reported as pending + pending_logs = [call for call in mock_logger.warning.call_args_list if "did not terminate in time during shutdown" in call[0][0]] + assert len(pending_logs) == 2 + finally: + hang_event.set() + await asyncio.sleep(0.01) From baaf3b8bbcad5dc9aab89de0cbcb5f6cdfdeb54f Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:19:28 +0100 Subject: [PATCH 08/13] Bump version to 0.26.9 --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 76b47c1ddb..5252f582fe 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.8" +__version__ = "0.26.9" From 582bcb16670f489a986df52072ff5c825eee632d Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 7 Sep 2026 09:47:02 +0000 Subject: [PATCH 09/13] Prevent background tasks from being garbage collected during shutdown --- api_app/main.py | 18 ++++++++++++------ api_app/tests_ma/test_main_lifecycle.py | 6 ++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/api_app/main.py b/api_app/main.py index bd241d27f8..5e69d638c0 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -43,7 +43,9 @@ def get_tasks(self) -> list[asyncio.Task[Any]]: @asynccontextmanager async def lifespan(app: FastAPI): - app.state.background_tasks = BackgroundTaskManager() + + background_tasks = BackgroundTaskManager() + app.state.background_tasks = background_tasks while not await bootstrap_database(): await asyncio.sleep(5) @@ -55,10 +57,12 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() + shutdown_inspected_tasks: set[asyncio.Task[Any]] = set() + def track(task: asyncio.Task[Any]) -> None: def _done_callback(task: asyncio.Task[Any]) -> None: - app.state.background_tasks.discard(task) - if app.state.background_tasks.is_shutting_down: + background_tasks.discard(task) + if task in shutdown_inspected_tasks: return if task.cancelled(): @@ -75,7 +79,7 @@ def _done_callback(task: asyncio.Task[Any]) -> None: exc_info=(type(exception), exception, exception.__traceback__) ) - app.state.background_tasks.add(task) + background_tasks.add(task) task.add_done_callback(_done_callback) track(asyncio.create_task( @@ -90,8 +94,9 @@ def _done_callback(task: asyncio.Task[Any]) -> None: try: yield finally: - app.state.background_tasks.is_shutting_down = True - tasks = app.state.background_tasks.get_tasks() + background_tasks.is_shutting_down = True + tasks = background_tasks.get_tasks() + shutdown_inspected_tasks.update(tasks) logger.info(f"Cancelling {len(tasks)} background tasks") for task in tasks: @@ -116,6 +121,7 @@ def _done_callback(task: asyncio.Task[Any]) -> None: ) if pending: + shutdown_inspected_tasks.difference_update(pending) logger.error("Timeout waiting for background tasks to shutdown") for task in pending: logger.warning(f"Task {task.get_name()} did not terminate in time during shutdown") diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py index af2ecc539f..75ee9e2474 100644 --- a/api_app/tests_ma/test_main_lifecycle.py +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -157,6 +157,7 @@ async def hang_on_cancellation(): await asyncio.sleep(5) except asyncio.CancelledError: await hang_event.wait() + raise RuntimeError("Airlock teardown failed late") mock_airlock_instance = MagicMock() mock_airlock_instance.init_repos = AsyncMock() @@ -185,6 +186,11 @@ async def hang_on_cancellation(): finally: hang_event.set() await asyncio.sleep(0.01) + late_error_logs = [ + call for call in mock_logger.error.call_args_list + if "Background task airlock-status-updater failed" in call[0][0] + ] + assert len(late_error_logs) == 1 @pytest.mark.asyncio From 2e2d61f77210a854ef567654b71a9d42f3268c1a Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:10:58 +0000 Subject: [PATCH 10/13] Bump version to 0.26.10 --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 5252f582fe..29dd6604f8 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.9" +__version__ = "0.26.10" From 37fefb567b8ba0b66d1100d088b4fba7a5f3418a Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:29:41 +0000 Subject: [PATCH 11/13] Refactor shutdown task exception handling and update tests for logging --- api_app/main.py | 33 ++++++++++++++----------- api_app/tests_ma/test_main_lifecycle.py | 8 +++--- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/api_app/main.py b/api_app/main.py index 5e69d638c0..99cf907a0f 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -58,11 +58,29 @@ async def lifespan(app: FastAPI): await airlockStatusUpdater.init_repos() shutdown_inspected_tasks: set[asyncio.Task[Any]] = set() + shutdown_reported_tasks: set[asyncio.Task[Any]] = set() + + def inspect_shutdown_task(task: asyncio.Task[Any]) -> None: + if task.cancelled() or task in shutdown_reported_tasks: + return + + try: + exception = task.exception() + except asyncio.CancelledError: + return + + if exception is not None: + shutdown_reported_tasks.add(task) + logger.warning( + f"Background task {task.get_name()} raised exception during shutdown: {exception}", + exc_info=(type(exception), exception, exception.__traceback__) + ) def track(task: asyncio.Task[Any]) -> None: def _done_callback(task: asyncio.Task[Any]) -> None: background_tasks.discard(task) if task in shutdown_inspected_tasks: + inspect_shutdown_task(task) return if task.cancelled(): @@ -106,22 +124,9 @@ def _done_callback(task: asyncio.Task[Any]) -> None: if tasks: done, pending = await asyncio.wait(tasks, timeout=SHUTDOWN_TIMEOUT) for task in done: - if task.cancelled(): - continue - - try: - exception = task.exception() - except asyncio.CancelledError: - continue - - if exception is not None: - logger.warning( - f"Background task {task.get_name()} raised exception during shutdown: {exception}", - exc_info=(type(exception), exception, exception.__traceback__) - ) + inspect_shutdown_task(task) if pending: - shutdown_inspected_tasks.difference_update(pending) logger.error("Timeout waiting for background tasks to shutdown") for task in pending: logger.warning(f"Task {task.get_name()} did not terminate in time during shutdown") diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py index 75ee9e2474..290b666fea 100644 --- a/api_app/tests_ma/test_main_lifecycle.py +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -186,11 +186,11 @@ async def hang_on_cancellation(): finally: hang_event.set() await asyncio.sleep(0.01) - late_error_logs = [ - call for call in mock_logger.error.call_args_list - if "Background task airlock-status-updater failed" in call[0][0] + late_shutdown_logs = [ + call for call in mock_logger.warning.call_args_list + if "Background task airlock-status-updater raised exception during shutdown" in call[0][0] ] - assert len(late_error_logs) == 1 + assert len(late_shutdown_logs) == 1 @pytest.mark.asyncio From 9f11bd15a3d534313b20e12540c1033eef474f8e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 20 Oct 2026 11:40:24 +0000 Subject: [PATCH 12/13] Apply Ruff formatting to API lifecycle code and tests --- CHANGELOG.md | 2 +- api_app/main.py | 14 +++----- api_app/tests_ma/test_main_lifecycle.py | 43 +++++++++++++++++++------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecd81ebd3..7306db17cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) +* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. ([#4923](https://github.com/microsoft/AzureTRE/issues/4923)) * Protect active validation environments and require successful smoke tests before custom tests. Validate shared-service command arguments and forward explicit Nexus EULA consent. (`devops` 0.6.20, `build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Give E2E cleanup separate deadlines and allow 30 seconds for smoke health requests. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Verify TLS for identity authentication and keep response content and URL parameters out of E2E operation polling logs. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) @@ -111,7 +112,6 @@ BUG FIXES: * Fix vulnerable GitHub script dependencies with compatible patched versions, add regression tests, and run the helper suite in CI. ([#5064](https://github.com/microsoft/AzureTRE/pull/5064)) * Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055)) * Fix pytest deprecation warnings in API test fixtures. ([#5069](https://github.com/microsoft/AzureTRE/pull/5069)) -* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. ([#4923](https://github.com/microsoft/AzureTRE/issues/4923)) * Fix Service Bus client lifecycle management in the API and resource processor to prevent connection socket and AMQP channel leaks. Close credentials on errors and handle cancellation and retry delays. ([#4930](https://github.com/microsoft/AzureTRE/pull/4930)) COMPONENTS: diff --git a/api_app/main.py b/api_app/main.py index 87a896e73e..c5d7e9b79b 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -73,7 +73,7 @@ def inspect_shutdown_task(task: asyncio.Task[Any]) -> None: shutdown_reported_tasks.add(task) logger.warning( f"Background task {task.get_name()} raised exception during shutdown: {exception}", - exc_info=(type(exception), exception, exception.__traceback__) + exc_info=(type(exception), exception, exception.__traceback__), ) def track(task: asyncio.Task[Any]) -> None: @@ -94,20 +94,14 @@ def _done_callback(task: asyncio.Task[Any]) -> None: if exception is not None: logger.error( f"Background task {task.get_name()} failed", - exc_info=(type(exception), exception, exception.__traceback__) + exc_info=(type(exception), exception, exception.__traceback__), ) background_tasks.add(task) task.add_done_callback(_done_callback) - track(asyncio.create_task( - deploymentStatusUpdater.receive_messages(), - name="deployment-status-updater" - )) - track(asyncio.create_task( - airlockStatusUpdater.receive_messages(), - name="airlock-status-updater" - )) + track(asyncio.create_task(deploymentStatusUpdater.receive_messages(), name="deployment-status-updater")) + track(asyncio.create_task(airlockStatusUpdater.receive_messages(), name="airlock-status-updater")) try: yield diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py index 290b666fea..facfe13af8 100644 --- a/api_app/tests_ma/test_main_lifecycle.py +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -83,7 +83,11 @@ async def fail_immediately(): await asyncio.sleep(0.1) # Verify that the runtime failure was logged as an error - error_logs = [call for call in mock_logger.error.call_args_list if "Background task deployment-status-updater failed" in call[0][0]] + error_logs = [ + call + for call in mock_logger.error.call_args_list + if "Background task deployment-status-updater failed" in call[0][0] + ] assert len(error_logs) == 1 @@ -122,7 +126,9 @@ async def raise_teardown_exception(): assert "Background task deployment-status-updater failed" not in call[0][0] # Instead, it should be caught during shutdown inspection and logged as warning - warning_logs = [call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]] + warning_logs = [ + call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0] + ] assert len(warning_logs) == 1 assert "deployment-status-updater" in warning_logs[0][0][0] @@ -133,7 +139,9 @@ async def raise_teardown_exception(): @patch("main.AirlockStatusUpdater") @patch("main.logger") @patch("main.SHUTDOWN_TIMEOUT", 0.05) -async def test_lifespan_timeout_preserves_completed_task_exceptions(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): +async def test_lifespan_timeout_preserves_completed_task_exceptions( + mock_logger, mock_airlock, mock_deployment, mock_bootstrap +): from main import lifespan mock_deployment_instance = MagicMock() @@ -171,23 +179,34 @@ async def hang_on_cancellation(): await asyncio.sleep(0.01) # 1. Completed task exception is still inspected and logged as warning despite timeout - warning_logs = [call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]] + warning_logs = [ + call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0] + ] assert len(warning_logs) == 1 assert "deployment-status-updater" in warning_logs[0][0][0] # 2. Timeout error is logged - error_logs = [call for call in mock_logger.error.call_args_list if "Timeout waiting for background tasks to shutdown" in call[0][0]] + error_logs = [ + call + for call in mock_logger.error.call_args_list + if "Timeout waiting for background tasks to shutdown" in call[0][0] + ] assert len(error_logs) == 1 # 3. Pending task is logged with warning - pending_logs = [call for call in mock_logger.warning.call_args_list if "did not terminate in time during shutdown" in call[0][0]] + pending_logs = [ + call + for call in mock_logger.warning.call_args_list + if "did not terminate in time during shutdown" in call[0][0] + ] assert len(pending_logs) == 1 assert "airlock-status-updater" in pending_logs[0][0][0] finally: hang_event.set() await asyncio.sleep(0.01) late_shutdown_logs = [ - call for call in mock_logger.warning.call_args_list + call + for call in mock_logger.warning.call_args_list if "Background task airlock-status-updater raised exception during shutdown" in call[0][0] ] assert len(late_shutdown_logs) == 1 @@ -199,7 +218,9 @@ async def hang_on_cancellation(): @patch("main.AirlockStatusUpdater") @patch("main.logger") @patch("main.SHUTDOWN_TIMEOUT", 0.05) -async def test_lifespan_cancellation_suppression_strictly_bounded(mock_logger, mock_airlock, mock_deployment, mock_bootstrap): +async def test_lifespan_cancellation_suppression_strictly_bounded( + mock_logger, mock_airlock, mock_deployment, mock_bootstrap +): import time from main import lifespan @@ -235,7 +256,11 @@ async def suppress_cancellation(): assert elapsed < 0.5 # Both tasks should be reported as pending - pending_logs = [call for call in mock_logger.warning.call_args_list if "did not terminate in time during shutdown" in call[0][0]] + pending_logs = [ + call + for call in mock_logger.warning.call_args_list + if "did not terminate in time during shutdown" in call[0][0] + ] assert len(pending_logs) == 2 finally: hang_event.set() From f02bc6f1975a62be77173d5667fc4de2f36e5d45 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 22 Oct 2026 09:02:56 +0000 Subject: [PATCH 13/13] bump version --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 90c08ed819..92f2eb1585 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.7" +__version__ = "0.28.8"