Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
74c89c8
implement app-scoped lifecycle management to prevent silent garbage c…
Jun 5, 2026
7be55fe
changelog
Jun 5, 2026
4f4d616
Merge remote-tracking branch 'origin/main' into asyncio-prevent-garba…
Jun 5, 2026
eb3bf72
bump api
Jun 5, 2026
c2a4be7
implement copilot suggestions
Jun 5, 2026
a51b3eb
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jun 9, 2026
4cd1a7b
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jun 24, 2026
f3f0b01
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jun 30, 2026
422d029
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jul 3, 2026
2d043f3
Bump version to 0.25.27 and implement BackgroundTaskManager for impro…
Jul 6, 2026
d819444
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jul 6, 2026
8a403f8
suppress bandit warning
Jul 7, 2026
95075a9
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jul 21, 2026
8a5e234
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Jul 31, 2026
619344f
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Sep 4, 2026
1c7e021
Refactor shutdown handling in lifespan to improve task exception logg…
Sep 4, 2026
baaf3b8
Bump version to 0.26.9
JC-wk Sep 4, 2026
0e46e29
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Sep 4, 2026
582bcb1
Prevent background tasks from being garbage collected during shutdown
Sep 7, 2026
2fca0e9
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Sep 9, 2026
2e2d61f
Bump version to 0.26.10
JC-wk Sep 9, 2026
37fefb5
Refactor shutdown task exception handling and update tests for logging
JC-wk Sep 9, 2026
f16f6e4
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Sep 11, 2026
efa00ce
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Oct 8, 2026
72d62e5
Merge branch 'main' into asyncio-prevent-garbage-collection
ChrisChapman-gh Oct 16, 2026
ecfbd8e
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Oct 20, 2026
9f11bd1
Apply Ruff formatting to API lifecycle code and tests
Oct 20, 2026
ca1283f
Merge branch 'main' into asyncio-prevent-garbage-collection
ChrisChapman-gh Oct 20, 2026
f02bc6f
bump version
Oct 22, 2026
0a8cdca
Merge branch 'main' into asyncio-prevent-garbage-collection
JC-wk Oct 22, 2026
11485a7
Merge branch 'main' into asyncio-prevent-garbage-collection
ChrisChapman-gh Oct 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ BUG FIXES:
* Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274))
* Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270))
* Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266))
* Prevent silent garbage collection of background API task workers by implementing app-scoped lifecycle management. ([#4923](https://github.com/microsoft/AzureTRE/issues/4923))
* Protect active validation environments and require successful smoke tests before custom tests. Validate shared-service command arguments and forward explicit Nexus EULA consent. (`devops` 0.6.20, `build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255))
* Give E2E cleanup separate deadlines and allow 30 seconds for smoke health requests. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255))
* Verify TLS for identity authentication and keep response content and URL parameters out of E2E operation polling logs. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255))
Expand Down
2 changes: 1 addition & 1 deletion api_app/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.28.7"
__version__ = "0.28.8"
93 changes: 90 additions & 3 deletions api_app/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
from typing import Any
import uvicorn

from fastapi import FastAPI
Expand All @@ -22,8 +23,30 @@
from service_bus.airlock_request_status_update import AirlockStatusUpdater


class BackgroundTaskManager:
def __init__(self) -> None:
self._tasks: set[asyncio.Task[Any]] = set()
self.is_shutting_down: bool = False

def add(self, task: asyncio.Task[Any]) -> None:
self._tasks.add(task)

def discard(self, task: asyncio.Task[Any]) -> None:
self._tasks.discard(task)

def get_tasks(self) -> list[asyncio.Task[Any]]:
return list(self._tasks)


SHUTDOWN_TIMEOUT = 10.0


@asynccontextmanager
async def lifespan(app: FastAPI):

background_tasks = BackgroundTaskManager()
app.state.background_tasks = background_tasks

while not await bootstrap_database():
await asyncio.sleep(5)
logger.warning("Database connection could not be established")
Expand All @@ -34,9 +57,73 @@ async def lifespan(app: FastAPI):
airlockStatusUpdater = AirlockStatusUpdater()
await airlockStatusUpdater.init_repos()

asyncio.create_task(deploymentStatusUpdater.receive_messages())
asyncio.create_task(airlockStatusUpdater.receive_messages())
yield
shutdown_inspected_tasks: set[asyncio.Task[Any]] = set()
shutdown_reported_tasks: set[asyncio.Task[Any]] = set()

def inspect_shutdown_task(task: asyncio.Task[Any]) -> None:
if task.cancelled() or task in shutdown_reported_tasks:
return

try:
exception = task.exception()
except asyncio.CancelledError:
return

if exception is not None:
shutdown_reported_tasks.add(task)
logger.warning(
f"Background task {task.get_name()} raised exception during shutdown: {exception}",
exc_info=(type(exception), exception, exception.__traceback__),
)

def track(task: asyncio.Task[Any]) -> None:
def _done_callback(task: asyncio.Task[Any]) -> None:
background_tasks.discard(task)
if task in shutdown_inspected_tasks:
inspect_shutdown_task(task)
return

if task.cancelled():
return

try:
exception = task.exception()
except asyncio.CancelledError:
return

if exception is not None:
logger.error(
f"Background task {task.get_name()} failed",
exc_info=(type(exception), exception, exception.__traceback__),
)

background_tasks.add(task)
task.add_done_callback(_done_callback)

track(asyncio.create_task(deploymentStatusUpdater.receive_messages(), name="deployment-status-updater"))
track(asyncio.create_task(airlockStatusUpdater.receive_messages(), name="airlock-status-updater"))

try:
yield
finally:
background_tasks.is_shutting_down = True
tasks = background_tasks.get_tasks()
shutdown_inspected_tasks.update(tasks)
logger.info(f"Cancelling {len(tasks)} background tasks")

for task in tasks:
logger.debug(f"Cancelling task {task.get_name()}")
task.cancel()

if tasks:
done, pending = await asyncio.wait(tasks, timeout=SHUTDOWN_TIMEOUT)
for task in done:
inspect_shutdown_task(task)

if pending:
logger.error("Timeout waiting for background tasks to shutdown")
for task in pending:
logger.warning(f"Task {task.get_name()} did not terminate in time during shutdown")
Comment thread
JC-wk marked this conversation as resolved.
Resolved


def get_application() -> FastAPI:
Expand Down
267 changes: 267 additions & 0 deletions api_app/tests_ma/test_main_lifecycle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,267 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import FastAPI


@pytest.mark.asyncio
async def test_background_task_manager():
from main import BackgroundTaskManager

manager = BackgroundTaskManager()
assert manager.is_shutting_down is False

async def dummy():
pass

task = asyncio.create_task(dummy())
manager.add(task)
assert task in manager.get_tasks()

manager.discard(task)
assert task not in manager.get_tasks()

await task


@pytest.mark.asyncio
@patch("main.bootstrap_database", return_value=True)
@patch("main.DeploymentStatusUpdater")
@patch("main.AirlockStatusUpdater")
async def test_lifespan_lifecycle_normal(mock_airlock, mock_deployment, mock_bootstrap):
from main import BackgroundTaskManager, lifespan

mock_deployment_instance = MagicMock()
mock_deployment_instance.init_repos = AsyncMock()
mock_deployment_instance.receive_messages = AsyncMock()
mock_deployment.return_value = mock_deployment_instance

mock_airlock_instance = MagicMock()
mock_airlock_instance.init_repos = AsyncMock()
mock_airlock_instance.receive_messages = AsyncMock()
mock_airlock.return_value = mock_airlock_instance

app = FastAPI()

async with lifespan(app):
assert hasattr(app.state, "background_tasks")
assert isinstance(app.state.background_tasks, BackgroundTaskManager)
tasks = app.state.background_tasks.get_tasks()
assert len(tasks) == 2
names = {t.get_name() for t in tasks}
assert "deployment-status-updater" in names
assert "airlock-status-updater" in names

assert app.state.background_tasks.is_shutting_down is True


@pytest.mark.asyncio
@patch("main.bootstrap_database", return_value=True)
@patch("main.DeploymentStatusUpdater")
@patch("main.AirlockStatusUpdater")
@patch("main.logger")
async def test_lifespan_task_failure_during_runtime(mock_logger, mock_airlock, mock_deployment, mock_bootstrap):
from main import lifespan

mock_deployment_instance = MagicMock()
mock_deployment_instance.init_repos = AsyncMock()

async def fail_immediately():
raise ValueError("Runtime failure")

mock_deployment_instance.receive_messages = fail_immediately
mock_deployment.return_value = mock_deployment_instance

mock_airlock_instance = MagicMock()
mock_airlock_instance.init_repos = AsyncMock()
mock_airlock_instance.receive_messages = AsyncMock()
mock_airlock.return_value = mock_airlock_instance

app = FastAPI()

async with lifespan(app):
await asyncio.sleep(0.1)

# Verify that the runtime failure was logged as an error
error_logs = [
call
for call in mock_logger.error.call_args_list
if "Background task deployment-status-updater failed" in call[0][0]
]
assert len(error_logs) == 1


@pytest.mark.asyncio
@patch("main.bootstrap_database", return_value=True)
@patch("main.DeploymentStatusUpdater")
@patch("main.AirlockStatusUpdater")
@patch("main.logger")
async def test_lifespan_task_failure_during_shutdown(mock_logger, mock_airlock, mock_deployment, mock_bootstrap):
from main import lifespan

mock_deployment_instance = MagicMock()
mock_deployment_instance.init_repos = AsyncMock()

async def raise_teardown_exception():
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
raise RuntimeError("Database connection closed abruptly")

mock_deployment_instance.receive_messages = raise_teardown_exception
mock_deployment.return_value = mock_deployment_instance

mock_airlock_instance = MagicMock()
mock_airlock_instance.init_repos = AsyncMock()
mock_airlock_instance.receive_messages = AsyncMock()
mock_airlock.return_value = mock_airlock_instance

app = FastAPI()

async with lifespan(app):
await asyncio.sleep(0.1)

# The error should NOT be logged as error in _done_callback because it was gated on is_shutting_down
for call in mock_logger.error.call_args_list:
assert "Background task deployment-status-updater failed" not in call[0][0]

# Instead, it should be caught during shutdown inspection and logged as warning
warning_logs = [
call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]
]
assert len(warning_logs) == 1
assert "deployment-status-updater" in warning_logs[0][0][0]


@pytest.mark.asyncio
@patch("main.bootstrap_database", return_value=True)
@patch("main.DeploymentStatusUpdater")
@patch("main.AirlockStatusUpdater")
@patch("main.logger")
@patch("main.SHUTDOWN_TIMEOUT", 0.05)
async def test_lifespan_timeout_preserves_completed_task_exceptions(
mock_logger, mock_airlock, mock_deployment, mock_bootstrap
):
from main import lifespan

mock_deployment_instance = MagicMock()
mock_deployment_instance.init_repos = AsyncMock()

# Deployment updater completes quickly during cancellation by raising an exception
async def raise_teardown_exception():
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
raise RuntimeError("Deployment teardown failed")

mock_deployment_instance.receive_messages = raise_teardown_exception
mock_deployment.return_value = mock_deployment_instance

hang_event = asyncio.Event()

# Airlock updater suppresses cancellation and hangs past the timeout
async def hang_on_cancellation():
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
await hang_event.wait()
raise RuntimeError("Airlock teardown failed late")

mock_airlock_instance = MagicMock()
mock_airlock_instance.init_repos = AsyncMock()
mock_airlock_instance.receive_messages = hang_on_cancellation
mock_airlock.return_value = mock_airlock_instance

app = FastAPI()

try:
async with lifespan(app):
await asyncio.sleep(0.01)

# 1. Completed task exception is still inspected and logged as warning despite timeout
warning_logs = [
call for call in mock_logger.warning.call_args_list if "raised exception during shutdown" in call[0][0]
]
assert len(warning_logs) == 1
assert "deployment-status-updater" in warning_logs[0][0][0]

# 2. Timeout error is logged
error_logs = [
call
for call in mock_logger.error.call_args_list
if "Timeout waiting for background tasks to shutdown" in call[0][0]
]
assert len(error_logs) == 1

# 3. Pending task is logged with warning
pending_logs = [
call
for call in mock_logger.warning.call_args_list
if "did not terminate in time during shutdown" in call[0][0]
]
assert len(pending_logs) == 1
assert "airlock-status-updater" in pending_logs[0][0][0]
finally:
hang_event.set()
await asyncio.sleep(0.01)
late_shutdown_logs = [
call
for call in mock_logger.warning.call_args_list
if "Background task airlock-status-updater raised exception during shutdown" in call[0][0]
]
assert len(late_shutdown_logs) == 1


@pytest.mark.asyncio
@patch("main.bootstrap_database", return_value=True)
@patch("main.DeploymentStatusUpdater")
@patch("main.AirlockStatusUpdater")
@patch("main.logger")
@patch("main.SHUTDOWN_TIMEOUT", 0.05)
async def test_lifespan_cancellation_suppression_strictly_bounded(
mock_logger, mock_airlock, mock_deployment, mock_bootstrap
):
import time
from main import lifespan

mock_deployment_instance = MagicMock()
mock_deployment_instance.init_repos = AsyncMock()

hang_event = asyncio.Event()

# Worker suppresses cancellation and ignores cancel signal
async def suppress_cancellation():
try:
await asyncio.sleep(5)
except asyncio.CancelledError:
await hang_event.wait()

mock_deployment_instance.receive_messages = suppress_cancellation
mock_deployment.return_value = mock_deployment_instance

mock_airlock_instance = MagicMock()
mock_airlock_instance.init_repos = AsyncMock()
mock_airlock_instance.receive_messages = suppress_cancellation
mock_airlock.return_value = mock_airlock_instance

app = FastAPI()

try:
start_time = time.monotonic()
async with lifespan(app):
await asyncio.sleep(0.01)
elapsed = time.monotonic() - start_time

# Teardown should return promptly near SHUTDOWN_TIMEOUT and NOT hang
assert elapsed < 0.5

# Both tasks should be reported as pending
pending_logs = [
call
for call in mock_logger.warning.call_args_list
if "did not terminate in time during shutdown" in call[0][0]
]
assert len(pending_logs) == 2
finally:
hang_event.set()
await asyncio.sleep(0.01)
Loading