From c83471e12db234605e8fbfc036913f959d72371d Mon Sep 17 00:00:00 2001 From: JC-wk <196318169+JC-wk@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:04:07 +0000 Subject: [PATCH 01/90] Implement address space removal --- CHANGELOG.md | 1 + api_app/_version.py | 2 +- api_app/api/routes/workspaces.py | 21 +++++++++- .../authoring-workspace-templates.md | 2 +- .../azureml/template_schema.json | 7 ++++ .../databricks/template_schema.json | 39 +++++++++++++++---- 6 files changed, 61 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a09d0c7605..5baac30a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ BUG FIXES: * Fix issue where multiple lists in config.yaml incorrectly caused a validation error ([#4711](https://github.com/microsoft/AzureTRE/pull/4711)) * Updated config_schema.json to include missing values. ([#4712](https://github.com/microsoft/AzureTRE/issues/4712))([#4714](https://github.com/microsoft/AzureTRE/issues/4714)) * Remove workspace upgrade step from databricks template ([#4726](https://github.com/microsoft/AzureTRE/pull/4726)) +* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) * Update Starlette and FastAPI versions ([#4738](https://github.com/microsoft/AzureTRE/pull/4738)) ## 0.25.0 (July 18, 2025) diff --git a/api_app/_version.py b/api_app/_version.py index 71290cd566..97b0e6226c 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.1" +__version__ = "0.25.2" diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index a2dcbe52fa..0f99b0b2b2 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -310,7 +310,26 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(get_current_workspace_owner_user)]) -async def delete_workspace_service(response: Response, user=Depends(get_current_workspace_owner_user), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: +async def delete_workspace_service(response: Response, user=Depends(get_current_workspace_owner_user), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository))) -> OperationInResponse: + # If the workspace service owns an allocated address_space, remove it from the parent workspace before uninstall + try: + address_to_free = workspace_service.properties.get("address_space") + if address_to_free: + # ensure the workspace has the address_spaces property + workspace_address_spaces = workspace.properties.get("address_spaces", []) + if address_to_free in workspace_address_spaces: + new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": new_address_spaces} + try: + await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) + except CosmosAccessConditionFailedError: + # let the client try again or fail the uninstall flow due to etag conflict + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) + except Exception: + # don't block uninstall if patching the workspace fails for unexpected reasons + logger.exception("Failed to free workspace address space before uninstall") + if await delete_validation(workspace_service, workspace_service_repo): operation = await send_uninstall_message( resource=workspace_service, diff --git a/docs/tre-workspace-authors/authoring-workspace-templates.md b/docs/tre-workspace-authors/authoring-workspace-templates.md index 23614bb29f..801b3c4423 100644 --- a/docs/tre-workspace-authors/authoring-workspace-templates.md +++ b/docs/tre-workspace-authors/authoring-workspace-templates.md @@ -109,7 +109,7 @@ The size of the `address_space` will default to `/24`, however other sizes can b The `address_space` allocation will only take place during the install phase of a deployment, as this is a breaking change to your template you should increment the major version of your template, this means a you must deploy a new resource instead of upgrading an existing one. -In your install pipeline you also need to include a workspace upgrade step for the workspace to update it's `address_spaces` property. +In your install and uninstall pipelines you also need to include a workspace upgrade step for the workspace to update it's `address_spaces` property. ```json "pipeline": { diff --git a/templates/workspace_services/azureml/template_schema.json b/templates/workspace_services/azureml/template_schema.json index 581b8993df..8c8b910e63 100644 --- a/templates/workspace_services/azureml/template_schema.json +++ b/templates/workspace_services/azureml/template_schema.json @@ -391,6 +391,13 @@ }, { "stepId": "main" + }, + { + "stepId": "f720975a-c81e-477e-854e-53fde86e5e57", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } diff --git a/templates/workspace_services/databricks/template_schema.json b/templates/workspace_services/databricks/template_schema.json index 1e43ba0395..531d9ddb37 100644 --- a/templates/workspace_services/databricks/template_schema.json +++ b/templates/workspace_services/databricks/template_schema.json @@ -78,7 +78,9 @@ "name": "databricks", "description": "Communication with Azure Databricks dependancies.", "source_addresses": "{{ resource.properties.databricks_address_prefixes }}", - "destination_addresses": [ "AzureDatabricks"], + "destination_addresses": [ + "AzureDatabricks" + ], "destination_ports": [ "443" ], @@ -114,9 +116,15 @@ "name": "AzureAD", "description": "AAD access", "source_addresses": "{{ resource.properties.workspace_address_spaces }}", - "destination_addresses": ["AzureActiveDirectory"], - "destination_ports": ["*"], - "protocols": ["TCP"] + "destination_addresses": [ + "AzureActiveDirectory" + ], + "destination_ports": [ + "*" + ], + "protocols": [ + "TCP" + ] } ] } @@ -212,7 +220,9 @@ "name": "databricks", "description": "Communication with Azure Databricks dependancies.", "source_addresses": "{{ resource.properties.databricks_address_prefixes }}", - "destination_addresses": [ "AzureDatabricks"], + "destination_addresses": [ + "AzureDatabricks" + ], "destination_ports": [ "443" ], @@ -248,9 +258,15 @@ "name": "AzureAD", "description": "AAD access", "source_addresses": "{{ resource.properties.workspace_address_spaces }}", - "destination_addresses": ["AzureActiveDirectory"], - "destination_ports": ["*"], - "protocols": ["TCP"] + "destination_addresses": [ + "AzureActiveDirectory" + ], + "destination_ports": [ + "*" + ], + "protocols": [ + "TCP" + ] } ] } @@ -352,6 +368,13 @@ }, { "stepId": "main" + }, + { + "stepId": "9c4dc64b-8fbf-4e77-a7f6-48fb33423504", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } From 029f5068dba30b64d87bb1f893531b59e01f2ce1 Mon Sep 17 00:00:00 2001 From: JC-wk <196318169+JC-wk@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:10:53 +0000 Subject: [PATCH 02/90] update template versions --- templates/workspace_services/azureml/porter.yaml | 2 +- templates/workspace_services/databricks/porter.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index 0cf012d630..ff9c631345 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-azureml -version: 0.10.0 +version: 0.10.1 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/databricks/porter.yaml b/templates/workspace_services/databricks/porter.yaml index 8fdffa2db8..e3222da2f5 100644 --- a/templates/workspace_services/databricks/porter.yaml +++ b/templates/workspace_services/databricks/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-databricks -version: 1.0.14 +version: 1.0.15 description: "An Azure TRE service for Azure Databricks." registry: azuretre dockerfile: Dockerfile.tmpl From c9f9307590e67b9870f5284195953cc48292f947 Mon Sep 17 00:00:00 2001 From: JC-wk <196318169+JC-wk@users.noreply.github.com> Date: Wed, 12 Nov 2025 17:30:13 +0000 Subject: [PATCH 03/90] Move address_space deletion to service_bus --- api_app/api/routes/workspaces.py | 21 +----------- .../service_bus/deployment_status_updater.py | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 0f99b0b2b2..a2dcbe52fa 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -310,26 +310,7 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(get_current_workspace_owner_user)]) -async def delete_workspace_service(response: Response, user=Depends(get_current_workspace_owner_user), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository))) -> OperationInResponse: - # If the workspace service owns an allocated address_space, remove it from the parent workspace before uninstall - try: - address_to_free = workspace_service.properties.get("address_space") - if address_to_free: - # ensure the workspace has the address_spaces property - workspace_address_spaces = workspace.properties.get("address_spaces", []) - if address_to_free in workspace_address_spaces: - new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] - workspace_patch = ResourcePatch() - workspace_patch.properties = {"address_spaces": new_address_spaces} - try: - await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) - except CosmosAccessConditionFailedError: - # let the client try again or fail the uninstall flow due to etag conflict - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) - except Exception: - # don't block uninstall if patching the workspace fails for unexpected reasons - logger.exception("Failed to free workspace address space before uninstall") - +async def delete_workspace_service(response: Response, user=Depends(get_current_workspace_owner_user), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: if await delete_validation(workspace_service, workspace_service_repo): operation = await send_uninstall_message( resource=workspace_service, diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 41670464c7..43c7852765 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -6,7 +6,7 @@ from pydantic import ValidationError, parse_obj_as from api.routes.resource_helpers import get_timestamp -from models.domain.resource import Output +from models.domain.resource import Output, ResourceType from db.repositories.resources_history import ResourceHistoryRepository from models.domain.request_action import RequestAction from db.repositories.resource_templates import ResourceTemplateRepository @@ -21,6 +21,9 @@ from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings from services.logging import logger, tracer +from db.repositories.workspaces import WorkspaceRepository +from models.schemas.resource import ResourcePatch +from azure.cosmos.exceptions import CosmosAccessConditionFailedError class DeploymentStatusUpdater(): @@ -187,6 +190,34 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) + # If the 'main' step succeeded for an uninstall operation, free any allocated address space + # owned by a WorkspaceService resource. We trigger cleanup when the step with templateStepId == 'main' + # is successful; this ensures the primary resource has been destroyed successfully before attempting to free the ip address space + try: + # if the step that just succeeded is the main step for this operation, and this is an uninstall, + # proceed with post-uninstall cleanup. No need to scan the operation.steps list again. + if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: + if resource_to_persist.get("resourceType") == ResourceType.WorkspaceService: + address_to_free = resource_to_persist.get("properties", {}).get("address_space") + parent_workspace_id = resource_to_persist.get("workspaceId") + if address_to_free and parent_workspace_id: + try: + workspace_repo = await WorkspaceRepository.create() + workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) + workspace_address_spaces = workspace.properties.get("address_spaces", []) + if address_to_free in workspace_address_spaces: + new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": new_address_spaces} + try: + await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, self.resource_template_repo, self.resource_history_repo, operation.user, False) + logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") + except CosmosAccessConditionFailedError: + logger.exception("ETag conflict when freeing workspace address space after successful uninstall") + except Exception: + logger.exception("Failed to free workspace address space after successful uninstall") + except Exception: + logger.exception("Unexpected error during post-uninstall address space cleanup") result = True From 6ebeb08c160977bbf80c453d4d6ef7a3c458e422 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 9 Jan 2026 10:22:32 +0000 Subject: [PATCH 04/90] bump api version --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 5461be1190..4f3cae6929 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.7" +__version__ = "0.25.8" From b32d4ef0038e863b2b3b61e4ab0cd52f014c66a0 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 6 Feb 2026 17:06:51 +0000 Subject: [PATCH 05/90] changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ded92b9dc7..4e4b2d3911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * _No changes yet_ ENHANCEMENTS: -* _No changes yet_ +* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) BUG FIXES: * Fix property substitution not occuring where there is only a main step in the pipeline ([#4824](https://github.com/microsoft/AzureTRE/issues/4824)) @@ -117,7 +117,6 @@ BUG FIXES: * Add Loading spinner to shared services page ([#4718](https://github.com/microsoft/AzureTRE/pull/4718)) * Updated config_schema.json to include missing values. ([#4712](https://github.com/microsoft/AzureTRE/issues/4712))([#4714](https://github.com/microsoft/AzureTRE/issues/4714)) * Remove workspace upgrade step from databricks template ([#4726](https://github.com/microsoft/AzureTRE/pull/4726)) -* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) * Update Starlette and FastAPI versions ([#4738](https://github.com/microsoft/AzureTRE/pull/4738)) COMPONENTS: From fc9ba33876fe62f262708cfe0e4af5402eba08fa Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 9 Jun 2026 10:02:20 +0000 Subject: [PATCH 06/90] Copilot feedback --- CHANGELOG.md | 4 +- .../service_bus/deployment_status_updater.py | 14 +- .../test_deployment_status_update.py | 284 ++++++++++++++++++ 3 files changed, 297 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a656612c20..115cfdf3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,13 @@ * Remove Windows 10 and dsvm image support from Guacamole. ([#4890](https://github.com/microsoft/AzureTRE/issues/4890)) ENHANCEMENTS: -* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) * Specify default_outbound_access_enabled = false setting for all subnets ([#4757](https://github.com/microsoft/AzureTRE/pull/4757)) * Pin all GitHub Actions workflow steps to full commit SHAs to prevent supply chain attacks plus update to latest releases ([#4886](https://github.com/microsoft/AzureTRE/pull/4886)) * Add Windows Server 2025 image support to Guacamole. ([#4890](https://github.com/microsoft/AzureTRE/issues/4890)) +BUG FIXES: +* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) + ## (0.28.0) (March 2, 2026) **BREAKING CHANGES** * Sonatype Nexus shared service now requires explicit EULA acceptance (`accept_nexus_eula: true`) when deploying. This ensures compliance with Sonatype Nexus Community Edition licensing. ([#4842](https://github.com/microsoft/AzureTRE/issues/4842)) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 43c7852765..3d6a190232 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -203,17 +203,23 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if address_to_free and parent_workspace_id: try: workspace_repo = await WorkspaceRepository.create() - workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) - workspace_address_spaces = workspace.properties.get("address_spaces", []) - if address_to_free in workspace_address_spaces: + max_retries = 3 + for attempt in range(max_retries): + workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) + workspace_address_spaces = workspace.properties.get("address_spaces", []) + if address_to_free not in workspace_address_spaces: + break new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] workspace_patch = ResourcePatch() workspace_patch.properties = {"address_spaces": new_address_spaces} try: await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, self.resource_template_repo, self.resource_history_repo, operation.user, False) logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") + break except CosmosAccessConditionFailedError: - logger.exception("ETag conflict when freeing workspace address space after successful uninstall") + if attempt == max_retries - 1: + raise + logger.warning(f"ETag conflict when freeing workspace address space after successful uninstall. Retrying (attempt {attempt + 1}/{max_retries})...") except Exception: logger.exception("Failed to free workspace address space after successful uninstall") except Exception: diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index db80c5b1f7..6c9b328cea 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -413,3 +413,287 @@ async def test_convert_outputs_to_dict(): 'list2': ['one', 'two'] } assert status_updater.convert_outputs_to_dict(deployment_status_update_message.outputs) == expected_result + + +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_frees_address_space( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + # Mock the operation showing RequestAction.UnInstall + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + # Mock WorkspaceService resource returned by get_resource_by_id + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + # Mock resource dict representation returned by get_resource_dict_by_id + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + # Mock parent workspace containing the address space to free + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + workspace_repo.patch_workspace.assert_called_once() + called_args = workspace_repo.patch_workspace.call_args[0] + assert called_args[0] == parent_workspace + assert called_args[1].properties == {"address_spaces": ["10.0.0.0/22"]} + assert called_args[2] == "parent-workspace-etag" + + +@pytest.mark.parametrize("missing_property", ["address_space", "workspaceId"]) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_does_not_free_address_space_if_missing( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + missing_property +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + if missing_property == "address_space": + del workspace_service_dict["properties"]["address_space"] + elif missing_property == "workspaceId": + del workspace_service_dict["workspaceId"] + + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + workspace_repo = AsyncMock() + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + workspace_repo.patch_workspace.assert_not_called() + + +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_frees_address_space_with_retry_on_etag_conflict( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock +): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + # Mock the operation showing RequestAction.UnInstall + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + # Mock WorkspaceService resource returned by get_resource_by_id + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + # Mock resource dict representation returned by get_resource_dict_by_id + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + # Mock parent workspace containing the address space to free + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + + # First attempt raises CosmosAccessConditionFailedError, second succeeds + workspace_repo.patch_workspace.side_effect = [CosmosAccessConditionFailedError, None] + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + # Assert get_workspace_by_id called twice due to retry + assert workspace_repo.get_workspace_by_id.call_count == 2 + # Assert patch_workspace called twice + assert workspace_repo.patch_workspace.call_count == 2 + + +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +@patch('services.logging.logger.exception') +async def test_workspace_service_uninstall_logs_error_after_max_retries( + logging_mock, + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock +): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + # Mock the operation showing RequestAction.UnInstall + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + # Mock WorkspaceService resource returned by get_resource_by_id + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + # Mock resource dict representation returned by get_resource_dict_by_id + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + # Mock parent workspace containing the address space to free + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + + # All attempts raise CosmosAccessConditionFailedError + workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + # Note: complete_message is still True because we caught the exception in the outer block and logged it. + assert complete_message is True + # Assert get_workspace_by_id and patch_workspace called max_retries = 3 times + assert workspace_repo.get_workspace_by_id.call_count == 3 + assert workspace_repo.patch_workspace.call_count == 3 + # Assert we logged the final failure + logging_mock.assert_called_once_with("Failed to free workspace address space after successful uninstall") + + From 69f0b3e469018c9e750b1deb00743967be1766ed Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 9 Jun 2026 11:22:04 +0000 Subject: [PATCH 07/90] remove spaces --- .../tests_ma/test_service_bus/test_deployment_status_update.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 6c9b328cea..9a65e12628 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -695,5 +695,3 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( assert workspace_repo.patch_workspace.call_count == 3 # Assert we logged the final failure logging_mock.assert_called_once_with("Failed to free workspace address space after successful uninstall") - - From e962d53f61d1ac6f0c1cc7654b31666d2c2e8cb3 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:23:42 +0100 Subject: [PATCH 08/90] Bump version to 0.25.27 --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index eda2a21a48..7df7ad28da 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.25.26" +__version__ = "0.25.27" From 702f07896789ac5c2b53a273d1bc8777066593c7 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 16:20:31 +0000 Subject: [PATCH 09/90] ### Summary of Changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Ordering & Interaction between direct patch and enqueued upgrade step (deployment_status_updater.py): • Moved _free_workspace_address_space to execute before if is_last_step is False: (which enqueues the next step). • patch_workspace directly updates the Workspace resource document in Cosmos DB to remove address_to_free from workspace. properties["address_spaces"]. • By running _free_workspace_address_space first, when update_resource_for_step runs for the trailing workspace upgrade step, it queries the workspace document from Cosmos DB with the address space ALREADY removed. • The trailing workspace upgrade step then executes Terraform against Azure infrastructure using the updated Cosmos DB state. 2. Cleaned up try/except error logging (deployment_status_updater.py): • Extracted cleanup into _free_workspace_address_space and collapsed double try/except loops. • Emits structured, alertable logs logger.error("[ADDRESS_SPACE_CLEANUP_FAILED] ...", exc_info=True) when cleanup exhausts retries or fails. 3. Confirmed patch_workspace(..., False) behavior: • patch_workspace with force_version_update=False only updates the database document and records history; it does not trigger a new deployment operation or send messages to Service Bus. Added clarifying comments in the code. 4. Extracted private method and constant (deployment_status_updater.py): • Created MAX_CLEANUP_RETRIES = 3 at module scope. • Extracted async def _free_workspace_address_space(self, resource_to_persist: dict, operation: Operation). 5. Updated & added unit tests (test_deployment_status_update.py): • Refactored test_workspace_service_uninstall_logs_error_after_max_retries to patch @patch('service_bus.deployment_status_updater. logger') and assert on the module logger. • Added test_workspace_service_uninstall_frees_address_space_before_enqueuing_next_step to verify patch_workspace runs before send_deployment_message. 6. Full coverage of bundled workspace services (template_schema.json, porter.yaml): • Audited all workspace services (azureml, databricks, gitea, guacamole, health-services, mysql, ohdsi, openai, azuresql). • Identified that ohdsi also defines an address_space parameter (for PostgreSQL subnet). • Added trailing workspace upgrade step to ohdsi's uninstall pipeline and bumped ohdsi porter bundle version to 0.3.8. ### Verification • Ran full test suite (pytest api_app/tests_ma): 730 passed in 17.86s. --- .../service_bus/deployment_status_updater.py | 161 +++++++++--------- .../test_deployment_status_update.py | 117 ++++++++++++- .../workspace_services/ohdsi/porter.yaml | 2 +- .../ohdsi/template_schema.json | 7 + 4 files changed, 207 insertions(+), 80 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 3d6a190232..1ead6bf1fc 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -26,6 +26,9 @@ from azure.cosmos.exceptions import CosmosAccessConditionFailedError +MAX_CLEANUP_RETRIES = 3 + + class DeploymentStatusUpdater(): def __init__(self): pass @@ -43,60 +46,44 @@ async def receive_messages(self): with tracer.start_as_current_span("deployment_status_receive_messages"): last_heartbeat_time = 0 polling_count = 0 - while True: - try: - current_time = time.time() - polling_count += 1 - # Log a heartbeat message every 60 seconds to show the service is still working - if current_time - last_heartbeat_time >= 60: - logger.info(f"Queue reader heartbeat: Polled {config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE} queue {polling_count} times in the last minute") - last_heartbeat_time = current_time - polling_count = 0 - - async with credentials.get_credential_async_context() as credential: - service_bus_client = ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) - - logger.debug(f"Looking for new messages on {config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE} queue...") - # max_wait_time=1 -> don't hold the session open after processing of the message has finished - async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=1, session_id=NEXT_AVAILABLE_SESSION) as receiver: - logger.info(f"Got a session containing messages: {receiver.session.session_id}") - async with AutoLockRenewer() as renewer: + complete_message = True + async with credentials.get_credential_async() as credential: + async with ServiceBusClient(fully_qualified_namespace=config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential=credential) as service_bus_client: + try: + logger.debug("Creating Deployment Status receiver session") + async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=config.SERVICE_BUS_MAX_WAIT_TIME, session_id=NEXT_AVAILABLE_SESSION) as receiver: + renewer = AutoLockRenewer() renewer.register(receiver, receiver.session, max_lock_renewal_duration=60) + async for msg in receiver: complete_message = await self.process_message(msg) if complete_message: await receiver.complete_message(msg) else: - # could have been any kind of transient issue, we'll abandon back to the queue, and retry await receiver.abandon_message(msg) - logger.info(f"Closing session: {receiver.session.session_id}") - - except OperationTimeoutError: - # Timeout occurred whilst connecting to a session - this is expected and indicates no non-empty sessions are available - logger.debug("No sessions for this process. Will look again...") - - except ServiceBusConnectionError: - # Occasionally there will be a transient / network-level error in connecting to SB. - logger.info("Unknown Service Bus connection error. Will retry...") - except Exception as e: - # Catch all other exceptions, log them via .exception to get the stack trace, and reconnect - logger.exception(f"Unknown exception. Will retry - {e}") - - async def process_message(self, msg): + await renewer.close() + polling_count = 0 + + except OperationTimeoutError: + polling_count += 1 + # log a heartbeat every ~5 minutes when queue is idle (max_wait_time is usually ~10s) + if time.time() - last_heartbeat_time > 300: + logger.info(f"Deployment status updater polling... (idle checks: {polling_count})") + last_heartbeat_time = time.time() + except ServiceBusConnectionError as e: + logger.warning(f"Service Bus connection error in deployment status updater, will retry: {e}") + await asyncio.sleep(5) + except Exception: + logger.exception("Unexpected error in deployment status receiver loop") + await asyncio.sleep(5) + + async def process_message(self, msg) -> bool: complete_message = False - message = "" - - with tracer.start_as_current_span("process_message") as current_span: + with tracer.start_as_current_span("process_message"): try: message = parse_obj_as(DeploymentStatusUpdateMessage, json.loads(str(msg))) - - current_span.set_attribute("step_id", message.stepId) - current_span.set_attribute("operation_id", message.operationId) - current_span.set_attribute("status", message.status) - - logger.info(f"Received and parsed JSON for: {msg.correlation_id}") complete_message = await self.update_status_in_database(message) logger.info(f"Update status in DB for {message.operationId} - {message.status}") except (json.JSONDecodeError, ValidationError): @@ -160,6 +147,13 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_to_persist = self.create_updated_resource_document(resource, message) await self.resource_repo.update_item_dict(resource_to_persist) + # If the 'main' step succeeded for an uninstall operation, free any allocated address space + # owned by a WorkspaceService resource. MUST be executed BEFORE preparing/enqueuing any next step + # (such as a trailing workspace upgrade step), so Cosmos DB state is updated before the next step + # queries the workspace document. + if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: + await self._free_workspace_address_space(resource_to_persist, operation) + # more steps in the op to do? if is_last_step is False: assert current_step_index < (len(operation.steps) - 1) @@ -190,40 +184,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) - # If the 'main' step succeeded for an uninstall operation, free any allocated address space - # owned by a WorkspaceService resource. We trigger cleanup when the step with templateStepId == 'main' - # is successful; this ensures the primary resource has been destroyed successfully before attempting to free the ip address space - try: - # if the step that just succeeded is the main step for this operation, and this is an uninstall, - # proceed with post-uninstall cleanup. No need to scan the operation.steps list again. - if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: - if resource_to_persist.get("resourceType") == ResourceType.WorkspaceService: - address_to_free = resource_to_persist.get("properties", {}).get("address_space") - parent_workspace_id = resource_to_persist.get("workspaceId") - if address_to_free and parent_workspace_id: - try: - workspace_repo = await WorkspaceRepository.create() - max_retries = 3 - for attempt in range(max_retries): - workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) - workspace_address_spaces = workspace.properties.get("address_spaces", []) - if address_to_free not in workspace_address_spaces: - break - new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] - workspace_patch = ResourcePatch() - workspace_patch.properties = {"address_spaces": new_address_spaces} - try: - await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, self.resource_template_repo, self.resource_history_repo, operation.user, False) - logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") - break - except CosmosAccessConditionFailedError: - if attempt == max_retries - 1: - raise - logger.warning(f"ETag conflict when freeing workspace address space after successful uninstall. Retrying (attempt {attempt + 1}/{max_retries})...") - except Exception: - logger.exception("Failed to free workspace address space after successful uninstall") - except Exception: - logger.exception("Unexpected error during post-uninstall address space cleanup") result = True @@ -236,6 +196,55 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result + async def _free_workspace_address_space(self, resource_to_persist: dict, operation: Operation): + """ + Frees any allocated address space owned by a WorkspaceService resource after a successful uninstall main step. + Updates the parent Workspace document in Cosmos DB so that subsequent pipeline steps (e.g., workspace upgrade) + or future operations rely on the updated address_spaces property. + """ + if resource_to_persist.get("resourceType") != ResourceType.WorkspaceService: + return + + address_to_free = resource_to_persist.get("properties", {}).get("address_space") + parent_workspace_id = resource_to_persist.get("workspaceId") + resource_id = resource_to_persist.get("id") + + if not address_to_free or not parent_workspace_id: + return + + try: + workspace_repo = await WorkspaceRepository.create() + for attempt in range(MAX_CLEANUP_RETRIES): + try: + workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) + workspace_address_spaces = workspace.properties.get("address_spaces", []) + if address_to_free not in workspace_address_spaces: + break + new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": new_address_spaces} + + # Note: patch_workspace with force_version_update=False updates the Cosmos DB record only; + # it does not trigger an independent deployment operation. Infrastructure updates (Terraform) + # are driven by the trailing workspace upgrade step in the uninstall pipeline. + await workspace_repo.patch_workspace( + workspace, + workspace_patch, + workspace.etag, + self.resource_template_repo, + self.resource_history_repo, + operation.user, + False + ) + logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") + break + except CosmosAccessConditionFailedError: + if attempt == MAX_CLEANUP_RETRIES - 1: + raise + logger.warning(f"ETag conflict when freeing workspace address space after successful uninstall. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + except Exception as e: + logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after uninstalling {resource_id}: {e}", exc_info=True) + async def update_overall_operation_status(self, operation: Operation, step: OperationStep, is_last_step: bool): operation.updatedWhen = get_timestamp() diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 9a65e12628..7e4f7449a5 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -626,7 +626,7 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta @patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -@patch('services.logging.logger.exception') +@patch('service_bus.deployment_status_updater.logger') async def test_workspace_service_uninstall_logs_error_after_max_retries( logging_mock, resource_repo, @@ -693,5 +693,116 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( # Assert get_workspace_by_id and patch_workspace called max_retries = 3 times assert workspace_repo.get_workspace_by_id.call_count == 3 assert workspace_repo.patch_workspace.call_count == 3 - # Assert we logged the final failure - logging_mock.assert_called_once_with("Failed to free workspace address space after successful uninstall") + # Assert we logged the final failure using the module's logger + logging_mock.error.assert_called_once() + assert "[ADDRESS_SPACE_CLEANUP_FAILED]" in logging_mock.error.call_args[0][0] + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.update_resource_for_step') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_next_step( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + update_resource_for_step_mock, + send_deployment_message_mock +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + call_order = [] + + # Create a 2-step uninstall operation (step 1: main, step 2: workspace upgrade) + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId="workspace-upgrade", + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "step-1", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + + async def mock_patch_workspace(*args, **kwargs): + call_order.append("patch_workspace") + return parent_workspace, MagicMock() + + workspace_repo.patch_workspace = AsyncMock(side_effect=mock_patch_workspace) + workspace_repo_mock.return_value = workspace_repo + + resource_to_send_mock = MagicMock() + resource_to_send_mock.id = parent_workspace_id + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_for_step_mock.return_value = resource_to_send_mock + + async def mock_send_deployment_message(*args, **kwargs): + call_order.append("send_deployment_message") + + send_deployment_message_mock.side_effect = mock_send_deployment_message + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + assert call_order == ["patch_workspace", "send_deployment_message"] diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index ca5e7ef556..7c9aa6caae 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.7 +version: 0.3.8 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/ohdsi/template_schema.json b/templates/workspace_services/ohdsi/template_schema.json index 1fa56948ba..3ff191a51e 100644 --- a/templates/workspace_services/ohdsi/template_schema.json +++ b/templates/workspace_services/ohdsi/template_schema.json @@ -371,6 +371,13 @@ }, { "stepId": "main" + }, + { + "stepId": "f720975a-c81e-477e-854e-53fde86e5e58", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } From 232c0887e2ff5075ea65a8e74863ed48189b95f7 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 16:32:30 +0000 Subject: [PATCH 10/90] copilot feedback --- api_app/service_bus/deployment_status_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 1ead6bf1fc..20e5be62b3 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -48,7 +48,7 @@ async def receive_messages(self): polling_count = 0 while True: complete_message = True - async with credentials.get_credential_async() as credential: + async with credentials.get_credential_async_context() as credential: async with ServiceBusClient(fully_qualified_namespace=config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential=credential) as service_bus_client: try: logger.debug("Creating Deployment Status receiver session") From a622859ee2fd5f4b4ef8d2d2bab01d4b537d6699 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 16:51:09 +0000 Subject: [PATCH 11/90] =?UTF-8?q?=20=20=E2=80=A2=20Refactored=20AutoLockRe?= =?UTF-8?q?newer=20in=20deployment=5Fstatus=5Fupdater.py=20to=20use=20an?= =?UTF-8?q?=20asynchronous=20context=20manager=20(async=20with=20AutoLockR?= =?UTF-8?q?enewer()=20as=20=20=20renewer:).=20=20=20=E2=80=A2=20This=20ens?= =?UTF-8?q?ures=20that=20renewer.close()=20is=20guaranteed=20to=20execute?= =?UTF-8?q?=20upon=20exiting=20the=20block,=20even=20if=20an=20exception?= =?UTF-8?q?=20occurs=20while=20iterating=20over=20=20=20messages=20or=20co?= =?UTF-8?q?mpleting/abandoning=20a=20message.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service_bus/deployment_status_updater.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 20e5be62b3..54a412b158 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -53,17 +53,16 @@ async def receive_messages(self): try: logger.debug("Creating Deployment Status receiver session") async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=config.SERVICE_BUS_MAX_WAIT_TIME, session_id=NEXT_AVAILABLE_SESSION) as receiver: - renewer = AutoLockRenewer() - renewer.register(receiver, receiver.session, max_lock_renewal_duration=60) + async with AutoLockRenewer() as renewer: + renewer.register(receiver, receiver.session, max_lock_renewal_duration=60) - async for msg in receiver: - complete_message = await self.process_message(msg) - if complete_message: - await receiver.complete_message(msg) - else: - await receiver.abandon_message(msg) + async for msg in receiver: + complete_message = await self.process_message(msg) + if complete_message: + await receiver.complete_message(msg) + else: + await receiver.abandon_message(msg) - await renewer.close() polling_count = 0 except OperationTimeoutError: From 1d65c721c4c50072928f57e87976328e216e7530 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 17:00:43 +0000 Subject: [PATCH 12/90] =?UTF-8?q?=20=201.=20**deployment=5Fstatus=5Fupdate?= =?UTF-8?q?r.py**:=20=20=20=20=20=20=20=E2=80=A2=20Captured=20the=20active?= =?UTF-8?q?=20tracing=20span=20context=20via=20with=20tracer.start=5Fas=5F?= =?UTF-8?q?current=5Fspan("process=5Fmessage")=20as=20current=5Fspan:.=20?= =?UTF-8?q?=20=20=20=20=20=20=E2=80=A2=20Set=20the=20attributes=20step=5Fi?= =?UTF-8?q?d,=20operation=5Fid,=20and=20status=20on=20current=5Fspan=20aft?= =?UTF-8?q?er=20parsing=20the=20DeploymentStatusUpdateMessage.=20=20=202.?= =?UTF-8?q?=20**test=5Fdeployment=5Fstatus=5Fupdate.py**:=20=20=20=20=20?= =?UTF-8?q?=20=20=E2=80=A2=20Added=20a=20unit=20test=20test=5Fprocess=5Fme?= =?UTF-8?q?ssage=5Fsets=5Fspan=5Fattributes=20to=20verify=20that=20set=5Fa?= =?UTF-8?q?ttribute=20is=20called=20with=20the=20expected=20values.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service_bus/deployment_status_updater.py | 7 +++++- .../test_deployment_status_update.py | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 54a412b158..a3906ca513 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -80,9 +80,14 @@ async def receive_messages(self): async def process_message(self, msg) -> bool: complete_message = False - with tracer.start_as_current_span("process_message"): + with tracer.start_as_current_span("process_message") as current_span: try: message = parse_obj_as(DeploymentStatusUpdateMessage, json.loads(str(msg))) + + current_span.set_attribute("step_id", message.stepId) + current_span.set_attribute("operation_id", message.operationId) + current_span.set_attribute("status", message.status) + complete_message = await self.update_status_in_database(message) logger.info(f"Update status in DB for {message.operationId} - {message.status}") except (json.JSONDecodeError, ValidationError): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 7e4f7449a5..07a7fcaaa4 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -154,6 +154,30 @@ async def test_receiving_good_message(logging_mock, resource_repo, operation_rep logging_mock.assert_not_called() +@patch('service_bus.deployment_status_updater.tracer') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_process_message_sets_span_attributes(resource_repo, operation_repo, _, __, tracer_mock): + expected_workspace = create_sample_workspace_object(test_sb_message["id"]) + resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.dict() + + operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) + operation_repo.return_value.get_operation_by_id.return_value = operation + + mock_span = MagicMock() + tracer_mock.start_as_current_span.return_value.__enter__.return_value = mock_span + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + await status_updater.process_message(ServiceBusReceivedMessageMock(test_sb_message)) + + mock_span.set_attribute.assert_any_call("step_id", test_sb_message["stepId"]) + mock_span.set_attribute.assert_any_call("operation_id", uuid.UUID(test_sb_message["operationId"])) + mock_span.set_attribute.assert_any_call("status", test_sb_message["status"]) + + @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') From bf72041369755aa62b6c5717f665deca64b947fe Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 17:30:42 +0000 Subject: [PATCH 13/90] Add SERVICE_BUS_MAX_WAIT_TIME configuration and improve error handling in DeploymentStatusUpdater --- api_app/core/config.py | 1 + .../service_bus/deployment_status_updater.py | 30 +++++++++---------- .../test_deployment_status_update.py | 24 +++++++++++++++ 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/api_app/core/config.py b/api_app/core/config.py index d2f1cf1fa4..2e0f3ecdd2 100644 --- a/api_app/core/config.py +++ b/api_app/core/config.py @@ -41,6 +41,7 @@ SERVICE_BUS_RESOURCE_REQUEST_QUEUE: str = config("SERVICE_BUS_RESOURCE_REQUEST_QUEUE", default="") SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE: str = config("SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE", default="") SERVICE_BUS_STEP_RESULT_QUEUE: str = config("SERVICE_BUS_STEP_RESULT_QUEUE", default="") +SERVICE_BUS_MAX_WAIT_TIME: int = config("SERVICE_BUS_MAX_WAIT_TIME", cast=int, default=10) # Event grid configuration EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT: str = config("EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT", default="") diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index a3906ca513..ba5b643c98 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -48,9 +48,9 @@ async def receive_messages(self): polling_count = 0 while True: complete_message = True - async with credentials.get_credential_async_context() as credential: - async with ServiceBusClient(fully_qualified_namespace=config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential=credential) as service_bus_client: - try: + try: + async with credentials.get_credential_async_context() as credential: + async with ServiceBusClient(fully_qualified_namespace=config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential=credential) as service_bus_client: logger.debug("Creating Deployment Status receiver session") async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=config.SERVICE_BUS_MAX_WAIT_TIME, session_id=NEXT_AVAILABLE_SESSION) as receiver: async with AutoLockRenewer() as renewer: @@ -65,18 +65,18 @@ async def receive_messages(self): polling_count = 0 - except OperationTimeoutError: - polling_count += 1 - # log a heartbeat every ~5 minutes when queue is idle (max_wait_time is usually ~10s) - if time.time() - last_heartbeat_time > 300: - logger.info(f"Deployment status updater polling... (idle checks: {polling_count})") - last_heartbeat_time = time.time() - except ServiceBusConnectionError as e: - logger.warning(f"Service Bus connection error in deployment status updater, will retry: {e}") - await asyncio.sleep(5) - except Exception: - logger.exception("Unexpected error in deployment status receiver loop") - await asyncio.sleep(5) + except OperationTimeoutError: + polling_count += 1 + # log a heartbeat every ~5 minutes when queue is idle (max_wait_time is usually ~10s) + if time.time() - last_heartbeat_time > 300: + logger.info(f"Deployment status updater polling... (idle checks: {polling_count})") + last_heartbeat_time = time.time() + except ServiceBusConnectionError as e: + logger.warning(f"Service Bus connection error in deployment status updater, will retry: {e}") + await asyncio.sleep(5) + except Exception: + logger.exception("Unexpected error in deployment status receiver loop") + await asyncio.sleep(5) async def process_message(self, msg) -> bool: complete_message = False diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 07a7fcaaa4..e1f2e7f2c4 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1,3 +1,4 @@ +import asyncio import copy import json from unittest.mock import MagicMock, ANY @@ -830,3 +831,26 @@ async def mock_send_deployment_message(*args, **kwargs): assert complete_message is True assert call_order == ["patch_workspace", "send_deployment_message"] + + +@patch("service_bus.deployment_status_updater.config") +@patch("service_bus.deployment_status_updater.credentials") +@patch("service_bus.deployment_status_updater.logger") +async def test_receive_messages_handles_credential_context_failure(logger_mock, credentials_mock, config_mock): + config_mock.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE = "test_namespace" + config_mock.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE = "test_queue" + config_mock.SERVICE_BUS_MAX_WAIT_TIME = 10 + + # Simulate exception when entering credential context manager + credentials_mock.get_credential_async_context.side_effect = Exception("Credential error") + + status_updater = DeploymentStatusUpdater() + + # Run receive_messages in a task and cancel after one loop iteration + task = asyncio.create_task(status_updater.receive_messages()) + await asyncio.sleep(0.1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + logger_mock.exception.assert_called_with("Unexpected error in deployment status receiver loop") From 04be27dfd5ebada7f72d0d42e3f5dfa50bab18d5 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 17:50:50 +0000 Subject: [PATCH 14/90] Fix CosmosAccessConditionFailedError instantiation in tests for airlock request and resource patching --- .../test_repositories/test_airlock_request_repository.py | 2 +- .../test_service_bus/test_deployment_status_update.py | 4 ++-- .../tests_ma/test_service_bus/test_resource_request_sender.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index 18a75e8d1e..f62905d8bc 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -147,7 +147,7 @@ async def test_update_airlock_request_with_forbidden_status_should_fail_on_valid await airlock_request_repo.update_airlock_request(mock_existing_request, user, new_status) -@patch("db.repositories.airlock_requests.AirlockRequestRepository.update_airlock_request_item", side_effect=[CosmosAccessConditionFailedError, None]) +@patch("db.repositories.airlock_requests.AirlockRequestRepository.update_airlock_request_item", side_effect=[CosmosAccessConditionFailedError(), None]) @patch("db.repositories.airlock_requests.AirlockRequestRepository.get_airlock_request_by_id", return_value=airlock_request_mock(status=DRAFT)) async def test_update_airlock_request_should_retry_update_when_etag_is_not_up_to_date(_, update_airlock_request_item_mock, airlock_request_repo): expected_update_attempts = 2 diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index e1f2e7f2c4..dbae204d58 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -631,7 +631,7 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta workspace_repo.get_workspace_by_id.return_value = parent_workspace # First attempt raises CosmosAccessConditionFailedError, second succeeds - workspace_repo.patch_workspace.side_effect = [CosmosAccessConditionFailedError, None] + workspace_repo.patch_workspace.side_effect = [CosmosAccessConditionFailedError(), None] workspace_repo_mock.return_value = workspace_repo status_updater = DeploymentStatusUpdater() @@ -706,7 +706,7 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( workspace_repo.get_workspace_by_id.return_value = parent_workspace # All attempts raise CosmosAccessConditionFailedError - workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError + workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() workspace_repo_mock.return_value = workspace_repo status_updater = DeploymentStatusUpdater() diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 5f51770f02..3485d265d1 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -168,7 +168,7 @@ async def test_multi_step_document_retries( ) # simulate an etag mismatch - resource_repo.patch_resource.side_effect = CosmosAccessConditionFailedError + resource_repo.patch_resource.side_effect = CosmosAccessConditionFailedError() num_retries = 5 try: From ad7bdb65c1b5f483f429eea6cf14ec98e1ea9fec Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 31 Jul 2026 18:07:30 +0000 Subject: [PATCH 15/90] Fix type casting for span attributes in DeploymentStatusUpdater and update test assertions --- api_app/service_bus/deployment_status_updater.py | 6 +++--- .../test_service_bus/test_deployment_status_update.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index ba5b643c98..9dd96a2e39 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -84,9 +84,9 @@ async def process_message(self, msg) -> bool: try: message = parse_obj_as(DeploymentStatusUpdateMessage, json.loads(str(msg))) - current_span.set_attribute("step_id", message.stepId) - current_span.set_attribute("operation_id", message.operationId) - current_span.set_attribute("status", message.status) + current_span.set_attribute("step_id", str(message.stepId)) + current_span.set_attribute("operation_id", str(message.operationId)) + current_span.set_attribute("status", str(message.status)) complete_message = await self.update_status_in_database(message) logger.info(f"Update status in DB for {message.operationId} - {message.status}") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index dbae204d58..efa2ab7864 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -175,7 +175,7 @@ async def test_process_message_sets_span_attributes(resource_repo, operation_rep await status_updater.process_message(ServiceBusReceivedMessageMock(test_sb_message)) mock_span.set_attribute.assert_any_call("step_id", test_sb_message["stepId"]) - mock_span.set_attribute.assert_any_call("operation_id", uuid.UUID(test_sb_message["operationId"])) + mock_span.set_attribute.assert_any_call("operation_id", test_sb_message["operationId"]) mock_span.set_attribute.assert_any_call("status", test_sb_message["status"]) From 1ce7cb0db3bcc14cb30832828c60439d7c51f150 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 09:30:18 +0000 Subject: [PATCH 16/90] Handle workspace address cleanup failures --- .../service_bus/deployment_status_updater.py | 25 +++++++++++-------- .../test_deployment_status_update.py | 8 +++--- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 3e2e179b3a..e5b771106a 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -36,6 +36,7 @@ def __init__(self): async def init_repos(self): self.operations_repo = await OperationRepository.create() self.resource_repo = await ResourceRepository.create() + self.workspace_repo = await WorkspaceRepository.create() self.resource_template_repo = await ResourceTemplateRepository.create() self.resource_history_repo = await ResourceHistoryRepository.create() @@ -156,7 +157,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # (such as a trailing workspace upgrade step), so Cosmos DB state is updated before the next step # queries the workspace document. if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: - await self._free_workspace_address_space(resource_to_persist, operation) + if not await self._free_workspace_address_space(resource_to_persist, operation): + return False # more steps in the op to do? if is_last_step is False: @@ -200,30 +202,30 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result - async def _free_workspace_address_space(self, resource_to_persist: dict, operation: Operation): + async def _free_workspace_address_space(self, resource_to_persist: dict, operation: Operation) -> bool: """ Frees any allocated address space owned by a WorkspaceService resource after a successful uninstall main step. Updates the parent Workspace document in Cosmos DB so that subsequent pipeline steps (e.g., workspace upgrade) or future operations rely on the updated address_spaces property. """ - if resource_to_persist.get("resourceType") != ResourceType.WorkspaceService: - return + resource_type = resource_to_persist.get("resourceType") + if resource_type not in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value): + return True address_to_free = resource_to_persist.get("properties", {}).get("address_space") parent_workspace_id = resource_to_persist.get("workspaceId") resource_id = resource_to_persist.get("id") if not address_to_free or not parent_workspace_id: - return + return True try: - workspace_repo = await WorkspaceRepository.create() for attempt in range(MAX_CLEANUP_RETRIES): try: - workspace = await workspace_repo.get_workspace_by_id(parent_workspace_id) + workspace = await self.workspace_repo.get_workspace_by_id(parent_workspace_id) workspace_address_spaces = workspace.properties.get("address_spaces", []) if address_to_free not in workspace_address_spaces: - break + return True new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] workspace_patch = ResourcePatch() workspace_patch.properties = {"address_spaces": new_address_spaces} @@ -231,7 +233,7 @@ async def _free_workspace_address_space(self, resource_to_persist: dict, operati # Note: patch_workspace with force_version_update=False updates the Cosmos DB record only; # it does not trigger an independent deployment operation. Infrastructure updates (Terraform) # are driven by the trailing workspace upgrade step in the uninstall pipeline. - await workspace_repo.patch_workspace( + await self.workspace_repo.patch_workspace( workspace, workspace_patch, workspace.etag, @@ -241,13 +243,16 @@ async def _free_workspace_address_space(self, resource_to_persist: dict, operati False ) logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") - break + return True except CosmosAccessConditionFailedError: if attempt == MAX_CLEANUP_RETRIES - 1: raise logger.warning(f"ETag conflict when freeing workspace address space after successful uninstall. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") except Exception as e: logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after uninstalling {resource_id}: {e}", exc_info=True) + return False + + return False async def update_overall_operation_status(self, operation: Operation, step: OperationStep, is_last_step: bool): operation.updatedWhen = get_timestamp() diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index aa6c243956..b6e874d7f1 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -162,7 +162,7 @@ async def test_receiving_good_message(logging_mock, resource_repo, operation_rep @patch('service_bus.deployment_status_updater.ResourceRepository.create') async def test_process_message_sets_span_attributes(resource_repo, operation_repo, _, __, tracer_mock): expected_workspace = create_sample_workspace_object(test_sb_message["id"]) - resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.dict() + resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.model_dump() operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) operation_repo.return_value.get_operation_by_id.return_value = operation @@ -480,7 +480,7 @@ async def test_workspace_service_uninstall_frees_address_space( # Mock resource dict representation returned by get_resource_dict_by_id workspace_service_dict = { "id": workspace_service_id, - "resourceType": ResourceType.WorkspaceService, + "resourceType": ResourceType.WorkspaceService.value, "workspaceId": parent_workspace_id, "properties": { "address_space": address_space @@ -713,8 +713,8 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( await status_updater.init_repos() complete_message = await status_updater.process_message(service_bus_received_message_mock) - # Note: complete_message is still True because we caught the exception in the outer block and logged it. - assert complete_message is True + # Cleanup failure must abandon the message so the operation can retry before progressing. + assert complete_message is False # Assert get_workspace_by_id and patch_workspace called max_retries = 3 times assert workspace_repo.get_workspace_by_id.call_count == 3 assert workspace_repo.patch_workspace.call_count == 3 From 8ac5d24d86104f7d8d7b4dd72ea2674d1b2e7e5b Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 12:59:51 +0000 Subject: [PATCH 17/90] Defer workspace address cleanup until upgrade --- .../service_bus/deployment_status_updater.py | 90 +++++++++++++------ .../test_deployment_status_update.py | 46 ++++++++-- 2 files changed, 102 insertions(+), 34 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index e5b771106a..47cf678683 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -152,12 +152,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_to_persist = self.create_updated_resource_document(resource, message) await self.resource_repo.update_item_dict(resource_to_persist) - # If the 'main' step succeeded for an uninstall operation, free any allocated address space - # owned by a WorkspaceService resource. MUST be executed BEFORE preparing/enqueuing any next step - # (such as a trailing workspace upgrade step), so Cosmos DB state is updated before the next step - # queries the workspace document. + # Keep the address space reserved until Azure has applied the workspace update. if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: - if not await self._free_workspace_address_space(resource_to_persist, operation): + if self._is_workspace_service_with_address_space(resource_to_persist) and not self._has_workspace_upgrade_step(operation, current_step_index): + operation.steps.append(self._create_workspace_upgrade_step(resource_to_persist)) + is_last_step = False + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + + # A workspace upgrade is the point at which the address space is no longer + # in use by Azure, so it can be released for a future allocation. + if (step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) + and step_to_update.resourceAction == RequestAction.Upgrade + and step_to_update.is_success() + and operation.action == RequestAction.UnInstall): + if not await self._free_workspace_address_space(operation): return False # more steps in the op to do? @@ -168,17 +176,23 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # catch any errors in updating the resource - maybe Cosmos / schema invalid etc, and report them back to the op try: # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values - parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) - resource_to_send = await update_resource_for_step( - operation_step=next_step, - resource_repo=self.resource_repo, - resource_template_repo=self.resource_template_repo, - resource_history_repo=self.resource_history_repo, - root_resource=None, - step_resource=parent_resource, - resource_to_update_id=next_step.resourceId, - primary_action=operation.action, - user=operation.user) + if next_step.templateStepId == "address-space-cleanup": + resource_to_send = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) + else: + parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) + resource_to_send = await update_resource_for_step( + operation_step=next_step, + resource_repo=self.resource_repo, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + root_resource=None, + step_resource=parent_resource, + resource_to_update_id=next_step.resourceId, + primary_action=operation.action, + user=operation.user) + + if next_step.templateStepId == "address-space-cleanup": + await self.operations_repo.update_item(operation) # create + send the message logger.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") @@ -202,16 +216,40 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result - async def _free_workspace_address_space(self, resource_to_persist: dict, operation: Operation) -> bool: + def _is_workspace_service_with_address_space(self, resource: dict) -> bool: + return (resource.get("resourceType") in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value) + and bool(resource.get("properties", {}).get("address_space")) + and bool(resource.get("workspaceId"))) + + def _has_workspace_upgrade_step(self, operation: Operation, current_step_index: int) -> bool: + return any( + step.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) + and step.resourceAction == RequestAction.Upgrade + for step in operation.steps[current_step_index + 1:] + ) + + def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: + return OperationStep( + id=str(uuid.uuid4()), + templateStepId="address-space-cleanup", + stepTitle="Update workspace address spaces", + resourceId=resource["workspaceId"], + resourceTemplateName="", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.AwaitingUpdate, + sourceTemplateResourceId=resource["id"] + ) + + async def _free_workspace_address_space(self, operation: Operation) -> bool: """ - Frees any allocated address space owned by a WorkspaceService resource after a successful uninstall main step. - Updates the parent Workspace document in Cosmos DB so that subsequent pipeline steps (e.g., workspace upgrade) - or future operations rely on the updated address_spaces property. + Frees the address space owned by the uninstalled WorkspaceService after + the corresponding workspace upgrade has succeeded. """ - resource_type = resource_to_persist.get("resourceType") - if resource_type not in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value): + main_step = next((step for step in operation.steps if step.templateStepId == "main"), None) + if main_step is None: return True - + resource_to_persist = await self.resource_repo.get_resource_dict_by_id(main_step.resourceId) address_to_free = resource_to_persist.get("properties", {}).get("address_space") parent_workspace_id = resource_to_persist.get("workspaceId") resource_id = resource_to_persist.get("id") @@ -242,14 +280,14 @@ async def _free_workspace_address_space(self, resource_to_persist: dict, operati operation.user, False ) - logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful uninstall of {resource_id}") + logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful workspace upgrade for {resource_id}") return True except CosmosAccessConditionFailedError: if attempt == MAX_CLEANUP_RETRIES - 1: raise - logger.warning(f"ETag conflict when freeing workspace address space after successful uninstall. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + logger.warning(f"ETag conflict when freeing workspace address space after successful workspace upgrade. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") except Exception as e: - logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after uninstalling {resource_id}: {e}", exc_info=True) + logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after upgrading the workspace for {resource_id}: {e}", exc_info=True) return False return False diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index b6e874d7f1..24b0bdacac 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -440,6 +440,7 @@ async def test_convert_outputs_to_dict(): assert status_updater.convert_outputs_to_dict(deployment_status_update_message.outputs) == expected_result +@patch('service_bus.deployment_status_updater.send_deployment_message') @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @@ -452,7 +453,8 @@ async def test_workspace_service_uninstall_frees_address_space( _, __, ___, - workspace_repo_mock + workspace_repo_mock, + send_deployment_message_mock ): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -502,6 +504,9 @@ async def test_workspace_service_uninstall_frees_address_space( complete_message = await status_updater.process_message(service_bus_received_message_mock) assert complete_message is True + send_deployment_message_mock.assert_called_once() + workspace_repo.patch_workspace.assert_not_called() + await status_updater._free_workspace_address_space(operation) workspace_repo.patch_workspace.assert_called_once() called_args = workspace_repo.patch_workspace.call_args[0] assert called_args[0] == parent_workspace @@ -572,6 +577,7 @@ async def test_workspace_service_uninstall_does_not_free_address_space_if_missin workspace_repo.patch_workspace.assert_not_called() +@patch('service_bus.deployment_status_updater.send_deployment_message') @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @@ -584,7 +590,8 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta _, __, ___, - workspace_repo_mock + workspace_repo_mock, + send_deployment_message_mock ): from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -639,12 +646,15 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta complete_message = await status_updater.process_message(service_bus_received_message_mock) assert complete_message is True - # Assert get_workspace_by_id called twice due to retry - assert workspace_repo.get_workspace_by_id.call_count == 2 + send_deployment_message_mock.assert_called_once() + await status_updater._free_workspace_address_space(operation) + # One read prepares the synthetic upgrade, then two reads cover the retry. + assert workspace_repo.get_workspace_by_id.call_count == 3 # Assert patch_workspace called twice assert workspace_repo.patch_workspace.call_count == 2 +@patch('service_bus.deployment_status_updater.send_deployment_message') @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @@ -659,7 +669,8 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( _, __, ___, - workspace_repo_mock + workspace_repo_mock, + send_deployment_message_mock ): from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -713,10 +724,14 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( await status_updater.init_repos() complete_message = await status_updater.process_message(service_bus_received_message_mock) - # Cleanup failure must abandon the message so the operation can retry before progressing. + # The uninstall message is complete; cleanup is deferred to the upgrade step. + assert complete_message is True + send_deployment_message_mock.assert_called_once() + workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() + complete_message = await status_updater._free_workspace_address_space(operation) assert complete_message is False # Assert get_workspace_by_id and patch_workspace called max_retries = 3 times - assert workspace_repo.get_workspace_by_id.call_count == 3 + assert workspace_repo.get_workspace_by_id.call_count == 4 assert workspace_repo.patch_workspace.call_count == 3 # Assert we logged the final failure using the module's logger logging_mock.error.assert_called_once() @@ -830,7 +845,22 @@ async def mock_send_deployment_message(*args, **kwargs): complete_message = await status_updater.process_message(service_bus_received_message_mock) assert complete_message is True - assert call_order == ["patch_workspace", "send_deployment_message"] + + # The address remains reserved while the workspace upgrade is queued. + assert call_order == ["send_deployment_message"] + + upgrade_message = ServiceBusReceivedMessageMock({ + "operationId": OPERATION_ID, + "stepId": "step-2", + "id": parent_workspace_id, + "status": Status.Updated, + "message": "workspace upgrade succeeded", + "correlation_id": "test_correlation_id" + }) + complete_message = await status_updater.process_message(upgrade_message) + + assert complete_message is True + assert call_order == ["send_deployment_message", "patch_workspace"] @patch("service_bus.deployment_status_updater.config") From 2711e28096bd7c3faefaadb02201d95e893f9246 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 15:25:07 +0000 Subject: [PATCH 18/90] Select the matching resource when freeing workspace address space --- .../service_bus/deployment_status_updater.py | 6 +++- .../test_deployment_status_update.py | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 47cf678683..76936e8bf6 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -246,7 +246,11 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: Frees the address space owned by the uninstalled WorkspaceService after the corresponding workspace upgrade has succeeded. """ - main_step = next((step for step in operation.steps if step.templateStepId == "main"), None) + main_step = next( + (step for step in operation.steps + if step.templateStepId == "main" and step.resourceId == operation.resourceId), + None + ) if main_step is None: return True resource_to_persist = await self.resource_repo.get_resource_dict_by_id(main_step.resourceId) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 24b0bdacac..7ddaccaff6 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -514,6 +514,42 @@ async def test_workspace_service_uninstall_frees_address_space( assert called_args[2] == "parent-workspace-etag" +async def test_workspace_service_uninstall_frees_address_space_from_root_main_step(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + child_resource_id = "69b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps.insert(0, OperationStep( + id="child-main", + templateStepId="main", + resourceId=child_resource_id, + resourceType=ResourceType.UserResource, + resourceAction=RequestAction.UnInstall + )) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.workspace_repo = AsyncMock() + status_updater.resource_template_repo = AsyncMock() + status_updater.resource_history_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": [address_space]} + parent_workspace.etag = "parent-workspace-etag" + status_updater.workspace_repo.get_workspace_by_id.return_value = parent_workspace + + assert await status_updater._free_workspace_address_space(operation) is True + status_updater.resource_repo.get_resource_dict_by_id.assert_awaited_once_with(workspace_service_id) + status_updater.workspace_repo.patch_workspace.assert_awaited_once() + + @pytest.mark.parametrize("missing_property", ["address_space", "workspaceId"]) @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') From 5d668e817396013de389dd4068bb662bc09cfd5e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 16:34:38 +0000 Subject: [PATCH 19/90] Mock workspace repository in deployment status tests --- .../test_deployment_status_update.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 7ddaccaff6..efeaedf8b3 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -133,12 +133,13 @@ async def test_receiving_bad_json_logs_error(logging_mock, payload): assert error_message.startswith(strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_receiving_good_message(logging_mock, resource_repo, operation_repo, _, __): +async def test_receiving_good_message(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): expected_workspace = create_sample_workspace_object(test_sb_message["id"]) resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.model_dump() @@ -156,11 +157,12 @@ async def test_receiving_good_message(logging_mock, resource_repo, operation_rep @patch('service_bus.deployment_status_updater.tracer') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_process_message_sets_span_attributes(resource_repo, operation_repo, _, __, tracer_mock): +async def test_process_message_sets_span_attributes(resource_repo, operation_repo, _, __, workspace_repo, tracer_mock): expected_workspace = create_sample_workspace_object(test_sb_message["id"]) resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.model_dump() @@ -179,12 +181,13 @@ async def test_process_message_sets_span_attributes(resource_repo, operation_rep mock_span.set_attribute.assert_any_call("status", test_sb_message["status"]) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock, resource_repo, operation_repo, _, __): +async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): resource_repo.return_value.get_resource_dict_by_id.side_effect = EntityDoesNotExist operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) @@ -199,12 +202,13 @@ async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock logging_mock.assert_called_once_with(expected_error_message) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_when_updating_and_state_store_exception(logging_mock, resource_repo, operation_repo, _, __): +async def test_when_updating_and_state_store_exception(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): resource_repo.return_value.get_resource_dict_by_id.side_effect = Exception operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) @@ -218,12 +222,13 @@ async def test_when_updating_and_state_store_exception(logging_mock, resource_re assert complete_message is False +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_state_transitions_from_deployed_to_deleted(resource_repo, operations_repo_mock, _, __, ___): +async def test_state_transitions_from_deployed_to_deleted(resource_repo, operations_repo_mock, _, __, ___, workspace_repo): updated_message = test_sb_message updated_message["status"] = Status.Deleted updated_message["message"] = "Has been deleted" @@ -250,11 +255,12 @@ async def test_state_transitions_from_deployed_to_deleted(resource_repo, operati operations_repo_mock.return_value.update_item.assert_called_once_with(expected_operation) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo, _, __): +async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo, _, __, workspace_repo): received_message = test_sb_message_with_outputs received_message["status"] = Status.Deployed service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -287,11 +293,12 @@ async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo resource_repo.return_value.update_item_dict.assert_called_once_with(expected_resource.model_dump()) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_properties_dont_change_with_no_outputs(resource_repo, operations_repo, _, __): +async def test_properties_dont_change_with_no_outputs(resource_repo, operations_repo, _, __, workspace_repo): received_message = test_sb_message received_message["status"] = Status.Deployed service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -313,13 +320,14 @@ async def test_properties_dont_change_with_no_outputs(resource_repo, operations_ resource_repo.return_value.update_item_dict.assert_called_once_with(expected_resource.model_dump()) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.update_resource_for_step') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('service_bus.helpers.ServiceBusClient') -async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_repo, operations_repo, update_resource_for_step, _, __, multi_step_operation, user_resource_multi, basic_shared_service): +async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_repo, operations_repo, update_resource_for_step, _, __, workspace_repo, multi_step_operation, user_resource_multi, basic_shared_service): received_message = test_sb_message_multi_step_1_complete received_message["status"] = Status.Updated service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -368,12 +376,13 @@ async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_r sb_sender_client().get_queue_sender().send_messages.assert_called_once() +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('service_bus.helpers.ServiceBusClient') -async def test_multi_step_operation_ends_at_last_step(sb_sender_client, resource_repo, operations_repo, _, __, multi_step_operation, user_resource_multi, basic_shared_service): +async def test_multi_step_operation_ends_at_last_step(sb_sender_client, resource_repo, operations_repo, _, __, workspace_repo, multi_step_operation, user_resource_multi, basic_shared_service): received_message = test_sb_message_multi_step_3_complete received_message["status"] = Status.Updated service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) From 60e52b62e18a648fc866f4c57a7c8faf3e33be2c Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 17:35:26 +0000 Subject: [PATCH 20/90] Require successful root step for workspace address cleanup --- .../service_bus/deployment_status_updater.py | 7 +++-- .../test_deployment_status_update.py | 27 ++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 76936e8bf6..1b1c02d3e7 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -161,7 +161,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # A workspace upgrade is the point at which the address space is no longer # in use by Azure, so it can be released for a future allocation. - if (step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) + if (step_to_update.templateStepId == "address-space-cleanup" + and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) and step_to_update.resourceAction == RequestAction.Upgrade and step_to_update.is_success() and operation.action == RequestAction.UnInstall): @@ -248,7 +249,9 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: """ main_step = next( (step for step in operation.steps - if step.templateStepId == "main" and step.resourceId == operation.resourceId), + if step.templateStepId == "main" + and step.resourceId == operation.resourceId + and step.is_success()), None ) if main_step is None: diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index efeaedf8b3..e24e4a6b13 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -530,6 +530,7 @@ async def test_workspace_service_uninstall_frees_address_space_from_root_main_st address_space = "10.1.0.0/22" operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted operation.steps.insert(0, OperationStep( id="child-main", templateStepId="main", @@ -559,6 +560,30 @@ async def test_workspace_service_uninstall_frees_address_space_from_root_main_st status_updater.workspace_repo.patch_workspace.assert_awaited_once() +async def test_workspace_service_uninstall_does_not_free_address_space_if_root_main_failed(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.DeletingFailed + operation.steps.append(OperationStep( + id="address-space-cleanup", + templateStepId="address-space-cleanup", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.Updated + )) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.workspace_repo = AsyncMock() + + assert await status_updater._free_workspace_address_space(operation) is True + status_updater.resource_repo.get_resource_dict_by_id.assert_not_awaited() + status_updater.workspace_repo.patch_workspace.assert_not_awaited() + + @pytest.mark.parametrize("missing_property", ["address_space", "workspaceId"]) @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @@ -823,7 +848,7 @@ async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_ resourceId=parent_workspace_id, resourceType=ResourceType.Workspace, resourceAction=RequestAction.Upgrade, - templateStepId="workspace-upgrade", + templateStepId="address-space-cleanup", sourceTemplateResourceId=parent_workspace_id, status=Status.AwaitingUpdate ) From 9e78958988f573f01c2665aa09085b7a51ed0815 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 21 Aug 2026 17:52:15 +0000 Subject: [PATCH 21/90] standardize the deallocation of IP addresses for workspace services using address-space-cleanup stepId --- .../pipeline-templates/pipeline-schema.md | 17 ++++++++++++++++- .../azureml/template_schema.json | 4 ++-- .../databricks/template_schema.json | 4 ++-- .../ohdsi/template_schema.json | 4 ++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/tre-templates/pipeline-templates/pipeline-schema.md b/docs/tre-templates/pipeline-templates/pipeline-schema.md index 411e5f4302..aabc62fbbe 100644 --- a/docs/tre-templates/pipeline-templates/pipeline-schema.md +++ b/docs/tre-templates/pipeline-templates/pipeline-schema.md @@ -8,7 +8,7 @@ This document will help you write a valid `pipeline: {}` block in your template. "pipeline": { "install": [ // <-- [install | upgrade | uninstall] { - "stepId": "a unique string value here", + "stepId": "a unique string value within this template", "stepTitle": "Friendly description of the step here - will be displayed in the UI", "resourceTemplateName": "name of the resource template to update", // only required for shared_service targets "resourceType": "shared-service", // [ shared-service | user-resource | workspace-service | workspace ] @@ -83,6 +83,21 @@ To do so, the `arraySubstitutionAction` field supports the following values: - `replace` - find this object in the array (using the `arrayMatchField` value), and replace it with this value - `remove` - remove this property from the array (useful for `uninstall` actions) +## Reserved Cleanup Step +Use `address-space-cleanup` as the `stepId` for a trailing workspace upgrade that releases a workspace service's address space during uninstall: + +```json +{ + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] +} +``` + +This identifier is reserved for that cleanup behavior and may be used by multiple templates. Step IDs only need to be unique within a single template's pipeline. + ## Notes - Each step is executed in serial, in the order defined in the template - Theoretically any number of steps could be created diff --git a/templates/workspace_services/azureml/template_schema.json b/templates/workspace_services/azureml/template_schema.json index 32c4f2bc60..6e4cdd2c69 100644 --- a/templates/workspace_services/azureml/template_schema.json +++ b/templates/workspace_services/azureml/template_schema.json @@ -414,7 +414,7 @@ "stepId": "main" }, { - "stepId": "f720975a-c81e-477e-854e-53fde86e5e57", + "stepId": "address-space-cleanup", "stepTitle": "Upgrade to ensure workspace is aware of address space removal", "resourceType": "workspace", "resourceAction": "upgrade", @@ -422,4 +422,4 @@ } ] } -} +} \ No newline at end of file diff --git a/templates/workspace_services/databricks/template_schema.json b/templates/workspace_services/databricks/template_schema.json index fbde2fbe3f..d643c05855 100644 --- a/templates/workspace_services/databricks/template_schema.json +++ b/templates/workspace_services/databricks/template_schema.json @@ -368,7 +368,7 @@ "stepId": "main" }, { - "stepId": "9c4dc64b-8fbf-4e77-a7f6-48fb33423504", + "stepId": "address-space-cleanup", "stepTitle": "Upgrade to ensure workspace is aware of address space removal", "resourceType": "workspace", "resourceAction": "upgrade", @@ -376,4 +376,4 @@ } ] } -} +} \ No newline at end of file diff --git a/templates/workspace_services/ohdsi/template_schema.json b/templates/workspace_services/ohdsi/template_schema.json index b3b931cadf..30f249afee 100644 --- a/templates/workspace_services/ohdsi/template_schema.json +++ b/templates/workspace_services/ohdsi/template_schema.json @@ -372,7 +372,7 @@ "stepId": "main" }, { - "stepId": "f720975a-c81e-477e-854e-53fde86e5e58", + "stepId": "address-space-cleanup", "stepTitle": "Upgrade to ensure workspace is aware of address space removal", "resourceType": "workspace", "resourceAction": "upgrade", @@ -380,4 +380,4 @@ } ] } -} +} \ No newline at end of file From 9fb1eb3ce88d9d21e21db94cf922c2affd5b6e92 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:56:19 +0100 Subject: [PATCH 22/90] CHANGELOG Removed duplicate 'BUG FIXES' section and updated entries. --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3419b3d1e..324a34862f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,6 @@ ENHANCEMENTS: BUG FIXES: * Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055)) - -BUG FIXES: * `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) ## (0.29.0) (August 14, 2026) From 6e71bb1e2181bd4af5d5c83ac881dc439efb0e41 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 11:29:38 +0000 Subject: [PATCH 23/90] Fix address space cleanup handling for cascade uninstalls --- api_app/db/repositories/operations.py | 7 +- api_app/resources/strings.py | 1 + .../service_bus/deployment_status_updater.py | 36 +++++++---- .../test_operation_repository.py | 40 +++++++++++- .../test_deployment_status_update.py | 64 +++++++++++++++++++ 5 files changed, 131 insertions(+), 17 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index f85d69af68..331e3583d0 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -73,7 +73,8 @@ async def create_operation_item(self, resource_id: str, resource_list: List, act resource_repo=resource_repo, resource_id=resource["id"], status=status, - message=message + message=message, + is_cascade_operation=resource["id"] != resource_id ) # if no pipeline is defined for this action, create a main step only @@ -100,10 +101,12 @@ async def create_operation_item(self, resource_id: str, resource_list: List, act await self.save_item(operation) return operation - async def build_step_list(self, steps: List[OperationStep], resource_template_dict: dict, action: str, resource_repo: ResourceRepository, resource_id: str, status: Status, message: str): + async def build_step_list(self, steps: List[OperationStep], resource_template_dict: dict, action: str, resource_repo: ResourceRepository, resource_id: str, status: Status, message: str, is_cascade_operation: bool = False): if "pipeline" in resource_template_dict and resource_template_dict["pipeline"] is not None: if action in resource_template_dict["pipeline"] and resource_template_dict["pipeline"][action] is not None: for step in resource_template_dict["pipeline"][action]: + if is_cascade_operation and step["stepId"] == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + continue if step["stepId"] == "main": steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource_id, status=status, message=message)) else: diff --git a/api_app/resources/strings.py b/api_app/resources/strings.py index 09432e64a3..0d889e5155 100644 --- a/api_app/resources/strings.py +++ b/api_app/resources/strings.py @@ -181,6 +181,7 @@ RESOURCE_ACTION_INSTALL = "install" RESOURCE_ACTION_UPDATE = "upgrade" RESOURCE_ACTION_UNINSTALL = "uninstall" +ADDRESS_SPACE_CLEANUP_STEP_ID = "address-space-cleanup" # Resource Action Status diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 1b1c02d3e7..d54a02bedd 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -153,21 +153,30 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.resource_repo.update_item_dict(resource_to_persist) # Keep the address space reserved until Azure has applied the workspace update. - if step_to_update.templateStepId == "main" and step_to_update.is_success() and operation.action == RequestAction.UnInstall: - if self._is_workspace_service_with_address_space(resource_to_persist) and not self._has_workspace_upgrade_step(operation, current_step_index): - operation.steps.append(self._create_workspace_upgrade_step(resource_to_persist)) - is_last_step = False - await self.update_overall_operation_status(operation, step_to_update, is_last_step) + if (step_to_update.templateStepId == "main" + and step_to_update.resourceId == operation.resourceId + and step_to_update.is_success() + and operation.action == RequestAction.UnInstall + and self._is_workspace_service_with_address_space(resource_to_persist) + and not self._has_workspace_upgrade_step(operation, current_step_index)): + operation.steps.append(self._create_workspace_upgrade_step(resource_to_persist)) + is_last_step = False + await self.update_overall_operation_status(operation, step_to_update, is_last_step) # A workspace upgrade is the point at which the address space is no longer # in use by Azure, so it can be released for a future allocation. - if (step_to_update.templateStepId == "address-space-cleanup" + if (step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) and step_to_update.resourceAction == RequestAction.Upgrade and step_to_update.is_success() - and operation.action == RequestAction.UnInstall): - if not await self._free_workspace_address_space(operation): - return False + and operation.action == RequestAction.UnInstall + and not await self._free_workspace_address_space(operation)): + cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." + step_to_update.status = Status.UpdatingFailed + step_to_update.message = cleanup_failure_message + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + return False # more steps in the op to do? if is_last_step is False: @@ -177,7 +186,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # catch any errors in updating the resource - maybe Cosmos / schema invalid etc, and report them back to the op try: # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values - if next_step.templateStepId == "address-space-cleanup": + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: resource_to_send = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) else: parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) @@ -192,7 +201,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage primary_action=operation.action, user=operation.user) - if next_step.templateStepId == "address-space-cleanup": + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: await self.operations_repo.update_item(operation) # create + send the message @@ -224,15 +233,14 @@ def _is_workspace_service_with_address_space(self, resource: dict) -> bool: def _has_workspace_upgrade_step(self, operation: Operation, current_step_index: int) -> bool: return any( - step.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) - and step.resourceAction == RequestAction.Upgrade + step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps[current_step_index + 1:] ) def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: return OperationStep( id=str(uuid.uuid4()), - templateStepId="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, stepTitle="Update workspace address spaces", resourceId=resource["workspaceId"], resourceTemplateName="", diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index c0c5f9b09c..5550052d66 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -1,10 +1,12 @@ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import uuid import pytest_asyncio import pytest from mock import patch from db.repositories.resource_templates import ResourceTemplateRepository from models.domain.operation import Status +from models.domain.request_action import RequestAction +from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository from db.repositories.operations import OperationRepository from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP @@ -65,3 +67,39 @@ async def test_create_operation_steps_from_multi_step_template(_, __, ___, resou ) assert operation.model_dump() == expected_op.model_dump() + + +async def test_create_operation_steps_excludes_address_space_cleanup_for_cascade( + resource_repo, test_user, operations_repo, basic_shared_service, resource_template_repo +): + resource_template = MagicMock() + resource_template.model_dump.return_value = { + "name": "workspace-service", + "resourceType": ResourceType.WorkspaceService, + "pipeline": { + "uninstall": [ + {"stepId": "main"}, + { + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade workspace", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ] + } + } + resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=resource_template) + operations_repo.save_item = AsyncMock() + + operation = await operations_repo.create_operation_item( + resource_id="workspace-root", + resource_list=[{**basic_shared_service.__dict__, "id": "descendant-service"}], + action=RequestAction.UnInstall, + resource_path="/workspaces/workspace-root", + resource_version=0, + user=test_user, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo + ) + + assert [step.templateStepId for step in operation.steps] == ["main"] diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index e24e4a6b13..d964e31145 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -808,6 +808,70 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( assert "[ADDRESS_SPACE_CLEANUP_FAILED]" in logging_mock.error.call_args[0][0] +async def test_address_space_cleanup_failure_is_persisted_before_message_redelivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted + operation.steps.append(OperationStep( + id="address-space-cleanup-step", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.Updating + )) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = MagicMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = create_sample_workspace_object(parent_workspace_id).model_dump() + status_updater.workspace_repo = AsyncMock() + status_updater.resource_template_repo = AsyncMock() + status_updater.resource_history_repo = AsyncMock() + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="address-space-cleanup-step", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + complete_message = await status_updater.update_status_in_database(message) + + assert complete_message is False + assert operation.steps[-1].status == Status.UpdatingFailed + assert "will be retried" in operation.steps[-1].message + assert status_updater.operations_repo.update_item.await_args_list[-1].args[0] == operation + + +async def test_unrelated_workspace_upgrade_does_not_suppress_address_space_cleanup(): + operation = create_sample_operation("workspace-service-id", RequestAction.UnInstall) + status_updater = DeploymentStatusUpdater() + + operation.steps.append(OperationStep( + id="workspace-upgrade", + templateStepId="unrelated-upgrade", + resourceId="workspace-id", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + )) + assert status_updater._has_workspace_upgrade_step(operation, 0) is False + + operation.steps.append(OperationStep( + id="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + resourceId="workspace-id", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + )) + assert status_updater._has_workspace_upgrade_step(operation, 0) is True + + @patch('service_bus.deployment_status_updater.send_deployment_message') @patch('service_bus.deployment_status_updater.update_resource_for_step') @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') From b2d187f42078856cdd34012cb6703b1149513911 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 11:49:01 +0000 Subject: [PATCH 24/90] Fix workspace address space removal payload and failure status handling --- .../service_bus/deployment_status_updater.py | 46 ++++++++++---- .../test_operation_repository.py | 3 +- .../test_deployment_status_update.py | 61 +++++++++++++++++-- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index d54a02bedd..2e48ca8093 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -1,7 +1,9 @@ import asyncio +import copy import json import uuid import time +from typing import Optional, Tuple from pydantic import ValidationError, TypeAdapter @@ -123,7 +125,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage is_last_step = True if step_to_update is None: - raise f"Error finding step {message.stepId} in operation {message.operationId}" + raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") # update the step status step_to_update.status = message.status @@ -162,6 +164,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage operation.steps.append(self._create_workspace_upgrade_step(resource_to_persist)) is_last_step = False await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) # A workspace upgrade is the point at which the address space is no longer # in use by Azure, so it can be released for a future allocation. @@ -176,6 +179,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage step_to_update.message = cleanup_failure_message await self.update_overall_operation_status(operation, step_to_update, is_last_step) await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) return False # more steps in the op to do? @@ -187,7 +193,16 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage try: # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: - resource_to_send = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) + workspace = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) + resource_to_send = copy.deepcopy(workspace) + cleanup_details = await self._get_address_cleanup_details(operation) + if cleanup_details: + address_to_free, _, _ = cleanup_details + workspace_address_spaces = resource_to_send.properties.get("address_spaces", []) + if address_to_free and isinstance(workspace_address_spaces, list): + resource_to_send.properties["address_spaces"] = [ + a for a in workspace_address_spaces if a != address_to_free + ] else: parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) resource_to_send = await update_resource_for_step( @@ -250,11 +265,7 @@ def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: sourceTemplateResourceId=resource["id"] ) - async def _free_workspace_address_space(self, operation: Operation) -> bool: - """ - Frees the address space owned by the uninstalled WorkspaceService after - the corresponding workspace upgrade has succeeded. - """ + async def _get_address_cleanup_details(self, operation: Operation) -> Optional[Tuple[str, str, str]]: main_step = next( (step for step in operation.steps if step.templateStepId == "main" @@ -263,15 +274,28 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: None ) if main_step is None: - return True + return None resource_to_persist = await self.resource_repo.get_resource_dict_by_id(main_step.resourceId) address_to_free = resource_to_persist.get("properties", {}).get("address_space") parent_workspace_id = resource_to_persist.get("workspaceId") resource_id = resource_to_persist.get("id") if not address_to_free or not parent_workspace_id: + return None + + return address_to_free, parent_workspace_id, resource_id + + async def _free_workspace_address_space(self, operation: Operation) -> bool: + """ + Frees the address space owned by the uninstalled WorkspaceService after + the corresponding workspace upgrade has succeeded. + """ + cleanup_details = await self._get_address_cleanup_details(operation) + if cleanup_details is None: return True + address_to_free, parent_workspace_id, resource_id = cleanup_details + try: for attempt in range(MAX_CLEANUP_RETRIES): try: @@ -325,9 +349,9 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper # pipeline failed - update the primary resource (from the main step) as failed too main_step = None - for i, step in enumerate(operation.steps): - if step.templateStepId == "main": - main_step = step + for op_step in operation.steps: + if op_step.templateStepId == "main": + main_step = op_step break if main_step: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 5550052d66..8af2296fed 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -9,6 +9,7 @@ from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository from db.repositories.operations import OperationRepository +from resources import strings from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP RESOURCE_ID = str(uuid.uuid4()) @@ -80,7 +81,7 @@ async def test_create_operation_steps_excludes_address_space_cleanup_for_cascade "uninstall": [ {"stepId": "main"}, { - "stepId": "address-space-cleanup", + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "stepTitle": "Upgrade workspace", "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade, diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index d964e31145..3fc659a6f5 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -514,6 +514,9 @@ async def test_workspace_service_uninstall_frees_address_space( assert complete_message is True send_deployment_message_mock.assert_called_once() + sent_payload = json.loads(send_deployment_message_mock.call_args.kwargs["content"]) + assert sent_payload["parameters"]["address_spaces"] == ["10.0.0.0/22"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22", address_space] workspace_repo.patch_workspace.assert_not_called() await status_updater._free_workspace_address_space(operation) workspace_repo.patch_workspace.assert_called_once() @@ -568,7 +571,7 @@ async def test_workspace_service_uninstall_does_not_free_address_space_if_root_m operation.steps[0].status = Status.DeletingFailed operation.steps.append(OperationStep( id="address-space-cleanup", - templateStepId="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, resourceId=parent_workspace_id, resourceType=ResourceType.Workspace, resourceAction=RequestAction.Upgrade, @@ -847,6 +850,7 @@ async def test_address_space_cleanup_failure_is_persisted_before_message_redeliv assert operation.steps[-1].status == Status.UpdatingFailed assert "will be retried" in operation.steps[-1].message assert status_updater.operations_repo.update_item.await_args_list[-1].args[0] == operation + assert status_updater.resource_repo.update_item.await_args_list[-1].args[0].deploymentStatus == Status.UpdatingFailed async def test_unrelated_workspace_upgrade_does_not_suppress_address_space_cleanup(): @@ -880,7 +884,7 @@ async def test_unrelated_workspace_upgrade_does_not_suppress_address_space_clean @patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_next_step( +async def test_workspace_service_uninstall_defers_address_space_cleanup_until_workspace_upgrade_succeeds( resource_repo, operations_repo_mock, _, @@ -895,6 +899,7 @@ async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_ address_space = "10.1.0.0/22" call_order = [] + sent_payloads = [] # Create a 2-step uninstall operation (step 1: main, step 2: workspace upgrade) step1 = OperationStep( @@ -912,7 +917,7 @@ async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_ resourceId=parent_workspace_id, resourceType=ResourceType.Workspace, resourceAction=RequestAction.Upgrade, - templateStepId="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, sourceTemplateResourceId=parent_workspace_id, status=Status.AwaitingUpdate ) @@ -959,6 +964,7 @@ async def test_workspace_service_uninstall_frees_address_space_before_enqueuing_ async def mock_patch_workspace(*args, **kwargs): call_order.append("patch_workspace") + parent_workspace.properties = kwargs.get("workspace_patch", args[1] if len(args) > 1 else None).properties return parent_workspace, MagicMock() workspace_repo.patch_workspace = AsyncMock(side_effect=mock_patch_workspace) @@ -971,6 +977,7 @@ async def mock_patch_workspace(*args, **kwargs): async def mock_send_deployment_message(*args, **kwargs): call_order.append("send_deployment_message") + sent_payloads.append(json.loads(kwargs["content"])) send_deployment_message_mock.side_effect = mock_send_deployment_message @@ -980,8 +987,11 @@ async def mock_send_deployment_message(*args, **kwargs): assert complete_message is True - # The address remains reserved while the workspace upgrade is queued. + # The address remains reserved while the workspace upgrade is queued, + # but the deployment message sent to Terraform excludes the target address space. assert call_order == ["send_deployment_message"] + assert sent_payloads[0]["parameters"]["address_spaces"] == ["10.0.0.0/22"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22", address_space] upgrade_message = ServiceBusReceivedMessageMock({ "operationId": OPERATION_ID, @@ -995,6 +1005,7 @@ async def mock_send_deployment_message(*args, **kwargs): assert complete_message is True assert call_order == ["send_deployment_message", "patch_workspace"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22"] @patch("service_bus.deployment_status_updater.config") @@ -1018,3 +1029,45 @@ async def test_receive_messages_handles_credential_context_failure(logger_mock, await task logger_mock.exception.assert_called_with("Unexpected error in deployment status receiver loop") + + +async def test_workspace_upgrade_failure_does_not_free_address_space(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted + operation.steps.append(OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating + )) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = MagicMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = create_sample_workspace_object(parent_workspace_id).model_dump() + status_updater.workspace_repo = AsyncMock() + + failed_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.UpdatingFailed, + message="workspace upgrade failed" + ) + + with patch.object(status_updater, "_free_workspace_address_space") as free_mock: + complete_message = await status_updater.update_status_in_database(failed_message) + + assert complete_message is True + free_mock.assert_not_called() + assert operation.steps[-1].status == Status.UpdatingFailed + assert operation.status == Status.DeletingFailed From bdca9b2c5017c1859a852934a73af2b28c427f80 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 13:30:07 +0000 Subject: [PATCH 25/90] Serialize workspace address allocation with active operations and restore primary resource on retried cleanup --- api_app/api/routes/workspaces.py | 2 + api_app/db/repositories/operations.py | 15 +++ api_app/resources/strings.py | 1 + .../service_bus/deployment_status_updater.py | 49 +++++++--- .../test_api/test_routes/test_workspaces.py | 20 +++- .../test_operation_repository.py | 16 ++++ .../test_deployment_status_update.py | 91 +++++++++++++++++++ 7 files changed, 181 insertions(+), 13 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 04ea2d6513..d8af3b265c 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -262,6 +262,8 @@ async def create_workspace_service(response: Response, workspace_service_input: # check workspace has address_spaces property if not workspace.properties.get("address_spaces"): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) workspace_patch = ResourcePatch() workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 331e3583d0..723e10aaaa 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -197,3 +197,18 @@ async def resource_has_deployed_operation(self, resource_id: str) -> bool: query = self.operations_query() + f' c.resourceId = "{resource_id}" AND ((c.action = "{RequestAction.Install}" AND c.status = "{Status.Deployed}") OR (c.action = "{RequestAction.Upgrade}" AND c.status = "{Status.Updated}"))' operations = await self.query(query=query) return len(operations) > 0 + + async def resource_has_active_operation(self, resource_id: str) -> bool: + active_statuses = ( + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + ) + status_filter = ", ".join(f'"{s}"' for s in active_statuses) + query = ( + self.operations_query() + + f' (c.resourceId = "{resource_id}" OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true))' + + f' AND c.status IN ({status_filter})' + ) + operations = await self.query(query=query) + return len(operations) > 0 diff --git a/api_app/resources/strings.py b/api_app/resources/strings.py index 0d889e5155..f1a5722263 100644 --- a/api_app/resources/strings.py +++ b/api_app/resources/strings.py @@ -143,6 +143,7 @@ WORKSPACE_SERVICE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION = "The workspace service needs to be disabled before you can delete it" WORKSPACE_SERVICES_NEED_TO_BE_DELETED_BEFORE_WORKSPACE = "All workspace services need to be deleted before you can delete the workspace" WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY = "Workspace does not have address_spaces property" +WORKSPACE_HAS_ACTIVE_OPERATION = "Workspace has an active operation in progress" WORKSPACE_TEMPLATE_VERSION_EXISTS = "A template with the same version, or a newer version already exists" OPERATION_DOES_NOT_EXIST = "Operation does not exist" CUSTOM_ACTION_NOT_DEFINED = "The specified custom action isn't defined in the targeted resource." diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 2e48ca8093..fa318fb620 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -172,17 +172,30 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) and step_to_update.resourceAction == RequestAction.Upgrade and step_to_update.is_success() - and operation.action == RequestAction.UnInstall - and not await self._free_workspace_address_space(operation)): - cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." - step_to_update.status = Status.UpdatingFailed - step_to_update.message = cleanup_failure_message - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) - resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) - resource_to_update.deploymentStatus = step_to_update.status - await self.resource_repo.update_item(resource_to_update) - return False + and operation.action == RequestAction.UnInstall): + if not await self._free_workspace_address_space(operation): + cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." + step_to_update.status = Status.UpdatingFailed + step_to_update.message = cleanup_failure_message + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + return False + else: + # Restore the primary resource to Deleted when cleanup succeeds + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) # more steps in the op to do? if is_last_step is False: @@ -350,7 +363,7 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper # pipeline failed - update the primary resource (from the main step) as failed too main_step = None for op_step in operation.steps: - if op_step.templateStepId == "main": + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId: main_step = op_step break @@ -363,6 +376,18 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.status = self.get_success_status_for_action(operation.action) operation.message = "Multi step pipeline completed successfully" + # pipeline succeeded - update the primary resource (from the main step) as succeeded too + main_step = None + for op_step in operation.steps: + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId: + main_step = op_step + break + + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + def get_success_status_for_action(self, action: RequestAction): status = Status.ActionSucceeded diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 986bb8d450..88fbf9d6d4 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -776,8 +776,9 @@ async def test_post_workspace_services_creates_workspace_service(self, _, __, __ @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") - async def test_post_workspace_services_creates_workspace_service_with_address_space(self, create_workspace_service_item_mock, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): + async def test_post_workspace_services_creates_workspace_service_with_address_space(self, create_workspace_service_item_mock, _, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): etag = "some-etag-value" workspace = sample_workspace() workspace.properties["address_spaces"] = ["192.168.0.1/24"] @@ -803,6 +804,23 @@ async def test_post_workspace_services_creates_workspace_service_with_address_sp assert response.status_code == status.HTTP_202_ACCEPTED assert response.json()["operation"]["resourceId"] == SERVICE_ID + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") + async def test_post_workspace_services_creates_workspace_service_with_address_space_raises_409_if_workspace_has_active_operation(self, create_workspace_service_item_mock, _, get_workspace_mock, __, app, client, workspace_service_input, basic_workspace_service_template): + workspace = sample_workspace() + workspace.properties["address_spaces"] = ["192.168.0.1/24"] + get_workspace_mock.return_value = workspace + basic_workspace_service_template.properties["address_space"]: str = Field() + create_workspace_service_item_mock.return_value = [sample_workspace_service(), basic_workspace_service_template] + + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 8af2296fed..56e8d1c502 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -104,3 +104,19 @@ async def test_create_operation_steps_excludes_address_space_cleanup_for_cascade ) assert [step.templateStepId for step in operation.steps] == ["main"] + + +async def test_resource_has_active_operation_returns_true_when_active_operation_exists(operations_repo): + operations_repo.query = AsyncMock(return_value=[{"id": "op-1"}]) + result = await operations_repo.resource_has_active_operation("workspace-123") + assert result is True + operations_repo.query.assert_called_once() + query_str = operations_repo.query.call_args[1]["query"] if "query" in operations_repo.query.call_args[1] else operations_repo.query.call_args[0][0] + assert "workspace-123" in query_str + assert "ARRAY_CONTAINS" in query_str + + +async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): + operations_repo.query = AsyncMock(return_value=[]) + result = await operations_repo.resource_has_active_operation("workspace-123") + assert result is False diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 3fc659a6f5..430005aba6 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1071,3 +1071,94 @@ async def test_workspace_upgrade_failure_does_not_free_address_space(): free_mock.assert_not_called() assert operation.steps[-1].status == Status.UpdatingFailed assert operation.status == Status.DeletingFailed + + +async def test_workspace_service_uninstall_address_space_cleanup_fail_then_succeed_restores_primary_resource(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + # 1. First attempt: cleanup fails after retries; message is abandoned for redelivery + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is False + assert step2.status == Status.UpdatingFailed + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + # 2. Redelivery attempt: cleanup succeeds + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert step2.status == Status.Updated + assert operation.status == Status.Deleted + # Primary workspace service must be restored to Deleted, not left as DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.Updated From 1732ec2b2eb3cf8dd1244c251633a22d5e6ea79e Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 14:29:03 +0000 Subject: [PATCH 26/90] Serialize workspace mutations against active cleanup and ensure cleanup idempotency --- api_app/api/routes/workspaces.py | 16 ++- api_app/models/domain/resource.py | 9 +- api_app/resources/strings.py | 1 + .../service_bus/deployment_status_updater.py | 98 ++++++++++------ .../test_api/test_routes/test_workspaces.py | 63 +++++++++++ api_app/tests_ma/test_models/test_resource.py | 20 ++++ .../test_deployment_status_update.py | 106 ++++++++++++++++++ 7 files changed, 278 insertions(+), 35 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index d8af3b265c..1f6f52e813 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -129,6 +129,8 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -157,6 +159,8 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use @workspaces_core_router.delete("/workspaces/{workspace_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def delete_workspace(response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) if await delete_validation(workspace, workspace_repo): operation = await send_uninstall_message( resource=workspace, @@ -175,6 +179,8 @@ async def delete_workspace(response: Response, user=Depends(require_tre_admin), @workspaces_core_router.post("/workspaces/{workspace_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def invoke_action_on_workspace(response: Response, action: str, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace, resource_repo=workspace_repo, @@ -243,6 +249,8 @@ async def retrieve_workspace_service_by_id(workspace_service=Depends(get_workspa @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_CREATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def create_workspace_service(response: Response, workspace_service_input: WorkspaceServiceInCreate, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), workspace=Depends(get_deployed_workspace_by_id_from_path)) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) try: workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) @@ -262,8 +270,6 @@ async def create_workspace_service(response: Response, workspace_service_input: # check workspace has address_spaces property if not workspace.properties.get("address_spaces"): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) - if await operations_repo.resource_has_active_operation(workspace.id): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) workspace_patch = ResourcePatch() workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} @@ -290,6 +296,8 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -316,6 +324,8 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def delete_workspace_service(response: Response, user=Depends(require_workspace_owner), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) if await delete_validation(workspace_service, workspace_service_repo): operation = await send_uninstall_message( resource=workspace_service, @@ -334,6 +344,8 @@ async def delete_workspace_service(response: Response, user=Depends(require_work @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def invoke_action_on_workspace_service(response: Response, action: str, user=Depends(require_workspace_owner), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace_service, resource_repo=workspace_service_repo, diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index 7677f96b13..8a18630b8d 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -1,3 +1,4 @@ +import copy from enum import StrEnum from typing import Optional, Union, List from pydantic import field_validator, BaseModel, Field @@ -80,6 +81,12 @@ def convert_user_to_dict(cls, value): return value def get_resource_request_message_payload(self, operation_id: str, step_id: str, action: RequestAction) -> dict: + parameters = copy.deepcopy(self.properties) if self.properties else {} + if self.resourceType == ResourceType.Workspace: + pending_removal = parameters.pop("address_spaces_pending_removal", []) + if pending_removal and "address_spaces" in parameters and isinstance(parameters["address_spaces"], list): + parameters["address_spaces"] = [a for a in parameters["address_spaces"] if a not in pending_removal] + payload = { "operationId": operation_id, "stepId": step_id, @@ -87,7 +94,7 @@ def get_resource_request_message_payload(self, operation_id: str, step_id: str, "id": self.id, "name": self.templateName, "version": self.templateVersion, - "parameters": self.properties + "parameters": parameters } if self.resourceType == ResourceType.WorkspaceService: diff --git a/api_app/resources/strings.py b/api_app/resources/strings.py index f1a5722263..c58ac279c1 100644 --- a/api_app/resources/strings.py +++ b/api_app/resources/strings.py @@ -183,6 +183,7 @@ RESOURCE_ACTION_UPDATE = "upgrade" RESOURCE_ACTION_UNINSTALL = "uninstall" ADDRESS_SPACE_CLEANUP_STEP_ID = "address-space-cleanup" +ADDRESS_SPACE_CLEANUP_SUCCESS = "address_space_cleanup_success" # Resource Action Status diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index fa318fb620..5eef9a8f18 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -24,6 +24,7 @@ from resources import strings from services.logging import logger, tracer from db.repositories.workspaces import WorkspaceRepository +from db.repositories.workspace_services import WorkspaceServiceRepository from models.schemas.resource import ResourcePatch from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -39,6 +40,7 @@ async def init_repos(self): self.operations_repo = await OperationRepository.create() self.resource_repo = await ResourceRepository.create() self.workspace_repo = await WorkspaceRepository.create() + self.workspace_services_repo = await WorkspaceServiceRepository.create() self.resource_template_repo = await ResourceTemplateRepository.create() self.resource_history_repo = await ResourceHistoryRepository.create() @@ -132,6 +134,34 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() + resource_id = uuid.UUID(step_to_update.resourceId) + + # Is this the address space cleanup step? + is_cleanup_step = ( + step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) + and step_to_update.resourceAction == RequestAction.Upgrade + and operation.action == RequestAction.UnInstall + ) + + # For the address space cleanup step, we MUST run cleanup before persisting the + # final successful operation status. This keeps the operation active in Cosmos DB + # (so resource_has_active_operation blocks new allocations and workspace updates) + # until the cleanup has been durably committed. + if is_cleanup_step and step_to_update.is_success(): + if not await self._free_workspace_address_space(operation): + cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." + step_to_update.status = Status.UpdatingFailed + step_to_update.message = cleanup_failure_message + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + return False + else: + step_to_update.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + # update the overall headline operation status await self.update_overall_operation_status(operation, step_to_update, is_last_step) @@ -139,8 +169,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.operations_repo.update_item(operation) # copy the step status to the resource item, for convenience - resource_id = uuid.UUID(step_to_update.resourceId) - resource = await self.resource_repo.get_resource_by_id(resource_id) resource.deploymentStatus = step_to_update.status await self.resource_repo.update_item(resource) @@ -166,36 +194,19 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.update_overall_operation_status(operation, step_to_update, is_last_step) await self.operations_repo.update_item(operation) - # A workspace upgrade is the point at which the address space is no longer - # in use by Azure, so it can be released for a future allocation. - if (step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID - and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) - and step_to_update.resourceAction == RequestAction.Upgrade - and step_to_update.is_success() - and operation.action == RequestAction.UnInstall): - if not await self._free_workspace_address_space(operation): - cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." - step_to_update.status = Status.UpdatingFailed - step_to_update.message = cleanup_failure_message - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) - resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) - resource_to_update.deploymentStatus = step_to_update.status - await self.resource_repo.update_item(resource_to_update) - return False - else: - # Restore the primary resource to Deleted when cleanup succeeds - main_step = next( - (op_step for op_step in operation.steps - if op_step.templateStepId == "main" - and op_step.resourceId == operation.resourceId), - None - ) - if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - if primary_resource.deploymentStatus != Status.Deleted: - primary_resource.deploymentStatus = Status.Deleted - await self.resource_repo.update_item(primary_resource) + if is_cleanup_step and step_to_update.is_success(): + # Restore the primary resource to Deleted when cleanup succeeds + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) # more steps in the op to do? if is_last_step is False: @@ -309,6 +320,29 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: address_to_free, parent_workspace_id, resource_id = cleanup_details + # Check if cleanup for this step was already completed (idempotency on redelivery) + for step in operation.steps: + if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID and step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS: + return True + + # Check if address_to_free has been assigned to another active workspace service on this workspace + workspace_services_repo = getattr(self, "workspace_services_repo", None) + if workspace_services_repo is None: + try: + from db.repositories.workspace_services import WorkspaceServiceRepository + workspace_services_repo = await WorkspaceServiceRepository.create() + except Exception: + workspace_services_repo = None + + if workspace_services_repo is not None: + try: + active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) + if any(s.id != resource_id and s.properties.get("address_space") == address_to_free for s in active_services): + logger.info(f"Address space {address_to_free} is currently allocated to an active workspace service; skipping removal.") + return True + except Exception as e: + logger.warning(f"Failed to check active workspace services during address space cleanup: {e}") + try: for attempt in range(MAX_CLEANUP_RETRIES): try: diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 88fbf9d6d4..18103fffca 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -684,6 +684,30 @@ async def test_patch_workspace_returns_409_if_bad_etag(self, _, __, update_item_ assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.ETAG_CONFLICT + # [PATCH] /workspaces/{workspace_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_patch_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE, workspace_id=WORKSPACE_ID), json={"isEnabled": False}, headers={"etag": "some-etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_delete_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.delete(app.url_path_for(strings.API_DELETE_WORKSPACE, workspace_id=WORKSPACE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/invoke-action + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_invoke_action_on_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE, workspace_id=WORKSPACE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [DELETE] /workspaces/{workspace_id} @patch("api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", return_value=[sample_workspace().__dict__]) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @@ -821,6 +845,45 @@ async def test_post_workspace_services_creates_workspace_service_with_address_sp assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + async def test_post_workspace_services_raises_409_if_workspace_has_active_operation(self, _, get_workspace_mock, __, app, client, workspace_service_input): + get_workspace_mock.return_value = sample_workspace() + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id}/workspace-services/{service_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_delete_workspace_service_raises_409_if_workspace_has_active_operation(self, _, __, ___, app, client): + response = await client.delete(app.url_path_for(strings.API_DELETE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_patch_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_invoke_action_on_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") diff --git a/api_app/tests_ma/test_models/test_resource.py b/api_app/tests_ma/test_models/test_resource.py index 61963d3256..57e4a4212a 100644 --- a/api_app/tests_ma/test_models/test_resource.py +++ b/api_app/tests_ma/test_models/test_resource.py @@ -8,6 +8,7 @@ from models.domain.resource import Output, Resource, ResourceHistoryItem, ResourceType from models.domain.resource_template import Property from models.domain.user_resource import UserResource +from models.domain.workspace import Workspace from models.domain.workspace_service import WorkspaceService from models.schemas.resource import ResourceHistoryInList from models.schemas.shared_service_template import SharedServiceTemplateInCreate @@ -55,6 +56,25 @@ def test_workspace_service_get_resource_request_message_payload_augments_payload assert message_payload["workspaceId"] == workspace_id +def test_workspace_get_resource_request_message_payload_excludes_pending_removal_address_spaces(): + workspace = Workspace( + id="123", + templateName="workspace-template", + templateVersion="1.0", + etag="", + resourcePath="test", + properties={ + "address_spaces": ["10.0.0.0/22", "10.1.0.0/22"], + "address_spaces_pending_removal": ["10.1.0.0/22"] + } + ) + + message_payload = workspace.get_resource_request_message_payload(OPERATION_ID, STEP_ID, RequestAction.Upgrade) + + assert message_payload["parameters"]["address_spaces"] == ["10.0.0.0/22"] + assert "address_spaces_pending_removal" not in message_payload["parameters"] + + def test_legacy_actor_dicts_validate_without_user_required_fields(): resource = Resource.model_validate({ "id": "resource-id", diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 430005aba6..56fb2e22cc 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1162,3 +1162,109 @@ async def mock_get_resource_by_id(resource_uuid): # Primary workspace service must be restored to Deleted, not left as DeletingFailed assert workspace_service_mock.deploymentStatus == Status.Deleted assert parent_workspace_mock.deploymentStatus == Status.Updated + + +async def test_free_workspace_address_space_idempotent_on_redelivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is True + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_skips_when_address_owned_by_active_service(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + new_active_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + active_service_mock = MagicMock() + active_service_mock.id = new_active_service_id + active_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [active_service_mock] + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is True + status_updater.workspace_repo.patch_workspace.assert_not_called() From 3586aac6c3227c3dfadb24920ecbad1406336e9b Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 14:52:22 +0000 Subject: [PATCH 27/90] Lock workspace from start via resourcePath, keep active status on retry, and fail closed on ownership check --- api_app/db/repositories/operations.py | 4 +- .../service_bus/deployment_status_updater.py | 43 +++---- .../test_operation_repository.py | 1 + .../test_deployment_status_update.py | 119 ++++++++++++++++-- 4 files changed, 138 insertions(+), 29 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 723e10aaaa..bbe7695780 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -207,7 +207,9 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: status_filter = ", ".join(f'"{s}"' for s in active_statuses) query = ( self.operations_query() - + f' (c.resourceId = "{resource_id}" OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true))' + + f' (c.resourceId = "{resource_id}"' + + f' OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true)' + + f' OR CONTAINS(c.resourcePath, "{resource_id}"))' + f' AND c.status IN ({status_filter})' ) operations = await self.query(query=query) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 5eef9a8f18..8d1c5d8cdf 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -151,9 +151,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if is_cleanup_step and step_to_update.is_success(): if not await self._free_workspace_address_space(operation): cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." - step_to_update.status = Status.UpdatingFailed + step_to_update.status = Status.Updating step_to_update.message = cleanup_failure_message - await self.update_overall_operation_status(operation, step_to_update, is_last_step) + operation.status = Status.PipelineRunning await self.operations_repo.update_item(operation) resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) resource_to_update.deploymentStatus = step_to_update.status @@ -221,7 +221,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_to_send = copy.deepcopy(workspace) cleanup_details = await self._get_address_cleanup_details(operation) if cleanup_details: - address_to_free, _, _ = cleanup_details + address_to_free = cleanup_details[0] workspace_address_spaces = resource_to_send.properties.get("address_spaces", []) if address_to_free and isinstance(workspace_address_spaces, list): resource_to_send.properties["address_spaces"] = [ @@ -289,7 +289,7 @@ def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: sourceTemplateResourceId=resource["id"] ) - async def _get_address_cleanup_details(self, operation: Operation) -> Optional[Tuple[str, str, str]]: + async def _get_address_cleanup_details(self, operation: Operation) -> Optional[Tuple[str, str, str, dict]]: main_step = next( (step for step in operation.steps if step.templateStepId == "main" @@ -307,7 +307,7 @@ async def _get_address_cleanup_details(self, operation: Operation) -> Optional[T if not address_to_free or not parent_workspace_id: return None - return address_to_free, parent_workspace_id, resource_id + return address_to_free, parent_workspace_id, resource_id, resource_to_persist async def _free_workspace_address_space(self, operation: Operation) -> bool: """ @@ -318,37 +318,36 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: if cleanup_details is None: return True - address_to_free, parent_workspace_id, resource_id = cleanup_details + address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details # Check if cleanup for this step was already completed (idempotency on redelivery) for step in operation.steps: if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID and step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS: return True - # Check if address_to_free has been assigned to another active workspace service on this workspace - workspace_services_repo = getattr(self, "workspace_services_repo", None) - if workspace_services_repo is None: - try: + # Check if the uninstalled service already has the address_space_freed marker + if service_dict.get("properties", {}).get("address_space_freed"): + return True + + try: + # Check if address_to_free has been assigned to another active workspace service on this workspace (fail closed) + workspace_services_repo = getattr(self, "workspace_services_repo", None) + if workspace_services_repo is None: from db.repositories.workspace_services import WorkspaceServiceRepository workspace_services_repo = await WorkspaceServiceRepository.create() - except Exception: - workspace_services_repo = None - if workspace_services_repo is not None: - try: - active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) - if any(s.id != resource_id and s.properties.get("address_space") == address_to_free for s in active_services): - logger.info(f"Address space {address_to_free} is currently allocated to an active workspace service; skipping removal.") - return True - except Exception as e: - logger.warning(f"Failed to check active workspace services during address space cleanup: {e}") + active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) + if any(s.id != resource_id and s.properties.get("address_space") == address_to_free for s in active_services): + logger.error(f"[ADDRESS_SPACE_CONFLICT] Address space {address_to_free} is allocated to another active service in workspace {parent_workspace_id} but was removed from Azure during cleanup.") + return False - try: for attempt in range(MAX_CLEANUP_RETRIES): try: workspace = await self.workspace_repo.get_workspace_by_id(parent_workspace_id) workspace_address_spaces = workspace.properties.get("address_spaces", []) if address_to_free not in workspace_address_spaces: + service_dict["properties"]["address_space_freed"] = True + await self.resource_repo.update_item_dict(service_dict) return True new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] workspace_patch = ResourcePatch() @@ -366,6 +365,8 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: operation.user, False ) + service_dict["properties"]["address_space_freed"] = True + await self.resource_repo.update_item_dict(service_dict) logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful workspace upgrade for {resource_id}") return True except CosmosAccessConditionFailedError: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 56e8d1c502..e40fab20a7 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -114,6 +114,7 @@ async def test_resource_has_active_operation_returns_true_when_active_operation_ query_str = operations_repo.query.call_args[1]["query"] if "query" in operations_repo.query.call_args[1] else operations_repo.query.call_args[0][0] assert "workspace-123" in query_str assert "ARRAY_CONTAINS" in query_str + assert 'CONTAINS(c.resourcePath, "workspace-123")' in query_str async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 56fb2e22cc..1864abd4a6 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -450,6 +450,7 @@ async def test_convert_outputs_to_dict(): @patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.WorkspaceServiceRepository.create') @patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @@ -463,6 +464,7 @@ async def test_workspace_service_uninstall_frees_address_space( __, ___, workspace_repo_mock, + workspace_service_repo_mock, send_deployment_message_mock ): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -507,6 +509,9 @@ async def test_workspace_service_uninstall_frees_address_space( workspace_repo = AsyncMock() workspace_repo.get_workspace_by_id.return_value = parent_workspace workspace_repo_mock.return_value = workspace_repo + workspace_service_repo = AsyncMock() + workspace_service_repo.get_active_workspace_services_for_workspace.return_value = [] + workspace_service_repo_mock.return_value = workspace_service_repo status_updater = DeploymentStatusUpdater() await status_updater.init_repos() @@ -847,10 +852,11 @@ async def test_address_space_cleanup_failure_is_persisted_before_message_redeliv complete_message = await status_updater.update_status_in_database(message) assert complete_message is False - assert operation.steps[-1].status == Status.UpdatingFailed + assert operation.steps[-1].status == Status.Updating + assert operation.status == Status.PipelineRunning assert "will be retried" in operation.steps[-1].message assert status_updater.operations_repo.update_item.await_args_list[-1].args[0] == operation - assert status_updater.resource_repo.update_item.await_args_list[-1].args[0].deploymentStatus == Status.UpdatingFailed + assert status_updater.resource_repo.update_item.await_args_list[-1].args[0].deploymentStatus == Status.Updating async def test_unrelated_workspace_upgrade_does_not_suppress_address_space_cleanup(): @@ -1147,10 +1153,9 @@ async def mock_get_resource_by_id(resource_uuid): result = await status_updater.update_status_in_database(cleanup_message) assert result is False - assert step2.status == Status.UpdatingFailed - assert operation.status == Status.DeletingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed - assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + assert step2.status == Status.Updating + assert operation.status == Status.PipelineRunning + assert parent_workspace_mock.deploymentStatus == Status.Updating # 2. Redelivery attempt: cleanup succeeds with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): @@ -1214,7 +1219,7 @@ async def test_free_workspace_address_space_idempotent_on_redelivery(): status_updater.workspace_repo.patch_workspace.assert_not_called() -async def test_free_workspace_address_space_skips_when_address_owned_by_active_service(): +async def test_free_workspace_address_space_fails_when_address_owned_by_other_active_service(): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" new_active_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -1266,5 +1271,105 @@ async def test_free_workspace_address_space_skips_when_address_owned_by_active_s result = await status_updater._free_workspace_address_space(operation) + assert result is False + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_fails_closed_when_active_services_query_errors(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.side_effect = Exception("DB error") + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is False + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_idempotent_when_marker_present(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space, "address_space_freed": True} + } + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + assert result is True status_updater.workspace_repo.patch_workspace.assert_not_called() From 0fc006d9e333a53383d53adf6564e8546bc73a83 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 15:08:17 +0000 Subject: [PATCH 28/90] Append fallback step before persisting status and treat address conflicts as terminal failures --- .../service_bus/deployment_status_updater.py | 69 ++++++-- .../test_deployment_status_update.py | 150 +++++++++++++++++- 2 files changed, 199 insertions(+), 20 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 8d1c5d8cdf..9352894ea1 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -32,6 +32,11 @@ MAX_CLEANUP_RETRIES = 3 +class AddressSpaceConflictError(Exception): + """Raised when address space cleanup encounters a terminal conflict with another active service.""" + pass + + class DeploymentStatusUpdater(): def __init__(self): pass @@ -136,6 +141,23 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_id = uuid.UUID(step_to_update.resourceId) + # If this is the main step of an uninstall succeeding for a workspace service with an address space, + # ensure the trailing workspace upgrade fallback step is appended BEFORE calculating + # and persisting the main step's overall status. This ensures is_last_step is False + # and the operation is persisted as PipelineRunning continuously in Cosmos DB, eliminating any + # window where resource_has_active_operation sees the operation as Deleted. + if ( + step_to_update.templateStepId == "main" + and step_to_update.resourceId == operation.resourceId + and step_to_update.is_success() + and operation.action == RequestAction.UnInstall + and not self._has_workspace_upgrade_step(operation, current_step_index) + ): + resource_dict = await self.resource_repo.get_resource_dict_by_id(resource_id) + if self._is_workspace_service_with_address_space(resource_dict): + operation.steps.append(self._create_workspace_upgrade_step(resource_dict)) + is_last_step = False + # Is this the address space cleanup step? is_cleanup_step = ( step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID @@ -149,7 +171,30 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # (so resource_has_active_operation blocks new allocations and workspace updates) # until the cleanup has been durably committed. if is_cleanup_step and step_to_update.is_success(): - if not await self._free_workspace_address_space(operation): + try: + freed = await self._free_workspace_address_space(operation) + except AddressSpaceConflictError as e: + logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") + step_to_update.status = Status.UpdatingFailed + step_to_update.message = f"Terminal address space conflict: {e}" + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + return True + + if not freed: cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." step_to_update.status = Status.Updating step_to_update.message = cleanup_failure_message @@ -182,18 +227,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_to_persist = self.create_updated_resource_document(resource, message) await self.resource_repo.update_item_dict(resource_to_persist) - # Keep the address space reserved until Azure has applied the workspace update. - if (step_to_update.templateStepId == "main" - and step_to_update.resourceId == operation.resourceId - and step_to_update.is_success() - and operation.action == RequestAction.UnInstall - and self._is_workspace_service_with_address_space(resource_to_persist) - and not self._has_workspace_upgrade_step(operation, current_step_index)): - operation.steps.append(self._create_workspace_upgrade_step(resource_to_persist)) - is_last_step = False - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) - if is_cleanup_step and step_to_update.is_success(): # Restore the primary resource to Deleted when cleanup succeeds main_step = next( @@ -337,9 +370,11 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: workspace_services_repo = await WorkspaceServiceRepository.create() active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) - if any(s.id != resource_id and s.properties.get("address_space") == address_to_free for s in active_services): - logger.error(f"[ADDRESS_SPACE_CONFLICT] Address space {address_to_free} is allocated to another active service in workspace {parent_workspace_id} but was removed from Azure during cleanup.") - return False + conflicting_services = [s for s in active_services if s.id != resource_id and s.properties.get("address_space") == address_to_free] + if conflicting_services: + raise AddressSpaceConflictError( + f"Address space {address_to_free} is allocated to active service {conflicting_services[0].id} in workspace {parent_workspace_id}." + ) for attempt in range(MAX_CLEANUP_RETRIES): try: @@ -373,6 +408,8 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: if attempt == MAX_CLEANUP_RETRIES - 1: raise logger.warning(f"ETag conflict when freeing workspace address space after successful workspace upgrade. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + except AddressSpaceConflictError: + raise except Exception as e: logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after upgrading the workspace for {resource_id}: {e}", exc_info=True) return False diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 1864abd4a6..97c2f741a1 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -15,7 +15,7 @@ from models.domain.workspace import Workspace from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings -from service_bus.deployment_status_updater import DeploymentStatusUpdater +from service_bus.deployment_status_updater import DeploymentStatusUpdater, AddressSpaceConflictError pytestmark = pytest.mark.asyncio @@ -1269,9 +1269,8 @@ async def test_free_workspace_address_space_fails_when_address_owned_by_other_ac status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [active_service_mock] status_updater.workspace_repo = AsyncMock() - result = await status_updater._free_workspace_address_space(operation) - - assert result is False + with pytest.raises(AddressSpaceConflictError): + await status_updater._free_workspace_address_space(operation) status_updater.workspace_repo.patch_workspace.assert_not_called() @@ -1373,3 +1372,146 @@ async def test_free_workspace_address_space_idempotent_when_marker_present(): assert result is True status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_terminal_address_space_conflict_marks_operation_and_resource_failed_and_completes_message(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", side_effect=AddressSpaceConflictError("Conflict")): + result = await status_updater.update_status_in_database(cleanup_message) + + # Terminal conflict: completes message (returns True) so it is not endlessly retried + assert result is True + assert step2.status == Status.UpdatingFailed + assert "Terminal address space conflict" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_legacy_template_appends_workspace_upgrade_step_before_persisting_main_status(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.resource_template_repo = AsyncMock() + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + # Verify fallback step was appended + assert len(operation.steps) == 2 + assert operation.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + # Operation was never saved as Deleted; it was saved as PipelineRunning + first_saved_op = status_updater.operations_repo.update_item.call_args_list[0].args[0] + assert first_saved_op.status == Status.PipelineRunning From 69dded368310eaa00c027ca935cdeddf553036b4 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 15:38:57 +0000 Subject: [PATCH 29/90] Validate ownership before enqueue, make enqueue retryable, and handle final delivery --- .../service_bus/deployment_status_updater.py | 103 ++++++- .../test_deployment_status_update.py | 272 ++++++++++++++++++ 2 files changed, 363 insertions(+), 12 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 9352894ea1..19af7dad4b 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -30,6 +30,7 @@ MAX_CLEANUP_RETRIES = 3 +MAX_CLEANUP_DELIVERY_COUNT = 10 class AddressSpaceConflictError(Exception): @@ -98,7 +99,10 @@ async def process_message(self, msg) -> bool: current_span.set_attribute("operation_id", str(message.operationId)) current_span.set_attribute("status", str(message.status)) - complete_message = await self.update_status_in_database(message) + delivery_count = getattr(msg, "delivery_count", 1) + is_final_delivery = delivery_count >= MAX_CLEANUP_DELIVERY_COUNT + + complete_message = await self.update_status_in_database(message, is_final_delivery=is_final_delivery) logger.info(f"Update status in DB for {message.operationId} - {message.status}") except (json.JSONDecodeError, ValidationError): logger.exception(f"{strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT}: {msg.correlation_id}") @@ -107,7 +111,7 @@ async def process_message(self, msg) -> bool: return complete_message - async def update_status_in_database(self, message: DeploymentStatusUpdateMessage): + async def update_status_in_database(self, message: DeploymentStatusUpdateMessage, is_final_delivery: bool = False): """ Get the operation the message references, and find the step within the operation that is to be updated Update the status of the step. If it's a single step operation, copy the status into the operation status. If it's a multi step, @@ -195,15 +199,37 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return True if not freed: - cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." - step_to_update.status = Status.Updating - step_to_update.message = cleanup_failure_message - operation.status = Status.PipelineRunning - await self.operations_repo.update_item(operation) - resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) - resource_to_update.deploymentStatus = step_to_update.status - await self.resource_repo.update_item(resource_to_update) - return False + if is_final_delivery: + cleanup_failure_message = f"Address space cleanup failed after {MAX_CLEANUP_DELIVERY_COUNT} delivery attempts. Marking operation as failed to unblock workspace." + logger.error(f"[ADDRESS_SPACE_CLEANUP_MAX_DELIVERIES] {cleanup_failure_message}") + step_to_update.status = Status.UpdatingFailed + step_to_update.message = cleanup_failure_message + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + return True + else: + cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." + step_to_update.status = Status.Updating + step_to_update.message = cleanup_failure_message + operation.status = Status.PipelineRunning + await self.operations_repo.update_item(operation) + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + return False else: step_to_update.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS @@ -254,7 +280,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_to_send = copy.deepcopy(workspace) cleanup_details = await self._get_address_cleanup_details(operation) if cleanup_details: - address_to_free = cleanup_details[0] + address_to_free, parent_workspace_id, service_resource_id, _ = cleanup_details + # Validate active-service ownership before enqueueing this upgrade + workspace_services_repo = getattr(self, "workspace_services_repo", None) + if workspace_services_repo is None: + from db.repositories.workspace_services import WorkspaceServiceRepository + workspace_services_repo = await WorkspaceServiceRepository.create() + + active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) + conflicting_services = [s for s in active_services if s.id != service_resource_id and s.properties.get("address_space") == address_to_free] + if conflicting_services: + raise AddressSpaceConflictError( + f"Address space {address_to_free} is allocated to active service {conflicting_services[0].id} in workspace {parent_workspace_id}." + ) + workspace_address_spaces = resource_to_send.properties.get("address_spaces", []) if address_to_free and isinstance(workspace_address_spaces, list): resource_to_send.properties["address_spaces"] = [ @@ -280,8 +319,48 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage logger.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=next_step.id, action=next_step.resourceAction)) await send_deployment_message(content=content, correlation_id=operation.id, session_id=resource_to_send.id, action=next_step.resourceAction) + except AddressSpaceConflictError as e: + logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") + next_step.message = f"Terminal address space conflict: {e}" + next_step.status = Status.UpdatingFailed + await self.update_overall_operation_status(operation, next_step, is_last_step=True) + await self.operations_repo.update_item(operation) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + return True except Exception as e: logger.exception("Unable to send update for resource in pipeline step") + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if is_final_delivery: + next_step.message = f"Failed to enqueue address space cleanup after {MAX_CLEANUP_DELIVERY_COUNT} deliveries: {e}" + next_step.status = Status.UpdatingFailed + await self.update_overall_operation_status(operation, next_step, is_last_step=True) + await self.operations_repo.update_item(operation) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + return True + else: + next_step.status = Status.AwaitingUpdate + next_step.message = f"Failed to enqueue address space cleanup, will retry: {e}" + operation.status = Status.PipelineRunning + await self.operations_repo.update_item(operation) + return False next_step.message = repr(e) next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 97c2f741a1..f4db038d8c 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1515,3 +1515,275 @@ async def test_legacy_template_appends_workspace_upgrade_step_before_persisting_ # Operation was never saved as Deleted; it was saved as PipelineRunning first_saved_op = status_updater.operations_repo.update_item.call_args_list[0].args[0] assert first_saved_op.status == Status.PipelineRunning + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_enqueue_cleanup_validates_ownership_and_aborts_before_azure_send(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + other_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + other_service_mock = MagicMock() + other_service_mock.id = other_service_id + other_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [other_service_mock] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + # Completed so it does not redeliver endlessly + assert result is True + # send_deployment_message was NOT called: Terraform was NOT told to remove the address from Azure + send_deployment_message_mock.assert_not_called() + assert operation.status == Status.DeletingFailed + assert operation.steps[1].status == Status.UpdatingFailed + assert "Terminal address space conflict" in operation.steps[1].message + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + + +@patch('service_bus.deployment_status_updater.send_deployment_message', side_effect=Exception("Service Bus send failed")) +async def test_enqueue_cleanup_send_failure_abandons_message_for_retry(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=False) + + # Abandons message so Service Bus redelivers and retries enqueue + assert result is False + assert operation.status == Status.PipelineRunning + assert operation.steps[1].status == Status.AwaitingUpdate + assert "will retry" in operation.steps[1].message + + +@patch('service_bus.deployment_status_updater.send_deployment_message', side_effect=Exception("Service Bus send failed")) +async def test_enqueue_cleanup_send_failure_on_final_delivery_marks_failed(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=True) + + # Completes message on final delivery so workspace is unblocked + assert result is True + assert operation.status == Status.DeletingFailed + assert operation.steps[1].status == Status.UpdatingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + + +async def test_cleanup_failure_on_final_delivery_marks_failed_and_completes_message(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + # On final delivery, completes message and marks operation failed to unblock workspace + assert result is True + assert step2.status == Status.UpdatingFailed + assert "delivery attempts" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed From 624de536df25fbd332e0e4b3a96a7c3c23018c90 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 16:04:29 +0000 Subject: [PATCH 30/90] revert service bus changes in favour of PR 4930 --- api_app/core/config.py | 1 - .../service_bus/deployment_status_updater.py | 59 +++++++++++-------- .../test_deployment_status_update.py | 24 -------- 3 files changed, 34 insertions(+), 50 deletions(-) diff --git a/api_app/core/config.py b/api_app/core/config.py index 2e0f3ecdd2..d2f1cf1fa4 100644 --- a/api_app/core/config.py +++ b/api_app/core/config.py @@ -41,7 +41,6 @@ SERVICE_BUS_RESOURCE_REQUEST_QUEUE: str = config("SERVICE_BUS_RESOURCE_REQUEST_QUEUE", default="") SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE: str = config("SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE", default="") SERVICE_BUS_STEP_RESULT_QUEUE: str = config("SERVICE_BUS_STEP_RESULT_QUEUE", default="") -SERVICE_BUS_MAX_WAIT_TIME: int = config("SERVICE_BUS_MAX_WAIT_TIME", cast=int, default=10) # Event grid configuration EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT: str = config("EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT", default="") diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 19af7dad4b..996dc33a43 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -57,37 +57,46 @@ async def receive_messages(self): with tracer.start_as_current_span("deployment_status_receive_messages"): last_heartbeat_time = 0 polling_count = 0 + while True: - complete_message = True try: + current_time = time.time() + polling_count += 1 + # Log a heartbeat message every 60 seconds to show the service is still working + if current_time - last_heartbeat_time >= 60: + logger.info(f"Queue reader heartbeat: Polled {config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE} queue {polling_count} times in the last minute") + last_heartbeat_time = current_time + polling_count = 0 + async with credentials.get_credential_async_context() as credential: - async with ServiceBusClient(fully_qualified_namespace=config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential=credential) as service_bus_client: - logger.debug("Creating Deployment Status receiver session") - async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=config.SERVICE_BUS_MAX_WAIT_TIME, session_id=NEXT_AVAILABLE_SESSION) as receiver: - async with AutoLockRenewer() as renewer: - renewer.register(receiver, receiver.session, max_lock_renewal_duration=60) + service_bus_client = ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) + + logger.debug(f"Looking for new messages on {config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE} queue...") + # max_wait_time=1 -> don't hold the session open after processing of the message has finished + async with service_bus_client.get_queue_receiver(queue_name=config.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE, max_wait_time=1, session_id=NEXT_AVAILABLE_SESSION) as receiver: + logger.info(f"Got a session containing messages: {receiver.session.session_id}") + async with AutoLockRenewer() as renewer: + renewer.register(receiver, receiver.session, max_lock_renewal_duration=60) + async for msg in receiver: + complete_message = await self.process_message(msg) + if complete_message: + await receiver.complete_message(msg) + else: + # could have been any kind of transient issue, we'll abandon back to the queue, and retry + await receiver.abandon_message(msg) + logger.info(f"Closing session: {receiver.session.session_id}") - async for msg in receiver: - complete_message = await self.process_message(msg) - if complete_message: - await receiver.complete_message(msg) - else: - await receiver.abandon_message(msg) + except OperationTimeoutError: + # Timeout occurred whilst connecting to a session - this is expected and indicates no non-empty sessions are available + logger.debug("No sessions for this process. Will look again...") - polling_count = 0 + except ServiceBusConnectionError: + # Occasionally there will be a transient / network-level error in connecting to SB. + logger.info("Unknown Service Bus connection error. Will retry...") - except OperationTimeoutError: - polling_count += 1 - # log a heartbeat every ~5 minutes when queue is idle (max_wait_time is usually ~10s) - if time.time() - last_heartbeat_time > 300: - logger.info(f"Deployment status updater polling... (idle checks: {polling_count})") - last_heartbeat_time = time.time() - except ServiceBusConnectionError as e: - logger.warning(f"Service Bus connection error in deployment status updater, will retry: {e}") - await asyncio.sleep(5) - except Exception: - logger.exception("Unexpected error in deployment status receiver loop") - await asyncio.sleep(5) + except Exception as e: + # Catch all other exceptions, log them via .exception to get the stack trace, and reconnect + logger.exception(f"Unknown exception. Will retry - {e}") async def process_message(self, msg) -> bool: complete_message = False diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index f4db038d8c..b186e4328b 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1,4 +1,3 @@ -import asyncio import copy import json from unittest.mock import MagicMock, ANY @@ -1014,29 +1013,6 @@ async def mock_send_deployment_message(*args, **kwargs): assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22"] -@patch("service_bus.deployment_status_updater.config") -@patch("service_bus.deployment_status_updater.credentials") -@patch("service_bus.deployment_status_updater.logger") -async def test_receive_messages_handles_credential_context_failure(logger_mock, credentials_mock, config_mock): - config_mock.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE = "test_namespace" - config_mock.SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE = "test_queue" - config_mock.SERVICE_BUS_MAX_WAIT_TIME = 10 - - # Simulate exception when entering credential context manager - credentials_mock.get_credential_async_context.side_effect = Exception("Credential error") - - status_updater = DeploymentStatusUpdater() - - # Run receive_messages in a task and cancel after one loop iteration - task = asyncio.create_task(status_updater.receive_messages()) - await asyncio.sleep(0.1) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - logger_mock.exception.assert_called_with("Unexpected error in deployment status receiver loop") - - async def test_workspace_upgrade_failure_does_not_free_address_space(): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" From 95e9305635aa0945622ebe5c0f14657ed61b86f0 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 3 Sep 2026 16:20:48 +0000 Subject: [PATCH 31/90] Adjust delivery count handling for final cleanup in DeploymentStatusUpdater and add tests for delivery count scenarios --- .../service_bus/deployment_status_updater.py | 10 ++++++-- .../test_deployment_status_update.py | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 996dc33a43..f74e1c6e8f 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -31,6 +31,11 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 +# Azure Service Bus default max delivery count is 10. +# The AMQP delivery_count reported by ServiceBusReceivedMessage is zero-based: +# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. +# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 class AddressSpaceConflictError(Exception): @@ -108,8 +113,9 @@ async def process_message(self, msg) -> bool: current_span.set_attribute("operation_id", str(message.operationId)) current_span.set_attribute("status", str(message.status)) - delivery_count = getattr(msg, "delivery_count", 1) - is_final_delivery = delivery_count >= MAX_CLEANUP_DELIVERY_COUNT + raw_delivery_count = getattr(msg, "delivery_count", 0) + delivery_count = raw_delivery_count if raw_delivery_count is not None else 0 + is_final_delivery = delivery_count >= FINAL_CLEANUP_DELIVERY_COUNT complete_message = await self.update_status_in_database(message, is_final_delivery=is_final_delivery) logger.info(f"Update status in DB for {message.operationId} - {message.status}") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index b186e4328b..2056d04543 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -70,10 +70,11 @@ class ServiceBusReceivedMessageMock: - def __init__(self, message: dict): + def __init__(self, message: dict, delivery_count: int = 0): self.message = json.dumps(message) self.correlation_id = "test_correlation_id" self.session_id = "test_session_id" + self.delivery_count = delivery_count def __str__(self): return self.message @@ -1763,3 +1764,24 @@ async def mock_get_resource_by_id(resource_uuid): assert operation.status == Status.DeletingFailed assert workspace_service_mock.deploymentStatus == Status.DeletingFailed assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +@pytest.mark.parametrize("delivery_count, expected_final", [ + (0, False), + (1, False), + (8, False), + (9, True), + (10, True), + (None, False), +]) +async def test_process_message_delivery_count_handling(delivery_count, expected_final): + status_updater = DeploymentStatusUpdater() + status_updater.update_status_in_database = AsyncMock(return_value=True) + + msg = ServiceBusReceivedMessageMock(test_sb_message, delivery_count=delivery_count) + + await status_updater.process_message(msg) + + status_updater.update_status_in_database.assert_awaited_once() + _, kwargs = status_updater.update_status_in_database.call_args + assert kwargs.get("is_final_delivery") is expected_final From 4b2b20f2a22f07b4f073bba389dfecab36335765 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 4 Sep 2026 08:26:01 +0000 Subject: [PATCH 32/90] Guard completed address space cleanup against redelivered status messages --- .../service_bus/deployment_status_updater.py | 46 +++++- .../test_deployment_status_update.py | 152 ++++++++++++++++++ 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index f74e1c6e8f..bfdbf863fb 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -160,6 +160,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_id = uuid.UUID(step_to_update.resourceId) + # If address space cleanup for this operation has already completed, + # do not reopen the operation or re-enqueue cleanup on redelivered main-step messages. + if ( + step_to_update.templateStepId == "main" + and step_to_update.is_success() + and operation.action == RequestAction.UnInstall + and self._is_cleanup_completed(operation) + ): + logger.info( + f"Address space cleanup already completed for operation {operation.id}. " + f"Skipping redelivery of step {message.stepId} to prevent reopening operation." + ) + return True + # If this is the main step of an uninstall succeeding for a workspace service with an address space, # ensure the trailing workspace upgrade fallback step is appended BEFORE calculating # and persisting the main step's overall status. This ensures is_last_step is False @@ -291,6 +305,17 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage try: # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if ( + self._is_cleanup_completed(operation) + or next_step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + or next_step.status == Status.Updating + ): + logger.info( + f"Address space cleanup step {next_step.id} in operation {operation.id} " + f"is already completed or in progress. Skipping enqueue." + ) + return True + workspace = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) resource_to_send = copy.deepcopy(workspace) cleanup_details = await self._get_address_cleanup_details(operation) @@ -403,6 +428,16 @@ def _has_workspace_upgrade_step(self, operation: Operation, current_step_index: for step in operation.steps[current_step_index + 1:] ) + def _is_cleanup_completed(self, operation: Operation) -> bool: + for step in operation.steps: + if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS: + return True + if operation.action == RequestAction.UnInstall and operation.status == Status.Deleted: + if any(step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps): + return True + return False + def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: return OperationStep( id=str(uuid.uuid4()), @@ -448,9 +483,8 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details # Check if cleanup for this step was already completed (idempotency on redelivery) - for step in operation.steps: - if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID and step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS: - return True + if self._is_cleanup_completed(operation): + return True # Check if the uninstalled service already has the address_space_freed marker if service_dict.get("properties", {}).get("address_space_freed"): @@ -519,6 +553,12 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.message = step.message return + # If address space cleanup has completed for an uninstall, do not reopen to PipelineRunning + if self._is_cleanup_completed(operation) and operation.action == RequestAction.UnInstall: + operation.status = Status.Deleted + operation.message = "Multi step pipeline completed successfully" + return + operation.status = Status.PipelineRunning operation.message = "Multi step pipeline running. See steps for details." diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 2056d04543..2bf3dfb1e6 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1785,3 +1785,155 @@ async def test_process_message_delivery_count_handling(delivery_count, expected_ status_updater.update_status_in_database.assert_awaited_once() _, kwargs = status_updater.update_status_in_database.call_args assert kwargs.get("is_final_delivery") is expected_final + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_does_not_rerun_completed_cleanup_or_fail_on_reallocated_address(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + new_active_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + new_active_service_mock = MagicMock() + new_active_service_mock.id = new_active_service_id + new_active_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space, "address_space_freed": True} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + # The address space has been reallocated to a new active service + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [new_active_service_mock] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + # Operation and resource must remain successfully Deleted, not failed or reopened + assert operation.status == Status.Deleted + assert step2.status == Status.Updated + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + assert workspace_service_mock.deploymentStatus == Status.Deleted + status_updater.operations_repo.update_item.assert_not_called() + send_deployment_message_mock.assert_not_called() + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_skips_enqueue_when_cleanup_in_progress(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + message="" + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + send_deployment_message_mock.assert_not_called() From 9e8207aa047f925b8787be8a7d8e8b2d69e6ccbf Mon Sep 17 00:00:00 2001 From: James Chapman Date: Fri, 4 Sep 2026 09:00:04 +0000 Subject: [PATCH 33/90] Resolve cleanup redelivery, terminal failure, and concurrent workspace write consistency issues --- .../service_bus/deployment_status_updater.py | 96 +++- .../test_deployment_status_update.py | 447 ++++++++++++++++++ 2 files changed, 518 insertions(+), 25 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index bfdbf863fb..e7981b3609 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -153,27 +153,41 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") - # update the step status - step_to_update.status = message.status - step_to_update.message = message.message - step_to_update.updatedWhen = get_timestamp() - - resource_id = uuid.UUID(step_to_update.resourceId) - - # If address space cleanup for this operation has already completed, + # If address space cleanup for this operation is already in a terminal state (completed or failed), # do not reopen the operation or re-enqueue cleanup on redelivered main-step messages. if ( step_to_update.templateStepId == "main" - and step_to_update.is_success() + and (step_to_update.is_success() or message.status in [Status.Deployed, Status.Deleted, Status.Updated, Status.ActionSucceeded]) and operation.action == RequestAction.UnInstall - and self._is_cleanup_completed(operation) + and self._is_cleanup_terminal(operation) ): logger.info( - f"Address space cleanup already completed for operation {operation.id}. " + f"Address space cleanup already in terminal state for operation {operation.id}. " f"Skipping redelivery of step {message.stepId} to prevent reopening operation." ) return True + # If the uninstall cleanup is already terminal, ignore delayed or duplicate cleanup-step messages + # (such as an intermediate Updating or delayed UpdatingFailed) to prevent writing stale status + # to the workspace resource or reopening the operation. + if ( + step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + and operation.action == RequestAction.UnInstall + and self._is_cleanup_terminal(operation) + ): + logger.info( + f"Cleanup step message {message.stepId} with status {message.status} received for operation {operation.id}, " + f"but uninstall cleanup is already terminal. Ignoring to prevent stale writes." + ) + return True + + # update the step status + step_to_update.status = message.status + step_to_update.message = message.message + step_to_update.updatedWhen = get_timestamp() + + resource_id = uuid.UUID(step_to_update.resourceId) + # If this is the main step of an uninstall succeeding for a workspace service with an address space, # ensure the trailing workspace upgrade fallback step is appended BEFORE calculating # and persisting the main step's overall status. This ensures is_last_step is False @@ -210,8 +224,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") step_to_update.status = Status.UpdatingFailed step_to_update.message = f"Terminal address space conflict: {e}" - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) resource_to_update.deploymentStatus = step_to_update.status await self.resource_repo.update_item(resource_to_update) @@ -225,6 +237,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) primary_resource.deploymentStatus = Status.DeletingFailed await self.resource_repo.update_item(primary_resource) + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) return True if not freed: @@ -233,8 +247,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage logger.error(f"[ADDRESS_SPACE_CLEANUP_MAX_DELIVERIES] {cleanup_failure_message}") step_to_update.status = Status.UpdatingFailed step_to_update.message = cleanup_failure_message - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) resource_to_update.deploymentStatus = step_to_update.status await self.resource_repo.update_item(resource_to_update) @@ -248,26 +260,22 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) primary_resource.deploymentStatus = Status.DeletingFailed await self.resource_repo.update_item(primary_resource) + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) return True else: cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." step_to_update.status = Status.Updating step_to_update.message = cleanup_failure_message - operation.status = Status.PipelineRunning - await self.operations_repo.update_item(operation) resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) resource_to_update.deploymentStatus = step_to_update.status await self.resource_repo.update_item(resource_to_update) + operation.status = Status.PipelineRunning + await self.operations_repo.update_item(operation) return False else: step_to_update.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS - # update the overall headline operation status - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - - # save the operation - await self.operations_repo.update_item(operation) - # copy the step status to the resource item, for convenience resource = await self.resource_repo.get_resource_by_id(resource_id) resource.deploymentStatus = step_to_update.status @@ -275,6 +283,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. if not step_to_update.is_success(): + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) return True # update the resource doc to persist any outputs @@ -296,6 +306,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage primary_resource.deploymentStatus = Status.Deleted await self.resource_repo.update_item(primary_resource) + # update the overall headline operation status + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + + # save the operation with final status LAST, after all resource updates are committed + await self.operations_repo.update_item(operation) + # more steps in the op to do? if is_last_step is False: assert current_step_index < (len(operation.steps) - 1) @@ -306,9 +322,10 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: if ( - self._is_cleanup_completed(operation) + self._is_cleanup_terminal(operation) or next_step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS - or next_step.status == Status.Updating + or next_step.is_failure() + or next_step.status in (Status.Updating, Status.UpdatingFailed) ): logger.info( f"Address space cleanup step {next_step.id} in operation {operation.id} " @@ -438,6 +455,19 @@ def _is_cleanup_completed(self, operation: Operation) -> bool: return True return False + def _is_cleanup_failed(self, operation: Operation) -> bool: + for step in operation.steps: + if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if step.is_failure() or step.status == Status.UpdatingFailed: + return True + if operation.action == RequestAction.UnInstall and operation.status == Status.DeletingFailed: + if any(step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps): + return True + return False + + def _is_cleanup_terminal(self, operation: Operation) -> bool: + return self._is_cleanup_completed(operation) or self._is_cleanup_failed(operation) + def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: return OperationStep( id=str(uuid.uuid4()), @@ -559,6 +589,22 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.message = "Multi step pipeline completed successfully" return + # If address space cleanup has failed terminally for an uninstall, do not reopen to PipelineRunning + if self._is_cleanup_failed(operation) and operation.action == RequestAction.UnInstall: + operation.status = Status.DeletingFailed + operation.message = f"Multi step pipeline failed on step {strings.ADDRESS_SPACE_CLEANUP_STEP_ID}" + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != operation.status: + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + return + operation.status = Status.PipelineRunning operation.message = "Multi step pipeline running. See steps for details." diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 2bf3dfb1e6..6ff603b481 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1937,3 +1937,450 @@ async def test_redelivered_main_status_skips_enqueue_when_cleanup_in_progress(se assert result is True send_deployment_message_mock.assert_not_called() + + +async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success(): + """ + Addresses review comment 1: + Cleanup-status redeliveries are not idempotent after success. If a delayed duplicate + Updating or UpdatingFailed message arrives after the operation is Deleted, ignore it + to avoid writing stale status to the workspace resource. + """ + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + + # Delayed / duplicate Updating message arrives for the cleanup step + delayed_cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updating, + message="in progress..." + ) + + result = await status_updater.update_status_in_database(delayed_cleanup_message) + + assert result is True + # Verify no stale updates written to workspace resource or operation + status_updater.resource_repo.update_item.assert_not_called() + status_updater.resource_repo.update_item_dict.assert_not_called() + status_updater.operations_repo.update_item.assert_not_called() + assert operation.status == Status.Deleted + assert step2.status == Status.Updated + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send_deployment_message_mock): + """ + Addresses review comment 3: + Redelivered main-step message reopening terminal cleanup failure: _is_cleanup_completed was false + for UpdatingFailed/DeletingFailed. Redelivery of the main-step message must not reset operation.status + from DeletingFailed back to PipelineRunning or re-enqueue cleanup. + """ + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.UpdatingFailed, + message="Cleanup failed terminally" + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.DeletingFailed, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.DeletingFailed + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + # Operation and resource must remain DeletingFailed, not reopened to PipelineRunning + assert operation.status == Status.DeletingFailed + assert step2.status == Status.UpdatingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + status_updater.operations_repo.update_item.assert_not_called() + send_deployment_message_mock.assert_not_called() + + +async def test_cleanup_success_persists_workspace_and_primary_resource_before_operation_deleted(): + """ + Addresses review comment 2: + Keep the operation active (PipelineRunning) until all workspace and primary resource writes + are complete, persisting operation with final status (Deleted) last. + """ + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + async def tracking_resource_dict_update(res_dict): + call_order.append(("resource_repo.update_item_dict", res_dict.get("id"))) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": parent_workspace_id, + "resourceType": ResourceType.Workspace, + "properties": {"address_spaces": [address_space]} + } + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.resource_repo.update_item_dict = AsyncMock(side_effect=tracking_resource_dict_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert operation.status == Status.Deleted + + # Check that operations_repo.update_item with Status.Deleted was the LAST call + assert len(call_order) >= 3 + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.Deleted) + + # All resource_repo calls must have occurred BEFORE operations_repo.update_item + resource_calls = [i for i, call in enumerate(call_order) if call[0].startswith("resource_repo")] + for r_idx in resource_calls: + assert r_idx < op_update_index + + +async def test_cleanup_conflict_persists_workspace_and_primary_resource_before_operation_failed(): + """ + Addresses review comment 2 (failure path): + In AddressSpaceConflictError, update workspace and primary resource before saving operation as DeletingFailed. + """ + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", side_effect=AddressSpaceConflictError("Conflict")): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert operation.status == Status.DeletingFailed + + # Verify operations_repo.update_item with Status.DeletingFailed was called AFTER resource updates + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) + + resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] + assert len(resource_calls) == 2 # workspace and primary resource + for r_idx in resource_calls: + assert r_idx < op_update_index + + +async def test_cleanup_final_delivery_persists_workspace_and_primary_resource_before_operation_failed(): + """ + Addresses review comment 2 (final delivery failure path): + On final delivery cleanup failure, update workspace and primary resource before saving operation as DeletingFailed. + """ + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert operation.status == Status.DeletingFailed + + # Verify operations_repo.update_item with Status.DeletingFailed was called AFTER resource updates + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) + + resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] + assert len(resource_calls) == 2 # workspace and primary resource + for r_idx in resource_calls: + assert r_idx < op_update_index From 4ac97bf8317ec4ab896cebe70f248a3b6ae4ab26 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:41:01 +0000 Subject: [PATCH 34/90] Enhance deployment message handling with deterministic message IDs and update Service Bus queue settings for duplicate detection --- .../service_bus/deployment_status_updater.py | 12 +- api_app/service_bus/helpers.py | 25 ++- .../test_deployment_status_update.py | 179 ++++++++++++++++++ core/terraform/servicebus.tf | 6 +- core/version.txt | 2 +- 5 files changed, 214 insertions(+), 10 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index e7981b3609..4016d8be27 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -369,13 +369,17 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage primary_action=operation.action, user=operation.user) - if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: - await self.operations_repo.update_item(operation) - # create + send the message logger.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=next_step.id, action=next_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=resource_to_send.id, action=next_step.resourceAction) + step_message_id = f"{operation.id}_{next_step.id}" + await send_deployment_message( + content=content, + correlation_id=operation.id, + session_id=resource_to_send.id, + action=next_step.resourceAction, + message_id=step_message_id, + ) except AddressSpaceConflictError as e: logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") next_step.message = f"Terminal address space conflict: {e}" diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index e1b2daf74d..ca27fbfee5 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -1,3 +1,5 @@ +import json +from typing import Optional from azure.servicebus import ServiceBusMessage from azure.servicebus.aio import ServiceBusClient from pydantic import TypeAdapter @@ -35,9 +37,26 @@ async def _send_message(message: ServiceBusMessage, queue: str): await sender.send_messages(message) -async def send_deployment_message(content, correlation_id, session_id, action): - resource_request_message = ServiceBusMessage(body=content, correlation_id=correlation_id, session_id=session_id) - logger.info(f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, action: {action}") +async def send_deployment_message(content, correlation_id, session_id, action, message_id: Optional[str] = None): + if not message_id: + try: + payload = content if isinstance(content, dict) else json.loads(content) + op_id = payload.get("operationId", correlation_id) + step_id = payload.get("stepId") + if op_id and step_id: + message_id = f"{op_id}_{step_id}" + except Exception: + pass + if not message_id: + message_id = f"{correlation_id}_{action}" + + resource_request_message = ServiceBusMessage( + body=content, + correlation_id=correlation_id, + session_id=session_id, + message_id=message_id, + ) + logger.info(f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, message ID {resource_request_message.message_id}, action: {action}") await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 6ff603b481..571b0d0b32 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -2384,3 +2384,182 @@ async def tracking_resource_update(res): assert len(resource_calls) == 2 # workspace and primary resource for r_idx in resource_calls: assert r_idx < op_update_index + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_cleanup_dispatch_uses_deterministic_message_id(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + status_before_send = None + + async def mock_send_deployment_message(*args, **kwargs): + nonlocal status_before_send + status_before_send = operation.steps[1].status + + send_deployment_message_mock.side_effect = mock_send_deployment_message + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert len(operation.steps) == 2 + cleanup_step = operation.steps[1] + assert cleanup_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + assert status_before_send == Status.AwaitingUpdate + + send_deployment_message_mock.assert_called_once() + assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step.id}" + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_before_updating_dispatches_with_same_deterministic_message_id(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + cleanup_step_id = "step-2-cleanup-id" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id=cleanup_step_id, + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + send_deployment_message_mock.assert_called_once() + assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step_id}" + + +@patch('service_bus.helpers._send_message') +async def test_send_deployment_message_attaches_deterministic_message_id(_send_message_mock): + from service_bus.helpers import send_deployment_message + + # Derived from JSON payload with operationId and stepId + payload = {"operationId": "op-123", "stepId": "step-456", "action": "upgrade"} + await send_deployment_message( + content=json.dumps(payload), + correlation_id="op-123", + session_id="session-1", + action="upgrade" + ) + sent_msg = _send_message_mock.call_args.args[0] + assert sent_msg.message_id == "op-123_step-456" + + # Explicit message_id takes precedence + await send_deployment_message( + content=json.dumps(payload), + correlation_id="op-123", + session_id="session-1", + action="upgrade", + message_id="custom-id" + ) + sent_msg = _send_message_mock.call_args.args[0] + assert sent_msg.message_id == "custom-id" + + # Fallback when payload lacks stepId + await send_deployment_message( + content=json.dumps({"some": "data"}), + correlation_id="op-789", + session_id="session-2", + action="install" + ) + sent_msg = _send_message_mock.call_args.args[0] + assert sent_msg.message_id == "op-789_install" diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..a7336264c9 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,8 +54,10 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + requires_duplicate_detection = true + duplicate_detection_history_time_window = "PT4H" } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 12d80d0630..670ad2c1cf 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.16.18" From 7d830ecee081fbc71a8afda8426d24420cec185c Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:13:31 +0000 Subject: [PATCH 35/90] Refactor workspace service operations and update delivery count handling in deployment status updater --- api_app/api/routes/workspaces.py | 4 ---- api_app/db/repositories/operations.py | 2 +- api_app/models/domain/resource.py | 9 +-------- .../service_bus/deployment_status_updater.py | 12 +++++------ .../test_api/test_routes/test_workspaces.py | 20 ------------------- .../test_airlock_request_repository.py | 8 ++++++-- .../test_operation_repository.py | 1 + api_app/tests_ma/test_models/test_resource.py | 20 ------------------- .../test_deployment_status_update.py | 5 +++-- core/terraform/servicebus.tf | 6 ++---- core/version.txt | 2 +- 11 files changed, 21 insertions(+), 68 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 1f6f52e813..bee1d1bda0 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -296,8 +296,6 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: - if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -344,8 +342,6 @@ async def delete_workspace_service(response: Response, user=Depends(require_work @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def invoke_action_on_workspace_service(response: Response, action: str, user=Depends(require_workspace_owner), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: - if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace_service, resource_repo=workspace_service_repo, diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index bbe7695780..c0fa978336 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -209,7 +209,7 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: self.operations_query() + f' (c.resourceId = "{resource_id}"' + f' OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true)' - + f' OR CONTAINS(c.resourcePath, "{resource_id}"))' + + f' OR (CONTAINS(c.resourcePath, "{resource_id}") AND NOT CONTAINS(c.resourcePath, "/user-resources/")))' + f' AND c.status IN ({status_filter})' ) operations = await self.query(query=query) diff --git a/api_app/models/domain/resource.py b/api_app/models/domain/resource.py index 8a18630b8d..7677f96b13 100644 --- a/api_app/models/domain/resource.py +++ b/api_app/models/domain/resource.py @@ -1,4 +1,3 @@ -import copy from enum import StrEnum from typing import Optional, Union, List from pydantic import field_validator, BaseModel, Field @@ -81,12 +80,6 @@ def convert_user_to_dict(cls, value): return value def get_resource_request_message_payload(self, operation_id: str, step_id: str, action: RequestAction) -> dict: - parameters = copy.deepcopy(self.properties) if self.properties else {} - if self.resourceType == ResourceType.Workspace: - pending_removal = parameters.pop("address_spaces_pending_removal", []) - if pending_removal and "address_spaces" in parameters and isinstance(parameters["address_spaces"], list): - parameters["address_spaces"] = [a for a in parameters["address_spaces"] if a not in pending_removal] - payload = { "operationId": operation_id, "stepId": step_id, @@ -94,7 +87,7 @@ def get_resource_request_message_payload(self, operation_id: str, step_id: str, "id": self.id, "name": self.templateName, "version": self.templateVersion, - "parameters": parameters + "parameters": self.properties } if self.resourceType == ResourceType.WorkspaceService: diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 4016d8be27..7642114a79 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -32,10 +32,10 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 # Azure Service Bus default max delivery count is 10. -# The AMQP delivery_count reported by ServiceBusReceivedMessage is zero-based: -# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. -# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. -FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 +# The AMQP delivery_count reported by ServiceBusReceivedMessage is 1-based: +# attempt 1 has delivery_count = 1, and attempt 10 (the final processable attempt) has delivery_count = 10. +# When delivery_count >= MAX_CLEANUP_DELIVERY_COUNT, the message is on its final attempt before dead-lettering. +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT class AddressSpaceConflictError(Exception): @@ -113,8 +113,8 @@ async def process_message(self, msg) -> bool: current_span.set_attribute("operation_id", str(message.operationId)) current_span.set_attribute("status", str(message.status)) - raw_delivery_count = getattr(msg, "delivery_count", 0) - delivery_count = raw_delivery_count if raw_delivery_count is not None else 0 + raw_delivery_count = getattr(msg, "delivery_count", 1) + delivery_count = raw_delivery_count if raw_delivery_count is not None else 1 is_final_delivery = delivery_count >= FINAL_CLEANUP_DELIVERY_COUNT complete_message = await self.update_status_in_database(message, is_final_delivery=is_final_delivery) diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 18103fffca..e7d46f7222 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -864,26 +864,6 @@ async def test_delete_workspace_service_raises_409_if_workspace_has_active_opera assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION - # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id} - @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) - @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") - @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") - async def test_patch_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): - get_service_mock.return_value = sample_workspace_service() - response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) - assert response.status_code == status.HTTP_409_CONFLICT - assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION - - # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action - @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) - @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") - @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") - async def test_invoke_action_on_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): - get_service_mock.return_value = sample_workspace_service() - response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), params={"action": "restart"}) - assert response.status_code == status.HTTP_409_CONFLICT - assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION - # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index 6c2a3edfe4..c3bf8c3659 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -94,16 +94,20 @@ def airlock_request_mock(status=AirlockRequestStatus.Draft): def get_allowed_status_changes(): + changes = [] for current_status, allowed_new_statuses in ALLOWED_STATUS_CHANGES.items(): for new_status in allowed_new_statuses: - yield current_status, new_status + changes.append((current_status, new_status)) + return changes def get_forbidden_status_changes(): + changes = [] for current_status, allowed_new_statuses in ALLOWED_STATUS_CHANGES.items(): forbidden_new_statuses = list(set(ALL_STATUSES) - set(allowed_new_statuses)) for new_status in forbidden_new_statuses: - yield current_status, new_status + changes.append((current_status, new_status)) + return changes async def test_get_airlock_request_by_id(airlock_request_repo): diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index e40fab20a7..7cee7c66ff 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -115,6 +115,7 @@ async def test_resource_has_active_operation_returns_true_when_active_operation_ assert "workspace-123" in query_str assert "ARRAY_CONTAINS" in query_str assert 'CONTAINS(c.resourcePath, "workspace-123")' in query_str + assert 'NOT CONTAINS(c.resourcePath, "/user-resources/")' in query_str async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): diff --git a/api_app/tests_ma/test_models/test_resource.py b/api_app/tests_ma/test_models/test_resource.py index 57e4a4212a..61963d3256 100644 --- a/api_app/tests_ma/test_models/test_resource.py +++ b/api_app/tests_ma/test_models/test_resource.py @@ -8,7 +8,6 @@ from models.domain.resource import Output, Resource, ResourceHistoryItem, ResourceType from models.domain.resource_template import Property from models.domain.user_resource import UserResource -from models.domain.workspace import Workspace from models.domain.workspace_service import WorkspaceService from models.schemas.resource import ResourceHistoryInList from models.schemas.shared_service_template import SharedServiceTemplateInCreate @@ -56,25 +55,6 @@ def test_workspace_service_get_resource_request_message_payload_augments_payload assert message_payload["workspaceId"] == workspace_id -def test_workspace_get_resource_request_message_payload_excludes_pending_removal_address_spaces(): - workspace = Workspace( - id="123", - templateName="workspace-template", - templateVersion="1.0", - etag="", - resourcePath="test", - properties={ - "address_spaces": ["10.0.0.0/22", "10.1.0.0/22"], - "address_spaces_pending_removal": ["10.1.0.0/22"] - } - ) - - message_payload = workspace.get_resource_request_message_payload(OPERATION_ID, STEP_ID, RequestAction.Upgrade) - - assert message_payload["parameters"]["address_spaces"] == ["10.0.0.0/22"] - assert "address_spaces_pending_removal" not in message_payload["parameters"] - - def test_legacy_actor_dicts_validate_without_user_required_fields(): resource = Resource.model_validate({ "id": "resource-id", diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 571b0d0b32..b4ac9802bf 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -70,7 +70,7 @@ class ServiceBusReceivedMessageMock: - def __init__(self, message: dict, delivery_count: int = 0): + def __init__(self, message: dict, delivery_count: int = 1): self.message = json.dumps(message) self.correlation_id = "test_correlation_id" self.session_id = "test_session_id" @@ -1770,8 +1770,9 @@ async def mock_get_resource_by_id(resource_uuid): (0, False), (1, False), (8, False), - (9, True), + (9, False), (10, True), + (11, True), (None, False), ]) async def test_process_message_delivery_count_handling(delivery_count, expected_final): diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index a7336264c9..7fbfb99af8 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,10 +54,8 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order - requires_duplicate_detection = true - duplicate_detection_history_time_window = "PT4H" + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 670ad2c1cf..12d80d0630 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.18" +__version__ = "0.16.17" From 9b2751f1faea85ba7d035dce774f6163b4c88d2d Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:15:00 +0000 Subject: [PATCH 36/90] Address review feedback on deduplication, delivery counting, and workspace operation guards --- api_app/api/routes/workspaces.py | 4 ++++ .../service_bus/deployment_status_updater.py | 12 +++++------ .../test_api/test_routes/test_workspaces.py | 20 +++++++++++++++++++ .../test_deployment_status_update.py | 5 ++--- core/terraform/servicebus.tf | 6 ++++-- core/version.txt | 2 +- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index bee1d1bda0..1f6f52e813 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -296,6 +296,8 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -342,6 +344,8 @@ async def delete_workspace_service(response: Response, user=Depends(require_work @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def invoke_action_on_workspace_service(response: Response, action: str, user=Depends(require_workspace_owner), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace_service, resource_repo=workspace_service_repo, diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 7642114a79..4016d8be27 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -32,10 +32,10 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 # Azure Service Bus default max delivery count is 10. -# The AMQP delivery_count reported by ServiceBusReceivedMessage is 1-based: -# attempt 1 has delivery_count = 1, and attempt 10 (the final processable attempt) has delivery_count = 10. -# When delivery_count >= MAX_CLEANUP_DELIVERY_COUNT, the message is on its final attempt before dead-lettering. -FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT +# The AMQP delivery_count reported by ServiceBusReceivedMessage is zero-based: +# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. +# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 class AddressSpaceConflictError(Exception): @@ -113,8 +113,8 @@ async def process_message(self, msg) -> bool: current_span.set_attribute("operation_id", str(message.operationId)) current_span.set_attribute("status", str(message.status)) - raw_delivery_count = getattr(msg, "delivery_count", 1) - delivery_count = raw_delivery_count if raw_delivery_count is not None else 1 + raw_delivery_count = getattr(msg, "delivery_count", 0) + delivery_count = raw_delivery_count if raw_delivery_count is not None else 0 is_final_delivery = delivery_count >= FINAL_CLEANUP_DELIVERY_COUNT complete_message = await self.update_status_in_database(message, is_final_delivery=is_final_delivery) diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index e7d46f7222..18103fffca 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -864,6 +864,26 @@ async def test_delete_workspace_service_raises_409_if_workspace_has_active_opera assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_patch_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_invoke_action_on_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index b4ac9802bf..571b0d0b32 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -70,7 +70,7 @@ class ServiceBusReceivedMessageMock: - def __init__(self, message: dict, delivery_count: int = 1): + def __init__(self, message: dict, delivery_count: int = 0): self.message = json.dumps(message) self.correlation_id = "test_correlation_id" self.session_id = "test_session_id" @@ -1770,9 +1770,8 @@ async def mock_get_resource_by_id(resource_uuid): (0, False), (1, False), (8, False), - (9, False), + (9, True), (10, True), - (11, True), (None, False), ]) async def test_process_message_delivery_count_handling(delivery_count, expected_final): diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..a7336264c9 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,8 +54,10 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + requires_duplicate_detection = true + duplicate_detection_history_time_window = "PT4H" } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 12d80d0630..670ad2c1cf 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.16.18" From f94c4eaf2c62e1c95e5499a03340a5e595f03311 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:49:47 +0000 Subject: [PATCH 37/90] Make cleanup dispatch idempotent in application and revert queue recreation settings --- api_app/service_bus/deployment_status_updater.py | 4 ++++ .../test_service_bus/test_deployment_status_update.py | 7 +++++++ core/terraform/servicebus.tf | 6 ++---- core/version.txt | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 4016d8be27..0f953b495d 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -380,6 +380,10 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage action=next_step.resourceAction, message_id=step_message_id, ) + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + next_step.status = Status.Updating + next_step.updatedWhen = get_timestamp() + await self.operations_repo.update_item(operation) except AddressSpaceConflictError as e: logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") next_step.message = f"Terminal address space conflict: {e}" diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 571b0d0b32..b66f48d958 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -2453,6 +2453,7 @@ async def mock_send_deployment_message(*args, **kwargs): cleanup_step = operation.steps[1] assert cleanup_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID assert status_before_send == Status.AwaitingUpdate + assert cleanup_step.status == Status.Updating send_deployment_message_mock.assert_called_once() assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step.id}" @@ -2526,6 +2527,12 @@ async def test_redelivered_main_status_before_updating_dispatches_with_same_dete assert result is True send_deployment_message_mock.assert_called_once() assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step_id}" + assert step2.status == Status.Updating + + # Subsequent redelivery of main status message skips enqueueing because step2 is now Updating + result2 = await status_updater.update_status_in_database(redelivered_main_message) + assert result2 is True + send_deployment_message_mock.assert_called_once() @patch('service_bus.helpers._send_message') diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index a7336264c9..7fbfb99af8 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,10 +54,8 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order - requires_duplicate_detection = true - duplicate_detection_history_time_window = "PT4H" + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 670ad2c1cf..12d80d0630 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.18" +__version__ = "0.16.17" From 0d551dc0ea7c5954323023390828a141275b345b Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:59:09 +0000 Subject: [PATCH 38/90] Enable duplicate detection on workspacequeue and bump core version to 0.16.18 --- core/terraform/servicebus.tf | 6 ++++-- core/version.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..a7336264c9 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,8 +54,10 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + requires_duplicate_detection = true + duplicate_detection_history_time_window = "PT4H" } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 12d80d0630..670ad2c1cf 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.16.18" From 5722c2aba7796eb8a35f7c008c79b8a1a480185b Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:05:30 +0000 Subject: [PATCH 39/90] Handle cleanup dispatch idempotency in application and revert Service Bus queue changes --- .../service_bus/deployment_status_updater.py | 2 - api_app/service_bus/helpers.py | 25 +------- .../test_deployment_status_update.py | 57 +++++-------------- core/terraform/servicebus.tf | 6 +- core/version.txt | 2 +- 5 files changed, 20 insertions(+), 72 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 0f953b495d..e95bebb346 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -372,13 +372,11 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # create + send the message logger.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=next_step.id, action=next_step.resourceAction)) - step_message_id = f"{operation.id}_{next_step.id}" await send_deployment_message( content=content, correlation_id=operation.id, session_id=resource_to_send.id, action=next_step.resourceAction, - message_id=step_message_id, ) if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: next_step.status = Status.Updating diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index ca27fbfee5..e1b2daf74d 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -1,5 +1,3 @@ -import json -from typing import Optional from azure.servicebus import ServiceBusMessage from azure.servicebus.aio import ServiceBusClient from pydantic import TypeAdapter @@ -37,26 +35,9 @@ async def _send_message(message: ServiceBusMessage, queue: str): await sender.send_messages(message) -async def send_deployment_message(content, correlation_id, session_id, action, message_id: Optional[str] = None): - if not message_id: - try: - payload = content if isinstance(content, dict) else json.loads(content) - op_id = payload.get("operationId", correlation_id) - step_id = payload.get("stepId") - if op_id and step_id: - message_id = f"{op_id}_{step_id}" - except Exception: - pass - if not message_id: - message_id = f"{correlation_id}_{action}" - - resource_request_message = ServiceBusMessage( - body=content, - correlation_id=correlation_id, - session_id=session_id, - message_id=message_id, - ) - logger.info(f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, message ID {resource_request_message.message_id}, action: {action}") +async def send_deployment_message(content, correlation_id, session_id, action): + resource_request_message = ServiceBusMessage(body=content, correlation_id=correlation_id, session_id=session_id) + logger.info(f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, action: {action}") await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index b66f48d958..521b655d82 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -2387,7 +2387,7 @@ async def tracking_resource_update(res): @patch('service_bus.deployment_status_updater.send_deployment_message') -async def test_cleanup_dispatch_uses_deterministic_message_id(send_deployment_message_mock): +async def test_cleanup_dispatch_transitions_step_to_updating(send_deployment_message_mock): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" address_space = "10.1.0.0/22" @@ -2455,12 +2455,16 @@ async def mock_send_deployment_message(*args, **kwargs): assert status_before_send == Status.AwaitingUpdate assert cleanup_step.status == Status.Updating - send_deployment_message_mock.assert_called_once() - assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step.id}" + send_deployment_message_mock.assert_called_once_with( + content=ANY, + correlation_id=OPERATION_ID, + session_id=parent_workspace_id, + action=RequestAction.Upgrade + ) @patch('service_bus.deployment_status_updater.send_deployment_message') -async def test_redelivered_main_status_before_updating_dispatches_with_same_deterministic_message_id(send_deployment_message_mock): +async def test_redelivered_main_status_dispatches_and_subsequent_redelivery_skips(send_deployment_message_mock): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" cleanup_step_id = "step-2-cleanup-id" @@ -2525,48 +2529,15 @@ async def test_redelivered_main_status_before_updating_dispatches_with_same_dete result = await status_updater.update_status_in_database(redelivered_main_message) assert result is True - send_deployment_message_mock.assert_called_once() - assert send_deployment_message_mock.call_args.kwargs["message_id"] == f"{OPERATION_ID}_{cleanup_step_id}" + send_deployment_message_mock.assert_called_once_with( + content=ANY, + correlation_id=OPERATION_ID, + session_id=parent_workspace_id, + action=RequestAction.Upgrade + ) assert step2.status == Status.Updating # Subsequent redelivery of main status message skips enqueueing because step2 is now Updating result2 = await status_updater.update_status_in_database(redelivered_main_message) assert result2 is True send_deployment_message_mock.assert_called_once() - - -@patch('service_bus.helpers._send_message') -async def test_send_deployment_message_attaches_deterministic_message_id(_send_message_mock): - from service_bus.helpers import send_deployment_message - - # Derived from JSON payload with operationId and stepId - payload = {"operationId": "op-123", "stepId": "step-456", "action": "upgrade"} - await send_deployment_message( - content=json.dumps(payload), - correlation_id="op-123", - session_id="session-1", - action="upgrade" - ) - sent_msg = _send_message_mock.call_args.args[0] - assert sent_msg.message_id == "op-123_step-456" - - # Explicit message_id takes precedence - await send_deployment_message( - content=json.dumps(payload), - correlation_id="op-123", - session_id="session-1", - action="upgrade", - message_id="custom-id" - ) - sent_msg = _send_message_mock.call_args.args[0] - assert sent_msg.message_id == "custom-id" - - # Fallback when payload lacks stepId - await send_deployment_message( - content=json.dumps({"some": "data"}), - correlation_id="op-789", - session_id="session-2", - action="install" - ) - sent_msg = _send_message_mock.call_args.args[0] - assert sent_msg.message_id == "op-789_install" diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index a7336264c9..7fbfb99af8 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -54,10 +54,8 @@ resource "azurerm_servicebus_queue" "workspacequeue" { name = "workspacequeue" namespace_id = azurerm_servicebus_namespace.sb.id - partitioning_enabled = false - requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order - requires_duplicate_detection = true - duplicate_detection_history_time_window = "PT4H" + partitioning_enabled = false + requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index 670ad2c1cf..12d80d0630 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.18" +__version__ = "0.16.17" From 13ae0738b8519283358c17702a97800eb2079de3 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:16:13 +0000 Subject: [PATCH 40/90] Inject empty active-services mock for workspace_services_repo in deployment status update tests --- .../test_service_bus/test_deployment_status_update.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 521b655d82..7bfdb5dc1d 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -800,6 +800,8 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( status_updater = DeploymentStatusUpdater() await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] complete_message = await status_updater.process_message(service_bus_received_message_mock) # The uninstall message is complete; cleanup is deferred to the upgrade step. @@ -989,6 +991,8 @@ async def mock_send_deployment_message(*args, **kwargs): status_updater = DeploymentStatusUpdater() await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] complete_message = await status_updater.process_message(service_bus_received_message_mock) assert complete_message is True From eaf6150aca255a97177f2c1bd3e189740efa888d Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:38:05 +0000 Subject: [PATCH 41/90] Address Copilot review on terminal-state guards, resource lookups, and cleanup pipeline validation - Extend terminal-state guard in deployment status updater to ignore messages for any step once uninstall cleanup is terminal - Persist terminal failure operation state when a step resource does not exist in Cosmos DB - Wrap _get_address_cleanup_details in try block and handle exceptions gracefully on final delivery - Validate that address-space-cleanup step is strictly the final step in uninstall pipelines and check all steps succeed before marking Deleted - Add comprehensive unit tests for delayed duplicate messages, missing resources, cleanup failures on final delivery, and pipeline trailing steps --- api_app/db/repositories/resource_templates.py | 12 +- .../service_bus/deployment_status_updater.py | 83 ++++--- .../test_resource_templates_repository.py | 30 +++ .../test_deployment_status_update.py | 233 ++++++++++++++++++ 4 files changed, 317 insertions(+), 41 deletions(-) diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 36ac905b94..f97cd88194 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -7,9 +7,11 @@ from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput from db.repositories.base import BaseRepository from models.domain.resource import ResourceType +from models.domain.request_action import RequestAction from models.domain.resource_template import ResourceTemplate from models.domain.user_resource_template import UserResourceTemplate from models.schemas.resource_template import ResourceTemplateInCreate, ResourceTemplateInformation +from resources import strings from services.schema_service import enrich_shared_service_template, enrich_workspace_template, enrich_workspace_service_template, enrich_user_resource_template @@ -181,7 +183,9 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline): step_ids = [] for action in pipeline: num_of_main_steps = 0 - for step in pipeline[action]: + steps = pipeline[action] + num_steps = len(steps) + for i, step in enumerate(steps): step_id = step["stepId"] if step_id == "main": @@ -190,5 +194,11 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline): if step_id in step_ids or num_of_main_steps > 1: raise InvalidInput(f"Invalid template - duplicate stepIds are not allowed. stepId: {step_id}") + if step_id == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if action != RequestAction.UnInstall: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in the '{RequestAction.UnInstall}' pipeline") + if i != num_steps - 1: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must be the final step in the '{RequestAction.UnInstall}' pipeline") + if step_id != "main": step_ids.append(step_id) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index e95bebb346..da5fb95d96 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -134,11 +134,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage If there is another step in the operation after this one, process the substitutions + patch, then enqueue a message to process it. """ result = False + operation = None + step_to_update = None try: # update the op operation = await self.operations_repo.get_operation_by_id(str(message.operationId)) - step_to_update = None is_last_step = False current_step_index = 0 @@ -153,31 +154,16 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") - # If address space cleanup for this operation is already in a terminal state (completed or failed), - # do not reopen the operation or re-enqueue cleanup on redelivered main-step messages. + # If address space cleanup for this uninstall operation is already in a terminal state (completed or failed), + # the entire operation is already finalized. Ignore any delayed or duplicate messages for ANY step in this + # operation to prevent overwriting resources with stale status or reopening the operation. if ( - step_to_update.templateStepId == "main" - and (step_to_update.is_success() or message.status in [Status.Deployed, Status.Deleted, Status.Updated, Status.ActionSucceeded]) - and operation.action == RequestAction.UnInstall - and self._is_cleanup_terminal(operation) - ): - logger.info( - f"Address space cleanup already in terminal state for operation {operation.id}. " - f"Skipping redelivery of step {message.stepId} to prevent reopening operation." - ) - return True - - # If the uninstall cleanup is already terminal, ignore delayed or duplicate cleanup-step messages - # (such as an intermediate Updating or delayed UpdatingFailed) to prevent writing stale status - # to the workspace resource or reopening the operation. - if ( - step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID - and operation.action == RequestAction.UnInstall + operation.action == RequestAction.UnInstall and self._is_cleanup_terminal(operation) ): logger.info( - f"Cleanup step message {message.stepId} with status {message.status} received for operation {operation.id}, " - f"but uninstall cleanup is already terminal. Ignoring to prevent stale writes." + f"Message {message.stepId} (step {step_to_update.templateStepId}, status {message.status}) received for operation {operation.id}, " + f"but uninstall cleanup is already terminal ({operation.status}). Ignoring to prevent stale writes." ) return True @@ -211,6 +197,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) and step_to_update.resourceAction == RequestAction.Upgrade and operation.action == RequestAction.UnInstall + and is_last_step ) # For the address space cleanup step, we MUST run cleanup before persisting the @@ -240,6 +227,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.update_overall_operation_status(operation, step_to_update, is_last_step) await self.operations_repo.update_item(operation) return True + except Exception as e: + logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Unexpected error freeing workspace address space: {e}", exc_info=True) + freed = False if not freed: if is_final_delivery: @@ -435,6 +425,18 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # Marking as true as this message will never succeed anyways and should be removed from the queue. result = True logger.exception(strings.DEPLOYMENT_STATUS_ID_NOT_FOUND.format(message.id)) + if operation is not None and step_to_update is not None: + try: + step_to_update.status = ( + Status.UpdatingFailed if operation.action == RequestAction.Upgrade + else Status.DeletingFailed if operation.action == RequestAction.UnInstall + else Status.DeploymentFailed + ) + step_to_update.message = f"Resource {message.id} not found in database: operation aborted." + await self.update_overall_operation_status(operation, step_to_update, is_last_step=True) + await self.operations_repo.update_item(operation) + except Exception: + logger.exception(f"Failed to persist terminal failure status for operation {operation.id} when resource {message.id} was missing") except Exception: logger.exception("Failed to update status") @@ -512,21 +514,20 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: Frees the address space owned by the uninstalled WorkspaceService after the corresponding workspace upgrade has succeeded. """ - cleanup_details = await self._get_address_cleanup_details(operation) - if cleanup_details is None: - return True - - address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details + try: + cleanup_details = await self._get_address_cleanup_details(operation) + if cleanup_details is None: + return True - # Check if cleanup for this step was already completed (idempotency on redelivery) - if self._is_cleanup_completed(operation): - return True + address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details - # Check if the uninstalled service already has the address_space_freed marker - if service_dict.get("properties", {}).get("address_space_freed"): - return True + # Check if cleanup for this step was already completed (idempotency on redelivery) + if self._is_cleanup_completed(operation): + return True - try: + # Check if the uninstalled service already has the address_space_freed marker + if service_dict.get("properties", {}).get("address_space_freed"): + return True # Check if address_to_free has been assigned to another active workspace service on this workspace (fail closed) workspace_services_repo = getattr(self, "workspace_services_repo", None) if workspace_services_repo is None: @@ -589,11 +590,12 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.message = step.message return - # If address space cleanup has completed for an uninstall, do not reopen to PipelineRunning + # If address space cleanup has completed for an uninstall, only finalize as Deleted if it is the last step and all steps succeeded if self._is_cleanup_completed(operation) and operation.action == RequestAction.UnInstall: - operation.status = Status.Deleted - operation.message = "Multi step pipeline completed successfully" - return + if is_last_step and all(op_step.is_success() for op_step in operation.steps): + operation.status = Status.Deleted + operation.message = "Multi step pipeline completed successfully" + return # If address space cleanup has failed terminally for an uninstall, do not reopen to PipelineRunning if self._is_cleanup_failed(operation) and operation.action == RequestAction.UnInstall: @@ -631,8 +633,9 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper await self.resource_repo.update_item(primary_resource) if step.is_success() and is_last_step: - operation.status = self.get_success_status_for_action(operation.action) - operation.message = "Multi step pipeline completed successfully" + if all(op_step.is_success() for op_step in operation.steps): + operation.status = self.get_success_status_for_action(operation.action) + operation.message = "Multi step pipeline completed successfully" # pipeline succeeded - update the primary resource (from the main step) as succeeded too main_step = None diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 970b4721e7..7a285229ce 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -8,6 +8,7 @@ from models.domain.resource import ResourceType from models.domain.resource_template import ResourceTemplate from models.schemas.workspace_template import WorkspaceTemplateInCreate +from resources import strings pytestmark = pytest.mark.asyncio @@ -293,3 +294,32 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel input_user_resource_template.json_schema["pipeline"] = None created = await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) assert created.pipeline is None + + +@pytest.mark.parametrize( + "pipeline", + [ + { + "install": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + }, + { + "upgrade": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + }, + { + "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}, {"stepId": "main"}], + }, + ], +) +async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails(resource_template_repo, input_user_resource_template, pipeline): + input_user_resource_template.json_schema["pipeline"] = pipeline + with pytest.raises(InvalidInput): + await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) + + +@patch('db.repositories.resource_templates.ResourceTemplateRepository.save_item') +async def test_create_template_with_valid_address_space_cleanup_as_final_uninstall_step_succeeds(_, resource_template_repo, input_user_resource_template): + input_user_resource_template.json_schema["pipeline"] = { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + } + created = await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) + assert created.pipeline diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 7bfdb5dc1d..e563d6896e 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -2545,3 +2545,236 @@ async def test_redelivered_main_status_dispatches_and_subsequent_redelivery_skip result2 = await status_updater.update_status_in_database(redelivered_main_message) assert result2 is True send_deployment_message_mock.assert_called_once() + + +async def test_delayed_earlier_step_message_ignored_when_cleanup_terminal(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + + # Delayed failure message arriving for step-1 after cleanup has already finalized the operation + delayed_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.DeletingFailed, + message="delayed failure notification", + ) + + result = await status_updater.update_status_in_database(delayed_message) + + assert result is True + assert operation.status == Status.Deleted + assert step1.status == Status.Deleted + status_updater.resource_repo.update_item.assert_not_called() + status_updater.operations_repo.update_item.assert_not_called() + + +async def test_missing_step_resource_persists_terminal_failure_operation_state(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="resource deleted", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is True + assert step1.status == Status.DeletingFailed + assert "not found in database" in step1.message + assert operation.status == Status.DeletingFailed + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + +async def test_cleanup_failure_in_get_address_cleanup_details_on_final_delivery_marks_failed(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + # Simulate Cosmos DB read failure when reading cleanup details + status_updater._get_address_cleanup_details = AsyncMock(side_effect=Exception("Cosmos read error")) + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert step2.status == Status.UpdatingFailed + assert "delivery attempts" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +async def test_cleanup_step_with_trailing_step_does_not_prematurely_finalize_as_deleted(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + trailing_resource_id = "2222c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + step3 = OperationStep( + id="step-3", + stepTitle="Firewall post-cleanup step", + resourceId=trailing_resource_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId="trailing-step", + sourceTemplateResourceId=trailing_resource_id, + status=Status.AwaitingUpdate, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2, step3], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.resource_repo = AsyncMock() + + # When step2 completes but is not the last step in the operation + await status_updater.update_overall_operation_status(operation, step2, is_last_step=False) + + assert operation.status == Status.PipelineRunning + assert operation.status != Status.Deleted From 45bf9367ad68ea1fe86f6115d52d843abacf6d31 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:13:30 +0000 Subject: [PATCH 42/90] Enhance address space cleanup validation and improve deployment status update handling --- api_app/db/repositories/resource_templates.py | 3 + .../service_bus/deployment_status_updater.py | 163 +++++++++--- .../test_resource_templates_repository.py | 25 +- .../test_deployment_status_update.py | 245 ++++++++++++++++++ 4 files changed, 398 insertions(+), 38 deletions(-) diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index f97cd88194..d7d5a1d635 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -199,6 +199,9 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline): raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in the '{RequestAction.UnInstall}' pipeline") if i != num_steps - 1: raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must be the final step in the '{RequestAction.UnInstall}' pipeline") + if (step.get("resourceType") != ResourceType.Workspace + or step.get("resourceAction") != RequestAction.Upgrade): + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must upgrade a workspace") if step_id != "main": step_ids.append(step_id) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index da5fb95d96..1735dc87d6 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -266,41 +266,94 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage else: step_to_update.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS - # copy the step status to the resource item, for convenience - resource = await self.resource_repo.get_resource_by_id(resource_id) - resource.deploymentStatus = step_to_update.status - await self.resource_repo.update_item(resource) + try: + # copy the step status to the resource item, for convenience + resource = await self.resource_repo.get_resource_by_id(resource_id) + resource.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource) - # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. - if not step_to_update.is_success(): - await self.update_overall_operation_status(operation, step_to_update, is_last_step) - await self.operations_repo.update_item(operation) - return True + # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. + if not step_to_update.is_success(): + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + return True - # update the resource doc to persist any outputs - resource = await self.resource_repo.get_resource_dict_by_id(resource_id) - resource_to_persist = self.create_updated_resource_document(resource, message) - await self.resource_repo.update_item_dict(resource_to_persist) + # update the resource doc to persist any outputs + resource = await self.resource_repo.get_resource_dict_by_id(resource_id) + resource_to_persist = self.create_updated_resource_document(resource, message) + await self.resource_repo.update_item_dict(resource_to_persist) - if is_cleanup_step and step_to_update.is_success(): - # Restore the primary resource to Deleted when cleanup succeeds - main_step = next( - (op_step for op_step in operation.steps - if op_step.templateStepId == "main" - and op_step.resourceId == operation.resourceId), - None - ) - if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - if primary_resource.deploymentStatus != Status.Deleted: - primary_resource.deploymentStatus = Status.Deleted - await self.resource_repo.update_item(primary_resource) + if is_cleanup_step and step_to_update.is_success(): + # Restore the primary resource to Deleted when cleanup succeeds + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) + + # update the overall headline operation status + await self.update_overall_operation_status(operation, step_to_update, is_last_step) - # update the overall headline operation status - await self.update_overall_operation_status(operation, step_to_update, is_last_step) + # save the operation with final status LAST, after all resource updates are committed + for attempt in range(MAX_CLEANUP_RETRIES): + try: + await self.operations_repo.update_item(operation) + break + except CosmosAccessConditionFailedError: + if attempt == MAX_CLEANUP_RETRIES - 1: + raise + logger.warning(f"ETag conflict when saving operation {operation.id}. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + operation = await self.operations_repo.get_operation_by_id(operation.id) + for s in operation.steps: + if s.id == step_to_update.id: + s.status = step_to_update.status + s.message = step_to_update.message + s.updatedWhen = get_timestamp() + break + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + except Exception as e: + if is_cleanup_step and step_to_update.is_success(): + logger.error(f"[CLEANUP_FINALIZATION_FAILED] Error in post-cleanup writes: {e}", exc_info=True) + if is_final_delivery: + # Final delivery handling: must persist terminal state and complete message + try: + fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) + for s in fresh_op.steps: + if s.id == step_to_update.id: + s.status = step_to_update.status + s.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + s.updatedWhen = get_timestamp() + break + await self.update_overall_operation_status(fresh_op, step_to_update, is_last_step) + await self.operations_repo.update_item(fresh_op) + except Exception as op_err: + logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) - # save the operation with final status LAST, after all resource updates are committed - await self.operations_repo.update_item(operation) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) + except Exception as prim_err: + logger.error(f"Failed to restore primary resource to Deleted on final delivery: {prim_err}", exc_info=True) + return True + else: + return False + else: + raise # more steps in the op to do? if is_last_step is False: @@ -368,10 +421,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage session_id=resource_to_send.id, action=next_step.resourceAction, ) - if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: - next_step.status = Status.Updating - next_step.updatedWhen = get_timestamp() - await self.operations_repo.update_item(operation) except AddressSpaceConflictError as e: logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") next_step.message = f"Terminal address space conflict: {e}" @@ -418,6 +467,51 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) + return True + + # Post-send persistence: send succeeded, now update operation status for cleanup step + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + for attempt in range(MAX_CLEANUP_RETRIES): + try: + next_step.status = Status.Updating + next_step.updatedWhen = get_timestamp() + await self.operations_repo.update_item(operation) + break + except CosmosAccessConditionFailedError: + if attempt == MAX_CLEANUP_RETRIES - 1: + logger.warning( + f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id} " + f"in operation {operation.id} after successful dispatch." + ) + else: + logger.warning( + f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id}. " + f"Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." + ) + try: + fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + for s in fresh_op.steps: + if s.id == next_step.id: + s.status = Status.Updating + s.updatedWhen = get_timestamp() + break + await self.operations_repo.update_item(fresh_op) + operation = fresh_op + break + except Exception: + pass + except Exception as post_send_err: + if attempt == MAX_CLEANUP_RETRIES - 1: + logger.error( + f"Failed to persist {Status.Updating} status for address space cleanup step {next_step.id} " + f"in operation {operation.id} after successful dispatch: {post_send_err}", + exc_info=True + ) + else: + logger.warning( + f"Transient error persisting {Status.Updating} status for cleanup step {next_step.id}. " + f"Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." + ) result = True @@ -514,6 +608,7 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: Frees the address space owned by the uninstalled WorkspaceService after the corresponding workspace upgrade has succeeded. """ + address_to_free = parent_workspace_id = resource_id = "" try: cleanup_details = await self._get_address_cleanup_details(operation) if cleanup_details is None: diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 7a285229ce..4392027743 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -5,6 +5,7 @@ from db.repositories.resource_templates import ResourceTemplateRepository from db.errors import EntityDoesNotExist, InvalidInput +from models.domain.request_action import RequestAction from models.domain.resource import ResourceType from models.domain.resource_template import ResourceTemplate from models.schemas.workspace_template import WorkspaceTemplateInCreate @@ -300,13 +301,22 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel "pipeline", [ { - "install": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + "install": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], }, { - "upgrade": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + "upgrade": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], }, { - "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}, {"stepId": "main"}], + "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}, {"stepId": "main"}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.SharedService, "resourceAction": RequestAction.Upgrade}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.UnInstall}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], }, ], ) @@ -319,7 +329,14 @@ async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails @patch('db.repositories.resource_templates.ResourceTemplateRepository.save_item') async def test_create_template_with_valid_address_space_cleanup_as_final_uninstall_step_succeeds(_, resource_template_repo, input_user_resource_template): input_user_resource_template.json_schema["pipeline"] = { - "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + "uninstall": [ + {"stepId": "main"}, + { + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ], } created = await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) assert created.pipeline diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index e563d6896e..2ec28766ff 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -10,6 +10,7 @@ from models.domain.request_action import RequestAction from models.domain.resource import ResourceType +from azure.cosmos.exceptions import CosmosAccessConditionFailedError from db.errors import EntityDoesNotExist from models.domain.workspace import Workspace from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status @@ -2778,3 +2779,247 @@ async def test_cleanup_step_with_trailing_step_does_not_prematurely_finalize_as_ assert operation.status == Status.PipelineRunning assert operation.status != Status.Deleted + + +async def test_cleanup_post_cleanup_write_failure_retries_on_non_final_delivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item.side_effect = Exception("Cosmos write failed") + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=False) + + assert result is False + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + assert operation.status != Status.DeletingFailed + + +async def test_cleanup_post_cleanup_write_failure_persists_terminal_state_on_final_delivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Updating + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + attempt_count = 0 + + async def mock_op_update(op): + nonlocal attempt_count + attempt_count += 1 + if attempt_count <= 3: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_op_update + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert operation.status == Status.Deleted + assert workspace_service_mock.deploymentStatus == Status.Deleted + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_cleanup_post_send_persistence_failure_does_not_terminally_fail_enqueued_cleanup(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + send_called = False + + async def mock_op_update(op): + if send_called: + raise Exception("Cosmos write error during post-send status update") + return op + + async def mock_send_deployment_message(*args, **kwargs): + nonlocal send_called + send_called = True + + send_deployment_message_mock.side_effect = mock_send_deployment_message + status_updater.operations_repo.update_item.side_effect = mock_op_update + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=True) + + assert result is True + send_deployment_message_mock.assert_called_once() + cleanup_step = operation.steps[1] + assert cleanup_step.status != Status.UpdatingFailed + assert operation.status != Status.DeletingFailed + assert not status_updater._is_cleanup_terminal(operation) + + +async def test_free_workspace_address_space_read_failure_avoids_unbound_local(): + status_updater = DeploymentStatusUpdater() + status_updater._get_address_cleanup_details = AsyncMock(side_effect=Exception("Cosmos read error")) + + operation = MagicMock() + freed = await status_updater._free_workspace_address_space(operation) + + assert freed is False From 3d6371b105fb99ca775862f2c157a74d95d12832 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:47:21 +0000 Subject: [PATCH 43/90] Implement ETag handling in Operation updates and enhance step merging logic in DeploymentStatusUpdater --- api_app/db/repositories/operations.py | 35 ++- api_app/models/domain/operation.py | 8 + .../service_bus/deployment_status_updater.py | 102 +++++++-- .../test_operation_repository.py | 56 ++++- .../test_deployment_status_update.py | 208 ++++++++++++++++++ 5 files changed, 386 insertions(+), 23 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index c0fa978336..26ec3da7f9 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -1,7 +1,8 @@ from datetime import datetime, UTC import uuid -from typing import List +from typing import List, Optional +from azure.core import MatchConditions from pydantic import TypeAdapter from db.repositories.resource_templates import ResourceTemplateRepository from resources import strings @@ -23,6 +24,38 @@ async def create(cls): await super().create(config.STATE_STORE_OPERATIONS_CONTAINER) return cls + async def save_item(self, item: Operation): + item_dict = item.model_dump() + item_dict.pop("_etag", None) + response = await self.container.create_item(body=item_dict) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + + async def update_item(self, item: Operation, etag: Optional[str] = None) -> Operation: + etag_to_match = etag or getattr(item, "etag", None) + item_dict = item.model_dump() + item_dict.pop("_etag", None) + if etag_to_match: + response = await self.container.replace_item( + item=item.id, + body=item_dict, + etag=etag_to_match, + match_condition=MatchConditions.IfNotModified + ) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + else: + response = await self.container.upsert_item(body=item_dict) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + return item + + async def update_item_with_etag(self, item: Operation, etag: str) -> Operation: + return await self.update_item(item, etag=etag) + @staticmethod def operations_query(): return 'SELECT * FROM c WHERE' diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 9cb7b258ff..d4da0239fe 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -94,6 +94,14 @@ class Operation(AzureTREModel): updatedWhen: float = Field(0.0, title="POSIX Timestamp for When the operation was updated") user: dict = Field(default_factory=dict) steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") + etag: Optional[str] = Field(None, title="_etag", alias="_etag") + + # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 + @field_validator("etag", mode="before") + @classmethod + def parse_etag_to_remove_escaped_quotes(cls, value): + if value: + return value.replace('\"', '') @field_validator("user", mode="before") @classmethod diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 1735dc87d6..de3b8061cd 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -309,29 +309,34 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if attempt == MAX_CLEANUP_RETRIES - 1: raise logger.warning(f"ETag conflict when saving operation {operation.id}. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") - operation = await self.operations_repo.get_operation_by_id(operation.id) - for s in operation.steps: - if s.id == step_to_update.id: - s.status = step_to_update.status - s.message = step_to_update.message - s.updatedWhen = get_timestamp() - break - await self.update_overall_operation_status(operation, step_to_update, is_last_step) + fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + step_to_update, is_last_step, current_step_index = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=step_to_update.message, + ) + await self.update_overall_operation_status(fresh_op, step_to_update, is_last_step) + operation = fresh_op except Exception as e: if is_cleanup_step and step_to_update.is_success(): logger.error(f"[CLEANUP_FINALIZATION_FAILED] Error in post-cleanup writes: {e}", exc_info=True) if is_final_delivery: # Final delivery handling: must persist terminal state and complete message + terminal_persisted = False try: fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) - for s in fresh_op.steps: - if s.id == step_to_update.id: - s.status = step_to_update.status - s.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS - s.updatedWhen = get_timestamp() - break - await self.update_overall_operation_status(fresh_op, step_to_update, is_last_step) + fresh_step, fresh_is_last, _ = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) await self.operations_repo.update_item(fresh_op) + terminal_persisted = True except Exception as op_err: logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) @@ -349,7 +354,17 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.resource_repo.update_item(primary_resource) except Exception as prim_err: logger.error(f"Failed to restore primary resource to Deleted on final delivery: {prim_err}", exc_info=True) - return True + + if terminal_persisted: + logger.info(f"Terminal operation status successfully persisted on final delivery for operation {operation.id}. Completing message.") + return True + else: + logger.error( + f"[CLEANUP_FINALIZATION_DEAD_LETTER] Could not persist terminal operation status for " + f"operation {operation.id} on final delivery {MAX_CLEANUP_DELIVERY_COUNT}. " + f"Returning False so message is dead-lettered for manual recovery." + ) + return False else: return False else: @@ -490,11 +505,13 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage ) try: fresh_op = await self.operations_repo.get_operation_by_id(operation.id) - for s in fresh_op.steps: - if s.id == next_step.id: - s.status = Status.Updating - s.updatedWhen = get_timestamp() - break + self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=next_step.id, + step_status=Status.Updating, + step_message=next_step.message or "", + ) await self.operations_repo.update_item(fresh_op) operation = fresh_op break @@ -536,6 +553,49 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result + def _merge_operation_steps( + self, + in_memory_op: Operation, + fresh_op: Operation, + step_id_to_update: str, + step_status: Status, + step_message: str, + ) -> Tuple[OperationStep, bool, int]: + """ + Merges in-memory appended steps (e.g. dynamically appended cleanup step on legacy operations) + into the fresh operation reloaded from Cosmos DB during an ETag conflict retry. + Recomputes step_to_update, is_last_step, and current_step_index for fresh_op. + """ + if fresh_op.steps is None: + fresh_op.steps = [] + + if in_memory_op.steps: + fresh_step_ids = {s.id for s in fresh_op.steps} + for s in in_memory_op.steps: + if s.id not in fresh_step_ids: + fresh_op.steps.append(s) + fresh_step_ids.add(s.id) + + step_to_update = None + current_step_index = 0 + is_last_step = False + for i, s in enumerate(fresh_op.steps): + if s.id == step_id_to_update: + step_to_update = s + current_step_index = i + if i == (len(fresh_op.steps) - 1): + is_last_step = True + break + + if step_to_update is None: + raise Exception(f"Error finding step {step_id_to_update} in reloaded operation {fresh_op.id}") + + step_to_update.status = step_status + step_to_update.message = step_message + step_to_update.updatedWhen = get_timestamp() + + return step_to_update, is_last_step, current_step_index + def _is_workspace_service_with_address_space(self, resource: dict) -> bool: return (resource.get("resourceType") in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value) and bool(resource.get("properties", {}).get("address_space")) diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 7cee7c66ff..f86a7a92dd 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -4,7 +4,7 @@ import pytest from mock import patch from db.repositories.resource_templates import ResourceTemplateRepository -from models.domain.operation import Status +from models.domain.operation import Operation, Status from models.domain.request_action import RequestAction from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository @@ -122,3 +122,57 @@ async def test_resource_has_active_operation_returns_false_when_no_active_operat operations_repo.query = AsyncMock(return_value=[]) result = await operations_repo.resource_has_active_operation("workspace-123") assert result is False + + +async def test_update_item_with_etag_calls_replace_item(operations_repo): + from azure.core import MatchConditions + operations_repo._container = MagicMock() + operations_repo._container.replace_item = AsyncMock(return_value={"_etag": "\"new-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="old-etag" + ) + result = await operations_repo.update_item(op) + + operations_repo._container.replace_item.assert_awaited_once() + call_kwargs = operations_repo._container.replace_item.call_args[1] + assert call_kwargs["item"] == "op-1" + assert call_kwargs["etag"] == "old-etag" + assert call_kwargs["match_condition"] == MatchConditions.IfNotModified + assert result.etag == "new-etag" + + +async def test_update_item_without_etag_calls_upsert_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.upsert_item = AsyncMock(return_value={"_etag": "\"new-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + ) + result = await operations_repo.update_item(op) + + operations_repo._container.upsert_item.assert_awaited_once() + assert result.etag == "new-etag" + + +async def test_update_item_raises_cosmos_access_condition_failed_error_on_mismatch(operations_repo): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + operations_repo._container = MagicMock() + operations_repo._container.replace_item = AsyncMock(side_effect=CosmosAccessConditionFailedError()) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="stale-etag" + ) + with pytest.raises(CosmosAccessConditionFailedError): + await operations_repo.update_item(op) diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 2ec28766ff..3484f750de 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -3023,3 +3023,211 @@ async def test_free_workspace_address_space_read_failure_avoids_unbound_local(): freed = await status_updater._free_workspace_address_space(operation) assert freed is False + + +async def test_merge_operation_steps_appends_missing_steps_and_recomputes_indices(): + status_updater = DeploymentStatusUpdater() + step1 = OperationStep( + id="step-1", + stepTitle="Step 1", + resourceId="res-1", + templateStepId="main", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup step", + resourceId="res-ws", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId="res-1", + resourcePath="/workspaces/res-ws/workspace-services/res-1", + action=RequestAction.UnInstall, + steps=[step1, step2], + ) + fresh_step1 = copy.deepcopy(step1) + fresh_op = Operation( + id=OPERATION_ID, + resourceId="res-1", + resourcePath="/workspaces/res-ws/workspace-services/res-1", + action=RequestAction.UnInstall, + steps=[fresh_step1], + ) + + step_to_update, is_last_step, current_step_index = status_updater._merge_operation_steps( + in_memory_op=in_memory_op, + fresh_op=fresh_op, + step_id_to_update="step-1", + step_status=Status.Deleted, + step_message="Uninstall done", + ) + + assert len(fresh_op.steps) == 2 + assert step_to_update.id == "step-1" + assert step_to_update.status == Status.Deleted + assert is_last_step is False + assert current_step_index == 0 + assert fresh_op.steps[1].id == "step-2" + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_etag_conflict_retry_merges_legacy_synthetic_cleanup_step(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + # Legacy operation with only 1 step initially in Cosmos DB + legacy_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleting, + steps=[step1], + etag="initial-etag", + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + # When get_operation_by_id is called on retry, it returns a 1-step operation as still stored in Cosmos DB + def mock_get_op(op_id): + return copy.deepcopy(legacy_op) + + status_updater.operations_repo.get_operation_by_id.side_effect = mock_get_op + + saved_operations = [] + attempt_count = 0 + + async def mock_op_update(op): + nonlocal attempt_count + attempt_count += 1 + saved_operations.append(copy.deepcopy(op)) + if attempt_count == 1: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_op_update + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + # Verify retry occurred + assert attempt_count >= 2 + # On the 2nd attempt, the merged operation must have 2 steps and status PipelineRunning + second_saved_op = saved_operations[1] + assert len(second_saved_op.steps) == 2 + assert second_saved_op.status == Status.PipelineRunning + assert second_saved_op.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + # send_deployment_message must have been called for the cleanup step (no AssertionError!) + send_deployment_message_mock.assert_called_once() + + +async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_terminal_persist_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup address space", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Updating + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + # All attempts to write operation fail (including the final-delivery write) + status_updater.operations_repo.update_item.side_effect = CosmosAccessConditionFailedError() + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + # Must return False so Service Bus abandons/dead-letters the message instead of discarding it + assert result is False From 148bd618930d5fbd467c6aca530b9ee35ebfbf49 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:59:11 +0000 Subject: [PATCH 44/90] Address Copilot feedback on etag exclusion, template validation, and terminal cleanup persistence --- api_app/db/repositories/operations.py | 4 +- api_app/db/repositories/resource_templates.py | 6 +- .../service_bus/deployment_status_updater.py | 65 ++++++++----- .../test_operation_repository.py | 25 +++++ .../test_resource_templates_repository.py | 35 +++++-- .../test_deployment_status_update.py | 95 +++++++++++++++++++ 6 files changed, 198 insertions(+), 32 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 26ec3da7f9..15eabe92c7 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -25,7 +25,7 @@ async def create(cls): return cls async def save_item(self, item: Operation): - item_dict = item.model_dump() + item_dict = item.model_dump(exclude={"etag"}) item_dict.pop("_etag", None) response = await self.container.create_item(body=item_dict) if isinstance(response, dict) and "_etag" in response: @@ -34,7 +34,7 @@ async def save_item(self, item: Operation): async def update_item(self, item: Operation, etag: Optional[str] = None) -> Operation: etag_to_match = etag or getattr(item, "etag", None) - item_dict = item.model_dump() + item_dict = item.model_dump(exclude={"etag"}) item_dict.pop("_etag", None) if etag_to_match: response = await self.container.replace_item( diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index d7d5a1d635..e9f19be744 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -139,7 +139,7 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if "pipeline" in template_input.json_schema: pipeline = template_input.json_schema["pipeline"] - self._validate_pipeline_has_unique_step_ids(pipeline) + self._validate_pipeline_has_unique_step_ids(pipeline, resource_type) template["pipeline"] = pipeline if "allOf" in template_input.json_schema: @@ -176,7 +176,7 @@ async def create_and_validate_template(self, template_input: ResourceTemplateInC created_template = await self.create_template(template_input, resource_type, workspace_service_template_name) return self.enrich_template(created_template) - def _validate_pipeline_has_unique_step_ids(self, pipeline): + def _validate_pipeline_has_unique_step_ids(self, pipeline, resource_type: ResourceType = None): if pipeline is None: return @@ -195,6 +195,8 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline): raise InvalidInput(f"Invalid template - duplicate stepIds are not allowed. stepId: {step_id}") if step_id == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if resource_type != ResourceType.WorkspaceService: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in '{ResourceType.WorkspaceService}' templates") if action != RequestAction.UnInstall: raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in the '{RequestAction.UnInstall}' pipeline") if i != num_steps - 1: diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index de3b8061cd..d425de0265 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -323,23 +323,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if is_cleanup_step and step_to_update.is_success(): logger.error(f"[CLEANUP_FINALIZATION_FAILED] Error in post-cleanup writes: {e}", exc_info=True) if is_final_delivery: - # Final delivery handling: must persist terminal state and complete message - terminal_persisted = False + # Final delivery handling: must retry and verify all required resource and operation writes + workspace_persisted = False try: - fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) - fresh_step, fresh_is_last, _ = self._merge_operation_steps( - in_memory_op=operation, - fresh_op=fresh_op, - step_id_to_update=step_to_update.id, - step_status=step_to_update.status, - step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, - ) - await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) - await self.operations_repo.update_item(fresh_op) - terminal_persisted = True - except Exception as op_err: - logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) - + resource = await self.resource_repo.get_resource_by_id(resource_id) + resource.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource) + resource_dict = await self.resource_repo.get_resource_dict_by_id(resource_id) + resource_to_persist = self.create_updated_resource_document(resource_dict, message) + await self.resource_repo.update_item_dict(resource_to_persist) + workspace_persisted = True + except Exception as ws_err: + logger.error(f"Failed to persist workspace resource status/outputs on final delivery: {ws_err}", exc_info=True) + + primary_persisted = False main_step = next( (op_step for op_step in operation.steps if op_step.templateStepId == "main" @@ -352,16 +349,37 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if primary_resource.deploymentStatus != Status.Deleted: primary_resource.deploymentStatus = Status.Deleted await self.resource_repo.update_item(primary_resource) + primary_persisted = True except Exception as prim_err: logger.error(f"Failed to restore primary resource to Deleted on final delivery: {prim_err}", exc_info=True) + else: + primary_persisted = True - if terminal_persisted: - logger.info(f"Terminal operation status successfully persisted on final delivery for operation {operation.id}. Completing message.") + operation_persisted = False + if workspace_persisted and primary_persisted: + try: + fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) + fresh_step, fresh_is_last, _ = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) + await self.operations_repo.update_item(fresh_op) + operation_persisted = True + except Exception as op_err: + logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) + + if workspace_persisted and primary_persisted and operation_persisted: + logger.info(f"All required resource and operation writes successfully persisted on final delivery for operation {operation.id}. Completing message.") return True else: logger.error( - f"[CLEANUP_FINALIZATION_DEAD_LETTER] Could not persist terminal operation status for " - f"operation {operation.id} on final delivery {MAX_CLEANUP_DELIVERY_COUNT}. " + f"[CLEANUP_FINALIZATION_DEAD_LETTER] Could not persist all required states for " + f"operation {operation.id} on final delivery {MAX_CLEANUP_DELIVERY_COUNT} " + f"(workspace={workspace_persisted}, primary={primary_persisted}, op={operation_persisted}). " f"Returning False so message is dead-lettered for manual recovery." ) return False @@ -654,6 +672,9 @@ async def _get_address_cleanup_details(self, operation: Operation) -> Optional[T if main_step is None: return None resource_to_persist = await self.resource_repo.get_resource_dict_by_id(main_step.resourceId) + if resource_to_persist.get("resourceType") not in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value): + return None + address_to_free = resource_to_persist.get("properties", {}).get("address_space") parent_workspace_id = resource_to_persist.get("workspaceId") resource_id = resource_to_persist.get("id") @@ -699,8 +720,8 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: for attempt in range(MAX_CLEANUP_RETRIES): try: workspace = await self.workspace_repo.get_workspace_by_id(parent_workspace_id) - workspace_address_spaces = workspace.properties.get("address_spaces", []) - if address_to_free not in workspace_address_spaces: + workspace_address_spaces = workspace.properties.get("address_spaces") or [] + if not isinstance(workspace_address_spaces, list) or address_to_free not in workspace_address_spaces: service_dict["properties"]["address_space_freed"] = True await self.resource_repo.update_item_dict(service_dict) return True diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index f86a7a92dd..59af8e9817 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -143,6 +143,8 @@ async def test_update_item_with_etag_calls_replace_item(operations_repo): assert call_kwargs["item"] == "op-1" assert call_kwargs["etag"] == "old-etag" assert call_kwargs["match_condition"] == MatchConditions.IfNotModified + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] assert result.etag == "new-etag" @@ -159,9 +161,32 @@ async def test_update_item_without_etag_calls_upsert_item(operations_repo): result = await operations_repo.update_item(op) operations_repo._container.upsert_item.assert_awaited_once() + call_kwargs = operations_repo._container.upsert_item.call_args[1] + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] assert result.etag == "new-etag" +async def test_save_item_excludes_etag_from_body(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={"_etag": "\"created-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="initial-etag" + ) + await operations_repo.save_item(op) + + operations_repo._container.create_item.assert_awaited_once() + call_kwargs = operations_repo._container.create_item.call_args[1] + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] + assert op.etag == "created-etag" + + async def test_update_item_raises_cosmos_access_condition_failed_error_on_mismatch(operations_repo): from azure.cosmos.exceptions import CosmosAccessConditionFailedError operations_repo._container = MagicMock() diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 4392027743..1d9fae45d6 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -320,14 +320,21 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel }, ], ) -async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails(resource_template_repo, input_user_resource_template, pipeline): - input_user_resource_template.json_schema["pipeline"] = pipeline +async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails(resource_template_repo, input_workspace_service_template, pipeline): + input_workspace_service_template.json_schema["pipeline"] = pipeline with pytest.raises(InvalidInput): - await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) + await resource_template_repo.create_template(input_workspace_service_template, ResourceType.WorkspaceService) -@patch('db.repositories.resource_templates.ResourceTemplateRepository.save_item') -async def test_create_template_with_valid_address_space_cleanup_as_final_uninstall_step_succeeds(_, resource_template_repo, input_user_resource_template): +@pytest.mark.parametrize( + "non_ws_resource_type", + [ + ResourceType.UserResource, + ResourceType.Workspace, + ResourceType.SharedService, + ], +) +async def test_create_template_with_address_space_cleanup_on_non_workspace_service_fails(resource_template_repo, input_user_resource_template, non_ws_resource_type): input_user_resource_template.json_schema["pipeline"] = { "uninstall": [ {"stepId": "main"}, @@ -338,5 +345,21 @@ async def test_create_template_with_valid_address_space_cleanup_as_final_uninsta }, ], } - created = await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) + with pytest.raises(InvalidInput, match=f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in '{ResourceType.WorkspaceService}' templates"): + await resource_template_repo.create_template(input_user_resource_template, non_ws_resource_type) + + +@patch('db.repositories.resource_templates.ResourceTemplateRepository.save_item') +async def test_create_template_with_valid_address_space_cleanup_as_final_uninstall_step_succeeds(_, resource_template_repo, input_workspace_service_template): + input_workspace_service_template.json_schema["pipeline"] = { + "uninstall": [ + {"stepId": "main"}, + { + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ], + } + created = await resource_template_repo.create_template(input_workspace_service_template, ResourceType.WorkspaceService) assert created.pipeline diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 3484f750de..821f0cfc47 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -3231,3 +3231,98 @@ async def mock_get_resource_by_id(resource_uuid): # Must return False so Service Bus abandons/dead-letters the message instead of discarding it assert result is False + + +async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_workspace_resource_write_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup address space", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item = AsyncMock() + + status_updater.resource_repo = AsyncMock() + # Workspace resource update always fails + status_updater.resource_repo.update_item.side_effect = Exception("Cosmos DB unavailable") + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + # Must return False so message is dead-lettered, and operation must NOT be marked Deleted + assert result is False + assert operation.status != Status.Deleted + + +async def test_get_address_cleanup_details_returns_none_if_resource_type_not_workspace_service(): + user_resource_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step = OperationStep( + id="step-1", + stepTitle="Uninstall user resource", + resourceId=user_resource_id, + resourceType=ResourceType.UserResource, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=user_resource_id, + resourcePath=f"/workspaces/{parent_workspace_id}/user-resources/{user_resource_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": user_resource_id, + "resourceType": ResourceType.UserResource, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.0.1.0/24"}, + } + + details = await status_updater._get_address_cleanup_details(operation) + assert details is None From 577036484481428b2a183e5d1f64ed9682507740 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:09:01 +0000 Subject: [PATCH 45/90] Correct delivery_count indexing in updater and update pipeline schema documentation --- api_app/service_bus/deployment_status_updater.py | 8 ++++---- .../test_service_bus/test_deployment_status_update.py | 3 ++- docs/tre-templates/pipeline-templates/pipeline-schema.md | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index d425de0265..09eaf5ff24 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -32,10 +32,10 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 # Azure Service Bus default max delivery count is 10. -# The AMQP delivery_count reported by ServiceBusReceivedMessage is zero-based: -# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. -# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. -FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 +# ServiceBusReceivedMessage.delivery_count is 1-based in Azure Service Bus: +# attempt 1 has delivery_count = 1, and attempt 10 (the final processable attempt before DLQ) has delivery_count = 10. +# When delivery_count >= MAX_CLEANUP_DELIVERY_COUNT, this is the final delivery attempt before Service Bus dead-letters the message. +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT class AddressSpaceConflictError(Exception): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 821f0cfc47..1c96a35d4e 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1775,8 +1775,9 @@ async def mock_get_resource_by_id(resource_uuid): (0, False), (1, False), (8, False), - (9, True), + (9, False), (10, True), + (11, True), (None, False), ]) async def test_process_message_delivery_count_handling(delivery_count, expected_final): diff --git a/docs/tre-templates/pipeline-templates/pipeline-schema.md b/docs/tre-templates/pipeline-templates/pipeline-schema.md index aabc62fbbe..065730e192 100644 --- a/docs/tre-templates/pipeline-templates/pipeline-schema.md +++ b/docs/tre-templates/pipeline-templates/pipeline-schema.md @@ -96,7 +96,7 @@ Use `address-space-cleanup` as the `stepId` for a trailing workspace upgrade tha } ``` -This identifier is reserved for that cleanup behavior and may be used by multiple templates. Step IDs only need to be unique within a single template's pipeline. +This identifier is reserved for that cleanup behavior and is only permitted as the final step in a `workspace-service` `uninstall` pipeline. Step IDs only need to be unique within a single template's pipeline. ## Notes - Each step is executed in serial, in the order defined in the template From ab994151804b3c0f6e4cf6e19bd2e63e6fbba3c5 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:19:12 +0000 Subject: [PATCH 46/90] Roll back operations on dispatch failure and correct repository mocking in cleanup tests --- .../service_bus/resource_request_sender.py | 58 +++++++++----- .../test_deployment_status_update.py | 4 + .../test_resource_request_sender.py | 76 ++++++++++++++++++- 3 files changed, 118 insertions(+), 20 deletions(-) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e5fe83ba84..c7567fcc3c 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -8,10 +8,10 @@ from models.domain.request_action import RequestAction from models.domain.resource import Resource -from models.domain.operation import Operation +from models.domain.operation import Operation, Status from db.repositories.operations import OperationRepository -from services.logging import tracer +from services.logging import logger, tracer async def send_resource_request_message(resource: Resource, operations_repo: OperationRepository, resource_repo: ResourceRepository, user: User, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, action: RequestAction = RequestAction.Install, is_cascade: str = False) -> Operation: @@ -45,23 +45,43 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope resource_template_repo=resource_template_repo) current_span.set_attribute("operation_id", operation.id) - # prep the first step to send in SB - # resource at this point is the original object with unmaskked values - first_step = operation.steps[0] - current_span.set_attribute("step_id", first_step.id) - resource_to_send = await update_resource_for_step( - operation_step=first_step, - resource_repo=resource_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - root_resource=resource, - step_resource=None, - resource_to_update_id=first_step.resourceId, - primary_action=action, - user=user) + try: + # prep the first step to send in SB + # resource at this point is the original object with unmaskked values + first_step = operation.steps[0] + current_span.set_attribute("step_id", first_step.id) + resource_to_send = await update_resource_for_step( + operation_step=first_step, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + root_resource=resource, + step_resource=None, + resource_to_update_id=first_step.resourceId, + primary_action=action, + user=user) - # create + send the message - content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) + # create + send the message + content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) + await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) + except Exception: + logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") + try: + await operations_repo.delete_item(operation.id) + except Exception: + logger.exception(f"Failed to delete undispatched operation {operation.id}, attempting to mark failed") + try: + failure_status = ( + Status.DeploymentFailed if action == RequestAction.Install + else Status.DeletingFailed if action == RequestAction.UnInstall + else Status.UpdatingFailed if action == RequestAction.Upgrade + else Status.ActionFailed + ) + operation.status = failure_status + operation.message = f"Failed to dispatch initial deployment message: {action}" + await operations_repo.update_item(operation) + except Exception: + logger.exception(f"Failed to mark operation {operation.id} as failed") + raise return operation diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 1c96a35d4e..7335bcaac3 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -553,6 +553,8 @@ async def test_workspace_service_uninstall_frees_address_space_from_root_main_st status_updater.workspace_repo = AsyncMock() status_updater.resource_template_repo = AsyncMock() status_updater.resource_history_repo = AsyncMock() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] status_updater.resource_repo.get_resource_dict_by_id.return_value = { "id": workspace_service_id, "resourceType": ResourceType.WorkspaceService, @@ -722,6 +724,8 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta status_updater = DeploymentStatusUpdater() await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] complete_message = await status_updater.process_message(service_bus_received_message_mock) assert complete_message is True diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 3485d265d1..95cdb562bd 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -4,7 +4,7 @@ import uuid from azure.servicebus import ServiceBusMessage -from mock import AsyncMock, patch +from mock import AsyncMock, patch, MagicMock from resources import strings from models.schemas.resource import ResourcePatch from service_bus.helpers import ( @@ -21,6 +21,7 @@ send_resource_request_message, RequestAction, ) +from models.domain.operation import Status from azure.cosmos.exceptions import CosmosAccessConditionFailedError pytestmark = pytest.mark.asyncio @@ -191,3 +192,76 @@ async def test_multi_step_document_retries( # check it tried to patch and re-get the item the first time + all the retries assert len(resource_repo.patch_resource.mock_calls) == (num_retries + 1) assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_deletes_operation_on_dispatch_failure( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.delete_item = AsyncMock() + + resource_repo_mock = AsyncMock() + resource_template_repo_mock = AsyncMock() + resource_history_repo_mock = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=resource_repo_mock, + user=create_test_user(), + resource_template_repo=resource_template_repo_mock, + resource_history_repo=resource_history_repo_mock, + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_marks_operation_failed_when_delete_fails( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.delete_item = AsyncMock(side_effect=Exception("Cosmos delete error")) + operations_repo_mock.update_item = AsyncMock() + + resource_repo_mock = AsyncMock() + resource_template_repo_mock = AsyncMock() + resource_history_repo_mock = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=resource_repo_mock, + user=create_test_user(), + resource_template_repo=resource_template_repo_mock, + resource_history_repo=resource_history_repo_mock, + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) + operations_repo_mock.update_item.assert_awaited_once_with(operation) + assert operation.status == Status.DeploymentFailed From 5c1892f20adaed60856ebb72e50e578636b8cebb Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:29:28 +0000 Subject: [PATCH 47/90] Persist failure status on dispatch failure and restore 0-based final delivery threshold --- .../service_bus/deployment_status_updater.py | 8 +-- .../service_bus/resource_request_sender.py | 27 +++++---- .../test_deployment_status_update.py | 3 +- .../test_resource_request_sender.py | 57 ++++++------------- 4 files changed, 34 insertions(+), 61 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 09eaf5ff24..55b24cfafc 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -32,10 +32,10 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 # Azure Service Bus default max delivery count is 10. -# ServiceBusReceivedMessage.delivery_count is 1-based in Azure Service Bus: -# attempt 1 has delivery_count = 1, and attempt 10 (the final processable attempt before DLQ) has delivery_count = 10. -# When delivery_count >= MAX_CLEANUP_DELIVERY_COUNT, this is the final delivery attempt before Service Bus dead-letters the message. -FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT +# The AMQP delivery_count reported by ServiceBusReceivedMessage represents prior unsuccessful delivery attempts (0-based): +# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. +# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 class AddressSpaceConflictError(Exception): diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index c7567fcc3c..e1c5838f4b 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -67,21 +67,20 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope except Exception: logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") try: - await operations_repo.delete_item(operation.id) + failure_status = ( + Status.DeploymentFailed if action == RequestAction.Install + else Status.DeletingFailed if action == RequestAction.UnInstall + else Status.UpdatingFailed if action == RequestAction.Upgrade + else Status.ActionFailed + ) + operation.status = failure_status + operation.message = f"Failed to dispatch initial deployment message: {action}" + if operation.steps: + operation.steps[0].status = failure_status + operation.steps[0].message = f"Failed to dispatch initial deployment message: {action}" + await operations_repo.update_item(operation) except Exception: - logger.exception(f"Failed to delete undispatched operation {operation.id}, attempting to mark failed") - try: - failure_status = ( - Status.DeploymentFailed if action == RequestAction.Install - else Status.DeletingFailed if action == RequestAction.UnInstall - else Status.UpdatingFailed if action == RequestAction.Upgrade - else Status.ActionFailed - ) - operation.status = failure_status - operation.message = f"Failed to dispatch initial deployment message: {action}" - await operations_repo.update_item(operation) - except Exception: - logger.exception(f"Failed to mark operation {operation.id} as failed") + logger.exception(f"Failed to persist failure state for operation {operation.id}") raise return operation diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 7335bcaac3..034261229f 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1779,9 +1779,8 @@ async def mock_get_resource_by_id(resource_uuid): (0, False), (1, False), (8, False), - (9, False), + (9, True), (10, True), - (11, True), (None, False), ]) async def test_process_message_delivery_count_handling(delivery_count, expected_final): diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 95cdb562bd..9f1409d819 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -194,53 +194,27 @@ async def test_multi_step_document_retries( assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) +@pytest.mark.parametrize( + "action, expected_status", + [ + (RequestAction.Install, Status.DeploymentFailed), + (RequestAction.Upgrade, Status.UpdatingFailed), + (RequestAction.UnInstall, Status.DeletingFailed), + ], +) @patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) @patch("service_bus.resource_request_sender.update_resource_for_step") -async def test_send_resource_request_message_deletes_operation_on_dispatch_failure( - update_resource_mock, - send_deployment_mock, -): - resource = create_test_resource() - operation = create_sample_operation(resource.id, RequestAction.Install) - - operations_repo_mock = AsyncMock() - operations_repo_mock.create_operation_item.return_value = operation - operations_repo_mock.delete_item = AsyncMock() - - resource_repo_mock = AsyncMock() - resource_template_repo_mock = AsyncMock() - resource_history_repo_mock = AsyncMock() - - resource_to_send_mock = MagicMock() - resource_to_send_mock.get_resource_request_message_payload.return_value = {} - update_resource_mock.return_value = resource_to_send_mock - - with pytest.raises(Exception, match="Service Bus unavailable"): - await send_resource_request_message( - resource=resource, - operations_repo=operations_repo_mock, - resource_repo=resource_repo_mock, - user=create_test_user(), - resource_template_repo=resource_template_repo_mock, - resource_history_repo=resource_history_repo_mock, - action=RequestAction.Install, - ) - - operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) - - -@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) -@patch("service_bus.resource_request_sender.update_resource_for_step") -async def test_send_resource_request_message_marks_operation_failed_when_delete_fails( +async def test_send_resource_request_message_persists_failure_state_on_dispatch_failure( update_resource_mock, send_deployment_mock, + action, + expected_status, ): resource = create_test_resource() - operation = create_sample_operation(resource.id, RequestAction.Install) + operation = create_sample_operation(resource.id, action) operations_repo_mock = AsyncMock() operations_repo_mock.create_operation_item.return_value = operation - operations_repo_mock.delete_item = AsyncMock(side_effect=Exception("Cosmos delete error")) operations_repo_mock.update_item = AsyncMock() resource_repo_mock = AsyncMock() @@ -259,9 +233,10 @@ async def test_send_resource_request_message_marks_operation_failed_when_delete_ user=create_test_user(), resource_template_repo=resource_template_repo_mock, resource_history_repo=resource_history_repo_mock, - action=RequestAction.Install, + action=action, ) - operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) operations_repo_mock.update_item.assert_awaited_once_with(operation) - assert operation.status == Status.DeploymentFailed + assert operation.status == expected_status + assert operation.steps[0].status == expected_status + assert "Failed to dispatch" in operation.message From 18d5faf41501dbc3d350d82af8909c3c86fd19ba Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:39:38 +0000 Subject: [PATCH 48/90] Fix terminal status mapping and missing resource failure persistence in updater --- .../service_bus/deployment_status_updater.py | 74 +++++++---- .../test_deployment_status_update.py | 116 +++++++++++++++++- 2 files changed, 163 insertions(+), 27 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 55b24cfafc..2340d33ab3 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -247,9 +247,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass await self.update_overall_operation_status(operation, step_to_update, is_last_step) await self.operations_repo.update_item(operation) return True @@ -292,10 +295,13 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - if primary_resource.deploymentStatus != Status.Deleted: - primary_resource.deploymentStatus = Status.Deleted - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass # update the overall headline operation status await self.update_overall_operation_status(operation, step_to_update, is_last_step) @@ -551,21 +557,30 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage result = True except EntityDoesNotExist: - # Marking as true as this message will never succeed anyways and should be removed from the queue. - result = True logger.exception(strings.DEPLOYMENT_STATUS_ID_NOT_FOUND.format(message.id)) if operation is not None and step_to_update is not None: try: - step_to_update.status = ( - Status.UpdatingFailed if operation.action == RequestAction.Upgrade - else Status.DeletingFailed if operation.action == RequestAction.UnInstall - else Status.DeploymentFailed - ) + failure_status = self.get_failure_status_for_action(operation.action) + step_to_update.status = failure_status step_to_update.message = f"Resource {message.id} not found in database: operation aborted." - await self.update_overall_operation_status(operation, step_to_update, is_last_step=True) + operation.status = failure_status + operation.message = f"Resource {message.id} not found in database: operation aborted." + operation.updatedWhen = get_timestamp() + if operation.resourceId and operation.resourceId != str(message.id): + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) + primary_resource.deploymentStatus = failure_status + await self.resource_repo.update_item(primary_resource) + except Exception: + pass await self.operations_repo.update_item(operation) + result = True except Exception: logger.exception(f"Failed to persist terminal failure status for operation {operation.id} when resource {message.id} was missing") + result = False + else: + # Marking as true as this message will never succeed anyways and should be removed from the queue. + result = True except Exception: logger.exception("Failed to update status") @@ -783,10 +798,13 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - if primary_resource.deploymentStatus != operation.status: - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != operation.status: + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass return operation.status = Status.PipelineRunning @@ -804,9 +822,12 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper break if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass if step.is_success() and is_last_step: if all(op_step.is_success() for op_step in operation.steps): @@ -821,9 +842,12 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper break if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass def get_success_status_for_action(self, action: RequestAction): status = Status.ActionSucceeded diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 034261229f..a0380fe45d 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -2609,7 +2609,63 @@ async def test_delayed_earlier_step_message_ignored_when_cleanup_terminal(): status_updater.operations_repo.update_item.assert_not_called() -async def test_missing_step_resource_persists_terminal_failure_operation_state(): +@pytest.mark.parametrize( + "action, expected_status", + [ + (RequestAction.Install, Status.DeploymentFailed), + (RequestAction.UnInstall, Status.DeletingFailed), + (RequestAction.Upgrade, Status.UpdatingFailed), + ("custom-action", Status.ActionFailed), + ], +) +async def test_missing_step_resource_persists_terminal_failure_operation_state(action, expected_status): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Execute action on workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=action, + templateStepId="main", + status=Status.Deploying, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=action, + status=Status.PipelineRunning, + steps=[step1], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deployed, + message="status update", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is True + assert step1.status == expected_status + assert "not found in database" in step1.message + assert operation.status == expected_status + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + +async def test_missing_primary_resource_in_multi_step_operation_persists_failure_without_raising(): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -2622,6 +2678,15 @@ async def test_missing_step_resource_persists_terminal_failure_operation_state() templateStepId="main", status=Status.Deleting, ) + step2 = OperationStep( + id="step-2", + stepTitle="Address space cleanup", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId="address-space-cleanup", + status=Status.AwaitingUpdate, + ) operation = Operation( id=OPERATION_ID, resourceId=workspace_service_id, @@ -2629,7 +2694,7 @@ async def test_missing_step_resource_persists_terminal_failure_operation_state() resourceVersion=0, action=RequestAction.UnInstall, status=Status.PipelineRunning, - steps=[step1], + steps=[step1, step2], ) status_updater = DeploymentStatusUpdater() @@ -2656,6 +2721,53 @@ async def test_missing_step_resource_persists_terminal_failure_operation_state() status_updater.operations_repo.update_item.assert_called_once_with(operation) +async def test_missing_resource_returns_false_when_operation_update_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Install workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.Install, + templateStepId="main", + status=Status.Deploying, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.Install, + status=Status.Deploying, + steps=[step1], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item.side_effect = Exception("Cosmos DB connection timeout") + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deployed, + message="deployed", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is False + assert step1.status == Status.DeploymentFailed + assert operation.status == Status.DeploymentFailed + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + async def test_cleanup_failure_in_get_address_cleanup_details_on_final_delivery_marks_failed(): workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" From c1ae6c5bfc9a4383f3a876eab94ce5dfd10890f7 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:49:33 +0000 Subject: [PATCH 49/90] Reduce comment verbosity and remove obvious comments across cleanup implementation and tests --- .../service_bus/deployment_status_updater.py | 48 +++------- .../test_deployment_status_update.py | 87 ++----------------- 2 files changed, 17 insertions(+), 118 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 2340d33ab3..99d257b3c5 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -31,10 +31,7 @@ MAX_CLEANUP_RETRIES = 3 MAX_CLEANUP_DELIVERY_COUNT = 10 -# Azure Service Bus default max delivery count is 10. -# The AMQP delivery_count reported by ServiceBusReceivedMessage represents prior unsuccessful delivery attempts (0-based): -# attempt 1 has delivery_count = 0, and attempt 10 (the final processable attempt) has delivery_count = 9. -# Waiting for delivery_count >= 10 would cause attempt 10 (count 9) to be abandoned into the dead-letter queue. +# 0-based: delivery_count counts prior unsuccessful delivery attempts before DLQ FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 @@ -154,9 +151,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") - # If address space cleanup for this uninstall operation is already in a terminal state (completed or failed), - # the entire operation is already finalized. Ignore any delayed or duplicate messages for ANY step in this - # operation to prevent overwriting resources with stale status or reopening the operation. + # Ignore duplicate messages if uninstall cleanup is already terminal if ( operation.action == RequestAction.UnInstall and self._is_cleanup_terminal(operation) @@ -174,11 +169,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_id = uuid.UUID(step_to_update.resourceId) - # If this is the main step of an uninstall succeeding for a workspace service with an address space, - # ensure the trailing workspace upgrade fallback step is appended BEFORE calculating - # and persisting the main step's overall status. This ensures is_last_step is False - # and the operation is persisted as PipelineRunning continuously in Cosmos DB, eliminating any - # window where resource_has_active_operation sees the operation as Deleted. + # Ensure workspace upgrade step is appended before calculating overall status if ( step_to_update.templateStepId == "main" and step_to_update.resourceId == operation.resourceId @@ -200,10 +191,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage and is_last_step ) - # For the address space cleanup step, we MUST run cleanup before persisting the - # final successful operation status. This keeps the operation active in Cosmos DB - # (so resource_has_active_operation blocks new allocations and workspace updates) - # until the cleanup has been durably committed. + # Run cleanup before marking the operation complete if is_cleanup_step and step_to_update.is_success(): try: freed = await self._free_workspace_address_space(operation) @@ -287,7 +275,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.resource_repo.update_item_dict(resource_to_persist) if is_cleanup_step and step_to_update.is_success(): - # Restore the primary resource to Deleted when cleanup succeeds main_step = next( (op_step for op_step in operation.steps if op_step.templateStepId == "main" @@ -306,7 +293,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # update the overall headline operation status await self.update_overall_operation_status(operation, step_to_update, is_last_step) - # save the operation with final status LAST, after all resource updates are committed for attempt in range(MAX_CLEANUP_RETRIES): try: await self.operations_repo.update_item(operation) @@ -329,7 +315,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if is_cleanup_step and step_to_update.is_success(): logger.error(f"[CLEANUP_FINALIZATION_FAILED] Error in post-cleanup writes: {e}", exc_info=True) if is_final_delivery: - # Final delivery handling: must retry and verify all required resource and operation writes workspace_persisted = False try: resource = await self.resource_repo.get_resource_by_id(resource_id) @@ -420,7 +405,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage cleanup_details = await self._get_address_cleanup_details(operation) if cleanup_details: address_to_free, parent_workspace_id, service_resource_id, _ = cleanup_details - # Validate active-service ownership before enqueueing this upgrade workspace_services_repo = getattr(self, "workspace_services_repo", None) if workspace_services_repo is None: from db.repositories.workspace_services import WorkspaceServiceRepository @@ -508,7 +492,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.operations_repo.update_item(operation) return True - # Post-send persistence: send succeeded, now update operation status for cleanup step if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: for attempt in range(MAX_CLEANUP_RETRIES): try: @@ -594,11 +577,7 @@ def _merge_operation_steps( step_status: Status, step_message: str, ) -> Tuple[OperationStep, bool, int]: - """ - Merges in-memory appended steps (e.g. dynamically appended cleanup step on legacy operations) - into the fresh operation reloaded from Cosmos DB during an ETag conflict retry. - Recomputes step_to_update, is_last_step, and current_step_index for fresh_op. - """ + """Merge in-memory steps into fresh operation reloaded during ETag retry.""" if fresh_op.steps is None: fresh_op.steps = [] @@ -700,10 +679,7 @@ async def _get_address_cleanup_details(self, operation: Operation) -> Optional[T return address_to_free, parent_workspace_id, resource_id, resource_to_persist async def _free_workspace_address_space(self, operation: Operation) -> bool: - """ - Frees the address space owned by the uninstalled WorkspaceService after - the corresponding workspace upgrade has succeeded. - """ + """Free address space from workspace after successful upgrade.""" address_to_free = parent_workspace_id = resource_id = "" try: cleanup_details = await self._get_address_cleanup_details(operation) @@ -712,14 +688,12 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details - # Check if cleanup for this step was already completed (idempotency on redelivery) if self._is_cleanup_completed(operation): return True - # Check if the uninstalled service already has the address_space_freed marker if service_dict.get("properties", {}).get("address_space_freed"): return True - # Check if address_to_free has been assigned to another active workspace service on this workspace (fail closed) + workspace_services_repo = getattr(self, "workspace_services_repo", None) if workspace_services_repo is None: from db.repositories.workspace_services import WorkspaceServiceRepository @@ -744,9 +718,7 @@ async def _free_workspace_address_space(self, operation: Operation) -> bool: workspace_patch = ResourcePatch() workspace_patch.properties = {"address_spaces": new_address_spaces} - # Note: patch_workspace with force_version_update=False updates the Cosmos DB record only; - # it does not trigger an independent deployment operation. Infrastructure updates (Terraform) - # are driven by the trailing workspace upgrade step in the uninstall pipeline. + # Update address spaces in Cosmos DB without triggering deployment await self.workspace_repo.patch_workspace( workspace, workspace_patch, @@ -781,14 +753,14 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.message = step.message return - # If address space cleanup has completed for an uninstall, only finalize as Deleted if it is the last step and all steps succeeded + # Finalize as Deleted only when all steps including cleanup succeed if self._is_cleanup_completed(operation) and operation.action == RequestAction.UnInstall: if is_last_step and all(op_step.is_success() for op_step in operation.steps): operation.status = Status.Deleted operation.message = "Multi step pipeline completed successfully" return - # If address space cleanup has failed terminally for an uninstall, do not reopen to PipelineRunning + # Do not reopen failed cleanup uninstall to PipelineRunning if self._is_cleanup_failed(operation) and operation.action == RequestAction.UnInstall: operation.status = Status.DeletingFailed operation.message = f"Multi step pipeline failed on step {strings.ADDRESS_SPACE_CLEANUP_STEP_ID}" diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index a0380fe45d..cb2ab00061 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -482,16 +482,13 @@ async def test_workspace_service_uninstall_frees_address_space( } service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) - # Mock the operation showing RequestAction.UnInstall operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) operations_repo_mock.return_value.get_operation_by_id.return_value = operation - # Mock WorkspaceService resource returned by get_resource_by_id workspace_service_mock = MagicMock() workspace_service_mock.deploymentStatus = None resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock - # Mock resource dict representation returned by get_resource_dict_by_id workspace_service_dict = { "id": workspace_service_id, "resourceType": ResourceType.WorkspaceService.value, @@ -502,7 +499,6 @@ async def test_workspace_service_uninstall_frees_address_space( } resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict - # Mock parent workspace containing the address space to free parent_workspace = create_sample_workspace_object(parent_workspace_id) parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} parent_workspace.etag = "parent-workspace-etag" @@ -690,16 +686,13 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta } service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) - # Mock the operation showing RequestAction.UnInstall operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) operations_repo_mock.return_value.get_operation_by_id.return_value = operation - # Mock WorkspaceService resource returned by get_resource_by_id workspace_service_mock = MagicMock() workspace_service_mock.deploymentStatus = None resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock - # Mock resource dict representation returned by get_resource_dict_by_id workspace_service_dict = { "id": workspace_service_id, "resourceType": ResourceType.WorkspaceService, @@ -710,15 +703,12 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta } resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict - # Mock parent workspace containing the address space to free parent_workspace = create_sample_workspace_object(parent_workspace_id) parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} parent_workspace.etag = "parent-workspace-etag" workspace_repo = AsyncMock() workspace_repo.get_workspace_by_id.return_value = parent_workspace - - # First attempt raises CosmosAccessConditionFailedError, second succeeds workspace_repo.patch_workspace.side_effect = [CosmosAccessConditionFailedError(), None] workspace_repo_mock.return_value = workspace_repo @@ -731,9 +721,7 @@ async def test_workspace_service_uninstall_frees_address_space_with_retry_on_eta assert complete_message is True send_deployment_message_mock.assert_called_once() await status_updater._free_workspace_address_space(operation) - # One read prepares the synthetic upgrade, then two reads cover the retry. assert workspace_repo.get_workspace_by_id.call_count == 3 - # Assert patch_workspace called twice assert workspace_repo.patch_workspace.call_count == 2 @@ -771,16 +759,13 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( } service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) - # Mock the operation showing RequestAction.UnInstall operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) operations_repo_mock.return_value.get_operation_by_id.return_value = operation - # Mock WorkspaceService resource returned by get_resource_by_id workspace_service_mock = MagicMock() workspace_service_mock.deploymentStatus = None resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock - # Mock resource dict representation returned by get_resource_dict_by_id workspace_service_dict = { "id": workspace_service_id, "resourceType": ResourceType.WorkspaceService, @@ -791,15 +776,12 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( } resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict - # Mock parent workspace containing the address space to free parent_workspace = create_sample_workspace_object(parent_workspace_id) parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} parent_workspace.etag = "parent-workspace-etag" workspace_repo = AsyncMock() workspace_repo.get_workspace_by_id.return_value = parent_workspace - - # All attempts raise CosmosAccessConditionFailedError workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() workspace_repo_mock.return_value = workspace_repo @@ -809,16 +791,13 @@ async def test_workspace_service_uninstall_logs_error_after_max_retries( status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] complete_message = await status_updater.process_message(service_bus_received_message_mock) - # The uninstall message is complete; cleanup is deferred to the upgrade step. assert complete_message is True send_deployment_message_mock.assert_called_once() workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() complete_message = await status_updater._free_workspace_address_space(operation) assert complete_message is False - # Assert get_workspace_by_id and patch_workspace called max_retries = 3 times assert workspace_repo.get_workspace_by_id.call_count == 4 assert workspace_repo.patch_workspace.call_count == 3 - # Assert we logged the final failure using the module's logger logging_mock.error.assert_called_once() assert "[ADDRESS_SPACE_CLEANUP_FAILED]" in logging_mock.error.call_args[0][0] @@ -1134,7 +1113,6 @@ async def mock_get_resource_by_id(resource_uuid): message="workspace upgrade succeeded" ) - # 1. First attempt: cleanup fails after retries; message is abandoned for redelivery with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): result = await status_updater.update_status_in_database(cleanup_message) @@ -1143,14 +1121,12 @@ async def mock_get_resource_by_id(resource_uuid): assert operation.status == Status.PipelineRunning assert parent_workspace_mock.deploymentStatus == Status.Updating - # 2. Redelivery attempt: cleanup succeeds with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): result = await status_updater.update_status_in_database(cleanup_message) assert result is True assert step2.status == Status.Updated assert operation.status == Status.Deleted - # Primary workspace service must be restored to Deleted, not left as DeletingFailed assert workspace_service_mock.deploymentStatus == Status.Deleted assert parent_workspace_mock.deploymentStatus == Status.Updated @@ -1432,7 +1408,6 @@ async def mock_get_resource_by_id(resource_uuid): with patch.object(status_updater, "_free_workspace_address_space", side_effect=AddressSpaceConflictError("Conflict")): result = await status_updater.update_status_in_database(cleanup_message) - # Terminal conflict: completes message (returns True) so it is not endlessly retried assert result is True assert step2.status == Status.UpdatingFailed assert "Terminal address space conflict" in step2.message @@ -1495,10 +1470,8 @@ async def test_legacy_template_appends_workspace_upgrade_step_before_persisting_ result = await status_updater.update_status_in_database(main_message) assert result is True - # Verify fallback step was appended assert len(operation.steps) == 2 assert operation.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID - # Operation was never saved as Deleted; it was saved as PipelineRunning first_saved_op = status_updater.operations_repo.update_item.call_args_list[0].args[0] assert first_saved_op.status == Status.PipelineRunning @@ -1562,9 +1535,7 @@ async def test_enqueue_cleanup_validates_ownership_and_aborts_before_azure_send( result = await status_updater.update_status_in_database(main_message) - # Completed so it does not redeliver endlessly assert result is True - # send_deployment_message was NOT called: Terraform was NOT told to remove the address from Azure send_deployment_message_mock.assert_not_called() assert operation.status == Status.DeletingFailed assert operation.steps[1].status == Status.UpdatingFailed @@ -1626,7 +1597,6 @@ async def test_enqueue_cleanup_send_failure_abandons_message_for_retry(send_depl result = await status_updater.update_status_in_database(main_message, is_final_delivery=False) - # Abandons message so Service Bus redelivers and retries enqueue assert result is False assert operation.status == Status.PipelineRunning assert operation.steps[1].status == Status.AwaitingUpdate @@ -1687,7 +1657,6 @@ async def test_enqueue_cleanup_send_failure_on_final_delivery_marks_failed(send_ result = await status_updater.update_status_in_database(main_message, is_final_delivery=True) - # Completes message on final delivery so workspace is unblocked assert result is True assert operation.status == Status.DeletingFailed assert operation.steps[1].status == Status.UpdatingFailed @@ -1766,7 +1735,6 @@ async def mock_get_resource_by_id(resource_uuid): with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) - # On final delivery, completes message and marks operation failed to unblock workspace assert result is True assert step2.status == Status.UpdatingFailed assert "delivery attempts" in step2.message @@ -1855,7 +1823,6 @@ async def test_redelivered_main_status_does_not_rerun_completed_cleanup_or_fail_ status_updater.workspace_repo = AsyncMock() status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) status_updater.workspace_services_repo = AsyncMock() - # The address space has been reallocated to a new active service status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [new_active_service_mock] redelivered_main_message = DeploymentStatusUpdateMessage( @@ -1869,7 +1836,6 @@ async def test_redelivered_main_status_does_not_rerun_completed_cleanup_or_fail_ result = await status_updater.update_status_in_database(redelivered_main_message) assert result is True - # Operation and resource must remain successfully Deleted, not failed or reopened assert operation.status == Status.Deleted assert step2.status == Status.Updated assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS @@ -1949,12 +1915,7 @@ async def test_redelivered_main_status_skips_enqueue_when_cleanup_in_progress(se async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success(): - """ - Addresses review comment 1: - Cleanup-status redeliveries are not idempotent after success. If a delayed duplicate - Updating or UpdatingFailed message arrives after the operation is Deleted, ignore it - to avoid writing stale status to the workspace resource. - """ + """Delayed cleanup update after terminal success should be ignored.""" workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -1993,7 +1954,6 @@ async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success( status_updater.operations_repo.get_operation_by_id.return_value = operation status_updater.resource_repo = AsyncMock() - # Delayed / duplicate Updating message arrives for the cleanup step delayed_cleanup_message = DeploymentStatusUpdateMessage( operationId=OPERATION_ID, stepId="step-2", @@ -2005,7 +1965,6 @@ async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success( result = await status_updater.update_status_in_database(delayed_cleanup_message) assert result is True - # Verify no stale updates written to workspace resource or operation status_updater.resource_repo.update_item.assert_not_called() status_updater.resource_repo.update_item_dict.assert_not_called() status_updater.operations_repo.update_item.assert_not_called() @@ -2016,12 +1975,7 @@ async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success( @patch('service_bus.deployment_status_updater.send_deployment_message') async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send_deployment_message_mock): - """ - Addresses review comment 3: - Redelivered main-step message reopening terminal cleanup failure: _is_cleanup_completed was false - for UpdatingFailed/DeletingFailed. Redelivery of the main-step message must not reset operation.status - from DeletingFailed back to PipelineRunning or re-enqueue cleanup. - """ + """Redelivered main-step message should not reopen terminal cleanup failure.""" workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" address_space = "10.1.0.0/22" @@ -2087,7 +2041,6 @@ async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send result = await status_updater.update_status_in_database(redelivered_main_message) assert result is True - # Operation and resource must remain DeletingFailed, not reopened to PipelineRunning assert operation.status == Status.DeletingFailed assert step2.status == Status.UpdatingFailed assert workspace_service_mock.deploymentStatus == Status.DeletingFailed @@ -2096,11 +2049,7 @@ async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send async def test_cleanup_success_persists_workspace_and_primary_resource_before_operation_deleted(): - """ - Addresses review comment 2: - Keep the operation active (PipelineRunning) until all workspace and primary resource writes - are complete, persisting operation with final status (Deleted) last. - """ + """Ensure operation is marked Deleted only after resource writes complete.""" workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" address_space = "10.1.0.0/22" @@ -2190,24 +2139,19 @@ async def tracking_resource_dict_update(res_dict): assert result is True assert operation.status == Status.Deleted - # Check that operations_repo.update_item with Status.Deleted was the LAST call assert len(call_order) >= 3 op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] assert len(op_update_calls) == 1 op_update_index = op_update_calls[0] assert call_order[op_update_index] == ("operations_repo.update_item", Status.Deleted) - # All resource_repo calls must have occurred BEFORE operations_repo.update_item resource_calls = [i for i, call in enumerate(call_order) if call[0].startswith("resource_repo")] for r_idx in resource_calls: assert r_idx < op_update_index async def test_cleanup_conflict_persists_workspace_and_primary_resource_before_operation_failed(): - """ - Addresses review comment 2 (failure path): - In AddressSpaceConflictError, update workspace and primary resource before saving operation as DeletingFailed. - """ + """On address space conflict, update resources before saving operation as DeletingFailed.""" workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -2287,23 +2231,19 @@ async def tracking_resource_update(res): assert result is True assert operation.status == Status.DeletingFailed - # Verify operations_repo.update_item with Status.DeletingFailed was called AFTER resource updates op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] assert len(op_update_calls) == 1 op_update_index = op_update_calls[0] assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] - assert len(resource_calls) == 2 # workspace and primary resource + assert len(resource_calls) == 2 for r_idx in resource_calls: assert r_idx < op_update_index async def test_cleanup_final_delivery_persists_workspace_and_primary_resource_before_operation_failed(): - """ - Addresses review comment 2 (final delivery failure path): - On final delivery cleanup failure, update workspace and primary resource before saving operation as DeletingFailed. - """ + """On final delivery cleanup failure, update resources before saving operation as DeletingFailed.""" workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" @@ -2383,14 +2323,13 @@ async def tracking_resource_update(res): assert result is True assert operation.status == Status.DeletingFailed - # Verify operations_repo.update_item with Status.DeletingFailed was called AFTER resource updates op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] assert len(op_update_calls) == 1 op_update_index = op_update_calls[0] assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] - assert len(resource_calls) == 2 # workspace and primary resource + assert len(resource_calls) == 2 for r_idx in resource_calls: assert r_idx < op_update_index @@ -2546,7 +2485,6 @@ async def test_redelivered_main_status_dispatches_and_subsequent_redelivery_skip ) assert step2.status == Status.Updating - # Subsequent redelivery of main status message skips enqueueing because step2 is now Updating result2 = await status_updater.update_status_in_database(redelivered_main_message) assert result2 is True send_deployment_message_mock.assert_called_once() @@ -2591,7 +2529,6 @@ async def test_delayed_earlier_step_message_ignored_when_cleanup_terminal(): status_updater.operations_repo.get_operation_by_id.return_value = operation status_updater.resource_repo = AsyncMock() - # Delayed failure message arriving for step-1 after cleanup has already finalized the operation delayed_message = DeploymentStatusUpdateMessage( operationId=OPERATION_ID, stepId="step-1", @@ -2820,7 +2757,6 @@ async def mock_get_resource_by_id(resource_uuid): status_updater.operations_repo.get_operation_by_id.return_value = operation status_updater.resource_repo = AsyncMock() status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) - # Simulate Cosmos DB read failure when reading cleanup details status_updater._get_address_cleanup_details = AsyncMock(side_effect=Exception("Cosmos read error")) cleanup_message = DeploymentStatusUpdateMessage( @@ -2890,7 +2826,6 @@ async def test_cleanup_step_with_trailing_step_does_not_prematurely_finalize_as_ status_updater.operations_repo = AsyncMock() status_updater.resource_repo = AsyncMock() - # When step2 completes but is not the last step in the operation await status_updater.update_overall_operation_status(operation, step2, is_last_step=False) assert operation.status == Status.PipelineRunning @@ -3223,7 +3158,6 @@ async def test_etag_conflict_retry_merges_legacy_synthetic_cleanup_step(send_dep status_updater = DeploymentStatusUpdater() status_updater.operations_repo = AsyncMock() - # When get_operation_by_id is called on retry, it returns a 1-step operation as still stored in Cosmos DB def mock_get_op(op_id): return copy.deepcopy(legacy_op) @@ -3265,14 +3199,11 @@ async def mock_op_update(op): result = await status_updater.update_status_in_database(main_message) assert result is True - # Verify retry occurred assert attempt_count >= 2 - # On the 2nd attempt, the merged operation must have 2 steps and status PipelineRunning second_saved_op = saved_operations[1] assert len(second_saved_op.steps) == 2 assert second_saved_op.status == Status.PipelineRunning assert second_saved_op.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID - # send_deployment_message must have been called for the cleanup step (no AssertionError!) send_deployment_message_mock.assert_called_once() @@ -3326,7 +3257,6 @@ async def mock_get_resource_by_id(resource_uuid): status_updater = DeploymentStatusUpdater() status_updater.operations_repo = AsyncMock() status_updater.operations_repo.get_operation_by_id.return_value = operation - # All attempts to write operation fail (including the final-delivery write) status_updater.operations_repo.update_item.side_effect = CosmosAccessConditionFailedError() status_updater.resource_repo = AsyncMock() @@ -3345,7 +3275,6 @@ async def mock_get_resource_by_id(resource_uuid): with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) - # Must return False so Service Bus abandons/dead-letters the message instead of discarding it assert result is False @@ -3387,7 +3316,6 @@ async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_wo status_updater.operations_repo.update_item = AsyncMock() status_updater.resource_repo = AsyncMock() - # Workspace resource update always fails status_updater.resource_repo.update_item.side_effect = Exception("Cosmos DB unavailable") status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} status_updater.workspace_repo = AsyncMock() @@ -3403,7 +3331,6 @@ async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_wo with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) - # Must return False so message is dead-lettered, and operation must NOT be marked Deleted assert result is False assert operation.status != Status.Deleted From 24740e407df94643122a5ef298ae6404586c5c24 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:56:55 +0000 Subject: [PATCH 50/90] Resolve stale-status concurrency on ETag conflict and derive step failure from step action --- .../service_bus/deployment_status_updater.py | 133 ++++++++--- .../service_bus/resource_request_sender.py | 11 +- .../test_deployment_status_update.py | 211 ++++++++++++++++++ .../test_resource_request_sender.py | 35 +++ 4 files changed, 361 insertions(+), 29 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 99d257b3c5..18346d16ed 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -302,6 +302,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage raise logger.warning(f"ETag conflict when saving operation {operation.id}. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + if ( + fresh_op is not operation + and ( + (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + logger.info( + f"Operation {fresh_op.id} is already terminal ({fresh_op.status}) after ETag conflict reload. " + f"Reconciling resource writes and skipping stale update." + ) + await self._reconcile_resources_for_terminal_operation(fresh_op) + return True + step_to_update, is_last_step, current_step_index = self._merge_operation_steps( in_memory_op=operation, fresh_op=fresh_op, @@ -350,16 +364,26 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if workspace_persisted and primary_persisted: try: fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) - fresh_step, fresh_is_last, _ = self._merge_operation_steps( - in_memory_op=operation, - fresh_op=fresh_op, - step_id_to_update=step_to_update.id, - step_status=step_to_update.status, - step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, - ) - await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) - await self.operations_repo.update_item(fresh_op) - operation_persisted = True + if ( + fresh_op is not operation + and ( + (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + await self._reconcile_resources_for_terminal_operation(fresh_op) + operation_persisted = True + else: + fresh_step, fresh_is_last, _ = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) + await self.operations_repo.update_item(fresh_op) + operation_persisted = True except Exception as op_err: logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) @@ -500,18 +524,27 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.operations_repo.update_item(operation) break except CosmosAccessConditionFailedError: - if attempt == MAX_CLEANUP_RETRIES - 1: - logger.warning( - f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id} " - f"in operation {operation.id} after successful dispatch." - ) - else: - logger.warning( - f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id}. " - f"Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." - ) - try: - fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + logger.warning( + f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id}. " + f"Reloading operation (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." + ) + try: + fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + if ( + fresh_op is not operation + and ( + (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + logger.info( + f"Operation {fresh_op.id} is already terminal ({fresh_op.status}) after cleanup step dispatch. " + f"Reconciling resource writes and skipping stale update." + ) + await self._reconcile_resources_for_terminal_operation(fresh_op) + return True + + if attempt < MAX_CLEANUP_RETRIES - 1: self._merge_operation_steps( in_memory_op=operation, fresh_op=fresh_op, @@ -522,8 +555,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.operations_repo.update_item(fresh_op) operation = fresh_op break - except Exception: - pass + except Exception: + pass except Exception as post_send_err: if attempt == MAX_CLEANUP_RETRIES - 1: logger.error( @@ -602,9 +635,21 @@ def _merge_operation_steps( if step_to_update is None: raise Exception(f"Error finding step {step_id_to_update} in reloaded operation {fresh_op.id}") - step_to_update.status = step_status - step_to_update.message = step_message - step_to_update.updatedWhen = get_timestamp() + step_is_terminal = step_to_update.is_success() or step_to_update.is_failure() + new_status_is_terminal = step_status in ( + Status.ActionSucceeded, + Status.Deployed, + Status.Deleted, + Status.Updated, + Status.ActionFailed, + Status.DeletingFailed, + Status.DeploymentFailed, + Status.UpdatingFailed, + ) + if not step_is_terminal or new_status_is_terminal: + step_to_update.status = step_status + step_to_update.message = step_message + step_to_update.updatedWhen = get_timestamp() return step_to_update, is_last_step, current_step_index @@ -642,6 +687,40 @@ def _is_cleanup_failed(self, operation: Operation) -> bool: def _is_cleanup_terminal(self, operation: Operation) -> bool: return self._is_cleanup_completed(operation) or self._is_cleanup_failed(operation) + async def _reconcile_resources_for_terminal_operation(self, operation: Operation) -> None: + """Reconcile resource deployment statuses against terminal operation state.""" + if operation.resourceId: + try: + primary = await self.resource_repo.get_resource_by_id(uuid.UUID(str(operation.resourceId))) + if primary.deploymentStatus != operation.status: + primary.deploymentStatus = operation.status + await self.resource_repo.update_item(primary) + except (EntityDoesNotExist, ValueError): + pass + except Exception: + logger.exception(f"Failed to reconcile primary resource for terminal operation {operation.id}") + + if operation.action == RequestAction.UnInstall: + cleanup_step = next( + (s for s in (operation.steps or []) if s.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID), + None + ) + if cleanup_step and cleanup_step.resourceId: + try: + ws_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(str(cleanup_step.resourceId))) + target_status = ( + Status.Updated if cleanup_step.is_success() or operation.status == Status.Deleted + else Status.UpdatingFailed if cleanup_step.is_failure() or operation.status == Status.DeletingFailed + else ws_resource.deploymentStatus + ) + if ws_resource.deploymentStatus != target_status: + ws_resource.deploymentStatus = target_status + await self.resource_repo.update_item(ws_resource) + except (EntityDoesNotExist, ValueError): + pass + except Exception: + logger.exception(f"Failed to reconcile workspace resource for terminal operation {operation.id}") + def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: return OperationStep( id=str(uuid.uuid4()), diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e1c5838f4b..4db6c1bb27 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -76,8 +76,15 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope operation.status = failure_status operation.message = f"Failed to dispatch initial deployment message: {action}" if operation.steps: - operation.steps[0].status = failure_status - operation.steps[0].message = f"Failed to dispatch initial deployment message: {action}" + first_step = operation.steps[0] + step_failure_status = ( + Status.DeploymentFailed if first_step.resourceAction == RequestAction.Install + else Status.DeletingFailed if first_step.resourceAction == RequestAction.UnInstall + else Status.UpdatingFailed if first_step.resourceAction == RequestAction.Upgrade + else Status.ActionFailed + ) + first_step.status = step_failure_status + first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" await operations_repo.update_item(operation) except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index cb2ab00061..2045407e95 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -3369,3 +3369,214 @@ async def test_get_address_cleanup_details_returns_none_if_resource_type_not_wor details = await status_updater._get_address_cleanup_details(operation) assert details is None + + +async def test_etag_conflict_on_earlier_step_reconciles_and_skips_when_fresh_op_already_terminal(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[copy.deepcopy(step1), copy.deepcopy(step2)], + ) + + terminal_step1 = copy.deepcopy(step1) + terminal_step1.status = Status.Deleted + terminal_step2 = copy.deepcopy(step2) + terminal_step2.status = Status.Updated + terminal_step2.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + fresh_terminal_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=1, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[terminal_step1, terminal_step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.side_effect = [ + in_memory_op, + fresh_terminal_op, + ] + status_updater.operations_repo.update_item.side_effect = CosmosAccessConditionFailedError() + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"}, + } + status_updater.workspace_repo = AsyncMock() + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert fresh_terminal_op.status == Status.Deleted + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.Updated + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_etag_conflict_after_cleanup_dispatch_skips_when_fresh_op_already_terminal(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[copy.deepcopy(step1), copy.deepcopy(step2)], + ) + + terminal_step2 = copy.deepcopy(step2) + terminal_step2.status = Status.Updated + terminal_step2.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + fresh_terminal_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=2, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[copy.deepcopy(step1), terminal_step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + call_count = 0 + + def mock_get_op(op_id): + nonlocal call_count + call_count += 1 + return in_memory_op if call_count == 1 else fresh_terminal_op + + status_updater.operations_repo.get_operation_by_id.side_effect = mock_get_op + + op_update_count = 0 + + async def mock_update_op(op): + nonlocal op_update_count + op_update_count += 1 + if op_update_count > 1: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_update_op + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + send_deployment_message_mock.assert_called_once() + assert fresh_terminal_op.steps[1].status == Status.Updated + assert fresh_terminal_op.status == Status.Deleted diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 9f1409d819..a42a465398 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -240,3 +240,38 @@ async def test_send_resource_request_message_persists_failure_state_on_dispatch_ assert operation.status == expected_status assert operation.steps[0].status == expected_status assert "Failed to dispatch" in operation.message + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_derives_step_failure_status_from_step_action( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.UnInstall) + operation.steps[0].resourceAction = RequestAction.Upgrade + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.UnInstall, + ) + + operations_repo_mock.update_item.assert_awaited_once_with(operation) + assert operation.status == Status.DeletingFailed + assert operation.steps[0].status == Status.UpdatingFailed + assert "Failed to dispatch" in operation.steps[0].message From d940017f1e0cf4544f2abc2147178a01072c5a84 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:26:05 +0000 Subject: [PATCH 51/90] Refactor failure status handling and improve UUID validation in resource operations --- api_app/db/repositories/operations.py | 8 +++-- api_app/models/domain/operation.py | 11 +++++++ .../service_bus/deployment_status_updater.py | 31 ++++++++++++------- .../service_bus/resource_request_sender.py | 18 ++--------- .../test_operation_repository.py | 16 +++++++--- 5 files changed, 51 insertions(+), 33 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 15eabe92c7..8e547b3764 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -53,9 +53,6 @@ async def update_item(self, item: Operation, etag: Optional[str] = None) -> Oper item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag return item - async def update_item_with_etag(self, item: Operation, etag: str) -> Operation: - return await self.update_item(item, etag=etag) - @staticmethod def operations_query(): return 'SELECT * FROM c WHERE' @@ -232,6 +229,11 @@ async def resource_has_deployed_operation(self, resource_id: str) -> bool: return len(operations) > 0 async def resource_has_active_operation(self, resource_id: str) -> bool: + # Guard against injection; resource_id should always be a valid UUID string + try: + uuid.UUID(resource_id) + except ValueError: + return False active_statuses = ( Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, Status.Deploying, Status.AwaitingDeletion, Status.Deleting, diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index d4da0239fe..3e48f51554 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -79,6 +79,16 @@ def is_action(self) -> bool: ) +def get_failure_status_for_action(action) -> Status: + """Return the terminal failure Status for a given RequestAction.""" + from models.domain.request_action import RequestAction # local import to avoid circular deps + return { + RequestAction.Install: Status.DeploymentFailed, + RequestAction.UnInstall: Status.DeletingFailed, + RequestAction.Upgrade: Status.UpdatingFailed, + }.get(action, Status.ActionFailed) + + class Operation(AzureTREModel): """ Operation model @@ -102,6 +112,7 @@ class Operation(AzureTREModel): def parse_etag_to_remove_escaped_quotes(cls, value): if value: return value.replace('\"', '') + return value @field_validator("user", mode="before") @classmethod diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 18346d16ed..9900b0e721 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -209,9 +209,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass await self.update_overall_operation_status(operation, step_to_update, is_last_step) await self.operations_repo.update_item(operation) return True @@ -481,9 +484,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass return True except Exception as e: logger.exception("Unable to send update for resource in pipeline step") @@ -500,9 +506,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage None ) if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = Status.DeletingFailed + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass return True else: next_step.status = Status.AwaitingUpdate @@ -577,7 +586,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if operation is not None and step_to_update is not None: try: failure_status = self.get_failure_status_for_action(operation.action) - step_to_update.status = failure_status + step_to_update.status = self.get_failure_status_for_action(step_to_update.resourceAction) step_to_update.message = f"Resource {message.id} not found in database: operation aborted." operation.status = failure_status operation.message = f"Resource {message.id} not found in database: operation aborted." @@ -677,7 +686,7 @@ def _is_cleanup_completed(self, operation: Operation) -> bool: def _is_cleanup_failed(self, operation: Operation) -> bool: for step in operation.steps: if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: - if step.is_failure() or step.status == Status.UpdatingFailed: + if step.is_failure(): return True if operation.action == RequestAction.UnInstall and operation.status == Status.DeletingFailed: if any(step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps): diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 4db6c1bb27..e5c22a70d6 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -8,7 +8,7 @@ from models.domain.request_action import RequestAction from models.domain.resource import Resource -from models.domain.operation import Operation, Status +from models.domain.operation import Operation, get_failure_status_for_action from db.repositories.operations import OperationRepository from services.logging import logger, tracer @@ -67,23 +67,11 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope except Exception: logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") try: - failure_status = ( - Status.DeploymentFailed if action == RequestAction.Install - else Status.DeletingFailed if action == RequestAction.UnInstall - else Status.UpdatingFailed if action == RequestAction.Upgrade - else Status.ActionFailed - ) - operation.status = failure_status + operation.status = get_failure_status_for_action(action) operation.message = f"Failed to dispatch initial deployment message: {action}" if operation.steps: first_step = operation.steps[0] - step_failure_status = ( - Status.DeploymentFailed if first_step.resourceAction == RequestAction.Install - else Status.DeletingFailed if first_step.resourceAction == RequestAction.UnInstall - else Status.UpdatingFailed if first_step.resourceAction == RequestAction.Upgrade - else Status.ActionFailed - ) - first_step.status = step_failure_status + first_step.status = get_failure_status_for_action(first_step.resourceAction) first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" await operations_repo.update_item(operation) except Exception: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 59af8e9817..b0c1355cf4 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -107,23 +107,31 @@ async def test_create_operation_steps_excludes_address_space_cleanup_for_cascade async def test_resource_has_active_operation_returns_true_when_active_operation_exists(operations_repo): + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" operations_repo.query = AsyncMock(return_value=[{"id": "op-1"}]) - result = await operations_repo.resource_has_active_operation("workspace-123") + result = await operations_repo.resource_has_active_operation(workspace_id) assert result is True operations_repo.query.assert_called_once() query_str = operations_repo.query.call_args[1]["query"] if "query" in operations_repo.query.call_args[1] else operations_repo.query.call_args[0][0] - assert "workspace-123" in query_str + assert workspace_id in query_str assert "ARRAY_CONTAINS" in query_str - assert 'CONTAINS(c.resourcePath, "workspace-123")' in query_str + assert f'CONTAINS(c.resourcePath, "{workspace_id}")' in query_str assert 'NOT CONTAINS(c.resourcePath, "/user-resources/")' in query_str async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): operations_repo.query = AsyncMock(return_value=[]) - result = await operations_repo.resource_has_active_operation("workspace-123") + result = await operations_repo.resource_has_active_operation("7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d") assert result is False +async def test_resource_has_active_operation_returns_false_for_non_uuid_resource_id(operations_repo): + operations_repo.query = AsyncMock() + result = await operations_repo.resource_has_active_operation("not-a-uuid") + assert result is False + operations_repo.query.assert_not_called() + + async def test_update_item_with_etag_calls_replace_item(operations_repo): from azure.core import MatchConditions operations_repo._container = MagicMock() From 92a52d399ce5a99225350373c25604973f0b65ce Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:47:20 +0000 Subject: [PATCH 52/90] Refactor resource path handling to include cascading operations and adjust deployment status updates based on main step outcomes --- api_app/db/repositories/operations.py | 7 ++++- .../service_bus/deployment_status_updater.py | 27 ++++++++++++++++--- .../test_api/test_routes/test_workspaces.py | 13 ++++++--- .../test_operation_repository.py | 2 +- .../test_deployment_status_update.py | 4 +-- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 8e547b3764..e02ec3ec3c 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -244,7 +244,12 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: self.operations_query() + f' (c.resourceId = "{resource_id}"' + f' OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true)' - + f' OR (CONTAINS(c.resourcePath, "{resource_id}") AND NOT CONTAINS(c.resourcePath, "/user-resources/")))' + # Include any operation whose resource path contains this resource id — this catches + # cascading operations on all descendant resources (workspace services AND user resources). + # The previous NOT CONTAINS(c.resourcePath, "/user-resources/") exclusion meant that + # active user-resource operations were invisible when checking a parent workspace or + # workspace-service, allowing duplicate cascading pipelines to start. + + f' OR CONTAINS(c.resourcePath, "{resource_id}"))' + f' AND c.status IN ({status_filter})' ) operations = await self.query(query=query) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 9900b0e721..e5973d1e8c 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -477,6 +477,10 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step=True) await self.operations_repo.update_item(operation) + # Preserve the primary resource's status based on what the main step actually achieved. + # If the main uninstall step succeeded the service is Deleted; only the address-space + # cleanup step failed. Forcing DeletingFailed here would resurrect the service as + # non-Deleted and cause ownership conflicts on every future operation. main_step = next( (op_step for op_step in operation.steps if op_step.templateStepId == "main" @@ -486,8 +490,15 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + # Use the status the main step actually achieved rather than + # unconditionally overwriting it with DeletingFailed. + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass return True @@ -508,8 +519,14 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + # Preserve the status the main step actually achieved. + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass return True @@ -708,6 +725,7 @@ async def _reconcile_resources_for_terminal_operation(self, operation: Operation pass except Exception: logger.exception(f"Failed to reconcile primary resource for terminal operation {operation.id}") + raise # propagate so Service Bus abandons and retries the message if operation.action == RequestAction.UnInstall: cleanup_step = next( @@ -729,6 +747,7 @@ async def _reconcile_resources_for_terminal_operation(self, operation: Operation pass except Exception: logger.exception(f"Failed to reconcile workspace resource for terminal operation {operation.id}") + raise # propagate so Service Bus abandons and retries the message def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: return OperationStep( diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 18103fffca..d69804210b 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -919,7 +919,8 @@ async def test_post_workspace_services_raises_400_bad_request_if_input_is_bad(se @patch("api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", return_value=[sample_workspace_service().__dict__]) @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") - async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabled(self, _, get_workspace_service_mock, __, app, client): + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) + async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabled(self, _mock_active_op, _, get_workspace_service_mock, __, app, client): workspace_service = sample_workspace_service() workspace_service.properties["enabled"] = True get_workspace_service_mock.return_value = workspace_service @@ -935,8 +936,9 @@ async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabl @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", return_value=disabled_workspace_service()) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) async def test_delete_workspace_service_raises_404_if_workspace_service_has_active_resources(self, - __, ___, ____, app, + _mock_active_op, __, ___, ____, app, client): response = await client.delete( app.url_path_for(strings.API_DELETE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, @@ -986,9 +988,10 @@ async def test_get_workspace_service_history_returns_empty_list_when_no_history( @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", return_value=disabled_workspace_service()) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service", return_value=[]) @patch("api.routes.workspaces.send_uninstall_message", return_value=sample_resource_operation(resource_id=SERVICE_ID, operation_id=OPERATION_ID)) - async def test_delete_workspace_service_sends_uninstall_message(self, send_uninstall_mock, __, ___, ____, resource_template_repo, _____, + async def test_delete_workspace_service_sends_uninstall_message(self, send_uninstall_mock, __, _mock_active_op, ___, ____, resource_template_repo, _____, app, client, basic_workspace_service_template): resource_template_repo.return_value = basic_workspace_service_template @@ -1000,9 +1003,10 @@ async def test_delete_workspace_service_sends_uninstall_message(self, send_unins @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service", return_value=[]) @patch("api.routes.workspaces.send_uninstall_message", return_value=sample_resource_operation(resource_id=SERVICE_ID, operation_id=OPERATION_ID)) - async def test_delete_workspace_service_returns_the_deleted_workspace_service_id(self, __, ___, ____, workspace_service_mock, resource_template_repo, _____, app, client, basic_workspace_service_template): + async def test_delete_workspace_service_returns_the_deleted_workspace_service_id(self, __, ___, _mock_active_op, ____, workspace_service_mock, resource_template_repo, _____, app, client, basic_workspace_service_template): workspace_service = disabled_workspace_service() workspace_service_mock.return_value = workspace_service resource_template_repo.return_value = basic_workspace_service_template @@ -1014,6 +1018,7 @@ async def test_delete_workspace_service_returns_the_deleted_workspace_service_id assert response.json()["operation"]["resourceId"] == workspace_service.id # GET /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources + @patch("api.routes.workspaces.enrich_resource_with_available_upgrades", return_value=None) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service") diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index b0c1355cf4..7b80741268 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -116,7 +116,7 @@ async def test_resource_has_active_operation_returns_true_when_active_operation_ assert workspace_id in query_str assert "ARRAY_CONTAINS" in query_str assert f'CONTAINS(c.resourcePath, "{workspace_id}")' in query_str - assert 'NOT CONTAINS(c.resourcePath, "/user-resources/")' in query_str + assert 'NOT CONTAINS(c.resourcePath, "/user-resources/")' not in query_str async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 2045407e95..9797951f58 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1540,7 +1540,7 @@ async def test_enqueue_cleanup_validates_ownership_and_aborts_before_azure_send( assert operation.status == Status.DeletingFailed assert operation.steps[1].status == Status.UpdatingFailed assert "Terminal address space conflict" in operation.steps[1].message - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted @patch('service_bus.deployment_status_updater.send_deployment_message', side_effect=Exception("Service Bus send failed")) @@ -1660,7 +1660,7 @@ async def test_enqueue_cleanup_send_failure_on_final_delivery_marks_failed(send_ assert result is True assert operation.status == Status.DeletingFailed assert operation.steps[1].status == Status.UpdatingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted async def test_cleanup_failure_on_final_delivery_marks_failed_and_completes_message(): From 098f3c5479c0483b01ce4f191d33316337a2d27f Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:17:06 +0000 Subject: [PATCH 53/90] Enforce workspace lock on descendant mutating routes and preserve deleted status on cleanup failure --- api_app/api/routes/resource_helpers.py | 7 +++ api_app/api/routes/workspaces.py | 12 ++++ api_app/pytest.ini | 2 + .../service_bus/deployment_status_updater.py | 57 ++++++++++++++----- .../service_bus/resource_request_sender.py | 10 ++++ .../test_api/test_routes/test_workspaces.py | 46 +++++++++++++++ .../test_deployment_status_update.py | 16 +++--- 7 files changed, 128 insertions(+), 22 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index eb9262663d..c12856aaa5 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -92,6 +92,9 @@ async def save_and_deploy_resource( action=RequestAction.Install, ) return operation + except HTTPException: + await resource_repo.delete_item(resource.id) + raise except Exception: await resource_repo.delete_item(resource.id) logger.exception("Failed send resource request message") @@ -183,6 +186,8 @@ async def send_uninstall_message( is_cascade=is_cascade ) return operation + except HTTPException: + raise except Exception: logger.exception(f"Failed to send {resource_type} resource delete message") raise HTTPException( @@ -234,6 +239,8 @@ async def send_custom_action_message( action=custom_action, ) return operation + except HTTPException: + raise except Exception: logger.exception(f"Failed to send {resource_type} resource custom action message") raise HTTPException( diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 1f6f52e813..210523c53d 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -425,6 +425,8 @@ async def create_user_resource( user=Depends(require_workspace_owner_or_researcher_or_airlock_manager), workspace=Depends(get_deployed_workspace_by_id_from_path), workspace_service=Depends(get_deployed_workspace_service_by_id_from_path)) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) owner_id: str = None @@ -483,6 +485,9 @@ async def delete_user_resource( resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: validate_user_has_valid_role_for_user_resource(user, user_resource) + if await operations_repo.resource_has_active_operation(user_resource.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + if user_resource.isEnabled: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.USER_RESOURCE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION) @@ -515,6 +520,9 @@ async def patch_user_resource( force_version_update: bool = False) -> OperationInResponse: validate_user_has_valid_role_for_user_resource(user, user_resource) + if await operations_repo.resource_has_active_operation(user_resource.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + try: operation = await update_user_resource(user_resource, user_resource_patch, force_version_update, user, etag, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) response.headers["Location"] = construct_location_header(operation) @@ -540,6 +548,10 @@ async def invoke_action_on_user_resource( resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), user=Depends(require_workspace_owner_or_researcher_or_airlock_manager)) -> OperationInResponse: validate_user_has_valid_role_for_user_resource(user, user_resource) + + if await operations_repo.resource_has_active_operation(user_resource.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation = await send_custom_action_message( resource=user_resource, resource_repo=user_resource_repo, diff --git a/api_app/pytest.ini b/api_app/pytest.ini index a86e89e4bb..f8537f0452 100644 --- a/api_app/pytest.ini +++ b/api_app/pytest.ini @@ -1,5 +1,7 @@ [pytest] filterwarnings = error + ignore::pytest.PytestRemovedIn10Warning asyncio_mode = strict asyncio_default_fixture_loop_scope = function + diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index e5973d1e8c..f72fc161d5 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -211,8 +211,13 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass await self.update_overall_operation_status(operation, step_to_update, is_last_step) @@ -240,8 +245,13 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = Status.DeletingFailed - await self.resource_repo.update_item(primary_resource) + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass await self.update_overall_operation_status(operation, step_to_update, is_last_step) @@ -476,11 +486,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.message = f"Terminal address space conflict: {e}" next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step=True) - await self.operations_repo.update_item(operation) # Preserve the primary resource's status based on what the main step actually achieved. - # If the main uninstall step succeeded the service is Deleted; only the address-space - # cleanup step failed. Forcing DeletingFailed here would resurrect the service as - # non-Deleted and cause ownership conflicts on every future operation. + # Reconcile and persist the resource state BEFORE saving the terminal operation so that + # transient failures cause message retry without stranding the resource in DeletingFailed. main_step = next( (op_step for op_step in operation.steps if op_step.templateStepId == "main" @@ -501,6 +509,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass + await self.operations_repo.update_item(operation) return True except Exception as e: logger.exception("Unable to send update for resource in pipeline step") @@ -509,7 +518,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.message = f"Failed to enqueue address space cleanup after {MAX_CLEANUP_DELIVERY_COUNT} deliveries: {e}" next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step=True) - await self.operations_repo.update_item(operation) main_step = next( (op_step for op_step in operation.steps if op_step.templateStepId == "main" @@ -529,6 +537,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass + await self.operations_repo.update_item(operation) return True else: next_step.status = Status.AwaitingUpdate @@ -718,8 +727,17 @@ async def _reconcile_resources_for_terminal_operation(self, operation: Operation if operation.resourceId: try: primary = await self.resource_repo.get_resource_by_id(uuid.UUID(str(operation.resourceId))) - if primary.deploymentStatus != operation.status: - primary.deploymentStatus = operation.status + main_step = next( + (op_step for op_step in (operation.steps or []) + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId), + None + ) + target_status = ( + Status.Deleted if (operation.action == RequestAction.UnInstall and main_step and main_step.is_success()) + else operation.status + ) + if primary.deploymentStatus != target_status: + primary.deploymentStatus = target_status await self.resource_repo.update_item(primary) except (EntityDoesNotExist, ValueError): pass @@ -879,8 +897,12 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - if primary_resource.deploymentStatus != operation.status: - primary_resource.deploymentStatus = operation.status + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass @@ -903,8 +925,13 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper if main_step: try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + primary_status = ( + Status.Deleted if (operation.action == RequestAction.UnInstall and main_step.is_success()) + else operation.status + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except EntityDoesNotExist: pass diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e5c22a70d6..2fe5b31f3a 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -9,6 +9,8 @@ from models.domain.request_action import RequestAction from models.domain.resource import Resource from models.domain.operation import Operation, get_failure_status_for_action +from resources import strings +from fastapi import HTTPException, status from db.repositories.operations import OperationRepository from services.logging import logger, tracer @@ -26,6 +28,14 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope current_span.set_attribute("resource_id", resource.id) current_span.set_attribute("action", action) + target_workspace_id = getattr(resource, "workspaceId", None) + if target_workspace_id and hasattr(operations_repo, "resource_has_active_operation"): + active_op_check = operations_repo.resource_has_active_operation(target_workspace_id) + if hasattr(active_op_check, "__await__"): + is_active = await active_op_check + if is_active is True: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + # Construct the resources to build an operation item for resources_list = [] if is_cascade: diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index d69804210b..0a63bdc31f 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -1586,6 +1586,52 @@ async def test_get_user_resource_raises_404_if_resource_not_found(self, _, __, a response = await client.get(app.url_path_for(strings.API_GET_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID)) assert response.status_code == status.HTTP_404_NOT_FOUND + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_deployed_workspace_service_by_id") + async def test_post_user_resources_raises_409_if_workspace_has_active_operation(self, _, __, get_workspace_mock, app, client, sample_user_resource_input_data): + get_workspace_mock.return_value = sample_workspace() + response = await client.post(app.url_path_for(strings.API_CREATE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json=sample_user_resource_input_data) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id} + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_delete_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.delete(app.url_path_for(strings.API_DELETE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id} + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_patch_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.patch(app.url_path_for(strings.API_UPDATE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}/invoke-action + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_invoke_action_on_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_deployed_workspace_service_by_id") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 9797951f58..b74be557b4 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1412,7 +1412,7 @@ async def mock_get_resource_by_id(resource_uuid): assert step2.status == Status.UpdatingFailed assert "Terminal address space conflict" in step2.message assert operation.status == Status.DeletingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed @@ -1739,7 +1739,7 @@ async def mock_get_resource_by_id(resource_uuid): assert step2.status == Status.UpdatingFailed assert "delivery attempts" in step2.message assert operation.status == Status.DeletingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed @@ -2012,7 +2012,7 @@ async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send workspace_service_mock = MagicMock() workspace_service_mock.id = workspace_service_id - workspace_service_mock.deploymentStatus = Status.DeletingFailed + workspace_service_mock.deploymentStatus = Status.Deleted status_updater = DeploymentStatusUpdater() status_updater.operations_repo = AsyncMock() @@ -2043,7 +2043,7 @@ async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send assert result is True assert operation.status == Status.DeletingFailed assert step2.status == Status.UpdatingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted status_updater.operations_repo.update_item.assert_not_called() send_deployment_message_mock.assert_not_called() @@ -2186,7 +2186,7 @@ async def test_cleanup_conflict_persists_workspace_and_primary_resource_before_o workspace_service_mock = MagicMock() workspace_service_mock.id = workspace_service_id - workspace_service_mock.deploymentStatus = Status.Deleted + workspace_service_mock.deploymentStatus = Status.Deleting parent_workspace_mock = MagicMock() parent_workspace_mock.id = parent_workspace_id @@ -2230,6 +2230,7 @@ async def tracking_resource_update(res): assert result is True assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] assert len(op_update_calls) == 1 @@ -2278,7 +2279,7 @@ async def test_cleanup_final_delivery_persists_workspace_and_primary_resource_be workspace_service_mock = MagicMock() workspace_service_mock.id = workspace_service_id - workspace_service_mock.deploymentStatus = Status.Deleted + workspace_service_mock.deploymentStatus = Status.Deleting parent_workspace_mock = MagicMock() parent_workspace_mock.id = parent_workspace_id @@ -2322,6 +2323,7 @@ async def tracking_resource_update(res): assert result is True assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] assert len(op_update_calls) == 1 @@ -2773,7 +2775,7 @@ async def mock_get_resource_by_id(resource_uuid): assert step2.status == Status.UpdatingFailed assert "delivery attempts" in step2.message assert operation.status == Status.DeletingFailed - assert workspace_service_mock.deploymentStatus == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed From ae0608cc54b1760815bb1c6b181194b8eccfdc89 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:35:25 +0000 Subject: [PATCH 54/90] Acquire atomic workspace lease on operation admission and roll back address space on failure --- api_app/api/routes/workspaces.py | 33 +++- api_app/db/repositories/operations.py | 182 +++++++++++++----- .../service_bus/resource_request_sender.py | 10 - .../test_api/test_routes/test_workspaces.py | 36 +++- .../test_operation_repository.py | 53 +++++ 5 files changed, 251 insertions(+), 63 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 210523c53d..37ef056514 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -281,14 +281,31 @@ async def create_workspace_service(response: Response, workspace_service_input: except CosmosAccessConditionFailedError: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) - operation = await save_and_deploy_resource( - resource=workspace_service, - resource_repo=workspace_service_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) + try: + operation = await save_and_deploy_resource( + resource=workspace_service, + resource_repo=workspace_service_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template) + except Exception: + if resource_template.properties.get("address_space") and workspace_service.properties.get("address_space"): + try: + latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) + current_spaces = latest_workspace.properties.get("address_spaces", []) + allocated_space = workspace_service.properties["address_space"] + if allocated_space in current_spaces: + updated_spaces = [s for s in current_spaces if s != allocated_space] + rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) + await workspace_repo.patch_workspace( + latest_workspace, rollback_patch, latest_workspace.etag, + resource_template_repo, resource_history_repo, user, False + ) + except Exception: + logger.exception("Failed to rollback allocated address space on workspace") + raise response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index e02ec3ec3c..2a924ec39a 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -3,6 +3,9 @@ from typing import List, Optional from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError +from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosAccessConditionFailedError +from fastapi import HTTPException, status as http_status from pydantic import TypeAdapter from db.repositories.resource_templates import ResourceTemplateRepository from resources import strings @@ -17,6 +20,14 @@ from models.domain.operation import Operation, OperationStep, Status +def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: + if resource_path and resource_path.startswith("/workspaces/"): + parts = resource_path.split("/") + if len(parts) > 2: + return parts[2] + return None + + class OperationRepository(BaseRepository): @classmethod async def create(cls): @@ -51,8 +62,82 @@ async def update_item(self, item: Operation, etag: Optional[str] = None) -> Oper if isinstance(response, dict) and "_etag" in response: new_etag = response["_etag"] item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + + active_statuses = ( + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + ) + if item.status not in active_statuses: + target_workspace_id = extract_workspace_id_from_resource_path(item.resourcePath) + if target_workspace_id: + await self.release_workspace_lease(target_workspace_id, item.id) + return item + async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> bool: + if not hasattr(self, "_container") or self._container is None: + return True + + if hasattr(self, "resource_has_active_operation"): + check = self.resource_has_active_operation(workspace_id) + if hasattr(check, "__await__"): + is_active = await check + if is_active is True: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + lease_id = f"lease_{workspace_id}" + timestamp = self.get_timestamp() + lease_body = { + "id": lease_id, + "workspaceId": workspace_id, + "operationId": operation_id, + "createdWhen": timestamp, + } + + try: + await self.container.create_item(body=lease_body) + return True + except (CosmosResourceExistsError, ResourceExistsError): + try: + existing_lease = await self.read_item_by_id(lease_id) + current_op_id = existing_lease.get("operationId") + lease_created = existing_lease.get("createdWhen", 0.0) + + if current_op_id != operation_id: + is_active = False + if hasattr(self, "resource_has_active_operation"): + active_check = self.resource_has_active_operation(workspace_id) + if hasattr(active_check, "__await__"): + is_active = (await active_check) is True + if (timestamp - lease_created < 60.0) or is_active: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + etag = existing_lease.get("_etag") + if etag: + await self.container.replace_item( + item=lease_id, + body=lease_body, + etag=etag, + match_condition=MatchConditions.IfNotModified + ) + else: + await self.container.upsert_item(body=lease_body) + return True + except (CosmosAccessConditionFailedError, ResourceExistsError): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + async def release_workspace_lease(self, workspace_id: str, operation_id: Optional[str] = None) -> None: + if not hasattr(self, "_container") or self._container is None: + return + lease_id = f"lease_{workspace_id}" + try: + existing_lease = await self.read_item_by_id(lease_id) + if not operation_id or existing_lease.get("operationId") == operation_id: + await self.delete_item(lease_id) + except Exception: + pass + @staticmethod def operations_query(): return 'SELECT * FROM c WHERE' @@ -82,54 +167,63 @@ def create_main_step(self, resource_template: dict, action: str, resource_id: st async def create_operation_item(self, resource_id: str, resource_list: List, action: str, resource_path: str, resource_version: int, user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository) -> Operation: operation_id = self.create_operation_id() - # get the right "awaiting" message based on the action - status, message = self.get_initial_status(action) - all_steps = [] - for resource in resource_list: - name = resource["templateName"] - version = resource["templateVersion"] - resource_type = ResourceType(resource["resourceType"]) - primary_parent_service_name = None - if resource_type == ResourceType.UserResource: - primary_parent_workspace_service = await resource_repo.get_resource_by_id(resource["parentWorkspaceServiceId"]) - primary_parent_service_name = primary_parent_workspace_service.templateName - resource_template = await resource_template_repo.get_template_by_name_and_version(name, version, resource_type, primary_parent_service_name) - resource_template_dict = resource_template.model_dump(exclude_none=True) - # if the template has a pipeline defined for this action, copy over all the steps to the ops document - steps = await self.build_step_list( - steps=[], - resource_template_dict=resource_template_dict, - action=action, - resource_repo=resource_repo, - resource_id=resource["id"], + target_workspace_id = extract_workspace_id_from_resource_path(resource_path) + if target_workspace_id: + await self.acquire_workspace_lease(target_workspace_id, operation_id) + + try: + # get the right "awaiting" message based on the action + status, message = self.get_initial_status(action) + all_steps = [] + for resource in resource_list: + name = resource["templateName"] + version = resource["templateVersion"] + resource_type = ResourceType(resource["resourceType"]) + primary_parent_service_name = None + if resource_type == ResourceType.UserResource: + primary_parent_workspace_service = await resource_repo.get_resource_by_id(resource["parentWorkspaceServiceId"]) + primary_parent_service_name = primary_parent_workspace_service.templateName + resource_template = await resource_template_repo.get_template_by_name_and_version(name, version, resource_type, primary_parent_service_name) + resource_template_dict = resource_template.model_dump(exclude_none=True) + # if the template has a pipeline defined for this action, copy over all the steps to the ops document + steps = await self.build_step_list( + steps=[], + resource_template_dict=resource_template_dict, + action=action, + resource_repo=resource_repo, + resource_id=resource["id"], + status=status, + message=message, + is_cascade_operation=resource["id"] != resource_id + ) + + # if no pipeline is defined for this action, create a main step only + if len(steps) == 0: + all_steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource["id"], status=status, message=message)) + else: + all_steps.extend(steps) + + timestamp = self.get_timestamp() + operation = Operation( + id=operation_id, + resourceId=resource_id, + resourcePath=resource_path, + resourceVersion=resource_version, status=status, + createdWhen=timestamp, + updatedWhen=timestamp, + action=action, message=message, - is_cascade_operation=resource["id"] != resource_id + user=user.model_dump(), + steps=all_steps ) - # if no pipeline is defined for this action, create a main step only - if len(steps) == 0: - all_steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource["id"], status=status, message=message)) - else: - all_steps.extend(steps) - - timestamp = self.get_timestamp() - operation = Operation( - id=operation_id, - resourceId=resource_id, - resourcePath=resource_path, - resourceVersion=resource_version, - status=status, - createdWhen=timestamp, - updatedWhen=timestamp, - action=action, - message=message, - user=user.model_dump(), - steps=all_steps - ) - - await self.save_item(operation) - return operation + await self.save_item(operation) + return operation + except Exception: + if target_workspace_id: + await self.release_workspace_lease(target_workspace_id, operation_id) + raise async def build_step_list(self, steps: List[OperationStep], resource_template_dict: dict, action: str, resource_repo: ResourceRepository, resource_id: str, status: Status, message: str, is_cascade_operation: bool = False): if "pipeline" in resource_template_dict and resource_template_dict["pipeline"] is not None: diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 2fe5b31f3a..e5c22a70d6 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -9,8 +9,6 @@ from models.domain.request_action import RequestAction from models.domain.resource import Resource from models.domain.operation import Operation, get_failure_status_for_action -from resources import strings -from fastapi import HTTPException, status from db.repositories.operations import OperationRepository from services.logging import logger, tracer @@ -28,14 +26,6 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope current_span.set_attribute("resource_id", resource.id) current_span.set_attribute("action", action) - target_workspace_id = getattr(resource, "workspaceId", None) - if target_workspace_id and hasattr(operations_repo, "resource_has_active_operation"): - active_op_check = operations_repo.resource_has_active_operation(target_workspace_id) - if hasattr(active_op_check, "__await__"): - is_active = await active_op_check - if is_active is True: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - # Construct the resources to build an operation item for resources_list = [] if is_cascade: diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 0a63bdc31f..ae28d1246a 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -5,7 +5,7 @@ import pytest from mock import patch -from fastapi import status +from fastapi import status, HTTPException from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP, FAKE_UPDATE_TIMESTAMP from tests_ma.test_api.conftest import create_admin_user, create_test_user, create_workspace_owner_user, create_workspace_researcher_user @@ -828,6 +828,40 @@ async def test_post_workspace_services_creates_workspace_service_with_address_sp assert response.status_code == status.HTTP_202_ACCEPTED assert response.json()["operation"]["resourceId"] == SERVICE_ID + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.ResourceHistoryRepository.save_item", return_value=AsyncMock()) + @patch("api.routes.workspaces.save_and_deploy_resource", side_effect=HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION)) + @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) + @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") + @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) + @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") + async def test_post_workspace_services_rolls_back_address_space_if_deploy_fails(self, create_workspace_service_item_mock, _, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): + etag = "some-etag-value" + workspace = sample_workspace() + workspace.properties["address_spaces"] = ["192.168.0.1/24"] + workspace.etag = etag + get_workspace_mock.return_value = workspace + basic_workspace_service_template.properties["address_space"] = {"type": "string"} + create_workspace_service_item_mock.return_value = [sample_workspace_service(), basic_workspace_service_template] + basic_resource_template.properties["address_spaces"] = {"type": "array", "updateable": True} + resource_template_repo.return_value = basic_resource_template + + modified_workspace = sample_workspace() + modified_workspace.properties["address_spaces"] = ["192.168.0.1/24", "10.1.4.0/24"] + modified_workspace.etag = etag + update_item_mock.return_value = modified_workspace + + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + + assert response.status_code == status.HTTP_409_CONFLICT + assert update_item_mock.call_count == 2 + rollback_call_workspace = update_item_mock.call_args_list[1][0][0] + assert rollback_call_workspace.properties["address_spaces"] == ["192.168.0.1/24"] + # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 7b80741268..ccdc054a55 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -209,3 +209,56 @@ async def test_update_item_raises_cosmos_access_condition_failed_error_on_mismat ) with pytest.raises(CosmosAccessConditionFailedError): await operations_repo.update_item(op) + + +async def test_acquire_workspace_lease_creates_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.create_item.assert_awaited_once() + body = operations_repo._container.create_item.call_args[1]["body"] + assert body["id"] == "lease_ws-1" + assert body["workspaceId"] == "ws-1" + assert body["operationId"] == "op-1" + + +async def test_acquire_workspace_lease_raises_409_if_active_operation_exists(operations_repo): + from fastapi import HTTPException + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=True) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_workspace_lease_raises_409_if_recent_lease_exists(operations_repo): + from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + + +async def test_release_workspace_lease_deletes_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1" + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_workspace_lease("ws-1", "op-1") + operations_repo.delete_item.assert_awaited_once_with("lease_ws-1") From 57facf64bf00738c89a8bce9c96cf022d7471a2f Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:53:28 +0000 Subject: [PATCH 55/90] Acquire workspace lease before resource mutations, keep lease ownership until release, and validate cleanup step --- api_app/api/routes/resource_helpers.py | 38 ++-- api_app/api/routes/workspaces.py | 187 +++++++++++------- api_app/db/repositories/operations.py | 85 +++++--- api_app/db/repositories/resource_templates.py | 2 + .../service_bus/deployment_status_updater.py | 16 +- .../service_bus/resource_request_sender.py | 6 +- .../test_operation_repository.py | 38 +++- .../test_resource_templates_repository.py | 3 + 8 files changed, 263 insertions(+), 112 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index c12856aaa5..fb72ebfd57 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -63,6 +63,7 @@ async def save_and_deploy_resource( resource_history_repo: ResourceHistoryRepository, user: User, resource_template: ResourceTemplate, + operation_id: Optional[str] = None, ) -> Operation: try: resource.user = user.model_dump() @@ -82,6 +83,9 @@ async def save_and_deploy_resource( ) try: + kwargs = {} + if operation_id is not None: + kwargs["operation_id"] = operation_id operation = await send_resource_request_message( resource=resource, operations_repo=operations_repo, @@ -90,6 +94,7 @@ async def save_and_deploy_resource( resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Install, + **kwargs, ) return operation except HTTPException: @@ -303,17 +308,28 @@ async def update_user_resource( operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> Operation: - patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update) - operation = await send_resource_request_message( - resource=patched_user_resource, - operations_repo=operations_repo, - resource_repo=user_resource_repo, - user=user, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - action=RequestAction.Upgrade) - - return operation + target_workspace_id = getattr(workspace_service, "workspaceId", None) or getattr(user_resource, "workspaceId", None) + operation_id = operations_repo.create_operation_id() + if target_workspace_id and hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(target_workspace_id, operation_id) + + try: + patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update) + operation = await send_resource_request_message( + resource=patched_user_resource, + operations_repo=operations_repo, + resource_repo=user_resource_repo, + user=user, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + action=RequestAction.Upgrade, + operation_id=operation_id) + + return operation + except Exception: + if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(target_workspace_id, operation_id) + raise async def enrich_resource_with_available_upgrades(resource: Resource, resource_template_repo: ResourceTemplateRepository): diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 37ef056514..bdfb55248f 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -129,8 +129,9 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: - if await operations_repo.resource_has_active_operation(workspace.id): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -145,16 +146,27 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Upgrade, - is_cascade=is_disablement) + is_cascade=is_disablement, + operation_id=operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) except CosmosAccessConditionFailedError: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @workspaces_core_router.delete("/workspaces/{workspace_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) @@ -252,36 +264,40 @@ async def create_workspace_service(response: Response, workspace_service_input: if await operations_repo.resource_has_active_operation(workspace.id): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - try: - workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) - except (ValidationError, ValueError) as e: - logger.exception("Failed create workspace service model instance") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except UserNotAuthorizedToUseTemplate as e: - logger.exception("User not authorized to use template") - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) - # Get the workspace subscription id (if set) - if workspace.properties.get("workspace_subscription_id"): - workspace_service.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] - - # if template has address_space get an address space - if resource_template.properties.get("address_space"): - # check workspace has address_spaces property - if not workspace.properties.get("address_spaces"): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) - workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) - workspace_patch = ResourcePatch() - workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} - # IP address allocation is managed by the API. Ideally this request would happen as a result of the workspace - # service deployment via the reosurce processor. there is no such functionality so the database is being - # updated directly, and an "update" on the workspace is called by the workspace service pipeline. + try: try: - await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) - except CosmosAccessConditionFailedError: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) + workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) + except (ValidationError, ValueError) as e: + logger.exception("Failed create workspace service model instance") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except UserNotAuthorizedToUseTemplate as e: + logger.exception("User not authorized to use template") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + + # Get the workspace subscription id (if set) + if workspace.properties.get("workspace_subscription_id"): + workspace_service.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] + + # if template has address_space get an address space + if resource_template.properties.get("address_space"): + # check workspace has address_spaces property + if not workspace.properties.get("address_spaces"): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) + workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} + # IP address allocation is managed by the API. Ideally this request would happen as a result of the workspace + # service deployment via the reosurce processor. there is no such functionality so the database is being + # updated directly, and an "update" on the workspace is called by the workspace service pipeline. + try: + await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) + except CosmosAccessConditionFailedError: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) - try: operation = await save_and_deploy_resource( resource=workspace_service, resource_repo=workspace_service_repo, @@ -289,9 +305,10 @@ async def create_workspace_service(response: Response, workspace_service_input: resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, user=user, - resource_template=resource_template) + resource_template=resource_template, + operation_id=operation_id) except Exception: - if resource_template.properties.get("address_space") and workspace_service.properties.get("address_space"): + if 'resource_template' in locals() and 'workspace_service' in locals() and resource_template.properties.get("address_space") and workspace_service.properties.get("address_space"): try: latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) current_spaces = latest_workspace.properties.get("address_spaces", []) @@ -305,6 +322,8 @@ async def create_workspace_service(response: Response, workspace_service_input: ) except Exception: logger.exception("Failed to rollback allocated address space on workspace") + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise response.headers["Location"] = construct_location_header(operation) @@ -313,8 +332,9 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: - if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace_service.workspaceId, operation_id) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -328,15 +348,26 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Upgrade, - is_cascade=is_disablement) + is_cascade=is_disablement, + operation_id=operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) except CosmosAccessConditionFailedError: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) + raise @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) @@ -445,49 +476,59 @@ async def create_user_resource( if await operations_repo.resource_has_active_operation(workspace.id): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - owner_id: str = None - - # Check for assign_to_another_user logic - if ( - hasattr(user_resource_create, "properties") - and isinstance(user_resource_create.properties, dict) - and user_resource_create.properties.get("assign_to_another_user") is True - ): - if user_resource_create.properties.get("owner_id"): - owner_id = user_resource_create.properties.get("owner_id") + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) try: - user_resource, resource_template = await user_resource_repo.create_user_resource_item( - user_resource_create, - workspace.id, - workspace_service.id, - workspace_service.templateName, - user.id, - user.roles, - owner_id - ) - except (ValidationError, ValueError) as e: - logger.exception("Failed create user resource model instance") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except UserNotAuthorizedToUseTemplate as e: - logger.exception("User not authorized to use template") - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + owner_id: str = None - # Get the workspace subscription id (if set) - if workspace.properties.get("workspace_subscription_id"): - user_resource.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] + # Check for assign_to_another_user logic + if ( + hasattr(user_resource_create, "properties") + and isinstance(user_resource_create.properties, dict) + and user_resource_create.properties.get("assign_to_another_user") is True + ): + if user_resource_create.properties.get("owner_id"): + owner_id = user_resource_create.properties.get("owner_id") - operation = await save_and_deploy_resource( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) - response.headers["Location"] = construct_location_header(operation) + try: + user_resource, resource_template = await user_resource_repo.create_user_resource_item( + user_resource_create, + workspace.id, + workspace_service.id, + workspace_service.templateName, + user.id, + user.roles, + owner_id + ) + except (ValidationError, ValueError) as e: + logger.exception("Failed create user resource model instance") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except UserNotAuthorizedToUseTemplate as e: + logger.exception("User not authorized to use template") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + + # Get the workspace subscription id (if set) + if workspace.properties.get("workspace_subscription_id"): + user_resource.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] - return OperationInResponse(operation=operation) + operation = await save_and_deploy_resource( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template, + operation_id=operation_id) + response.headers["Location"] = construct_location_header(operation) + + return OperationInResponse(operation=operation) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @user_resources_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}", response_model=OperationInResponse, name=strings.API_DELETE_USER_RESOURCE) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 2a924ec39a..76d14b99e6 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -4,7 +4,7 @@ from azure.core import MatchConditions from azure.core.exceptions import ResourceExistsError -from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosAccessConditionFailedError +from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosAccessConditionFailedError, CosmosResourceNotFoundError from fastapi import HTTPException, status as http_status from pydantic import TypeAdapter from db.repositories.resource_templates import ResourceTemplateRepository @@ -43,6 +43,18 @@ async def save_item(self, item: Operation): new_etag = response["_etag"] item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + async def get_active_operations_for_resource(self, resource_id: str) -> List[Operation]: + query = self.operations_query() + f' c.resourceId = "{resource_id}" AND NOT ARRAY_CONTAINS(["deployed", "deleted", "updated", "invoking_action_failed"], c.status)' + operations = await self.query(query=query) + return [TypeAdapter(Operation).validate_python(op) for op in operations] + + async def get_last_operation_for_resource(self, resource_id: str) -> Operation: + query = self.operations_query() + f' c.resourceId = "{resource_id}" ORDER BY c.createdWhen DESC OFFSET 0 LIMIT 1' + operations = await self.query(query=query) + if not operations: + raise EntityDoesNotExist + return TypeAdapter(Operation).validate_python(operations[0]) + async def update_item(self, item: Operation, etag: Optional[str] = None) -> Operation: etag_to_match = etag or getattr(item, "etag", None) item_dict = item.model_dump(exclude={"etag"}) @@ -83,8 +95,10 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> check = self.resource_has_active_operation(workspace_id) if hasattr(check, "__await__"): is_active = await check - if is_active is True: - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + else: + is_active = check + if is_active is True: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) lease_id = f"lease_{workspace_id}" timestamp = self.get_timestamp() @@ -96,33 +110,54 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> } try: - await self.container.create_item(body=lease_body) + create_call = self.container.create_item(body=lease_body) + if hasattr(create_call, "__await__"): + await create_call return True except (CosmosResourceExistsError, ResourceExistsError): try: - existing_lease = await self.read_item_by_id(lease_id) - current_op_id = existing_lease.get("operationId") - lease_created = existing_lease.get("createdWhen", 0.0) - - if current_op_id != operation_id: - is_active = False - if hasattr(self, "resource_has_active_operation"): - active_check = self.resource_has_active_operation(workspace_id) - if hasattr(active_check, "__await__"): - is_active = (await active_check) is True - if (timestamp - lease_created < 60.0) or is_active: + read_call = self.read_item_by_id(lease_id) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + current_op_id = existing_lease.get("operationId") if isinstance(existing_lease, dict) else None + + if current_op_id == operation_id: + return True + + if current_op_id: + try: + op_call = self.get_operation_by_id(current_op_id) + existing_op = await op_call if hasattr(op_call, "__await__") else op_call + terminal_statuses = { + Status.Deployed, + Status.DeploymentFailed, + Status.Deleted, + Status.DeletingFailed, + Status.Updated, + Status.UpdatingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + } + if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + except (EntityDoesNotExist, CosmosResourceNotFoundError): + # Operation doc not written yet (still constructing cascade/operation). + # Keep ownership until explicit release! raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - etag = existing_lease.get("_etag") + etag = existing_lease.get("_etag") if isinstance(existing_lease, dict) else None if etag: - await self.container.replace_item( + rep_call = self.container.replace_item( item=lease_id, body=lease_body, etag=etag, match_condition=MatchConditions.IfNotModified ) + if hasattr(rep_call, "__await__"): + await rep_call else: - await self.container.upsert_item(body=lease_body) + up_call = self.container.upsert_item(body=lease_body) + if hasattr(up_call, "__await__"): + await up_call return True except (CosmosAccessConditionFailedError, ResourceExistsError): raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) @@ -132,9 +167,12 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona return lease_id = f"lease_{workspace_id}" try: - existing_lease = await self.read_item_by_id(lease_id) - if not operation_id or existing_lease.get("operationId") == operation_id: - await self.delete_item(lease_id) + read_call = self.read_item_by_id(lease_id) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + if isinstance(existing_lease, dict) and (not operation_id or existing_lease.get("operationId") == operation_id): + del_call = self.delete_item(lease_id) + if hasattr(del_call, "__await__"): + await del_call except Exception: pass @@ -164,8 +202,9 @@ def create_main_step(self, resource_template: dict, action: str, resource_id: st message=message, updatedWhen=self.get_timestamp()) - async def create_operation_item(self, resource_id: str, resource_list: List, action: str, resource_path: str, resource_version: int, user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository) -> Operation: - operation_id = self.create_operation_id() + async def create_operation_item(self, resource_id: str, resource_list: List, action: str, resource_path: str, resource_version: int, user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, operation_id: Optional[str] = None) -> Operation: + if not operation_id: + operation_id = self.create_operation_id() target_workspace_id = extract_workspace_id_from_resource_path(resource_path) if target_workspace_id: diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index e9f19be744..27399a1e8f 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -201,6 +201,8 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline, resource_type: Resour raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in the '{RequestAction.UnInstall}' pipeline") if i != num_steps - 1: raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must be the final step in the '{RequestAction.UnInstall}' pipeline") + if num_of_main_steps == 0: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' requires a preceding 'main' step in the '{RequestAction.UnInstall}' pipeline") if (step.get("resourceType") != ResourceType.Workspace or step.get("resourceAction") != RequestAction.Upgrade): raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must upgrade a workspace") diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index f72fc161d5..1ad534e147 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -620,8 +620,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if operation.resourceId and operation.resourceId != str(message.id): try: primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) - primary_resource.deploymentStatus = failure_status - await self.resource_repo.update_item(primary_resource) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + primary_status = ( + Status.Deleted + if (operation.action == RequestAction.UnInstall and main_step and main_step.is_success()) + else failure_status + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) except Exception: pass await self.operations_repo.update_item(operation) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e5c22a70d6..baa1f5178d 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -1,4 +1,5 @@ import json +from typing import Optional from db.repositories.resources import ResourceRepository from db.repositories.resource_templates import ResourceTemplateRepository @@ -14,7 +15,7 @@ from services.logging import logger, tracer -async def send_resource_request_message(resource: Resource, operations_repo: OperationRepository, resource_repo: ResourceRepository, user: User, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, action: RequestAction = RequestAction.Install, is_cascade: str = False) -> Operation: +async def send_resource_request_message(resource: Resource, operations_repo: OperationRepository, resource_repo: ResourceRepository, user: User, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, action: RequestAction = RequestAction.Install, is_cascade: str = False, operation_id: Optional[str] = None) -> Operation: """ Creates and sends a resource request message for the resource to the Service Bus. The resource ID is added to the message to serve as an correlation ID for the deployment process. @@ -42,7 +43,8 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope resource_version=resource.resourceVersion, user=user, resource_repo=resource_repo, - resource_template_repo=resource_template_repo) + resource_template_repo=resource_template_repo, + operation_id=operation_id) current_span.set_attribute("operation_id", operation.id) try: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index ccdc054a55..27bb43a707 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -9,6 +9,7 @@ from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository from db.repositories.operations import OperationRepository +from db.errors import EntityDoesNotExist from resources import strings from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP @@ -235,7 +236,7 @@ async def test_acquire_workspace_lease_raises_409_if_active_operation_exists(ope assert exc.value.status_code == 409 -async def test_acquire_workspace_lease_raises_409_if_recent_lease_exists(operations_repo): +async def test_acquire_workspace_lease_raises_409_if_existing_lease_operation_not_found(operations_repo): from fastapi import HTTPException from azure.cosmos.exceptions import CosmosResourceExistsError operations_repo._container = MagicMock() @@ -246,12 +247,47 @@ async def test_acquire_workspace_lease_raises_409_if_recent_lease_exists(operati "operationId": "other-op", "createdWhen": operations_repo.get_timestamp(), }) + operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) with pytest.raises(HTTPException) as exc: await operations_repo.acquire_workspace_lease("ws-1", "op-1") assert exc.value.status_code == 409 +async def test_acquire_workspace_lease_succeeds_if_same_operation_holds_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1", + "createdWhen": operations_repo.get_timestamp(), + }) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + + +async def test_acquire_workspace_lease_replaces_lease_if_previous_operation_terminal(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp() - 1000, + "_etag": "old-etag", + }) + operations_repo.get_operation_by_id = AsyncMock(return_value=MagicMock(status=Status.Deployed)) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.replace_item.assert_awaited_once() + + async def test_release_workspace_lease_deletes_item(operations_repo): operations_repo._container = MagicMock() operations_repo.read_item_by_id = AsyncMock(return_value={ diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 1d9fae45d6..7dc02ecb1a 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -318,6 +318,9 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel { "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], }, + { + "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + }, ], ) async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails(resource_template_repo, input_workspace_service_template, pipeline): From f32fe944f2f02b184960c43d2de2a42917781dc5 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:05:09 +0000 Subject: [PATCH 56/90] Reclaim orphaned workspace leases after bounded expiry and retry on conflict-delete races --- api_app/db/repositories/operations.py | 89 +++++++++++-------- .../test_operation_repository.py | 33 +++++++ 2 files changed, 87 insertions(+), 35 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 76d14b99e6..d6f62e2945 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -3,7 +3,7 @@ from typing import List, Optional from azure.core import MatchConditions -from azure.core.exceptions import ResourceExistsError +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosAccessConditionFailedError, CosmosResourceNotFoundError from fastapi import HTTPException, status as http_status from pydantic import TypeAdapter @@ -19,6 +19,8 @@ from db.errors import EntityDoesNotExist from models.domain.operation import Operation, OperationStep, Status +WORKSPACE_LEASE_EXPIRY_SECONDS = 300.0 + def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: if resource_path and resource_path.startswith("/workspaces/"): @@ -101,28 +103,41 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) lease_id = f"lease_{workspace_id}" - timestamp = self.get_timestamp() - lease_body = { - "id": lease_id, - "workspaceId": workspace_id, - "operationId": operation_id, - "createdWhen": timestamp, - } + max_attempts = 3 + + for attempt in range(max_attempts): + timestamp = self.get_timestamp() + lease_body = { + "id": lease_id, + "workspaceId": workspace_id, + "operationId": operation_id, + "createdWhen": timestamp, + } - try: - create_call = self.container.create_item(body=lease_body) - if hasattr(create_call, "__await__"): - await create_call - return True - except (CosmosResourceExistsError, ResourceExistsError): try: - read_call = self.read_item_by_id(lease_id) - existing_lease = await read_call if hasattr(read_call, "__await__") else read_call - current_op_id = existing_lease.get("operationId") if isinstance(existing_lease, dict) else None + create_call = self.container.create_item(body=lease_body) + if hasattr(create_call, "__await__"): + await create_call + return True + except (CosmosResourceExistsError, ResourceExistsError): + try: + read_call = self.read_item_by_id(lease_id) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + # The lease was released between create_item conflict and read_item_by_id; retry + if attempt < max_attempts - 1: + continue + raise + + if not isinstance(existing_lease, dict): + return True + current_op_id = existing_lease.get("operationId") if current_op_id == operation_id: return True + lease_created = existing_lease.get("createdWhen", 0.0) + if current_op_id: try: op_call = self.get_operation_by_id(current_op_id) @@ -141,26 +156,30 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) except (EntityDoesNotExist, CosmosResourceNotFoundError): # Operation doc not written yet (still constructing cascade/operation). - # Keep ownership until explicit release! - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + # Safely reclaim orphaned leases after a bounded interval (WORKSPACE_LEASE_EXPIRY_SECONDS). + if timestamp - lease_created < WORKSPACE_LEASE_EXPIRY_SECONDS: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) etag = existing_lease.get("_etag") if isinstance(existing_lease, dict) else None - if etag: - rep_call = self.container.replace_item( - item=lease_id, - body=lease_body, - etag=etag, - match_condition=MatchConditions.IfNotModified - ) - if hasattr(rep_call, "__await__"): - await rep_call - else: - up_call = self.container.upsert_item(body=lease_body) - if hasattr(up_call, "__await__"): - await up_call - return True - except (CosmosAccessConditionFailedError, ResourceExistsError): - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + try: + if etag: + rep_call = self.container.replace_item( + item=lease_id, + body=lease_body, + etag=etag, + match_condition=MatchConditions.IfNotModified + ) + if hasattr(rep_call, "__await__"): + await rep_call + else: + up_call = self.container.upsert_item(body=lease_body) + if hasattr(up_call, "__await__"): + await up_call + return True + except (CosmosAccessConditionFailedError, ResourceExistsError): + if attempt < max_attempts - 1: + continue + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) async def release_workspace_lease(self, workspace_id: str, operation_id: Optional[str] = None) -> None: if not hasattr(self, "_container") or self._container is None: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 27bb43a707..a3130c91cd 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -288,6 +288,39 @@ async def test_acquire_workspace_lease_replaces_lease_if_previous_operation_term operations_repo._container.replace_item.assert_awaited_once() +async def test_acquire_workspace_lease_reclaims_orphaned_lease_when_operation_not_found_after_expiry(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "orphaned-op", + "createdWhen": operations_repo.get_timestamp() - 500, + "_etag": "old-etag", + }) + operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.replace_item.assert_awaited_once() + + +async def test_acquire_workspace_lease_retries_and_succeeds_when_lease_deleted_during_conflict_race(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosResourceNotFoundError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + # 1st attempt: create fails with conflict, read fails because lease was deleted + # 2nd attempt: create succeeds! + operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) + operations_repo.read_item_by_id = AsyncMock(side_effect=CosmosResourceNotFoundError()) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + assert operations_repo._container.create_item.await_count == 2 + + async def test_release_workspace_lease_deletes_item(operations_repo): operations_repo._container = MagicMock() operations_repo.read_item_by_id = AsyncMock(return_value={ From c3cd999afd2bf6a7c28c72cb6c18493fa0c4ef16 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:19:55 +0000 Subject: [PATCH 57/90] Reconcile stale operations and use conditional lease release --- CHANGELOG.md | 2 +- api_app/api/routes/workspaces.py | 6 ++ api_app/db/repositories/base.py | 9 ++- api_app/db/repositories/operations.py | 63 +++++++++++++---- .../test_operation_repository.py | 70 ++++++++++++++++++- 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 324a34862f..355803d3f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ENHANCEMENTS: BUG FIXES: * Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055)) -* `address_spaces` will now be removed from a workspace when a workspace service that uses an `address_space` is deleted to prevent IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) +* A workspace service's allocated `address_space` will now be removed from its parent workspace's `address_spaces` list when the service is deleted, preventing IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) ## (0.29.0) (August 14, 2026) **BREAKING CHANGES** diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index bdfb55248f..55c9ddeb1f 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -129,6 +129,9 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) @@ -332,6 +335,9 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace_service.workspaceId, operation_id) diff --git a/api_app/db/repositories/base.py b/api_app/db/repositories/base.py index a4cd5aed4b..609344444d 100644 --- a/api_app/db/repositories/base.py +++ b/api_app/db/repositories/base.py @@ -44,8 +44,13 @@ async def upsert_item_with_etag(self, item: BaseModel, etag: str) -> BaseModel: async def update_item_dict(self, item_dict: dict): await self.container.upsert_item(body=item_dict) - async def delete_item(self, item_id: str): - await self.container.delete_item(item=item_id, partition_key=item_id) + async def delete_item(self, item_id: str, etag: Optional[str] = None, match_condition: Optional[MatchConditions] = None): + kwargs = {} + if etag is not None: + kwargs["etag"] = etag + if match_condition is not None: + kwargs["match_condition"] = match_condition + await self.container.delete_item(item=item_id, partition_key=item_id, **kwargs) async def rename_field_name(self, old_field_name: str, new_field_name: str): for item in await self.query('SELECT * FROM c'): diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index d6f62e2945..fd7618ed41 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -17,7 +17,7 @@ from db.repositories.base import BaseRepository from db.errors import EntityDoesNotExist -from models.domain.operation import Operation, OperationStep, Status +from models.domain.operation import Operation, OperationStep, Status, get_failure_status_for_action WORKSPACE_LEASE_EXPIRY_SECONDS = 300.0 @@ -93,15 +93,6 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> if not hasattr(self, "_container") or self._container is None: return True - if hasattr(self, "resource_has_active_operation"): - check = self.resource_has_active_operation(workspace_id) - if hasattr(check, "__await__"): - is_active = await check - else: - is_active = check - if is_active is True: - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - lease_id = f"lease_{workspace_id}" max_attempts = 3 @@ -153,7 +144,19 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> Status.ActionFailed, } if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + op_time = getattr(existing_op, "updatedWhen", 0.0) or getattr(existing_op, "createdWhen", 0.0) + if timestamp - op_time < WORKSPACE_LEASE_EXPIRY_SECONDS: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + # Stale operation from interrupted dispatch; reconcile to failed status + try: + existing_op.status = get_failure_status_for_action(existing_op.action) + existing_op.message = "Operation timed out or was interrupted before completion" + existing_op.updatedWhen = timestamp + update_call = self.update_item(existing_op) + if hasattr(update_call, "__await__"): + await update_call + except Exception: + pass except (EntityDoesNotExist, CosmosResourceNotFoundError): # Operation doc not written yet (still constructing cascade/operation). # Safely reclaim orphaned leases after a bounded interval (WORKSPACE_LEASE_EXPIRY_SECONDS). @@ -189,7 +192,11 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona read_call = self.read_item_by_id(lease_id) existing_lease = await read_call if hasattr(read_call, "__await__") else read_call if isinstance(existing_lease, dict) and (not operation_id or existing_lease.get("operationId") == operation_id): - del_call = self.delete_item(lease_id) + etag = existing_lease.get("_etag") + if etag: + del_call = self.delete_item(lease_id, etag=etag, match_condition=MatchConditions.IfNotModified) + else: + del_call = self.delete_item(lease_id) if hasattr(del_call, "__await__"): await del_call except Exception: @@ -405,4 +412,34 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: + f' AND c.status IN ({status_filter})' ) operations = await self.query(query=query) - return len(operations) > 0 + if not operations: + return False + + timestamp = self.get_timestamp() + has_active = False + for op_dict in operations: + op_status = op_dict.get("status") if isinstance(op_dict, dict) else getattr(op_dict, "status", None) + if op_status is not None and op_status not in active_statuses: + continue + op_time = ( + (op_dict.get("updatedWhen") or op_dict.get("createdWhen")) + if isinstance(op_dict, dict) + else (getattr(op_dict, "updatedWhen", None) or getattr(op_dict, "createdWhen", None)) + ) + # If timestamp is present and exceeds expiry, it is stale + if op_time is not None and (timestamp - op_time >= WORKSPACE_LEASE_EXPIRY_SECONDS): + # Stale active operation from interrupted dispatch; reconcile to failure status + try: + op = TypeAdapter(Operation).validate_python(op_dict) + op.status = get_failure_status_for_action(op.action) + op.message = "Operation timed out or was interrupted before completion" + op.updatedWhen = timestamp + update_call = self.update_item(op) + if hasattr(update_call, "__await__"): + await update_call + except Exception: + pass + else: + has_active = True + + return has_active diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index a3130c91cd..e8f438650c 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -228,8 +228,17 @@ async def test_acquire_workspace_lease_creates_item(operations_repo): async def test_acquire_workspace_lease_raises_409_if_active_operation_exists(operations_repo): from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError operations_repo._container = MagicMock() - operations_repo.resource_has_active_operation = AsyncMock(return_value=True) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": now, + }) + active_op = MagicMock(status=Status.Deploying, updatedWhen=now, createdWhen=now) + operations_repo.get_operation_by_id = AsyncMock(return_value=active_op) with pytest.raises(HTTPException) as exc: await operations_repo.acquire_workspace_lease("ws-1", "op-1") @@ -331,3 +340,62 @@ async def test_release_workspace_lease_deletes_item(operations_repo): await operations_repo.release_workspace_lease("ws-1", "op-1") operations_repo.delete_item.assert_awaited_once_with("lease_ws-1") + + +async def test_release_workspace_lease_deletes_item_with_etag(operations_repo): + from azure.core import MatchConditions + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1", + "_etag": "lease-etag", + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_workspace_lease("ws-1", "op-1") + operations_repo.delete_item.assert_awaited_once_with("lease_ws-1", etag="lease-etag", match_condition=MatchConditions.IfNotModified) + + +async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_replaces_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": now - 500, + "_etag": "lease-etag", + }) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 500, createdWhen=now - 500) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock() + operations_repo._container.replace_item = AsyncMock(return_value={}) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo.update_item.assert_awaited_once() + assert stale_op.status == Status.DeploymentFailed + operations_repo._container.replace_item.assert_awaited_once() + + +async def test_resource_has_active_operation_reconciles_stale_operation(operations_repo): + now = operations_repo.get_timestamp() + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" + stale_op_dict = { + "id": "op-stale", + "resourceId": workspace_id, + "resourcePath": f"/workspaces/{workspace_id}", + "action": "install", + "status": Status.AwaitingDeployment, + "createdWhen": now - 500, + "updatedWhen": now - 500, + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock() + + result = await operations_repo.resource_has_active_operation(workspace_id) + assert result is False + operations_repo.update_item.assert_awaited_once() + saved_op = operations_repo.update_item.call_args[0][0] + assert saved_op.status == Status.DeploymentFailed From f91b6cda080759101502fb70ff94ee121a7192e5 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:25:38 +0000 Subject: [PATCH 58/90] bump version --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 76b47c1ddb..5252f582fe 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.8" +__version__ = "0.26.9" From 7265c8b72c41951794534298a1494f24f353b7e4 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:31:22 +0000 Subject: [PATCH 59/90] Address Copilot review on lease expiry, acquisition retry, and fail-closed reconciliation --- api_app/db/repositories/operations.py | 27 +++++++-- .../test_operation_repository.py | 59 ++++++++++++++++--- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index fd7618ed41..2e57b52bb2 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -16,10 +16,16 @@ from core import config from db.repositories.base import BaseRepository +from services.logging import logger from db.errors import EntityDoesNotExist from models.domain.operation import Operation, OperationStep, Status, get_failure_status_for_action -WORKSPACE_LEASE_EXPIRY_SECONDS = 300.0 +# The resource processor auto-renews Porter session locks for up to 3600 seconds (1 hour) +# during deployment execution (see resource_processor/vmss_porter/runner.py:68). +# Lease expiry must safely cover this maximum execution duration rather than aging out +# active operations after 300 seconds, preventing premature lease reclamation and +# overlapping Terraform operations. +WORKSPACE_LEASE_EXPIRY_SECONDS = 3600.0 def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: @@ -155,8 +161,14 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> update_call = self.update_item(existing_op) if hasattr(update_call, "__await__"): await update_call - except Exception: - pass + except Exception as e: + logger.exception(f"Failed to reconcile stale operation {existing_op.id}: {e}") + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + # update_item persists a terminal status and releases/deletes the workspace lease. + # Restart lease acquisition rather than replacing the now-deleted lease document. + continue + except HTTPException: + raise except (EntityDoesNotExist, CosmosResourceNotFoundError): # Operation doc not written yet (still constructing cascade/operation). # Safely reclaim orphaned leases after a bounded interval (WORKSPACE_LEASE_EXPIRY_SECONDS). @@ -179,11 +191,13 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> if hasattr(up_call, "__await__"): await up_call return True - except (CosmosAccessConditionFailedError, ResourceExistsError): + except (CosmosAccessConditionFailedError, ResourceExistsError, CosmosResourceNotFoundError, ResourceNotFoundError): if attempt < max_attempts - 1: continue raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + async def release_workspace_lease(self, workspace_id: str, operation_id: Optional[str] = None) -> None: if not hasattr(self, "_container") or self._container is None: return @@ -437,8 +451,9 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: update_call = self.update_item(op) if hasattr(update_call, "__await__"): await update_call - except Exception: - pass + except Exception as e: + logger.exception(f"Failed to reconcile stale active operation {op_dict.get('id') if isinstance(op_dict, dict) else getattr(op_dict, 'id', None)}: {e}") + has_active = True else: has_active = True diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index e8f438650c..2b596d877c 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -305,7 +305,7 @@ async def test_acquire_workspace_lease_reclaims_orphaned_lease_when_operation_no operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", "operationId": "orphaned-op", - "createdWhen": operations_repo.get_timestamp() - 500, + "createdWhen": operations_repo.get_timestamp() - 4000, "_etag": "old-etag", }) operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) @@ -356,27 +356,49 @@ async def test_release_workspace_lease_deletes_item_with_etag(operations_repo): operations_repo.delete_item.assert_awaited_once_with("lease_ws-1", etag="lease-etag", match_condition=MatchConditions.IfNotModified) -async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_replaces_lease(operations_repo): +async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_reacquires_lease(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError operations_repo._container = MagicMock() - operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + # 1st attempt: conflict on create, reads lease, reconciles stale op, restarts loop + # 2nd attempt: create succeeds because old lease was released by update_item + operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) now = operations_repo.get_timestamp() operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", "operationId": "stale-op", - "createdWhen": now - 500, + "createdWhen": now - 4000, "_etag": "lease-etag", }) - stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 500, createdWhen=now - 500) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 4000, createdWhen=now - 4000) operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock() - operations_repo._container.replace_item = AsyncMock(return_value={}) res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") assert res is True operations_repo.update_item.assert_awaited_once() assert stale_op.status == Status.DeploymentFailed - operations_repo._container.replace_item.assert_awaited_once() + assert operations_repo._container.create_item.await_count == 2 + + +async def test_acquire_workspace_lease_fails_closed_when_reconciliation_fails(operations_repo): + from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": now - 4000, + "_etag": "lease-etag", + }) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 4000, createdWhen=now - 4000) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 async def test_resource_has_active_operation_reconciles_stale_operation(operations_repo): @@ -388,8 +410,8 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio "resourcePath": f"/workspaces/{workspace_id}", "action": "install", "status": Status.AwaitingDeployment, - "createdWhen": now - 500, - "updatedWhen": now - 500, + "createdWhen": now - 4000, + "updatedWhen": now - 4000, } operations_repo.query = AsyncMock(return_value=[stale_op_dict]) operations_repo.update_item = AsyncMock() @@ -399,3 +421,22 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio operations_repo.update_item.assert_awaited_once() saved_op = operations_repo.update_item.call_args[0][0] assert saved_op.status == Status.DeploymentFailed + + +async def test_resource_has_active_operation_fails_closed_when_reconciliation_fails(operations_repo): + now = operations_repo.get_timestamp() + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" + stale_op_dict = { + "id": "op-stale", + "resourceId": workspace_id, + "resourcePath": f"/workspaces/{workspace_id}", + "action": "install", + "status": Status.AwaitingDeployment, + "createdWhen": now - 4000, + "updatedWhen": now - 4000, + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) + + result = await operations_repo.resource_has_active_operation(workspace_id) + assert result is True From 64d16714d3a0dd6ac0008e7796ad2b9da96a4e61 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:39:27 +0000 Subject: [PATCH 60/90] Exclude terminal statuses in active ops query and require non-empty stepTitle for cleanup step --- api_app/db/repositories/operations.py | 15 ++++++- api_app/db/repositories/resource_templates.py | 2 + .../test_operation_repository.py | 43 +++++++++++++++++++ .../test_resource_templates_repository.py | 7 ++- 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 2e57b52bb2..5cf213bdc6 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -52,7 +52,18 @@ async def save_item(self, item: Operation): item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag async def get_active_operations_for_resource(self, resource_id: str) -> List[Operation]: - query = self.operations_query() + f' c.resourceId = "{resource_id}" AND NOT ARRAY_CONTAINS(["deployed", "deleted", "updated", "invoking_action_failed"], c.status)' + terminal_statuses = ( + Status.Deployed, + Status.DeploymentFailed, + Status.Updated, + Status.UpdatingFailed, + Status.Deleted, + Status.DeletingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + ) + status_filter = ", ".join(f'\"{s}\"' for s in terminal_statuses) + query = self.operations_query() + f' c.resourceId = "{resource_id}" AND NOT ARRAY_CONTAINS([{status_filter}], c.status)' operations = await self.query(query=query) return [TypeAdapter(Operation).validate_python(op) for op in operations] @@ -341,7 +352,7 @@ async def build_step_list(self, steps: List[OperationStep], resource_template_di steps.append(OperationStep( id=str(uuid.uuid4()), templateStepId=step["stepId"], - stepTitle=step["stepTitle"], + stepTitle=step.get("stepTitle"), resourceId=resource_for_step.id, resourceTemplateName=resource_for_step.templateName, resourceType=resource_for_step.resourceType, diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 27399a1e8f..5033c848ef 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -206,6 +206,8 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline, resource_type: Resour if (step.get("resourceType") != ResourceType.Workspace or step.get("resourceAction") != RequestAction.Upgrade): raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must upgrade a workspace") + if not step.get("stepTitle") or not str(step.get("stepTitle")).strip(): + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' requires a non-empty 'stepTitle'") if step_id != "main": step_ids.append(step_id) diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 2b596d877c..2415ad8237 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -440,3 +440,46 @@ async def test_resource_has_active_operation_fails_closed_when_reconciliation_fa result = await operations_repo.resource_has_active_operation(workspace_id) assert result is True + + +async def test_get_active_operations_for_resource_excludes_terminal_statuses(operations_repo): + operations_repo.query = AsyncMock(return_value=[]) + + await operations_repo.get_active_operations_for_resource("res-1") + operations_repo.query.assert_awaited_once() + query_str = operations_repo.query.call_args[1]["query"] + assert 'c.resourceId = "res-1"' in query_str + assert 'NOT ARRAY_CONTAINS(["deployed", "deployment_failed", "updated", "updating_failed", "deleted", "deleting_failed", "action_succeeded", "action_failed"], c.status)' in query_str + + +async def test_build_step_list_tolerates_missing_step_title(operations_repo): + ws_id = str(uuid.uuid4()) + svc_id = str(uuid.uuid4()) + resource_template_dict = { + "pipeline": { + "upgrade": [ + { + "stepId": "step-1", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + } + ] + } + } + mock_resource_repo = MagicMock() + mock_workspace = MagicMock(id=ws_id, templateName="tre-workspace", resourceType=ResourceType.Workspace) + mock_service = MagicMock(id=svc_id, workspaceId=ws_id, resourceType=ResourceType.WorkspaceService) + mock_resource_repo.get_resource_by_id = AsyncMock(side_effect=[mock_service, mock_workspace]) + + steps = await operations_repo.build_step_list( + steps=[], + resource_template_dict=resource_template_dict, + action="upgrade", + resource_repo=mock_resource_repo, + resource_id=svc_id, + status=Status.AwaitingUpdate, + message="updating", + ) + assert len(steps) == 1 + assert steps[0].stepTitle is None + assert steps[0].templateStepId == "step-1" diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 7dc02ecb1a..a61f995fb1 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -319,7 +319,10 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], }, { - "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade, "stepTitle": ""}], }, ], ) @@ -343,6 +346,7 @@ async def test_create_template_with_address_space_cleanup_on_non_workspace_servi {"stepId": "main"}, { "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "stepTitle": "Cleanup address space", "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade, }, @@ -359,6 +363,7 @@ async def test_create_template_with_valid_address_space_cleanup_as_final_uninsta {"stepId": "main"}, { "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "stepTitle": "Cleanup address space", "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade, }, From ce885d088e6ef0ea872237af2e7769e97e4c79cd Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:50:19 +0000 Subject: [PATCH 61/90] Extend lease expiry margin, keep lease during caller rollback, and serialize airlock review deletions --- api_app/db/repositories/operations.py | 11 +++-- .../service_bus/resource_request_sender.py | 27 ++++++++++-- api_app/services/airlock.py | 44 ++++++++++++++++++- .../test_operation_repository.py | 18 ++++---- .../test_resource_request_sender.py | 38 +++++++++++++++- 5 files changed, 116 insertions(+), 22 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 5cf213bdc6..31fa87afdf 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -22,10 +22,9 @@ # The resource processor auto-renews Porter session locks for up to 3600 seconds (1 hour) # during deployment execution (see resource_processor/vmss_porter/runner.py:68). -# Lease expiry must safely cover this maximum execution duration rather than aging out -# active operations after 300 seconds, preventing premature lease reclamation and -# overlapping Terraform operations. -WORKSPACE_LEASE_EXPIRY_SECONDS = 3600.0 +# Lease expiry includes a safe margin beyond the 3600-second execution window and status-delivery +# latency (7200 seconds / 2 hours) to ensure still-running operations are not prematurely reclaimed. +WORKSPACE_LEASE_EXPIRY_SECONDS = 7200.0 def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: @@ -74,7 +73,7 @@ async def get_last_operation_for_resource(self, resource_id: str) -> Operation: raise EntityDoesNotExist return TypeAdapter(Operation).validate_python(operations[0]) - async def update_item(self, item: Operation, etag: Optional[str] = None) -> Operation: + async def update_item(self, item: Operation, etag: Optional[str] = None, release_lease: bool = True) -> Operation: etag_to_match = etag or getattr(item, "etag", None) item_dict = item.model_dump(exclude={"etag"}) item_dict.pop("_etag", None) @@ -99,7 +98,7 @@ async def update_item(self, item: Operation, etag: Optional[str] = None) -> Oper Status.Deploying, Status.AwaitingDeletion, Status.Deleting, Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning ) - if item.status not in active_statuses: + if release_lease and item.status not in active_statuses: target_workspace_id = extract_workspace_id_from_resource_path(item.resourcePath) if target_workspace_id: await self.release_workspace_lease(target_workspace_id, item.id) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index baa1f5178d..16e3779ed1 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -8,10 +8,10 @@ from models.domain.authentication import User from models.domain.request_action import RequestAction -from models.domain.resource import Resource +from models.domain.resource import Resource, ResourceType from models.domain.operation import Operation, get_failure_status_for_action -from db.repositories.operations import OperationRepository +from db.repositories.operations import OperationRepository, extract_workspace_id_from_resource_path from services.logging import logger, tracer @@ -75,9 +75,30 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope first_step = operation.steps[0] first_step.status = get_failure_status_for_action(first_step.resourceAction) first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" - await operations_repo.update_item(operation) + # Keep lease held during caller compensation; caller will release lease upon completing rollback + await operations_repo.update_item(operation, release_lease=False) except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") + # Fallback: remove orphaned operation and release its workspace lease + try: + del_call = operations_repo.delete_item(operation.id) + if hasattr(del_call, "__await__"): + await del_call + except Exception: + logger.exception(f"Failed to delete orphaned operation {operation.id}") + target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) + if not target_workspace_id: + if getattr(resource, "resourceType", None) == ResourceType.Workspace: + target_workspace_id = resource.id + elif hasattr(resource, "workspaceId") and resource.workspaceId: + target_workspace_id = resource.workspaceId + if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + try: + rel_call = operations_repo.release_workspace_lease(target_workspace_id, operation.id) + if hasattr(rel_call, "__await__"): + await rel_call + except Exception: + logger.exception(f"Failed to release workspace lease for {target_workspace_id}") raise return operation diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 3c84f56192..5b8237f22a 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -1,3 +1,5 @@ +import asyncio +import time from datetime import datetime, timedelta, UTC from services.logging import logger @@ -8,7 +10,7 @@ from models.domain.authentication import User from models.domain.workspace import Workspace from models.domain.user_resource import UserResource -from models.domain.operation import Operation +from models.domain.operation import Operation, Status from models.domain.resource import ResourceType from models.domain.workspace_service import WorkspaceService from models.schemas.airlock_request import AirlockReviewInCreate @@ -402,6 +404,40 @@ def enrich_requests_with_allowed_actions(requests: List[AirlockRequest], user: U return enriched_requests +TERMINAL_STATUSES = { + Status.Deployed, + Status.DeploymentFailed, + Status.Updated, + Status.UpdatingFailed, + Status.Deleted, + Status.DeletingFailed, + Status.ActionSucceeded, + Status.ActionFailed, +} + + +async def wait_for_terminal_operation( + operations_repo: OperationRepository, + operation_id: str, + timeout: float = 30.0, + poll_interval: float = 0.5, +) -> Optional[Operation]: + if not hasattr(operations_repo, "get_operation_by_id"): + return None + start = time.time() + while time.time() - start < timeout: + try: + op_call = operations_repo.get_operation_by_id(operation_id) + op = await op_call if hasattr(op_call, "__await__") else op_call + status = getattr(op, "status", None) + if op is None or not isinstance(status, (str, Status)) or status in TERMINAL_STATUSES: + return op + except Exception: + pass + await asyncio.sleep(poll_interval) + return None + + async def delete_review_user_resource( user_resource: UserResource, user_resource_repo: UserResourceRepository, @@ -414,7 +450,9 @@ async def delete_review_user_resource( service_id=user_resource.parentWorkspaceServiceId) # disable might contain logic that we need to execute before the deletion of the resource - _ = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) + disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) + if disable_op and hasattr(disable_op, "id"): + await wait_for_terminal_operation(operations_repo, disable_op.id) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -472,6 +510,8 @@ async def delete_all_review_user_resources( user=user ) operations.append(operation) + if operation and hasattr(operation, "id"): + await wait_for_terminal_operation(operations_repo, operation.id) logger.info(f"Started {len(operations)} operations on deleting user resources") return operations diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 2415ad8237..f489e00e39 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -305,7 +305,7 @@ async def test_acquire_workspace_lease_reclaims_orphaned_lease_when_operation_no operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", "operationId": "orphaned-op", - "createdWhen": operations_repo.get_timestamp() - 4000, + "createdWhen": operations_repo.get_timestamp() - 8000, "_etag": "old-etag", }) operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) @@ -366,10 +366,10 @@ async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_rea operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", "operationId": "stale-op", - "createdWhen": now - 4000, + "createdWhen": now - 8000, "_etag": "lease-etag", }) - stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 4000, createdWhen=now - 4000) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 8000, createdWhen=now - 8000) operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock() @@ -389,10 +389,10 @@ async def test_acquire_workspace_lease_fails_closed_when_reconciliation_fails(op operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", "operationId": "stale-op", - "createdWhen": now - 4000, + "createdWhen": now - 8000, "_etag": "lease-etag", }) - stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 4000, createdWhen=now - 4000) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 8000, createdWhen=now - 8000) operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) @@ -410,8 +410,8 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio "resourcePath": f"/workspaces/{workspace_id}", "action": "install", "status": Status.AwaitingDeployment, - "createdWhen": now - 4000, - "updatedWhen": now - 4000, + "createdWhen": now - 8000, + "updatedWhen": now - 8000, } operations_repo.query = AsyncMock(return_value=[stale_op_dict]) operations_repo.update_item = AsyncMock() @@ -432,8 +432,8 @@ async def test_resource_has_active_operation_fails_closed_when_reconciliation_fa "resourcePath": f"/workspaces/{workspace_id}", "action": "install", "status": Status.AwaitingDeployment, - "createdWhen": now - 4000, - "updatedWhen": now - 4000, + "createdWhen": now - 8000, + "updatedWhen": now - 8000, } operations_repo.query = AsyncMock(return_value=[stale_op_dict]) operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index a9341efeaa..509f917909 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -236,7 +236,7 @@ async def test_send_resource_request_message_persists_failure_state_on_dispatch_ action=action, ) - operations_repo_mock.update_item.assert_awaited_once_with(operation) + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) assert operation.status == expected_status assert operation.steps[0].status == expected_status assert "Failed to dispatch" in operation.message @@ -271,7 +271,41 @@ async def test_send_resource_request_message_derives_step_failure_status_from_st action=RequestAction.UnInstall, ) - operations_repo_mock.update_item.assert_awaited_once_with(operation) + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) assert operation.status == Status.DeletingFailed assert operation.steps[0].status == Status.UpdatingFailed assert "Failed to dispatch" in operation.steps[0].message + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_cleans_up_orphaned_operation_when_persisting_failure_fails( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock() + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) + operations_repo_mock.release_workspace_lease.assert_awaited_once_with(resource.id, operation.id) From 4efc6507d037199a0f7ea105ca70e398031ed0d3 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:03:01 +0000 Subject: [PATCH 62/90] Release lease on internal dispatch failures and serialize airlock operations until success --- .../service_bus/resource_request_sender.py | 7 +- api_app/services/airlock.py | 61 +++++++++++--- .../test_resource_request_sender.py | 37 ++++++++- .../tests_ma/test_services/test_airlock.py | 81 +++++++++++++++++++ 4 files changed, 173 insertions(+), 13 deletions(-) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 16e3779ed1..dd197d9b66 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -75,8 +75,11 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope first_step = operation.steps[0] first_step.status = get_failure_status_for_action(first_step.resourceAction) first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" - # Keep lease held during caller compensation; caller will release lease upon completing rollback - await operations_repo.update_item(operation, release_lease=False) + # If the caller provided operation_id, they manage the operation lifecycle and outer compensation, + # so keep the lease held. Otherwise, this function created and owns the operation, so release the + # workspace lease when persisting terminal failure. + should_release_lease = operation_id is None + await operations_repo.update_item(operation, release_lease=should_release_lease) except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") # Fallback: remove orphaned operation and release its workspace lease diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 5b8237f22a..c7880ff81f 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -25,6 +25,7 @@ from api.routes.resource_helpers import save_and_deploy_resource, send_uninstall_message, update_user_resource +from db.errors import EntityDoesNotExist from db.repositories.user_resources import UserResourceRepository from db.repositories.workspace_services import WorkspaceServiceRepository from db.repositories.operations import OperationRepository @@ -261,7 +262,7 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": logger.info("Deleting existing user resource...") - _ = await delete_review_user_resource( + del_op = await delete_review_user_resource( user_resource=existing_resource, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -270,6 +271,8 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR resource_history_repo=resource_history_repo, user=user ) + if del_op and hasattr(del_op, "id"): + await wait_for_successful_operation(operation_repo, del_op.id) async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest, airlock_request_repo: AirlockRequestRepository, user: User, workspace: Workspace): @@ -416,26 +419,66 @@ def enrich_requests_with_allowed_actions(requests: List[AirlockRequest], user: U } -async def wait_for_terminal_operation( +SUCCESS_STATUSES = { + Status.Deployed, + Status.Updated, + Status.Deleted, + Status.ActionSucceeded, +} + +FAILURE_STATUSES = { + Status.DeploymentFailed, + Status.UpdatingFailed, + Status.DeletingFailed, + Status.ActionFailed, +} + + +async def wait_for_successful_operation( operations_repo: OperationRepository, operation_id: str, - timeout: float = 30.0, + timeout: float = 300.0, poll_interval: float = 0.5, ) -> Optional[Operation]: - if not hasattr(operations_repo, "get_operation_by_id"): + if not hasattr(operations_repo, "get_operation_by_id") or not isinstance(operation_id, str): return None start = time.time() while time.time() - start < timeout: try: op_call = operations_repo.get_operation_by_id(operation_id) op = await op_call if hasattr(op_call, "__await__") else op_call - status = getattr(op, "status", None) - if op is None or not isinstance(status, (str, Status)) or status in TERMINAL_STATUSES: + op_status = getattr(op, "status", None) + if op is None or not isinstance(op_status, (str, Status)): + return op + if op_status in SUCCESS_STATUSES: return op + if op_status in FAILURE_STATUSES: + logger.error(f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}" + ) + except HTTPException: + raise + except EntityDoesNotExist: + return None except Exception: pass await asyncio.sleep(poll_interval) - return None + logger.error(f"Timed out waiting for operation {operation_id} to complete successfully") + raise HTTPException( + status_code=status.HTTP_504_GATEWAY_TIMEOUT, + detail=f"Timed out waiting for operation {operation_id} to complete successfully" + ) + + +async def wait_for_terminal_operation( + operations_repo: OperationRepository, + operation_id: str, + timeout: float = 300.0, + poll_interval: float = 0.5, +) -> Optional[Operation]: + return await wait_for_successful_operation(operations_repo, operation_id, timeout=timeout, poll_interval=poll_interval) async def delete_review_user_resource( @@ -452,7 +495,7 @@ async def delete_review_user_resource( # disable might contain logic that we need to execute before the deletion of the resource disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) if disable_op and hasattr(disable_op, "id"): - await wait_for_terminal_operation(operations_repo, disable_op.id) + await wait_for_successful_operation(operations_repo, disable_op.id) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -511,7 +554,7 @@ async def delete_all_review_user_resources( ) operations.append(operation) if operation and hasattr(operation, "id"): - await wait_for_terminal_operation(operations_repo, operation.id) + await wait_for_successful_operation(operations_repo, operation.id) logger.info(f"Started {len(operations)} operations on deleting user resources") return operations diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 509f917909..72dd580492 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -236,7 +236,7 @@ async def test_send_resource_request_message_persists_failure_state_on_dispatch_ action=action, ) - operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=True) assert operation.status == expected_status assert operation.steps[0].status == expected_status assert "Failed to dispatch" in operation.message @@ -271,12 +271,45 @@ async def test_send_resource_request_message_derives_step_failure_status_from_st action=RequestAction.UnInstall, ) - operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=True) assert operation.status == Status.DeletingFailed assert operation.steps[0].status == Status.UpdatingFailed assert "Failed to dispatch" in operation.steps[0].message +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_retains_lease_when_caller_provides_operation_id( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + operation_id=operation.id, + ) + + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) + assert operation.status == Status.DeploymentFailed + + @patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) @patch("service_bus.resource_request_sender.update_resource_for_step") async def test_send_resource_request_message_cleans_up_orphaned_operation_when_persisting_failure_fails( diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 65c61e6d62..54896cff28 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -732,3 +732,84 @@ def test_authenticated_user_with_empty_email_persists_and_builds_notification(): assert notification.created_by.email == "" assert notification.updated_by.email == "" + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_success(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.Updated + repo.get_operation_by_id.return_value = op + + res = await wait_for_successful_operation(repo, "op-123", timeout=1.0) + assert res == op + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_failure(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.UpdatingFailed + op.message = "Failed step" + repo.get_operation_by_id.return_value = op + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=1.0) + assert exc_info.value.status_code == 500 + assert "Failed step" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_timeout(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.Updating + repo.get_operation_by_id.return_value = op + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=0.1, poll_interval=0.02) + assert exc_info.value.status_code == 504 + + +@pytest.mark.asyncio +@patch("services.airlock.disable_user_resource") +@patch("services.airlock.send_uninstall_message") +async def test_delete_review_user_resource_aborts_if_disable_fails(mock_send_uninstall, mock_disable): + from models.domain.operation import Status, Operation + + disable_op = MagicMock(spec=Operation) + disable_op.id = "op-disable" + mock_disable.return_value = disable_op + + operations_repo = AsyncMock() + op_record = MagicMock(spec=Operation) + op_record.status = Status.UpdatingFailed + op_record.message = "Could not disable" + operations_repo.get_operation_by_id.return_value = op_record + + workspace_service_repo = AsyncMock() + user_resource = AsyncMock() + user_resource.workspaceId = "ws-1" + user_resource.parentWorkspaceServiceId = "svc-1" + + with pytest.raises(HTTPException) as exc_info: + await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=AsyncMock(), + workspace_service_repo=workspace_service_repo, + resource_template_repo=AsyncMock(), + operations_repo=operations_repo, + resource_history_repo=AsyncMock(), + user=create_test_user(), + ) + assert exc_info.value.status_code == 500 + mock_send_uninstall.assert_not_called() From 7877eebbba45c91243b8eb3b6cd602c1beb03ebc Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:17:38 +0000 Subject: [PATCH 63/90] Address Copilot review: conditional rollback, ETag delete check, and background review VM cleanup --- CHANGELOG.md | 2 +- api_app/api/routes/workspaces.py | 4 +- .../service_bus/resource_request_sender.py | 54 +++++++++++++------ api_app/services/airlock.py | 37 ++++++------- .../test_resource_request_sender.py | 39 +++++++++++++- .../tests_ma/test_services/test_airlock.py | 13 +++++ 6 files changed, 111 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7acdf89b62..bb840d5154 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ENHANCEMENTS: BUG FIXES: * Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055)) -* A workspace service's allocated `address_space` will now be removed from its parent workspace's `address_spaces` list when the service is deleted, preventing IP address range exhaustion ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) +* A workspace service's allocated `address_space` will now be removed from its parent workspace's `address_spaces` list when the service is deleted, preventing IP address range exhaustion, and workspace-level operation leasing ensures safe serialization of concurrent workspace mutations ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) * Fix inconsistent ServiceBusClient lifecycle management in deployment_status_updater.py, airlock_request_status_update.py, and runner.py to prevent connection socket and AMQP channel leaks ([#4930](https://github.com/microsoft/AzureTRE/pull/4930)) ## (0.29.0) (August 14, 2026) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 55c9ddeb1f..4444bd4dd5 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -271,6 +271,7 @@ async def create_workspace_service(response: Response, workspace_service_input: if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + address_space_added = False try: try: workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) @@ -298,6 +299,7 @@ async def create_workspace_service(response: Response, workspace_service_input: # updated directly, and an "update" on the workspace is called by the workspace service pipeline. try: await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) + address_space_added = True except CosmosAccessConditionFailedError: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) @@ -311,7 +313,7 @@ async def create_workspace_service(response: Response, workspace_service_input: resource_template=resource_template, operation_id=operation_id) except Exception: - if 'resource_template' in locals() and 'workspace_service' in locals() and resource_template.properties.get("address_space") and workspace_service.properties.get("address_space"): + if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): try: latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) current_spaces = latest_workspace.properties.get("address_spaces", []) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index dd197d9b66..1d179da9ae 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -11,6 +11,9 @@ from models.domain.resource import Resource, ResourceType from models.domain.operation import Operation, get_failure_status_for_action +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceModifiedError +from azure.cosmos.exceptions import CosmosAccessConditionFailedError from db.repositories.operations import OperationRepository, extract_workspace_id_from_resource_path from services.logging import logger, tracer @@ -82,26 +85,43 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope await operations_repo.update_item(operation, release_lease=should_release_lease) except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") - # Fallback: remove orphaned operation and release its workspace lease + # Fallback: remove orphaned operation and release its workspace lease only if it has not been advanced + lease_released = False try: - del_call = operations_repo.delete_item(operation.id) - if hasattr(del_call, "__await__"): - await del_call + creation_etag = getattr(operation, "etag", None) + if hasattr(operations_repo, "delete_item"): + if creation_etag: + del_call = operations_repo.delete_item( + operation.id, + etag=creation_etag, + match_condition=MatchConditions.IfNotModified + ) + else: + del_call = operations_repo.delete_item(operation.id) + if hasattr(del_call, "__await__"): + await del_call + lease_released = True + except (CosmosAccessConditionFailedError, ResourceModifiedError, ResourceExistsError): + logger.warning( + f"Operation {operation.id} was modified concurrently; retaining operation and workspace lease" + ) except Exception: logger.exception(f"Failed to delete orphaned operation {operation.id}") - target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) - if not target_workspace_id: - if getattr(resource, "resourceType", None) == ResourceType.Workspace: - target_workspace_id = resource.id - elif hasattr(resource, "workspaceId") and resource.workspaceId: - target_workspace_id = resource.workspaceId - if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): - try: - rel_call = operations_repo.release_workspace_lease(target_workspace_id, operation.id) - if hasattr(rel_call, "__await__"): - await rel_call - except Exception: - logger.exception(f"Failed to release workspace lease for {target_workspace_id}") + + if lease_released: + target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) + if not target_workspace_id: + if getattr(resource, "resourceType", None) == ResourceType.Workspace: + target_workspace_id = resource.id + elif hasattr(resource, "workspaceId") and resource.workspaceId: + target_workspace_id = resource.workspaceId + if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + try: + rel_call = operations_repo.release_workspace_lease(target_workspace_id, operation.id) + if hasattr(rel_call, "__await__"): + await rel_call + except Exception: + logger.exception(f"Failed to release workspace lease for {target_workspace_id}") raise return operation diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index c7880ff81f..a37304c79a 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -155,7 +155,7 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - _ = await delete_all_review_user_resources( + asyncio.create_task(delete_all_review_user_resources( airlock_request=airlock_request, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -163,7 +163,8 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai operations_repo=operation_repo, resource_history_repo=resource_history_repo, user=user - ) + )) + await asyncio.sleep(0) return updated_airlock_request @@ -262,7 +263,7 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": logger.info("Deleting existing user resource...") - del_op = await delete_review_user_resource( + _ = await delete_review_user_resource( user_resource=existing_resource, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -271,8 +272,6 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR resource_history_repo=resource_history_repo, user=user ) - if del_op and hasattr(del_op, "id"): - await wait_for_successful_operation(operation_repo, del_op.id) async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest, airlock_request_repo: AirlockRequestRepository, user: User, workspace: Workspace): @@ -447,21 +446,22 @@ async def wait_for_successful_operation( try: op_call = operations_repo.get_operation_by_id(operation_id) op = await op_call if hasattr(op_call, "__await__") else op_call - op_status = getattr(op, "status", None) - if op is None or not isinstance(op_status, (str, Status)): - return op - if op_status in SUCCESS_STATUSES: - return op - if op_status in FAILURE_STATUSES: - logger.error(f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}" - ) + if op is not None: + op_status = getattr(op, "status", None) + if not isinstance(op_status, (str, Status)): + return op + if op_status in SUCCESS_STATUSES: + return op + if op_status in FAILURE_STATUSES: + logger.error(f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}" + ) except HTTPException: raise except EntityDoesNotExist: - return None + pass except Exception: pass await asyncio.sleep(poll_interval) @@ -564,7 +564,8 @@ async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) + asyncio.create_task(delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user)) + await asyncio.sleep(0) return updated_request diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 72dd580492..bb1610cb35 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -22,6 +22,7 @@ RequestAction, ) from models.domain.operation import Status +from azure.core import MatchConditions from azure.cosmos.exceptions import CosmosAccessConditionFailedError pytestmark = pytest.mark.asyncio @@ -318,6 +319,7 @@ async def test_send_resource_request_message_cleans_up_orphaned_operation_when_p ): resource = create_test_resource() operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" operations_repo_mock = AsyncMock() operations_repo_mock.create_operation_item.return_value = operation @@ -340,5 +342,40 @@ async def test_send_resource_request_message_cleans_up_orphaned_operation_when_p action=RequestAction.Install, ) - operations_repo_mock.delete_item.assert_awaited_once_with(operation.id) + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) operations_repo_mock.release_workspace_lease.assert_awaited_once_with(resource.id, operation.id) + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_retains_lease_when_concurrently_modified( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock(side_effect=CosmosAccessConditionFailedError) + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) + operations_repo_mock.release_workspace_lease.assert_not_called() diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 54896cff28..33ca1a1f4f 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -813,3 +813,16 @@ async def test_delete_review_user_resource_aborts_if_disable_fails(mock_send_uni ) assert exc_info.value.status_code == 500 mock_send_uninstall.assert_not_called() + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_retries_on_entity_does_not_exist(): + from services.airlock import wait_for_successful_operation + from db.errors import EntityDoesNotExist + + repo = AsyncMock() + repo.get_operation_by_id.side_effect = EntityDoesNotExist + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=0.1, poll_interval=0.02) + assert exc_info.value.status_code == 504 From 6248f86557b8b8e275232faf99596889e3c820a1 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:27:07 +0000 Subject: [PATCH 64/90] Address Copilot review feedback on background tasks, lease retention, and rollback --- api_app/api/routes/workspaces.py | 33 +++++--- .../service_bus/resource_request_sender.py | 2 +- api_app/services/airlock.py | 84 ++++++++++++++----- .../test_resource_request_sender.py | 36 ++++++++ .../tests_ma/test_services/test_airlock.py | 54 ++++++++++++ 5 files changed, 172 insertions(+), 37 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 4444bd4dd5..07582b4a27 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -314,19 +314,26 @@ async def create_workspace_service(response: Response, workspace_service_input: operation_id=operation_id) except Exception: if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): - try: - latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) - current_spaces = latest_workspace.properties.get("address_spaces", []) - allocated_space = workspace_service.properties["address_space"] - if allocated_space in current_spaces: - updated_spaces = [s for s in current_spaces if s != allocated_space] - rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) - await workspace_repo.patch_workspace( - latest_workspace, rollback_patch, latest_workspace.etag, - resource_template_repo, resource_history_repo, user, False - ) - except Exception: - logger.exception("Failed to rollback allocated address space on workspace") + max_rollback_attempts = 3 + for attempt in range(max_rollback_attempts): + try: + latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) + current_spaces = latest_workspace.properties.get("address_spaces", []) + allocated_space = workspace_service.properties["address_space"] + if allocated_space in current_spaces: + updated_spaces = [s for s in current_spaces if s != allocated_space] + rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) + await workspace_repo.patch_workspace( + latest_workspace, rollback_patch, latest_workspace.etag, + resource_template_repo, resource_history_repo, user, False + ) + break + except CosmosAccessConditionFailedError: + if attempt == max_rollback_attempts - 1: + logger.exception("Failed to rollback allocated address space due to repeated ETag conflicts") + except Exception: + logger.exception("Failed to rollback allocated address space on workspace") + break if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) raise diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 1d179da9ae..7191825f67 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -108,7 +108,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope except Exception: logger.exception(f"Failed to delete orphaned operation {operation.id}") - if lease_released: + if lease_released and should_release_lease: target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) if not target_workspace_id: if getattr(resource, "resourceType", None) == ResourceType.Workspace: diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index a37304c79a..125f08cf56 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -28,7 +28,7 @@ from db.errors import EntityDoesNotExist from db.repositories.user_resources import UserResourceRepository from db.repositories.workspace_services import WorkspaceServiceRepository -from db.repositories.operations import OperationRepository +from db.repositories.operations import OperationRepository, WORKSPACE_LEASE_EXPIRY_SECONDS from db.repositories.airlock_requests import AirlockRequestRepository from db.repositories.resource_templates import ResourceTemplateRepository from db.repositories.resources_history import ResourceHistoryRepository @@ -38,6 +38,27 @@ STORAGE_ENDPOINT = config.STORAGE_ENDPOINT_SUFFIX +_background_cleanup_tasks: set = set() + + +def _handle_background_cleanup_result(task: asyncio.Task) -> None: + _background_cleanup_tasks.discard(task) + try: + exc = task.exception() + if exc: + logger.error(f"Background cleanup task failed with exception: {exc}") + except asyncio.CancelledError: + logger.warning("Background cleanup task was cancelled") + except Exception: + logger.exception("Failed to handle background cleanup task result") + + +def run_background_cleanup_task(coro) -> asyncio.Task: + task = asyncio.create_task(coro) + _background_cleanup_tasks.add(task) + task.add_done_callback(_handle_background_cleanup_result) + return task + def get_account_by_request(airlock_request: AirlockRequest, workspace: Workspace) -> str: tre_id = config.TRE_ID @@ -155,7 +176,7 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - asyncio.create_task(delete_all_review_user_resources( + run_background_cleanup_task(delete_all_review_user_resources( airlock_request=airlock_request, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -436,7 +457,7 @@ def enrich_requests_with_allowed_actions(requests: List[AirlockRequest], user: U async def wait_for_successful_operation( operations_repo: OperationRepository, operation_id: str, - timeout: float = 300.0, + timeout: float = WORKSPACE_LEASE_EXPIRY_SECONDS, poll_interval: float = 0.5, ) -> Optional[Operation]: if not hasattr(operations_repo, "get_operation_by_id") or not isinstance(operation_id, str): @@ -475,7 +496,7 @@ async def wait_for_successful_operation( async def wait_for_terminal_operation( operations_repo: OperationRepository, operation_id: str, - timeout: float = 300.0, + timeout: float = WORKSPACE_LEASE_EXPIRY_SECONDS, poll_interval: float = 0.5, ) -> Optional[Operation]: return await wait_for_successful_operation(operations_repo, operation_id, timeout=timeout, poll_interval=poll_interval) @@ -537,24 +558,41 @@ async def delete_all_review_user_resources( user: User) -> List[Operation]: operations: List[Operation] = [] for review_ur in airlock_request.reviewUserResources.values(): - user_resource = await user_resource_repo.get_user_resource_by_id( - workspace_id=review_ur.workspaceId, - service_id=review_ur.workspaceServiceId, - resource_id=review_ur.userResourceId - ) - - operation = await delete_review_user_resource( - user_resource=user_resource, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operations_repo, - resource_history_repo=resource_history_repo, - user=user - ) - operations.append(operation) - if operation and hasattr(operation, "id"): - await wait_for_successful_operation(operations_repo, operation.id) + max_attempts = 3 + for attempt in range(max_attempts): + try: + user_resource = await user_resource_repo.get_user_resource_by_id( + workspace_id=review_ur.workspaceId, + service_id=review_ur.workspaceServiceId, + resource_id=review_ur.userResourceId + ) + + operation = await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operations_repo, + resource_history_repo=resource_history_repo, + user=user + ) + operations.append(operation) + if operation and hasattr(operation, "id"): + await wait_for_successful_operation(operations_repo, operation.id) + break + except EntityDoesNotExist: + logger.info(f"Review user resource {review_ur.userResourceId} already deleted or does not exist") + break + except Exception as ex: + if attempt == max_attempts - 1: + logger.exception( + f"Failed to delete review user resource {review_ur.userResourceId} after {max_attempts} attempts: {ex}" + ) + raise + logger.warning( + f"Retrying deletion for review user resource {review_ur.userResourceId} (attempt {attempt + 1}/{max_attempts}): {ex}" + ) + await asyncio.sleep(1.0) logger.info(f"Started {len(operations)} operations on deleting user resources") return operations @@ -564,7 +602,7 @@ async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - asyncio.create_task(delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user)) + run_background_cleanup_task(delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user)) await asyncio.sleep(0) return updated_request diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index bb1610cb35..28c6c8526b 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -379,3 +379,39 @@ async def test_send_resource_request_message_retains_lease_when_concurrently_mod operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) operations_repo_mock.release_workspace_lease.assert_not_called() + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_does_not_release_lease_in_fallback_when_operation_id_provided( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock() + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + operation_id="custom-op-id", + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) + operations_repo_mock.release_workspace_lease.assert_not_called() diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 33ca1a1f4f..dd417a7cb7 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -1,3 +1,4 @@ +import asyncio from fastapi import HTTPException, status import pytest import pytest_asyncio @@ -826,3 +827,56 @@ async def test_wait_for_successful_operation_retries_on_entity_does_not_exist(): with pytest.raises(HTTPException) as exc_info: await wait_for_successful_operation(repo, "op-123", timeout=0.1, poll_interval=0.02) assert exc_info.value.status_code == 504 + + +@pytest.mark.asyncio +async def test_run_background_cleanup_task_tracks_and_cleans_up(): + from services.airlock import run_background_cleanup_task, _background_cleanup_tasks + + async def sample_coro(): + await asyncio.sleep(0.01) + return "done" + + task = run_background_cleanup_task(sample_coro()) + assert task in _background_cleanup_tasks + await task + assert task not in _background_cleanup_tasks + + +@pytest.mark.asyncio +@patch("services.airlock.delete_review_user_resource") +async def test_delete_all_review_user_resources_retries_on_failure(mock_delete_review): + from services.airlock import delete_all_review_user_resources + from models.domain.operation import Status + + airlock_request = sample_airlock_request() + + user_resource_repo = AsyncMock() + # First attempt fails with transient exception, second attempt succeeds + user_resource_repo.get_user_resource_by_id.side_effect = [ + Exception("Cosmos transient error"), + AsyncMock(), + ] + + mock_op = MagicMock() + mock_op.id = "op-1" + mock_delete_review.return_value = mock_op + + operations_repo = AsyncMock() + op_record = MagicMock() + op_record.status = Status.Deleted + operations_repo.get_operation_by_id.return_value = op_record + + with patch("asyncio.sleep", new_callable=AsyncMock): + ops = await delete_all_review_user_resources( + airlock_request=airlock_request, + user_resource_repo=user_resource_repo, + workspace_service_repo=AsyncMock(), + resource_template_repo=AsyncMock(), + operations_repo=operations_repo, + resource_history_repo=AsyncMock(), + user=create_test_user(), + ) + + assert len(ops) == 1 + assert user_resource_repo.get_user_resource_by_id.call_count == 2 From 9acf41aca9c42daa5160db0d521e314b6aeb61a4 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:39:07 +0000 Subject: [PATCH 65/90] Retain lease on ambiguous dispatch, await airlock cleanup, and retry lease contention with backoff --- api_app/api/routes/resource_helpers.py | 11 +-- api_app/api/routes/workspaces.py | 81 ++++++++++++------- .../service_bus/resource_request_sender.py | 17 ++-- api_app/services/airlock.py | 62 ++++++-------- .../test_resource_request_sender.py | 7 +- .../tests_ma/test_services/test_airlock.py | 34 +++----- 6 files changed, 109 insertions(+), 103 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index fb72ebfd57..2506e84ff1 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -100,13 +100,14 @@ async def save_and_deploy_resource( except HTTPException: await resource_repo.delete_item(resource.id) raise - except Exception: - await resource_repo.delete_item(resource.id) + except Exception as ex: + if not getattr(ex, "lease_retained", False): + await resource_repo.delete_item(resource.id) logger.exception("Failed send resource request message") raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, - ) + ) from ex def mask_sensitive_properties( @@ -326,8 +327,8 @@ async def update_user_resource( operation_id=operation_id) return operation - except Exception: - if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + except Exception as ex: + if not getattr(ex, "lease_retained", False) and target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(target_workspace_id, operation_id) raise diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 07582b4a27..33733f954a 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -43,6 +43,28 @@ user_resources_workspace_router = APIRouter(dependencies=[Depends(require_workspace_owner_or_researcher_or_airlock_manager)]) +async def _rollback_address_space(workspace_repo, workspace_id, allocated_space, resource_template_repo, resource_history_repo, user): + max_rollback_attempts = 3 + for attempt in range(max_rollback_attempts): + try: + latest_workspace = await workspace_repo.get_workspace_by_id(workspace_id) + current_spaces = latest_workspace.properties.get("address_spaces", []) + if allocated_space in current_spaces: + updated_spaces = [s for s in current_spaces if s != allocated_space] + rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) + await workspace_repo.patch_workspace( + latest_workspace, rollback_patch, latest_workspace.etag, + resource_template_repo, resource_history_repo, user, False + ) + break + except CosmosAccessConditionFailedError: + if attempt == max_rollback_attempts - 1: + logger.exception("Failed to rollback allocated address space due to repeated ETag conflicts") + except Exception: + logger.exception("Failed to rollback allocated address space on workspace") + break + + def validate_user_has_valid_role_for_user_resource(user, user_resource): if "WorkspaceOwner" in user.roles: return @@ -166,7 +188,12 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except Exception: + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) raise @@ -312,30 +339,20 @@ async def create_workspace_service(response: Response, workspace_service_input: user=user, resource_template=resource_template, operation_id=operation_id) - except Exception: - if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): - max_rollback_attempts = 3 - for attempt in range(max_rollback_attempts): - try: - latest_workspace = await workspace_repo.get_workspace_by_id(workspace.id) - current_spaces = latest_workspace.properties.get("address_spaces", []) - allocated_space = workspace_service.properties["address_space"] - if allocated_space in current_spaces: - updated_spaces = [s for s in current_spaces if s != allocated_space] - rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) - await workspace_repo.patch_workspace( - latest_workspace, rollback_patch, latest_workspace.etag, - resource_template_repo, resource_history_repo, user, False - ) - break - except CosmosAccessConditionFailedError: - if attempt == max_rollback_attempts - 1: - logger.exception("Failed to rollback allocated address space due to repeated ETag conflicts") - except Exception: - logger.exception("Failed to rollback allocated address space on workspace") - break - if hasattr(operations_repo, "release_workspace_lease"): - await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception as err: + if not getattr(err, "lease_retained", False): + if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): + await _rollback_address_space( + workspace_repo, workspace.id, workspace_service.properties["address_space"], + resource_template_repo, resource_history_repo, user + ) + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) raise response.headers["Location"] = construct_location_header(operation) @@ -379,7 +396,12 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except Exception: + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise @@ -540,7 +562,12 @@ async def create_user_resource( response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) - except Exception: + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) raise diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 7191825f67..9c0d8b6588 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -69,8 +69,11 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope # create + send the message content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) - except Exception: + except Exception as ex: logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") + lease_released = False + lease_retained = False + should_release_lease = operation_id is None try: operation.status = get_failure_status_for_action(action) operation.message = f"Failed to dispatch initial deployment message: {action}" @@ -78,15 +81,12 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope first_step = operation.steps[0] first_step.status = get_failure_status_for_action(first_step.resourceAction) first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" - # If the caller provided operation_id, they manage the operation lifecycle and outer compensation, - # so keep the lease held. Otherwise, this function created and owns the operation, so release the - # workspace lease when persisting terminal failure. - should_release_lease = operation_id is None await operations_repo.update_item(operation, release_lease=should_release_lease) + if should_release_lease: + lease_released = True except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") # Fallback: remove orphaned operation and release its workspace lease only if it has not been advanced - lease_released = False try: creation_etag = getattr(operation, "etag", None) if hasattr(operations_repo, "delete_item"): @@ -105,6 +105,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope logger.warning( f"Operation {operation.id} was modified concurrently; retaining operation and workspace lease" ) + lease_retained = True except Exception: logger.exception(f"Failed to delete orphaned operation {operation.id}") @@ -122,6 +123,8 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope await rel_call except Exception: logger.exception(f"Failed to release workspace lease for {target_workspace_id}") - raise + + setattr(ex, "lease_retained", lease_retained) + raise ex return operation diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 125f08cf56..d15f3bdcda 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -38,27 +38,6 @@ STORAGE_ENDPOINT = config.STORAGE_ENDPOINT_SUFFIX -_background_cleanup_tasks: set = set() - - -def _handle_background_cleanup_result(task: asyncio.Task) -> None: - _background_cleanup_tasks.discard(task) - try: - exc = task.exception() - if exc: - logger.error(f"Background cleanup task failed with exception: {exc}") - except asyncio.CancelledError: - logger.warning("Background cleanup task was cancelled") - except Exception: - logger.exception("Failed to handle background cleanup task result") - - -def run_background_cleanup_task(coro) -> asyncio.Task: - task = asyncio.create_task(coro) - _background_cleanup_tasks.add(task) - task.add_done_callback(_handle_background_cleanup_result) - return task - def get_account_by_request(airlock_request: AirlockRequest, workspace: Workspace) -> str: tre_id = config.TRE_ID @@ -176,7 +155,7 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - run_background_cleanup_task(delete_all_review_user_resources( + await delete_all_review_user_resources( airlock_request=airlock_request, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -184,8 +163,7 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai operations_repo=operation_repo, resource_history_repo=resource_history_repo, user=user - )) - await asyncio.sleep(0) + ) return updated_airlock_request @@ -558,8 +536,10 @@ async def delete_all_review_user_resources( user: User) -> List[Operation]: operations: List[Operation] = [] for review_ur in airlock_request.reviewUserResources.values(): - max_attempts = 3 - for attempt in range(max_attempts): + start_time = time.time() + max_lease_wait = WORKSPACE_LEASE_EXPIRY_SECONDS + poll_interval = 2.0 + while True: try: user_resource = await user_resource_repo.get_user_resource_by_id( workspace_id=review_ur.workspaceId, @@ -583,16 +563,25 @@ async def delete_all_review_user_resources( except EntityDoesNotExist: logger.info(f"Review user resource {review_ur.userResourceId} already deleted or does not exist") break - except Exception as ex: - if attempt == max_attempts - 1: - logger.exception( - f"Failed to delete review user resource {review_ur.userResourceId} after {max_attempts} attempts: {ex}" + except HTTPException as ex: + if ex.status_code == status.HTTP_409_CONFLICT: + elapsed = time.time() - start_time + if elapsed >= max_lease_wait: + logger.error( + f"Timed out waiting for workspace lease to delete review user resource {review_ur.userResourceId}" + ) + raise + logger.info( + f"Workspace lease contention for review user resource {review_ur.userResourceId}; " + f"waiting for active operation to complete (elapsed {elapsed:.1f}s)..." ) - raise - logger.warning( - f"Retrying deletion for review user resource {review_ur.userResourceId} (attempt {attempt + 1}/{max_attempts}): {ex}" - ) - await asyncio.sleep(1.0) + await asyncio.sleep(min(poll_interval, max_lease_wait - elapsed)) + poll_interval = min(poll_interval * 1.5, 30.0) + continue + raise + except Exception as ex: + logger.exception(f"Failed to delete review user resource {review_ur.userResourceId}: {ex}") + raise logger.info(f"Started {len(operations)} operations on deleting user resources") return operations @@ -602,8 +591,7 @@ async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - run_background_cleanup_task(delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user)) - await asyncio.sleep(0) + await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) return updated_request diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 28c6c8526b..bd935cd10e 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -383,7 +383,7 @@ async def test_send_resource_request_message_retains_lease_when_concurrently_mod @patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) @patch("service_bus.resource_request_sender.update_resource_for_step") -async def test_send_resource_request_message_does_not_release_lease_in_fallback_when_operation_id_provided( +async def test_send_resource_request_message_sets_lease_retained_on_concurrent_modification( update_resource_mock, send_deployment_mock, ): @@ -394,14 +394,14 @@ async def test_send_resource_request_message_does_not_release_lease_in_fallback_ operations_repo_mock = AsyncMock() operations_repo_mock.create_operation_item.return_value = operation operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) - operations_repo_mock.delete_item = AsyncMock() + operations_repo_mock.delete_item = AsyncMock(side_effect=CosmosAccessConditionFailedError) operations_repo_mock.release_workspace_lease = AsyncMock() resource_to_send_mock = MagicMock() resource_to_send_mock.get_resource_request_message_payload.return_value = {} update_resource_mock.return_value = resource_to_send_mock - with pytest.raises(Exception, match="Service Bus unavailable"): + with pytest.raises(Exception, match="Service Bus unavailable") as exc_info: await send_resource_request_message( resource=resource, operations_repo=operations_repo_mock, @@ -413,5 +413,6 @@ async def test_send_resource_request_message_does_not_release_lease_in_fallback_ operation_id="custom-op-id", ) + assert getattr(exc_info.value, "lease_retained", False) is True operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) operations_repo_mock.release_workspace_lease.assert_not_called() diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index dd417a7cb7..11e0cd26ec 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -1,4 +1,3 @@ -import asyncio from fastapi import HTTPException, status import pytest import pytest_asyncio @@ -829,45 +828,31 @@ async def test_wait_for_successful_operation_retries_on_entity_does_not_exist(): assert exc_info.value.status_code == 504 -@pytest.mark.asyncio -async def test_run_background_cleanup_task_tracks_and_cleans_up(): - from services.airlock import run_background_cleanup_task, _background_cleanup_tasks - - async def sample_coro(): - await asyncio.sleep(0.01) - return "done" - - task = run_background_cleanup_task(sample_coro()) - assert task in _background_cleanup_tasks - await task - assert task not in _background_cleanup_tasks - - @pytest.mark.asyncio @patch("services.airlock.delete_review_user_resource") -async def test_delete_all_review_user_resources_retries_on_failure(mock_delete_review): +async def test_delete_all_review_user_resources_retries_on_lease_contention(mock_delete_review): from services.airlock import delete_all_review_user_resources from models.domain.operation import Status airlock_request = sample_airlock_request() user_resource_repo = AsyncMock() - # First attempt fails with transient exception, second attempt succeeds - user_resource_repo.get_user_resource_by_id.side_effect = [ - Exception("Cosmos transient error"), - AsyncMock(), - ] + user_resource_repo.get_user_resource_by_id.return_value = AsyncMock() + # First attempt raises 409 Conflict (lease contention), second succeeds mock_op = MagicMock() mock_op.id = "op-1" - mock_delete_review.return_value = mock_op + mock_delete_review.side_effect = [ + HTTPException(status_code=409, detail="Workspace has an active operation"), + mock_op, + ] operations_repo = AsyncMock() op_record = MagicMock() op_record.status = Status.Deleted operations_repo.get_operation_by_id.return_value = op_record - with patch("asyncio.sleep", new_callable=AsyncMock): + with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: ops = await delete_all_review_user_resources( airlock_request=airlock_request, user_resource_repo=user_resource_repo, @@ -879,4 +864,5 @@ async def test_delete_all_review_user_resources_retries_on_failure(mock_delete_r ) assert len(ops) == 1 - assert user_resource_repo.get_user_resource_by_id.call_count == 2 + assert mock_delete_review.call_count == 2 + mock_sleep.assert_awaited() From 54440d6314158794087a4e089f272534aee41b36 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:46:52 +0000 Subject: [PATCH 66/90] Propagate rollback errors, preserve lease_retained on HTTPException, and return early on airlock uninstall --- api_app/api/routes/resource_helpers.py | 9 ++++++--- api_app/api/routes/workspaces.py | 5 +++-- api_app/services/airlock.py | 4 +--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 2506e84ff1..5ad6146d7d 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -101,13 +101,16 @@ async def save_and_deploy_resource( await resource_repo.delete_item(resource.id) raise except Exception as ex: - if not getattr(ex, "lease_retained", False): + lease_retained = getattr(ex, "lease_retained", False) + if not lease_retained: await resource_repo.delete_item(resource.id) logger.exception("Failed send resource request message") - raise HTTPException( + http_ex = HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, - ) from ex + ) + setattr(http_ex, "lease_retained", lease_retained) + raise http_ex from ex def mask_sensitive_properties( diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 33733f954a..cbb5e9e52b 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -56,13 +56,14 @@ async def _rollback_address_space(workspace_repo, workspace_id, allocated_space, latest_workspace, rollback_patch, latest_workspace.etag, resource_template_repo, resource_history_repo, user, False ) - break + return except CosmosAccessConditionFailedError: if attempt == max_rollback_attempts - 1: logger.exception("Failed to rollback allocated address space due to repeated ETag conflicts") + raise except Exception: logger.exception("Failed to rollback allocated address space on workspace") - break + raise def validate_user_has_valid_role_for_user_resource(user, user_resource): diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index d15f3bdcda..ef741917bd 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -494,7 +494,7 @@ async def delete_review_user_resource( # disable might contain logic that we need to execute before the deletion of the resource disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) if disable_op and hasattr(disable_op, "id"): - await wait_for_successful_operation(operations_repo, disable_op.id) + await wait_for_successful_operation(operations_repo, disable_op.id, timeout=30.0) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -557,8 +557,6 @@ async def delete_all_review_user_resources( user=user ) operations.append(operation) - if operation and hasattr(operation, "id"): - await wait_for_successful_operation(operations_repo, operation.id) break except EntityDoesNotExist: logger.info(f"Review user resource {review_ur.userResourceId} already deleted or does not exist") From a08268f4e6834d5ed205eb3db17e5f37207de11c Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:01:23 +0000 Subject: [PATCH 67/90] Address Copilot feedback: try-finally lease release, reconciled operation guard, and lease retention --- api_app/api/routes/workspaces.py | 16 ++++---- api_app/db/repositories/operations.py | 14 +++++++ api_app/models/domain/operation.py | 1 + .../service_bus/deployment_status_updater.py | 17 +++++++-- .../service_bus/resource_request_sender.py | 3 ++ api_app/services/airlock.py | 4 +- .../test_operation_repository.py | 1 + .../test_deployment_status_update.py | 38 +++++++++++++++++++ 8 files changed, 81 insertions(+), 13 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index cbb5e9e52b..cfa69e616c 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -342,13 +342,15 @@ async def create_workspace_service(response: Response, workspace_service_input: operation_id=operation_id) except Exception as err: if not getattr(err, "lease_retained", False): - if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): - await _rollback_address_space( - workspace_repo, workspace.id, workspace_service.properties["address_space"], - resource_template_repo, resource_history_repo, user - ) - if hasattr(operations_repo, "release_workspace_lease"): - await operations_repo.release_workspace_lease(workspace.id, operation_id) + try: + if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): + await _rollback_address_space( + workspace_repo, workspace.id, workspace_service.properties["address_space"], + resource_template_repo, resource_history_repo, user + ) + finally: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) if getattr(err, "lease_retained", False): raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 31fa87afdf..470b94fd88 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -168,6 +168,13 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> existing_op.status = get_failure_status_for_action(existing_op.action) existing_op.message = "Operation timed out or was interrupted before completion" existing_op.updatedWhen = timestamp + existing_op.reconciled = True + if getattr(existing_op, "steps", None): + for step in existing_op.steps: + if not step.is_failure() and not step.is_success(): + step.status = get_failure_status_for_action(existing_op.action) + step.message = "Operation timed out or was interrupted before completion" + step.updatedWhen = timestamp update_call = self.update_item(existing_op) if hasattr(update_call, "__await__"): await update_call @@ -458,6 +465,13 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: op.status = get_failure_status_for_action(op.action) op.message = "Operation timed out or was interrupted before completion" op.updatedWhen = timestamp + op.reconciled = True + if getattr(op, "steps", None): + for step in op.steps: + if not step.is_failure() and not step.is_success(): + step.status = get_failure_status_for_action(op.action) + step.message = "Operation timed out or was interrupted before completion" + step.updatedWhen = timestamp update_call = self.update_item(op) if hasattr(update_call, "__await__"): await update_call diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 3e48f51554..4f2e290e5b 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -104,6 +104,7 @@ class Operation(AzureTREModel): updatedWhen: float = Field(0.0, title="POSIX Timestamp for When the operation was updated") user: dict = Field(default_factory=dict) steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") + reconciled: Optional[bool] = Field(default=False, title="reconciled", description="Whether operation was reconciled due to timeout or interruption") etag: Optional[str] = Field(None, title="_etag", alias="_etag") # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 67fbef08a4..680e01e9c8 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -160,6 +160,14 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") + # Reject messages for reconciled/timed-out operations to prevent resurrecting stale operations + if getattr(operation, "reconciled", False): + logger.warning( + f"Message {message.stepId} received for operation {operation.id}, " + f"but operation was reconciled due to timeout/interruption. Rejecting to prevent stale writes." + ) + return True + # Ignore duplicate messages if uninstall cleanup is already terminal if ( operation.action == RequestAction.UnInstall @@ -327,7 +335,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if ( fresh_op is not operation and ( - (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) or fresh_op.status in (Status.Deleted, Status.DeletingFailed) ) ): @@ -389,7 +398,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if ( fresh_op is not operation and ( - (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) or fresh_op.status in (Status.Deleted, Status.DeletingFailed) ) ): @@ -577,7 +587,8 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if ( fresh_op is not operation and ( - (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) or fresh_op.status in (Status.Deleted, Status.DeletingFailed) ) ): diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 9c0d8b6588..6592449c78 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -87,6 +87,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope except Exception: logger.exception(f"Failed to persist failure state for operation {operation.id}") # Fallback: remove orphaned operation and release its workspace lease only if it has not been advanced + lease_retained = True try: creation_etag = getattr(operation, "etag", None) if hasattr(operations_repo, "delete_item"): @@ -101,6 +102,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope if hasattr(del_call, "__await__"): await del_call lease_released = True + lease_retained = False except (CosmosAccessConditionFailedError, ResourceModifiedError, ResourceExistsError): logger.warning( f"Operation {operation.id} was modified concurrently; retaining operation and workspace lease" @@ -108,6 +110,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope lease_retained = True except Exception: logger.exception(f"Failed to delete orphaned operation {operation.id}") + lease_retained = True if lease_released and should_release_lease: target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index ef741917bd..798bcb4b69 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -461,8 +461,6 @@ async def wait_for_successful_operation( raise except EntityDoesNotExist: pass - except Exception: - pass await asyncio.sleep(poll_interval) logger.error(f"Timed out waiting for operation {operation_id} to complete successfully") raise HTTPException( @@ -494,7 +492,7 @@ async def delete_review_user_resource( # disable might contain logic that we need to execute before the deletion of the resource disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) if disable_op and hasattr(disable_op, "id"): - await wait_for_successful_operation(operations_repo, disable_op.id, timeout=30.0) + await wait_for_successful_operation(operations_repo, disable_op.id) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index f489e00e39..9eada81dff 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -421,6 +421,7 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio operations_repo.update_item.assert_awaited_once() saved_op = operations_repo.update_item.call_args[0][0] assert saved_op.status == Status.DeploymentFailed + assert saved_op.reconciled is True async def test_resource_has_active_operation_fails_closed_when_reconciliation_fails(operations_repo): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index a76aceb045..5184041f24 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -3670,3 +3670,41 @@ async def mock_update_op(op): send_deployment_message_mock.assert_called_once() assert fresh_terminal_op.steps[1].status == Status.Updated assert fresh_terminal_op.status == Status.Deleted + + +async def test_update_status_in_database_rejects_reconciled_operation(): + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + res_id = uuid.uuid4() + reconciled_op = Operation( + id=OPERATION_ID, + resourceId=str(res_id), + resourcePath="/workspaces/ws-1", + action=RequestAction.Install, + status=Status.DeploymentFailed, + reconciled=True, + steps=[ + OperationStep( + id="step-1", + templateStepId="main", + resourceId=str(res_id), + status=Status.DeploymentFailed, + ) + ], + ) + status_updater.operations_repo.get_operation_by_id = AsyncMock(return_value=reconciled_op) + status_updater.resource_repo = AsyncMock() + + msg = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=res_id, + status=Status.Deployed, + message="late success", + ) + + result = await status_updater.update_status_in_database(msg) + assert result is True + status_updater.operations_repo.update_item.assert_not_called() + status_updater.resource_repo.update_item.assert_not_called() From 15f67879fdfabfbb9ae0e34c001df44557feb689 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:10:53 +0000 Subject: [PATCH 68/90] Schedule review user resource deletion via BackgroundTasks to avoid HTTP timeout --- api_app/api/routes/airlock.py | 10 ++++--- api_app/services/airlock.py | 50 ++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/api_app/api/routes/airlock.py b/api_app/api/routes/airlock.py index dbf96cc117..fd7cb93348 100644 --- a/api_app/api/routes/airlock.py +++ b/api_app/api/routes/airlock.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, status as status_code, Response +from fastapi import APIRouter, Depends, HTTPException, status as status_code, Response, BackgroundTasks from jsonschema.exceptions import ValidationError from api.helpers import get_repository @@ -99,7 +99,8 @@ async def create_submit_request(airlock_request=Depends(get_airlock_request_by_i @airlock_workspace_router.post("/workspaces/{workspace_id}/requests/{airlock_request_id}/cancel", status_code=status_code.HTTP_200_OK, response_model=AirlockRequestWithAllowedUserActions, name=strings.API_CANCEL_AIRLOCK_REQUEST, dependencies=[Depends(require_workspace_owner_or_researcher), Depends(get_workspace_by_id_from_path)]) -async def create_cancel_request(airlock_request=Depends(get_airlock_request_by_id_from_path), +async def create_cancel_request(background_tasks: BackgroundTasks, + airlock_request=Depends(get_airlock_request_by_id_from_path), user=Depends(require_workspace_owner_or_researcher), workspace=Depends(get_workspace_by_id_from_path), airlock_request_repo=Depends(get_repository(AirlockRequestRepository)), @@ -108,7 +109,7 @@ async def create_cancel_request(airlock_request=Depends(get_airlock_request_by_i resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), operation_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)),) -> AirlockRequestWithAllowedUserActions: - updated_request = await cancel_request(airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, resource_template_repo, operation_repo, resource_history_repo) + updated_request = await cancel_request(airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, resource_template_repo, operation_repo, resource_history_repo, background_tasks=background_tasks) allowed_actions = get_allowed_actions(updated_request, user, airlock_request_repo) return AirlockRequestWithAllowedUserActions(airlockRequest=updated_request, allowedUserActions=allowed_actions) @@ -170,6 +171,7 @@ async def create_review_user_resource( Depends(get_workspace_by_id_from_path)]) async def create_airlock_review( airlock_review_input: AirlockReviewInCreate, + background_tasks: BackgroundTasks, airlock_request=Depends(get_airlock_request_by_id_from_path), user=Depends(require_airlock_manager), workspace=Depends(get_deployed_workspace_by_id_from_path), @@ -180,7 +182,7 @@ async def create_airlock_review( resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> AirlockRequestWithAllowedUserActions: try: - updated_airlock_request = await review_airlock_request(airlock_review_input, airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) + updated_airlock_request = await review_airlock_request(airlock_review_input, airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo, background_tasks=background_tasks) allowed_actions = get_allowed_actions(updated_airlock_request, user, airlock_request_repo) return AirlockRequestWithAllowedUserActions(airlockRequest=updated_airlock_request, allowedUserActions=allowed_actions) except (ValidationError, ValueError) as e: diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 798bcb4b69..a373a14826 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -139,7 +139,8 @@ def get_account_url(account_name: str) -> str: async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, airlock_request: AirlockRequest, user: User, workspace: Workspace, airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo, operation_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, - resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: + resource_history_repo: ResourceHistoryRepository, + background_tasks=None) -> AirlockRequest: airlock_review = airlock_request_repo.create_airlock_review_item(airlock_review_input, user) # Store review with new status in cosmos, and send status_changed event if airlock_review.reviewDecision.value == AirlockReviewDecision.Approved: @@ -155,15 +156,27 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - await delete_all_review_user_resources( - airlock_request=airlock_request, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operation_repo, - resource_history_repo=resource_history_repo, - user=user - ) + if background_tasks is not None: + background_tasks.add_task( + delete_all_review_user_resources, + airlock_request=airlock_request, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operation_repo, + resource_history_repo=resource_history_repo, + user=user + ) + else: + await delete_all_review_user_resources( + airlock_request=airlock_request, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operation_repo, + resource_history_repo=resource_history_repo, + user=user + ) return updated_airlock_request @@ -585,9 +598,22 @@ async def delete_all_review_user_resources( async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: Workspace, airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: + resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, + background_tasks=None) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) + if background_tasks is not None: + background_tasks.add_task( + delete_all_review_user_resources, + airlock_request, + user_resource_repo, + workspace_service_repo, + resource_template_repo, + operations_repo, + resource_history_repo, + user + ) + else: + await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) return updated_request From ecc79b5d45cf1dff527c2f609ad5193b433d8d7b Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:22:10 +0000 Subject: [PATCH 69/90] Use step resourceAction for stale step status and await uninstall in VM replacement --- api_app/db/repositories/operations.py | 4 +- api_app/services/airlock.py | 4 +- .../test_api/test_routes/test_airlock.py | 4 +- .../test_operation_repository.py | 84 ++++++++++++++++++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 470b94fd88..c73c34e6ef 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -172,7 +172,7 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> if getattr(existing_op, "steps", None): for step in existing_op.steps: if not step.is_failure() and not step.is_success(): - step.status = get_failure_status_for_action(existing_op.action) + step.status = get_failure_status_for_action(step.resourceAction or existing_op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp update_call = self.update_item(existing_op) @@ -469,7 +469,7 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: if getattr(op, "steps", None): for step in op.steps: if not step.is_failure() and not step.is_success(): - step.status = get_failure_status_for_action(op.action) + step.status = get_failure_status_for_action(step.resourceAction or op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp update_call = self.update_item(op) diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index a373a14826..4a2ce496cc 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -275,7 +275,7 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": logger.info("Deleting existing user resource...") - _ = await delete_review_user_resource( + delete_op = await delete_review_user_resource( user_resource=existing_resource, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, @@ -284,6 +284,8 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR resource_history_repo=resource_history_repo, user=user ) + if delete_op and hasattr(delete_op, "id"): + await wait_for_successful_operation(operation_repo, delete_op.id) async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest, airlock_request_repo: AirlockRequestRepository, user: User, workspace: Workspace): diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index 842aec038f..dc3c8da029 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -409,6 +409,7 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert response.status_code == status.HTTP_409_CONFLICT + @patch("services.airlock.wait_for_successful_operation") @patch("services.airlock.delete_review_user_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="uninstall", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.get_azure_resource_status", return_value={}) @@ -419,9 +420,10 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r @patch("services.airlock.UserResourceRepository.create_user_resource_item", return_value=(UserResource(id=USER_RESOURCE_ID, templateName="test", templateVersion="0.0.1", _etag="123"), "test")) @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id", return_value=sample_workspace(workspace_properties=sample_airlock_review_config())) - async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, app, client): + async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, wait_for_op_mock, app, client): await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert delete_review_resource_mock.call_count == 1 + assert wait_for_op_mock.call_count == 1 assert deploy_resource_mock.call_count == 1 # [POST] /workspaces/{workspace_id}/requests/{airlock_request_id}/revoke diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 9eada81dff..7d60bb55b6 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -4,7 +4,7 @@ import pytest from mock import patch from db.repositories.resource_templates import ResourceTemplateRepository -from models.domain.operation import Operation, Status +from models.domain.operation import Operation, OperationStep, Status from models.domain.request_action import RequestAction from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository @@ -484,3 +484,85 @@ async def test_build_step_list_tolerates_missing_step_title(operations_repo): assert len(steps) == 1 assert steps[0].stepTitle is None assert steps[0].templateStepId == "step-1" + + +async def test_reconcile_stale_operation_uses_step_resource_action(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) + + stale_time = operations_repo.get_timestamp() - WORKSPACE_LEASE_EXPIRY_SECONDS - 100 + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": stale_time, + }) + + step1 = OperationStep( + id="step-1", + templateStepId="main", + resourceAction="uninstall", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + templateStepId="cleanup", + resourceAction="upgrade", + status=Status.Updating, + ) + stale_op = Operation( + id="stale-op", + resourceId="ws-1", + resourcePath="ws/ws-1", + action="uninstall", + status=Status.Deleting, + createdWhen=stale_time, + updatedWhen=stale_time, + steps=[step1, step2], + ) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock(return_value=None) + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + + await operations_repo.acquire_workspace_lease("ws-1", "new-op") + + operations_repo.update_item.assert_awaited_once() + saved_op = operations_repo.update_item.call_args[0][0] + assert saved_op.status == Status.DeletingFailed + assert saved_op.steps[0].status == Status.DeletingFailed + assert saved_op.steps[1].status == Status.UpdatingFailed + + +async def test_resource_has_active_operation_reconciles_stale_step_with_step_resource_action(operations_repo): + from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS + + stale_time = operations_repo.get_timestamp() - WORKSPACE_LEASE_EXPIRY_SECONDS - 100 + res_id = str(uuid.uuid4()) + stale_op_dict = { + "id": "stale-op", + "resourceId": res_id, + "resourcePath": f"res/{res_id}", + "action": "uninstall", + "status": "deleting", + "createdWhen": stale_time, + "updatedWhen": stale_time, + "steps": [ + { + "id": "step-1", + "templateStepId": "cleanup", + "resourceAction": "upgrade", + "status": "updating", + } + ], + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock(return_value=None) + + has_active = await operations_repo.resource_has_active_operation(res_id) + assert has_active is False + operations_repo.update_item.assert_awaited_once() + saved_op = operations_repo.update_item.call_args[0][0] + assert saved_op.status == Status.DeletingFailed + assert saved_op.steps[0].status == Status.UpdatingFailed From ef5a1b6edd558e97a0c563829eaccb97c8809a96 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:45:00 +0100 Subject: [PATCH 70/90] Enhance error handling for CosmosAccessConditionFailedError Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- api_app/api/routes/workspaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index cfa69e616c..0e8a2d4270 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -177,8 +177,8 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) - except CosmosAccessConditionFailedError: - if hasattr(operations_repo, "release_workspace_lease"): + except CosmosAccessConditionFailedError as err: + if not getattr(err, "lease_retained", False) and hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: From f17e48d7248b2657e899dd14eb7fb67374381eb5 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 7 Sep 2026 13:42:17 +0000 Subject: [PATCH 71/90] Await airlock resource cleanup during review and cancellation --- api_app/api/routes/airlock.py | 10 +++---- api_app/services/airlock.py | 50 +++++++++-------------------------- 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/api_app/api/routes/airlock.py b/api_app/api/routes/airlock.py index fd7cb93348..dbf96cc117 100644 --- a/api_app/api/routes/airlock.py +++ b/api_app/api/routes/airlock.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, status as status_code, Response, BackgroundTasks +from fastapi import APIRouter, Depends, HTTPException, status as status_code, Response from jsonschema.exceptions import ValidationError from api.helpers import get_repository @@ -99,8 +99,7 @@ async def create_submit_request(airlock_request=Depends(get_airlock_request_by_i @airlock_workspace_router.post("/workspaces/{workspace_id}/requests/{airlock_request_id}/cancel", status_code=status_code.HTTP_200_OK, response_model=AirlockRequestWithAllowedUserActions, name=strings.API_CANCEL_AIRLOCK_REQUEST, dependencies=[Depends(require_workspace_owner_or_researcher), Depends(get_workspace_by_id_from_path)]) -async def create_cancel_request(background_tasks: BackgroundTasks, - airlock_request=Depends(get_airlock_request_by_id_from_path), +async def create_cancel_request(airlock_request=Depends(get_airlock_request_by_id_from_path), user=Depends(require_workspace_owner_or_researcher), workspace=Depends(get_workspace_by_id_from_path), airlock_request_repo=Depends(get_repository(AirlockRequestRepository)), @@ -109,7 +108,7 @@ async def create_cancel_request(background_tasks: BackgroundTasks, resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), operation_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)),) -> AirlockRequestWithAllowedUserActions: - updated_request = await cancel_request(airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, resource_template_repo, operation_repo, resource_history_repo, background_tasks=background_tasks) + updated_request = await cancel_request(airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, resource_template_repo, operation_repo, resource_history_repo) allowed_actions = get_allowed_actions(updated_request, user, airlock_request_repo) return AirlockRequestWithAllowedUserActions(airlockRequest=updated_request, allowedUserActions=allowed_actions) @@ -171,7 +170,6 @@ async def create_review_user_resource( Depends(get_workspace_by_id_from_path)]) async def create_airlock_review( airlock_review_input: AirlockReviewInCreate, - background_tasks: BackgroundTasks, airlock_request=Depends(get_airlock_request_by_id_from_path), user=Depends(require_airlock_manager), workspace=Depends(get_deployed_workspace_by_id_from_path), @@ -182,7 +180,7 @@ async def create_airlock_review( resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> AirlockRequestWithAllowedUserActions: try: - updated_airlock_request = await review_airlock_request(airlock_review_input, airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo, background_tasks=background_tasks) + updated_airlock_request = await review_airlock_request(airlock_review_input, airlock_request, user, workspace, airlock_request_repo, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) allowed_actions = get_allowed_actions(updated_airlock_request, user, airlock_request_repo) return AirlockRequestWithAllowedUserActions(airlockRequest=updated_airlock_request, allowedUserActions=allowed_actions) except (ValidationError, ValueError) as e: diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 4a2ce496cc..681b1d4011 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -139,8 +139,7 @@ def get_account_url(account_name: str) -> str: async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, airlock_request: AirlockRequest, user: User, workspace: Workspace, airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo, operation_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, - resource_history_repo: ResourceHistoryRepository, - background_tasks=None) -> AirlockRequest: + resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: airlock_review = airlock_request_repo.create_airlock_review_item(airlock_review_input, user) # Store review with new status in cosmos, and send status_changed event if airlock_review.reviewDecision.value == AirlockReviewDecision.Approved: @@ -156,27 +155,15 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - if background_tasks is not None: - background_tasks.add_task( - delete_all_review_user_resources, - airlock_request=airlock_request, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operation_repo, - resource_history_repo=resource_history_repo, - user=user - ) - else: - await delete_all_review_user_resources( - airlock_request=airlock_request, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operation_repo, - resource_history_repo=resource_history_repo, - user=user - ) + await delete_all_review_user_resources( + airlock_request=airlock_request, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operation_repo, + resource_history_repo=resource_history_repo, + user=user + ) return updated_airlock_request @@ -600,22 +587,9 @@ async def delete_all_review_user_resources( async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: Workspace, airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, - background_tasks=None) -> AirlockRequest: + resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - if background_tasks is not None: - background_tasks.add_task( - delete_all_review_user_resources, - airlock_request, - user_resource_repo, - workspace_service_repo, - resource_template_repo, - operations_repo, - resource_history_repo, - user - ) - else: - await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) + await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) return updated_request From 35305536de9d246553fdbd5c85d66c95f4aa95b8 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Mon, 7 Sep 2026 16:49:59 +0000 Subject: [PATCH 72/90] Improve lease retention, stale operation reconciliation, and review resource cleanup --- api_app/api/routes/resource_helpers.py | 5 +- api_app/api/routes/workspaces.py | 4 +- api_app/db/repositories/operations.py | 18 +++++++ api_app/services/airlock.py | 50 ++++++++++++++----- .../test_operation_repository.py | 4 ++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 5ad6146d7d..48456a0056 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -97,8 +97,9 @@ async def save_and_deploy_resource( **kwargs, ) return operation - except HTTPException: - await resource_repo.delete_item(resource.id) + except HTTPException as ex: + if not getattr(ex, "lease_retained", False): + await resource_repo.delete_item(resource.id) raise except Exception as ex: lease_retained = getattr(ex, "lease_retained", False) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 0e8a2d4270..620c9b08af 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -387,8 +387,8 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo operation_id=operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) - except CosmosAccessConditionFailedError: - if hasattr(operations_repo, "release_workspace_lease"): + except CosmosAccessConditionFailedError as err: + if not getattr(err, "lease_retained", False) and hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index c73c34e6ef..8b2f3356eb 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -105,6 +105,18 @@ async def update_item(self, item: Operation, etag: Optional[str] = None, release return item + async def _reconcile_resource_status(self, resource_id: str, status: Status, message: str): + resource_repo = await ResourceRepository.create() + resource = await resource_repo.get_resource_by_id(resource_id) + if resource.deploymentStatus in { + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + }: + resource.deploymentStatus = status + resource.updatedWhen = self.get_timestamp() + await resource_repo.update_item(resource) + async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> bool: if not hasattr(self, "_container") or self._container is None: return True @@ -175,6 +187,11 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> step.status = get_failure_status_for_action(step.resourceAction or existing_op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp + await self._reconcile_resource_status( + existing_op.resourceId, + existing_op.status, + existing_op.message, + ) update_call = self.update_item(existing_op) if hasattr(update_call, "__await__"): await update_call @@ -472,6 +489,7 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: step.status = get_failure_status_for_action(step.resourceAction or op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp + await self._reconcile_resource_status(op.resourceId, op.status, op.message) update_call = self.update_item(op) if hasattr(update_call, "__await__"): await update_call diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 681b1d4011..d654b2dbf9 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -176,7 +176,8 @@ def get_airlock_container_link(airlock_request: AirlockRequest, user, workspace) async def create_review_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository) -> Tuple[UserResource, Operation]: + operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + ) -> Tuple[AirlockRequest, Operation]: if airlock_request.type == AirlockRequestType.Import: config = workspace.properties["airlock_review_config"]["import"] review_workspace_id = config["import_vm_workspace_id"] @@ -248,7 +249,8 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor async def _handle_existing_review_resource(existing_resource: AirlockReviewUserResource, user: User, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - operation_repo: OperationRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository): + operation_repo: OperationRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + wait_for_completion: bool = True) -> Optional[Operation]: # Is the existing resource enabled, deployed, and can we get its power state information if existing_resource.isEnabled and existing_resource.deploymentStatus == "deployed" and 'azure_resource_id' in existing_resource.properties: resource_status = get_azure_resource_status(existing_resource.properties['azure_resource_id']) @@ -261,18 +263,42 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR # If it wasn't healthy or running, we'll delete the existing resource if not already deleted, and then create a new one logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": - logger.info("Deleting existing user resource...") - delete_op = await delete_review_user_resource( - user_resource=existing_resource, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, + if wait_for_completion: + logger.info("Deleting existing user resource...") + operation = await delete_review_user_resource( + user_resource=existing_resource, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operation_repo, + resource_history_repo=resource_history_repo, + user=user) + if operation and hasattr(operation, "id"): + await wait_for_successful_operation(operation_repo, operation.id) + return operation + + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace_id=existing_resource.workspaceId, + service_id=existing_resource.parentWorkspaceServiceId) + disable_op = await disable_user_resource( + existing_resource, user, workspace_service, user_resource_repo, + resource_template_repo, operation_repo, resource_history_repo) + if disable_op and hasattr(disable_op, "id"): + if not wait_for_completion: + return disable_op + await wait_for_successful_operation(operation_repo, disable_op.id) + + logger.info(f"Deleting user resource {existing_resource.id} in workspace service {workspace_service.id}") + operation = await send_uninstall_message( + resource=existing_resource, + resource_repo=user_resource_repo, operations_repo=operation_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, - user=user - ) - if delete_op and hasattr(delete_op, "id"): - await wait_for_successful_operation(operation_repo, delete_op.id) + user=user) + logger.info(f"Started operation {operation}") + return operation async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest, airlock_request_repo: AirlockRequestRepository, user: User, workspace: Workspace): diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 7d60bb55b6..5bb413e15b 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -372,6 +372,7 @@ async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_rea stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 8000, createdWhen=now - 8000) operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock() + operations_repo._reconcile_resource_status = AsyncMock() res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") assert res is True @@ -415,6 +416,7 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio } operations_repo.query = AsyncMock(return_value=[stale_op_dict]) operations_repo.update_item = AsyncMock() + operations_repo._reconcile_resource_status = AsyncMock() result = await operations_repo.resource_has_active_operation(workspace_id) assert result is False @@ -524,6 +526,7 @@ async def test_reconcile_stale_operation_uses_step_resource_action(operations_re ) operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock(return_value=None) + operations_repo._reconcile_resource_status = AsyncMock() operations_repo.resource_has_active_operation = AsyncMock(return_value=False) await operations_repo.acquire_workspace_lease("ws-1", "new-op") @@ -559,6 +562,7 @@ async def test_resource_has_active_operation_reconciles_stale_step_with_step_res } operations_repo.query = AsyncMock(return_value=[stale_op_dict]) operations_repo.update_item = AsyncMock(return_value=None) + operations_repo._reconcile_resource_status = AsyncMock() has_active = await operations_repo.resource_has_active_operation(res_id) assert has_active is False From e205c2e959891fbdcdd097b90848402aa0f2a1fe Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 8 Sep 2026 13:13:53 +0000 Subject: [PATCH 73/90] Improve workspace lease handling and stale operation reconciliation --- api_app/db/repositories/operations.py | 32 ++++++++++++----- .../service_bus/resource_request_sender.py | 20 ++++++++--- .../test_operation_repository.py | 22 ++++++++++++ .../test_resource_request_sender.py | 35 +++++++++++++++++++ 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 8b2f3356eb..fe40c20c77 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -117,6 +117,11 @@ async def _reconcile_resource_status(self, resource_id: str, status: Status, mes resource.updatedWhen = self.get_timestamp() await resource_repo.update_item(resource) + async def _reconcile_operation_resources(self, operation: Operation, step_resources=None): + await self._reconcile_resource_status(operation.resourceId, operation.status, operation.message) + for resource_id, status, message in step_resources or []: + await self._reconcile_resource_status(resource_id, status, message) + async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> bool: if not hasattr(self, "_container") or self._container is None: return True @@ -137,6 +142,9 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> create_call = self.container.create_item(body=lease_body) if hasattr(create_call, "__await__"): await create_call + if await self.resource_has_active_operation(workspace_id, exclude_operation_id=operation_id): + await self.release_workspace_lease(workspace_id, operation_id) + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) return True except (CosmosResourceExistsError, ResourceExistsError): try: @@ -181,20 +189,19 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> existing_op.message = "Operation timed out or was interrupted before completion" existing_op.updatedWhen = timestamp existing_op.reconciled = True + affected_step_resources = [] if getattr(existing_op, "steps", None): for step in existing_op.steps: if not step.is_failure() and not step.is_success(): step.status = get_failure_status_for_action(step.resourceAction or existing_op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp - await self._reconcile_resource_status( - existing_op.resourceId, - existing_op.status, - existing_op.message, - ) - update_call = self.update_item(existing_op) + affected_step_resources.append((step.resourceId, step.status, step.message)) + update_call = self.update_item(existing_op, release_lease=False) if hasattr(update_call, "__await__"): await update_call + await self._reconcile_operation_resources(existing_op, affected_step_resources) + await self.release_workspace_lease(workspace_id, existing_op.id) except Exception as e: logger.exception(f"Failed to reconcile stale operation {existing_op.id}: {e}") raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) @@ -435,7 +442,7 @@ async def resource_has_deployed_operation(self, resource_id: str) -> bool: operations = await self.query(query=query) return len(operations) > 0 - async def resource_has_active_operation(self, resource_id: str) -> bool: + async def resource_has_active_operation(self, resource_id: str, exclude_operation_id: Optional[str] = None) -> bool: # Guard against injection; resource_id should always be a valid UUID string try: uuid.UUID(resource_id) @@ -459,6 +466,8 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: + f' OR CONTAINS(c.resourcePath, "{resource_id}"))' + f' AND c.status IN ({status_filter})' ) + if exclude_operation_id: + query += f' AND c.id != "{exclude_operation_id}"' operations = await self.query(query=query) if not operations: return False @@ -483,16 +492,21 @@ async def resource_has_active_operation(self, resource_id: str) -> bool: op.message = "Operation timed out or was interrupted before completion" op.updatedWhen = timestamp op.reconciled = True + affected_step_resources = [] if getattr(op, "steps", None): for step in op.steps: if not step.is_failure() and not step.is_success(): step.status = get_failure_status_for_action(step.resourceAction or op.action) step.message = "Operation timed out or was interrupted before completion" step.updatedWhen = timestamp - await self._reconcile_resource_status(op.resourceId, op.status, op.message) - update_call = self.update_item(op) + affected_step_resources.append((step.resourceId, step.status, step.message)) + update_call = self.update_item(op, release_lease=False) if hasattr(update_call, "__await__"): await update_call + await self._reconcile_operation_resources(op, affected_step_resources) + target_workspace_id = extract_workspace_id_from_resource_path(op.resourcePath) + if target_workspace_id: + await self.release_workspace_lease(target_workspace_id, op.id) except Exception as e: logger.exception(f"Failed to reconcile stale active operation {op_dict.get('id') if isinstance(op_dict, dict) else getattr(op_dict, 'id', None)}: {e}") has_active = True diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index 6592449c78..c0ea7b7482 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -29,13 +29,23 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope with tracer.start_as_current_span("send_resource_request_message") as current_span: current_span.set_attribute("resource_id", resource.id) current_span.set_attribute("action", action) + caller_operation_id = operation_id + operation_id = operation_id or OperationRepository.create_operation_id() + target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) # Construct the resources to build an operation item for resources_list = [] - if is_cascade: - resources_list = await resource_repo.get_resource_dependency_list(resource) - else: - resources_list.append(resource.__dict__) + try: + if target_workspace_id: + await operations_repo.acquire_workspace_lease(target_workspace_id, operation_id) + if is_cascade: + resources_list = await resource_repo.get_resource_dependency_list(resource) + else: + resources_list.append(resource.__dict__) + except Exception: + if target_workspace_id: + await operations_repo.release_workspace_lease(target_workspace_id, operation_id) + raise # add the operation to the db - this will create all the steps needed (if any are defined in the template) operation = await operations_repo.create_operation_item( @@ -73,7 +83,7 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") lease_released = False lease_retained = False - should_release_lease = operation_id is None + should_release_lease = caller_operation_id is None try: operation.status = get_failure_status_for_action(action) operation.message = f"Failed to dispatch initial deployment message: {action}" diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 5bb413e15b..9f1a796f72 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -226,6 +226,22 @@ async def test_acquire_workspace_lease_creates_item(operations_repo): assert body["operationId"] == "op-1" +async def test_acquire_workspace_lease_rejects_legacy_active_operation_after_create(operations_repo): + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + operations_repo.resource_has_active_operation = AsyncMock(return_value=True) + operations_repo.release_workspace_lease = AsyncMock() + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + + assert exc.value.status_code == 409 + operations_repo.resource_has_active_operation.assert_awaited_once_with("ws-1", exclude_operation_id="op-1") + operations_repo.release_workspace_lease.assert_awaited_once_with("ws-1", "op-1") + + async def test_acquire_workspace_lease_raises_409_if_active_operation_exists(operations_repo): from fastapi import HTTPException from azure.cosmos.exceptions import CosmosResourceExistsError @@ -570,3 +586,9 @@ async def test_resource_has_active_operation_reconciles_stale_step_with_step_res saved_op = operations_repo.update_item.call_args[0][0] assert saved_op.status == Status.DeletingFailed assert saved_op.steps[0].status == Status.UpdatingFailed + operations_repo.update_item.assert_awaited_once_with(saved_op, release_lease=False) + assert operations_repo._reconcile_resource_status.await_args_list[1].args == ( + saved_op.steps[0].resourceId, + Status.UpdatingFailed, + saved_op.steps[0].message, + ) diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index bd935cd10e..13351cc50d 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -195,6 +195,41 @@ async def test_multi_step_document_retries( assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) +async def test_cascade_acquires_workspace_lease_before_dependency_snapshot(): + resource = create_test_resource() + resource.resourcePath = f"/workspaces/{resource.id}" + operation = create_sample_operation(resource.id, RequestAction.UnInstall) + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + resource_repo_mock = AsyncMock() + resource_repo_mock.get_resource_dependency_list.return_value = [resource.__dict__] + resource_template_repo_mock = AsyncMock() + resource_history_repo_mock = AsyncMock() + order = [] + operations_repo_mock.acquire_workspace_lease.side_effect = lambda *args: order.append(("lease", args)) + resource_repo_mock.get_resource_dependency_list.side_effect = lambda *args: order.append(("snapshot", args)) or [resource.__dict__] + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + + with patch("service_bus.resource_request_sender.update_resource_for_step", new=AsyncMock(return_value=resource_to_send_mock)), \ + patch("service_bus.resource_request_sender.send_deployment_message", new=AsyncMock()): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=resource_repo_mock, + user=create_test_user(), + resource_template_repo=resource_template_repo_mock, + resource_history_repo=resource_history_repo_mock, + action=RequestAction.UnInstall, + is_cascade=True, + ) + + assert order[0][0] == "lease" + assert order[1][0] == "snapshot" + operations_repo_mock.create_operation_item.assert_awaited_once() + assert operations_repo_mock.create_operation_item.call_args.kwargs["operation_id"] == order[0][1][1] + + @pytest.mark.parametrize( "action, expected_status", [ From 7ebff4ec2810c98615d11bfad74660b0a5cbd4a2 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 8 Sep 2026 13:58:31 +0000 Subject: [PATCH 74/90] Improve Airlock VM replacement and workspace lease cleanup --- api_app/db/repositories/operations.py | 13 ++- api_app/services/airlock.py | 84 +++++++++++-------- .../test_api/test_routes/test_airlock.py | 14 ++-- .../tests_ma/test_services/test_airlock.py | 29 +++++++ 4 files changed, 97 insertions(+), 43 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index fe40c20c77..bb17ced373 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -131,6 +131,7 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> for attempt in range(max_attempts): timestamp = self.get_timestamp() + lease_created = False lease_body = { "id": lease_id, "workspaceId": workspace_id, @@ -142,8 +143,17 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> create_call = self.container.create_item(body=lease_body) if hasattr(create_call, "__await__"): await create_call - if await self.resource_has_active_operation(workspace_id, exclude_operation_id=operation_id): + lease_created = True + try: + has_active_operation = await self.resource_has_active_operation( + workspace_id, exclude_operation_id=operation_id) + except Exception: + await self.release_workspace_lease(workspace_id, operation_id) + lease_created = False + raise + if has_active_operation: await self.release_workspace_lease(workspace_id, operation_id) + lease_created = False raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) return True except (CosmosResourceExistsError, ResourceExistsError): @@ -155,7 +165,6 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> if attempt < max_attempts - 1: continue raise - if not isinstance(existing_lease, dict): return True diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index d654b2dbf9..801c3233c3 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -196,7 +196,18 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac existing_resource = airlock_request.reviewUserResources[user.id] existing_resource = await user_resource_repo.get_user_resource_by_id(workspace_id=existing_resource.workspaceId, service_id=existing_resource.workspaceServiceId, resource_id=existing_resource.userResourceId) logger.info("User already has an existing review resource") - await _handle_existing_review_resource(existing_resource, user, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) + delete_operation = await _handle_existing_review_resource( + existing_resource, user, user_resource_repo, workspace_service_repo, + operation_repo, resource_template_repo, resource_history_repo, + wait_for_completion=False) + if delete_operation and hasattr(delete_operation, "id"): + asyncio.create_task(_redeploy_review_vm_after_delete( + airlock_request, user, workspace, review_workspace_id, + review_workspace_service_id, user_resource_template_name, + user_resource_repo, workspace_service_repo, operation_repo, + airlock_request_repo, resource_template_repo, + resource_history_repo, delete_operation.id)) + return airlock_request, delete_operation # Create the VM user_resource, operation = await _deploy_vm(airlock_request, user, workspace, review_workspace_id, review_workspace_service_id, user_resource_template_name, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) @@ -217,6 +228,31 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac return updated_resource, operation +async def _redeploy_review_vm_after_delete( + airlock_request: AirlockRequest, user: User, workspace: Workspace, + review_workspace_id: str, review_workspace_service_id: str, + user_resource_template_name: str, user_resource_repo: UserResourceRepository, + workspace_service_repo: WorkspaceServiceRepository, + operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, + resource_template_repo: ResourceTemplateRepository, + resource_history_repo: ResourceHistoryRepository, delete_operation_id: str): + try: + await wait_for_successful_operation(operation_repo, delete_operation_id) + user_resource, _ = await _deploy_vm( + airlock_request, user, workspace, review_workspace_id, + review_workspace_service_id, user_resource_template_name, + user_resource_repo, workspace_service_repo, operation_repo, + resource_template_repo, resource_history_repo) + await update_and_publish_event_airlock_request( + airlock_request, airlock_request_repo, user, workspace, + review_user_resource=AirlockReviewUserResource( + workspaceId=review_workspace_id, + workspaceServiceId=review_workspace_service_id, + userResourceId=user_resource.id)) + except Exception: + logger.exception("Failed to replace Airlock review VM after delete operation %s", delete_operation_id) + + async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, review_workspace_id: str, review_workspace_service_id: str, user_resource_template_name: str, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, operation_repo: OperationRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository): @@ -263,41 +299,17 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR # If it wasn't healthy or running, we'll delete the existing resource if not already deleted, and then create a new one logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": - if wait_for_completion: - logger.info("Deleting existing user resource...") - operation = await delete_review_user_resource( - user_resource=existing_resource, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operation_repo, - resource_history_repo=resource_history_repo, - user=user) - if operation and hasattr(operation, "id"): - await wait_for_successful_operation(operation_repo, operation.id) - return operation - - workspace_service = await workspace_service_repo.get_workspace_service_by_id( - workspace_id=existing_resource.workspaceId, - service_id=existing_resource.parentWorkspaceServiceId) - disable_op = await disable_user_resource( - existing_resource, user, workspace_service, user_resource_repo, - resource_template_repo, operation_repo, resource_history_repo) - if disable_op and hasattr(disable_op, "id"): - if not wait_for_completion: - return disable_op - await wait_for_successful_operation(operation_repo, disable_op.id) - - logger.info(f"Deleting user resource {existing_resource.id} in workspace service {workspace_service.id}") - operation = await send_uninstall_message( - resource=existing_resource, - resource_repo=user_resource_repo, - operations_repo=operation_repo, - resource_type=ResourceType.UserResource, + logger.info("Deleting existing user resource...") + operation = await delete_review_user_resource( + user_resource=existing_resource, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, resource_template_repo=resource_template_repo, + operations_repo=operation_repo, resource_history_repo=resource_history_repo, - user=user) - logger.info(f"Started operation {operation}") + user=user, wait_for_completion=wait_for_completion) + if wait_for_completion and operation and hasattr(operation, "id"): + await wait_for_successful_operation(operation_repo, operation.id) return operation @@ -513,13 +525,13 @@ async def delete_review_user_resource( resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, - user: User) -> Operation: + user: User, wait_for_completion: bool = True) -> Operation: workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=user_resource.workspaceId, service_id=user_resource.parentWorkspaceServiceId) # disable might contain logic that we need to execute before the deletion of the resource disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) - if disable_op and hasattr(disable_op, "id"): + if wait_for_completion and disable_op and hasattr(disable_op, "id"): await wait_for_successful_operation(operations_repo, disable_op.id) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index dc3c8da029..11c08a2cb5 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -409,7 +409,8 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert response.status_code == status.HTTP_409_CONFLICT - @patch("services.airlock.wait_for_successful_operation") + @patch("services.airlock._redeploy_review_vm_after_delete") + @patch("services.airlock.asyncio.create_task", side_effect=lambda coroutine: coroutine.close()) @patch("services.airlock.delete_review_user_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="uninstall", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.get_azure_resource_status", return_value={}) @@ -420,11 +421,14 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r @patch("services.airlock.UserResourceRepository.create_user_resource_item", return_value=(UserResource(id=USER_RESOURCE_ID, templateName="test", templateVersion="0.0.1", _etag="123"), "test")) @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id", return_value=sample_workspace(workspace_properties=sample_airlock_review_config())) - async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, wait_for_op_mock, app, client): - await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) + async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, create_task_mock, redeploy_mock, app, client): + response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) + assert response.status_code == status.HTTP_202_ACCEPTED assert delete_review_resource_mock.call_count == 1 - assert wait_for_op_mock.call_count == 1 - assert deploy_resource_mock.call_count == 1 + assert response.json()["operation"]["action"] == "uninstall" + assert create_task_mock.call_count == 1 + redeploy_mock.assert_called_once() + deploy_resource_mock.assert_not_called() # [POST] /workspaces/{workspace_id}/requests/{airlock_request_id}/revoke @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.Approved)) diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 11e0cd26ec..d9e9e10f78 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -866,3 +866,32 @@ async def test_delete_all_review_user_resources_retries_on_lease_contention(mock assert len(ops) == 1 assert mock_delete_review.call_count == 2 mock_sleep.assert_awaited() + + +@pytest.mark.asyncio +async def test_redeploy_review_vm_after_delete_waits_deploys_and_publishes(): + from services.airlock import _redeploy_review_vm_after_delete + + airlock_request = sample_airlock_request(status=AirlockRequestStatus.InReview) + user = create_test_user() + workspace = sample_workspace() + user_resource = MagicMock(id="replacement-resource") + operation_repo = AsyncMock() + airlock_request_repo = AsyncMock() + user_resource_repo = AsyncMock() + workspace_service_repo = AsyncMock() + resource_template_repo = AsyncMock() + resource_history_repo = AsyncMock() + + with patch("services.airlock.wait_for_successful_operation", new_callable=AsyncMock) as wait_mock, \ + patch("services.airlock._deploy_vm", new_callable=AsyncMock, return_value=(user_resource, MagicMock())) as deploy_mock, \ + patch("services.airlock.update_and_publish_event_airlock_request", new_callable=AsyncMock) as publish_mock: + await _redeploy_review_vm_after_delete( + airlock_request, user, workspace, "review-workspace", "review-service", "review-template", + user_resource_repo, workspace_service_repo, operation_repo, airlock_request_repo, + resource_template_repo, resource_history_repo, "delete-operation") + + wait_mock.assert_awaited_once_with(operation_repo, "delete-operation") + deploy_mock.assert_awaited_once() + publish_mock.assert_awaited_once() + assert publish_mock.await_args.kwargs["review_user_resource"].userResourceId == "replacement-resource" From 28656358c8cb51a6f2fc292eb25a6a231b264961 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Tue, 8 Sep 2026 15:11:25 +0000 Subject: [PATCH 75/90] Bump API version to 0.26.10 --- api_app/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/_version.py b/api_app/_version.py index 5252f582fe..29dd6604f8 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.9" +__version__ = "0.26.10" From 705d1bc98e3a61e71f13eb736012a6904d592b95 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Wed, 9 Sep 2026 09:02:01 +0000 Subject: [PATCH 76/90] Improve Airlock resource cleanup and redeployment workflows --- api_app/core/config.py | 1 + api_app/db/repositories/airlock_requests.py | 9 +- api_app/db/repositories/operations.py | 32 +++++- api_app/main.py | 4 + api_app/service_bus/airlock_workflow.py | 108 ++++++++++++++++++ api_app/service_bus/helpers.py | 5 + api_app/services/airlock.py | 44 +++++-- .../test_airlock_request_repository.py | 14 ++- .../test_operation_repository.py | 36 +++++- .../tests_ma/test_services/test_airlock.py | 34 +++++- core/terraform/api-webapp.tf | 1 + core/terraform/outputs.tf | 4 + core/terraform/servicebus.tf | 6 + core/version.txt | 2 +- 14 files changed, 276 insertions(+), 24 deletions(-) create mode 100644 api_app/service_bus/airlock_workflow.py diff --git a/api_app/core/config.py b/api_app/core/config.py index d2f1cf1fa4..85567b582e 100644 --- a/api_app/core/config.py +++ b/api_app/core/config.py @@ -41,6 +41,7 @@ SERVICE_BUS_RESOURCE_REQUEST_QUEUE: str = config("SERVICE_BUS_RESOURCE_REQUEST_QUEUE", default="") SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE: str = config("SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE", default="") SERVICE_BUS_STEP_RESULT_QUEUE: str = config("SERVICE_BUS_STEP_RESULT_QUEUE", default="") +SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE: str = config("SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE", default="") # Event grid configuration EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT: str = config("EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT", default="") diff --git a/api_app/db/repositories/airlock_requests.py b/api_app/db/repositories/airlock_requests.py index 21184c6e61..34881d68e9 100644 --- a/api_app/db/repositories/airlock_requests.py +++ b/api_app/db/repositories/airlock_requests.py @@ -208,7 +208,14 @@ async def update_airlock_request( except CosmosAccessConditionFailedError: logger.warning(f"ETag mismatch for request ID: '{original_request.id}'. Retrying.") original_request = await self.get_airlock_request_by_id(original_request.id) - updated_request = self._build_updated_request(original_request=original_request, new_status=new_status, request_files=request_files, status_message=status_message, airlock_review=airlock_review) + updated_request = self._build_updated_request( + original_request=original_request, + new_status=new_status, + request_files=request_files, + status_message=status_message, + airlock_review=airlock_review, + review_user_resource=review_user_resource, + updated_by=updated_by) db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) return db_response diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index bb17ced373..e87001582e 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -188,6 +188,24 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> Status.ActionSucceeded, Status.ActionFailed, } + if existing_op and getattr(existing_op, "status", None) in terminal_statuses and not getattr(existing_op, "reconciled", False): + try: + affected_step_resources = [ + (step.resourceId, step.status, step.message) + for step in getattr(existing_op, "steps", None) or [] + if step.resourceId and step.status + ] + await self._reconcile_operation_resources(existing_op, affected_step_resources) + existing_op.reconciled = True + existing_op.updatedWhen = timestamp + update_call = self.update_item(existing_op, release_lease=False) + if hasattr(update_call, "__await__"): + await update_call + await self.release_workspace_lease(workspace_id, existing_op.id) + except Exception as e: + logger.exception(f"Failed to reconcile pending operation {existing_op.id}: {e}") + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + continue if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: op_time = getattr(existing_op, "updatedWhen", 0.0) or getattr(existing_op, "createdWhen", 0.0) if timestamp - op_time < WORKSPACE_LEASE_EXPIRY_SECONDS: @@ -197,7 +215,7 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> existing_op.status = get_failure_status_for_action(existing_op.action) existing_op.message = "Operation timed out or was interrupted before completion" existing_op.updatedWhen = timestamp - existing_op.reconciled = True + existing_op.reconciled = False affected_step_resources = [] if getattr(existing_op, "steps", None): for step in existing_op.steps: @@ -210,12 +228,15 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> if hasattr(update_call, "__await__"): await update_call await self._reconcile_operation_resources(existing_op, affected_step_resources) + existing_op.reconciled = True + update_call = self.update_item(existing_op, release_lease=False) + if hasattr(update_call, "__await__"): + await update_call await self.release_workspace_lease(workspace_id, existing_op.id) except Exception as e: logger.exception(f"Failed to reconcile stale operation {existing_op.id}: {e}") raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - # update_item persists a terminal status and releases/deletes the workspace lease. - # Restart lease acquisition rather than replacing the now-deleted lease document. + # Restart lease acquisition rather than replacing the now-reconciled lease document. continue except HTTPException: raise @@ -263,8 +284,9 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona del_call = self.delete_item(lease_id) if hasattr(del_call, "__await__"): await del_call - except Exception: - pass + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, + CosmosAccessConditionFailedError): + return @staticmethod def operations_query(): diff --git a/api_app/main.py b/api_app/main.py index 0bdc769141..0bfcf7a4a0 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -20,6 +20,7 @@ from services.logging import initialize_logging, logger from service_bus.deployment_status_updater import DeploymentStatusUpdater from service_bus.airlock_request_status_update import AirlockStatusUpdater +from service_bus.airlock_workflow import AirlockWorkflowUpdater @asynccontextmanager @@ -33,9 +34,12 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() + airlockWorkflow = AirlockWorkflowUpdater() + await airlockWorkflow.init_repos() asyncio.create_task(deploymentStatusUpdater.receive_messages()) asyncio.create_task(airlockStatusUpdater.receive_messages()) + asyncio.create_task(airlockWorkflow.receive_messages()) yield diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py new file mode 100644 index 0000000000..99167b990b --- /dev/null +++ b/api_app/service_bus/airlock_workflow.py @@ -0,0 +1,108 @@ +import asyncio +import json + +from azure.servicebus.aio import AutoLockRenewer, ServiceBusClient +from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError +from pydantic import ValidationError + +from core import config, credentials +from db.repositories.airlock_requests import AirlockRequestRepository +from db.repositories.operations import OperationRepository +from db.repositories.resource_templates import ResourceTemplateRepository +from db.repositories.resources_history import ResourceHistoryRepository +from db.repositories.user_resources import UserResourceRepository +from db.repositories.workspace_services import WorkspaceServiceRepository +from db.repositories.workspaces import WorkspaceRepository +from models.domain.authentication import User +from models.domain.airlock_request import AirlockReviewUserResource +from models.domain.resource import ResourceType +from services.airlock import _deploy_vm, wait_for_successful_operation, update_and_publish_event_airlock_request, disable_user_resource +from api.routes.resource_helpers import send_uninstall_message +from services.logging import logger + + +class AirlockWorkflowUpdater: + async def init_repos(self): + self.airlock_request_repo = await AirlockRequestRepository.create() + self.operations_repo = await OperationRepository.create() + self.resource_template_repo = await ResourceTemplateRepository.create() + self.resource_history_repo = await ResourceHistoryRepository.create() + self.user_resource_repo = await UserResourceRepository.create() + self.workspace_service_repo = await WorkspaceServiceRepository.create() + self.workspace_repo = await WorkspaceRepository.create() + + async def receive_messages(self): + while True: + try: + async with credentials.get_credential_async_context() as credential: + async with ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) as client: + receiver = client.get_queue_receiver(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + async with receiver: + messages = await receiver.receive_messages(max_message_count=10, max_wait_time=1) + for message in messages: + async with AutoLockRenewer() as renewer: + renewer.register(receiver, message, max_lock_renewal_duration=60) + if await self.process_message(message): + await receiver.complete_message(message) + else: + await receiver.abandon_message(message) + except OperationTimeoutError: + logger.debug("No Airlock workflow messages available") + except ServiceBusConnectionError: + logger.warning("Airlock workflow Service Bus connection failed; retrying") + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Airlock workflow receiver failed; retrying") + await asyncio.sleep(10) + + async def process_message(self, message) -> bool: + try: + payload = json.loads(str(message)) + user = User.model_validate(payload["user"]) + user_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=payload["review_workspace_id"], + service_id=payload["review_workspace_service_id"], + resource_id=payload["user_resource_id"]) + if payload["workflow"] == "cleanup": + workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( + workspace_id=payload["review_workspace_id"], + service_id=payload["review_workspace_service_id"]) + disable_operation = await disable_user_resource( + user_resource, user, workspace_service, self.user_resource_repo, + self.resource_template_repo, self.operations_repo, self.resource_history_repo) + await wait_for_successful_operation(self.operations_repo, disable_operation.id) + else: + await wait_for_successful_operation(self.operations_repo, payload["operation_id"]) + delete_operation = await send_uninstall_message( + resource=user_resource, + resource_repo=self.user_resource_repo, + operations_repo=self.operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + user=user) + await wait_for_successful_operation(self.operations_repo, delete_operation.id) + + if payload["workflow"] == "redeploy": + airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) + workspace = await self.workspace_repo.get_workspace_by_id(payload["workspace_id"]) + user_resource, _ = await _deploy_vm( + airlock_request, user, workspace, + payload["review_workspace_id"], payload["review_workspace_service_id"], + payload["user_resource_template_name"], self.user_resource_repo, + self.workspace_service_repo, self.operations_repo, + self.resource_template_repo, self.resource_history_repo) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + review_user_resource=AirlockReviewUserResource( + workspaceId=payload["review_workspace_id"], + workspaceServiceId=payload["review_workspace_service_id"], + userResourceId=user_resource.id)) + return True + except (json.JSONDecodeError, KeyError, ValidationError): + logger.exception("Invalid Airlock workflow message") + return True + except Exception: + logger.exception("Airlock workflow message failed; it will be retried") + return False diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index e1b2daf74d..f01f9346e0 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -1,3 +1,4 @@ +import json from azure.servicebus import ServiceBusMessage from azure.servicebus.aio import ServiceBusClient from pydantic import TypeAdapter @@ -41,6 +42,10 @@ async def send_deployment_message(content, correlation_id, session_id, action): await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) +async def send_airlock_workflow_message(content): + await _send_message(ServiceBusMessage(body=json.dumps(content)), config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + + async def update_resource_for_step(operation_step: OperationStep, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, root_resource: Resource, step_resource: Resource, resource_to_update_id: str, primary_action: str, user: User) -> Resource: # step_resource is the resource instance where the step was defined. e.g. 'add firewall rule' step defined in Guacamole template -> the step_resource is the Guacamole ws service. # root_resource is theresource on which the user chose to update, i.e. the top most resource in cascaded action or the same resource in a non-cascaded action. diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 801c3233c3..e55550ab56 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -35,6 +35,7 @@ from collections import defaultdict from event_grid.event_sender import send_status_changed_event, send_airlock_notification_event +from service_bus.helpers import send_airlock_workflow_message STORAGE_ENDPOINT = config.STORAGE_ENDPOINT_SUFFIX @@ -201,12 +202,17 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac operation_repo, resource_template_repo, resource_history_repo, wait_for_completion=False) if delete_operation and hasattr(delete_operation, "id"): - asyncio.create_task(_redeploy_review_vm_after_delete( - airlock_request, user, workspace, review_workspace_id, - review_workspace_service_id, user_resource_template_name, - user_resource_repo, workspace_service_repo, operation_repo, - airlock_request_repo, resource_template_repo, - resource_history_repo, delete_operation.id)) + await send_airlock_workflow_message({ + "workflow": "redeploy", + "airlock_request_id": airlock_request.id, + "user": user.model_dump(), + "workspace_id": workspace.id, + "review_workspace_id": review_workspace_id, + "review_workspace_service_id": review_workspace_service_id, + "user_resource_template_name": user_resource_template_name, + "user_resource_id": existing_resource.id, + "operation_id": delete_operation.id, + }) return airlock_request, delete_operation # Create the VM @@ -235,7 +241,10 @@ async def _redeploy_review_vm_after_delete( workspace_service_repo: WorkspaceServiceRepository, operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, resource_template_repo: ResourceTemplateRepository, - resource_history_repo: ResourceHistoryRepository, delete_operation_id: str): + resource_history_repo: ResourceHistoryRepository, delete_operation_id: str, + existing_resource_workspace_id: Optional[str] = None, + existing_resource_service_id: Optional[str] = None, + existing_resource_id: Optional[str] = None): try: await wait_for_successful_operation(operation_repo, delete_operation_id) user_resource, _ = await _deploy_vm( @@ -243,6 +252,7 @@ async def _redeploy_review_vm_after_delete( review_workspace_service_id, user_resource_template_name, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) + airlock_request = await airlock_request_repo.get_airlock_request_by_id(airlock_request.id) await update_and_publish_event_airlock_request( airlock_request, airlock_request_repo, user, workspace, review_user_resource=AirlockReviewUserResource( @@ -533,6 +543,8 @@ async def delete_review_user_resource( disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) if wait_for_completion and disable_op and hasattr(disable_op, "id"): await wait_for_successful_operation(operations_repo, disable_op.id) + if not wait_for_completion: + return disable_op logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -571,7 +583,18 @@ async def delete_all_review_user_resources( resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, - user: User) -> List[Operation]: + user: User, enqueue: bool = False) -> List[Operation]: + if enqueue: + for review_resource in airlock_request.reviewUserResources.values(): + await send_airlock_workflow_message({ + "workflow": "cleanup", + "user": user.model_dump(), + "review_workspace_id": review_resource.workspaceId, + "review_workspace_service_id": review_resource.workspaceServiceId, + "user_resource_id": review_resource.userResourceId, + }) + return [] + operations: List[Operation] = [] for review_ur in airlock_request.reviewUserResources.values(): start_time = time.time() @@ -592,7 +615,8 @@ async def delete_all_review_user_resources( resource_template_repo=resource_template_repo, operations_repo=operations_repo, resource_history_repo=resource_history_repo, - user=user + user=user, + enqueue=True ) operations.append(operation) break @@ -627,7 +651,7 @@ async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) + await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user, enqueue=True) return updated_request diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index dd1d03f305..9edd3d104f 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -7,7 +7,7 @@ from models.domain.workspace import Workspace from tests_ma.test_api.conftest import create_test_user from models.schemas.airlock_request import AirlockRequestInCreate -from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource from db.repositories.airlock_requests import AirlockRequestRepository from db.errors import EntityDoesNotExist @@ -191,8 +191,18 @@ async def test_update_airlock_request_should_retry_update_when_etag_is_not_up_to expected_update_attempts = 2 user = create_test_user() mock_existing_request = airlock_request_mock(status=DRAFT) - await airlock_request_repo.update_airlock_request(original_request=mock_existing_request, updated_by=user, new_status=SUBMITTED) + review_user_resource = AirlockReviewUserResource( + workspaceId="review-workspace", + workspaceServiceId="review-service", + userResourceId="replacement-resource") + await airlock_request_repo.update_airlock_request( + original_request=mock_existing_request, + updated_by=user, + new_status=SUBMITTED, + review_user_resource=review_user_resource) assert update_airlock_request_item_mock.call_count == expected_update_attempts + retried_request = update_airlock_request_item_mock.call_args_list[1].args[1] + assert review_user_resource in retried_request.reviewUserResources.values() async def test_get_airlock_requests_queries_db(airlock_request_repo): diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 9f1a796f72..ecdc593751 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -392,11 +392,42 @@ async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_rea res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") assert res is True - operations_repo.update_item.assert_awaited_once() + assert operations_repo.update_item.await_count == 2 assert stale_op.status == Status.DeploymentFailed assert operations_repo._container.create_item.await_count == 2 +async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "terminal-op", + "createdWhen": operations_repo.get_timestamp(), + "_etag": "lease-etag", + }) + terminal_op = MagicMock(id="terminal-op", status=Status.DeploymentFailed, reconciled=False, steps=[]) + operations_repo.get_operation_by_id = AsyncMock(return_value=terminal_op) + operations_repo._reconcile_operation_resources = AsyncMock(side_effect=Exception("Cosmos write error")) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "new-op") + + assert exc.value.status_code == 409 + operations_repo._container.replace_item.assert_not_called() + + operations_repo._reconcile_operation_resources.side_effect = None + operations_repo.update_item = AsyncMock() + operations_repo.release_workspace_lease = AsyncMock() + assert await operations_repo.acquire_workspace_lease("ws-1", "new-op") is True + assert terminal_op.reconciled is True + operations_repo.update_item.assert_awaited_once_with(terminal_op, release_lease=False) + operations_repo.release_workspace_lease.assert_awaited_once_with("ws-1", "terminal-op") + + async def test_acquire_workspace_lease_fails_closed_when_reconciliation_fails(operations_repo): from fastapi import HTTPException from azure.cosmos.exceptions import CosmosResourceExistsError @@ -543,11 +574,12 @@ async def test_reconcile_stale_operation_uses_step_resource_action(operations_re operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) operations_repo.update_item = AsyncMock(return_value=None) operations_repo._reconcile_resource_status = AsyncMock() + operations_repo._container.delete_item = AsyncMock() operations_repo.resource_has_active_operation = AsyncMock(return_value=False) await operations_repo.acquire_workspace_lease("ws-1", "new-op") - operations_repo.update_item.assert_awaited_once() + assert operations_repo.update_item.await_count == 2 saved_op = operations_repo.update_item.call_args[0][0] assert saved_op.status == Status.DeletingFailed assert saved_op.steps[0].status == Status.DeletingFailed diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index d9e9e10f78..8ca8e312e3 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -556,9 +556,9 @@ async def test_revoke_request_calls_update_with_revoked_status(update_mock, airl @pytest.mark.asyncio -@patch("services.airlock.delete_review_user_resource") +@patch("services.airlock.send_airlock_workflow_message", new_callable=AsyncMock) @patch("services.airlock.update_and_publish_event_airlock_request") -async def test_cancel_request_deletes_review_resource(_, delete_review_user_resource, airlock_request_repo_mock): +async def test_cancel_request_enqueues_review_resource_cleanup(_, send_airlock_workflow_message, airlock_request_repo_mock): await cancel_request( airlock_request=sample_airlock_request(), user=create_test_user(), @@ -570,7 +570,8 @@ async def test_cancel_request_deletes_review_resource(_, delete_review_user_reso operations_repo=AsyncMock(), resource_history_repo=AsyncMock()) - delete_review_user_resource.assert_called_once() + send_airlock_workflow_message.assert_awaited_once() + assert send_airlock_workflow_message.await_args.args[0]["workflow"] == "cleanup" @pytest.mark.asyncio @@ -815,6 +816,32 @@ async def test_delete_review_user_resource_aborts_if_disable_fails(mock_send_uni mock_send_uninstall.assert_not_called() +@pytest.mark.asyncio +@patch("services.airlock.wait_for_successful_operation", new_callable=AsyncMock) +@patch("services.airlock.disable_user_resource") +@patch("services.airlock.send_uninstall_message") +async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mock_send_uninstall, mock_disable, wait_mock): + from models.domain.operation import Operation + + disable_op = Operation(id="disable", resourceId="resource", resourcePath="/workspaces/ws", action="upgrade") + delete_op = Operation(id="delete", resourceId="resource", resourcePath="/workspaces/ws", action="uninstall") + mock_disable.return_value = disable_op + mock_send_uninstall.return_value = delete_op + + user_resource = MagicMock(workspaceId="ws-1", parentWorkspaceServiceId="svc-1", id="resource") + await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=AsyncMock(), + workspace_service_repo=AsyncMock(), + resource_template_repo=AsyncMock(), + operations_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + user=create_test_user(), + wait_for_completion=False) + + mock_send_uninstall.assert_not_called() + + @pytest.mark.asyncio async def test_wait_for_successful_operation_retries_on_entity_does_not_exist(): from services.airlock import wait_for_successful_operation @@ -893,5 +920,6 @@ async def test_redeploy_review_vm_after_delete_waits_deploys_and_publishes(): wait_mock.assert_awaited_once_with(operation_repo, "delete-operation") deploy_mock.assert_awaited_once() + airlock_request_repo.get_airlock_request_by_id.assert_awaited_once_with(airlock_request.id) publish_mock.assert_awaited_once() assert publish_mock.await_args.kwargs["review_user_resource"].userResourceId == "replacement-resource" diff --git a/core/terraform/api-webapp.tf b/core/terraform/api-webapp.tf index 47afeb83cb..07a90a1fc6 100644 --- a/core/terraform/api-webapp.tf +++ b/core/terraform/api-webapp.tf @@ -45,6 +45,7 @@ resource "azurerm_linux_web_app" "api" { "SERVICE_BUS_RESOURCE_REQUEST_QUEUE" = azurerm_servicebus_queue.workspacequeue.name "SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE" = azurerm_servicebus_queue.service_bus_deployment_status_update_queue.name "SERVICE_BUS_STEP_RESULT_QUEUE" = module.airlock_resources.service_bus_step_result_queue + "SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE" = azurerm_servicebus_queue.airlock_workflow.name "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.id.client_id "TRE_ID" = var.tre_id "RESOURCE_LOCATION" = azurerm_resource_group.core.location diff --git a/core/terraform/outputs.tf b/core/terraform/outputs.tf index e4e22f97eb..96152f0091 100644 --- a/core/terraform/outputs.tf +++ b/core/terraform/outputs.tf @@ -50,6 +50,10 @@ output "service_bus_deployment_status_queue" { value = azurerm_servicebus_queue.service_bus_deployment_status_update_queue.name } +output "service_bus_airlock_workflow_queue" { + value = azurerm_servicebus_queue.airlock_workflow.name +} + output "state_store_resource_id" { value = azurerm_cosmosdb_account.tre_db_account.id } diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..dbb3427a86 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -70,6 +70,12 @@ resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" requires_session = true } +resource "azurerm_servicebus_queue" "airlock_workflow" { + name = "airlock-workflow" + namespace_id = azurerm_servicebus_namespace.sb.id + partitioning_enabled = false +} + resource "azurerm_private_dns_zone" "servicebus" { name = module.terraform_azurerm_environment_configuration.private_links["privatelink.servicebus.windows.net"] resource_group_name = azurerm_resource_group.core.name diff --git a/core/version.txt b/core/version.txt index 12d80d0630..670ad2c1cf 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.16.18" From 8596924741a35e110ba3dfe68ce62ac22dc57361 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Wed, 9 Sep 2026 12:16:45 +0000 Subject: [PATCH 77/90] Improve Airlock workflow cleanup and lease handling --- api_app/api/routes/workspaces.py | 9 +++ api_app/service_bus/airlock_workflow.py | 44 ++++++---- api_app/services/airlock.py | 17 +++- .../test_api/test_routes/test_airlock.py | 14 ++-- .../test_service_bus/test_airlock_workflow.py | 81 +++++++++++++++++++ .../tests_ma/test_services/test_airlock.py | 5 +- 6 files changed, 141 insertions(+), 29 deletions(-) create mode 100644 api_app/tests_ma/test_service_bus/test_airlock_workflow.py diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 620c9b08af..561e65341f 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -158,6 +158,7 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + workspace = await workspace_repo.get_workspace_by_id(workspace.id) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -298,6 +299,7 @@ async def create_workspace_service(response: Response, workspace_service_input: operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) address_space_added = False try: @@ -370,6 +372,8 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace_service.workspaceId, operation_id) + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace_service.workspaceId, workspace_service.id) try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -507,6 +511,8 @@ async def create_user_resource( response: Response, user_resource_create: UserResourceInCreate, user_resource_repo=Depends(get_repository(UserResourceRepository)), + workspace_repo=Depends(get_repository(WorkspaceRepository)), + workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), @@ -519,6 +525,9 @@ async def create_user_resource( operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + workspace_service = await workspace_service_repo.get_deployed_workspace_service_by_id( + workspace.id, workspace_service.id, operations_repo) try: owner_id: str = None diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index 99167b990b..faacf0b67e 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -13,6 +13,8 @@ from db.repositories.user_resources import UserResourceRepository from db.repositories.workspace_services import WorkspaceServiceRepository from db.repositories.workspaces import WorkspaceRepository +from db.errors import EntityDoesNotExist +from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS from models.domain.authentication import User from models.domain.airlock_request import AirlockReviewUserResource from models.domain.resource import ResourceType @@ -38,10 +40,13 @@ async def receive_messages(self): async with ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) as client: receiver = client.get_queue_receiver(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) async with receiver: - messages = await receiver.receive_messages(max_message_count=10, max_wait_time=1) + messages = await receiver.receive_messages(max_message_count=1, max_wait_time=1) for message in messages: async with AutoLockRenewer() as renewer: - renewer.register(receiver, message, max_lock_renewal_duration=60) + renewer.register( + receiver, + message, + max_lock_renewal_duration=2 * WORKSPACE_LEASE_EXPIRY_SECONDS) if await self.process_message(message): await receiver.complete_message(message) else: @@ -60,29 +65,38 @@ async def process_message(self, message) -> bool: try: payload = json.loads(str(message)) user = User.model_validate(payload["user"]) - user_resource = await self.user_resource_repo.get_user_resource_by_id( - workspace_id=payload["review_workspace_id"], - service_id=payload["review_workspace_service_id"], - resource_id=payload["user_resource_id"]) + try: + user_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=payload["review_workspace_id"], + service_id=payload["review_workspace_service_id"], + resource_id=payload["user_resource_id"]) + except EntityDoesNotExist: + if payload["workflow"] == "cleanup": + logger.info("Airlock review resource %s is already deleted", payload["user_resource_id"]) + return True + user_resource = None if payload["workflow"] == "cleanup": workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( workspace_id=payload["review_workspace_id"], service_id=payload["review_workspace_service_id"]) + if user_resource is None: + return True disable_operation = await disable_user_resource( user_resource, user, workspace_service, self.user_resource_repo, self.resource_template_repo, self.operations_repo, self.resource_history_repo) await wait_for_successful_operation(self.operations_repo, disable_operation.id) else: await wait_for_successful_operation(self.operations_repo, payload["operation_id"]) - delete_operation = await send_uninstall_message( - resource=user_resource, - resource_repo=self.user_resource_repo, - operations_repo=self.operations_repo, - resource_type=ResourceType.UserResource, - resource_template_repo=self.resource_template_repo, - resource_history_repo=self.resource_history_repo, - user=user) - await wait_for_successful_operation(self.operations_repo, delete_operation.id) + if user_resource is not None and not payload.get("uninstall_started", False): + delete_operation = await send_uninstall_message( + resource=user_resource, + resource_repo=self.user_resource_repo, + operations_repo=self.operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + user=user) + await wait_for_successful_operation(self.operations_repo, delete_operation.id) if payload["workflow"] == "redeploy": airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index e55550ab56..8b30315a17 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -163,7 +163,8 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai resource_template_repo=resource_template_repo, operations_repo=operation_repo, resource_history_repo=resource_history_repo, - user=user + user=user, + enqueue=True ) return updated_airlock_request @@ -212,6 +213,7 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac "user_resource_template_name": user_resource_template_name, "user_resource_id": existing_resource.id, "operation_id": delete_operation.id, + "uninstall_started": True, }) return airlock_request, delete_operation @@ -544,7 +546,15 @@ async def delete_review_user_resource( if wait_for_completion and disable_op and hasattr(disable_op, "id"): await wait_for_successful_operation(operations_repo, disable_op.id) if not wait_for_completion: - return disable_op + logger.info(f"Starting deletion of user resource {user_resource.id} after disable operation {disable_op.id}") + return await send_uninstall_message( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user) logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -615,8 +625,7 @@ async def delete_all_review_user_resources( resource_template_repo=resource_template_repo, operations_repo=operations_repo, resource_history_repo=resource_history_repo, - user=user, - enqueue=True + user=user ) operations.append(operation) break diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index b0b8205365..90e506f527 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -351,7 +351,7 @@ async def test_post_create_airlock_review_with_illegal_status_change_returns_400 response = await client.post(app.url_path_for(strings.API_REVIEW_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID), json=sample_airlock_review_input_data) assert response.status_code == status.HTTP_400_BAD_REQUEST - @patch("services.airlock.send_uninstall_message") + @patch("services.airlock.send_airlock_workflow_message") @patch("services.airlock.ResourceHistoryRepository.save_item") @patch("services.airlock.UserResourceRepository.update_item_with_etag") @patch("services.airlock.update_user_resource") @@ -362,10 +362,10 @@ async def test_post_create_airlock_review_with_illegal_status_change_returns_400 @patch("services.airlock.AirlockRequestRepository.create_airlock_review_item", return_value=sample_airlock_review_object()) @patch("services.airlock.update_and_publish_event_airlock_request", return_value=sample_airlock_request_object(status=AirlockRequestStatus.Approved, review_user_resource=True)) @patch("services.airlock.AirlockRequestRepository.save_item") - async def test_post_create_airlock_review_cleans_up_review_user_resources(self, _, __, ___, ____, _____, ______, _______, ________, _________, __________, send_uninstall_message_mock, app, client, sample_airlock_review_input_data): + async def test_post_create_airlock_review_cleans_up_review_user_resources(self, _, __, ___, ____, _____, ______, _______, ________, _________, __________, workflow_message_mock, app, client, sample_airlock_review_input_data): response = await client.post(app.url_path_for(strings.API_REVIEW_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID), json=sample_airlock_review_input_data) assert response.status_code == status.HTTP_200_OK - assert send_uninstall_message_mock.call_count == 1 + workflow_message_mock.assert_awaited_once() @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.update_and_publish_event_airlock_request", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @@ -411,8 +411,7 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert response.status_code == status.HTTP_409_CONFLICT - @patch("services.airlock._redeploy_review_vm_after_delete") - @patch("services.airlock.asyncio.create_task", side_effect=lambda coroutine: coroutine.close()) + @patch("services.airlock.send_airlock_workflow_message") @patch("services.airlock.delete_review_user_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="uninstall", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.get_azure_resource_status", return_value={}) @@ -423,13 +422,12 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r @patch("services.airlock.UserResourceRepository.create_user_resource_item", return_value=(UserResource(id=USER_RESOURCE_ID, templateName="test", templateVersion="0.0.1", _etag="123"), "test")) @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id", return_value=sample_workspace(workspace_properties=sample_airlock_review_config())) - async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, create_task_mock, redeploy_mock, app, client): + async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, workflow_message_mock, app, client): response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert response.status_code == status.HTTP_202_ACCEPTED assert delete_review_resource_mock.call_count == 1 assert response.json()["operation"]["action"] == "uninstall" - assert create_task_mock.call_count == 1 - redeploy_mock.assert_called_once() + workflow_message_mock.assert_awaited_once() deploy_resource_mock.assert_not_called() # [POST] /workspaces/{workspace_id}/requests/{airlock_request_id}/revoke diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py new file mode 100644 index 0000000000..27b8865482 --- /dev/null +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -0,0 +1,81 @@ +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from db.errors import EntityDoesNotExist +from models.domain.authentication import User +from service_bus.airlock_workflow import AirlockWorkflowUpdater + + +@pytest.fixture +def updater(): + workflow = AirlockWorkflowUpdater() + workflow.user_resource_repo = MagicMock() + workflow.workspace_service_repo = MagicMock() + workflow.operations_repo = MagicMock() + workflow.resource_template_repo = MagicMock() + workflow.resource_history_repo = MagicMock() + workflow.airlock_request_repo = MagicMock() + workflow.workspace_repo = MagicMock() + return workflow + + +def message(payload): + return json.dumps(payload) + + +def user_payload(): + return User(id="user-id", name="Test User", email="test@example.com").model_dump() + + +@pytest.mark.asyncio +async def test_process_message_completes_cleanup_when_resource_is_already_deleted(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + + result = await updater.process_message(message({ + "workflow": "cleanup", + "user": user_payload(), + "review_workspace_id": "workspace-id", + "review_workspace_service_id": "service-id", + "user_resource_id": "resource-id", + })) + + assert result is True + + +@pytest.mark.asyncio +async def test_process_message_redeploys_without_duplicate_uninstall(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + updater.operations_repo.get_operation_by_id = AsyncMock() + updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock() + updater.workspace_repo.get_workspace_by_id = AsyncMock() + + payload = { + "workflow": "redeploy", + "user": user_payload(), + "airlock_request_id": "request-id", + "workspace_id": "workspace-id", + "review_workspace_id": "review-workspace-id", + "review_workspace_service_id": "service-id", + "user_resource_template_name": "template", + "user_resource_id": "resource-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()) as uninstall, \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock(return_value=(MagicMock(id="new-resource-id"), MagicMock()))), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message(payload)) + + assert result is True + uninstall.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_message_acknowledges_invalid_message(updater): + result = await updater.process_message("not-json") + + assert result is True diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 8ca8e312e3..801f8eef41 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -829,7 +829,7 @@ async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mo mock_send_uninstall.return_value = delete_op user_resource = MagicMock(workspaceId="ws-1", parentWorkspaceServiceId="svc-1", id="resource") - await delete_review_user_resource( + result = await delete_review_user_resource( user_resource=user_resource, user_resource_repo=AsyncMock(), workspace_service_repo=AsyncMock(), @@ -839,7 +839,8 @@ async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mo user=create_test_user(), wait_for_completion=False) - mock_send_uninstall.assert_not_called() + mock_send_uninstall.assert_awaited_once() + assert result is delete_op @pytest.mark.asyncio From c983ac77f2a63206b30f84e32866050ec1e98ab6 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Wed, 9 Sep 2026 16:21:47 +0000 Subject: [PATCH 78/90] Improve Airlock redeployment recovery and workspace lease cleanup --- api_app/api/routes/workspaces.py | 46 +++++++++++-- api_app/db/repositories/airlock_requests.py | 11 ++- api_app/db/repositories/user_resources.py | 10 +++ api_app/models/domain/airlock_request.py | 10 +++ api_app/service_bus/airlock_workflow.py | 68 ++++++++++++++++--- api_app/services/airlock.py | 15 ++-- .../test_service_bus/test_airlock_workflow.py | 59 ++++++++++++++++ 7 files changed, 195 insertions(+), 24 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 561e65341f..24d1949729 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -158,7 +158,15 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) - workspace = await workspace_repo.get_workspace_by_id(workspace.id) + try: + workspace = await workspace_repo.get_workspace_by_id(workspace.id) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace reload failed") + raise try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -299,7 +307,15 @@ async def create_workspace_service(response: Response, workspace_service_input: operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) - workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + try: + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace reload failed") + raise address_space_added = False try: @@ -372,8 +388,16 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace_service.workspaceId, operation_id) - workspace_service = await workspace_service_repo.get_workspace_service_by_id( - workspace_service.workspaceId, workspace_service.id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace_service.workspaceId, workspace_service.id) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace service reload failed") + raise try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -525,9 +549,17 @@ async def create_user_resource( operation_id = operations_repo.create_operation_id() if hasattr(operations_repo, "acquire_workspace_lease"): await operations_repo.acquire_workspace_lease(workspace.id, operation_id) - workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) - workspace_service = await workspace_service_repo.get_deployed_workspace_service_by_id( - workspace.id, workspace_service.id, operations_repo) + try: + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + workspace_service = await workspace_service_repo.get_deployed_workspace_service_by_id( + workspace.id, workspace_service.id, operations_repo) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after resource reload failed") + raise try: owner_id: str = None diff --git a/api_app/db/repositories/airlock_requests.py b/api_app/db/repositories/airlock_requests.py index 34881d68e9..0e72e2c908 100644 --- a/api_app/db/repositories/airlock_requests.py +++ b/api_app/db/repositories/airlock_requests.py @@ -12,7 +12,7 @@ from models.domain.authentication import User from db.errors import EntityDoesNotExist from models.domain.airlock_request import AirlockFile, AirlockRequest, AirlockRequestStatus, \ - AirlockReview, AirlockReviewDecision, AirlockRequestHistoryItem, AirlockRequestType, AirlockReviewUserResource + AirlockReview, AirlockReviewDecision, AirlockRequestHistoryItem, AirlockRequestType, AirlockReviewUserResource, AirlockRedeployWorkflow from models.schemas.airlock_request import AirlockRequestInCreate, AirlockReviewInCreate from core import config from resources import strings @@ -194,7 +194,8 @@ async def update_airlock_request( request_files: Optional[List[AirlockFile]] = None, status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, - review_user_resource: Optional[AirlockReviewUserResource] = None) -> AirlockRequest: + review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None) -> AirlockRequest: updated_request = self._build_updated_request( original_request=original_request, new_status=new_status, @@ -202,6 +203,7 @@ async def update_airlock_request( status_message=status_message, airlock_review=airlock_review, review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow, updated_by=updated_by) try: db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) @@ -215,6 +217,7 @@ async def update_airlock_request( status_message=status_message, airlock_review=airlock_review, review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow, updated_by=updated_by) db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) @@ -258,6 +261,7 @@ def _build_updated_request( status_message: Optional[Optional[str]] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None, updated_by: Optional[Union[User, dict]] = None) -> AirlockRequest: updated_request = copy.deepcopy(original_request) @@ -282,6 +286,9 @@ def _build_updated_request( if reviewer_id: updated_request.reviewUserResources[reviewer_id] = review_user_resource + if redeploy_workflow is not None: + updated_request.redeployWorkflows[redeploy_workflow.workflowId] = redeploy_workflow + return updated_request def _validate_status_update(self, current_status, new_status): diff --git a/api_app/db/repositories/user_resources.py b/api_app/db/repositories/user_resources.py index 6df46adcc2..87b2a45518 100644 --- a/api_app/db/repositories/user_resources.py +++ b/api_app/db/repositories/user_resources.py @@ -85,6 +85,16 @@ async def get_user_resource_by_id(self, workspace_id: str, service_id: str, reso raise EntityDoesNotExist return TypeAdapter(UserResource).validate_python(user_resources[0]) + async def get_user_resource_by_workflow_id(self, workspace_id: str, service_id: str, workflow_id: str) -> UserResource: + query, parameters = self.user_resources_query(str(workspace_id), str(service_id)) + query += ' AND c.properties.airlock_redeploy_workflow_id = @workflowId AND c.deploymentStatus != @deletedStatus' + parameters.append({'name': '@workflowId', 'value': workflow_id}) + parameters.append({'name': '@deletedStatus', 'value': Status.Deleted}) + user_resources = await self.query(query=query, parameters=parameters) + if not user_resources: + raise EntityDoesNotExist + return TypeAdapter(UserResource).validate_python(user_resources[0]) + def get_user_resource_spec_params(self): return self.get_resource_base_spec_params() diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index 23ab8a626c..b87eb974ce 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -93,6 +93,15 @@ class AirlockReviewUserResource(AzureTREModel): userResourceId: str = Field(title="User Resource ID") +class AirlockRedeployWorkflow(AzureTREModel): + workflowId: str + phase: str + operationId: str + workspaceId: Optional[str] = None + workspaceServiceId: Optional[str] = None + userResourceId: Optional[str] = None + + class AirlockRequest(AzureTREModel): """ Airlock request @@ -114,6 +123,7 @@ class AirlockRequest(AzureTREModel): reviews: Optional[List[AirlockReview]] = None etag: Optional[str] = Field(None, title="_etag", alias="_etag") reviewUserResources: Dict[str, AirlockReviewUserResource] = Field(default_factory=dict, title="User resources created for Airlock Reviews") + redeployWorkflows: Dict[str, AirlockRedeployWorkflow] = Field(default_factory=dict) # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 @field_validator("etag", mode="before") diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index faacf0b67e..d4ea2dfa0f 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -1,5 +1,6 @@ import asyncio import json +import uuid from azure.servicebus.aio import AutoLockRenewer, ServiceBusClient from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError @@ -16,7 +17,7 @@ from db.errors import EntityDoesNotExist from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS from models.domain.authentication import User -from models.domain.airlock_request import AirlockReviewUserResource +from models.domain.airlock_request import AirlockReviewUserResource, AirlockRedeployWorkflow from models.domain.resource import ResourceType from services.airlock import _deploy_vm, wait_for_successful_operation, update_and_publish_event_airlock_request, disable_user_resource from api.routes.resource_helpers import send_uninstall_message @@ -64,6 +65,10 @@ async def receive_messages(self): async def process_message(self, message) -> bool: try: payload = json.loads(str(message)) + workflow = payload["workflow"] + if workflow not in ("cleanup", "redeploy"): + logger.warning("Unknown Airlock workflow type: %s", workflow) + return True user = User.model_validate(payload["user"]) try: user_resource = await self.user_resource_repo.get_user_resource_by_id( @@ -71,11 +76,11 @@ async def process_message(self, message) -> bool: service_id=payload["review_workspace_service_id"], resource_id=payload["user_resource_id"]) except EntityDoesNotExist: - if payload["workflow"] == "cleanup": + if workflow == "cleanup": logger.info("Airlock review resource %s is already deleted", payload["user_resource_id"]) return True user_resource = None - if payload["workflow"] == "cleanup": + if workflow == "cleanup": workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( workspace_id=payload["review_workspace_id"], service_id=payload["review_workspace_service_id"]) @@ -98,21 +103,62 @@ async def process_message(self, message) -> bool: user=user) await wait_for_successful_operation(self.operations_repo, delete_operation.id) - if payload["workflow"] == "redeploy": + if workflow == "redeploy": airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) workspace = await self.workspace_repo.get_workspace_by_id(payload["workspace_id"]) - user_resource, _ = await _deploy_vm( - airlock_request, user, workspace, - payload["review_workspace_id"], payload["review_workspace_service_id"], - payload["user_resource_template_name"], self.user_resource_repo, - self.workspace_service_repo, self.operations_repo, - self.resource_template_repo, self.resource_history_repo) + workflow_id = payload.get("redeploy_workflow_id", f"{airlock_request.id}:{user.id}") + workflows = airlock_request.redeployWorkflows + workflow_state = workflows.get(workflow_id) if isinstance(workflows, dict) else None + if workflow_state is not None and workflow_state.phase == "completed": + return True + + replacement_resource = None + operation_id = workflow_state.operationId if workflow_state is not None else str(uuid.uuid4()) + if workflow_state is not None and workflow_state.phase == "deploying": + try: + if workflow_state.userResourceId: + replacement_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=workflow_state.workspaceId, + service_id=workflow_state.workspaceServiceId, + resource_id=workflow_state.userResourceId) + else: + replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( + workspace_id=payload["review_workspace_id"], + service_id=payload["review_workspace_service_id"], + workflow_id=workflow_id) + except EntityDoesNotExist: + pass + + if replacement_resource is None: + workflow_state = AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId=operation_id) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + redeploy_workflow=workflow_state) + replacement_resource, _ = await _deploy_vm( + airlock_request, user, workspace, + payload["review_workspace_id"], payload["review_workspace_service_id"], + payload["user_resource_template_name"], self.user_resource_repo, + self.workspace_service_repo, self.operations_repo, + self.resource_template_repo, self.resource_history_repo, + workflow_id=workflow_id, operation_id=operation_id) + + workflow_state = AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="completed", + operationId=operation_id, + workspaceId=payload["review_workspace_id"], + workspaceServiceId=payload["review_workspace_service_id"], + userResourceId=replacement_resource.id) await update_and_publish_event_airlock_request( airlock_request, self.airlock_request_repo, user, workspace, review_user_resource=AirlockReviewUserResource( workspaceId=payload["review_workspace_id"], workspaceServiceId=payload["review_workspace_service_id"], - userResourceId=user_resource.id)) + userResourceId=replacement_resource.id), + redeploy_workflow=workflow_state) return True except (json.JSONDecodeError, KeyError, ValidationError): logger.exception("Invalid Airlock workflow message") diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 8b30315a17..5dc277377b 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -6,7 +6,7 @@ from azure.storage.blob import generate_container_sas, ContainerSasPermissions, BlobServiceClient from fastapi import HTTPException, status from core import config, credentials -from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource, AirlockReviewDecision, AirlockActions, AirlockFile, AirlockReview +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource, AirlockReviewDecision, AirlockActions, AirlockFile, AirlockReview, AirlockRedeployWorkflow from models.domain.authentication import User from models.domain.workspace import Workspace from models.domain.user_resource import UserResource @@ -214,6 +214,7 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac "user_resource_id": existing_resource.id, "operation_id": delete_operation.id, "uninstall_started": True, + "redeploy_workflow_id": f"{airlock_request.id}:{user.id}", }) return airlock_request, delete_operation @@ -267,7 +268,8 @@ async def _redeploy_review_vm_after_delete( async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, review_workspace_id: str, review_workspace_service_id: str, user_resource_template_name: str, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, operation_repo: OperationRepository, - resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository): + resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + workflow_id: Optional[str] = None, operation_id: Optional[str] = None): logger.info(f"Creating review VM in workspace:{review_workspace_id} service:{review_workspace_service_id} using template:{user_resource_template_name}") workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=review_workspace_id, service_id=review_workspace_service_id) airlock_request_sas_url = get_airlock_container_link(airlock_request, user, workspace) @@ -280,6 +282,8 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor "airlock_request_sas_url": airlock_request_sas_url } ) + if workflow_id is not None: + user_resource_create.properties["airlock_redeploy_workflow_id"] = workflow_id user_resource, resource_template = await user_resource_repo.create_user_resource_item( user_resource_create, review_workspace_id, review_workspace_service_id, workspace_service.templateName, user.id, user.roles) @@ -291,7 +295,8 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, user=user, - resource_template=resource_template) + resource_template=resource_template, + operation_id=operation_id) return user_resource, operation @@ -366,6 +371,7 @@ async def update_and_publish_event_airlock_request( status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None, ) -> AirlockRequest: try: logger.debug(f"Updating airlock request item: {airlock_request.id}") @@ -376,7 +382,8 @@ async def update_and_publish_event_airlock_request( request_files=request_files, status_message=status_message, airlock_review=airlock_review, - review_user_resource=review_user_resource) + review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow) except Exception as e: logger.exception(f'Failed updating airlock_request item {airlock_request}') # If the validation failed, the error was not related to the saving itself diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py index 27b8865482..8866b9ebc1 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -5,6 +5,7 @@ from db.errors import EntityDoesNotExist from models.domain.authentication import User +from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow from service_bus.airlock_workflow import AirlockWorkflowUpdater @@ -74,6 +75,64 @@ async def test_process_message_redeploys_without_duplicate_uninstall(updater): uninstall.assert_not_awaited() +@pytest.mark.asyncio +async def test_process_message_acknowledges_unknown_workflow_without_side_effects(updater): + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()) as wait_for_operation, \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()) as uninstall: + result = await updater.process_message(message({ + "workflow": "future-workflow", + "user": user_payload(), + "review_workspace_id": "workspace-id", + "review_workspace_service_id": "service-id", + "user_resource_id": "resource-id", + })) + + assert result is True + updater.user_resource_repo.get_user_resource_by_id.assert_not_called() + wait_for_operation.assert_not_awaited() + uninstall.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_message_recovers_deploying_redeploy_without_deploying_again(updater): + workflow_id = "request-id:user-id" + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + updater.user_resource_repo.get_user_resource_by_workflow_id = AsyncMock( + return_value=MagicMock(id="replacement-resource-id")) + updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + redeployWorkflows={workflow_id: AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId="replacement-operation-id")} + )) + updater.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock()) + + payload = { + "workflow": "redeploy", + "redeploy_workflow_id": workflow_id, + "user": user_payload(), + "airlock_request_id": "request-id", + "workspace_id": "workspace-id", + "review_workspace_id": "review-workspace-id", + "review_workspace_service_id": "service-id", + "user_resource_template_name": "template", + "user_resource_id": "resource-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock()) as deploy, \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()) as update: + result = await updater.process_message(message(payload)) + + assert result is True + deploy.assert_not_awaited() + assert update.await_args.kwargs["redeploy_workflow"].phase == "completed" + assert update.await_args.kwargs["redeploy_workflow"].operationId == "replacement-operation-id" + + @pytest.mark.asyncio async def test_process_message_acknowledges_invalid_message(updater): result = await updater.process_message("not-json") From 3e17a009f7f31f00c8501b1f2a52ed0b74cbecae Mon Sep 17 00:00:00 2001 From: James Chapman Date: Wed, 9 Sep 2026 16:44:51 +0000 Subject: [PATCH 79/90] fix tests --- .../tests_ma/test_api/test_routes/test_resource_helpers.py | 1 + api_app/tests_ma/test_api/test_routes/test_workspaces.py | 6 ++++++ .../test_service_bus/test_airlock_request_status_update.py | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index 31fb31e5f1..562f217692 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -276,6 +276,7 @@ async def test_save_and_deploy_masks_secrets(self, send_deployment_message_mock, resource_repo.save_item = AsyncMock(return_value=None) resource_repo.get_resource_by_id = AsyncMock(return_value=resource) operations_repo.create_operation_item = AsyncMock(return_value=operation) + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=basic_resource_template) diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 362a70c185..2fafa0250a 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -37,6 +37,12 @@ pytestmark = pytest.mark.asyncio +@pytest.fixture(autouse=True) +def mock_workspace_lease_release(): + with patch("api.routes.workspaces.OperationRepository.release_workspace_lease", new=AsyncMock()): + yield + + WORKSPACE_ID = '933ad738-7265-4b5f-9eae-a1a62928772e' SERVICE_ID = 'abcad738-7265-4b5f-9eae-a1a62928772e' USER_RESOURCE_ID = 'a33ad738-7265-4b5f-9eae-a1a62928772a' diff --git a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py index 6c5463bf76..0f9c722460 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py @@ -212,11 +212,12 @@ async def test_receiving_good_message(_, logging_mock, workspace_repo, airlock_r airlock_request_repo.return_value.update_airlock_request.assert_called_once_with( original_request=expected_airlock_request, updated_by=expected_airlock_request.updatedBy, - new_status=test_sb_step_result_message["data"]["new_status"], + new_status=AirlockRequestStatus.InReview, request_files=None, status_message=None, airlock_review=None, - review_user_resource=None) + review_user_resource=None, + redeploy_workflow=None) assert eg_client().send.call_count == 2 logging_mock.assert_not_called() From 3ed812eb97479ca9a58e4fffd33298bceaa0a4c3 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Wed, 9 Sep 2026 17:21:45 +0000 Subject: [PATCH 80/90] Fix Airlock deletion sequencing and operation reconciliation --- api_app/db/repositories/operations.py | 5 +++++ api_app/services/airlock.py | 13 +++---------- .../test_repositories/test_operation_repository.py | 14 ++++++++++++-- api_app/tests_ma/test_services/test_airlock.py | 6 ++---- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index e87001582e..dd3dcee8a0 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -195,6 +195,11 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> for step in getattr(existing_op, "steps", None) or [] if step.resourceId and step.status ] + existing_op.reconciled = False + existing_op.updatedWhen = timestamp + update_call = self.update_item(existing_op, release_lease=False) + if hasattr(update_call, "__await__"): + await update_call await self._reconcile_operation_resources(existing_op, affected_step_resources) existing_op.reconciled = True existing_op.updatedWhen = timestamp diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 5dc277377b..ce1bf1b2b9 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -213,7 +213,7 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac "user_resource_template_name": user_resource_template_name, "user_resource_id": existing_resource.id, "operation_id": delete_operation.id, - "uninstall_started": True, + "uninstall_started": False, "redeploy_workflow_id": f"{airlock_request.id}:{user.id}", }) return airlock_request, delete_operation @@ -553,15 +553,8 @@ async def delete_review_user_resource( if wait_for_completion and disable_op and hasattr(disable_op, "id"): await wait_for_successful_operation(operations_repo, disable_op.id) if not wait_for_completion: - logger.info(f"Starting deletion of user resource {user_resource.id} after disable operation {disable_op.id}") - return await send_uninstall_message( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operations_repo, - resource_type=ResourceType.UserResource, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user) + logger.info(f"Deferring deletion of user resource {user_resource.id} until disable operation {disable_op.id} completes") + return disable_op logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index ecdc593751..3a46fee452 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -411,6 +411,8 @@ async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(o }) terminal_op = MagicMock(id="terminal-op", status=Status.DeploymentFailed, reconciled=False, steps=[]) operations_repo.get_operation_by_id = AsyncMock(return_value=terminal_op) + persisted_reconciled_states = [] + operations_repo.update_item = AsyncMock(side_effect=lambda item, **kwargs: persisted_reconciled_states.append(item.reconciled)) operations_repo._reconcile_operation_resources = AsyncMock(side_effect=Exception("Cosmos write error")) with pytest.raises(HTTPException) as exc: @@ -418,13 +420,21 @@ async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(o assert exc.value.status_code == 409 operations_repo._container.replace_item.assert_not_called() + operations_repo.update_item.assert_awaited_once_with(terminal_op, release_lease=False) + assert persisted_reconciled_states == [False] + assert terminal_op.reconciled is False operations_repo._reconcile_operation_resources.side_effect = None - operations_repo.update_item = AsyncMock() + operations_repo.update_item.reset_mock() operations_repo.release_workspace_lease = AsyncMock() assert await operations_repo.acquire_workspace_lease("ws-1", "new-op") is True assert terminal_op.reconciled is True - operations_repo.update_item.assert_awaited_once_with(terminal_op, release_lease=False) + assert operations_repo.update_item.await_count == 2 + assert persisted_reconciled_states == [False, False, True] + assert operations_repo.update_item.await_args_list[0].args == (terminal_op,) + assert operations_repo.update_item.await_args_list[0].kwargs == {"release_lease": False} + assert operations_repo.update_item.await_args_list[1].args == (terminal_op,) + assert operations_repo.update_item.await_args_list[1].kwargs == {"release_lease": False} operations_repo.release_workspace_lease.assert_awaited_once_with("ws-1", "terminal-op") diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 801f8eef41..0ded913453 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -824,9 +824,7 @@ async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mo from models.domain.operation import Operation disable_op = Operation(id="disable", resourceId="resource", resourcePath="/workspaces/ws", action="upgrade") - delete_op = Operation(id="delete", resourceId="resource", resourcePath="/workspaces/ws", action="uninstall") mock_disable.return_value = disable_op - mock_send_uninstall.return_value = delete_op user_resource = MagicMock(workspaceId="ws-1", parentWorkspaceServiceId="svc-1", id="resource") result = await delete_review_user_resource( @@ -839,8 +837,8 @@ async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mo user=create_test_user(), wait_for_completion=False) - mock_send_uninstall.assert_awaited_once() - assert result is delete_op + mock_send_uninstall.assert_not_awaited() + assert result is disable_op @pytest.mark.asyncio From 3022bda866d7314e5923a247bf4426bf89364158 Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:55:07 +0100 Subject: [PATCH 81/90] Include operation ID in redeploy workflow ID Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- api_app/services/airlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index ce1bf1b2b9..08ca35c7fb 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -214,7 +214,7 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac "user_resource_id": existing_resource.id, "operation_id": delete_operation.id, "uninstall_started": False, - "redeploy_workflow_id": f"{airlock_request.id}:{user.id}", + "redeploy_workflow_id": f"{airlock_request.id}:{user.id}:{delete_operation.id}", }) return airlock_request, delete_operation From ad6cefbd482abad4d2e507ce88ea55c4cfe79f2a Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 13:03:26 +0000 Subject: [PATCH 82/90] Serialize address space allocation with allocator leases --- api_app/api/routes/workspaces.py | 43 ++++++++++++++++++++------- api_app/db/repositories/operations.py | 7 +++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 24d1949729..0732715b00 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -118,36 +118,48 @@ async def retrieve_workspace_scope_id_by_workspace_id(workspace=Depends(get_work @workspaces_core_router.post("/workspaces", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_CREATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def create_workspace(workspace_create: WorkspaceInCreate, response: Response, user=Depends(require_tre_admin), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + address_space_operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_address_space_allocator_lease(address_space_operation_id) try: # TODO: This requires Directory.ReadAll ( Application.Read.All ) to be enabled in the Azure AD application to enable a users workspaces to be listed. This should be made optional. auth_info = extract_auth_information(workspace_create.properties) workspace, resource_template = await workspace_repo.create_workspace_item(workspace_create, auth_info, user.id, user.roles) except (ValidationError, ValueError) as e: logger.exception("Failed to create workspace model instance") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except StorageAccountNameGenerationTimeout: logger.exception("Storage name availability check timed out") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Storage name availability check timed out. Please try again.") except StorageAccountNameCheckFailed: logger.exception("Storage name availability check failed") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Storage name availability check failed. Please try again.") except UserNotAuthorizedToUseTemplate as e: logger.exception("User not authorized to use template") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) except InvalidInput as e: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(e)) + except Exception: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) + raise - operation = await save_and_deploy_resource( - resource=workspace, - resource_repo=workspace_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) - response.headers["Location"] = construct_location_header(operation) - - return OperationInResponse(operation=operation) + try: + operation = await save_and_deploy_resource( + resource=workspace, + resource_repo=workspace_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template) + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + finally: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) @@ -318,6 +330,7 @@ async def create_workspace_service(response: Response, workspace_service_input: raise address_space_added = False + address_space_lease_acquired = False try: try: workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) @@ -334,6 +347,8 @@ async def create_workspace_service(response: Response, workspace_service_input: # if template has address_space get an address space if resource_template.properties.get("address_space"): + await operations_repo.acquire_address_space_allocator_lease(operation_id) + address_space_lease_acquired = True # check workspace has address_spaces property if not workspace.properties.get("address_spaces"): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) @@ -367,14 +382,20 @@ async def create_workspace_service(response: Response, workspace_service_input: resource_template_repo, resource_history_repo, user ) finally: + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) if hasattr(operations_repo, "release_workspace_lease"): await operations_repo.release_workspace_lease(workspace.id, operation_id) if getattr(err, "lease_retained", False): + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, ) raise + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index dd3dcee8a0..85e9952ecc 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -25,6 +25,7 @@ # Lease expiry includes a safe margin beyond the 3600-second execution window and status-delivery # latency (7200 seconds / 2 hours) to ensure still-running operations are not prematurely reclaimed. WORKSPACE_LEASE_EXPIRY_SECONDS = 7200.0 +ADDRESS_SPACE_ALLOCATOR_LEASE_ID = "address_space_allocator" def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: @@ -293,6 +294,12 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona CosmosAccessConditionFailedError): return + async def acquire_address_space_allocator_lease(self, operation_id: str) -> bool: + return await self.acquire_workspace_lease(ADDRESS_SPACE_ALLOCATOR_LEASE_ID, operation_id) + + async def release_address_space_allocator_lease(self, operation_id: Optional[str] = None) -> None: + await self.release_workspace_lease(ADDRESS_SPACE_ALLOCATOR_LEASE_ID, operation_id) + @staticmethod def operations_query(): return 'SELECT * FROM c WHERE' From f0d00405ca144143132c6ff63ec8ff778d305be8 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 14:48:09 +0000 Subject: [PATCH 83/90] Serialize resource deletion and Airlock redeployment operations --- api_app/api/routes/resource_helpers.py | 9 +- api_app/api/routes/workspaces.py | 90 +++++++++++------- api_app/db/repositories/operations.py | 68 +++++++++++++- api_app/service_bus/airlock_workflow.py | 91 ++++++++++++++----- api_app/services/airlock.py | 50 +++++----- .../test_operation_repository.py | 83 ++++++++++++++++- .../test_service_bus/test_airlock_workflow.py | 79 +++++++++++----- 7 files changed, 353 insertions(+), 117 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 48456a0056..0daac51a3d 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -182,9 +182,13 @@ async def send_uninstall_message( resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, user: User, - is_cascade: str = False + is_cascade: str = False, + operation_id: Optional[str] = None, ) -> Operation: try: + operation_kwargs = {} + if operation_id is not None: + operation_kwargs["operation_id"] = operation_id operation = await send_resource_request_message( resource=resource, operations_repo=operations_repo, @@ -193,7 +197,8 @@ async def send_uninstall_message( resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.UnInstall, - is_cascade=is_cascade + is_cascade=is_cascade, + **operation_kwargs, ) return operation except HTTPException: diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 0732715b00..b4be108c58 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -460,23 +460,32 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) -async def delete_workspace_service(response: Response, user=Depends(require_workspace_owner), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: - if await operations_repo.resource_has_active_operation(workspace.id): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - if await delete_validation(workspace_service, workspace_service_repo): - operation = await send_uninstall_message( - resource=workspace_service, - resource_repo=workspace_service_repo, - operations_repo=operations_repo, - resource_type=ResourceType.WorkspaceService, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - is_cascade=True) - - response.headers["Location"] = construct_location_header(operation) - - return OperationInResponse(operation=operation) +async def delete_workspace_service(response: Response, user=Depends(require_workspace_owner), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace.id, workspace_service.id) + if await operations_repo.resource_has_active_operation(workspace.id, exclude_operation_id=operation_id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + if await delete_validation(workspace_service, workspace_service_repo): + operation = await send_uninstall_message( + resource=workspace_service, + resource_repo=workspace_service_repo, + operations_repo=operations_repo, + resource_type=ResourceType.WorkspaceService, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + is_cascade=True, + operation_id=operation_id) + + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + except Exception as err: + if not getattr(err, "lease_retained", False): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) @@ -645,29 +654,44 @@ async def delete_user_resource( user_resource=Depends(get_user_resource_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), user_resource_repo=Depends(get_repository(UserResourceRepository)), + workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: - validate_user_has_valid_role_for_user_resource(user, user_resource) - - if await operations_repo.resource_has_active_operation(user_resource.workspaceId): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_workspace_lease(user_resource.workspaceId, operation_id) + try: + user_resource = await user_resource_repo.get_user_resource_by_id( + workspace_id=user_resource.workspaceId, + service_id=workspace_service.id, + resource_id=user_resource.id) + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + user_resource.workspaceId, workspace_service.id) + validate_user_has_valid_role_for_user_resource(user, user_resource) - if user_resource.isEnabled: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.USER_RESOURCE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION) + if await operations_repo.resource_has_active_operation( + user_resource.workspaceId, exclude_operation_id=operation_id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - operation = await send_uninstall_message( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operations_repo, - resource_type=ResourceType.UserResource, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user) + if user_resource.isEnabled: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.USER_RESOURCE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION) - response.headers["Location"] = construct_location_header(operation) + operation = await send_uninstall_message( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + operation_id=operation_id) - return OperationInResponse(operation=operation) + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + except Exception as err: + if not getattr(err, "lease_retained", False): + await operations_repo.release_workspace_lease(user_resource.workspaceId, operation_id) + raise @user_resources_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_USER_RESOURCE, dependencies=[Depends(get_workspace_by_id_from_path), Depends(get_workspace_service_by_id_from_path)]) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 85e9952ecc..ec38af5237 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -26,6 +26,7 @@ # latency (7200 seconds / 2 hours) to ensure still-running operations are not prematurely reclaimed. WORKSPACE_LEASE_EXPIRY_SECONDS = 7200.0 ADDRESS_SPACE_ALLOCATOR_LEASE_ID = "address_space_allocator" +ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS = 300.0 def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: @@ -291,14 +292,69 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona if hasattr(del_call, "__await__"): await del_call except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, - CosmosAccessConditionFailedError): + CosmosAccessConditionFailedError, TypeError): return async def acquire_address_space_allocator_lease(self, operation_id: str) -> bool: - return await self.acquire_workspace_lease(ADDRESS_SPACE_ALLOCATOR_LEASE_ID, operation_id) + if not hasattr(self, "_container") or self._container is None: + return True + + lease_body = { + "id": ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + "operationId": operation_id, + "createdWhen": self.get_timestamp(), + } + try: + create_call = self.container.create_item(body=lease_body) + if hasattr(create_call, "__await__"): + await create_call + return True + except (CosmosResourceExistsError, ResourceExistsError): + try: + read_call = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + return await self.acquire_address_space_allocator_lease(operation_id) + + if not isinstance(existing_lease, dict): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + if existing_lease.get("operationId") == operation_id: + return True + if self.get_timestamp() - existing_lease.get("createdWhen", 0.0) < ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + etag = existing_lease.get("_etag") + try: + if not etag: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, + detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + replace_call = self.container.replace_item( + item=ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + body=lease_body, + etag=etag, + match_condition=MatchConditions.IfNotModified) + if hasattr(replace_call, "__await__"): + await replace_call + return True + except (CosmosAccessConditionFailedError, ResourceExistsError, CosmosResourceNotFoundError, ResourceNotFoundError): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) async def release_address_space_allocator_lease(self, operation_id: Optional[str] = None) -> None: - await self.release_workspace_lease(ADDRESS_SPACE_ALLOCATOR_LEASE_ID, operation_id) + if not hasattr(self, "_container") or self._container is None: + return + try: + read_call = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + if isinstance(existing_lease, dict) and (not operation_id or existing_lease.get("operationId") == operation_id): + etag = existing_lease.get("_etag") + delete_call = self.delete_item( + ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + **({"etag": etag, "match_condition": MatchConditions.IfNotModified} if etag else {})) + if hasattr(delete_call, "__await__"): + await delete_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, + CosmosAccessConditionFailedError, TypeError): + return @staticmethod def operations_query(): @@ -534,7 +590,7 @@ async def resource_has_active_operation(self, resource_id: str, exclude_operatio op.status = get_failure_status_for_action(op.action) op.message = "Operation timed out or was interrupted before completion" op.updatedWhen = timestamp - op.reconciled = True + op.reconciled = False affected_step_resources = [] if getattr(op, "steps", None): for step in op.steps: @@ -547,6 +603,10 @@ async def resource_has_active_operation(self, resource_id: str, exclude_operatio if hasattr(update_call, "__await__"): await update_call await self._reconcile_operation_resources(op, affected_step_resources) + op.reconciled = True + update_call = self.update_item(op, release_lease=False) + if hasattr(update_call, "__await__"): + await update_call target_workspace_id = extract_workspace_id_from_resource_path(op.resourcePath) if target_workspace_id: await self.release_workspace_lease(target_workspace_id, op.id) diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index d4ea2dfa0f..514f902951 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -17,7 +17,7 @@ from db.errors import EntityDoesNotExist from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS from models.domain.authentication import User -from models.domain.airlock_request import AirlockReviewUserResource, AirlockRedeployWorkflow +from models.domain.airlock_request import AirlockRequestType, AirlockReviewUserResource, AirlockRedeployWorkflow from models.domain.resource import ResourceType from services.airlock import _deploy_vm, wait_for_successful_operation, update_and_publish_event_airlock_request, disable_user_resource from api.routes.resource_helpers import send_uninstall_message @@ -47,7 +47,7 @@ async def receive_messages(self): renewer.register( receiver, message, - max_lock_renewal_duration=2 * WORKSPACE_LEASE_EXPIRY_SECONDS) + max_lock_renewal_duration=3 * WORKSPACE_LEASE_EXPIRY_SECONDS) if await self.process_message(message): await receiver.complete_message(message) else: @@ -69,21 +69,50 @@ async def process_message(self, message) -> bool: if workflow not in ("cleanup", "redeploy"): logger.warning("Unknown Airlock workflow type: %s", workflow) return True - user = User.model_validate(payload["user"]) + airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) + review_user_id = payload.get("review_user_id") + workflow_id = payload.get("redeploy_workflow_id") + if workflow == "redeploy": + if not workflow_id: + raise ValueError("Redeploy workflow ID is required") + workflow_state = airlock_request.redeployWorkflows.get(workflow_id) + if workflow_state is not None and workflow_state.phase == "completed": + return True + workflow_user_id = workflow_id.split(":")[1] if ":" in workflow_id else None + review_user_id = review_user_id or workflow_user_id + if not review_user_id or review_user_id not in airlock_request.reviewUserResources: + raise ValueError("Airlock workflow user is not associated with the request") + user = User.model_validate(airlock_request.createdBy) + if user.id != review_user_id: + raise ValueError("Airlock workflow user does not match the request owner") + review_resource = airlock_request.reviewUserResources[review_user_id] + review_workspace_id = review_resource.workspaceId + review_workspace_service_id = review_resource.workspaceServiceId + user_resource_id = review_resource.userResourceId + workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) + if airlock_request.type == AirlockRequestType.Import: + review_config = workspace.properties["airlock_review_config"]["import"] + review_workspace_id = review_config["import_vm_workspace_id"] + review_workspace_service_id = review_config["import_vm_workspace_service_id"] + user_resource_template_name = review_config["import_vm_user_resource_template_name"] + else: + review_config = workspace.properties["airlock_review_config"]["export"] + review_workspace_service_id = review_config["export_vm_workspace_service_id"] + user_resource_template_name = review_config["export_vm_user_resource_template_name"] try: user_resource = await self.user_resource_repo.get_user_resource_by_id( - workspace_id=payload["review_workspace_id"], - service_id=payload["review_workspace_service_id"], - resource_id=payload["user_resource_id"]) + workspace_id=review_workspace_id, + service_id=review_workspace_service_id, + resource_id=user_resource_id) except EntityDoesNotExist: if workflow == "cleanup": - logger.info("Airlock review resource %s is already deleted", payload["user_resource_id"]) + logger.info("Airlock review resource %s is already deleted", user_resource_id) return True user_resource = None if workflow == "cleanup": workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( - workspace_id=payload["review_workspace_id"], - service_id=payload["review_workspace_service_id"]) + workspace_id=review_workspace_id, + service_id=review_workspace_service_id) if user_resource is None: return True disable_operation = await disable_user_resource( @@ -104,9 +133,6 @@ async def process_message(self, message) -> bool: await wait_for_successful_operation(self.operations_repo, delete_operation.id) if workflow == "redeploy": - airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) - workspace = await self.workspace_repo.get_workspace_by_id(payload["workspace_id"]) - workflow_id = payload.get("redeploy_workflow_id", f"{airlock_request.id}:{user.id}") workflows = airlock_request.redeployWorkflows workflow_state = workflows.get(workflow_id) if isinstance(workflows, dict) else None if workflow_state is not None and workflow_state.phase == "completed": @@ -123,8 +149,8 @@ async def process_message(self, message) -> bool: resource_id=workflow_state.userResourceId) else: replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( - workspace_id=payload["review_workspace_id"], - service_id=payload["review_workspace_service_id"], + workspace_id=review_workspace_id, + service_id=review_workspace_service_id, workflow_id=workflow_id) except EntityDoesNotExist: pass @@ -137,26 +163,41 @@ async def process_message(self, message) -> bool: await update_and_publish_event_airlock_request( airlock_request, self.airlock_request_repo, user, workspace, redeploy_workflow=workflow_state) - replacement_resource, _ = await _deploy_vm( - airlock_request, user, workspace, - payload["review_workspace_id"], payload["review_workspace_service_id"], - payload["user_resource_template_name"], self.user_resource_repo, - self.workspace_service_repo, self.operations_repo, - self.resource_template_repo, self.resource_history_repo, - workflow_id=workflow_id, operation_id=operation_id) + try: + replacement_resource, _ = await _deploy_vm( + airlock_request, user, workspace, + review_workspace_id, review_workspace_service_id, + user_resource_template_name, self.user_resource_repo, + self.workspace_service_repo, self.operations_repo, + self.resource_template_repo, self.resource_history_repo, + workflow_id=workflow_id, operation_id=operation_id) + except Exception: + try: + await self.operations_repo.release_workspace_lease( + review_workspace_id, operation_id) + except Exception: + logger.exception("Failed to release failed Airlock redeploy lease") + replacement_operation_id = str(uuid.uuid4()) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + redeploy_workflow=AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId=replacement_operation_id)) + raise workflow_state = AirlockRedeployWorkflow( workflowId=workflow_id, phase="completed", operationId=operation_id, - workspaceId=payload["review_workspace_id"], - workspaceServiceId=payload["review_workspace_service_id"], + workspaceId=review_workspace_id, + workspaceServiceId=review_workspace_service_id, userResourceId=replacement_resource.id) await update_and_publish_event_airlock_request( airlock_request, self.airlock_request_repo, user, workspace, review_user_resource=AirlockReviewUserResource( - workspaceId=payload["review_workspace_id"], - workspaceServiceId=payload["review_workspace_service_id"], + workspaceId=review_workspace_id, + workspaceServiceId=review_workspace_service_id, userResourceId=replacement_resource.id), redeploy_workflow=workflow_state) return True diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 08ca35c7fb..fd88a18198 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -1,5 +1,6 @@ import asyncio import time +import uuid from datetime import datetime, timedelta, UTC from services.logging import logger @@ -206,12 +207,6 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac await send_airlock_workflow_message({ "workflow": "redeploy", "airlock_request_id": airlock_request.id, - "user": user.model_dump(), - "workspace_id": workspace.id, - "review_workspace_id": review_workspace_id, - "review_workspace_service_id": review_workspace_service_id, - "user_resource_template_name": user_resource_template_name, - "user_resource_id": existing_resource.id, "operation_id": delete_operation.id, "uninstall_started": False, "redeploy_workflow_id": f"{airlock_request.id}:{user.id}:{delete_operation.id}", @@ -271,8 +266,14 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, workflow_id: Optional[str] = None, operation_id: Optional[str] = None): logger.info(f"Creating review VM in workspace:{review_workspace_id} service:{review_workspace_service_id} using template:{user_resource_template_name}") - workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=review_workspace_id, service_id=review_workspace_service_id) - airlock_request_sas_url = get_airlock_container_link(airlock_request, user, workspace) + deployment_operation_id = operation_id or str(uuid.uuid4()) + await operation_repo.acquire_workspace_lease(review_workspace_id, deployment_operation_id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=review_workspace_id, service_id=review_workspace_service_id) + airlock_request_sas_url = get_airlock_container_link(airlock_request, user, workspace) + except Exception: + await operation_repo.release_workspace_lease(review_workspace_id, deployment_operation_id) + raise user_resource_create = UserResourceInCreate( templateName=user_resource_template_name, @@ -285,18 +286,23 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor if workflow_id is not None: user_resource_create.properties["airlock_redeploy_workflow_id"] = workflow_id - user_resource, resource_template = await user_resource_repo.create_user_resource_item( - user_resource_create, review_workspace_id, review_workspace_service_id, workspace_service.templateName, user.id, user.roles) + try: + user_resource, resource_template = await user_resource_repo.create_user_resource_item( + user_resource_create, review_workspace_id, review_workspace_service_id, workspace_service.templateName, user.id, user.roles) - operation = await save_and_deploy_resource( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operation_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template, - operation_id=operation_id) + operation = await save_and_deploy_resource( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operation_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template, + operation_id=deployment_operation_id) + except Exception as ex: + if not getattr(ex, "lease_retained", False): + await operation_repo.release_workspace_lease(review_workspace_id, deployment_operation_id) + raise return user_resource, operation @@ -598,10 +604,8 @@ async def delete_all_review_user_resources( for review_resource in airlock_request.reviewUserResources.values(): await send_airlock_workflow_message({ "workflow": "cleanup", - "user": user.model_dump(), - "review_workspace_id": review_resource.workspaceId, - "review_workspace_service_id": review_resource.workspaceServiceId, - "user_resource_id": review_resource.userResourceId, + "airlock_request_id": airlock_request.id, + "review_user_id": next(user_id for user_id, resource in airlock_request.reviewUserResources.items() if resource == review_resource), }) return [] diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 3a46fee452..6f444c2117 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -459,6 +459,79 @@ async def test_acquire_workspace_lease_fails_closed_when_reconciliation_fails(op assert exc.value.status_code == 409 +async def test_acquire_address_space_allocator_lease_creates_item(operations_repo): + from db.repositories.operations import ADDRESS_SPACE_ALLOCATOR_LEASE_ID + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + body = operations_repo._container.create_item.call_args.kwargs["body"] + assert body["id"] == ADDRESS_SPACE_ALLOCATOR_LEASE_ID + assert body["operationId"] == "op-1" + + +async def test_acquire_address_space_allocator_lease_is_idempotent(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "op-1", + "createdWhen": operations_repo.get_timestamp(), + }) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + + +async def test_acquire_address_space_allocator_lease_rejects_active_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_address_space_allocator_lease("op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_address_space_allocator_lease_reclaims_expired_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp() - 301, + "_etag": "old-etag", + }) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + operations_repo._container.replace_item.assert_awaited_once() + assert operations_repo._container.replace_item.call_args.kwargs["etag"] == "old-etag" + + +async def test_release_address_space_allocator_lease_only_releases_owner(operations_repo): + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_address_space_allocator_lease("op-1") + operations_repo.delete_item.assert_not_awaited() + + async def test_resource_has_active_operation_reconciles_stale_operation(operations_repo): now = operations_repo.get_timestamp() workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" @@ -477,8 +550,8 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio result = await operations_repo.resource_has_active_operation(workspace_id) assert result is False - operations_repo.update_item.assert_awaited_once() - saved_op = operations_repo.update_item.call_args[0][0] + assert operations_repo.update_item.await_count == 2 + saved_op = operations_repo.update_item.call_args_list[-1].args[0] assert saved_op.status == Status.DeploymentFailed assert saved_op.reconciled is True @@ -624,11 +697,11 @@ async def test_resource_has_active_operation_reconciles_stale_step_with_step_res has_active = await operations_repo.resource_has_active_operation(res_id) assert has_active is False - operations_repo.update_item.assert_awaited_once() - saved_op = operations_repo.update_item.call_args[0][0] + assert operations_repo.update_item.await_count == 2 + saved_op = operations_repo.update_item.call_args_list[-1].args[0] assert saved_op.status == Status.DeletingFailed assert saved_op.steps[0].status == Status.UpdatingFailed - operations_repo.update_item.assert_awaited_once_with(saved_op, release_lease=False) + assert all(call.kwargs == {"release_lease": False} for call in operations_repo.update_item.call_args_list) assert operations_repo._reconcile_resource_status.await_args_list[1].args == ( saved_op.steps[0].resourceId, Status.UpdatingFailed, diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py index 8866b9ebc1..31fd59b6bd 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -5,7 +5,7 @@ from db.errors import EntityDoesNotExist from models.domain.authentication import User -from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow +from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow, AirlockRequestType, AirlockReviewUserResource from service_bus.airlock_workflow import AirlockWorkflowUpdater @@ -19,6 +19,22 @@ def updater(): workflow.resource_history_repo = MagicMock() workflow.airlock_request_repo = MagicMock() workflow.workspace_repo = MagicMock() + workflow.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User.model_validate(user_payload()), + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")} + )) + workflow.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock(properties={ + "airlock_review_config": {"import": { + "import_vm_workspace_id": "review-workspace-id", + "import_vm_workspace_service_id": "service-id", + "import_vm_user_resource_template_name": "template"}} + })) return workflow @@ -36,10 +52,8 @@ async def test_process_message_completes_cleanup_when_resource_is_already_delete result = await updater.process_message(message({ "workflow": "cleanup", - "user": user_payload(), - "review_workspace_id": "workspace-id", - "review_workspace_service_id": "service-id", - "user_resource_id": "resource-id", + "airlock_request_id": "request-id", + "review_user_id": "user-id", })) assert result is True @@ -48,21 +62,13 @@ async def test_process_message_completes_cleanup_when_resource_is_already_delete @pytest.mark.asyncio async def test_process_message_redeploys_without_duplicate_uninstall(updater): updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) - updater.operations_repo.get_operation_by_id = AsyncMock() - updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock() - updater.workspace_repo.get_workspace_by_id = AsyncMock() payload = { "workflow": "redeploy", - "user": user_payload(), "airlock_request_id": "request-id", - "workspace_id": "workspace-id", - "review_workspace_id": "review-workspace-id", - "review_workspace_service_id": "service-id", - "user_resource_template_name": "template", - "user_resource_id": "resource-id", "operation_id": "delete-operation-id", "uninstall_started": True, + "redeploy_workflow_id": "request-id:user-id:delete-operation-id", } with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ @@ -81,10 +87,7 @@ async def test_process_message_acknowledges_unknown_workflow_without_side_effect patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()) as uninstall: result = await updater.process_message(message({ "workflow": "future-workflow", - "user": user_payload(), - "review_workspace_id": "workspace-id", - "review_workspace_service_id": "service-id", - "user_resource_id": "resource-id", + "airlock_request_id": "request-id", })) assert result is True @@ -93,6 +96,26 @@ async def test_process_message_acknowledges_unknown_workflow_without_side_effect uninstall.assert_not_awaited() +@pytest.mark.asyncio +async def test_process_message_rejects_forged_review_user_context(updater): + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message({ + "workflow": "cleanup", + "airlock_request_id": "request-id", + "review_user_id": "attacker-id", + "user": user_payload(), + "review_workspace_id": "attacker-workspace-id", + "review_workspace_service_id": "attacker-service-id", + "user_resource_id": "attacker-resource-id", + })) + + assert result is False + updater.user_resource_repo.get_user_resource_by_id.assert_not_called() + + @pytest.mark.asyncio async def test_process_message_recovers_deploying_redeploy_without_deploying_again(updater): workflow_id = "request-id:user-id" @@ -101,23 +124,29 @@ async def test_process_message_recovers_deploying_redeploy_without_deploying_aga return_value=MagicMock(id="replacement-resource-id")) updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User.model_validate(user_payload()), + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")}, redeployWorkflows={workflow_id: AirlockRedeployWorkflow( workflowId=workflow_id, phase="deploying", operationId="replacement-operation-id")} )) - updater.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock()) + updater.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock(properties={ + "airlock_review_config": {"import": { + "import_vm_workspace_id": "review-workspace-id", + "import_vm_workspace_service_id": "service-id", + "import_vm_user_resource_template_name": "template"}} + })) payload = { "workflow": "redeploy", "redeploy_workflow_id": workflow_id, - "user": user_payload(), "airlock_request_id": "request-id", - "workspace_id": "workspace-id", - "review_workspace_id": "review-workspace-id", - "review_workspace_service_id": "service-id", - "user_resource_template_name": "template", - "user_resource_id": "resource-id", "operation_id": "delete-operation-id", "uninstall_started": True, } From 25a9c7bb779e82468c460823b742fb8b5607bf4c Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 15:34:28 +0000 Subject: [PATCH 84/90] Preserve lease state and Airlock reviewer identity --- api_app/api/routes/resource_helpers.py | 6 ++++-- api_app/models/domain/airlock_request.py | 8 ++++++++ api_app/service_bus/airlock_workflow.py | 14 ++++++++++---- api_app/services/airlock.py | 6 ++++-- .../test_routes/test_resource_helpers.py | 19 +++++++++++++++++++ 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 0daac51a3d..a1a8e0f4af 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -203,12 +203,14 @@ async def send_uninstall_message( return operation except HTTPException: raise - except Exception: + except Exception as ex: logger.exception(f"Failed to send {resource_type} resource delete message") - raise HTTPException( + http_ex = HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, ) + http_ex.lease_retained = getattr(ex, "lease_retained", False) + raise http_ex from ex async def send_custom_action_message( diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index b87eb974ce..24381e9dcb 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -91,6 +91,14 @@ class AirlockReviewUserResource(AzureTREModel): workspaceId: str = Field(title="Workspace ID") workspaceServiceId: str = Field(title="Workspace Service ID") userResourceId: str = Field(title="User Resource ID") + reviewer: dict = Field(default_factory=dict, title="User who created the review resource") + + @field_validator("reviewer", mode="before") + @classmethod + def convert_reviewer_to_dict(cls, value): + if hasattr(value, "model_dump"): + return value.model_dump() + return value class AirlockRedeployWorkflow(AzureTREModel): diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index 514f902951..bb65b2c72c 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -82,9 +82,6 @@ async def process_message(self, message) -> bool: review_user_id = review_user_id or workflow_user_id if not review_user_id or review_user_id not in airlock_request.reviewUserResources: raise ValueError("Airlock workflow user is not associated with the request") - user = User.model_validate(airlock_request.createdBy) - if user.id != review_user_id: - raise ValueError("Airlock workflow user does not match the request owner") review_resource = airlock_request.reviewUserResources[review_user_id] review_workspace_id = review_resource.workspaceId review_workspace_service_id = review_resource.workspaceServiceId @@ -109,6 +106,14 @@ async def process_message(self, message) -> bool: logger.info("Airlock review resource %s is already deleted", user_resource_id) return True user_resource = None + reviewer = (review_resource.reviewer + or (user_resource.user if user_resource is not None else {}) + or airlock_request.createdBy) + if not reviewer: + raise ValueError("Airlock workflow reviewer identity is missing") + user = User.model_validate(reviewer) + if user.id != review_user_id: + raise ValueError("Airlock workflow user does not match the review resource") if workflow == "cleanup": workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( workspace_id=review_workspace_id, @@ -198,7 +203,8 @@ async def process_message(self, message) -> bool: review_user_resource=AirlockReviewUserResource( workspaceId=review_workspace_id, workspaceServiceId=review_workspace_service_id, - userResourceId=replacement_resource.id), + userResourceId=replacement_resource.id, + reviewer=user), redeploy_workflow=workflow_state) return True except (json.JSONDecodeError, KeyError, ValidationError): diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index fd88a18198..6be204c932 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -225,7 +225,8 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac review_user_resource=AirlockReviewUserResource( workspaceId=review_workspace_id, workspaceServiceId=review_workspace_service_id, - userResourceId=user_resource.id + userResourceId=user_resource.id, + reviewer=user )) logger.info(f"Airlock Request {updated_resource.id} updated to include {updated_resource.reviewUserResources}") @@ -256,7 +257,8 @@ async def _redeploy_review_vm_after_delete( review_user_resource=AirlockReviewUserResource( workspaceId=review_workspace_id, workspaceServiceId=review_workspace_service_id, - userResourceId=user_resource.id)) + userResourceId=user_resource.id, + reviewer=user)) except Exception: logger.exception("Failed to replace Airlock review VM after delete operation %s", delete_operation_id) diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index 562f217692..cf75981bec 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -264,6 +264,25 @@ async def test_send_uninstall_message_raises_503_on_service_bus_exception(self, assert ex.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + @patch("api.routes.resource_helpers.send_resource_request_message") + @pytest.mark.asyncio + async def test_send_uninstall_message_preserves_retained_lease(self, send_request_mock, resource_repo, resource_history_repo): + dispatch_error = Exception() + dispatch_error.lease_retained = True + send_request_mock.side_effect = dispatch_error + + with pytest.raises(HTTPException) as ex: + await send_uninstall_message( + resource=sample_resource(), + resource_repo=resource_repo, + operations_repo=OperationRepository(), + resource_type=ResourceType.Workspace, + resource_template_repo=None, + resource_history_repo=resource_history_repo, + user=create_test_user()) + + assert ex.value.lease_retained is True + @patch("api.routes.workspaces.ResourceTemplateRepository") @patch("service_bus.resource_request_sender.send_deployment_message") @pytest.mark.asyncio From 609f49c43f2ba0e3927a6b77d6ae4534e3d6e233 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 15:37:27 +0000 Subject: [PATCH 85/90] Fix Airlock redeployment workspace and retry handling --- api_app/service_bus/airlock_workflow.py | 59 +++++++++++++++---------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index bb65b2c72c..83e9cfae4f 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -72,6 +72,7 @@ async def process_message(self, message) -> bool: airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) review_user_id = payload.get("review_user_id") workflow_id = payload.get("redeploy_workflow_id") + workflow_state = None if workflow == "redeploy": if not workflow_id: raise ValueError("Redeploy workflow ID is required") @@ -86,15 +87,18 @@ async def process_message(self, message) -> bool: review_workspace_id = review_resource.workspaceId review_workspace_service_id = review_resource.workspaceServiceId user_resource_id = review_resource.userResourceId + redeploy_workspace_id = review_workspace_id + redeploy_workspace_service_id = review_workspace_service_id workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) if airlock_request.type == AirlockRequestType.Import: review_config = workspace.properties["airlock_review_config"]["import"] - review_workspace_id = review_config["import_vm_workspace_id"] - review_workspace_service_id = review_config["import_vm_workspace_service_id"] + redeploy_workspace_id = review_config["import_vm_workspace_id"] + redeploy_workspace_service_id = review_config["import_vm_workspace_service_id"] user_resource_template_name = review_config["import_vm_user_resource_template_name"] else: review_config = workspace.properties["airlock_review_config"]["export"] - review_workspace_service_id = review_config["export_vm_workspace_service_id"] + redeploy_workspace_id = workspace.id + redeploy_workspace_service_id = review_config["export_vm_workspace_service_id"] user_resource_template_name = review_config["export_vm_user_resource_template_name"] try: user_resource = await self.user_resource_repo.get_user_resource_by_id( @@ -126,7 +130,13 @@ async def process_message(self, message) -> bool: await wait_for_successful_operation(self.operations_repo, disable_operation.id) else: await wait_for_successful_operation(self.operations_repo, payload["operation_id"]) - if user_resource is not None and not payload.get("uninstall_started", False): + uninstall_started = payload.get("uninstall_started", False) + uninstall_started = uninstall_started or ( + workflow == "redeploy" + and workflow_state is not None + and workflow_state.phase == "deploying" + ) + if user_resource is not None and not uninstall_started: delete_operation = await send_uninstall_message( resource=user_resource, resource_repo=self.user_resource_repo, @@ -154,8 +164,8 @@ async def process_message(self, message) -> bool: resource_id=workflow_state.userResourceId) else: replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( - workspace_id=review_workspace_id, - service_id=review_workspace_service_id, + workspace_id=redeploy_workspace_id, + service_id=redeploy_workspace_service_id, workflow_id=workflow_id) except EntityDoesNotExist: pass @@ -171,38 +181,39 @@ async def process_message(self, message) -> bool: try: replacement_resource, _ = await _deploy_vm( airlock_request, user, workspace, - review_workspace_id, review_workspace_service_id, + redeploy_workspace_id, redeploy_workspace_service_id, user_resource_template_name, self.user_resource_repo, self.workspace_service_repo, self.operations_repo, self.resource_template_repo, self.resource_history_repo, workflow_id=workflow_id, operation_id=operation_id) - except Exception: - try: - await self.operations_repo.release_workspace_lease( - review_workspace_id, operation_id) - except Exception: - logger.exception("Failed to release failed Airlock redeploy lease") - replacement_operation_id = str(uuid.uuid4()) - await update_and_publish_event_airlock_request( - airlock_request, self.airlock_request_repo, user, workspace, - redeploy_workflow=AirlockRedeployWorkflow( - workflowId=workflow_id, - phase="deploying", - operationId=replacement_operation_id)) + except Exception as ex: + if not getattr(ex, "lease_retained", False): + try: + await self.operations_repo.release_workspace_lease( + redeploy_workspace_id, operation_id) + except Exception: + logger.exception("Failed to release failed Airlock redeploy lease") + replacement_operation_id = str(uuid.uuid4()) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + redeploy_workflow=AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId=replacement_operation_id)) raise workflow_state = AirlockRedeployWorkflow( workflowId=workflow_id, phase="completed", operationId=operation_id, - workspaceId=review_workspace_id, - workspaceServiceId=review_workspace_service_id, + workspaceId=redeploy_workspace_id, + workspaceServiceId=redeploy_workspace_service_id, userResourceId=replacement_resource.id) await update_and_publish_event_airlock_request( airlock_request, self.airlock_request_repo, user, workspace, review_user_resource=AirlockReviewUserResource( - workspaceId=review_workspace_id, - workspaceServiceId=review_workspace_service_id, + workspaceId=redeploy_workspace_id, + workspaceServiceId=redeploy_workspace_service_id, userResourceId=replacement_resource.id, reviewer=user), redeploy_workflow=workflow_state) From 06acaf37d192b36230053c816290b4ed60fccae3 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 16:21:35 +0000 Subject: [PATCH 86/90] Improve Airlock workflow retries and lease handling --- api_app/.env.sample | 1 + api_app/db/repositories/operations.py | 80 +++---------------- api_app/service_bus/airlock_workflow.py | 57 ++++++++++--- api_app/service_bus/helpers.py | 18 ++++- .../test_operation_repository.py | 55 +++++-------- .../test_service_bus/test_airlock_workflow.py | 60 +++++++++++++- core/terraform/json-to-env.sh | 4 + 7 files changed, 156 insertions(+), 119 deletions(-) diff --git a/api_app/.env.sample b/api_app/.env.sample index 0cf370e342..1ece601152 100644 --- a/api_app/.env.sample +++ b/api_app/.env.sample @@ -43,6 +43,7 @@ RESOURCE_GROUP_NAME=__CHANGE_ME__ SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE=__CHANGE_ME__ SERVICE_BUS_RESOURCE_REQUEST_QUEUE=workspacequeue SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE=deploymentstatus +SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE=airlock-workflow SERVICE_BUS_STEP_RESULT_QUEUE=airlock-step-result # Event grid configuration diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index ec38af5237..30712e710a 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -18,7 +18,7 @@ from services.logging import logger from db.errors import EntityDoesNotExist -from models.domain.operation import Operation, OperationStep, Status, get_failure_status_for_action +from models.domain.operation import Operation, OperationStep, Status # The resource processor auto-renews Porter session locks for up to 3600 seconds (1 hour) # during deployment execution (see resource_processor/vmss_porter/runner.py:68). @@ -214,37 +214,7 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) continue if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: - op_time = getattr(existing_op, "updatedWhen", 0.0) or getattr(existing_op, "createdWhen", 0.0) - if timestamp - op_time < WORKSPACE_LEASE_EXPIRY_SECONDS: - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - # Stale operation from interrupted dispatch; reconcile to failed status - try: - existing_op.status = get_failure_status_for_action(existing_op.action) - existing_op.message = "Operation timed out or was interrupted before completion" - existing_op.updatedWhen = timestamp - existing_op.reconciled = False - affected_step_resources = [] - if getattr(existing_op, "steps", None): - for step in existing_op.steps: - if not step.is_failure() and not step.is_success(): - step.status = get_failure_status_for_action(step.resourceAction or existing_op.action) - step.message = "Operation timed out or was interrupted before completion" - step.updatedWhen = timestamp - affected_step_resources.append((step.resourceId, step.status, step.message)) - update_call = self.update_item(existing_op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call - await self._reconcile_operation_resources(existing_op, affected_step_resources) - existing_op.reconciled = True - update_call = self.update_item(existing_op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call - await self.release_workspace_lease(workspace_id, existing_op.id) - except Exception as e: - logger.exception(f"Failed to reconcile stale operation {existing_op.id}: {e}") - raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - # Restart lease acquisition rather than replacing the now-reconciled lease document. - continue + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) except HTTPException: raise except (EntityDoesNotExist, CosmosResourceNotFoundError): @@ -294,6 +264,9 @@ async def release_workspace_lease(self, workspace_id: str, operation_id: Optiona except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, CosmosAccessConditionFailedError, TypeError): return + except Exception: + logger.exception("Failed to release workspace lease") + raise async def acquire_address_space_allocator_lease(self, operation_id: str) -> bool: if not hasattr(self, "_container") or self._container is None: @@ -355,6 +328,9 @@ async def release_address_space_allocator_lease(self, operation_id: Optional[str except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, CosmosAccessConditionFailedError, TypeError): return + except Exception: + logger.exception("Failed to release address-space allocator lease") + raise @staticmethod def operations_query(): @@ -571,49 +547,11 @@ async def resource_has_active_operation(self, resource_id: str, exclude_operatio if not operations: return False - timestamp = self.get_timestamp() has_active = False for op_dict in operations: op_status = op_dict.get("status") if isinstance(op_dict, dict) else getattr(op_dict, "status", None) if op_status is not None and op_status not in active_statuses: continue - op_time = ( - (op_dict.get("updatedWhen") or op_dict.get("createdWhen")) - if isinstance(op_dict, dict) - else (getattr(op_dict, "updatedWhen", None) or getattr(op_dict, "createdWhen", None)) - ) - # If timestamp is present and exceeds expiry, it is stale - if op_time is not None and (timestamp - op_time >= WORKSPACE_LEASE_EXPIRY_SECONDS): - # Stale active operation from interrupted dispatch; reconcile to failure status - try: - op = TypeAdapter(Operation).validate_python(op_dict) - op.status = get_failure_status_for_action(op.action) - op.message = "Operation timed out or was interrupted before completion" - op.updatedWhen = timestamp - op.reconciled = False - affected_step_resources = [] - if getattr(op, "steps", None): - for step in op.steps: - if not step.is_failure() and not step.is_success(): - step.status = get_failure_status_for_action(step.resourceAction or op.action) - step.message = "Operation timed out or was interrupted before completion" - step.updatedWhen = timestamp - affected_step_resources.append((step.resourceId, step.status, step.message)) - update_call = self.update_item(op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call - await self._reconcile_operation_resources(op, affected_step_resources) - op.reconciled = True - update_call = self.update_item(op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call - target_workspace_id = extract_workspace_id_from_resource_path(op.resourcePath) - if target_workspace_id: - await self.release_workspace_lease(target_workspace_id, op.id) - except Exception as e: - logger.exception(f"Failed to reconcile stale active operation {op_dict.get('id') if isinstance(op_dict, dict) else getattr(op_dict, 'id', None)}: {e}") - has_active = True - else: - has_active = True + has_active = True return has_active diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index 83e9cfae4f..cd54245451 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -4,6 +4,7 @@ from azure.servicebus.aio import AutoLockRenewer, ServiceBusClient from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError +from fastapi import HTTPException from pydantic import ValidationError from core import config, credentials @@ -19,9 +20,14 @@ from models.domain.authentication import User from models.domain.airlock_request import AirlockRequestType, AirlockReviewUserResource, AirlockRedeployWorkflow from models.domain.resource import ResourceType +from models.domain.operation import Status from services.airlock import _deploy_vm, wait_for_successful_operation, update_and_publish_event_airlock_request, disable_user_resource from api.routes.resource_helpers import send_uninstall_message from services.logging import logger +from resources import strings +from service_bus.helpers import send_airlock_workflow_message + +MAX_LEASE_CONTENTION_RETRY_DELAY_SECONDS = int(WORKSPACE_LEASE_EXPIRY_SECONDS) class AirlockWorkflowUpdater: @@ -63,6 +69,7 @@ async def receive_messages(self): await asyncio.sleep(10) async def process_message(self, message) -> bool: + payload = None try: payload = json.loads(str(message)) workflow = payload["workflow"] @@ -110,8 +117,13 @@ async def process_message(self, message) -> bool: logger.info("Airlock review resource %s is already deleted", user_resource_id) return True user_resource = None + review_reviewer = next( + (review.reviewer for review in (airlock_request.reviews or []) + if review.reviewer.get("id") == review_user_id), + {}) reviewer = (review_resource.reviewer or (user_resource.user if user_resource is not None else {}) + or review_reviewer or airlock_request.createdBy) if not reviewer: raise ValueError("Airlock workflow reviewer identity is missing") @@ -157,18 +169,31 @@ async def process_message(self, message) -> bool: operation_id = workflow_state.operationId if workflow_state is not None else str(uuid.uuid4()) if workflow_state is not None and workflow_state.phase == "deploying": try: - if workflow_state.userResourceId: - replacement_resource = await self.user_resource_repo.get_user_resource_by_id( - workspace_id=workflow_state.workspaceId, - service_id=workflow_state.workspaceServiceId, - resource_id=workflow_state.userResourceId) + replacement_operation_call = self.operations_repo.get_operation_by_id(operation_id) + replacement_operation = ( + await replacement_operation_call + if hasattr(replacement_operation_call, "__await__") + else replacement_operation_call + ) + if replacement_operation.status in { + Status.DeploymentFailed, Status.UpdatingFailed, + Status.DeletingFailed, Status.ActionFailed, + }: + operation_id = str(uuid.uuid4()) else: - replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( - workspace_id=redeploy_workspace_id, - service_id=redeploy_workspace_service_id, - workflow_id=workflow_id) + if workflow_state.userResourceId: + replacement_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=workflow_state.workspaceId, + service_id=workflow_state.workspaceServiceId, + resource_id=workflow_state.userResourceId) + else: + replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( + workspace_id=redeploy_workspace_id, + service_id=redeploy_workspace_service_id, + workflow_id=workflow_id) + await wait_for_successful_operation(self.operations_repo, operation_id) except EntityDoesNotExist: - pass + operation_id = str(uuid.uuid4()) if replacement_resource is None: workflow_state = AirlockRedeployWorkflow( @@ -221,6 +246,18 @@ async def process_message(self, message) -> bool: except (json.JSONDecodeError, KeyError, ValidationError): logger.exception("Invalid Airlock workflow message") return True + except HTTPException as ex: + if ex.status_code == 409 and ex.detail == strings.WORKSPACE_HAS_ACTIVE_OPERATION and payload is not None: + retry_count = int(payload.get("lease_contention_retry_count", 0)) + delay_seconds = min(2 ** retry_count * 30, MAX_LEASE_CONTENTION_RETRY_DELAY_SECONDS) + payload["lease_contention_retry_count"] = retry_count + 1 + await send_airlock_workflow_message(payload, delay_seconds=delay_seconds) + logger.info( + "Rescheduled Airlock workflow %s after workspace lease contention in %s seconds", + payload.get("airlock_request_id"), delay_seconds) + return True + logger.exception("Airlock workflow message failed") + return False except Exception: logger.exception("Airlock workflow message failed; it will be retried") return False diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index f01f9346e0..ceb1f8001c 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timedelta, UTC from azure.servicebus import ServiceBusMessage from azure.servicebus.aio import ServiceBusClient from pydantic import TypeAdapter @@ -42,8 +43,21 @@ async def send_deployment_message(content, correlation_id, session_id, action): await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) -async def send_airlock_workflow_message(content): - await _send_message(ServiceBusMessage(body=json.dumps(content)), config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) +async def send_airlock_workflow_message(content, delay_seconds: int = 0): + message = ServiceBusMessage(body=json.dumps(content)) + if delay_seconds <= 0: + await _send_message(message, config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + return + + async with credentials.get_credential_async_context() as credential: + service_bus_client = ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) + async with service_bus_client: + sender = service_bus_client.get_queue_sender(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + async with sender: + await sender.schedule_messages( + message, + scheduled_enqueue_time_utc=datetime.now(UTC) + timedelta(seconds=delay_seconds), + ) async def update_resource_for_step(operation_step: OperationStep, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, root_resource: Resource, step_resource: Resource, resource_to_update_id: str, primary_action: str, user: User) -> Resource: diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 6f444c2117..104e2975d3 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -372,12 +372,11 @@ async def test_release_workspace_lease_deletes_item_with_etag(operations_repo): operations_repo.delete_item.assert_awaited_once_with("lease_ws-1", etag="lease-etag", match_condition=MatchConditions.IfNotModified) -async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_reacquires_lease(operations_repo): +async def test_acquire_workspace_lease_rejects_existing_active_operation(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError + from fastapi import HTTPException operations_repo._container = MagicMock() - # 1st attempt: conflict on create, reads lease, reconciles stale op, restarts loop - # 2nd attempt: create succeeds because old lease was released by update_item - operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) now = operations_repo.get_timestamp() operations_repo.read_item_by_id = AsyncMock(return_value={ "id": "lease_ws-1", @@ -390,11 +389,10 @@ async def test_acquire_workspace_lease_reconciles_stale_active_operation_and_rea operations_repo.update_item = AsyncMock() operations_repo._reconcile_resource_status = AsyncMock() - res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") - assert res is True - assert operations_repo.update_item.await_count == 2 - assert stale_op.status == Status.DeploymentFailed - assert operations_repo._container.create_item.await_count == 2 + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + operations_repo.update_item.assert_not_awaited() async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(operations_repo): @@ -532,7 +530,7 @@ async def test_release_address_space_allocator_lease_only_releases_owner(operati operations_repo.delete_item.assert_not_awaited() -async def test_resource_has_active_operation_reconciles_stale_operation(operations_repo): +async def test_resource_has_active_operation_rejects_stale_timestamp_without_heartbeat(operations_repo): now = operations_repo.get_timestamp() workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" stale_op_dict = { @@ -549,11 +547,8 @@ async def test_resource_has_active_operation_reconciles_stale_operation(operatio operations_repo._reconcile_resource_status = AsyncMock() result = await operations_repo.resource_has_active_operation(workspace_id) - assert result is False - assert operations_repo.update_item.await_count == 2 - saved_op = operations_repo.update_item.call_args_list[-1].args[0] - assert saved_op.status == Status.DeploymentFailed - assert saved_op.reconciled is True + assert result is True + operations_repo.update_item.assert_not_awaited() async def test_resource_has_active_operation_fails_closed_when_reconciliation_fails(operations_repo): @@ -618,9 +613,10 @@ async def test_build_step_list_tolerates_missing_step_title(operations_repo): assert steps[0].templateStepId == "step-1" -async def test_reconcile_stale_operation_uses_step_resource_action(operations_repo): +async def test_acquire_workspace_lease_does_not_reconcile_active_steps_by_timestamp(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS + from fastapi import HTTPException operations_repo._container = MagicMock() operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) @@ -660,16 +656,13 @@ async def test_reconcile_stale_operation_uses_step_resource_action(operations_re operations_repo._container.delete_item = AsyncMock() operations_repo.resource_has_active_operation = AsyncMock(return_value=False) - await operations_repo.acquire_workspace_lease("ws-1", "new-op") - - assert operations_repo.update_item.await_count == 2 - saved_op = operations_repo.update_item.call_args[0][0] - assert saved_op.status == Status.DeletingFailed - assert saved_op.steps[0].status == Status.DeletingFailed - assert saved_op.steps[1].status == Status.UpdatingFailed + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "new-op") + assert exc.value.status_code == 409 + operations_repo.update_item.assert_not_awaited() -async def test_resource_has_active_operation_reconciles_stale_step_with_step_resource_action(operations_repo): +async def test_resource_has_active_operation_keeps_stale_pipeline_active(operations_repo): from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS stale_time = operations_repo.get_timestamp() - WORKSPACE_LEASE_EXPIRY_SECONDS - 100 @@ -696,14 +689,6 @@ async def test_resource_has_active_operation_reconciles_stale_step_with_step_res operations_repo._reconcile_resource_status = AsyncMock() has_active = await operations_repo.resource_has_active_operation(res_id) - assert has_active is False - assert operations_repo.update_item.await_count == 2 - saved_op = operations_repo.update_item.call_args_list[-1].args[0] - assert saved_op.status == Status.DeletingFailed - assert saved_op.steps[0].status == Status.UpdatingFailed - assert all(call.kwargs == {"release_lease": False} for call in operations_repo.update_item.call_args_list) - assert operations_repo._reconcile_resource_status.await_args_list[1].args == ( - saved_op.steps[0].resourceId, - Status.UpdatingFailed, - saved_op.steps[0].message, - ) + assert has_active is True + operations_repo.update_item.assert_not_awaited() + operations_repo._reconcile_resource_status.assert_not_awaited() diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py index 31fd59b6bd..d1f07cb255 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -1,11 +1,12 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +from fastapi import HTTPException import pytest from db.errors import EntityDoesNotExist from models.domain.authentication import User -from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow, AirlockRequestType, AirlockReviewUserResource +from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow, AirlockRequestType, AirlockReview, AirlockReviewDecision, AirlockReviewUserResource from service_bus.airlock_workflow import AirlockWorkflowUpdater @@ -59,6 +60,26 @@ async def test_process_message_completes_cleanup_when_resource_is_already_delete assert result is True +@pytest.mark.asyncio +async def test_process_message_reschedules_workspace_lease_contention(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(return_value=MagicMock(user=user_payload())) + updater.workspace_service_repo.get_workspace_service_by_id = AsyncMock() + lease_error = HTTPException(status_code=409, detail="Workspace has an active operation in progress") + + with patch("service_bus.airlock_workflow.disable_user_resource", new=AsyncMock(side_effect=lease_error)), \ + patch("service_bus.airlock_workflow.send_airlock_workflow_message", new=AsyncMock()) as schedule: + result = await updater.process_message(message({ + "workflow": "cleanup", + "airlock_request_id": "request-id", + "review_user_id": "user-id", + })) + + assert result is True + schedule.assert_awaited_once() + assert schedule.await_args.kwargs["delay_seconds"] == 30 + assert schedule.await_args.args[0]["lease_contention_retry_count"] == 1 + + @pytest.mark.asyncio async def test_process_message_redeploys_without_duplicate_uninstall(updater): updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) @@ -81,6 +102,43 @@ async def test_process_message_redeploys_without_duplicate_uninstall(updater): uninstall.assert_not_awaited() +@pytest.mark.asyncio +async def test_process_message_redeploys_legacy_resource_using_review_data(updater): + workflow_id = "request-id:user-id:delete-operation-id" + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + updater.user_resource_repo.get_user_resource_by_workflow_id = AsyncMock( + return_value=MagicMock(id="replacement-resource-id")) + updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User(id="requester-id", name="Requester", email="requester@example.com").model_dump(), + reviews=[AirlockReview( + id="review-id", + reviewDecision=AirlockReviewDecision.Approved, + reviewer=User.model_validate(user_payload()))], + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")} + )) + + payload = { + "workflow": "redeploy", + "redeploy_workflow_id": workflow_id, + "airlock_request_id": "request-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock(return_value=(MagicMock(id="new-resource-id"), MagicMock()))), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message(payload)) + + assert result is True + + @pytest.mark.asyncio async def test_process_message_acknowledges_unknown_workflow_without_side_effects(updater): with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()) as wait_for_operation, \ diff --git a/core/terraform/json-to-env.sh b/core/terraform/json-to-env.sh index 0360fc2cc4..1439c47672 100755 --- a/core/terraform/json-to-env.sh +++ b/core/terraform/json-to-env.sh @@ -53,6 +53,10 @@ jq -r ' "path": "service_bus_deployment_status_queue", "env_var": "SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE" }, + { + "path": "service_bus_airlock_workflow_queue", + "env_var": "SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE" + }, { "path": "service_bus_step_result_queue", "env_var": "SERVICE_BUS_STEP_RESULT_QUEUE" From 8e94364016991da379d3f43ddc46a9c09d0faa8c Mon Sep 17 00:00:00 2001 From: James Chapman <196318169+JC-wk@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:37:02 +0100 Subject: [PATCH 87/90] Fix await usage in VM deployment function Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- api_app/service_bus/airlock_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index cd54245451..c7397adbdc 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -204,13 +204,14 @@ async def process_message(self, message) -> bool: airlock_request, self.airlock_request_repo, user, workspace, redeploy_workflow=workflow_state) try: - replacement_resource, _ = await _deploy_vm( + replacement_resource, replacement_operation = await _deploy_vm( airlock_request, user, workspace, redeploy_workspace_id, redeploy_workspace_service_id, user_resource_template_name, self.user_resource_repo, self.workspace_service_repo, self.operations_repo, self.resource_template_repo, self.resource_history_repo, workflow_id=workflow_id, operation_id=operation_id) + await wait_for_successful_operation(self.operations_repo, replacement_operation.id) except Exception as ex: if not getattr(ex, "lease_retained", False): try: From 8128e6975275483bafe87af1466e978299c4fffa Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 16:40:55 +0000 Subject: [PATCH 88/90] Reconcile expired workspace operations --- api_app/db/repositories/operations.py | 53 +++++++++++++------ .../test_operation_repository.py | 33 +++++++----- 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 30712e710a..54fd2e5610 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -18,7 +18,7 @@ from services.logging import logger from db.errors import EntityDoesNotExist -from models.domain.operation import Operation, OperationStep, Status +from models.domain.operation import Operation, OperationStep, Status, get_failure_status_for_action # The resource processor auto-renews Porter session locks for up to 3600 seconds (1 hour) # during deployment execution (see resource_processor/vmss_porter/runner.py:68). @@ -124,6 +124,27 @@ async def _reconcile_operation_resources(self, operation: Operation, step_resour for resource_id, status, message in step_resources or []: await self._reconcile_resource_status(resource_id, status, message) + async def _reconcile_operation(self, operation, timestamp: float, mark_failed: bool = False): + affected_step_resources = [ + ( + step.resourceId, + get_failure_status_for_action(step.resourceAction) if mark_failed else step.status, + step.message, + ) + for step in getattr(operation, "steps", None) or [] + if step.resourceId and step.status + ] + if mark_failed: + operation.status = get_failure_status_for_action(operation.action) + operation.message = "Operation was reconciled after its workspace lease expired" + operation.reconciled = False + operation.updatedWhen = timestamp + await self.update_item(operation, release_lease=False) + await self._reconcile_operation_resources(operation, affected_step_resources) + operation.reconciled = True + operation.updatedWhen = timestamp + await self.update_item(operation, release_lease=False) + async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> bool: if not hasattr(self, "_container") or self._container is None: return True @@ -192,28 +213,26 @@ async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> } if existing_op and getattr(existing_op, "status", None) in terminal_statuses and not getattr(existing_op, "reconciled", False): try: - affected_step_resources = [ - (step.resourceId, step.status, step.message) - for step in getattr(existing_op, "steps", None) or [] - if step.resourceId and step.status - ] - existing_op.reconciled = False - existing_op.updatedWhen = timestamp - update_call = self.update_item(existing_op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call - await self._reconcile_operation_resources(existing_op, affected_step_resources) - existing_op.reconciled = True - existing_op.updatedWhen = timestamp - update_call = self.update_item(existing_op, release_lease=False) - if hasattr(update_call, "__await__"): - await update_call + await self._reconcile_operation(existing_op, timestamp) await self.release_workspace_lease(workspace_id, existing_op.id) except Exception as e: logger.exception(f"Failed to reconcile pending operation {existing_op.id}: {e}") raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) continue if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: + last_activity = max( + lease_created, + getattr(existing_op, "createdWhen", 0.0), + getattr(existing_op, "updatedWhen", 0.0), + ) + if timestamp - last_activity >= WORKSPACE_LEASE_EXPIRY_SECONDS: + try: + await self._reconcile_operation(existing_op, timestamp, mark_failed=True) + await self.release_workspace_lease(workspace_id, existing_op.id) + except Exception as e: + logger.exception(f"Failed to reconcile expired operation {existing_op.id}: {e}") + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + continue raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) except HTTPException: raise diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 104e2975d3..7fd3990ed0 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -372,9 +372,8 @@ async def test_release_workspace_lease_deletes_item_with_etag(operations_repo): operations_repo.delete_item.assert_awaited_once_with("lease_ws-1", etag="lease-etag", match_condition=MatchConditions.IfNotModified) -async def test_acquire_workspace_lease_rejects_existing_active_operation(operations_repo): +async def test_acquire_workspace_lease_reconciles_expired_active_operation(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError - from fastapi import HTTPException operations_repo._container = MagicMock() operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) now = operations_repo.get_timestamp() @@ -389,10 +388,10 @@ async def test_acquire_workspace_lease_rejects_existing_active_operation(operati operations_repo.update_item = AsyncMock() operations_repo._reconcile_resource_status = AsyncMock() - with pytest.raises(HTTPException) as exc: - await operations_repo.acquire_workspace_lease("ws-1", "op-1") - assert exc.value.status_code == 409 - operations_repo.update_item.assert_not_awaited() + assert await operations_repo.acquire_workspace_lease("ws-1", "op-1") is True + assert stale_op.status == Status.DeploymentFailed + assert stale_op.reconciled is True + operations_repo.update_item.assert_awaited() async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(operations_repo): @@ -613,10 +612,9 @@ async def test_build_step_list_tolerates_missing_step_title(operations_repo): assert steps[0].templateStepId == "step-1" -async def test_acquire_workspace_lease_does_not_reconcile_active_steps_by_timestamp(operations_repo): +async def test_acquire_workspace_lease_reconciles_expired_active_steps(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS - from fastapi import HTTPException operations_repo._container = MagicMock() operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) @@ -631,12 +629,14 @@ async def test_acquire_workspace_lease_does_not_reconcile_active_steps_by_timest step1 = OperationStep( id="step-1", templateStepId="main", + resourceId="step-1-resource", resourceAction="uninstall", status=Status.Deleting, ) step2 = OperationStep( id="step-2", templateStepId="cleanup", + resourceId="step-2-resource", resourceAction="upgrade", status=Status.Updating, ) @@ -656,10 +656,19 @@ async def test_acquire_workspace_lease_does_not_reconcile_active_steps_by_timest operations_repo._container.delete_item = AsyncMock() operations_repo.resource_has_active_operation = AsyncMock(return_value=False) - with pytest.raises(HTTPException) as exc: - await operations_repo.acquire_workspace_lease("ws-1", "new-op") - assert exc.value.status_code == 409 - operations_repo.update_item.assert_not_awaited() + assert await operations_repo.acquire_workspace_lease("ws-1", "new-op") is True + assert stale_op.status == Status.DeletingFailed + assert stale_op.reconciled is True + assert operations_repo.update_item.await_count == 2 + operations_repo._reconcile_resource_status.assert_any_await( + "ws-1", Status.DeletingFailed, "Operation was reconciled after its workspace lease expired" + ) + operations_repo._reconcile_resource_status.assert_any_await( + "step-1-resource", Status.DeletingFailed, "" + ) + operations_repo._reconcile_resource_status.assert_any_await( + "step-2-resource", Status.UpdatingFailed, "" + ) async def test_resource_has_active_operation_keeps_stale_pipeline_active(operations_repo): From 6e33abb3761bb0842b208e48301eea220f12b33f Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 17:15:03 +0000 Subject: [PATCH 89/90] Enhance Airlock workflow processing and improve operation reconciliation logic --- api_app/db/repositories/operations.py | 16 ++++- api_app/service_bus/airlock_workflow.py | 71 ++++++++++++------- .../test_service_bus/test_airlock_workflow.py | 8 ++- 3 files changed, 67 insertions(+), 28 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 54fd2e5610..a8580f56d1 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -125,10 +125,24 @@ async def _reconcile_operation_resources(self, operation: Operation, step_resour await self._reconcile_resource_status(resource_id, status, message) async def _reconcile_operation(self, operation, timestamp: float, mark_failed: bool = False): + terminal_statuses = { + Status.Deployed, + Status.DeploymentFailed, + Status.Deleted, + Status.DeletingFailed, + Status.Updated, + Status.UpdatingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + } + if mark_failed: + for step in getattr(operation, "steps", None) or []: + if step.status not in terminal_statuses: + step.status = get_failure_status_for_action(step.resourceAction) affected_step_resources = [ ( step.resourceId, - get_failure_status_for_action(step.resourceAction) if mark_failed else step.status, + step.status, step.message, ) for step in getattr(operation, "steps", None) or [] diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index c7397adbdc..fa556fca24 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -28,6 +28,7 @@ from service_bus.helpers import send_airlock_workflow_message MAX_LEASE_CONTENTION_RETRY_DELAY_SECONDS = int(WORKSPACE_LEASE_EXPIRY_SECONDS) +MAX_CONCURRENT_AIRLOCK_WORKFLOWS = 10 class AirlockWorkflowUpdater: @@ -41,23 +42,43 @@ async def init_repos(self): self.workspace_repo = await WorkspaceRepository.create() async def receive_messages(self): + processing_tasks = set() + + async def process_received_message(message): + try: + async with AutoLockRenewer() as renewer: + renewer.register( + receiver, + message, + max_lock_renewal_duration=3 * WORKSPACE_LEASE_EXPIRY_SECONDS) + if await self.process_message(message): + await receiver.complete_message(message) + else: + await receiver.abandon_message(message) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Airlock workflow message acknowledgement failed") + while True: try: async with credentials.get_credential_async_context() as credential: async with ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) as client: receiver = client.get_queue_receiver(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) async with receiver: - messages = await receiver.receive_messages(max_message_count=1, max_wait_time=1) - for message in messages: - async with AutoLockRenewer() as renewer: - renewer.register( - receiver, - message, - max_lock_renewal_duration=3 * WORKSPACE_LEASE_EXPIRY_SECONDS) - if await self.process_message(message): - await receiver.complete_message(message) - else: - await receiver.abandon_message(message) + while True: + available_slots = MAX_CONCURRENT_AIRLOCK_WORKFLOWS - len(processing_tasks) + if available_slots <= 0: + await asyncio.sleep(1) + processing_tasks = {task for task in processing_tasks if not task.done()} + continue + messages = await receiver.receive_messages( + max_message_count=available_slots, + max_wait_time=1) + for message in messages: + task = asyncio.create_task(process_received_message(message)) + processing_tasks.add(task) + processing_tasks = {task for task in processing_tasks if not task.done()} except OperationTimeoutError: logger.debug("No Airlock workflow messages available") except ServiceBusConnectionError: @@ -96,17 +117,19 @@ async def process_message(self, message) -> bool: user_resource_id = review_resource.userResourceId redeploy_workspace_id = review_workspace_id redeploy_workspace_service_id = review_workspace_service_id - workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) - if airlock_request.type == AirlockRequestType.Import: - review_config = workspace.properties["airlock_review_config"]["import"] - redeploy_workspace_id = review_config["import_vm_workspace_id"] - redeploy_workspace_service_id = review_config["import_vm_workspace_service_id"] - user_resource_template_name = review_config["import_vm_user_resource_template_name"] - else: - review_config = workspace.properties["airlock_review_config"]["export"] - redeploy_workspace_id = workspace.id - redeploy_workspace_service_id = review_config["export_vm_workspace_service_id"] - user_resource_template_name = review_config["export_vm_user_resource_template_name"] + workspace = None + if workflow == "redeploy": + workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) + if airlock_request.type == AirlockRequestType.Import: + review_config = workspace.properties["airlock_review_config"]["import"] + redeploy_workspace_id = review_config["import_vm_workspace_id"] + redeploy_workspace_service_id = review_config["import_vm_workspace_service_id"] + user_resource_template_name = review_config["import_vm_user_resource_template_name"] + else: + review_config = workspace.properties["airlock_review_config"]["export"] + redeploy_workspace_id = workspace.id + redeploy_workspace_service_id = review_config["export_vm_workspace_service_id"] + user_resource_template_name = review_config["export_vm_user_resource_template_name"] try: user_resource = await self.user_resource_repo.get_user_resource_by_id( workspace_id=review_workspace_id, @@ -187,10 +210,10 @@ async def process_message(self, message) -> bool: service_id=workflow_state.workspaceServiceId, resource_id=workflow_state.userResourceId) else: - replacement_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( + replacement_resource = await self.user_resource_repo.get_user_resource_by_id( workspace_id=redeploy_workspace_id, service_id=redeploy_workspace_service_id, - workflow_id=workflow_id) + resource_id=replacement_operation.resourceId) await wait_for_successful_operation(self.operations_repo, operation_id) except EntityDoesNotExist: operation_id = str(uuid.uuid4()) diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py index d1f07cb255..c9232436ca 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -58,6 +58,7 @@ async def test_process_message_completes_cleanup_when_resource_is_already_delete })) assert result is True + updater.workspace_repo.get_workspace_by_id.assert_not_awaited() @pytest.mark.asyncio @@ -177,9 +178,10 @@ async def test_process_message_rejects_forged_review_user_context(updater): @pytest.mark.asyncio async def test_process_message_recovers_deploying_redeploy_without_deploying_again(updater): workflow_id = "request-id:user-id" - updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) - updater.user_resource_repo.get_user_resource_by_workflow_id = AsyncMock( - return_value=MagicMock(id="replacement-resource-id")) + updater.user_resource_repo.get_user_resource_by_id = AsyncMock( + side_effect=[EntityDoesNotExist, MagicMock(id="replacement-resource-id")]) + updater.operations_repo.get_operation_by_id = AsyncMock( + return_value=MagicMock(status="deploying", resourceId="replacement-resource-id")) updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( id="request-id", workspaceId="workspace-id", From 672526b2fae1256e6996e137fe89a71cb0657370 Mon Sep 17 00:00:00 2001 From: James Chapman Date: Thu, 10 Sep 2026 17:36:48 +0000 Subject: [PATCH 90/90] Improve address space lease and Airlock redeploy handling --- api_app/api/routes/workspaces.py | 1 + api_app/db/repositories/operations.py | 24 +++++++++++- api_app/service_bus/airlock_workflow.py | 11 ++++++ .../test_api/test_routes/test_workspaces.py | 3 +- .../test_operation_repository.py | 37 +++++++++++++++++++ 5 files changed, 73 insertions(+), 3 deletions(-) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index b4be108c58..0b25a53915 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -148,6 +148,7 @@ async def create_workspace(workspace_create: WorkspaceInCreate, response: Respon raise try: + await operations_repo.assert_address_space_allocator_lease(address_space_operation_id) operation = await save_and_deploy_resource( resource=workspace, resource_repo=workspace_repo, diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index a8580f56d1..cc51b7ea4d 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -120,8 +120,12 @@ async def _reconcile_resource_status(self, resource_id: str, status: Status, mes await resource_repo.update_item(resource) async def _reconcile_operation_resources(self, operation: Operation, step_resources=None): - await self._reconcile_resource_status(operation.resourceId, operation.status, operation.message) - for resource_id, status, message in step_resources or []: + resource_updates = [(operation.resourceId, operation.status, operation.message)] + resource_updates.extend(step_resources or []) + collapsed_updates = {} + for resource_id, status, message in resource_updates: + collapsed_updates[resource_id] = (status, message) + for resource_id, (status, message) in collapsed_updates.items(): await self._reconcile_resource_status(resource_id, status, message) async def _reconcile_operation(self, operation, timestamp: float, mark_failed: bool = False): @@ -345,6 +349,22 @@ async def acquire_address_space_allocator_lease(self, operation_id: str) -> bool except (CosmosAccessConditionFailedError, ResourceExistsError, CosmosResourceNotFoundError, ResourceNotFoundError): raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + async def assert_address_space_allocator_lease(self, operation_id: str) -> None: + if not hasattr(self, "_container") or self._container is None: + return + try: + lease = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + lease = await lease if hasattr(lease, "__await__") else lease + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + if ( + not isinstance(lease, dict) + or lease.get("operationId") != operation_id + or self.get_timestamp() - lease.get("createdWhen", 0.0) >= ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS + ): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + async def release_address_space_allocator_lease(self, operation_id: Optional[str] = None) -> None: if not hasattr(self, "_container") or self._container is None: return diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py index fa556fca24..31dc1ec384 100644 --- a/api_app/service_bus/airlock_workflow.py +++ b/api_app/service_bus/airlock_workflow.py @@ -216,6 +216,17 @@ async def process_message(self, message) -> bool: resource_id=replacement_operation.resourceId) await wait_for_successful_operation(self.operations_repo, operation_id) except EntityDoesNotExist: + try: + orphaned_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( + workspace_id=redeploy_workspace_id, + service_id=redeploy_workspace_service_id, + workflow_id=workflow_id) + await self.user_resource_repo.delete_item(orphaned_resource.id) + logger.warning( + "Removed orphaned Airlock redeploy resource %s before retrying workflow %s", + orphaned_resource.id, workflow_id) + except EntityDoesNotExist: + pass operation_id = str(uuid.uuid4()) if replacement_resource is None: diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 2fafa0250a..d17f1d30cb 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -39,7 +39,8 @@ @pytest.fixture(autouse=True) def mock_workspace_lease_release(): - with patch("api.routes.workspaces.OperationRepository.release_workspace_lease", new=AsyncMock()): + with patch("api.routes.workspaces.OperationRepository.release_workspace_lease", new=AsyncMock()), \ + patch("api.routes.workspaces.OperationRepository.assert_address_space_allocator_lease", new=AsyncMock()): yield diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 7fd3990ed0..c122adcb95 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -468,6 +468,43 @@ async def test_acquire_address_space_allocator_lease_creates_item(operations_rep assert body["operationId"] == "op-1" +async def test_assert_address_space_allocator_lease_requires_current_owner(operations_repo): + from db.repositories.operations import ADDRESS_SPACE_ALLOCATOR_LEASE_ID + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + + with pytest.raises(HTTPException) as exc: + await operations_repo.assert_address_space_allocator_lease("op-1") + + assert exc.value.status_code == 409 + + +async def test_reconcile_operation_resources_uses_last_status_per_resource(operations_repo): + operation = MagicMock( + resourceId="workspace-id", + status=Status.Updated, + message="workspace update", + ) + operations_repo._reconcile_resource_status = AsyncMock() + + await operations_repo._reconcile_operation_resources(operation, [ + ("workspace-id", Status.Updated, "first workspace update"), + ("workspace-id", Status.UpdatingFailed, "final workspace update"), + ("service-id", Status.DeletingFailed, "service cleanup"), + ]) + + assert [call.args for call in operations_repo._reconcile_resource_status.await_args_list] == [ + ("workspace-id", Status.UpdatingFailed, "final workspace update"), + ("service-id", Status.DeletingFailed, "service cleanup"), + ] + + async def test_acquire_address_space_allocator_lease_is_idempotent(operations_repo): from azure.cosmos.exceptions import CosmosResourceExistsError