diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9e5653c6..5bfa621d71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ENHANCEMENTS: BUG FIXES: * Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055)) +* A workspace service's allocated `address_space` will now be removed from its parent workspace's `address_spaces` list when the service is deleted, preventing IP address range exhaustion, and workspace-level operation leasing ensures safe serialization of concurrent workspace mutations ([#4727](https://github.com/microsoft/AzureTRE/issues/4727)) * Fix pytest warnings ([#5066](https://github.com/microsoft/AzureTRE/issues/5066)) * Fix inconsistent ServiceBusClient lifecycle management in deployment_status_updater.py, airlock_request_status_update.py, and runner.py to prevent connection socket and AMQP channel leaks ([#4930](https://github.com/microsoft/AzureTRE/pull/4930)) diff --git a/api_app/.env.sample b/api_app/.env.sample index 0cf370e342..1ece601152 100644 --- a/api_app/.env.sample +++ b/api_app/.env.sample @@ -43,6 +43,7 @@ RESOURCE_GROUP_NAME=__CHANGE_ME__ SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE=__CHANGE_ME__ SERVICE_BUS_RESOURCE_REQUEST_QUEUE=workspacequeue SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE=deploymentstatus +SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE=airlock-workflow SERVICE_BUS_STEP_RESULT_QUEUE=airlock-step-result # Event grid configuration diff --git a/api_app/_version.py b/api_app/_version.py index 5252f582fe..29dd6604f8 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.9" +__version__ = "0.26.10" diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index eb9262663d..a1a8e0f4af 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -63,6 +63,7 @@ async def save_and_deploy_resource( resource_history_repo: ResourceHistoryRepository, user: User, resource_template: ResourceTemplate, + operation_id: Optional[str] = None, ) -> Operation: try: resource.user = user.model_dump() @@ -82,6 +83,9 @@ async def save_and_deploy_resource( ) try: + kwargs = {} + if operation_id is not None: + kwargs["operation_id"] = operation_id operation = await send_resource_request_message( resource=resource, operations_repo=operations_repo, @@ -90,15 +94,24 @@ async def save_and_deploy_resource( resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Install, + **kwargs, ) return operation - except Exception: - await resource_repo.delete_item(resource.id) + except HTTPException as ex: + if not getattr(ex, "lease_retained", False): + await resource_repo.delete_item(resource.id) + raise + except Exception as ex: + lease_retained = getattr(ex, "lease_retained", False) + if not lease_retained: + await resource_repo.delete_item(resource.id) logger.exception("Failed send resource request message") - raise HTTPException( + http_ex = HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, ) + setattr(http_ex, "lease_retained", lease_retained) + raise http_ex from ex def mask_sensitive_properties( @@ -169,9 +182,13 @@ async def send_uninstall_message( resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, user: User, - is_cascade: str = False + is_cascade: str = False, + operation_id: Optional[str] = None, ) -> Operation: try: + operation_kwargs = {} + if operation_id is not None: + operation_kwargs["operation_id"] = operation_id operation = await send_resource_request_message( resource=resource, operations_repo=operations_repo, @@ -180,15 +197,20 @@ async def send_uninstall_message( resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.UnInstall, - is_cascade=is_cascade + is_cascade=is_cascade, + **operation_kwargs, ) return operation - except Exception: + except HTTPException: + raise + except Exception as ex: logger.exception(f"Failed to send {resource_type} resource delete message") - raise HTTPException( + http_ex = HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, ) + http_ex.lease_retained = getattr(ex, "lease_retained", False) + raise http_ex from ex async def send_custom_action_message( @@ -234,6 +256,8 @@ async def send_custom_action_message( action=custom_action, ) return operation + except HTTPException: + raise except Exception: logger.exception(f"Failed to send {resource_type} resource custom action message") raise HTTPException( @@ -296,17 +320,28 @@ async def update_user_resource( operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> Operation: - patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update) - operation = await send_resource_request_message( - resource=patched_user_resource, - operations_repo=operations_repo, - resource_repo=user_resource_repo, - user=user, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - action=RequestAction.Upgrade) - - return operation + target_workspace_id = getattr(workspace_service, "workspaceId", None) or getattr(user_resource, "workspaceId", None) + operation_id = operations_repo.create_operation_id() + if target_workspace_id and hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(target_workspace_id, operation_id) + + try: + patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update) + operation = await send_resource_request_message( + resource=patched_user_resource, + operations_repo=operations_repo, + resource_repo=user_resource_repo, + user=user, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + action=RequestAction.Upgrade, + operation_id=operation_id) + + return operation + except Exception as ex: + if not getattr(ex, "lease_retained", False) and target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(target_workspace_id, operation_id) + raise async def enrich_resource_with_available_upgrades(resource: Resource, resource_template_repo: ResourceTemplateRepository): diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index 04ea2d6513..0b25a53915 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -43,6 +43,29 @@ user_resources_workspace_router = APIRouter(dependencies=[Depends(require_workspace_owner_or_researcher_or_airlock_manager)]) +async def _rollback_address_space(workspace_repo, workspace_id, allocated_space, resource_template_repo, resource_history_repo, user): + max_rollback_attempts = 3 + for attempt in range(max_rollback_attempts): + try: + latest_workspace = await workspace_repo.get_workspace_by_id(workspace_id) + current_spaces = latest_workspace.properties.get("address_spaces", []) + if allocated_space in current_spaces: + updated_spaces = [s for s in current_spaces if s != allocated_space] + rollback_patch = ResourcePatch(properties={"address_spaces": updated_spaces}) + await workspace_repo.patch_workspace( + latest_workspace, rollback_patch, latest_workspace.etag, + resource_template_repo, resource_history_repo, user, False + ) + return + except CosmosAccessConditionFailedError: + if attempt == max_rollback_attempts - 1: + logger.exception("Failed to rollback allocated address space due to repeated ETag conflicts") + raise + except Exception: + logger.exception("Failed to rollback allocated address space on workspace") + raise + + def validate_user_has_valid_role_for_user_resource(user, user_resource): if "WorkspaceOwner" in user.roles: return @@ -95,40 +118,68 @@ async def retrieve_workspace_scope_id_by_workspace_id(workspace=Depends(get_work @workspaces_core_router.post("/workspaces", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_CREATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def create_workspace(workspace_create: WorkspaceInCreate, response: Response, user=Depends(require_tre_admin), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + address_space_operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_address_space_allocator_lease(address_space_operation_id) try: # TODO: This requires Directory.ReadAll ( Application.Read.All ) to be enabled in the Azure AD application to enable a users workspaces to be listed. This should be made optional. auth_info = extract_auth_information(workspace_create.properties) workspace, resource_template = await workspace_repo.create_workspace_item(workspace_create, auth_info, user.id, user.roles) except (ValidationError, ValueError) as e: logger.exception("Failed to create workspace model instance") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except StorageAccountNameGenerationTimeout: logger.exception("Storage name availability check timed out") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Storage name availability check timed out. Please try again.") except StorageAccountNameCheckFailed: logger.exception("Storage name availability check failed") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Storage name availability check failed. Please try again.") except UserNotAuthorizedToUseTemplate as e: logger.exception("User not authorized to use template") + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) except InvalidInput as e: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(e)) + except Exception: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) + raise - operation = await save_and_deploy_resource( - resource=workspace, - resource_repo=workspace_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) - response.headers["Location"] = construct_location_header(operation) - - return OperationInResponse(operation=operation) + try: + await operations_repo.assert_address_space_allocator_lease(address_space_operation_id) + operation = await save_and_deploy_resource( + resource=workspace, + resource_repo=workspace_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template) + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + finally: + await operations_repo.release_address_space_allocator_lease(address_space_operation_id) @workspaces_core_router.patch("/workspaces/{workspace_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def patch_workspace(resource_patch: ResourcePatch, response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), workspace_repo: WorkspaceRepository = Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + try: + workspace = await workspace_repo.get_workspace_by_id(workspace.id) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace reload failed") + raise try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -143,20 +194,38 @@ async def patch_workspace(resource_patch: ResourcePatch, response: Response, use resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Upgrade, - is_cascade=is_disablement) + is_cascade=is_disablement, + operation_id=operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) - except CosmosAccessConditionFailedError: + except CosmosAccessConditionFailedError as err: + if not getattr(err, "lease_retained", False) and hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @workspaces_core_router.delete("/workspaces/{workspace_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def delete_workspace(response: Response, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) if await delete_validation(workspace, workspace_repo): operation = await send_uninstall_message( resource=workspace, @@ -175,6 +244,8 @@ async def delete_workspace(response: Response, user=Depends(require_tre_admin), @workspaces_core_router.post("/workspaces/{workspace_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE, dependencies=[Depends(require_tre_admin)]) async def invoke_action_on_workspace(response: Response, action: str, user=Depends(require_tre_admin), workspace=Depends(get_workspace_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace, resource_repo=workspace_repo, @@ -243,44 +314,89 @@ async def retrieve_workspace_service_by_id(workspace_service=Depends(get_workspa @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_CREATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def create_workspace_service(response: Response, workspace_service_input: WorkspaceServiceInCreate, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), workspace=Depends(get_deployed_workspace_by_id_from_path)) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + try: + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace reload failed") + raise + + address_space_added = False + address_space_lease_acquired = False try: - workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) - except (ValidationError, ValueError) as e: - logger.exception("Failed create workspace service model instance") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except UserNotAuthorizedToUseTemplate as e: - logger.exception("User not authorized to use template") - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) - - # Get the workspace subscription id (if set) - if workspace.properties.get("workspace_subscription_id"): - workspace_service.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] - - # if template has address_space get an address space - if resource_template.properties.get("address_space"): - # check workspace has address_spaces property - if not workspace.properties.get("address_spaces"): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) - workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) - workspace_patch = ResourcePatch() - workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} - # IP address allocation is managed by the API. Ideally this request would happen as a result of the workspace - # service deployment via the reosurce processor. there is no such functionality so the database is being - # updated directly, and an "update" on the workspace is called by the workspace service pipeline. try: - await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) - except CosmosAccessConditionFailedError: - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) - - operation = await save_and_deploy_resource( - resource=workspace_service, - resource_repo=workspace_service_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) + workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item(workspace_service_input, workspace.id, user.roles) + except (ValidationError, ValueError) as e: + logger.exception("Failed create workspace service model instance") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except UserNotAuthorizedToUseTemplate as e: + logger.exception("User not authorized to use template") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + + # Get the workspace subscription id (if set) + if workspace.properties.get("workspace_subscription_id"): + workspace_service.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] + + # if template has address_space get an address space + if resource_template.properties.get("address_space"): + await operations_repo.acquire_address_space_allocator_lease(operation_id) + address_space_lease_acquired = True + # check workspace has address_spaces property + if not workspace.properties.get("address_spaces"): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY) + workspace_service.properties["address_space"] = await workspace_repo.get_address_space_based_on_size(workspace_service_input.properties) + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]} + # IP address allocation is managed by the API. Ideally this request would happen as a result of the workspace + # service deployment via the reosurce processor. there is no such functionality so the database is being + # updated directly, and an "update" on the workspace is called by the workspace service pipeline. + try: + await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False) + address_space_added = True + except CosmosAccessConditionFailedError: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) + + operation = await save_and_deploy_resource( + resource=workspace_service, + resource_repo=workspace_service_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template, + operation_id=operation_id) + except Exception as err: + if not getattr(err, "lease_retained", False): + try: + if address_space_added and 'workspace_service' in locals() and workspace_service.properties.get("address_space"): + await _rollback_address_space( + workspace_repo, workspace.id, workspace_service.properties["address_space"], + resource_template_repo, resource_history_repo, user + ) + finally: + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + if getattr(err, "lease_retained", False): + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) + raise + if address_space_lease_acquired: + await operations_repo.release_address_space_allocator_lease(operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) @@ -288,6 +404,22 @@ async def create_workspace_service(response: Response, workspace_service_input: @workspace_services_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner), Depends(get_workspace_by_id_from_path)]) async def patch_workspace_service(resource_patch: ResourcePatch, response: Response, user=Depends(require_workspace_owner), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), etag: str = Header(...), force_version_update: bool = False) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace_service.workspaceId, operation_id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace_service.workspaceId, workspace_service.id) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after workspace service reload failed") + raise try: is_disablement = resource_patch.isEnabled is not None and not resource_patch.isEnabled if is_disablement: @@ -301,37 +433,66 @@ async def patch_workspace_service(resource_patch: ResourcePatch, response: Respo resource_template_repo=resource_template_repo, resource_history_repo=resource_history_repo, action=RequestAction.Upgrade, - is_cascade=is_disablement) + is_cascade=is_disablement, + operation_id=operation_id) response.headers["Location"] = construct_location_header(operation) return OperationInResponse(operation=operation) - except CosmosAccessConditionFailedError: + except CosmosAccessConditionFailedError as err: + if not getattr(err, "lease_retained", False) and hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace_service.workspaceId, operation_id) + raise @workspace_services_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}", response_model=OperationInResponse, name=strings.API_DELETE_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) -async def delete_workspace_service(response: Response, user=Depends(require_workspace_owner), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: - if await delete_validation(workspace_service, workspace_service_repo): - operation = await send_uninstall_message( - resource=workspace_service, - resource_repo=workspace_service_repo, - operations_repo=operations_repo, - resource_type=ResourceType.WorkspaceService, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - is_cascade=True) - - response.headers["Location"] = construct_location_header(operation) - - return OperationInResponse(operation=operation) +async def delete_workspace_service(response: Response, user=Depends(require_workspace_owner), workspace=Depends(get_workspace_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), workspace_repo=Depends(get_repository(WorkspaceRepository)), user_resource_repo=Depends(get_repository(UserResourceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + workspace.id, workspace_service.id) + if await operations_repo.resource_has_active_operation(workspace.id, exclude_operation_id=operation_id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + if await delete_validation(workspace_service, workspace_service_repo): + operation = await send_uninstall_message( + resource=workspace_service, + resource_repo=workspace_service_repo, + operations_repo=operations_repo, + resource_type=ResourceType.WorkspaceService, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + is_cascade=True, + operation_id=operation_id) + + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + except Exception as err: + if not getattr(err, "lease_retained", False): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @workspace_services_workspace_router.post("/workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, dependencies=[Depends(require_workspace_owner)]) async def invoke_action_on_workspace_service(response: Response, action: str, user=Depends(require_workspace_owner), workspace_service=Depends(get_workspace_service_by_id_from_path), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace_service.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) operation = await send_custom_action_message( resource=workspace_service, resource_repo=workspace_service_repo, @@ -405,56 +566,86 @@ async def create_user_resource( response: Response, user_resource_create: UserResourceInCreate, user_resource_repo=Depends(get_repository(UserResourceRepository)), + workspace_repo=Depends(get_repository(WorkspaceRepository)), + workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), user=Depends(require_workspace_owner_or_researcher_or_airlock_manager), workspace=Depends(get_deployed_workspace_by_id_from_path), workspace_service=Depends(get_deployed_workspace_service_by_id_from_path)) -> OperationInResponse: + if await operations_repo.resource_has_active_operation(workspace.id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - owner_id: str = None - - # Check for assign_to_another_user logic - if ( - hasattr(user_resource_create, "properties") - and isinstance(user_resource_create.properties, dict) - and user_resource_create.properties.get("assign_to_another_user") is True - ): - if user_resource_create.properties.get("owner_id"): - owner_id = user_resource_create.properties.get("owner_id") + operation_id = operations_repo.create_operation_id() + if hasattr(operations_repo, "acquire_workspace_lease"): + await operations_repo.acquire_workspace_lease(workspace.id, operation_id) + try: + workspace = await workspace_repo.get_deployed_workspace_by_id(workspace.id, operations_repo) + workspace_service = await workspace_service_repo.get_deployed_workspace_service_by_id( + workspace.id, workspace_service.id, operations_repo) + except Exception: + if hasattr(operations_repo, "release_workspace_lease"): + try: + await operations_repo.release_workspace_lease(workspace.id, operation_id) + except Exception: + logger.exception("Failed to release workspace lease after resource reload failed") + raise try: - user_resource, resource_template = await user_resource_repo.create_user_resource_item( - user_resource_create, - workspace.id, - workspace_service.id, - workspace_service.templateName, - user.id, - user.roles, - owner_id - ) - except (ValidationError, ValueError) as e: - logger.exception("Failed create user resource model instance") - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - except UserNotAuthorizedToUseTemplate as e: - logger.exception("User not authorized to use template") - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + owner_id: str = None - # Get the workspace subscription id (if set) - if workspace.properties.get("workspace_subscription_id"): - user_resource.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] + # Check for assign_to_another_user logic + if ( + hasattr(user_resource_create, "properties") + and isinstance(user_resource_create.properties, dict) + and user_resource_create.properties.get("assign_to_another_user") is True + ): + if user_resource_create.properties.get("owner_id"): + owner_id = user_resource_create.properties.get("owner_id") - operation = await save_and_deploy_resource( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operations_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) - response.headers["Location"] = construct_location_header(operation) + try: + user_resource, resource_template = await user_resource_repo.create_user_resource_item( + user_resource_create, + workspace.id, + workspace_service.id, + workspace_service.templateName, + user.id, + user.roles, + owner_id + ) + except (ValidationError, ValueError) as e: + logger.exception("Failed create user resource model instance") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except UserNotAuthorizedToUseTemplate as e: + logger.exception("User not authorized to use template") + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(e)) + + # Get the workspace subscription id (if set) + if workspace.properties.get("workspace_subscription_id"): + user_resource.properties["workspace_subscription_id"] = workspace.properties["workspace_subscription_id"] + + operation = await save_and_deploy_resource( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operations_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template, + operation_id=operation_id) + response.headers["Location"] = construct_location_header(operation) - return OperationInResponse(operation=operation) + return OperationInResponse(operation=operation) + except Exception as err: + if getattr(err, "lease_retained", False): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE, + ) + if hasattr(operations_repo, "release_workspace_lease"): + await operations_repo.release_workspace_lease(workspace.id, operation_id) + raise @user_resources_workspace_router.delete("/workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}", response_model=OperationInResponse, name=strings.API_DELETE_USER_RESOURCE) @@ -464,26 +655,44 @@ async def delete_user_resource( user_resource=Depends(get_user_resource_by_id_from_path), workspace_service=Depends(get_workspace_service_by_id_from_path), user_resource_repo=Depends(get_repository(UserResourceRepository)), + workspace_service_repo=Depends(get_repository(WorkspaceServiceRepository)), operations_repo=Depends(get_repository(OperationRepository)), resource_template_repo=Depends(get_repository(ResourceTemplateRepository)), resource_history_repo=Depends(get_repository(ResourceHistoryRepository))) -> OperationInResponse: - validate_user_has_valid_role_for_user_resource(user, user_resource) + operation_id = operations_repo.create_operation_id() + await operations_repo.acquire_workspace_lease(user_resource.workspaceId, operation_id) + try: + user_resource = await user_resource_repo.get_user_resource_by_id( + workspace_id=user_resource.workspaceId, + service_id=workspace_service.id, + resource_id=user_resource.id) + workspace_service = await workspace_service_repo.get_workspace_service_by_id( + user_resource.workspaceId, workspace_service.id) + validate_user_has_valid_role_for_user_resource(user, user_resource) - if user_resource.isEnabled: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.USER_RESOURCE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION) + if await operations_repo.resource_has_active_operation( + user_resource.workspaceId, exclude_operation_id=operation_id): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) - operation = await send_uninstall_message( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operations_repo, - resource_type=ResourceType.UserResource, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user) + if user_resource.isEnabled: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=strings.USER_RESOURCE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION) - response.headers["Location"] = construct_location_header(operation) + operation = await send_uninstall_message( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + operation_id=operation_id) - return OperationInResponse(operation=operation) + response.headers["Location"] = construct_location_header(operation) + return OperationInResponse(operation=operation) + except Exception as err: + if not getattr(err, "lease_retained", False): + await operations_repo.release_workspace_lease(user_resource.workspaceId, operation_id) + raise @user_resources_workspace_router.patch("/workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}", status_code=status.HTTP_202_ACCEPTED, response_model=OperationInResponse, name=strings.API_UPDATE_USER_RESOURCE, dependencies=[Depends(get_workspace_by_id_from_path), Depends(get_workspace_service_by_id_from_path)]) @@ -501,6 +710,9 @@ async def patch_user_resource( force_version_update: bool = False) -> OperationInResponse: validate_user_has_valid_role_for_user_resource(user, user_resource) + if await operations_repo.resource_has_active_operation(user_resource.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + try: operation = await update_user_resource(user_resource, user_resource_patch, force_version_update, user, etag, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) response.headers["Location"] = construct_location_header(operation) @@ -526,6 +738,10 @@ async def invoke_action_on_user_resource( resource_history_repo=Depends(get_repository(ResourceHistoryRepository)), user=Depends(require_workspace_owner_or_researcher_or_airlock_manager)) -> OperationInResponse: validate_user_has_valid_role_for_user_resource(user, user_resource) + + if await operations_repo.resource_has_active_operation(user_resource.workspaceId): + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + operation = await send_custom_action_message( resource=user_resource, resource_repo=user_resource_repo, diff --git a/api_app/core/config.py b/api_app/core/config.py index d2f1cf1fa4..85567b582e 100644 --- a/api_app/core/config.py +++ b/api_app/core/config.py @@ -41,6 +41,7 @@ SERVICE_BUS_RESOURCE_REQUEST_QUEUE: str = config("SERVICE_BUS_RESOURCE_REQUEST_QUEUE", default="") SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE: str = config("SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE", default="") SERVICE_BUS_STEP_RESULT_QUEUE: str = config("SERVICE_BUS_STEP_RESULT_QUEUE", default="") +SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE: str = config("SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE", default="") # Event grid configuration EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT: str = config("EVENT_GRID_STATUS_CHANGED_TOPIC_ENDPOINT", default="") diff --git a/api_app/db/repositories/airlock_requests.py b/api_app/db/repositories/airlock_requests.py index 21184c6e61..0e72e2c908 100644 --- a/api_app/db/repositories/airlock_requests.py +++ b/api_app/db/repositories/airlock_requests.py @@ -12,7 +12,7 @@ from models.domain.authentication import User from db.errors import EntityDoesNotExist from models.domain.airlock_request import AirlockFile, AirlockRequest, AirlockRequestStatus, \ - AirlockReview, AirlockReviewDecision, AirlockRequestHistoryItem, AirlockRequestType, AirlockReviewUserResource + AirlockReview, AirlockReviewDecision, AirlockRequestHistoryItem, AirlockRequestType, AirlockReviewUserResource, AirlockRedeployWorkflow from models.schemas.airlock_request import AirlockRequestInCreate, AirlockReviewInCreate from core import config from resources import strings @@ -194,7 +194,8 @@ async def update_airlock_request( request_files: Optional[List[AirlockFile]] = None, status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, - review_user_resource: Optional[AirlockReviewUserResource] = None) -> AirlockRequest: + review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None) -> AirlockRequest: updated_request = self._build_updated_request( original_request=original_request, new_status=new_status, @@ -202,13 +203,22 @@ async def update_airlock_request( status_message=status_message, airlock_review=airlock_review, review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow, updated_by=updated_by) try: db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) except CosmosAccessConditionFailedError: logger.warning(f"ETag mismatch for request ID: '{original_request.id}'. Retrying.") original_request = await self.get_airlock_request_by_id(original_request.id) - updated_request = self._build_updated_request(original_request=original_request, new_status=new_status, request_files=request_files, status_message=status_message, airlock_review=airlock_review) + updated_request = self._build_updated_request( + original_request=original_request, + new_status=new_status, + request_files=request_files, + status_message=status_message, + airlock_review=airlock_review, + review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow, + updated_by=updated_by) db_response = await self.update_airlock_request_item(original_request, updated_request, updated_by, {"previousStatus": original_request.status}) return db_response @@ -251,6 +261,7 @@ def _build_updated_request( status_message: Optional[Optional[str]] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None, updated_by: Optional[Union[User, dict]] = None) -> AirlockRequest: updated_request = copy.deepcopy(original_request) @@ -275,6 +286,9 @@ def _build_updated_request( if reviewer_id: updated_request.reviewUserResources[reviewer_id] = review_user_resource + if redeploy_workflow is not None: + updated_request.redeployWorkflows[redeploy_workflow.workflowId] = redeploy_workflow + return updated_request def _validate_status_update(self, current_status, new_status): diff --git a/api_app/db/repositories/base.py b/api_app/db/repositories/base.py index a4cd5aed4b..609344444d 100644 --- a/api_app/db/repositories/base.py +++ b/api_app/db/repositories/base.py @@ -44,8 +44,13 @@ async def upsert_item_with_etag(self, item: BaseModel, etag: str) -> BaseModel: async def update_item_dict(self, item_dict: dict): await self.container.upsert_item(body=item_dict) - async def delete_item(self, item_id: str): - await self.container.delete_item(item=item_id, partition_key=item_id) + async def delete_item(self, item_id: str, etag: Optional[str] = None, match_condition: Optional[MatchConditions] = None): + kwargs = {} + if etag is not None: + kwargs["etag"] = etag + if match_condition is not None: + kwargs["match_condition"] = match_condition + await self.container.delete_item(item=item_id, partition_key=item_id, **kwargs) async def rename_field_name(self, old_field_name: str, new_field_name: str): for item in await self.query('SELECT * FROM c'): diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index f85d69af68..cc51b7ea4d 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -1,7 +1,11 @@ from datetime import datetime, UTC import uuid -from typing import List +from typing import List, Optional +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError +from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosAccessConditionFailedError, CosmosResourceNotFoundError +from fastapi import HTTPException, status as http_status from pydantic import TypeAdapter from db.repositories.resource_templates import ResourceTemplateRepository from resources import strings @@ -12,8 +16,25 @@ from core import config from db.repositories.base import BaseRepository +from services.logging import logger from db.errors import EntityDoesNotExist -from models.domain.operation import Operation, OperationStep, Status +from models.domain.operation import Operation, OperationStep, Status, get_failure_status_for_action + +# The resource processor auto-renews Porter session locks for up to 3600 seconds (1 hour) +# during deployment execution (see resource_processor/vmss_porter/runner.py:68). +# Lease expiry includes a safe margin beyond the 3600-second execution window and status-delivery +# latency (7200 seconds / 2 hours) to ensure still-running operations are not prematurely reclaimed. +WORKSPACE_LEASE_EXPIRY_SECONDS = 7200.0 +ADDRESS_SPACE_ALLOCATOR_LEASE_ID = "address_space_allocator" +ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS = 300.0 + + +def extract_workspace_id_from_resource_path(resource_path: str) -> Optional[str]: + if resource_path and resource_path.startswith("/workspaces/"): + parts = resource_path.split("/") + if len(parts) > 2: + return parts[2] + return None class OperationRepository(BaseRepository): @@ -23,6 +44,347 @@ async def create(cls): await super().create(config.STATE_STORE_OPERATIONS_CONTAINER) return cls + async def save_item(self, item: Operation): + item_dict = item.model_dump(exclude={"etag"}) + item_dict.pop("_etag", None) + response = await self.container.create_item(body=item_dict) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + + async def get_active_operations_for_resource(self, resource_id: str) -> List[Operation]: + terminal_statuses = ( + Status.Deployed, + Status.DeploymentFailed, + Status.Updated, + Status.UpdatingFailed, + Status.Deleted, + Status.DeletingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + ) + status_filter = ", ".join(f'\"{s}\"' for s in terminal_statuses) + query = self.operations_query() + f' c.resourceId = "{resource_id}" AND NOT ARRAY_CONTAINS([{status_filter}], c.status)' + operations = await self.query(query=query) + return [TypeAdapter(Operation).validate_python(op) for op in operations] + + async def get_last_operation_for_resource(self, resource_id: str) -> Operation: + query = self.operations_query() + f' c.resourceId = "{resource_id}" ORDER BY c.createdWhen DESC OFFSET 0 LIMIT 1' + operations = await self.query(query=query) + if not operations: + raise EntityDoesNotExist + return TypeAdapter(Operation).validate_python(operations[0]) + + async def update_item(self, item: Operation, etag: Optional[str] = None, release_lease: bool = True) -> Operation: + etag_to_match = etag or getattr(item, "etag", None) + item_dict = item.model_dump(exclude={"etag"}) + item_dict.pop("_etag", None) + if etag_to_match: + response = await self.container.replace_item( + item=item.id, + body=item_dict, + etag=etag_to_match, + match_condition=MatchConditions.IfNotModified + ) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + else: + response = await self.container.upsert_item(body=item_dict) + if isinstance(response, dict) and "_etag" in response: + new_etag = response["_etag"] + item.etag = new_etag.replace('\"', '') if isinstance(new_etag, str) else new_etag + + active_statuses = ( + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + ) + if release_lease and item.status not in active_statuses: + target_workspace_id = extract_workspace_id_from_resource_path(item.resourcePath) + if target_workspace_id: + await self.release_workspace_lease(target_workspace_id, item.id) + + return item + + async def _reconcile_resource_status(self, resource_id: str, status: Status, message: str): + resource_repo = await ResourceRepository.create() + resource = await resource_repo.get_resource_by_id(resource_id) + if resource.deploymentStatus in { + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + }: + resource.deploymentStatus = status + resource.updatedWhen = self.get_timestamp() + await resource_repo.update_item(resource) + + async def _reconcile_operation_resources(self, operation: Operation, step_resources=None): + resource_updates = [(operation.resourceId, operation.status, operation.message)] + resource_updates.extend(step_resources or []) + collapsed_updates = {} + for resource_id, status, message in resource_updates: + collapsed_updates[resource_id] = (status, message) + for resource_id, (status, message) in collapsed_updates.items(): + await self._reconcile_resource_status(resource_id, status, message) + + async def _reconcile_operation(self, operation, timestamp: float, mark_failed: bool = False): + terminal_statuses = { + Status.Deployed, + Status.DeploymentFailed, + Status.Deleted, + Status.DeletingFailed, + Status.Updated, + Status.UpdatingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + } + if mark_failed: + for step in getattr(operation, "steps", None) or []: + if step.status not in terminal_statuses: + step.status = get_failure_status_for_action(step.resourceAction) + affected_step_resources = [ + ( + step.resourceId, + step.status, + step.message, + ) + for step in getattr(operation, "steps", None) or [] + if step.resourceId and step.status + ] + if mark_failed: + operation.status = get_failure_status_for_action(operation.action) + operation.message = "Operation was reconciled after its workspace lease expired" + operation.reconciled = False + operation.updatedWhen = timestamp + await self.update_item(operation, release_lease=False) + await self._reconcile_operation_resources(operation, affected_step_resources) + operation.reconciled = True + operation.updatedWhen = timestamp + await self.update_item(operation, release_lease=False) + + async def acquire_workspace_lease(self, workspace_id: str, operation_id: str) -> bool: + if not hasattr(self, "_container") or self._container is None: + return True + + lease_id = f"lease_{workspace_id}" + max_attempts = 3 + + for attempt in range(max_attempts): + timestamp = self.get_timestamp() + lease_created = False + lease_body = { + "id": lease_id, + "workspaceId": workspace_id, + "operationId": operation_id, + "createdWhen": timestamp, + } + + try: + create_call = self.container.create_item(body=lease_body) + if hasattr(create_call, "__await__"): + await create_call + lease_created = True + try: + has_active_operation = await self.resource_has_active_operation( + workspace_id, exclude_operation_id=operation_id) + except Exception: + await self.release_workspace_lease(workspace_id, operation_id) + lease_created = False + raise + if has_active_operation: + await self.release_workspace_lease(workspace_id, operation_id) + lease_created = False + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + return True + except (CosmosResourceExistsError, ResourceExistsError): + try: + read_call = self.read_item_by_id(lease_id) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + # The lease was released between create_item conflict and read_item_by_id; retry + if attempt < max_attempts - 1: + continue + raise + if not isinstance(existing_lease, dict): + return True + + current_op_id = existing_lease.get("operationId") + if current_op_id == operation_id: + return True + + lease_created = existing_lease.get("createdWhen", 0.0) + + if current_op_id: + try: + op_call = self.get_operation_by_id(current_op_id) + existing_op = await op_call if hasattr(op_call, "__await__") else op_call + terminal_statuses = { + Status.Deployed, + Status.DeploymentFailed, + Status.Deleted, + Status.DeletingFailed, + Status.Updated, + Status.UpdatingFailed, + Status.ActionSucceeded, + Status.ActionFailed, + } + if existing_op and getattr(existing_op, "status", None) in terminal_statuses and not getattr(existing_op, "reconciled", False): + try: + await self._reconcile_operation(existing_op, timestamp) + await self.release_workspace_lease(workspace_id, existing_op.id) + except Exception as e: + logger.exception(f"Failed to reconcile pending operation {existing_op.id}: {e}") + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + continue + if existing_op and getattr(existing_op, "status", None) not in terminal_statuses: + last_activity = max( + lease_created, + getattr(existing_op, "createdWhen", 0.0), + getattr(existing_op, "updatedWhen", 0.0), + ) + if timestamp - last_activity >= WORKSPACE_LEASE_EXPIRY_SECONDS: + try: + await self._reconcile_operation(existing_op, timestamp, mark_failed=True) + await self.release_workspace_lease(workspace_id, existing_op.id) + except Exception as e: + logger.exception(f"Failed to reconcile expired operation {existing_op.id}: {e}") + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + continue + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + except HTTPException: + raise + except (EntityDoesNotExist, CosmosResourceNotFoundError): + # Operation doc not written yet (still constructing cascade/operation). + # Safely reclaim orphaned leases after a bounded interval (WORKSPACE_LEASE_EXPIRY_SECONDS). + if timestamp - lease_created < WORKSPACE_LEASE_EXPIRY_SECONDS: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + etag = existing_lease.get("_etag") if isinstance(existing_lease, dict) else None + try: + if etag: + rep_call = self.container.replace_item( + item=lease_id, + body=lease_body, + etag=etag, + match_condition=MatchConditions.IfNotModified + ) + if hasattr(rep_call, "__await__"): + await rep_call + else: + up_call = self.container.upsert_item(body=lease_body) + if hasattr(up_call, "__await__"): + await up_call + return True + except (CosmosAccessConditionFailedError, ResourceExistsError, CosmosResourceNotFoundError, ResourceNotFoundError): + if attempt < max_attempts - 1: + continue + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + async def release_workspace_lease(self, workspace_id: str, operation_id: Optional[str] = None) -> None: + if not hasattr(self, "_container") or self._container is None: + return + lease_id = f"lease_{workspace_id}" + try: + read_call = self.read_item_by_id(lease_id) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + if isinstance(existing_lease, dict) and (not operation_id or existing_lease.get("operationId") == operation_id): + etag = existing_lease.get("_etag") + if etag: + del_call = self.delete_item(lease_id, etag=etag, match_condition=MatchConditions.IfNotModified) + else: + del_call = self.delete_item(lease_id) + if hasattr(del_call, "__await__"): + await del_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, + CosmosAccessConditionFailedError, TypeError): + return + except Exception: + logger.exception("Failed to release workspace lease") + raise + + async def acquire_address_space_allocator_lease(self, operation_id: str) -> bool: + if not hasattr(self, "_container") or self._container is None: + return True + + lease_body = { + "id": ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + "operationId": operation_id, + "createdWhen": self.get_timestamp(), + } + try: + create_call = self.container.create_item(body=lease_body) + if hasattr(create_call, "__await__"): + await create_call + return True + except (CosmosResourceExistsError, ResourceExistsError): + try: + read_call = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + return await self.acquire_address_space_allocator_lease(operation_id) + + if not isinstance(existing_lease, dict): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + if existing_lease.get("operationId") == operation_id: + return True + if self.get_timestamp() - existing_lease.get("createdWhen", 0.0) < ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + etag = existing_lease.get("_etag") + try: + if not etag: + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, + detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + replace_call = self.container.replace_item( + item=ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + body=lease_body, + etag=etag, + match_condition=MatchConditions.IfNotModified) + if hasattr(replace_call, "__await__"): + await replace_call + return True + except (CosmosAccessConditionFailedError, ResourceExistsError, CosmosResourceNotFoundError, ResourceNotFoundError): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + async def assert_address_space_allocator_lease(self, operation_id: str) -> None: + if not hasattr(self, "_container") or self._container is None: + return + try: + lease = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + lease = await lease if hasattr(lease, "__await__") else lease + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + if ( + not isinstance(lease, dict) + or lease.get("operationId") != operation_id + or self.get_timestamp() - lease.get("createdWhen", 0.0) >= ADDRESS_SPACE_ALLOCATOR_LEASE_EXPIRY_SECONDS + ): + raise HTTPException(status_code=http_status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION) + + async def release_address_space_allocator_lease(self, operation_id: Optional[str] = None) -> None: + if not hasattr(self, "_container") or self._container is None: + return + try: + read_call = self.read_item_by_id(ADDRESS_SPACE_ALLOCATOR_LEASE_ID) + existing_lease = await read_call if hasattr(read_call, "__await__") else read_call + if isinstance(existing_lease, dict) and (not operation_id or existing_lease.get("operationId") == operation_id): + etag = existing_lease.get("_etag") + delete_call = self.delete_item( + ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + **({"etag": etag, "match_condition": MatchConditions.IfNotModified} if etag else {})) + if hasattr(delete_call, "__await__"): + await delete_call + except (CosmosResourceNotFoundError, ResourceNotFoundError, EntityDoesNotExist, + CosmosAccessConditionFailedError, TypeError): + return + except Exception: + logger.exception("Failed to release address-space allocator lease") + raise + @staticmethod def operations_query(): return 'SELECT * FROM c WHERE' @@ -49,61 +411,74 @@ def create_main_step(self, resource_template: dict, action: str, resource_id: st message=message, updatedWhen=self.get_timestamp()) - async def create_operation_item(self, resource_id: str, resource_list: List, action: str, resource_path: str, resource_version: int, user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository) -> Operation: - operation_id = self.create_operation_id() - - # get the right "awaiting" message based on the action - status, message = self.get_initial_status(action) - all_steps = [] - for resource in resource_list: - name = resource["templateName"] - version = resource["templateVersion"] - resource_type = ResourceType(resource["resourceType"]) - primary_parent_service_name = None - if resource_type == ResourceType.UserResource: - primary_parent_workspace_service = await resource_repo.get_resource_by_id(resource["parentWorkspaceServiceId"]) - primary_parent_service_name = primary_parent_workspace_service.templateName - resource_template = await resource_template_repo.get_template_by_name_and_version(name, version, resource_type, primary_parent_service_name) - resource_template_dict = resource_template.model_dump(exclude_none=True) - # if the template has a pipeline defined for this action, copy over all the steps to the ops document - steps = await self.build_step_list( - steps=[], - resource_template_dict=resource_template_dict, - action=action, - resource_repo=resource_repo, - resource_id=resource["id"], - status=status, - message=message - ) + async def create_operation_item(self, resource_id: str, resource_list: List, action: str, resource_path: str, resource_version: int, user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, operation_id: Optional[str] = None) -> Operation: + if not operation_id: + operation_id = self.create_operation_id() - # if no pipeline is defined for this action, create a main step only - if len(steps) == 0: - all_steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource["id"], status=status, message=message)) - else: - all_steps.extend(steps) + target_workspace_id = extract_workspace_id_from_resource_path(resource_path) + if target_workspace_id: + await self.acquire_workspace_lease(target_workspace_id, operation_id) - timestamp = self.get_timestamp() - operation = Operation( - id=operation_id, - resourceId=resource_id, - resourcePath=resource_path, - resourceVersion=resource_version, - status=status, - createdWhen=timestamp, - updatedWhen=timestamp, - action=action, - message=message, - user=user.model_dump(), - steps=all_steps - ) + try: + # get the right "awaiting" message based on the action + status, message = self.get_initial_status(action) + all_steps = [] + for resource in resource_list: + name = resource["templateName"] + version = resource["templateVersion"] + resource_type = ResourceType(resource["resourceType"]) + primary_parent_service_name = None + if resource_type == ResourceType.UserResource: + primary_parent_workspace_service = await resource_repo.get_resource_by_id(resource["parentWorkspaceServiceId"]) + primary_parent_service_name = primary_parent_workspace_service.templateName + resource_template = await resource_template_repo.get_template_by_name_and_version(name, version, resource_type, primary_parent_service_name) + resource_template_dict = resource_template.model_dump(exclude_none=True) + # if the template has a pipeline defined for this action, copy over all the steps to the ops document + steps = await self.build_step_list( + steps=[], + resource_template_dict=resource_template_dict, + action=action, + resource_repo=resource_repo, + resource_id=resource["id"], + status=status, + message=message, + is_cascade_operation=resource["id"] != resource_id + ) - await self.save_item(operation) - return operation + # if no pipeline is defined for this action, create a main step only + if len(steps) == 0: + all_steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource["id"], status=status, message=message)) + else: + all_steps.extend(steps) + + timestamp = self.get_timestamp() + operation = Operation( + id=operation_id, + resourceId=resource_id, + resourcePath=resource_path, + resourceVersion=resource_version, + status=status, + createdWhen=timestamp, + updatedWhen=timestamp, + action=action, + message=message, + user=user.model_dump(), + steps=all_steps + ) - async def build_step_list(self, steps: List[OperationStep], resource_template_dict: dict, action: str, resource_repo: ResourceRepository, resource_id: str, status: Status, message: str): + await self.save_item(operation) + return operation + except Exception: + if target_workspace_id: + await self.release_workspace_lease(target_workspace_id, operation_id) + raise + + async def build_step_list(self, steps: List[OperationStep], resource_template_dict: dict, action: str, resource_repo: ResourceRepository, resource_id: str, status: Status, message: str, is_cascade_operation: bool = False): if "pipeline" in resource_template_dict and resource_template_dict["pipeline"] is not None: if action in resource_template_dict["pipeline"] and resource_template_dict["pipeline"][action] is not None: for step in resource_template_dict["pipeline"][action]: + if is_cascade_operation and step["stepId"] == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + continue if step["stepId"] == "main": steps.append(self.create_main_step(resource_template=resource_template_dict, action=action, resource_id=resource_id, status=status, message=message)) else: @@ -135,7 +510,7 @@ async def build_step_list(self, steps: List[OperationStep], resource_template_di steps.append(OperationStep( id=str(uuid.uuid4()), templateStepId=step["stepId"], - stepTitle=step["stepTitle"], + stepTitle=step.get("stepTitle"), resourceId=resource_for_step.id, resourceTemplateName=resource_for_step.templateName, resourceType=resource_for_step.resourceType, @@ -194,3 +569,42 @@ async def resource_has_deployed_operation(self, resource_id: str) -> bool: query = self.operations_query() + f' c.resourceId = "{resource_id}" AND ((c.action = "{RequestAction.Install}" AND c.status = "{Status.Deployed}") OR (c.action = "{RequestAction.Upgrade}" AND c.status = "{Status.Updated}"))' operations = await self.query(query=query) return len(operations) > 0 + + async def resource_has_active_operation(self, resource_id: str, exclude_operation_id: Optional[str] = None) -> bool: + # Guard against injection; resource_id should always be a valid UUID string + try: + uuid.UUID(resource_id) + except ValueError: + return False + active_statuses = ( + Status.AwaitingAction, Status.InvokingAction, Status.AwaitingDeployment, + Status.Deploying, Status.AwaitingDeletion, Status.Deleting, + Status.AwaitingUpdate, Status.Updating, Status.PipelineRunning + ) + status_filter = ", ".join(f'"{s}"' for s in active_statuses) + query = ( + self.operations_query() + + f' (c.resourceId = "{resource_id}"' + + f' OR ARRAY_CONTAINS(c.steps, {{"resourceId": "{resource_id}"}}, true)' + # Include any operation whose resource path contains this resource id — this catches + # cascading operations on all descendant resources (workspace services AND user resources). + # The previous NOT CONTAINS(c.resourcePath, "/user-resources/") exclusion meant that + # active user-resource operations were invisible when checking a parent workspace or + # workspace-service, allowing duplicate cascading pipelines to start. + + f' OR CONTAINS(c.resourcePath, "{resource_id}"))' + + f' AND c.status IN ({status_filter})' + ) + if exclude_operation_id: + query += f' AND c.id != "{exclude_operation_id}"' + operations = await self.query(query=query) + if not operations: + return False + + has_active = False + for op_dict in operations: + op_status = op_dict.get("status") if isinstance(op_dict, dict) else getattr(op_dict, "status", None) + if op_status is not None and op_status not in active_statuses: + continue + has_active = True + + return has_active diff --git a/api_app/db/repositories/resource_templates.py b/api_app/db/repositories/resource_templates.py index 36ac905b94..5033c848ef 100644 --- a/api_app/db/repositories/resource_templates.py +++ b/api_app/db/repositories/resource_templates.py @@ -7,9 +7,11 @@ from db.errors import DuplicateEntity, EntityDoesNotExist, EntityVersionExist, InvalidInput from db.repositories.base import BaseRepository from models.domain.resource import ResourceType +from models.domain.request_action import RequestAction from models.domain.resource_template import ResourceTemplate from models.domain.user_resource_template import UserResourceTemplate from models.schemas.resource_template import ResourceTemplateInCreate, ResourceTemplateInformation +from resources import strings from services.schema_service import enrich_shared_service_template, enrich_workspace_template, enrich_workspace_service_template, enrich_user_resource_template @@ -137,7 +139,7 @@ async def create_template(self, template_input: ResourceTemplateInCreate, resour if "pipeline" in template_input.json_schema: pipeline = template_input.json_schema["pipeline"] - self._validate_pipeline_has_unique_step_ids(pipeline) + self._validate_pipeline_has_unique_step_ids(pipeline, resource_type) template["pipeline"] = pipeline if "allOf" in template_input.json_schema: @@ -174,14 +176,16 @@ async def create_and_validate_template(self, template_input: ResourceTemplateInC created_template = await self.create_template(template_input, resource_type, workspace_service_template_name) return self.enrich_template(created_template) - def _validate_pipeline_has_unique_step_ids(self, pipeline): + def _validate_pipeline_has_unique_step_ids(self, pipeline, resource_type: ResourceType = None): if pipeline is None: return step_ids = [] for action in pipeline: num_of_main_steps = 0 - for step in pipeline[action]: + steps = pipeline[action] + num_steps = len(steps) + for i, step in enumerate(steps): step_id = step["stepId"] if step_id == "main": @@ -190,5 +194,20 @@ def _validate_pipeline_has_unique_step_ids(self, pipeline): if step_id in step_ids or num_of_main_steps > 1: raise InvalidInput(f"Invalid template - duplicate stepIds are not allowed. stepId: {step_id}") + if step_id == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if resource_type != ResourceType.WorkspaceService: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in '{ResourceType.WorkspaceService}' templates") + if action != RequestAction.UnInstall: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in the '{RequestAction.UnInstall}' pipeline") + if i != num_steps - 1: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must be the final step in the '{RequestAction.UnInstall}' pipeline") + if num_of_main_steps == 0: + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' requires a preceding 'main' step in the '{RequestAction.UnInstall}' pipeline") + if (step.get("resourceType") != ResourceType.Workspace + or step.get("resourceAction") != RequestAction.Upgrade): + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' must upgrade a workspace") + if not step.get("stepTitle") or not str(step.get("stepTitle")).strip(): + raise InvalidInput(f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' requires a non-empty 'stepTitle'") + if step_id != "main": step_ids.append(step_id) diff --git a/api_app/db/repositories/user_resources.py b/api_app/db/repositories/user_resources.py index 6df46adcc2..87b2a45518 100644 --- a/api_app/db/repositories/user_resources.py +++ b/api_app/db/repositories/user_resources.py @@ -85,6 +85,16 @@ async def get_user_resource_by_id(self, workspace_id: str, service_id: str, reso raise EntityDoesNotExist return TypeAdapter(UserResource).validate_python(user_resources[0]) + async def get_user_resource_by_workflow_id(self, workspace_id: str, service_id: str, workflow_id: str) -> UserResource: + query, parameters = self.user_resources_query(str(workspace_id), str(service_id)) + query += ' AND c.properties.airlock_redeploy_workflow_id = @workflowId AND c.deploymentStatus != @deletedStatus' + parameters.append({'name': '@workflowId', 'value': workflow_id}) + parameters.append({'name': '@deletedStatus', 'value': Status.Deleted}) + user_resources = await self.query(query=query, parameters=parameters) + if not user_resources: + raise EntityDoesNotExist + return TypeAdapter(UserResource).validate_python(user_resources[0]) + def get_user_resource_spec_params(self): return self.get_resource_base_spec_params() diff --git a/api_app/main.py b/api_app/main.py index 0bdc769141..0bfcf7a4a0 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -20,6 +20,7 @@ from services.logging import initialize_logging, logger from service_bus.deployment_status_updater import DeploymentStatusUpdater from service_bus.airlock_request_status_update import AirlockStatusUpdater +from service_bus.airlock_workflow import AirlockWorkflowUpdater @asynccontextmanager @@ -33,9 +34,12 @@ async def lifespan(app: FastAPI): airlockStatusUpdater = AirlockStatusUpdater() await airlockStatusUpdater.init_repos() + airlockWorkflow = AirlockWorkflowUpdater() + await airlockWorkflow.init_repos() asyncio.create_task(deploymentStatusUpdater.receive_messages()) asyncio.create_task(airlockStatusUpdater.receive_messages()) + asyncio.create_task(airlockWorkflow.receive_messages()) yield diff --git a/api_app/models/domain/airlock_request.py b/api_app/models/domain/airlock_request.py index 23ab8a626c..24381e9dcb 100644 --- a/api_app/models/domain/airlock_request.py +++ b/api_app/models/domain/airlock_request.py @@ -91,6 +91,23 @@ class AirlockReviewUserResource(AzureTREModel): workspaceId: str = Field(title="Workspace ID") workspaceServiceId: str = Field(title="Workspace Service ID") userResourceId: str = Field(title="User Resource ID") + reviewer: dict = Field(default_factory=dict, title="User who created the review resource") + + @field_validator("reviewer", mode="before") + @classmethod + def convert_reviewer_to_dict(cls, value): + if hasattr(value, "model_dump"): + return value.model_dump() + return value + + +class AirlockRedeployWorkflow(AzureTREModel): + workflowId: str + phase: str + operationId: str + workspaceId: Optional[str] = None + workspaceServiceId: Optional[str] = None + userResourceId: Optional[str] = None class AirlockRequest(AzureTREModel): @@ -114,6 +131,7 @@ class AirlockRequest(AzureTREModel): reviews: Optional[List[AirlockReview]] = None etag: Optional[str] = Field(None, title="_etag", alias="_etag") reviewUserResources: Dict[str, AirlockReviewUserResource] = Field(default_factory=dict, title="User resources created for Airlock Reviews") + redeployWorkflows: Dict[str, AirlockRedeployWorkflow] = Field(default_factory=dict) # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 @field_validator("etag", mode="before") diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 9cb7b258ff..4f2e290e5b 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -79,6 +79,16 @@ def is_action(self) -> bool: ) +def get_failure_status_for_action(action) -> Status: + """Return the terminal failure Status for a given RequestAction.""" + from models.domain.request_action import RequestAction # local import to avoid circular deps + return { + RequestAction.Install: Status.DeploymentFailed, + RequestAction.UnInstall: Status.DeletingFailed, + RequestAction.Upgrade: Status.UpdatingFailed, + }.get(action, Status.ActionFailed) + + class Operation(AzureTREModel): """ Operation model @@ -94,6 +104,16 @@ class Operation(AzureTREModel): updatedWhen: float = Field(0.0, title="POSIX Timestamp for When the operation was updated") user: dict = Field(default_factory=dict) steps: Optional[List[OperationStep]] = Field(None, title="Operation Steps") + reconciled: Optional[bool] = Field(default=False, title="reconciled", description="Whether operation was reconciled due to timeout or interruption") + etag: Optional[str] = Field(None, title="_etag", alias="_etag") + + # SQL API CosmosDB saves ETag as an escaped string: https://github.com/microsoft/AzureTRE/issues/1931 + @field_validator("etag", mode="before") + @classmethod + def parse_etag_to_remove_escaped_quotes(cls, value): + if value: + return value.replace('\"', '') + return value @field_validator("user", mode="before") @classmethod diff --git a/api_app/pytest.ini b/api_app/pytest.ini index a86e89e4bb..f8537f0452 100644 --- a/api_app/pytest.ini +++ b/api_app/pytest.ini @@ -1,5 +1,7 @@ [pytest] filterwarnings = error + ignore::pytest.PytestRemovedIn10Warning asyncio_mode = strict asyncio_default_fixture_loop_scope = function + diff --git a/api_app/resources/strings.py b/api_app/resources/strings.py index 09432e64a3..c58ac279c1 100644 --- a/api_app/resources/strings.py +++ b/api_app/resources/strings.py @@ -143,6 +143,7 @@ WORKSPACE_SERVICE_NEEDS_TO_BE_DISABLED_BEFORE_DELETION = "The workspace service needs to be disabled before you can delete it" WORKSPACE_SERVICES_NEED_TO_BE_DELETED_BEFORE_WORKSPACE = "All workspace services need to be deleted before you can delete the workspace" WORKSPACE_DOES_NOT_HAVE_ADDRESS_SPACES_PROPERTY = "Workspace does not have address_spaces property" +WORKSPACE_HAS_ACTIVE_OPERATION = "Workspace has an active operation in progress" WORKSPACE_TEMPLATE_VERSION_EXISTS = "A template with the same version, or a newer version already exists" OPERATION_DOES_NOT_EXIST = "Operation does not exist" CUSTOM_ACTION_NOT_DEFINED = "The specified custom action isn't defined in the targeted resource." @@ -181,6 +182,8 @@ RESOURCE_ACTION_INSTALL = "install" RESOURCE_ACTION_UPDATE = "upgrade" RESOURCE_ACTION_UNINSTALL = "uninstall" +ADDRESS_SPACE_CLEANUP_STEP_ID = "address-space-cleanup" +ADDRESS_SPACE_CLEANUP_SUCCESS = "address_space_cleanup_success" # Resource Action Status diff --git a/api_app/service_bus/airlock_workflow.py b/api_app/service_bus/airlock_workflow.py new file mode 100644 index 0000000000..31dc1ec384 --- /dev/null +++ b/api_app/service_bus/airlock_workflow.py @@ -0,0 +1,298 @@ +import asyncio +import json +import uuid + +from azure.servicebus.aio import AutoLockRenewer, ServiceBusClient +from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError +from fastapi import HTTPException +from pydantic import ValidationError + +from core import config, credentials +from db.repositories.airlock_requests import AirlockRequestRepository +from db.repositories.operations import OperationRepository +from db.repositories.resource_templates import ResourceTemplateRepository +from db.repositories.resources_history import ResourceHistoryRepository +from db.repositories.user_resources import UserResourceRepository +from db.repositories.workspace_services import WorkspaceServiceRepository +from db.repositories.workspaces import WorkspaceRepository +from db.errors import EntityDoesNotExist +from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS +from models.domain.authentication import User +from models.domain.airlock_request import AirlockRequestType, AirlockReviewUserResource, AirlockRedeployWorkflow +from models.domain.resource import ResourceType +from models.domain.operation import Status +from services.airlock import _deploy_vm, wait_for_successful_operation, update_and_publish_event_airlock_request, disable_user_resource +from api.routes.resource_helpers import send_uninstall_message +from services.logging import logger +from resources import strings +from service_bus.helpers import send_airlock_workflow_message + +MAX_LEASE_CONTENTION_RETRY_DELAY_SECONDS = int(WORKSPACE_LEASE_EXPIRY_SECONDS) +MAX_CONCURRENT_AIRLOCK_WORKFLOWS = 10 + + +class AirlockWorkflowUpdater: + async def init_repos(self): + self.airlock_request_repo = await AirlockRequestRepository.create() + self.operations_repo = await OperationRepository.create() + self.resource_template_repo = await ResourceTemplateRepository.create() + self.resource_history_repo = await ResourceHistoryRepository.create() + self.user_resource_repo = await UserResourceRepository.create() + self.workspace_service_repo = await WorkspaceServiceRepository.create() + self.workspace_repo = await WorkspaceRepository.create() + + async def receive_messages(self): + processing_tasks = set() + + async def process_received_message(message): + try: + async with AutoLockRenewer() as renewer: + renewer.register( + receiver, + message, + max_lock_renewal_duration=3 * WORKSPACE_LEASE_EXPIRY_SECONDS) + if await self.process_message(message): + await receiver.complete_message(message) + else: + await receiver.abandon_message(message) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Airlock workflow message acknowledgement failed") + + while True: + try: + async with credentials.get_credential_async_context() as credential: + async with ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) as client: + receiver = client.get_queue_receiver(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + async with receiver: + while True: + available_slots = MAX_CONCURRENT_AIRLOCK_WORKFLOWS - len(processing_tasks) + if available_slots <= 0: + await asyncio.sleep(1) + processing_tasks = {task for task in processing_tasks if not task.done()} + continue + messages = await receiver.receive_messages( + max_message_count=available_slots, + max_wait_time=1) + for message in messages: + task = asyncio.create_task(process_received_message(message)) + processing_tasks.add(task) + processing_tasks = {task for task in processing_tasks if not task.done()} + except OperationTimeoutError: + logger.debug("No Airlock workflow messages available") + except ServiceBusConnectionError: + logger.warning("Airlock workflow Service Bus connection failed; retrying") + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Airlock workflow receiver failed; retrying") + await asyncio.sleep(10) + + async def process_message(self, message) -> bool: + payload = None + try: + payload = json.loads(str(message)) + workflow = payload["workflow"] + if workflow not in ("cleanup", "redeploy"): + logger.warning("Unknown Airlock workflow type: %s", workflow) + return True + airlock_request = await self.airlock_request_repo.get_airlock_request_by_id(payload["airlock_request_id"]) + review_user_id = payload.get("review_user_id") + workflow_id = payload.get("redeploy_workflow_id") + workflow_state = None + if workflow == "redeploy": + if not workflow_id: + raise ValueError("Redeploy workflow ID is required") + workflow_state = airlock_request.redeployWorkflows.get(workflow_id) + if workflow_state is not None and workflow_state.phase == "completed": + return True + workflow_user_id = workflow_id.split(":")[1] if ":" in workflow_id else None + review_user_id = review_user_id or workflow_user_id + if not review_user_id or review_user_id not in airlock_request.reviewUserResources: + raise ValueError("Airlock workflow user is not associated with the request") + review_resource = airlock_request.reviewUserResources[review_user_id] + review_workspace_id = review_resource.workspaceId + review_workspace_service_id = review_resource.workspaceServiceId + user_resource_id = review_resource.userResourceId + redeploy_workspace_id = review_workspace_id + redeploy_workspace_service_id = review_workspace_service_id + workspace = None + if workflow == "redeploy": + workspace = await self.workspace_repo.get_workspace_by_id(airlock_request.workspaceId) + if airlock_request.type == AirlockRequestType.Import: + review_config = workspace.properties["airlock_review_config"]["import"] + redeploy_workspace_id = review_config["import_vm_workspace_id"] + redeploy_workspace_service_id = review_config["import_vm_workspace_service_id"] + user_resource_template_name = review_config["import_vm_user_resource_template_name"] + else: + review_config = workspace.properties["airlock_review_config"]["export"] + redeploy_workspace_id = workspace.id + redeploy_workspace_service_id = review_config["export_vm_workspace_service_id"] + user_resource_template_name = review_config["export_vm_user_resource_template_name"] + try: + user_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=review_workspace_id, + service_id=review_workspace_service_id, + resource_id=user_resource_id) + except EntityDoesNotExist: + if workflow == "cleanup": + logger.info("Airlock review resource %s is already deleted", user_resource_id) + return True + user_resource = None + review_reviewer = next( + (review.reviewer for review in (airlock_request.reviews or []) + if review.reviewer.get("id") == review_user_id), + {}) + reviewer = (review_resource.reviewer + or (user_resource.user if user_resource is not None else {}) + or review_reviewer + or airlock_request.createdBy) + if not reviewer: + raise ValueError("Airlock workflow reviewer identity is missing") + user = User.model_validate(reviewer) + if user.id != review_user_id: + raise ValueError("Airlock workflow user does not match the review resource") + if workflow == "cleanup": + workspace_service = await self.workspace_service_repo.get_workspace_service_by_id( + workspace_id=review_workspace_id, + service_id=review_workspace_service_id) + if user_resource is None: + return True + disable_operation = await disable_user_resource( + user_resource, user, workspace_service, self.user_resource_repo, + self.resource_template_repo, self.operations_repo, self.resource_history_repo) + await wait_for_successful_operation(self.operations_repo, disable_operation.id) + else: + await wait_for_successful_operation(self.operations_repo, payload["operation_id"]) + uninstall_started = payload.get("uninstall_started", False) + uninstall_started = uninstall_started or ( + workflow == "redeploy" + and workflow_state is not None + and workflow_state.phase == "deploying" + ) + if user_resource is not None and not uninstall_started: + delete_operation = await send_uninstall_message( + resource=user_resource, + resource_repo=self.user_resource_repo, + operations_repo=self.operations_repo, + resource_type=ResourceType.UserResource, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + user=user) + await wait_for_successful_operation(self.operations_repo, delete_operation.id) + + if workflow == "redeploy": + workflows = airlock_request.redeployWorkflows + workflow_state = workflows.get(workflow_id) if isinstance(workflows, dict) else None + if workflow_state is not None and workflow_state.phase == "completed": + return True + + replacement_resource = None + operation_id = workflow_state.operationId if workflow_state is not None else str(uuid.uuid4()) + if workflow_state is not None and workflow_state.phase == "deploying": + try: + replacement_operation_call = self.operations_repo.get_operation_by_id(operation_id) + replacement_operation = ( + await replacement_operation_call + if hasattr(replacement_operation_call, "__await__") + else replacement_operation_call + ) + if replacement_operation.status in { + Status.DeploymentFailed, Status.UpdatingFailed, + Status.DeletingFailed, Status.ActionFailed, + }: + operation_id = str(uuid.uuid4()) + else: + if workflow_state.userResourceId: + replacement_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=workflow_state.workspaceId, + service_id=workflow_state.workspaceServiceId, + resource_id=workflow_state.userResourceId) + else: + replacement_resource = await self.user_resource_repo.get_user_resource_by_id( + workspace_id=redeploy_workspace_id, + service_id=redeploy_workspace_service_id, + resource_id=replacement_operation.resourceId) + await wait_for_successful_operation(self.operations_repo, operation_id) + except EntityDoesNotExist: + try: + orphaned_resource = await self.user_resource_repo.get_user_resource_by_workflow_id( + workspace_id=redeploy_workspace_id, + service_id=redeploy_workspace_service_id, + workflow_id=workflow_id) + await self.user_resource_repo.delete_item(orphaned_resource.id) + logger.warning( + "Removed orphaned Airlock redeploy resource %s before retrying workflow %s", + orphaned_resource.id, workflow_id) + except EntityDoesNotExist: + pass + operation_id = str(uuid.uuid4()) + + if replacement_resource is None: + workflow_state = AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId=operation_id) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + redeploy_workflow=workflow_state) + try: + replacement_resource, replacement_operation = await _deploy_vm( + airlock_request, user, workspace, + redeploy_workspace_id, redeploy_workspace_service_id, + user_resource_template_name, self.user_resource_repo, + self.workspace_service_repo, self.operations_repo, + self.resource_template_repo, self.resource_history_repo, + workflow_id=workflow_id, operation_id=operation_id) + await wait_for_successful_operation(self.operations_repo, replacement_operation.id) + except Exception as ex: + if not getattr(ex, "lease_retained", False): + try: + await self.operations_repo.release_workspace_lease( + redeploy_workspace_id, operation_id) + except Exception: + logger.exception("Failed to release failed Airlock redeploy lease") + replacement_operation_id = str(uuid.uuid4()) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + redeploy_workflow=AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId=replacement_operation_id)) + raise + + workflow_state = AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="completed", + operationId=operation_id, + workspaceId=redeploy_workspace_id, + workspaceServiceId=redeploy_workspace_service_id, + userResourceId=replacement_resource.id) + await update_and_publish_event_airlock_request( + airlock_request, self.airlock_request_repo, user, workspace, + review_user_resource=AirlockReviewUserResource( + workspaceId=redeploy_workspace_id, + workspaceServiceId=redeploy_workspace_service_id, + userResourceId=replacement_resource.id, + reviewer=user), + redeploy_workflow=workflow_state) + return True + except (json.JSONDecodeError, KeyError, ValidationError): + logger.exception("Invalid Airlock workflow message") + return True + except HTTPException as ex: + if ex.status_code == 409 and ex.detail == strings.WORKSPACE_HAS_ACTIVE_OPERATION and payload is not None: + retry_count = int(payload.get("lease_contention_retry_count", 0)) + delay_seconds = min(2 ** retry_count * 30, MAX_LEASE_CONTENTION_RETRY_DELAY_SECONDS) + payload["lease_contention_retry_count"] = retry_count + 1 + await send_airlock_workflow_message(payload, delay_seconds=delay_seconds) + logger.info( + "Rescheduled Airlock workflow %s after workspace lease contention in %s seconds", + payload.get("airlock_request_id"), delay_seconds) + return True + logger.exception("Airlock workflow message failed") + return False + except Exception: + logger.exception("Airlock workflow message failed; it will be retried") + return False diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 99f641f504..680e01e9c8 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -1,12 +1,14 @@ import asyncio +import copy import json import uuid import time +from typing import Optional, Tuple from pydantic import ValidationError, TypeAdapter from api.routes.resource_helpers import get_timestamp -from models.domain.resource import Output +from models.domain.resource import Output, ResourceType from db.repositories.resources_history import ResourceHistoryRepository from models.domain.request_action import RequestAction from db.repositories.resource_templates import ResourceTemplateRepository @@ -21,6 +23,21 @@ from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings from services.logging import logger, tracer +from db.repositories.workspaces import WorkspaceRepository +from db.repositories.workspace_services import WorkspaceServiceRepository +from models.schemas.resource import ResourcePatch +from azure.cosmos.exceptions import CosmosAccessConditionFailedError + + +MAX_CLEANUP_RETRIES = 3 +MAX_CLEANUP_DELIVERY_COUNT = 10 +# 0-based: delivery_count counts prior unsuccessful delivery attempts before DLQ +FINAL_CLEANUP_DELIVERY_COUNT = MAX_CLEANUP_DELIVERY_COUNT - 1 + + +class AddressSpaceConflictError(Exception): + """Raised when address space cleanup encounters a terminal conflict with another active service.""" + pass class DeploymentStatusUpdater(): @@ -30,6 +47,8 @@ def __init__(self): async def init_repos(self): self.operations_repo = await OperationRepository.create() self.resource_repo = await ResourceRepository.create() + self.workspace_repo = await WorkspaceRepository.create() + self.workspace_services_repo = await WorkspaceServiceRepository.create() self.resource_template_repo = await ResourceTemplateRepository.create() self.resource_history_repo = await ResourceHistoryRepository.create() @@ -90,20 +109,21 @@ async def receive_messages(self): logger.exception(f"Unknown exception. Will retry - {e}") await asyncio.sleep(10) - async def process_message(self, msg): + async def process_message(self, msg) -> bool: complete_message = False - message = "" - with tracer.start_as_current_span("process_message") as current_span: try: message = TypeAdapter(DeploymentStatusUpdateMessage).validate_python(json.loads(str(msg))) - current_span.set_attribute("step_id", message.stepId) - current_span.set_attribute("operation_id", message.operationId) - current_span.set_attribute("status", message.status) + current_span.set_attribute("step_id", str(message.stepId)) + current_span.set_attribute("operation_id", str(message.operationId)) + current_span.set_attribute("status", str(message.status)) + + raw_delivery_count = getattr(msg, "delivery_count", 0) + delivery_count = raw_delivery_count if raw_delivery_count is not None else 0 + is_final_delivery = delivery_count >= FINAL_CLEANUP_DELIVERY_COUNT - logger.info(f"Received and parsed JSON for: {msg.correlation_id}") - complete_message = await self.update_status_in_database(message) + complete_message = await self.update_status_in_database(message, is_final_delivery=is_final_delivery) logger.info(f"Update status in DB for {message.operationId} - {message.status}") except (json.JSONDecodeError, ValidationError): logger.exception(f"{strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT}: {msg.correlation_id}") @@ -112,7 +132,7 @@ async def process_message(self, msg): return complete_message - async def update_status_in_database(self, message: DeploymentStatusUpdateMessage): + async def update_status_in_database(self, message: DeploymentStatusUpdateMessage, is_final_delivery: bool = False): """ Get the operation the message references, and find the step within the operation that is to be updated Update the status of the step. If it's a single step operation, copy the status into the operation status. If it's a multi step, @@ -120,11 +140,12 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage If there is another step in the operation after this one, process the substitutions + patch, then enqueue a message to process it. """ result = False + operation = None + step_to_update = None try: # update the op operation = await self.operations_repo.get_operation_by_id(str(message.operationId)) - step_to_update = None is_last_step = False current_step_index = 0 @@ -137,34 +158,282 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage is_last_step = True if step_to_update is None: - raise f"Error finding step {message.stepId} in operation {message.operationId}" + raise Exception(f"Error finding step {message.stepId} in operation {message.operationId}") + + # Reject messages for reconciled/timed-out operations to prevent resurrecting stale operations + if getattr(operation, "reconciled", False): + logger.warning( + f"Message {message.stepId} received for operation {operation.id}, " + f"but operation was reconciled due to timeout/interruption. Rejecting to prevent stale writes." + ) + return True + + # Ignore duplicate messages if uninstall cleanup is already terminal + if ( + operation.action == RequestAction.UnInstall + and self._is_cleanup_terminal(operation) + ): + logger.info( + f"Message {message.stepId} (step {step_to_update.templateStepId}, status {message.status}) received for operation {operation.id}, " + f"but uninstall cleanup is already terminal ({operation.status}). Ignoring to prevent stale writes." + ) + return True # update the step status step_to_update.status = message.status step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() - # update the overall headline operation status - await self.update_overall_operation_status(operation, step_to_update, is_last_step) + resource_id = uuid.UUID(step_to_update.resourceId) - # save the operation - await self.operations_repo.update_item(operation) + # Ensure workspace upgrade step is appended before calculating overall status + if ( + step_to_update.templateStepId == "main" + and step_to_update.resourceId == operation.resourceId + and step_to_update.is_success() + and operation.action == RequestAction.UnInstall + and not self._has_workspace_upgrade_step(operation, current_step_index) + ): + resource_dict = await self.resource_repo.get_resource_dict_by_id(resource_id) + if self._is_workspace_service_with_address_space(resource_dict): + operation.steps.append(self._create_workspace_upgrade_step(resource_dict)) + is_last_step = False - # copy the step status to the resource item, for convenience - resource_id = uuid.UUID(step_to_update.resourceId) + # Is this the address space cleanup step? + is_cleanup_step = ( + step_to_update.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + and step_to_update.resourceType in (ResourceType.Workspace, ResourceType.Workspace.value) + and step_to_update.resourceAction == RequestAction.Upgrade + and operation.action == RequestAction.UnInstall + and is_last_step + ) - resource = await self.resource_repo.get_resource_by_id(resource_id) - resource.deploymentStatus = step_to_update.status - await self.resource_repo.update_item(resource) + # Run cleanup before marking the operation complete + if is_cleanup_step and step_to_update.is_success(): + try: + freed = await self._free_workspace_address_space(operation) + except AddressSpaceConflictError as e: + logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") + step_to_update.status = Status.UpdatingFailed + step_to_update.message = f"Terminal address space conflict: {e}" + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + return True + except Exception as e: + logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Unexpected error freeing workspace address space: {e}", exc_info=True) + freed = False - # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. - if not step_to_update.is_success(): - return True + if not freed: + if is_final_delivery: + cleanup_failure_message = f"Address space cleanup failed after {MAX_CLEANUP_DELIVERY_COUNT} delivery attempts. Marking operation as failed to unblock workspace." + logger.error(f"[ADDRESS_SPACE_CLEANUP_MAX_DELIVERIES] {cleanup_failure_message}") + step_to_update.status = Status.UpdatingFailed + step_to_update.message = cleanup_failure_message + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + return True + else: + cleanup_failure_message = "Address space cleanup failed after maximum retries; the message will be retried." + step_to_update.status = Status.Updating + step_to_update.message = cleanup_failure_message + resource_to_update = await self.resource_repo.get_resource_by_id(resource_id) + resource_to_update.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource_to_update) + operation.status = Status.PipelineRunning + await self.operations_repo.update_item(operation) + return False + else: + step_to_update.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + + try: + # copy the step status to the resource item, for convenience + resource = await self.resource_repo.get_resource_by_id(resource_id) + resource.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource) + + # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. + if not step_to_update.is_success(): + await self.update_overall_operation_status(operation, step_to_update, is_last_step) + await self.operations_repo.update_item(operation) + return True + + # update the resource doc to persist any outputs + resource = await self.resource_repo.get_resource_dict_by_id(resource_id) + resource_to_persist = self.create_updated_resource_document(resource, message) + await self.resource_repo.update_item_dict(resource_to_persist) + + if is_cleanup_step and step_to_update.is_success(): + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + + # update the overall headline operation status + await self.update_overall_operation_status(operation, step_to_update, is_last_step) - # update the resource doc to persist any outputs - resource = await self.resource_repo.get_resource_dict_by_id(resource_id) - resource_to_persist = self.create_updated_resource_document(resource, message) - await self.resource_repo.update_item_dict(resource_to_persist) + for attempt in range(MAX_CLEANUP_RETRIES): + try: + await self.operations_repo.update_item(operation) + break + except CosmosAccessConditionFailedError: + if attempt == MAX_CLEANUP_RETRIES - 1: + raise + logger.warning(f"ETag conflict when saving operation {operation.id}. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + if ( + fresh_op is not operation + and ( + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + logger.info( + f"Operation {fresh_op.id} is already terminal ({fresh_op.status}) after ETag conflict reload. " + f"Reconciling resource writes and skipping stale update." + ) + await self._reconcile_resources_for_terminal_operation(fresh_op) + return True + + step_to_update, is_last_step, current_step_index = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=step_to_update.message, + ) + await self.update_overall_operation_status(fresh_op, step_to_update, is_last_step) + operation = fresh_op + except Exception as e: + if is_cleanup_step and step_to_update.is_success(): + logger.error(f"[CLEANUP_FINALIZATION_FAILED] Error in post-cleanup writes: {e}", exc_info=True) + if is_final_delivery: + workspace_persisted = False + try: + resource = await self.resource_repo.get_resource_by_id(resource_id) + resource.deploymentStatus = step_to_update.status + await self.resource_repo.update_item(resource) + resource_dict = await self.resource_repo.get_resource_dict_by_id(resource_id) + resource_to_persist = self.create_updated_resource_document(resource_dict, message) + await self.resource_repo.update_item_dict(resource_to_persist) + workspace_persisted = True + except Exception as ws_err: + logger.error(f"Failed to persist workspace resource status/outputs on final delivery: {ws_err}", exc_info=True) + + primary_persisted = False + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + if primary_resource.deploymentStatus != Status.Deleted: + primary_resource.deploymentStatus = Status.Deleted + await self.resource_repo.update_item(primary_resource) + primary_persisted = True + except Exception as prim_err: + logger.error(f"Failed to restore primary resource to Deleted on final delivery: {prim_err}", exc_info=True) + else: + primary_persisted = True + + operation_persisted = False + if workspace_persisted and primary_persisted: + try: + fresh_op = await self.operations_repo.get_operation_by_id(str(message.operationId)) + if ( + fresh_op is not operation + and ( + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + await self._reconcile_resources_for_terminal_operation(fresh_op) + operation_persisted = True + else: + fresh_step, fresh_is_last, _ = self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=step_to_update.id, + step_status=step_to_update.status, + step_message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + await self.update_overall_operation_status(fresh_op, fresh_step, fresh_is_last) + await self.operations_repo.update_item(fresh_op) + operation_persisted = True + except Exception as op_err: + logger.error(f"Failed to persist terminal operation status on final delivery: {op_err}", exc_info=True) + + if workspace_persisted and primary_persisted and operation_persisted: + logger.info(f"All required resource and operation writes successfully persisted on final delivery for operation {operation.id}. Completing message.") + return True + else: + logger.error( + f"[CLEANUP_FINALIZATION_DEAD_LETTER] Could not persist all required states for " + f"operation {operation.id} on final delivery {MAX_CLEANUP_DELIVERY_COUNT} " + f"(workspace={workspace_persisted}, primary={primary_persisted}, op={operation_persisted}). " + f"Returning False so message is dead-lettered for manual recovery." + ) + return False + else: + return False + else: + raise # more steps in the op to do? if is_last_step is False: @@ -174,40 +443,464 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # catch any errors in updating the resource - maybe Cosmos / schema invalid etc, and report them back to the op try: # parent resource is always retrieved via cosmos, hence it is always with redacted sensitive values - parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) - resource_to_send = await update_resource_for_step( - operation_step=next_step, - resource_repo=self.resource_repo, - resource_template_repo=self.resource_template_repo, - resource_history_repo=self.resource_history_repo, - root_resource=None, - step_resource=parent_resource, - resource_to_update_id=next_step.resourceId, - primary_action=operation.action, - user=operation.user) + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if ( + self._is_cleanup_terminal(operation) + or next_step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + or next_step.is_failure() + or next_step.status in (Status.Updating, Status.UpdatingFailed) + ): + logger.info( + f"Address space cleanup step {next_step.id} in operation {operation.id} " + f"is already completed or in progress. Skipping enqueue." + ) + return True + + workspace = await self.workspace_repo.get_workspace_by_id(next_step.resourceId) + resource_to_send = copy.deepcopy(workspace) + cleanup_details = await self._get_address_cleanup_details(operation) + if cleanup_details: + address_to_free, parent_workspace_id, service_resource_id, _ = cleanup_details + workspace_services_repo = getattr(self, "workspace_services_repo", None) + if workspace_services_repo is None: + from db.repositories.workspace_services import WorkspaceServiceRepository + workspace_services_repo = await WorkspaceServiceRepository.create() + + active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) + conflicting_services = [s for s in active_services if s.id != service_resource_id and s.properties.get("address_space") == address_to_free] + if conflicting_services: + raise AddressSpaceConflictError( + f"Address space {address_to_free} is allocated to active service {conflicting_services[0].id} in workspace {parent_workspace_id}." + ) + + workspace_address_spaces = resource_to_send.properties.get("address_spaces", []) + if address_to_free and isinstance(workspace_address_spaces, list): + resource_to_send.properties["address_spaces"] = [ + a for a in workspace_address_spaces if a != address_to_free + ] + else: + parent_resource = await self.resource_repo.get_resource_by_id(next_step.sourceTemplateResourceId) + resource_to_send = await update_resource_for_step( + operation_step=next_step, + resource_repo=self.resource_repo, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + root_resource=None, + step_resource=parent_resource, + resource_to_update_id=next_step.resourceId, + primary_action=operation.action, + user=operation.user) # create + send the message logger.info(f"Sending next step in operation to deployment queue -> step_id: {next_step.templateStepId}, action: {next_step.resourceAction}") content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=next_step.id, action=next_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=resource_to_send.id, action=next_step.resourceAction) + await send_deployment_message( + content=content, + correlation_id=operation.id, + session_id=resource_to_send.id, + action=next_step.resourceAction, + ) + except AddressSpaceConflictError as e: + logger.error(f"[ADDRESS_SPACE_CONFLICT] {e}") + next_step.message = f"Terminal address space conflict: {e}" + next_step.status = Status.UpdatingFailed + await self.update_overall_operation_status(operation, next_step, is_last_step=True) + # Preserve the primary resource's status based on what the main step actually achieved. + # Reconcile and persist the resource state BEFORE saving the terminal operation so that + # transient failures cause message retry without stranding the resource in DeletingFailed. + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + # Use the status the main step actually achieved rather than + # unconditionally overwriting it with DeletingFailed. + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + await self.operations_repo.update_item(operation) + return True except Exception as e: logger.exception("Unable to send update for resource in pipeline step") + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if is_final_delivery: + next_step.message = f"Failed to enqueue address space cleanup after {MAX_CLEANUP_DELIVERY_COUNT} deliveries: {e}" + next_step.status = Status.UpdatingFailed + await self.update_overall_operation_status(operation, next_step, is_last_step=True) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + # Preserve the status the main step actually achieved. + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + await self.operations_repo.update_item(operation) + return True + else: + next_step.status = Status.AwaitingUpdate + next_step.message = f"Failed to enqueue address space cleanup, will retry: {e}" + operation.status = Status.PipelineRunning + await self.operations_repo.update_item(operation) + return False next_step.message = repr(e) next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) + return True + + if next_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + for attempt in range(MAX_CLEANUP_RETRIES): + try: + next_step.status = Status.Updating + next_step.updatedWhen = get_timestamp() + await self.operations_repo.update_item(operation) + break + except CosmosAccessConditionFailedError: + logger.warning( + f"ETag conflict persisting {Status.Updating} status for cleanup step {next_step.id}. " + f"Reloading operation (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." + ) + try: + fresh_op = await self.operations_repo.get_operation_by_id(operation.id) + if ( + fresh_op is not operation + and ( + getattr(fresh_op, "reconciled", False) + or (fresh_op.action == RequestAction.UnInstall and self._is_cleanup_terminal(fresh_op)) + or fresh_op.status in (Status.Deleted, Status.DeletingFailed) + ) + ): + logger.info( + f"Operation {fresh_op.id} is already terminal ({fresh_op.status}) after cleanup step dispatch. " + f"Reconciling resource writes and skipping stale update." + ) + await self._reconcile_resources_for_terminal_operation(fresh_op) + return True + + if attempt < MAX_CLEANUP_RETRIES - 1: + self._merge_operation_steps( + in_memory_op=operation, + fresh_op=fresh_op, + step_id_to_update=next_step.id, + step_status=Status.Updating, + step_message=next_step.message or "", + ) + await self.operations_repo.update_item(fresh_op) + operation = fresh_op + break + except Exception: + pass + except Exception as post_send_err: + if attempt == MAX_CLEANUP_RETRIES - 1: + logger.error( + f"Failed to persist {Status.Updating} status for address space cleanup step {next_step.id} " + f"in operation {operation.id} after successful dispatch: {post_send_err}", + exc_info=True + ) + else: + logger.warning( + f"Transient error persisting {Status.Updating} status for cleanup step {next_step.id}. " + f"Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})..." + ) result = True except EntityDoesNotExist: - # Marking as true as this message will never succeed anyways and should be removed from the queue. - result = True logger.exception(strings.DEPLOYMENT_STATUS_ID_NOT_FOUND.format(message.id)) + if operation is not None and step_to_update is not None: + try: + failure_status = self.get_failure_status_for_action(operation.action) + step_to_update.status = self.get_failure_status_for_action(step_to_update.resourceAction) + step_to_update.message = f"Resource {message.id} not found in database: operation aborted." + operation.status = failure_status + operation.message = f"Resource {message.id} not found in database: operation aborted." + operation.updatedWhen = get_timestamp() + if operation.resourceId and operation.resourceId != str(message.id): + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" + and op_step.resourceId == operation.resourceId), + None + ) + primary_status = ( + Status.Deleted + if (operation.action == RequestAction.UnInstall and main_step and main_step.is_success()) + else failure_status + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except Exception: + pass + await self.operations_repo.update_item(operation) + result = True + except Exception: + logger.exception(f"Failed to persist terminal failure status for operation {operation.id} when resource {message.id} was missing") + result = False + else: + # Marking as true as this message will never succeed anyways and should be removed from the queue. + result = True except Exception: logger.exception("Failed to update status") return result + def _merge_operation_steps( + self, + in_memory_op: Operation, + fresh_op: Operation, + step_id_to_update: str, + step_status: Status, + step_message: str, + ) -> Tuple[OperationStep, bool, int]: + """Merge in-memory steps into fresh operation reloaded during ETag retry.""" + if fresh_op.steps is None: + fresh_op.steps = [] + + if in_memory_op.steps: + fresh_step_ids = {s.id for s in fresh_op.steps} + for s in in_memory_op.steps: + if s.id not in fresh_step_ids: + fresh_op.steps.append(s) + fresh_step_ids.add(s.id) + + step_to_update = None + current_step_index = 0 + is_last_step = False + for i, s in enumerate(fresh_op.steps): + if s.id == step_id_to_update: + step_to_update = s + current_step_index = i + if i == (len(fresh_op.steps) - 1): + is_last_step = True + break + + if step_to_update is None: + raise Exception(f"Error finding step {step_id_to_update} in reloaded operation {fresh_op.id}") + + step_is_terminal = step_to_update.is_success() or step_to_update.is_failure() + new_status_is_terminal = step_status in ( + Status.ActionSucceeded, + Status.Deployed, + Status.Deleted, + Status.Updated, + Status.ActionFailed, + Status.DeletingFailed, + Status.DeploymentFailed, + Status.UpdatingFailed, + ) + if not step_is_terminal or new_status_is_terminal: + step_to_update.status = step_status + step_to_update.message = step_message + step_to_update.updatedWhen = get_timestamp() + + return step_to_update, is_last_step, current_step_index + + def _is_workspace_service_with_address_space(self, resource: dict) -> bool: + return (resource.get("resourceType") in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value) + and bool(resource.get("properties", {}).get("address_space")) + and bool(resource.get("workspaceId"))) + + def _has_workspace_upgrade_step(self, operation: Operation, current_step_index: int) -> bool: + return any( + step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + for step in operation.steps[current_step_index + 1:] + ) + + def _is_cleanup_completed(self, operation: Operation) -> bool: + for step in operation.steps: + if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if step.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS: + return True + if operation.action == RequestAction.UnInstall and operation.status == Status.Deleted: + if any(step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps): + return True + return False + + def _is_cleanup_failed(self, operation: Operation) -> bool: + for step in operation.steps: + if step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID: + if step.is_failure(): + return True + if operation.action == RequestAction.UnInstall and operation.status == Status.DeletingFailed: + if any(step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID for step in operation.steps): + return True + return False + + def _is_cleanup_terminal(self, operation: Operation) -> bool: + return self._is_cleanup_completed(operation) or self._is_cleanup_failed(operation) + + async def _reconcile_resources_for_terminal_operation(self, operation: Operation) -> None: + """Reconcile resource deployment statuses against terminal operation state.""" + if operation.resourceId: + try: + primary = await self.resource_repo.get_resource_by_id(uuid.UUID(str(operation.resourceId))) + main_step = next( + (op_step for op_step in (operation.steps or []) + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId), + None + ) + target_status = ( + Status.Deleted if (operation.action == RequestAction.UnInstall and main_step and main_step.is_success()) + else operation.status + ) + if primary.deploymentStatus != target_status: + primary.deploymentStatus = target_status + await self.resource_repo.update_item(primary) + except (EntityDoesNotExist, ValueError): + pass + except Exception: + logger.exception(f"Failed to reconcile primary resource for terminal operation {operation.id}") + raise # propagate so Service Bus abandons and retries the message + + if operation.action == RequestAction.UnInstall: + cleanup_step = next( + (s for s in (operation.steps or []) if s.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID), + None + ) + if cleanup_step and cleanup_step.resourceId: + try: + ws_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(str(cleanup_step.resourceId))) + target_status = ( + Status.Updated if cleanup_step.is_success() or operation.status == Status.Deleted + else Status.UpdatingFailed if cleanup_step.is_failure() or operation.status == Status.DeletingFailed + else ws_resource.deploymentStatus + ) + if ws_resource.deploymentStatus != target_status: + ws_resource.deploymentStatus = target_status + await self.resource_repo.update_item(ws_resource) + except (EntityDoesNotExist, ValueError): + pass + except Exception: + logger.exception(f"Failed to reconcile workspace resource for terminal operation {operation.id}") + raise # propagate so Service Bus abandons and retries the message + + def _create_workspace_upgrade_step(self, resource: dict) -> OperationStep: + return OperationStep( + id=str(uuid.uuid4()), + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + stepTitle="Update workspace address spaces", + resourceId=resource["workspaceId"], + resourceTemplateName="", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.AwaitingUpdate, + sourceTemplateResourceId=resource["id"] + ) + + async def _get_address_cleanup_details(self, operation: Operation) -> Optional[Tuple[str, str, str, dict]]: + main_step = next( + (step for step in operation.steps + if step.templateStepId == "main" + and step.resourceId == operation.resourceId + and step.is_success()), + None + ) + if main_step is None: + return None + resource_to_persist = await self.resource_repo.get_resource_dict_by_id(main_step.resourceId) + if resource_to_persist.get("resourceType") not in (ResourceType.WorkspaceService, ResourceType.WorkspaceService.value): + return None + + address_to_free = resource_to_persist.get("properties", {}).get("address_space") + parent_workspace_id = resource_to_persist.get("workspaceId") + resource_id = resource_to_persist.get("id") + + if not address_to_free or not parent_workspace_id: + return None + + return address_to_free, parent_workspace_id, resource_id, resource_to_persist + + async def _free_workspace_address_space(self, operation: Operation) -> bool: + """Free address space from workspace after successful upgrade.""" + address_to_free = parent_workspace_id = resource_id = "" + try: + cleanup_details = await self._get_address_cleanup_details(operation) + if cleanup_details is None: + return True + + address_to_free, parent_workspace_id, resource_id, service_dict = cleanup_details + + if self._is_cleanup_completed(operation): + return True + + if service_dict.get("properties", {}).get("address_space_freed"): + return True + + workspace_services_repo = getattr(self, "workspace_services_repo", None) + if workspace_services_repo is None: + from db.repositories.workspace_services import WorkspaceServiceRepository + workspace_services_repo = await WorkspaceServiceRepository.create() + + active_services = await workspace_services_repo.get_active_workspace_services_for_workspace(parent_workspace_id) + conflicting_services = [s for s in active_services if s.id != resource_id and s.properties.get("address_space") == address_to_free] + if conflicting_services: + raise AddressSpaceConflictError( + f"Address space {address_to_free} is allocated to active service {conflicting_services[0].id} in workspace {parent_workspace_id}." + ) + + for attempt in range(MAX_CLEANUP_RETRIES): + try: + workspace = await self.workspace_repo.get_workspace_by_id(parent_workspace_id) + workspace_address_spaces = workspace.properties.get("address_spaces") or [] + if not isinstance(workspace_address_spaces, list) or address_to_free not in workspace_address_spaces: + service_dict["properties"]["address_space_freed"] = True + await self.resource_repo.update_item_dict(service_dict) + return True + new_address_spaces = [a for a in workspace_address_spaces if a != address_to_free] + workspace_patch = ResourcePatch() + workspace_patch.properties = {"address_spaces": new_address_spaces} + + # Update address spaces in Cosmos DB without triggering deployment + await self.workspace_repo.patch_workspace( + workspace, + workspace_patch, + workspace.etag, + self.resource_template_repo, + self.resource_history_repo, + operation.user, + False + ) + service_dict["properties"]["address_space_freed"] = True + await self.resource_repo.update_item_dict(service_dict) + logger.info(f"Freed address space {address_to_free} from workspace {parent_workspace_id} after successful workspace upgrade for {resource_id}") + return True + except CosmosAccessConditionFailedError: + if attempt == MAX_CLEANUP_RETRIES - 1: + raise + logger.warning(f"ETag conflict when freeing workspace address space after successful workspace upgrade. Retrying (attempt {attempt + 1}/{MAX_CLEANUP_RETRIES})...") + except AddressSpaceConflictError: + raise + except Exception as e: + logger.error(f"[ADDRESS_SPACE_CLEANUP_FAILED] Failed to free workspace address space {address_to_free} for workspace {parent_workspace_id} after upgrading the workspace for {resource_id}: {e}", exc_info=True) + return False + + return False + async def update_overall_operation_status(self, operation: Operation, step: OperationStep, is_last_step: bool): operation.updatedWhen = get_timestamp() @@ -217,6 +910,36 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.message = step.message return + # Finalize as Deleted only when all steps including cleanup succeed + if self._is_cleanup_completed(operation) and operation.action == RequestAction.UnInstall: + if is_last_step and all(op_step.is_success() for op_step in operation.steps): + operation.status = Status.Deleted + operation.message = "Multi step pipeline completed successfully" + return + + # Do not reopen failed cleanup uninstall to PipelineRunning + if self._is_cleanup_failed(operation) and operation.action == RequestAction.UnInstall: + operation.status = Status.DeletingFailed + operation.message = f"Multi step pipeline failed on step {strings.ADDRESS_SPACE_CLEANUP_STEP_ID}" + main_step = next( + (op_step for op_step in operation.steps + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId), + None + ) + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_status = ( + Status.Deleted if main_step.is_success() + else Status.DeletingFailed + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass + return + operation.status = Status.PipelineRunning operation.message = "Multi step pipeline running. See steps for details." @@ -226,19 +949,43 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper # pipeline failed - update the primary resource (from the main step) as failed too main_step = None - for i, step in enumerate(operation.steps): - if step.templateStepId == "main": - main_step = step + for op_step in operation.steps: + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId: + main_step = op_step break if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_status = ( + Status.Deleted if (operation.action == RequestAction.UnInstall and main_step.is_success()) + else operation.status + ) + if primary_resource.deploymentStatus != primary_status: + primary_resource.deploymentStatus = primary_status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass if step.is_success() and is_last_step: - operation.status = self.get_success_status_for_action(operation.action) - operation.message = "Multi step pipeline completed successfully" + if all(op_step.is_success() for op_step in operation.steps): + operation.status = self.get_success_status_for_action(operation.action) + operation.message = "Multi step pipeline completed successfully" + + # pipeline succeeded - update the primary resource (from the main step) as succeeded too + main_step = None + for op_step in operation.steps: + if op_step.templateStepId == "main" and op_step.resourceId == operation.resourceId: + main_step = op_step + break + + if main_step: + try: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + except EntityDoesNotExist: + pass def get_success_status_for_action(self, action: RequestAction): status = Status.ActionSucceeded diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index e1b2daf74d..ceb1f8001c 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -1,3 +1,5 @@ +import json +from datetime import datetime, timedelta, UTC from azure.servicebus import ServiceBusMessage from azure.servicebus.aio import ServiceBusClient from pydantic import TypeAdapter @@ -41,6 +43,23 @@ async def send_deployment_message(content, correlation_id, session_id, action): await _send_message(resource_request_message, config.SERVICE_BUS_RESOURCE_REQUEST_QUEUE) +async def send_airlock_workflow_message(content, delay_seconds: int = 0): + message = ServiceBusMessage(body=json.dumps(content)) + if delay_seconds <= 0: + await _send_message(message, config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + return + + async with credentials.get_credential_async_context() as credential: + service_bus_client = ServiceBusClient(config.SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE, credential) + async with service_bus_client: + sender = service_bus_client.get_queue_sender(queue_name=config.SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE) + async with sender: + await sender.schedule_messages( + message, + scheduled_enqueue_time_utc=datetime.now(UTC) + timedelta(seconds=delay_seconds), + ) + + async def update_resource_for_step(operation_step: OperationStep, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, root_resource: Resource, step_resource: Resource, resource_to_update_id: str, primary_action: str, user: User) -> Resource: # step_resource is the resource instance where the step was defined. e.g. 'add firewall rule' step defined in Guacamole template -> the step_resource is the Guacamole ws service. # root_resource is theresource on which the user chose to update, i.e. the top most resource in cascaded action or the same resource in a non-cascaded action. diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e5fe83ba84..c0ea7b7482 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -1,4 +1,5 @@ import json +from typing import Optional from db.repositories.resources import ResourceRepository from db.repositories.resource_templates import ResourceTemplateRepository @@ -7,14 +8,17 @@ from models.domain.authentication import User from models.domain.request_action import RequestAction -from models.domain.resource import Resource -from models.domain.operation import Operation +from models.domain.resource import Resource, ResourceType +from models.domain.operation import Operation, get_failure_status_for_action -from db.repositories.operations import OperationRepository -from services.logging import tracer +from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError, ResourceModifiedError +from azure.cosmos.exceptions import CosmosAccessConditionFailedError +from db.repositories.operations import OperationRepository, extract_workspace_id_from_resource_path +from services.logging import logger, tracer -async def send_resource_request_message(resource: Resource, operations_repo: OperationRepository, resource_repo: ResourceRepository, user: User, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, action: RequestAction = RequestAction.Install, is_cascade: str = False) -> Operation: +async def send_resource_request_message(resource: Resource, operations_repo: OperationRepository, resource_repo: ResourceRepository, user: User, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, action: RequestAction = RequestAction.Install, is_cascade: str = False, operation_id: Optional[str] = None) -> Operation: """ Creates and sends a resource request message for the resource to the Service Bus. The resource ID is added to the message to serve as an correlation ID for the deployment process. @@ -25,13 +29,23 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope with tracer.start_as_current_span("send_resource_request_message") as current_span: current_span.set_attribute("resource_id", resource.id) current_span.set_attribute("action", action) + caller_operation_id = operation_id + operation_id = operation_id or OperationRepository.create_operation_id() + target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) # Construct the resources to build an operation item for resources_list = [] - if is_cascade: - resources_list = await resource_repo.get_resource_dependency_list(resource) - else: - resources_list.append(resource.__dict__) + try: + if target_workspace_id: + await operations_repo.acquire_workspace_lease(target_workspace_id, operation_id) + if is_cascade: + resources_list = await resource_repo.get_resource_dependency_list(resource) + else: + resources_list.append(resource.__dict__) + except Exception: + if target_workspace_id: + await operations_repo.release_workspace_lease(target_workspace_id, operation_id) + raise # add the operation to the db - this will create all the steps needed (if any are defined in the template) operation = await operations_repo.create_operation_item( @@ -42,26 +56,88 @@ async def send_resource_request_message(resource: Resource, operations_repo: Ope resource_version=resource.resourceVersion, user=user, resource_repo=resource_repo, - resource_template_repo=resource_template_repo) + resource_template_repo=resource_template_repo, + operation_id=operation_id) current_span.set_attribute("operation_id", operation.id) - # prep the first step to send in SB - # resource at this point is the original object with unmaskked values - first_step = operation.steps[0] - current_span.set_attribute("step_id", first_step.id) - resource_to_send = await update_resource_for_step( - operation_step=first_step, - resource_repo=resource_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - root_resource=resource, - step_resource=None, - resource_to_update_id=first_step.resourceId, - primary_action=action, - user=user) + try: + # prep the first step to send in SB + # resource at this point is the original object with unmaskked values + first_step = operation.steps[0] + current_span.set_attribute("step_id", first_step.id) + resource_to_send = await update_resource_for_step( + operation_step=first_step, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + root_resource=resource, + step_resource=None, + resource_to_update_id=first_step.resourceId, + primary_action=action, + user=user) + + # create + send the message + content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) + await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) + except Exception as ex: + logger.exception(f"Failed to dispatch initial deployment message for operation {operation.id}") + lease_released = False + lease_retained = False + should_release_lease = caller_operation_id is None + try: + operation.status = get_failure_status_for_action(action) + operation.message = f"Failed to dispatch initial deployment message: {action}" + if operation.steps: + first_step = operation.steps[0] + first_step.status = get_failure_status_for_action(first_step.resourceAction) + first_step.message = f"Failed to dispatch initial deployment message: {first_step.resourceAction}" + await operations_repo.update_item(operation, release_lease=should_release_lease) + if should_release_lease: + lease_released = True + except Exception: + logger.exception(f"Failed to persist failure state for operation {operation.id}") + # Fallback: remove orphaned operation and release its workspace lease only if it has not been advanced + lease_retained = True + try: + creation_etag = getattr(operation, "etag", None) + if hasattr(operations_repo, "delete_item"): + if creation_etag: + del_call = operations_repo.delete_item( + operation.id, + etag=creation_etag, + match_condition=MatchConditions.IfNotModified + ) + else: + del_call = operations_repo.delete_item(operation.id) + if hasattr(del_call, "__await__"): + await del_call + lease_released = True + lease_retained = False + except (CosmosAccessConditionFailedError, ResourceModifiedError, ResourceExistsError): + logger.warning( + f"Operation {operation.id} was modified concurrently; retaining operation and workspace lease" + ) + lease_retained = True + except Exception: + logger.exception(f"Failed to delete orphaned operation {operation.id}") + lease_retained = True + + if lease_released and should_release_lease: + target_workspace_id = extract_workspace_id_from_resource_path(resource.resourcePath) + if not target_workspace_id: + if getattr(resource, "resourceType", None) == ResourceType.Workspace: + target_workspace_id = resource.id + elif hasattr(resource, "workspaceId") and resource.workspaceId: + target_workspace_id = resource.workspaceId + if target_workspace_id and hasattr(operations_repo, "release_workspace_lease"): + try: + rel_call = operations_repo.release_workspace_lease(target_workspace_id, operation.id) + if hasattr(rel_call, "__await__"): + await rel_call + except Exception: + logger.exception(f"Failed to release workspace lease for {target_workspace_id}") - # create + send the message - content = json.dumps(resource_to_send.get_resource_request_message_payload(operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction)) - await send_deployment_message(content=content, correlation_id=operation.id, session_id=first_step.resourceId, action=first_step.resourceAction) + setattr(ex, "lease_retained", lease_retained) + raise ex return operation diff --git a/api_app/services/airlock.py b/api_app/services/airlock.py index 3c84f56192..6be204c932 100644 --- a/api_app/services/airlock.py +++ b/api_app/services/airlock.py @@ -1,14 +1,17 @@ +import asyncio +import time +import uuid from datetime import datetime, timedelta, UTC from services.logging import logger from azure.storage.blob import generate_container_sas, ContainerSasPermissions, BlobServiceClient from fastapi import HTTPException, status from core import config, credentials -from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource, AirlockReviewDecision, AirlockActions, AirlockFile, AirlockReview +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource, AirlockReviewDecision, AirlockActions, AirlockFile, AirlockReview, AirlockRedeployWorkflow from models.domain.authentication import User from models.domain.workspace import Workspace from models.domain.user_resource import UserResource -from models.domain.operation import Operation +from models.domain.operation import Operation, Status from models.domain.resource import ResourceType from models.domain.workspace_service import WorkspaceService from models.schemas.airlock_request import AirlockReviewInCreate @@ -23,15 +26,17 @@ from api.routes.resource_helpers import save_and_deploy_resource, send_uninstall_message, update_user_resource +from db.errors import EntityDoesNotExist from db.repositories.user_resources import UserResourceRepository from db.repositories.workspace_services import WorkspaceServiceRepository -from db.repositories.operations import OperationRepository +from db.repositories.operations import OperationRepository, WORKSPACE_LEASE_EXPIRY_SECONDS from db.repositories.airlock_requests import AirlockRequestRepository from db.repositories.resource_templates import ResourceTemplateRepository from db.repositories.resources_history import ResourceHistoryRepository from collections import defaultdict from event_grid.event_sender import send_status_changed_event, send_airlock_notification_event +from service_bus.helpers import send_airlock_workflow_message STORAGE_ENDPOINT = config.STORAGE_ENDPOINT_SUFFIX @@ -152,14 +157,15 @@ async def review_airlock_request(airlock_review_input: AirlockReviewInCreate, ai # If there was a VM created for the request, clean it up as it will no longer be needed # In this request, we aren't returning the operations for clean up of VMs, # however the operations still will be saved in the DB and displayed on the UI as normal. - _ = await delete_all_review_user_resources( + await delete_all_review_user_resources( airlock_request=airlock_request, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, resource_template_repo=resource_template_repo, operations_repo=operation_repo, resource_history_repo=resource_history_repo, - user=user + user=user, + enqueue=True ) return updated_airlock_request @@ -173,7 +179,8 @@ def get_airlock_container_link(airlock_request: AirlockRequest, user, workspace) async def create_review_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository) -> Tuple[UserResource, Operation]: + operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + ) -> Tuple[AirlockRequest, Operation]: if airlock_request.type == AirlockRequestType.Import: config = workspace.properties["airlock_review_config"]["import"] review_workspace_id = config["import_vm_workspace_id"] @@ -192,7 +199,19 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac existing_resource = airlock_request.reviewUserResources[user.id] existing_resource = await user_resource_repo.get_user_resource_by_id(workspace_id=existing_resource.workspaceId, service_id=existing_resource.workspaceServiceId, resource_id=existing_resource.userResourceId) logger.info("User already has an existing review resource") - await _handle_existing_review_resource(existing_resource, user, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) + delete_operation = await _handle_existing_review_resource( + existing_resource, user, user_resource_repo, workspace_service_repo, + operation_repo, resource_template_repo, resource_history_repo, + wait_for_completion=False) + if delete_operation and hasattr(delete_operation, "id"): + await send_airlock_workflow_message({ + "workflow": "redeploy", + "airlock_request_id": airlock_request.id, + "operation_id": delete_operation.id, + "uninstall_started": False, + "redeploy_workflow_id": f"{airlock_request.id}:{user.id}:{delete_operation.id}", + }) + return airlock_request, delete_operation # Create the VM user_resource, operation = await _deploy_vm(airlock_request, user, workspace, review_workspace_id, review_workspace_service_id, user_resource_template_name, user_resource_repo, workspace_service_repo, operation_repo, resource_template_repo, resource_history_repo) @@ -206,19 +225,57 @@ async def create_review_vm(airlock_request: AirlockRequest, user: User, workspac review_user_resource=AirlockReviewUserResource( workspaceId=review_workspace_id, workspaceServiceId=review_workspace_service_id, - userResourceId=user_resource.id + userResourceId=user_resource.id, + reviewer=user )) logger.info(f"Airlock Request {updated_resource.id} updated to include {updated_resource.reviewUserResources}") return updated_resource, operation +async def _redeploy_review_vm_after_delete( + airlock_request: AirlockRequest, user: User, workspace: Workspace, + review_workspace_id: str, review_workspace_service_id: str, + user_resource_template_name: str, user_resource_repo: UserResourceRepository, + workspace_service_repo: WorkspaceServiceRepository, + operation_repo: OperationRepository, airlock_request_repo: AirlockRequestRepository, + resource_template_repo: ResourceTemplateRepository, + resource_history_repo: ResourceHistoryRepository, delete_operation_id: str, + existing_resource_workspace_id: Optional[str] = None, + existing_resource_service_id: Optional[str] = None, + existing_resource_id: Optional[str] = None): + try: + await wait_for_successful_operation(operation_repo, delete_operation_id) + user_resource, _ = await _deploy_vm( + airlock_request, user, workspace, review_workspace_id, + review_workspace_service_id, user_resource_template_name, + user_resource_repo, workspace_service_repo, operation_repo, + resource_template_repo, resource_history_repo) + airlock_request = await airlock_request_repo.get_airlock_request_by_id(airlock_request.id) + await update_and_publish_event_airlock_request( + airlock_request, airlock_request_repo, user, workspace, + review_user_resource=AirlockReviewUserResource( + workspaceId=review_workspace_id, + workspaceServiceId=review_workspace_service_id, + userResourceId=user_resource.id, + reviewer=user)) + except Exception: + logger.exception("Failed to replace Airlock review VM after delete operation %s", delete_operation_id) + + async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Workspace, review_workspace_id: str, review_workspace_service_id: str, user_resource_template_name: str, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, operation_repo: OperationRepository, - resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository): + resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + workflow_id: Optional[str] = None, operation_id: Optional[str] = None): logger.info(f"Creating review VM in workspace:{review_workspace_id} service:{review_workspace_service_id} using template:{user_resource_template_name}") - workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=review_workspace_id, service_id=review_workspace_service_id) - airlock_request_sas_url = get_airlock_container_link(airlock_request, user, workspace) + deployment_operation_id = operation_id or str(uuid.uuid4()) + await operation_repo.acquire_workspace_lease(review_workspace_id, deployment_operation_id) + try: + workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=review_workspace_id, service_id=review_workspace_service_id) + airlock_request_sas_url = get_airlock_container_link(airlock_request, user, workspace) + except Exception: + await operation_repo.release_workspace_lease(review_workspace_id, deployment_operation_id) + raise user_resource_create = UserResourceInCreate( templateName=user_resource_template_name, @@ -228,24 +285,33 @@ async def _deploy_vm(airlock_request: AirlockRequest, user: User, workspace: Wor "airlock_request_sas_url": airlock_request_sas_url } ) + if workflow_id is not None: + user_resource_create.properties["airlock_redeploy_workflow_id"] = workflow_id - user_resource, resource_template = await user_resource_repo.create_user_resource_item( - user_resource_create, review_workspace_id, review_workspace_service_id, workspace_service.templateName, user.id, user.roles) + try: + user_resource, resource_template = await user_resource_repo.create_user_resource_item( + user_resource_create, review_workspace_id, review_workspace_service_id, workspace_service.templateName, user.id, user.roles) - operation = await save_and_deploy_resource( - resource=user_resource, - resource_repo=user_resource_repo, - operations_repo=operation_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - user=user, - resource_template=resource_template) + operation = await save_and_deploy_resource( + resource=user_resource, + resource_repo=user_resource_repo, + operations_repo=operation_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + user=user, + resource_template=resource_template, + operation_id=deployment_operation_id) + except Exception as ex: + if not getattr(ex, "lease_retained", False): + await operation_repo.release_workspace_lease(review_workspace_id, deployment_operation_id) + raise return user_resource, operation async def _handle_existing_review_resource(existing_resource: AirlockReviewUserResource, user: User, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, - operation_repo: OperationRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository): + operation_repo: OperationRepository, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, + wait_for_completion: bool = True) -> Optional[Operation]: # Is the existing resource enabled, deployed, and can we get its power state information if existing_resource.isEnabled and existing_resource.deploymentStatus == "deployed" and 'azure_resource_id' in existing_resource.properties: resource_status = get_azure_resource_status(existing_resource.properties['azure_resource_id']) @@ -259,15 +325,17 @@ async def _handle_existing_review_resource(existing_resource: AirlockReviewUserR logger.info("Existing review resource is in an unhealthy state.") if existing_resource.deploymentStatus != "deleted": logger.info("Deleting existing user resource...") - _ = await delete_review_user_resource( + operation = await delete_review_user_resource( user_resource=existing_resource, user_resource_repo=user_resource_repo, workspace_service_repo=workspace_service_repo, resource_template_repo=resource_template_repo, operations_repo=operation_repo, resource_history_repo=resource_history_repo, - user=user - ) + user=user, wait_for_completion=wait_for_completion) + if wait_for_completion and operation and hasattr(operation, "id"): + await wait_for_successful_operation(operation_repo, operation.id) + return operation async def save_and_publish_event_airlock_request(airlock_request: AirlockRequest, airlock_request_repo: AirlockRequestRepository, user: User, workspace: Workspace): @@ -311,6 +379,7 @@ async def update_and_publish_event_airlock_request( status_message: Optional[str] = None, airlock_review: Optional[AirlockReview] = None, review_user_resource: Optional[AirlockReviewUserResource] = None, + redeploy_workflow: Optional[AirlockRedeployWorkflow] = None, ) -> AirlockRequest: try: logger.debug(f"Updating airlock request item: {airlock_request.id}") @@ -321,7 +390,8 @@ async def update_and_publish_event_airlock_request( request_files=request_files, status_message=status_message, airlock_review=airlock_review, - review_user_resource=review_user_resource) + review_user_resource=review_user_resource, + redeploy_workflow=redeploy_workflow) except Exception as e: logger.exception(f'Failed updating airlock_request item {airlock_request}') # If the validation failed, the error was not related to the saving itself @@ -402,6 +472,79 @@ def enrich_requests_with_allowed_actions(requests: List[AirlockRequest], user: U return enriched_requests +TERMINAL_STATUSES = { + Status.Deployed, + Status.DeploymentFailed, + Status.Updated, + Status.UpdatingFailed, + Status.Deleted, + Status.DeletingFailed, + Status.ActionSucceeded, + Status.ActionFailed, +} + + +SUCCESS_STATUSES = { + Status.Deployed, + Status.Updated, + Status.Deleted, + Status.ActionSucceeded, +} + +FAILURE_STATUSES = { + Status.DeploymentFailed, + Status.UpdatingFailed, + Status.DeletingFailed, + Status.ActionFailed, +} + + +async def wait_for_successful_operation( + operations_repo: OperationRepository, + operation_id: str, + timeout: float = WORKSPACE_LEASE_EXPIRY_SECONDS, + poll_interval: float = 0.5, +) -> Optional[Operation]: + if not hasattr(operations_repo, "get_operation_by_id") or not isinstance(operation_id, str): + return None + start = time.time() + while time.time() - start < timeout: + try: + op_call = operations_repo.get_operation_by_id(operation_id) + op = await op_call if hasattr(op_call, "__await__") else op_call + if op is not None: + op_status = getattr(op, "status", None) + if not isinstance(op_status, (str, Status)): + return op + if op_status in SUCCESS_STATUSES: + return op + if op_status in FAILURE_STATUSES: + logger.error(f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Operation {operation_id} failed with status {op_status}: {getattr(op, 'message', '')}" + ) + except HTTPException: + raise + except EntityDoesNotExist: + pass + await asyncio.sleep(poll_interval) + logger.error(f"Timed out waiting for operation {operation_id} to complete successfully") + raise HTTPException( + status_code=status.HTTP_504_GATEWAY_TIMEOUT, + detail=f"Timed out waiting for operation {operation_id} to complete successfully" + ) + + +async def wait_for_terminal_operation( + operations_repo: OperationRepository, + operation_id: str, + timeout: float = WORKSPACE_LEASE_EXPIRY_SECONDS, + poll_interval: float = 0.5, +) -> Optional[Operation]: + return await wait_for_successful_operation(operations_repo, operation_id, timeout=timeout, poll_interval=poll_interval) + + async def delete_review_user_resource( user_resource: UserResource, user_resource_repo: UserResourceRepository, @@ -409,12 +552,17 @@ async def delete_review_user_resource( resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, - user: User) -> Operation: + user: User, wait_for_completion: bool = True) -> Operation: workspace_service = await workspace_service_repo.get_workspace_service_by_id(workspace_id=user_resource.workspaceId, service_id=user_resource.parentWorkspaceServiceId) # disable might contain logic that we need to execute before the deletion of the resource - _ = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) + disable_op = await disable_user_resource(user_resource, user, workspace_service, user_resource_repo, resource_template_repo, operations_repo, resource_history_repo) + if wait_for_completion and disable_op and hasattr(disable_op, "id"): + await wait_for_successful_operation(operations_repo, disable_op.id) + if not wait_for_completion: + logger.info(f"Deferring deletion of user resource {user_resource.id} until disable operation {disable_op.id} completes") + return disable_op logger.info(f"Deleting user resource {user_resource.id} in workspace service {workspace_service.id}") operation = await send_uninstall_message( @@ -453,25 +601,62 @@ async def delete_all_review_user_resources( resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository, - user: User) -> List[Operation]: + user: User, enqueue: bool = False) -> List[Operation]: + if enqueue: + for review_resource in airlock_request.reviewUserResources.values(): + await send_airlock_workflow_message({ + "workflow": "cleanup", + "airlock_request_id": airlock_request.id, + "review_user_id": next(user_id for user_id, resource in airlock_request.reviewUserResources.items() if resource == review_resource), + }) + return [] + operations: List[Operation] = [] for review_ur in airlock_request.reviewUserResources.values(): - user_resource = await user_resource_repo.get_user_resource_by_id( - workspace_id=review_ur.workspaceId, - service_id=review_ur.workspaceServiceId, - resource_id=review_ur.userResourceId - ) - - operation = await delete_review_user_resource( - user_resource=user_resource, - user_resource_repo=user_resource_repo, - workspace_service_repo=workspace_service_repo, - resource_template_repo=resource_template_repo, - operations_repo=operations_repo, - resource_history_repo=resource_history_repo, - user=user - ) - operations.append(operation) + start_time = time.time() + max_lease_wait = WORKSPACE_LEASE_EXPIRY_SECONDS + poll_interval = 2.0 + while True: + try: + user_resource = await user_resource_repo.get_user_resource_by_id( + workspace_id=review_ur.workspaceId, + service_id=review_ur.workspaceServiceId, + resource_id=review_ur.userResourceId + ) + + operation = await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=user_resource_repo, + workspace_service_repo=workspace_service_repo, + resource_template_repo=resource_template_repo, + operations_repo=operations_repo, + resource_history_repo=resource_history_repo, + user=user + ) + operations.append(operation) + break + except EntityDoesNotExist: + logger.info(f"Review user resource {review_ur.userResourceId} already deleted or does not exist") + break + except HTTPException as ex: + if ex.status_code == status.HTTP_409_CONFLICT: + elapsed = time.time() - start_time + if elapsed >= max_lease_wait: + logger.error( + f"Timed out waiting for workspace lease to delete review user resource {review_ur.userResourceId}" + ) + raise + logger.info( + f"Workspace lease contention for review user resource {review_ur.userResourceId}; " + f"waiting for active operation to complete (elapsed {elapsed:.1f}s)..." + ) + await asyncio.sleep(min(poll_interval, max_lease_wait - elapsed)) + poll_interval = min(poll_interval * 1.5, 30.0) + continue + raise + except Exception as ex: + logger.exception(f"Failed to delete review user resource {review_ur.userResourceId}: {ex}") + raise logger.info(f"Started {len(operations)} operations on deleting user resources") return operations @@ -481,7 +666,7 @@ async def cancel_request(airlock_request: AirlockRequest, user: User, workspace: airlock_request_repo: AirlockRequestRepository, user_resource_repo: UserResourceRepository, workspace_service_repo: WorkspaceServiceRepository, resource_template_repo: ResourceTemplateRepository, operations_repo: OperationRepository, resource_history_repo: ResourceHistoryRepository) -> AirlockRequest: updated_request = await update_and_publish_event_airlock_request(airlock_request=airlock_request, airlock_request_repo=airlock_request_repo, updated_by=user, workspace=workspace, new_status=AirlockRequestStatus.Cancelled) - await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user) + await delete_all_review_user_resources(airlock_request, user_resource_repo, workspace_service_repo, resource_template_repo, operations_repo, resource_history_repo, user, enqueue=True) return updated_request diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index 3d3a1c9bf6..90e506f527 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -351,7 +351,7 @@ async def test_post_create_airlock_review_with_illegal_status_change_returns_400 response = await client.post(app.url_path_for(strings.API_REVIEW_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID), json=sample_airlock_review_input_data) assert response.status_code == status.HTTP_400_BAD_REQUEST - @patch("services.airlock.send_uninstall_message") + @patch("services.airlock.send_airlock_workflow_message") @patch("services.airlock.ResourceHistoryRepository.save_item") @patch("services.airlock.UserResourceRepository.update_item_with_etag") @patch("services.airlock.update_user_resource") @@ -362,10 +362,10 @@ async def test_post_create_airlock_review_with_illegal_status_change_returns_400 @patch("services.airlock.AirlockRequestRepository.create_airlock_review_item", return_value=sample_airlock_review_object()) @patch("services.airlock.update_and_publish_event_airlock_request", return_value=sample_airlock_request_object(status=AirlockRequestStatus.Approved, review_user_resource=True)) @patch("services.airlock.AirlockRequestRepository.save_item") - async def test_post_create_airlock_review_cleans_up_review_user_resources(self, _, __, ___, ____, _____, ______, _______, ________, _________, __________, send_uninstall_message_mock, app, client, sample_airlock_review_input_data): + async def test_post_create_airlock_review_cleans_up_review_user_resources(self, _, __, ___, ____, _____, ______, _______, ________, _________, __________, workflow_message_mock, app, client, sample_airlock_review_input_data): response = await client.post(app.url_path_for(strings.API_REVIEW_AIRLOCK_REQUEST, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID), json=sample_airlock_review_input_data) assert response.status_code == status.HTTP_200_OK - assert send_uninstall_message_mock.call_count == 1 + workflow_message_mock.assert_awaited_once() @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.update_and_publish_event_airlock_request", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @@ -411,6 +411,7 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) assert response.status_code == status.HTTP_409_CONFLICT + @patch("services.airlock.send_airlock_workflow_message") @patch("services.airlock.delete_review_user_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="uninstall", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.save_and_deploy_resource", return_value=Operation(id="123", resourceId=USER_RESOURCE_ID, resourcePath="a/b", action="install", createdWhen=time.time(), updatedWhen=time.time())) @patch("services.airlock.get_azure_resource_status", return_value={}) @@ -421,10 +422,13 @@ async def test_post_create_review_user_resource_with_existing_healthy_resource_r @patch("services.airlock.UserResourceRepository.create_user_resource_item", return_value=(UserResource(id=USER_RESOURCE_ID, templateName="test", templateVersion="0.0.1", _etag="123"), "test")) @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview, review_user_resource=True)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id", return_value=sample_workspace(workspace_properties=sample_airlock_review_config())) - async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, app, client): - await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) + async def test_post_create_review_user_resource_with_existing_unhealthy_resource_deletes_previous_and_redeploys(self, _, __, ___, ____, _____, ______, _______, ________, deploy_resource_mock, delete_review_resource_mock, workflow_message_mock, app, client): + response = await client.post(app.url_path_for(strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, workspace_id=WORKSPACE_ID, airlock_request_id=AIRLOCK_REQUEST_ID)) + assert response.status_code == status.HTTP_202_ACCEPTED assert delete_review_resource_mock.call_count == 1 - assert deploy_resource_mock.call_count == 1 + assert response.json()["operation"]["action"] == "uninstall" + workflow_message_mock.assert_awaited_once() + deploy_resource_mock.assert_not_called() # [POST] /workspaces/{workspace_id}/requests/{airlock_request_id}/revoke @patch("services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.Approved)) diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index 31fb31e5f1..cf75981bec 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -264,6 +264,25 @@ async def test_send_uninstall_message_raises_503_on_service_bus_exception(self, assert ex.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + @patch("api.routes.resource_helpers.send_resource_request_message") + @pytest.mark.asyncio + async def test_send_uninstall_message_preserves_retained_lease(self, send_request_mock, resource_repo, resource_history_repo): + dispatch_error = Exception() + dispatch_error.lease_retained = True + send_request_mock.side_effect = dispatch_error + + with pytest.raises(HTTPException) as ex: + await send_uninstall_message( + resource=sample_resource(), + resource_repo=resource_repo, + operations_repo=OperationRepository(), + resource_type=ResourceType.Workspace, + resource_template_repo=None, + resource_history_repo=resource_history_repo, + user=create_test_user()) + + assert ex.value.lease_retained is True + @patch("api.routes.workspaces.ResourceTemplateRepository") @patch("service_bus.resource_request_sender.send_deployment_message") @pytest.mark.asyncio @@ -276,6 +295,7 @@ async def test_save_and_deploy_masks_secrets(self, send_deployment_message_mock, resource_repo.save_item = AsyncMock(return_value=None) resource_repo.get_resource_by_id = AsyncMock(return_value=resource) operations_repo.create_operation_item = AsyncMock(return_value=operation) + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=basic_resource_template) diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index aebe7b0eb1..d17f1d30cb 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -5,7 +5,7 @@ import pytest from mock import patch -from fastapi import status +from fastapi import status, HTTPException from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP, FAKE_UPDATE_TIMESTAMP from tests_ma.test_api.conftest import create_admin_user, create_test_user, create_workspace_owner_user, create_workspace_researcher_user @@ -37,6 +37,13 @@ pytestmark = pytest.mark.asyncio +@pytest.fixture(autouse=True) +def mock_workspace_lease_release(): + with patch("api.routes.workspaces.OperationRepository.release_workspace_lease", new=AsyncMock()), \ + patch("api.routes.workspaces.OperationRepository.assert_address_space_allocator_lease", new=AsyncMock()): + yield + + WORKSPACE_ID = '933ad738-7265-4b5f-9eae-a1a62928772e' SERVICE_ID = 'abcad738-7265-4b5f-9eae-a1a62928772e' USER_RESOURCE_ID = 'a33ad738-7265-4b5f-9eae-a1a62928772a' @@ -686,6 +693,30 @@ async def test_patch_workspace_returns_409_if_bad_etag(self, _, __, update_item_ assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.ETAG_CONFLICT + # [PATCH] /workspaces/{workspace_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_patch_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE, workspace_id=WORKSPACE_ID), json={"isEnabled": False}, headers={"etag": "some-etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_delete_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.delete(app.url_path_for(strings.API_DELETE_WORKSPACE, workspace_id=WORKSPACE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/invoke-action + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + async def test_invoke_action_on_workspace_raises_409_if_workspace_has_active_operation(self, _, __, app, client): + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE, workspace_id=WORKSPACE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [DELETE] /workspaces/{workspace_id} @patch("api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", return_value=[sample_workspace().__dict__]) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @@ -779,8 +810,9 @@ async def test_post_workspace_services_creates_workspace_service(self, _, __, __ @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") - async def test_post_workspace_services_creates_workspace_service_with_address_space(self, create_workspace_service_item_mock, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): + async def test_post_workspace_services_creates_workspace_service_with_address_space(self, create_workspace_service_item_mock, _, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): etag = "some-etag-value" workspace = sample_workspace() workspace.properties["address_spaces"] = ["192.168.0.1/24"] @@ -806,6 +838,96 @@ async def test_post_workspace_services_creates_workspace_service_with_address_sp assert response.status_code == status.HTTP_202_ACCEPTED assert response.json()["operation"]["resourceId"] == SERVICE_ID + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.ResourceHistoryRepository.save_item", return_value=AsyncMock()) + @patch("api.routes.workspaces.save_and_deploy_resource", side_effect=HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.WORKSPACE_HAS_ACTIVE_OPERATION)) + @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) + @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") + @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) + @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") + async def test_post_workspace_services_rolls_back_address_space_if_deploy_fails(self, create_workspace_service_item_mock, _, __, get_workspace_mock, resource_template_repo, ___, update_item_mock, ____, _____, ______, app, client, workspace_service_input, basic_workspace_service_template, basic_resource_template): + etag = "some-etag-value" + workspace = sample_workspace() + workspace.properties["address_spaces"] = ["192.168.0.1/24"] + workspace.etag = etag + get_workspace_mock.return_value = workspace + basic_workspace_service_template.properties["address_space"] = {"type": "string"} + create_workspace_service_item_mock.return_value = [sample_workspace_service(), basic_workspace_service_template] + basic_resource_template.properties["address_spaces"] = {"type": "array", "updateable": True} + resource_template_repo.return_value = basic_resource_template + + modified_workspace = sample_workspace() + modified_workspace.properties["address_spaces"] = ["192.168.0.1/24", "10.1.4.0/24"] + modified_workspace.etag = etag + update_item_mock.return_value = modified_workspace + + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + + assert response.status_code == status.HTTP_409_CONFLICT + assert update_item_mock.call_count == 2 + rollback_call_workspace = update_item_mock.call_args_list[1][0][0] + assert rollback_call_workspace.properties["address_spaces"] == ["192.168.0.1/24"] + + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") + async def test_post_workspace_services_creates_workspace_service_with_address_space_raises_409_if_workspace_has_active_operation(self, create_workspace_service_item_mock, _, get_workspace_mock, __, app, client, workspace_service_input, basic_workspace_service_template): + workspace = sample_workspace() + workspace.properties["address_spaces"] = ["192.168.0.1/24"] + get_workspace_mock.return_value = workspace + basic_workspace_service_template.properties["address_space"]: str = Field() + create_workspace_service_item_mock.return_value = [sample_workspace_service(), basic_workspace_service_template] + + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + async def test_post_workspace_services_raises_409_if_workspace_has_active_operation(self, _, get_workspace_mock, __, app, client, workspace_service_input): + get_workspace_mock.return_value = sample_workspace() + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), json=workspace_service_input) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id}/workspace-services/{service_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_delete_workspace_service_raises_409_if_workspace_has_active_operation(self, _, __, ___, app, client): + response = await client.delete(app.url_path_for(strings.API_DELETE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id} + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_patch_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/invoke-action + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_invoke_action_on_workspace_service_raises_409_if_workspace_has_active_operation(self, _, get_service_mock, __, app, client): + get_service_mock.return_value = sample_workspace_service() + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services @patch("api.dependencies.workspaces.WorkspaceRepository.get_new_address_space", return_value="10.1.4.0/24") @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @@ -841,7 +963,8 @@ async def test_post_workspace_services_raises_400_bad_request_if_input_is_bad(se @patch("api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", return_value=[sample_workspace_service().__dict__]) @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") - async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabled(self, _, get_workspace_service_mock, __, app, client): + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) + async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabled(self, _mock_active_op, _, get_workspace_service_mock, __, app, client): workspace_service = sample_workspace_service() workspace_service.properties["enabled"] = True get_workspace_service_mock.return_value = workspace_service @@ -857,8 +980,9 @@ async def test_delete_workspace_service_raises_400_if_workspace_service_is_enabl @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", return_value=disabled_workspace_service()) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) async def test_delete_workspace_service_raises_404_if_workspace_service_has_active_resources(self, - __, ___, ____, app, + _mock_active_op, __, ___, ____, app, client): response = await client.delete( app.url_path_for(strings.API_DELETE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, @@ -908,9 +1032,10 @@ async def test_get_workspace_service_history_returns_empty_list_when_no_history( @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", return_value=disabled_workspace_service()) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service", return_value=[]) @patch("api.routes.workspaces.send_uninstall_message", return_value=sample_resource_operation(resource_id=SERVICE_ID, operation_id=OPERATION_ID)) - async def test_delete_workspace_service_sends_uninstall_message(self, send_uninstall_mock, __, ___, ____, resource_template_repo, _____, + async def test_delete_workspace_service_sends_uninstall_message(self, send_uninstall_mock, __, _mock_active_op, ___, ____, resource_template_repo, _____, app, client, basic_workspace_service_template): resource_template_repo.return_value = basic_workspace_service_template @@ -922,9 +1047,10 @@ async def test_delete_workspace_service_sends_uninstall_message(self, send_unins @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=False) @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service", return_value=[]) @patch("api.routes.workspaces.send_uninstall_message", return_value=sample_resource_operation(resource_id=SERVICE_ID, operation_id=OPERATION_ID)) - async def test_delete_workspace_service_returns_the_deleted_workspace_service_id(self, __, ___, ____, workspace_service_mock, resource_template_repo, _____, app, client, basic_workspace_service_template): + async def test_delete_workspace_service_returns_the_deleted_workspace_service_id(self, __, ___, _mock_active_op, ____, workspace_service_mock, resource_template_repo, _____, app, client, basic_workspace_service_template): workspace_service = disabled_workspace_service() workspace_service_mock.return_value = workspace_service resource_template_repo.return_value = basic_workspace_service_template @@ -936,6 +1062,7 @@ async def test_delete_workspace_service_returns_the_deleted_workspace_service_id assert response.json()["operation"]["resourceId"] == workspace_service.id # GET /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources + @patch("api.routes.workspaces.enrich_resource_with_available_upgrades", return_value=None) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @patch("api.routes.workspaces.UserResourceRepository.get_user_resources_for_workspace_service") @@ -1504,6 +1631,52 @@ async def test_get_user_resource_raises_404_if_resource_not_found(self, _, __, a response = await client.get(app.url_path_for(strings.API_GET_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID)) assert response.status_code == status.HTTP_404_NOT_FOUND + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_deployed_workspace_service_by_id") + async def test_post_user_resources_raises_409_if_workspace_has_active_operation(self, _, __, get_workspace_mock, app, client, sample_user_resource_input_data): + get_workspace_mock.return_value = sample_workspace() + response = await client.post(app.url_path_for(strings.API_CREATE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), json=sample_user_resource_input_data) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [DELETE] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id} + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_delete_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.delete(app.url_path_for(strings.API_DELETE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID)) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [PATCH] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id} + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_patch_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.patch(app.url_path_for(strings.API_UPDATE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID), json={"isEnabled": False}, headers={"etag": "etag"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources/{resource_id}/invoke-action + @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") + @patch("api.routes.workspaces.OperationRepository.resource_has_active_operation", return_value=True) + @patch("api.dependencies.workspaces.UserResourceRepository.get_user_resource_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + async def test_invoke_action_on_user_resource_raises_409_if_workspace_has_active_operation(self, _, __, get_user_res_mock, ___, ____, app, client): + get_user_res_mock.return_value = sample_user_resource_object() + response = await client.post(app.url_path_for(strings.API_INVOKE_ACTION_ON_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID, resource_id=USER_RESOURCE_ID), params={"action": "restart"}) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.text == strings.WORKSPACE_HAS_ACTIVE_OPERATION + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_deployed_workspace_service_by_id") diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index 102503929a..9edd3d104f 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -7,7 +7,7 @@ from models.domain.workspace import Workspace from tests_ma.test_api.conftest import create_test_user from models.schemas.airlock_request import AirlockRequestInCreate -from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType, AirlockReviewUserResource from db.repositories.airlock_requests import AirlockRequestRepository from db.errors import EntityDoesNotExist @@ -94,16 +94,20 @@ def airlock_request_mock(status=AirlockRequestStatus.Draft): def get_allowed_status_changes(): + changes = [] for current_status, allowed_new_statuses in ALLOWED_STATUS_CHANGES.items(): for new_status in allowed_new_statuses: - yield current_status, new_status + changes.append((current_status, new_status)) + return changes def get_forbidden_status_changes(): + changes = [] for current_status, allowed_new_statuses in ALLOWED_STATUS_CHANGES.items(): forbidden_new_statuses = list(set(ALL_STATUSES) - set(allowed_new_statuses)) for new_status in forbidden_new_statuses: - yield current_status, new_status + changes.append((current_status, new_status)) + return changes async def test_get_airlock_request_by_id(airlock_request_repo): @@ -181,14 +185,24 @@ async def test_update_airlock_request_with_forbidden_status_should_fail_on_valid await airlock_request_repo.update_airlock_request(mock_existing_request, user, new_status) -@patch("db.repositories.airlock_requests.AirlockRequestRepository.update_airlock_request_item", side_effect=[CosmosAccessConditionFailedError, None]) +@patch("db.repositories.airlock_requests.AirlockRequestRepository.update_airlock_request_item", side_effect=[CosmosAccessConditionFailedError(), None]) @patch("db.repositories.airlock_requests.AirlockRequestRepository.get_airlock_request_by_id", return_value=airlock_request_mock(status=DRAFT)) async def test_update_airlock_request_should_retry_update_when_etag_is_not_up_to_date(_, update_airlock_request_item_mock, airlock_request_repo): expected_update_attempts = 2 user = create_test_user() mock_existing_request = airlock_request_mock(status=DRAFT) - await airlock_request_repo.update_airlock_request(original_request=mock_existing_request, updated_by=user, new_status=SUBMITTED) + review_user_resource = AirlockReviewUserResource( + workspaceId="review-workspace", + workspaceServiceId="review-service", + userResourceId="replacement-resource") + await airlock_request_repo.update_airlock_request( + original_request=mock_existing_request, + updated_by=user, + new_status=SUBMITTED, + review_user_resource=review_user_resource) assert update_airlock_request_item_mock.call_count == expected_update_attempts + retried_request = update_airlock_request_item_mock.call_args_list[1].args[1] + assert review_user_resource in retried_request.reviewUserResources.values() async def test_get_airlock_requests_queries_db(airlock_request_repo): diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index c0c5f9b09c..c122adcb95 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -1,12 +1,16 @@ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import uuid import pytest_asyncio import pytest from mock import patch from db.repositories.resource_templates import ResourceTemplateRepository -from models.domain.operation import Status +from models.domain.operation import Operation, OperationStep, Status +from models.domain.request_action import RequestAction +from models.domain.resource import ResourceType from db.repositories.resources import ResourceRepository from db.repositories.operations import OperationRepository +from db.errors import EntityDoesNotExist +from resources import strings from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP RESOURCE_ID = str(uuid.uuid4()) @@ -65,3 +69,672 @@ async def test_create_operation_steps_from_multi_step_template(_, __, ___, resou ) assert operation.model_dump() == expected_op.model_dump() + + +async def test_create_operation_steps_excludes_address_space_cleanup_for_cascade( + resource_repo, test_user, operations_repo, basic_shared_service, resource_template_repo +): + resource_template = MagicMock() + resource_template.model_dump.return_value = { + "name": "workspace-service", + "resourceType": ResourceType.WorkspaceService, + "pipeline": { + "uninstall": [ + {"stepId": "main"}, + { + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "stepTitle": "Upgrade workspace", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ] + } + } + resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=resource_template) + operations_repo.save_item = AsyncMock() + + operation = await operations_repo.create_operation_item( + resource_id="workspace-root", + resource_list=[{**basic_shared_service.__dict__, "id": "descendant-service"}], + action=RequestAction.UnInstall, + resource_path="/workspaces/workspace-root", + resource_version=0, + user=test_user, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo + ) + + assert [step.templateStepId for step in operation.steps] == ["main"] + + +async def test_resource_has_active_operation_returns_true_when_active_operation_exists(operations_repo): + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" + operations_repo.query = AsyncMock(return_value=[{"id": "op-1"}]) + result = await operations_repo.resource_has_active_operation(workspace_id) + assert result is True + operations_repo.query.assert_called_once() + query_str = operations_repo.query.call_args[1]["query"] if "query" in operations_repo.query.call_args[1] else operations_repo.query.call_args[0][0] + assert workspace_id in query_str + assert "ARRAY_CONTAINS" in query_str + assert f'CONTAINS(c.resourcePath, "{workspace_id}")' in query_str + assert 'NOT CONTAINS(c.resourcePath, "/user-resources/")' not in query_str + + +async def test_resource_has_active_operation_returns_false_when_no_active_operation_exists(operations_repo): + operations_repo.query = AsyncMock(return_value=[]) + result = await operations_repo.resource_has_active_operation("7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d") + assert result is False + + +async def test_resource_has_active_operation_returns_false_for_non_uuid_resource_id(operations_repo): + operations_repo.query = AsyncMock() + result = await operations_repo.resource_has_active_operation("not-a-uuid") + assert result is False + operations_repo.query.assert_not_called() + + +async def test_update_item_with_etag_calls_replace_item(operations_repo): + from azure.core import MatchConditions + operations_repo._container = MagicMock() + operations_repo._container.replace_item = AsyncMock(return_value={"_etag": "\"new-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="old-etag" + ) + result = await operations_repo.update_item(op) + + operations_repo._container.replace_item.assert_awaited_once() + call_kwargs = operations_repo._container.replace_item.call_args[1] + assert call_kwargs["item"] == "op-1" + assert call_kwargs["etag"] == "old-etag" + assert call_kwargs["match_condition"] == MatchConditions.IfNotModified + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] + assert result.etag == "new-etag" + + +async def test_update_item_without_etag_calls_upsert_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.upsert_item = AsyncMock(return_value={"_etag": "\"new-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + ) + result = await operations_repo.update_item(op) + + operations_repo._container.upsert_item.assert_awaited_once() + call_kwargs = operations_repo._container.upsert_item.call_args[1] + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] + assert result.etag == "new-etag" + + +async def test_save_item_excludes_etag_from_body(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={"_etag": "\"created-etag\""}) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="initial-etag" + ) + await operations_repo.save_item(op) + + operations_repo._container.create_item.assert_awaited_once() + call_kwargs = operations_repo._container.create_item.call_args[1] + assert "etag" not in call_kwargs["body"] + assert "_etag" not in call_kwargs["body"] + assert op.etag == "created-etag" + + +async def test_update_item_raises_cosmos_access_condition_failed_error_on_mismatch(operations_repo): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + operations_repo._container = MagicMock() + operations_repo._container.replace_item = AsyncMock(side_effect=CosmosAccessConditionFailedError()) + + op = Operation( + id="op-1", + resourceId="res-1", + resourcePath="/workspaces/res-1", + action="install", + etag="stale-etag" + ) + with pytest.raises(CosmosAccessConditionFailedError): + await operations_repo.update_item(op) + + +async def test_acquire_workspace_lease_creates_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.create_item.assert_awaited_once() + body = operations_repo._container.create_item.call_args[1]["body"] + assert body["id"] == "lease_ws-1" + assert body["workspaceId"] == "ws-1" + assert body["operationId"] == "op-1" + + +async def test_acquire_workspace_lease_rejects_legacy_active_operation_after_create(operations_repo): + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + operations_repo.resource_has_active_operation = AsyncMock(return_value=True) + operations_repo.release_workspace_lease = AsyncMock() + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + + assert exc.value.status_code == 409 + operations_repo.resource_has_active_operation.assert_awaited_once_with("ws-1", exclude_operation_id="op-1") + operations_repo.release_workspace_lease.assert_awaited_once_with("ws-1", "op-1") + + +async def test_acquire_workspace_lease_raises_409_if_active_operation_exists(operations_repo): + from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": now, + }) + active_op = MagicMock(status=Status.Deploying, updatedWhen=now, createdWhen=now) + operations_repo.get_operation_by_id = AsyncMock(return_value=active_op) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_workspace_lease_raises_409_if_existing_lease_operation_not_found(operations_repo): + from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_workspace_lease_succeeds_if_same_operation_holds_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1", + "createdWhen": operations_repo.get_timestamp(), + }) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + + +async def test_acquire_workspace_lease_replaces_lease_if_previous_operation_terminal(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp() - 1000, + "_etag": "old-etag", + }) + operations_repo.get_operation_by_id = AsyncMock(return_value=MagicMock(status=Status.Deployed)) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.replace_item.assert_awaited_once() + + +async def test_acquire_workspace_lease_reclaims_orphaned_lease_when_operation_not_found_after_expiry(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "orphaned-op", + "createdWhen": operations_repo.get_timestamp() - 8000, + "_etag": "old-etag", + }) + operations_repo.get_operation_by_id = AsyncMock(side_effect=EntityDoesNotExist()) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + operations_repo._container.replace_item.assert_awaited_once() + + +async def test_acquire_workspace_lease_retries_and_succeeds_when_lease_deleted_during_conflict_race(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosResourceNotFoundError + operations_repo._container = MagicMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + # 1st attempt: create fails with conflict, read fails because lease was deleted + # 2nd attempt: create succeeds! + operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) + operations_repo.read_item_by_id = AsyncMock(side_effect=CosmosResourceNotFoundError()) + + res = await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert res is True + assert operations_repo._container.create_item.await_count == 2 + + +async def test_release_workspace_lease_deletes_item(operations_repo): + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1" + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_workspace_lease("ws-1", "op-1") + operations_repo.delete_item.assert_awaited_once_with("lease_ws-1") + + +async def test_release_workspace_lease_deletes_item_with_etag(operations_repo): + from azure.core import MatchConditions + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "op-1", + "_etag": "lease-etag", + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_workspace_lease("ws-1", "op-1") + operations_repo.delete_item.assert_awaited_once_with("lease_ws-1", etag="lease-etag", match_condition=MatchConditions.IfNotModified) + + +async def test_acquire_workspace_lease_reconciles_expired_active_operation(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": now - 8000, + "_etag": "lease-etag", + }) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 8000, createdWhen=now - 8000) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock() + operations_repo._reconcile_resource_status = AsyncMock() + + assert await operations_repo.acquire_workspace_lease("ws-1", "op-1") is True + assert stale_op.status == Status.DeploymentFailed + assert stale_op.reconciled is True + operations_repo.update_item.assert_awaited() + + +async def test_acquire_workspace_lease_retries_unreconciled_terminal_operation(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "terminal-op", + "createdWhen": operations_repo.get_timestamp(), + "_etag": "lease-etag", + }) + terminal_op = MagicMock(id="terminal-op", status=Status.DeploymentFailed, reconciled=False, steps=[]) + operations_repo.get_operation_by_id = AsyncMock(return_value=terminal_op) + persisted_reconciled_states = [] + operations_repo.update_item = AsyncMock(side_effect=lambda item, **kwargs: persisted_reconciled_states.append(item.reconciled)) + operations_repo._reconcile_operation_resources = AsyncMock(side_effect=Exception("Cosmos write error")) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "new-op") + + assert exc.value.status_code == 409 + operations_repo._container.replace_item.assert_not_called() + operations_repo.update_item.assert_awaited_once_with(terminal_op, release_lease=False) + assert persisted_reconciled_states == [False] + assert terminal_op.reconciled is False + + operations_repo._reconcile_operation_resources.side_effect = None + operations_repo.update_item.reset_mock() + operations_repo.release_workspace_lease = AsyncMock() + assert await operations_repo.acquire_workspace_lease("ws-1", "new-op") is True + assert terminal_op.reconciled is True + assert operations_repo.update_item.await_count == 2 + assert persisted_reconciled_states == [False, False, True] + assert operations_repo.update_item.await_args_list[0].args == (terminal_op,) + assert operations_repo.update_item.await_args_list[0].kwargs == {"release_lease": False} + assert operations_repo.update_item.await_args_list[1].args == (terminal_op,) + assert operations_repo.update_item.await_args_list[1].kwargs == {"release_lease": False} + operations_repo.release_workspace_lease.assert_awaited_once_with("ws-1", "terminal-op") + + +async def test_acquire_workspace_lease_fails_closed_when_reconciliation_fails(operations_repo): + from fastapi import HTTPException + from azure.cosmos.exceptions import CosmosResourceExistsError + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + now = operations_repo.get_timestamp() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": now - 8000, + "_etag": "lease-etag", + }) + stale_op = MagicMock(action="install", status=Status.AwaitingDeployment, updatedWhen=now - 8000, createdWhen=now - 8000) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_workspace_lease("ws-1", "op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_address_space_allocator_lease_creates_item(operations_repo): + from db.repositories.operations import ADDRESS_SPACE_ALLOCATOR_LEASE_ID + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(return_value={}) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + body = operations_repo._container.create_item.call_args.kwargs["body"] + assert body["id"] == ADDRESS_SPACE_ALLOCATOR_LEASE_ID + assert body["operationId"] == "op-1" + + +async def test_assert_address_space_allocator_lease_requires_current_owner(operations_repo): + from db.repositories.operations import ADDRESS_SPACE_ALLOCATOR_LEASE_ID + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": ADDRESS_SPACE_ALLOCATOR_LEASE_ID, + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + + with pytest.raises(HTTPException) as exc: + await operations_repo.assert_address_space_allocator_lease("op-1") + + assert exc.value.status_code == 409 + + +async def test_reconcile_operation_resources_uses_last_status_per_resource(operations_repo): + operation = MagicMock( + resourceId="workspace-id", + status=Status.Updated, + message="workspace update", + ) + operations_repo._reconcile_resource_status = AsyncMock() + + await operations_repo._reconcile_operation_resources(operation, [ + ("workspace-id", Status.Updated, "first workspace update"), + ("workspace-id", Status.UpdatingFailed, "final workspace update"), + ("service-id", Status.DeletingFailed, "service cleanup"), + ]) + + assert [call.args for call in operations_repo._reconcile_resource_status.await_args_list] == [ + ("workspace-id", Status.UpdatingFailed, "final workspace update"), + ("service-id", Status.DeletingFailed, "service cleanup"), + ] + + +async def test_acquire_address_space_allocator_lease_is_idempotent(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "op-1", + "createdWhen": operations_repo.get_timestamp(), + }) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + + +async def test_acquire_address_space_allocator_lease_rejects_active_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from fastapi import HTTPException + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp(), + }) + + with pytest.raises(HTTPException) as exc: + await operations_repo.acquire_address_space_allocator_lease("op-1") + assert exc.value.status_code == 409 + + +async def test_acquire_address_space_allocator_lease_reclaims_expired_lease(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=CosmosResourceExistsError()) + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + "createdWhen": operations_repo.get_timestamp() - 301, + "_etag": "old-etag", + }) + operations_repo._container.replace_item = AsyncMock(return_value={}) + + assert await operations_repo.acquire_address_space_allocator_lease("op-1") is True + operations_repo._container.replace_item.assert_awaited_once() + assert operations_repo._container.replace_item.call_args.kwargs["etag"] == "old-etag" + + +async def test_release_address_space_allocator_lease_only_releases_owner(operations_repo): + operations_repo._container = MagicMock() + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "address_space_allocator", + "operationId": "other-op", + }) + operations_repo.delete_item = AsyncMock() + + await operations_repo.release_address_space_allocator_lease("op-1") + operations_repo.delete_item.assert_not_awaited() + + +async def test_resource_has_active_operation_rejects_stale_timestamp_without_heartbeat(operations_repo): + now = operations_repo.get_timestamp() + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" + stale_op_dict = { + "id": "op-stale", + "resourceId": workspace_id, + "resourcePath": f"/workspaces/{workspace_id}", + "action": "install", + "status": Status.AwaitingDeployment, + "createdWhen": now - 8000, + "updatedWhen": now - 8000, + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock() + operations_repo._reconcile_resource_status = AsyncMock() + + result = await operations_repo.resource_has_active_operation(workspace_id) + assert result is True + operations_repo.update_item.assert_not_awaited() + + +async def test_resource_has_active_operation_fails_closed_when_reconciliation_fails(operations_repo): + now = operations_repo.get_timestamp() + workspace_id = "7c5b2dc2-6b4c-4c7f-8d3e-1f5a9b0e2c4d" + stale_op_dict = { + "id": "op-stale", + "resourceId": workspace_id, + "resourcePath": f"/workspaces/{workspace_id}", + "action": "install", + "status": Status.AwaitingDeployment, + "createdWhen": now - 8000, + "updatedWhen": now - 8000, + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock(side_effect=Exception("Cosmos write error")) + + result = await operations_repo.resource_has_active_operation(workspace_id) + assert result is True + + +async def test_get_active_operations_for_resource_excludes_terminal_statuses(operations_repo): + operations_repo.query = AsyncMock(return_value=[]) + + await operations_repo.get_active_operations_for_resource("res-1") + operations_repo.query.assert_awaited_once() + query_str = operations_repo.query.call_args[1]["query"] + assert 'c.resourceId = "res-1"' in query_str + assert 'NOT ARRAY_CONTAINS(["deployed", "deployment_failed", "updated", "updating_failed", "deleted", "deleting_failed", "action_succeeded", "action_failed"], c.status)' in query_str + + +async def test_build_step_list_tolerates_missing_step_title(operations_repo): + ws_id = str(uuid.uuid4()) + svc_id = str(uuid.uuid4()) + resource_template_dict = { + "pipeline": { + "upgrade": [ + { + "stepId": "step-1", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + } + ] + } + } + mock_resource_repo = MagicMock() + mock_workspace = MagicMock(id=ws_id, templateName="tre-workspace", resourceType=ResourceType.Workspace) + mock_service = MagicMock(id=svc_id, workspaceId=ws_id, resourceType=ResourceType.WorkspaceService) + mock_resource_repo.get_resource_by_id = AsyncMock(side_effect=[mock_service, mock_workspace]) + + steps = await operations_repo.build_step_list( + steps=[], + resource_template_dict=resource_template_dict, + action="upgrade", + resource_repo=mock_resource_repo, + resource_id=svc_id, + status=Status.AwaitingUpdate, + message="updating", + ) + assert len(steps) == 1 + assert steps[0].stepTitle is None + assert steps[0].templateStepId == "step-1" + + +async def test_acquire_workspace_lease_reconciles_expired_active_steps(operations_repo): + from azure.cosmos.exceptions import CosmosResourceExistsError + from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS + + operations_repo._container = MagicMock() + operations_repo._container.create_item = AsyncMock(side_effect=[CosmosResourceExistsError(), {}]) + + stale_time = operations_repo.get_timestamp() - WORKSPACE_LEASE_EXPIRY_SECONDS - 100 + operations_repo.read_item_by_id = AsyncMock(return_value={ + "id": "lease_ws-1", + "operationId": "stale-op", + "createdWhen": stale_time, + }) + + step1 = OperationStep( + id="step-1", + templateStepId="main", + resourceId="step-1-resource", + resourceAction="uninstall", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + templateStepId="cleanup", + resourceId="step-2-resource", + resourceAction="upgrade", + status=Status.Updating, + ) + stale_op = Operation( + id="stale-op", + resourceId="ws-1", + resourcePath="ws/ws-1", + action="uninstall", + status=Status.Deleting, + createdWhen=stale_time, + updatedWhen=stale_time, + steps=[step1, step2], + ) + operations_repo.get_operation_by_id = AsyncMock(return_value=stale_op) + operations_repo.update_item = AsyncMock(return_value=None) + operations_repo._reconcile_resource_status = AsyncMock() + operations_repo._container.delete_item = AsyncMock() + operations_repo.resource_has_active_operation = AsyncMock(return_value=False) + + assert await operations_repo.acquire_workspace_lease("ws-1", "new-op") is True + assert stale_op.status == Status.DeletingFailed + assert stale_op.reconciled is True + assert operations_repo.update_item.await_count == 2 + operations_repo._reconcile_resource_status.assert_any_await( + "ws-1", Status.DeletingFailed, "Operation was reconciled after its workspace lease expired" + ) + operations_repo._reconcile_resource_status.assert_any_await( + "step-1-resource", Status.DeletingFailed, "" + ) + operations_repo._reconcile_resource_status.assert_any_await( + "step-2-resource", Status.UpdatingFailed, "" + ) + + +async def test_resource_has_active_operation_keeps_stale_pipeline_active(operations_repo): + from db.repositories.operations import WORKSPACE_LEASE_EXPIRY_SECONDS + + stale_time = operations_repo.get_timestamp() - WORKSPACE_LEASE_EXPIRY_SECONDS - 100 + res_id = str(uuid.uuid4()) + stale_op_dict = { + "id": "stale-op", + "resourceId": res_id, + "resourcePath": f"res/{res_id}", + "action": "uninstall", + "status": "deleting", + "createdWhen": stale_time, + "updatedWhen": stale_time, + "steps": [ + { + "id": "step-1", + "templateStepId": "cleanup", + "resourceAction": "upgrade", + "status": "updating", + } + ], + } + operations_repo.query = AsyncMock(return_value=[stale_op_dict]) + operations_repo.update_item = AsyncMock(return_value=None) + operations_repo._reconcile_resource_status = AsyncMock() + + has_active = await operations_repo.resource_has_active_operation(res_id) + assert has_active is True + operations_repo.update_item.assert_not_awaited() + operations_repo._reconcile_resource_status.assert_not_awaited() diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py index 970b4721e7..a61f995fb1 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_templates_repository.py @@ -5,9 +5,11 @@ from db.repositories.resource_templates import ResourceTemplateRepository from db.errors import EntityDoesNotExist, InvalidInput +from models.domain.request_action import RequestAction from models.domain.resource import ResourceType from models.domain.resource_template import ResourceTemplate from models.schemas.workspace_template import WorkspaceTemplateInCreate +from resources import strings pytestmark = pytest.mark.asyncio @@ -293,3 +295,79 @@ async def test_create_template_with_null_pipeline_creates_template_without_pipel input_user_resource_template.json_schema["pipeline"] = None created = await resource_template_repo.create_template(input_user_resource_template, ResourceType.UserResource) assert created.pipeline is None + + +@pytest.mark.parametrize( + "pipeline", + [ + { + "install": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + }, + { + "upgrade": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + }, + { + "uninstall": [{"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}, {"stepId": "main"}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.SharedService, "resourceAction": RequestAction.Upgrade}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.UnInstall}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade}], + }, + { + "uninstall": [{"stepId": "main"}, {"stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, "resourceType": ResourceType.Workspace, "resourceAction": RequestAction.Upgrade, "stepTitle": ""}], + }, + ], +) +async def test_create_template_with_invalid_address_space_cleanup_pipeline_fails(resource_template_repo, input_workspace_service_template, pipeline): + input_workspace_service_template.json_schema["pipeline"] = pipeline + with pytest.raises(InvalidInput): + await resource_template_repo.create_template(input_workspace_service_template, ResourceType.WorkspaceService) + + +@pytest.mark.parametrize( + "non_ws_resource_type", + [ + ResourceType.UserResource, + ResourceType.Workspace, + ResourceType.SharedService, + ], +) +async def test_create_template_with_address_space_cleanup_on_non_workspace_service_fails(resource_template_repo, input_user_resource_template, non_ws_resource_type): + input_user_resource_template.json_schema["pipeline"] = { + "uninstall": [ + {"stepId": "main"}, + { + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "stepTitle": "Cleanup address space", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ], + } + with pytest.raises(InvalidInput, match=f"Step '{strings.ADDRESS_SPACE_CLEANUP_STEP_ID}' is only allowed in '{ResourceType.WorkspaceService}' templates"): + await resource_template_repo.create_template(input_user_resource_template, non_ws_resource_type) + + +@patch('db.repositories.resource_templates.ResourceTemplateRepository.save_item') +async def test_create_template_with_valid_address_space_cleanup_as_final_uninstall_step_succeeds(_, resource_template_repo, input_workspace_service_template): + input_workspace_service_template.json_schema["pipeline"] = { + "uninstall": [ + {"stepId": "main"}, + { + "stepId": strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + "stepTitle": "Cleanup address space", + "resourceType": ResourceType.Workspace, + "resourceAction": RequestAction.Upgrade, + }, + ], + } + created = await resource_template_repo.create_template(input_workspace_service_template, ResourceType.WorkspaceService) + assert created.pipeline diff --git a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py index 6c5463bf76..0f9c722460 100644 --- a/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_airlock_request_status_update.py @@ -212,11 +212,12 @@ async def test_receiving_good_message(_, logging_mock, workspace_repo, airlock_r airlock_request_repo.return_value.update_airlock_request.assert_called_once_with( original_request=expected_airlock_request, updated_by=expected_airlock_request.updatedBy, - new_status=test_sb_step_result_message["data"]["new_status"], + new_status=AirlockRequestStatus.InReview, request_files=None, status_message=None, airlock_review=None, - review_user_resource=None) + review_user_resource=None, + redeploy_workflow=None) assert eg_client().send.call_count == 2 logging_mock.assert_not_called() diff --git a/api_app/tests_ma/test_service_bus/test_airlock_workflow.py b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py new file mode 100644 index 0000000000..c9232436ca --- /dev/null +++ b/api_app/tests_ma/test_service_bus/test_airlock_workflow.py @@ -0,0 +1,229 @@ +import json +from unittest.mock import AsyncMock, MagicMock, patch + +from fastapi import HTTPException +import pytest + +from db.errors import EntityDoesNotExist +from models.domain.authentication import User +from models.domain.airlock_request import AirlockRequest, AirlockRedeployWorkflow, AirlockRequestType, AirlockReview, AirlockReviewDecision, AirlockReviewUserResource +from service_bus.airlock_workflow import AirlockWorkflowUpdater + + +@pytest.fixture +def updater(): + workflow = AirlockWorkflowUpdater() + workflow.user_resource_repo = MagicMock() + workflow.workspace_service_repo = MagicMock() + workflow.operations_repo = MagicMock() + workflow.resource_template_repo = MagicMock() + workflow.resource_history_repo = MagicMock() + workflow.airlock_request_repo = MagicMock() + workflow.workspace_repo = MagicMock() + workflow.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User.model_validate(user_payload()), + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")} + )) + workflow.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock(properties={ + "airlock_review_config": {"import": { + "import_vm_workspace_id": "review-workspace-id", + "import_vm_workspace_service_id": "service-id", + "import_vm_user_resource_template_name": "template"}} + })) + return workflow + + +def message(payload): + return json.dumps(payload) + + +def user_payload(): + return User(id="user-id", name="Test User", email="test@example.com").model_dump() + + +@pytest.mark.asyncio +async def test_process_message_completes_cleanup_when_resource_is_already_deleted(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + + result = await updater.process_message(message({ + "workflow": "cleanup", + "airlock_request_id": "request-id", + "review_user_id": "user-id", + })) + + assert result is True + updater.workspace_repo.get_workspace_by_id.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_message_reschedules_workspace_lease_contention(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(return_value=MagicMock(user=user_payload())) + updater.workspace_service_repo.get_workspace_service_by_id = AsyncMock() + lease_error = HTTPException(status_code=409, detail="Workspace has an active operation in progress") + + with patch("service_bus.airlock_workflow.disable_user_resource", new=AsyncMock(side_effect=lease_error)), \ + patch("service_bus.airlock_workflow.send_airlock_workflow_message", new=AsyncMock()) as schedule: + result = await updater.process_message(message({ + "workflow": "cleanup", + "airlock_request_id": "request-id", + "review_user_id": "user-id", + })) + + assert result is True + schedule.assert_awaited_once() + assert schedule.await_args.kwargs["delay_seconds"] == 30 + assert schedule.await_args.args[0]["lease_contention_retry_count"] == 1 + + +@pytest.mark.asyncio +async def test_process_message_redeploys_without_duplicate_uninstall(updater): + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + + payload = { + "workflow": "redeploy", + "airlock_request_id": "request-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + "redeploy_workflow_id": "request-id:user-id:delete-operation-id", + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()) as uninstall, \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock(return_value=(MagicMock(id="new-resource-id"), MagicMock()))), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message(payload)) + + assert result is True + uninstall.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_message_redeploys_legacy_resource_using_review_data(updater): + workflow_id = "request-id:user-id:delete-operation-id" + updater.user_resource_repo.get_user_resource_by_id = AsyncMock(side_effect=EntityDoesNotExist) + updater.user_resource_repo.get_user_resource_by_workflow_id = AsyncMock( + return_value=MagicMock(id="replacement-resource-id")) + updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User(id="requester-id", name="Requester", email="requester@example.com").model_dump(), + reviews=[AirlockReview( + id="review-id", + reviewDecision=AirlockReviewDecision.Approved, + reviewer=User.model_validate(user_payload()))], + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")} + )) + + payload = { + "workflow": "redeploy", + "redeploy_workflow_id": workflow_id, + "airlock_request_id": "request-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock(return_value=(MagicMock(id="new-resource-id"), MagicMock()))), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message(payload)) + + assert result is True + + +@pytest.mark.asyncio +async def test_process_message_acknowledges_unknown_workflow_without_side_effects(updater): + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()) as wait_for_operation, \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()) as uninstall: + result = await updater.process_message(message({ + "workflow": "future-workflow", + "airlock_request_id": "request-id", + })) + + assert result is True + updater.user_resource_repo.get_user_resource_by_id.assert_not_called() + wait_for_operation.assert_not_awaited() + uninstall.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_message_rejects_forged_review_user_context(updater): + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.send_uninstall_message", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock()), \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()): + result = await updater.process_message(message({ + "workflow": "cleanup", + "airlock_request_id": "request-id", + "review_user_id": "attacker-id", + "user": user_payload(), + "review_workspace_id": "attacker-workspace-id", + "review_workspace_service_id": "attacker-service-id", + "user_resource_id": "attacker-resource-id", + })) + + assert result is False + updater.user_resource_repo.get_user_resource_by_id.assert_not_called() + + +@pytest.mark.asyncio +async def test_process_message_recovers_deploying_redeploy_without_deploying_again(updater): + workflow_id = "request-id:user-id" + updater.user_resource_repo.get_user_resource_by_id = AsyncMock( + side_effect=[EntityDoesNotExist, MagicMock(id="replacement-resource-id")]) + updater.operations_repo.get_operation_by_id = AsyncMock( + return_value=MagicMock(status="deploying", resourceId="replacement-resource-id")) + updater.airlock_request_repo.get_airlock_request_by_id = AsyncMock(return_value=AirlockRequest( + id="request-id", + workspaceId="workspace-id", + type=AirlockRequestType.Import, + createdBy=User.model_validate(user_payload()), + reviewUserResources={"user-id": AirlockReviewUserResource( + workspaceId="review-workspace-id", + workspaceServiceId="service-id", + userResourceId="resource-id")}, + redeployWorkflows={workflow_id: AirlockRedeployWorkflow( + workflowId=workflow_id, + phase="deploying", + operationId="replacement-operation-id")} + )) + updater.workspace_repo.get_workspace_by_id = AsyncMock(return_value=MagicMock(properties={ + "airlock_review_config": {"import": { + "import_vm_workspace_id": "review-workspace-id", + "import_vm_workspace_service_id": "service-id", + "import_vm_user_resource_template_name": "template"}} + })) + + payload = { + "workflow": "redeploy", + "redeploy_workflow_id": workflow_id, + "airlock_request_id": "request-id", + "operation_id": "delete-operation-id", + "uninstall_started": True, + } + + with patch("service_bus.airlock_workflow.wait_for_successful_operation", new=AsyncMock()), \ + patch("service_bus.airlock_workflow._deploy_vm", new=AsyncMock()) as deploy, \ + patch("service_bus.airlock_workflow.update_and_publish_event_airlock_request", new=AsyncMock()) as update: + result = await updater.process_message(message(payload)) + + assert result is True + deploy.assert_not_awaited() + assert update.await_args.kwargs["redeploy_workflow"].phase == "completed" + assert update.await_args.kwargs["redeploy_workflow"].operationId == "replacement-operation-id" + + +@pytest.mark.asyncio +async def test_process_message_acknowledges_invalid_message(updater): + result = await updater.process_message("not-json") + + assert result is True diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index f4562eb2cc..5184041f24 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -9,11 +9,12 @@ from models.domain.request_action import RequestAction from models.domain.resource import ResourceType +from azure.cosmos.exceptions import CosmosAccessConditionFailedError from db.errors import EntityDoesNotExist from models.domain.workspace import Workspace from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings -from service_bus.deployment_status_updater import DeploymentStatusUpdater +from service_bus.deployment_status_updater import DeploymentStatusUpdater, AddressSpaceConflictError from tests_ma.test_service_bus.test_helpers import ( StopReceiveMessages, credential_context, @@ -75,10 +76,11 @@ class ServiceBusReceivedMessageMock: - def __init__(self, message: dict): + def __init__(self, message: dict, delivery_count: int = 0): self.message = json.dumps(message) self.correlation_id = "test_correlation_id" self.session_id = "test_session_id" + self.delivery_count = delivery_count def __str__(self): return self.message @@ -220,12 +222,13 @@ async def test_receiving_bad_json_logs_error(logging_mock, payload): assert error_message.startswith(strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_receiving_good_message(logging_mock, resource_repo, operation_repo, _, __): +async def test_receiving_good_message(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): expected_workspace = create_sample_workspace_object(test_sb_message["id"]) resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.model_dump() @@ -242,12 +245,38 @@ async def test_receiving_good_message(logging_mock, resource_repo, operation_rep logging_mock.assert_not_called() +@patch('service_bus.deployment_status_updater.tracer') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_process_message_sets_span_attributes(resource_repo, operation_repo, _, __, workspace_repo, tracer_mock): + expected_workspace = create_sample_workspace_object(test_sb_message["id"]) + resource_repo.return_value.get_resource_dict_by_id.return_value = expected_workspace.model_dump() + + operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) + operation_repo.return_value.get_operation_by_id.return_value = operation + + mock_span = MagicMock() + tracer_mock.start_as_current_span.return_value.__enter__.return_value = mock_span + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + await status_updater.process_message(ServiceBusReceivedMessageMock(test_sb_message)) + + mock_span.set_attribute.assert_any_call("step_id", test_sb_message["stepId"]) + mock_span.set_attribute.assert_any_call("operation_id", test_sb_message["operationId"]) + mock_span.set_attribute.assert_any_call("status", test_sb_message["status"]) + + +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock, resource_repo, operation_repo, _, __): +async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): resource_repo.return_value.get_resource_dict_by_id.side_effect = EntityDoesNotExist operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) @@ -262,12 +291,13 @@ async def test_when_updating_non_existent_workspace_error_is_logged(logging_mock logging_mock.assert_called_once_with(expected_error_message) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('services.logging.logger.exception') -async def test_when_updating_and_state_store_exception(logging_mock, resource_repo, operation_repo, _, __): +async def test_when_updating_and_state_store_exception(logging_mock, resource_repo, operation_repo, _, __, workspace_repo): resource_repo.return_value.get_resource_dict_by_id.side_effect = Exception operation = create_sample_operation(test_sb_message["id"], RequestAction.Install) @@ -281,12 +311,13 @@ async def test_when_updating_and_state_store_exception(logging_mock, resource_re assert complete_message is False +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_state_transitions_from_deployed_to_deleted(resource_repo, operations_repo_mock, _, __, ___): +async def test_state_transitions_from_deployed_to_deleted(resource_repo, operations_repo_mock, _, __, ___, workspace_repo): updated_message = test_sb_message updated_message["status"] = Status.Deleted updated_message["message"] = "Has been deleted" @@ -313,11 +344,12 @@ async def test_state_transitions_from_deployed_to_deleted(resource_repo, operati operations_repo_mock.return_value.update_item.assert_called_once_with(expected_operation) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo, _, __): +async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo, _, __, workspace_repo): received_message = test_sb_message_with_outputs received_message["status"] = Status.Deployed service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -350,11 +382,12 @@ async def test_outputs_are_added_to_resource_item(resource_repo, operations_repo resource_repo.return_value.update_item_dict.assert_called_once_with(expected_resource.model_dump()) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') -async def test_properties_dont_change_with_no_outputs(resource_repo, operations_repo, _, __): +async def test_properties_dont_change_with_no_outputs(resource_repo, operations_repo, _, __, workspace_repo): received_message = test_sb_message received_message["status"] = Status.Deployed service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -376,13 +409,14 @@ async def test_properties_dont_change_with_no_outputs(resource_repo, operations_ resource_repo.return_value.update_item_dict.assert_called_once_with(expected_resource.model_dump()) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.update_resource_for_step') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('service_bus.helpers.ServiceBusClient') -async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_repo, operations_repo, update_resource_for_step, _, __, multi_step_operation, user_resource_multi, basic_shared_service): +async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_repo, operations_repo, update_resource_for_step, _, __, workspace_repo, multi_step_operation, user_resource_multi, basic_shared_service): received_message = test_sb_message_multi_step_1_complete received_message["status"] = Status.Updated service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -431,12 +465,13 @@ async def test_multi_step_operation_sends_next_step(sb_sender_client, resource_r sb_sender_client().get_queue_sender().send_messages.assert_called_once() +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') @patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') @patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') @patch('service_bus.deployment_status_updater.OperationRepository.create') @patch('service_bus.deployment_status_updater.ResourceRepository.create') @patch('service_bus.helpers.ServiceBusClient') -async def test_multi_step_operation_ends_at_last_step(sb_sender_client, resource_repo, operations_repo, _, __, multi_step_operation, user_resource_multi, basic_shared_service): +async def test_multi_step_operation_ends_at_last_step(sb_sender_client, resource_repo, operations_repo, _, __, workspace_repo, multi_step_operation, user_resource_multi, basic_shared_service): received_message = test_sb_message_multi_step_3_complete received_message["status"] = Status.Updated service_bus_received_message_mock = ServiceBusReceivedMessageMock(received_message) @@ -501,3 +536,3175 @@ async def test_convert_outputs_to_dict(): 'list2': ['one', 'two'] } assert status_updater.convert_outputs_to_dict(deployment_status_update_message.outputs) == expected_result + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.WorkspaceServiceRepository.create') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_frees_address_space( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + workspace_service_repo_mock, + send_deployment_message_mock +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService.value, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + workspace_repo_mock.return_value = workspace_repo + workspace_service_repo = AsyncMock() + workspace_service_repo.get_active_workspace_services_for_workspace.return_value = [] + workspace_service_repo_mock.return_value = workspace_service_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + send_deployment_message_mock.assert_called_once() + sent_payload = json.loads(send_deployment_message_mock.call_args.kwargs["content"]) + assert sent_payload["parameters"]["address_spaces"] == ["10.0.0.0/22"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22", address_space] + workspace_repo.patch_workspace.assert_not_called() + await status_updater._free_workspace_address_space(operation) + workspace_repo.patch_workspace.assert_called_once() + called_args = workspace_repo.patch_workspace.call_args[0] + assert called_args[0] == parent_workspace + assert called_args[1].properties == {"address_spaces": ["10.0.0.0/22"]} + assert called_args[2] == "parent-workspace-etag" + + +async def test_workspace_service_uninstall_frees_address_space_from_root_main_step(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + child_resource_id = "69b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted + operation.steps.insert(0, OperationStep( + id="child-main", + templateStepId="main", + resourceId=child_resource_id, + resourceType=ResourceType.UserResource, + resourceAction=RequestAction.UnInstall + )) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.workspace_repo = AsyncMock() + status_updater.resource_template_repo = AsyncMock() + status_updater.resource_history_repo = AsyncMock() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": [address_space]} + parent_workspace.etag = "parent-workspace-etag" + status_updater.workspace_repo.get_workspace_by_id.return_value = parent_workspace + + assert await status_updater._free_workspace_address_space(operation) is True + status_updater.resource_repo.get_resource_dict_by_id.assert_awaited_once_with(workspace_service_id) + status_updater.workspace_repo.patch_workspace.assert_awaited_once() + + +async def test_workspace_service_uninstall_does_not_free_address_space_if_root_main_failed(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.DeletingFailed + operation.steps.append(OperationStep( + id="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.Updated + )) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.workspace_repo = AsyncMock() + + assert await status_updater._free_workspace_address_space(operation) is True + status_updater.resource_repo.get_resource_dict_by_id.assert_not_awaited() + status_updater.workspace_repo.patch_workspace.assert_not_awaited() + + +@pytest.mark.parametrize("missing_property", ["address_space", "workspaceId"]) +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_does_not_free_address_space_if_missing( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + missing_property +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + if missing_property == "address_space": + del workspace_service_dict["properties"]["address_space"] + elif missing_property == "workspaceId": + del workspace_service_dict["workspaceId"] + + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + workspace_repo = AsyncMock() + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + workspace_repo.patch_workspace.assert_not_called() + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_frees_address_space_with_retry_on_etag_conflict( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + send_deployment_message_mock +): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + workspace_repo.patch_workspace.side_effect = [CosmosAccessConditionFailedError(), None] + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + send_deployment_message_mock.assert_called_once() + await status_updater._free_workspace_address_space(operation) + assert workspace_repo.get_workspace_by_id.call_count == 3 + assert workspace_repo.patch_workspace.call_count == 2 + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +@patch('service_bus.deployment_status_updater.logger') +async def test_workspace_service_uninstall_logs_error_after_max_retries( + logging_mock, + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + send_deployment_message_mock +): + from azure.cosmos.exceptions import CosmosAccessConditionFailedError + + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "random-uuid", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() + workspace_repo_mock.return_value = workspace_repo + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + send_deployment_message_mock.assert_called_once() + workspace_repo.patch_workspace.side_effect = CosmosAccessConditionFailedError() + complete_message = await status_updater._free_workspace_address_space(operation) + assert complete_message is False + assert workspace_repo.get_workspace_by_id.call_count == 4 + assert workspace_repo.patch_workspace.call_count == 3 + logging_mock.error.assert_called_once() + assert "[ADDRESS_SPACE_CLEANUP_FAILED]" in logging_mock.error.call_args[0][0] + + +async def test_address_space_cleanup_failure_is_persisted_before_message_redelivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted + operation.steps.append(OperationStep( + id="address-space-cleanup-step", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + status=Status.Updating + )) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = MagicMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = create_sample_workspace_object(parent_workspace_id).model_dump() + status_updater.workspace_repo = AsyncMock() + status_updater.resource_template_repo = AsyncMock() + status_updater.resource_history_repo = AsyncMock() + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="address-space-cleanup-step", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + complete_message = await status_updater.update_status_in_database(message) + + assert complete_message is False + assert operation.steps[-1].status == Status.Updating + assert operation.status == Status.PipelineRunning + assert "will be retried" in operation.steps[-1].message + assert status_updater.operations_repo.update_item.await_args_list[-1].args[0] == operation + assert status_updater.resource_repo.update_item.await_args_list[-1].args[0].deploymentStatus == Status.Updating + + +async def test_unrelated_workspace_upgrade_does_not_suppress_address_space_cleanup(): + operation = create_sample_operation("workspace-service-id", RequestAction.UnInstall) + status_updater = DeploymentStatusUpdater() + + operation.steps.append(OperationStep( + id="workspace-upgrade", + templateStepId="unrelated-upgrade", + resourceId="workspace-id", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + )) + assert status_updater._has_workspace_upgrade_step(operation, 0) is False + + operation.steps.append(OperationStep( + id="address-space-cleanup", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + resourceId="workspace-id", + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + )) + assert status_updater._has_workspace_upgrade_step(operation, 0) is True + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +@patch('service_bus.deployment_status_updater.update_resource_for_step') +@patch('service_bus.deployment_status_updater.WorkspaceRepository.create') +@patch('service_bus.deployment_status_updater.ResourceHistoryRepository.create') +@patch('service_bus.deployment_status_updater.ResourceTemplateRepository.create') +@patch("service_bus.deployment_status_updater.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +@patch('service_bus.deployment_status_updater.OperationRepository.create') +@patch('service_bus.deployment_status_updater.ResourceRepository.create') +async def test_workspace_service_uninstall_defers_address_space_cleanup_until_workspace_upgrade_succeeds( + resource_repo, + operations_repo_mock, + _, + __, + ___, + workspace_repo_mock, + update_resource_for_step_mock, + send_deployment_message_mock +): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + call_order = [] + sent_payloads = [] + + # Create a 2-step uninstall operation (step 1: main, step 2: workspace upgrade) + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + operations_repo_mock.return_value.get_operation_by_id.return_value = operation + + message_dict = { + "operationId": OPERATION_ID, + "stepId": "step-1", + "id": workspace_service_id, + "status": Status.Deleted, + "message": "uninstall succeeded", + "correlation_id": "test_correlation_id" + } + service_bus_received_message_mock = ServiceBusReceivedMessageMock(message_dict) + + workspace_service_mock = MagicMock() + workspace_service_mock.deploymentStatus = None + resource_repo.return_value.get_resource_by_id.return_value = workspace_service_mock + + workspace_service_dict = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": { + "address_space": address_space + } + } + resource_repo.return_value.get_resource_dict_by_id.return_value = workspace_service_dict + + parent_workspace = create_sample_workspace_object(parent_workspace_id) + parent_workspace.properties = {"address_spaces": ["10.0.0.0/22", address_space]} + parent_workspace.etag = "parent-workspace-etag" + + workspace_repo = AsyncMock() + workspace_repo.get_workspace_by_id.return_value = parent_workspace + + async def mock_patch_workspace(*args, **kwargs): + call_order.append("patch_workspace") + parent_workspace.properties = kwargs.get("workspace_patch", args[1] if len(args) > 1 else None).properties + return parent_workspace, MagicMock() + + workspace_repo.patch_workspace = AsyncMock(side_effect=mock_patch_workspace) + workspace_repo_mock.return_value = workspace_repo + + resource_to_send_mock = MagicMock() + resource_to_send_mock.id = parent_workspace_id + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_for_step_mock.return_value = resource_to_send_mock + + async def mock_send_deployment_message(*args, **kwargs): + call_order.append("send_deployment_message") + sent_payloads.append(json.loads(kwargs["content"])) + + send_deployment_message_mock.side_effect = mock_send_deployment_message + + status_updater = DeploymentStatusUpdater() + await status_updater.init_repos() + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + complete_message = await status_updater.process_message(service_bus_received_message_mock) + + assert complete_message is True + + # The address remains reserved while the workspace upgrade is queued, + # but the deployment message sent to Terraform excludes the target address space. + assert call_order == ["send_deployment_message"] + assert sent_payloads[0]["parameters"]["address_spaces"] == ["10.0.0.0/22"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22", address_space] + + upgrade_message = ServiceBusReceivedMessageMock({ + "operationId": OPERATION_ID, + "stepId": "step-2", + "id": parent_workspace_id, + "status": Status.Updated, + "message": "workspace upgrade succeeded", + "correlation_id": "test_correlation_id" + }) + complete_message = await status_updater.process_message(upgrade_message) + + assert complete_message is True + assert call_order == ["send_deployment_message", "patch_workspace"] + assert parent_workspace.properties["address_spaces"] == ["10.0.0.0/22"] + + +async def test_workspace_upgrade_failure_does_not_free_address_space(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + operation = create_sample_operation(workspace_service_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted + operation.steps.append(OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating + )) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = MagicMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = create_sample_workspace_object(parent_workspace_id).model_dump() + status_updater.workspace_repo = AsyncMock() + + failed_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.UpdatingFailed, + message="workspace upgrade failed" + ) + + with patch.object(status_updater, "_free_workspace_address_space") as free_mock: + complete_message = await status_updater.update_status_in_database(failed_message) + + assert complete_message is True + free_mock.assert_not_called() + assert operation.steps[-1].status == Status.UpdatingFailed + assert operation.status == Status.DeletingFailed + + +async def test_workspace_service_uninstall_address_space_cleanup_fail_then_succeed_restores_primary_resource(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is False + assert step2.status == Status.Updating + assert operation.status == Status.PipelineRunning + assert parent_workspace_mock.deploymentStatus == Status.Updating + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert step2.status == Status.Updated + assert operation.status == Status.Deleted + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.Updated + + +async def test_free_workspace_address_space_idempotent_on_redelivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is True + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_fails_when_address_owned_by_other_active_service(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + new_active_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + active_service_mock = MagicMock() + active_service_mock.id = new_active_service_id + active_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [active_service_mock] + status_updater.workspace_repo = AsyncMock() + + with pytest.raises(AddressSpaceConflictError): + await status_updater._free_workspace_address_space(operation) + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_fails_closed_when_active_services_query_errors(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.side_effect = Exception("DB error") + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is False + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_free_workspace_address_space_idempotent_when_marker_present(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space, "address_space_freed": True} + } + status_updater.workspace_repo = AsyncMock() + + result = await status_updater._free_workspace_address_space(operation) + + assert result is True + status_updater.workspace_repo.patch_workspace.assert_not_called() + + +async def test_terminal_address_space_conflict_marks_operation_and_resource_failed_and_completes_message(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", side_effect=AddressSpaceConflictError("Conflict")): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert step2.status == Status.UpdatingFailed + assert "Terminal address space conflict" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_legacy_template_appends_workspace_upgrade_step_before_persisting_main_status(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.resource_template_repo = AsyncMock() + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert len(operation.steps) == 2 + assert operation.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + first_saved_op = status_updater.operations_repo.update_item.call_args_list[0].args[0] + assert first_saved_op.status == Status.PipelineRunning + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_enqueue_cleanup_validates_ownership_and_aborts_before_azure_send(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + other_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + other_service_mock = MagicMock() + other_service_mock.id = other_service_id + other_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [other_service_mock] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + send_deployment_message_mock.assert_not_called() + assert operation.status == Status.DeletingFailed + assert operation.steps[1].status == Status.UpdatingFailed + assert "Terminal address space conflict" in operation.steps[1].message + assert workspace_service_mock.deploymentStatus == Status.Deleted + + +@patch('service_bus.deployment_status_updater.send_deployment_message', side_effect=Exception("Service Bus send failed")) +async def test_enqueue_cleanup_send_failure_abandons_message_for_retry(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=False) + + assert result is False + assert operation.status == Status.PipelineRunning + assert operation.steps[1].status == Status.AwaitingUpdate + assert "will retry" in operation.steps[1].message + + +@patch('service_bus.deployment_status_updater.send_deployment_message', side_effect=Exception("Service Bus send failed")) +async def test_enqueue_cleanup_send_failure_on_final_delivery_marks_failed(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=True) + + assert result is True + assert operation.status == Status.DeletingFailed + assert operation.steps[1].status == Status.UpdatingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + + +async def test_cleanup_failure_on_final_delivery_marks_failed_and_completes_message(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert step2.status == Status.UpdatingFailed + assert "delivery attempts" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +@pytest.mark.parametrize("delivery_count, expected_final", [ + (0, False), + (1, False), + (8, False), + (9, True), + (10, True), + (None, False), +]) +async def test_process_message_delivery_count_handling(delivery_count, expected_final): + status_updater = DeploymentStatusUpdater() + status_updater.update_status_in_database = AsyncMock(return_value=True) + + msg = ServiceBusReceivedMessageMock(test_sb_message, delivery_count=delivery_count) + + await status_updater.process_message(msg) + + status_updater.update_status_in_database.assert_awaited_once() + _, kwargs = status_updater.update_status_in_database.call_args + assert kwargs.get("is_final_delivery") is expected_final + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_does_not_rerun_completed_cleanup_or_fail_on_reallocated_address(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + new_active_service_id = "9999c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + new_active_service_mock = MagicMock() + new_active_service_mock.id = new_active_service_id + new_active_service_mock.properties = {"address_space": address_space} + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space, "address_space_freed": True} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [new_active_service_mock] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + assert operation.status == Status.Deleted + assert step2.status == Status.Updated + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + assert workspace_service_mock.deploymentStatus == Status.Deleted + status_updater.operations_repo.update_item.assert_not_called() + send_deployment_message_mock.assert_not_called() + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_skips_enqueue_when_cleanup_in_progress(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + message="" + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + send_deployment_message_mock.assert_not_called() + + +async def test_cleanup_redelivery_ignored_when_cleanup_already_terminal_success(): + """Delayed cleanup update after terminal success should be ignored.""" + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2] + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + + delayed_cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updating, + message="in progress..." + ) + + result = await status_updater.update_status_in_database(delayed_cleanup_message) + + assert result is True + status_updater.resource_repo.update_item.assert_not_called() + status_updater.resource_repo.update_item_dict.assert_not_called() + status_updater.operations_repo.update_item.assert_not_called() + assert operation.status == Status.Deleted + assert step2.status == Status.Updated + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_skips_when_cleanup_failed_terminally(send_deployment_message_mock): + """Redelivered main-step message should not reopen terminal cleanup failure.""" + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.UpdatingFailed, + message="Cleanup failed terminally" + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.DeletingFailed, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + assert operation.status == Status.DeletingFailed + assert step2.status == Status.UpdatingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + status_updater.operations_repo.update_item.assert_not_called() + send_deployment_message_mock.assert_not_called() + + +async def test_cleanup_success_persists_workspace_and_primary_resource_before_operation_deleted(): + """Ensure operation is marked Deleted only after resource writes complete.""" + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + async def tracking_resource_dict_update(res_dict): + call_order.append(("resource_repo.update_item_dict", res_dict.get("id"))) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": parent_workspace_id, + "resourceType": ResourceType.Workspace, + "properties": {"address_spaces": [address_space]} + } + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.resource_repo.update_item_dict = AsyncMock(side_effect=tracking_resource_dict_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert operation.status == Status.Deleted + + assert len(call_order) >= 3 + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.Deleted) + + resource_calls = [i for i, call in enumerate(call_order) if call[0].startswith("resource_repo")] + for r_idx in resource_calls: + assert r_idx < op_update_index + + +async def test_cleanup_conflict_persists_workspace_and_primary_resource_before_operation_failed(): + """On address space conflict, update resources before saving operation as DeletingFailed.""" + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", side_effect=AddressSpaceConflictError("Conflict")): + result = await status_updater.update_status_in_database(cleanup_message) + + assert result is True + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) + + resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] + assert len(resource_calls) == 2 + for r_idx in resource_calls: + assert r_idx < op_update_index + + +async def test_cleanup_final_delivery_persists_workspace_and_primary_resource_before_operation_failed(): + """On final delivery cleanup failure, update resources before saving operation as DeletingFailed.""" + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + call_order = [] + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + async def tracking_op_update(op): + call_order.append(("operations_repo.update_item", op.status)) + + async def tracking_resource_update(res): + call_order.append(("resource_repo.update_item", getattr(res, "id", None), res.deploymentStatus)) + + status_updater.operations_repo.update_item = AsyncMock(side_effect=tracking_op_update) + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.update_item = AsyncMock(side_effect=tracking_resource_update) + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded" + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=False)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + + op_update_calls = [i for i, call in enumerate(call_order) if call[0] == "operations_repo.update_item"] + assert len(op_update_calls) == 1 + op_update_index = op_update_calls[0] + assert call_order[op_update_index] == ("operations_repo.update_item", Status.DeletingFailed) + + resource_calls = [i for i, call in enumerate(call_order) if call[0] == "resource_repo.update_item"] + assert len(resource_calls) == 2 + for r_idx in resource_calls: + assert r_idx < op_update_index + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_cleanup_dispatch_transitions_step_to_updating(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + status_before_send = None + + async def mock_send_deployment_message(*args, **kwargs): + nonlocal status_before_send + status_before_send = operation.steps[1].status + + send_deployment_message_mock.side_effect = mock_send_deployment_message + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert len(operation.steps) == 2 + cleanup_step = operation.steps[1] + assert cleanup_step.templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + assert status_before_send == Status.AwaitingUpdate + assert cleanup_step.status == Status.Updating + + send_deployment_message_mock.assert_called_once_with( + content=ANY, + correlation_id=OPERATION_ID, + session_id=parent_workspace_id, + action=RequestAction.Upgrade + ) + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_redelivered_main_status_dispatches_and_subsequent_redelivery_skips(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + cleanup_step_id = "step-2-cleanup-id" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted + ) + step2 = OperationStep( + id=cleanup_step_id, + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f'/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}', + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2] + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"} + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + redelivered_main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded" + ) + + result = await status_updater.update_status_in_database(redelivered_main_message) + + assert result is True + send_deployment_message_mock.assert_called_once_with( + content=ANY, + correlation_id=OPERATION_ID, + session_id=parent_workspace_id, + action=RequestAction.Upgrade + ) + assert step2.status == Status.Updating + + result2 = await status_updater.update_status_in_database(redelivered_main_message) + assert result2 is True + send_deployment_message_mock.assert_called_once() + + +async def test_delayed_earlier_step_message_ignored_when_cleanup_terminal(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[step1, step2], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + + delayed_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.DeletingFailed, + message="delayed failure notification", + ) + + result = await status_updater.update_status_in_database(delayed_message) + + assert result is True + assert operation.status == Status.Deleted + assert step1.status == Status.Deleted + status_updater.resource_repo.update_item.assert_not_called() + status_updater.operations_repo.update_item.assert_not_called() + + +@pytest.mark.parametrize( + "action, expected_status", + [ + (RequestAction.Install, Status.DeploymentFailed), + (RequestAction.UnInstall, Status.DeletingFailed), + (RequestAction.Upgrade, Status.UpdatingFailed), + ("custom-action", Status.ActionFailed), + ], +) +async def test_missing_step_resource_persists_terminal_failure_operation_state(action, expected_status): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Execute action on workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=action, + templateStepId="main", + status=Status.Deploying, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=action, + status=Status.PipelineRunning, + steps=[step1], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deployed, + message="status update", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is True + assert step1.status == expected_status + assert "not found in database" in step1.message + assert operation.status == expected_status + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + +async def test_missing_primary_resource_in_multi_step_operation_persists_failure_without_raising(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Address space cleanup", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId="address-space-cleanup", + status=Status.AwaitingUpdate, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="resource deleted", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is True + assert step1.status == Status.DeletingFailed + assert "not found in database" in step1.message + assert operation.status == Status.DeletingFailed + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + +async def test_missing_resource_returns_false_when_operation_update_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Install workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.Install, + templateStepId="main", + status=Status.Deploying, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.Install, + status=Status.Deploying, + steps=[step1], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item.side_effect = Exception("Cosmos DB connection timeout") + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + status_updater.resource_repo.get_resource_dict_by_id.side_effect = EntityDoesNotExist + + message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deployed, + message="deployed", + ) + + result = await status_updater.update_status_in_database(message) + + assert result is False + assert step1.status == Status.DeploymentFailed + assert operation.status == Status.DeploymentFailed + status_updater.operations_repo.update_item.assert_called_once_with(operation) + + +async def test_cleanup_failure_in_get_address_cleanup_details_on_final_delivery_marks_failed(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater._get_address_cleanup_details = AsyncMock(side_effect=Exception("Cosmos read error")) + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert step2.status == Status.UpdatingFailed + assert "delivery attempts" in step2.message + assert operation.status == Status.DeletingFailed + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.UpdatingFailed + + +async def test_cleanup_step_with_trailing_step_does_not_prematurely_finalize_as_deleted(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + trailing_resource_id = "2222c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updated, + message=strings.ADDRESS_SPACE_CLEANUP_SUCCESS, + ) + step3 = OperationStep( + id="step-3", + stepTitle="Firewall post-cleanup step", + resourceId=trailing_resource_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId="trailing-step", + sourceTemplateResourceId=trailing_resource_id, + status=Status.AwaitingUpdate, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2, step3], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.resource_repo = AsyncMock() + + await status_updater.update_overall_operation_status(operation, step2, is_last_step=False) + + assert operation.status == Status.PipelineRunning + assert operation.status != Status.Deleted + + +async def test_cleanup_post_cleanup_write_failure_retries_on_non_final_delivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item.side_effect = Exception("Cosmos write failed") + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=False) + + assert result is False + assert step2.message == strings.ADDRESS_SPACE_CLEANUP_SUCCESS + assert operation.status != Status.DeletingFailed + + +async def test_cleanup_post_cleanup_write_failure_persists_terminal_state_on_final_delivery(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Updating + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + attempt_count = 0 + + async def mock_op_update(op): + nonlocal attempt_count + attempt_count += 1 + if attempt_count <= 3: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_op_update + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is True + assert operation.status == Status.Deleted + assert workspace_service_mock.deploymentStatus == Status.Deleted + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_cleanup_post_send_persistence_failure_does_not_terminally_fail_enqueued_cleanup(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + + send_called = False + + async def mock_op_update(op): + if send_called: + raise Exception("Cosmos write error during post-send status update") + return op + + async def mock_send_deployment_message(*args, **kwargs): + nonlocal send_called + send_called = True + + send_deployment_message_mock.side_effect = mock_send_deployment_message + status_updater.operations_repo.update_item.side_effect = mock_op_update + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message, is_final_delivery=True) + + assert result is True + send_deployment_message_mock.assert_called_once() + cleanup_step = operation.steps[1] + assert cleanup_step.status != Status.UpdatingFailed + assert operation.status != Status.DeletingFailed + assert not status_updater._is_cleanup_terminal(operation) + + +async def test_free_workspace_address_space_read_failure_avoids_unbound_local(): + status_updater = DeploymentStatusUpdater() + status_updater._get_address_cleanup_details = AsyncMock(side_effect=Exception("Cosmos read error")) + + operation = MagicMock() + freed = await status_updater._free_workspace_address_space(operation) + + assert freed is False + + +async def test_merge_operation_steps_appends_missing_steps_and_recomputes_indices(): + status_updater = DeploymentStatusUpdater() + step1 = OperationStep( + id="step-1", + stepTitle="Step 1", + resourceId="res-1", + templateStepId="main", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup step", + resourceId="res-ws", + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId="res-1", + resourcePath="/workspaces/res-ws/workspace-services/res-1", + action=RequestAction.UnInstall, + steps=[step1, step2], + ) + fresh_step1 = copy.deepcopy(step1) + fresh_op = Operation( + id=OPERATION_ID, + resourceId="res-1", + resourcePath="/workspaces/res-ws/workspace-services/res-1", + action=RequestAction.UnInstall, + steps=[fresh_step1], + ) + + step_to_update, is_last_step, current_step_index = status_updater._merge_operation_steps( + in_memory_op=in_memory_op, + fresh_op=fresh_op, + step_id_to_update="step-1", + step_status=Status.Deleted, + step_message="Uninstall done", + ) + + assert len(fresh_op.steps) == 2 + assert step_to_update.id == "step-1" + assert step_to_update.status == Status.Deleted + assert is_last_step is False + assert current_step_index == 0 + assert fresh_op.steps[1].id == "step-2" + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_etag_conflict_retry_merges_legacy_synthetic_cleanup_step(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + address_space = "10.1.0.0/22" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + # Legacy operation with only 1 step initially in Cosmos DB + legacy_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.Deleting, + steps=[step1], + etag="initial-etag", + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + def mock_get_op(op_id): + return copy.deepcopy(legacy_op) + + status_updater.operations_repo.get_operation_by_id.side_effect = mock_get_op + + saved_operations = [] + attempt_count = 0 + + async def mock_op_update(op): + nonlocal attempt_count + attempt_count += 1 + saved_operations.append(copy.deepcopy(op)) + if attempt_count == 1: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_op_update + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id.return_value = workspace_service_mock + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": address_space}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert attempt_count >= 2 + second_saved_op = saved_operations[1] + assert len(second_saved_op.steps) == 2 + assert second_saved_op.status == Status.PipelineRunning + assert second_saved_op.steps[1].templateStepId == strings.ADDRESS_SPACE_CLEANUP_STEP_ID + send_deployment_message_mock.assert_called_once() + + +async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_terminal_persist_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup address space", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Updating + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item.side_effect = CosmosAccessConditionFailedError() + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is False + + +async def test_cleanup_post_write_failure_on_final_delivery_dead_letters_when_workspace_resource_write_fails(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Cleanup address space", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + status=Status.Updating, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step1, step2], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.return_value = operation + status_updater.operations_repo.update_item = AsyncMock() + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.update_item.side_effect = Exception("Cosmos DB unavailable") + status_updater.resource_repo.get_resource_dict_by_id.return_value = {"id": parent_workspace_id, "properties": {}} + status_updater.workspace_repo = AsyncMock() + + cleanup_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-2", + id=parent_workspace_id, + status=Status.Updated, + message="workspace upgrade succeeded", + ) + + with patch.object(status_updater, "_free_workspace_address_space", new=AsyncMock(return_value=True)): + result = await status_updater.update_status_in_database(cleanup_message, is_final_delivery=True) + + assert result is False + assert operation.status != Status.Deleted + + +async def test_get_address_cleanup_details_returns_none_if_resource_type_not_workspace_service(): + user_resource_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step = OperationStep( + id="step-1", + stepTitle="Uninstall user resource", + resourceId=user_resource_id, + resourceType=ResourceType.UserResource, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + operation = Operation( + id=OPERATION_ID, + resourceId=user_resource_id, + resourcePath=f"/workspaces/{parent_workspace_id}/user-resources/{user_resource_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[step], + ) + + status_updater = DeploymentStatusUpdater() + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": user_resource_id, + "resourceType": ResourceType.UserResource, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.0.1.0/24"}, + } + + details = await status_updater._get_address_cleanup_details(operation) + assert details is None + + +async def test_etag_conflict_on_earlier_step_reconciles_and_skips_when_fresh_op_already_terminal(): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleting, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[copy.deepcopy(step1), copy.deepcopy(step2)], + ) + + terminal_step1 = copy.deepcopy(step1) + terminal_step1.status = Status.Deleted + terminal_step2 = copy.deepcopy(step2) + terminal_step2.status = Status.Updated + terminal_step2.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + fresh_terminal_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=1, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[terminal_step1, terminal_step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleting + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updating + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + status_updater.operations_repo.get_operation_by_id.side_effect = [ + in_memory_op, + fresh_terminal_op, + ] + status_updater.operations_repo.update_item.side_effect = CosmosAccessConditionFailedError() + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"}, + } + status_updater.workspace_repo = AsyncMock() + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + assert fresh_terminal_op.status == Status.Deleted + assert workspace_service_mock.deploymentStatus == Status.Deleted + assert parent_workspace_mock.deploymentStatus == Status.Updated + + +@patch('service_bus.deployment_status_updater.send_deployment_message') +async def test_etag_conflict_after_cleanup_dispatch_skips_when_fresh_op_already_terminal(send_deployment_message_mock): + workspace_service_id = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" + parent_workspace_id = "1111c8e7-5c42-4fcb-a7fd-294cfc27aa76" + + step1 = OperationStep( + id="step-1", + stepTitle="Uninstall workspace service", + resourceId=workspace_service_id, + resourceType=ResourceType.WorkspaceService, + resourceAction=RequestAction.UnInstall, + templateStepId="main", + status=Status.Deleted, + ) + step2 = OperationStep( + id="step-2", + stepTitle="Upgrade workspace", + resourceId=parent_workspace_id, + resourceType=ResourceType.Workspace, + resourceAction=RequestAction.Upgrade, + templateStepId=strings.ADDRESS_SPACE_CLEANUP_STEP_ID, + sourceTemplateResourceId=parent_workspace_id, + status=Status.AwaitingUpdate, + ) + in_memory_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=0, + action=RequestAction.UnInstall, + status=Status.PipelineRunning, + steps=[copy.deepcopy(step1), copy.deepcopy(step2)], + ) + + terminal_step2 = copy.deepcopy(step2) + terminal_step2.status = Status.Updated + terminal_step2.message = strings.ADDRESS_SPACE_CLEANUP_SUCCESS + fresh_terminal_op = Operation( + id=OPERATION_ID, + resourceId=workspace_service_id, + resourcePath=f"/workspaces/{parent_workspace_id}/workspace-services/{workspace_service_id}", + resourceVersion=2, + action=RequestAction.UnInstall, + status=Status.Deleted, + steps=[copy.deepcopy(step1), terminal_step2], + ) + + workspace_service_mock = MagicMock() + workspace_service_mock.id = workspace_service_id + workspace_service_mock.deploymentStatus = Status.Deleted + + parent_workspace_mock = MagicMock() + parent_workspace_mock.id = parent_workspace_id + parent_workspace_mock.deploymentStatus = Status.Updated + + async def mock_get_resource_by_id(resource_uuid): + if str(resource_uuid) == workspace_service_id: + return workspace_service_mock + elif str(resource_uuid) == parent_workspace_id: + return parent_workspace_mock + raise EntityDoesNotExist + + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + call_count = 0 + + def mock_get_op(op_id): + nonlocal call_count + call_count += 1 + return in_memory_op if call_count == 1 else fresh_terminal_op + + status_updater.operations_repo.get_operation_by_id.side_effect = mock_get_op + + op_update_count = 0 + + async def mock_update_op(op): + nonlocal op_update_count + op_update_count += 1 + if op_update_count > 1: + raise CosmosAccessConditionFailedError() + return op + + status_updater.operations_repo.update_item.side_effect = mock_update_op + + status_updater.resource_repo = AsyncMock() + status_updater.resource_repo.get_resource_by_id = AsyncMock(side_effect=mock_get_resource_by_id) + status_updater.resource_repo.get_resource_dict_by_id.return_value = { + "id": workspace_service_id, + "resourceType": ResourceType.WorkspaceService, + "workspaceId": parent_workspace_id, + "properties": {"address_space": "10.1.0.0/22"}, + } + status_updater.workspace_repo = AsyncMock() + status_updater.workspace_repo.get_workspace_by_id.return_value = create_sample_workspace_object(parent_workspace_id) + status_updater.workspace_services_repo = AsyncMock() + status_updater.workspace_services_repo.get_active_workspace_services_for_workspace.return_value = [] + + main_message = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=workspace_service_id, + status=Status.Deleted, + message="uninstall succeeded", + ) + + result = await status_updater.update_status_in_database(main_message) + + assert result is True + send_deployment_message_mock.assert_called_once() + assert fresh_terminal_op.steps[1].status == Status.Updated + assert fresh_terminal_op.status == Status.Deleted + + +async def test_update_status_in_database_rejects_reconciled_operation(): + status_updater = DeploymentStatusUpdater() + status_updater.operations_repo = AsyncMock() + + res_id = uuid.uuid4() + reconciled_op = Operation( + id=OPERATION_ID, + resourceId=str(res_id), + resourcePath="/workspaces/ws-1", + action=RequestAction.Install, + status=Status.DeploymentFailed, + reconciled=True, + steps=[ + OperationStep( + id="step-1", + templateStepId="main", + resourceId=str(res_id), + status=Status.DeploymentFailed, + ) + ], + ) + status_updater.operations_repo.get_operation_by_id = AsyncMock(return_value=reconciled_op) + status_updater.resource_repo = AsyncMock() + + msg = DeploymentStatusUpdateMessage( + operationId=OPERATION_ID, + stepId="step-1", + id=res_id, + status=Status.Deployed, + message="late success", + ) + + result = await status_updater.update_status_in_database(msg) + assert result is True + status_updater.operations_repo.update_item.assert_not_called() + status_updater.resource_repo.update_item.assert_not_called() diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 22db82ded6..13351cc50d 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -4,7 +4,7 @@ import uuid from azure.servicebus import ServiceBusMessage -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, patch, MagicMock from resources import strings from models.schemas.resource import ResourcePatch from service_bus.helpers import ( @@ -21,6 +21,8 @@ send_resource_request_message, RequestAction, ) +from models.domain.operation import Status +from azure.core import MatchConditions from azure.cosmos.exceptions import CosmosAccessConditionFailedError pytestmark = pytest.mark.asyncio @@ -168,7 +170,7 @@ async def test_multi_step_document_retries( ) # simulate an etag mismatch - resource_repo.patch_resource.side_effect = CosmosAccessConditionFailedError + resource_repo.patch_resource.side_effect = CosmosAccessConditionFailedError() num_retries = 5 try: @@ -191,3 +193,261 @@ async def test_multi_step_document_retries( # check it tried to patch and re-get the item the first time + all the retries assert len(resource_repo.patch_resource.mock_calls) == (num_retries + 1) assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) + + +async def test_cascade_acquires_workspace_lease_before_dependency_snapshot(): + resource = create_test_resource() + resource.resourcePath = f"/workspaces/{resource.id}" + operation = create_sample_operation(resource.id, RequestAction.UnInstall) + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + resource_repo_mock = AsyncMock() + resource_repo_mock.get_resource_dependency_list.return_value = [resource.__dict__] + resource_template_repo_mock = AsyncMock() + resource_history_repo_mock = AsyncMock() + order = [] + operations_repo_mock.acquire_workspace_lease.side_effect = lambda *args: order.append(("lease", args)) + resource_repo_mock.get_resource_dependency_list.side_effect = lambda *args: order.append(("snapshot", args)) or [resource.__dict__] + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + + with patch("service_bus.resource_request_sender.update_resource_for_step", new=AsyncMock(return_value=resource_to_send_mock)), \ + patch("service_bus.resource_request_sender.send_deployment_message", new=AsyncMock()): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=resource_repo_mock, + user=create_test_user(), + resource_template_repo=resource_template_repo_mock, + resource_history_repo=resource_history_repo_mock, + action=RequestAction.UnInstall, + is_cascade=True, + ) + + assert order[0][0] == "lease" + assert order[1][0] == "snapshot" + operations_repo_mock.create_operation_item.assert_awaited_once() + assert operations_repo_mock.create_operation_item.call_args.kwargs["operation_id"] == order[0][1][1] + + +@pytest.mark.parametrize( + "action, expected_status", + [ + (RequestAction.Install, Status.DeploymentFailed), + (RequestAction.Upgrade, Status.UpdatingFailed), + (RequestAction.UnInstall, Status.DeletingFailed), + ], +) +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_persists_failure_state_on_dispatch_failure( + update_resource_mock, + send_deployment_mock, + action, + expected_status, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, action) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock() + + resource_repo_mock = AsyncMock() + resource_template_repo_mock = AsyncMock() + resource_history_repo_mock = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=resource_repo_mock, + user=create_test_user(), + resource_template_repo=resource_template_repo_mock, + resource_history_repo=resource_history_repo_mock, + action=action, + ) + + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=True) + assert operation.status == expected_status + assert operation.steps[0].status == expected_status + assert "Failed to dispatch" in operation.message + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_derives_step_failure_status_from_step_action( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.UnInstall) + operation.steps[0].resourceAction = RequestAction.Upgrade + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.UnInstall, + ) + + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=True) + assert operation.status == Status.DeletingFailed + assert operation.steps[0].status == Status.UpdatingFailed + assert "Failed to dispatch" in operation.steps[0].message + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_retains_lease_when_caller_provides_operation_id( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + operation_id=operation.id, + ) + + operations_repo_mock.update_item.assert_awaited_once_with(operation, release_lease=False) + assert operation.status == Status.DeploymentFailed + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_cleans_up_orphaned_operation_when_persisting_failure_fails( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock() + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) + operations_repo_mock.release_workspace_lease.assert_awaited_once_with(resource.id, operation.id) + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_retains_lease_when_concurrently_modified( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock(side_effect=CosmosAccessConditionFailedError) + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable"): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + ) + + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) + operations_repo_mock.release_workspace_lease.assert_not_called() + + +@patch("service_bus.resource_request_sender.send_deployment_message", side_effect=Exception("Service Bus unavailable")) +@patch("service_bus.resource_request_sender.update_resource_for_step") +async def test_send_resource_request_message_sets_lease_retained_on_concurrent_modification( + update_resource_mock, + send_deployment_mock, +): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operation.etag = "creation-etag" + + operations_repo_mock = AsyncMock() + operations_repo_mock.create_operation_item.return_value = operation + operations_repo_mock.update_item = AsyncMock(side_effect=Exception("Cosmos error")) + operations_repo_mock.delete_item = AsyncMock(side_effect=CosmosAccessConditionFailedError) + operations_repo_mock.release_workspace_lease = AsyncMock() + + resource_to_send_mock = MagicMock() + resource_to_send_mock.get_resource_request_message_payload.return_value = {} + update_resource_mock.return_value = resource_to_send_mock + + with pytest.raises(Exception, match="Service Bus unavailable") as exc_info: + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo_mock, + resource_repo=AsyncMock(), + user=create_test_user(), + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + operation_id="custom-op-id", + ) + + assert getattr(exc_info.value, "lease_retained", False) is True + operations_repo_mock.delete_item.assert_awaited_once_with(operation.id, etag="creation-etag", match_condition=MatchConditions.IfNotModified) + operations_repo_mock.release_workspace_lease.assert_not_called() diff --git a/api_app/tests_ma/test_services/test_airlock.py b/api_app/tests_ma/test_services/test_airlock.py index 65c61e6d62..0ded913453 100644 --- a/api_app/tests_ma/test_services/test_airlock.py +++ b/api_app/tests_ma/test_services/test_airlock.py @@ -556,9 +556,9 @@ async def test_revoke_request_calls_update_with_revoked_status(update_mock, airl @pytest.mark.asyncio -@patch("services.airlock.delete_review_user_resource") +@patch("services.airlock.send_airlock_workflow_message", new_callable=AsyncMock) @patch("services.airlock.update_and_publish_event_airlock_request") -async def test_cancel_request_deletes_review_resource(_, delete_review_user_resource, airlock_request_repo_mock): +async def test_cancel_request_enqueues_review_resource_cleanup(_, send_airlock_workflow_message, airlock_request_repo_mock): await cancel_request( airlock_request=sample_airlock_request(), user=create_test_user(), @@ -570,7 +570,8 @@ async def test_cancel_request_deletes_review_resource(_, delete_review_user_reso operations_repo=AsyncMock(), resource_history_repo=AsyncMock()) - delete_review_user_resource.assert_called_once() + send_airlock_workflow_message.assert_awaited_once() + assert send_airlock_workflow_message.await_args.args[0]["workflow"] == "cleanup" @pytest.mark.asyncio @@ -732,3 +733,192 @@ def test_authenticated_user_with_empty_email_persists_and_builds_notification(): assert notification.created_by.email == "" assert notification.updated_by.email == "" + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_success(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.Updated + repo.get_operation_by_id.return_value = op + + res = await wait_for_successful_operation(repo, "op-123", timeout=1.0) + assert res == op + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_failure(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.UpdatingFailed + op.message = "Failed step" + repo.get_operation_by_id.return_value = op + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=1.0) + assert exc_info.value.status_code == 500 + assert "Failed step" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_timeout(): + from services.airlock import wait_for_successful_operation + from models.domain.operation import Status, Operation + + repo = AsyncMock() + op = MagicMock(spec=Operation) + op.status = Status.Updating + repo.get_operation_by_id.return_value = op + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=0.1, poll_interval=0.02) + assert exc_info.value.status_code == 504 + + +@pytest.mark.asyncio +@patch("services.airlock.disable_user_resource") +@patch("services.airlock.send_uninstall_message") +async def test_delete_review_user_resource_aborts_if_disable_fails(mock_send_uninstall, mock_disable): + from models.domain.operation import Status, Operation + + disable_op = MagicMock(spec=Operation) + disable_op.id = "op-disable" + mock_disable.return_value = disable_op + + operations_repo = AsyncMock() + op_record = MagicMock(spec=Operation) + op_record.status = Status.UpdatingFailed + op_record.message = "Could not disable" + operations_repo.get_operation_by_id.return_value = op_record + + workspace_service_repo = AsyncMock() + user_resource = AsyncMock() + user_resource.workspaceId = "ws-1" + user_resource.parentWorkspaceServiceId = "svc-1" + + with pytest.raises(HTTPException) as exc_info: + await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=AsyncMock(), + workspace_service_repo=workspace_service_repo, + resource_template_repo=AsyncMock(), + operations_repo=operations_repo, + resource_history_repo=AsyncMock(), + user=create_test_user(), + ) + assert exc_info.value.status_code == 500 + mock_send_uninstall.assert_not_called() + + +@pytest.mark.asyncio +@patch("services.airlock.wait_for_successful_operation", new_callable=AsyncMock) +@patch("services.airlock.disable_user_resource") +@patch("services.airlock.send_uninstall_message") +async def test_delete_review_user_resource_waits_for_disable_before_uninstall(mock_send_uninstall, mock_disable, wait_mock): + from models.domain.operation import Operation + + disable_op = Operation(id="disable", resourceId="resource", resourcePath="/workspaces/ws", action="upgrade") + mock_disable.return_value = disable_op + + user_resource = MagicMock(workspaceId="ws-1", parentWorkspaceServiceId="svc-1", id="resource") + result = await delete_review_user_resource( + user_resource=user_resource, + user_resource_repo=AsyncMock(), + workspace_service_repo=AsyncMock(), + resource_template_repo=AsyncMock(), + operations_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + user=create_test_user(), + wait_for_completion=False) + + mock_send_uninstall.assert_not_awaited() + assert result is disable_op + + +@pytest.mark.asyncio +async def test_wait_for_successful_operation_retries_on_entity_does_not_exist(): + from services.airlock import wait_for_successful_operation + from db.errors import EntityDoesNotExist + + repo = AsyncMock() + repo.get_operation_by_id.side_effect = EntityDoesNotExist + + with pytest.raises(HTTPException) as exc_info: + await wait_for_successful_operation(repo, "op-123", timeout=0.1, poll_interval=0.02) + assert exc_info.value.status_code == 504 + + +@pytest.mark.asyncio +@patch("services.airlock.delete_review_user_resource") +async def test_delete_all_review_user_resources_retries_on_lease_contention(mock_delete_review): + from services.airlock import delete_all_review_user_resources + from models.domain.operation import Status + + airlock_request = sample_airlock_request() + + user_resource_repo = AsyncMock() + user_resource_repo.get_user_resource_by_id.return_value = AsyncMock() + + # First attempt raises 409 Conflict (lease contention), second succeeds + mock_op = MagicMock() + mock_op.id = "op-1" + mock_delete_review.side_effect = [ + HTTPException(status_code=409, detail="Workspace has an active operation"), + mock_op, + ] + + operations_repo = AsyncMock() + op_record = MagicMock() + op_record.status = Status.Deleted + operations_repo.get_operation_by_id.return_value = op_record + + with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + ops = await delete_all_review_user_resources( + airlock_request=airlock_request, + user_resource_repo=user_resource_repo, + workspace_service_repo=AsyncMock(), + resource_template_repo=AsyncMock(), + operations_repo=operations_repo, + resource_history_repo=AsyncMock(), + user=create_test_user(), + ) + + assert len(ops) == 1 + assert mock_delete_review.call_count == 2 + mock_sleep.assert_awaited() + + +@pytest.mark.asyncio +async def test_redeploy_review_vm_after_delete_waits_deploys_and_publishes(): + from services.airlock import _redeploy_review_vm_after_delete + + airlock_request = sample_airlock_request(status=AirlockRequestStatus.InReview) + user = create_test_user() + workspace = sample_workspace() + user_resource = MagicMock(id="replacement-resource") + operation_repo = AsyncMock() + airlock_request_repo = AsyncMock() + user_resource_repo = AsyncMock() + workspace_service_repo = AsyncMock() + resource_template_repo = AsyncMock() + resource_history_repo = AsyncMock() + + with patch("services.airlock.wait_for_successful_operation", new_callable=AsyncMock) as wait_mock, \ + patch("services.airlock._deploy_vm", new_callable=AsyncMock, return_value=(user_resource, MagicMock())) as deploy_mock, \ + patch("services.airlock.update_and_publish_event_airlock_request", new_callable=AsyncMock) as publish_mock: + await _redeploy_review_vm_after_delete( + airlock_request, user, workspace, "review-workspace", "review-service", "review-template", + user_resource_repo, workspace_service_repo, operation_repo, airlock_request_repo, + resource_template_repo, resource_history_repo, "delete-operation") + + wait_mock.assert_awaited_once_with(operation_repo, "delete-operation") + deploy_mock.assert_awaited_once() + airlock_request_repo.get_airlock_request_by_id.assert_awaited_once_with(airlock_request.id) + publish_mock.assert_awaited_once() + assert publish_mock.await_args.kwargs["review_user_resource"].userResourceId == "replacement-resource" diff --git a/core/terraform/api-webapp.tf b/core/terraform/api-webapp.tf index 47afeb83cb..07a90a1fc6 100644 --- a/core/terraform/api-webapp.tf +++ b/core/terraform/api-webapp.tf @@ -45,6 +45,7 @@ resource "azurerm_linux_web_app" "api" { "SERVICE_BUS_RESOURCE_REQUEST_QUEUE" = azurerm_servicebus_queue.workspacequeue.name "SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE" = azurerm_servicebus_queue.service_bus_deployment_status_update_queue.name "SERVICE_BUS_STEP_RESULT_QUEUE" = module.airlock_resources.service_bus_step_result_queue + "SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE" = azurerm_servicebus_queue.airlock_workflow.name "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.id.client_id "TRE_ID" = var.tre_id "RESOURCE_LOCATION" = azurerm_resource_group.core.location diff --git a/core/terraform/json-to-env.sh b/core/terraform/json-to-env.sh index 0360fc2cc4..1439c47672 100755 --- a/core/terraform/json-to-env.sh +++ b/core/terraform/json-to-env.sh @@ -53,6 +53,10 @@ jq -r ' "path": "service_bus_deployment_status_queue", "env_var": "SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE" }, + { + "path": "service_bus_airlock_workflow_queue", + "env_var": "SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE" + }, { "path": "service_bus_step_result_queue", "env_var": "SERVICE_BUS_STEP_RESULT_QUEUE" diff --git a/core/terraform/outputs.tf b/core/terraform/outputs.tf index e4e22f97eb..96152f0091 100644 --- a/core/terraform/outputs.tf +++ b/core/terraform/outputs.tf @@ -50,6 +50,10 @@ output "service_bus_deployment_status_queue" { value = azurerm_servicebus_queue.service_bus_deployment_status_update_queue.name } +output "service_bus_airlock_workflow_queue" { + value = azurerm_servicebus_queue.airlock_workflow.name +} + output "state_store_resource_id" { value = azurerm_cosmosdb_account.tre_db_account.id } diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..dbb3427a86 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -70,6 +70,12 @@ resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" requires_session = true } +resource "azurerm_servicebus_queue" "airlock_workflow" { + name = "airlock-workflow" + namespace_id = azurerm_servicebus_namespace.sb.id + partitioning_enabled = false +} + resource "azurerm_private_dns_zone" "servicebus" { name = module.terraform_azurerm_environment_configuration.private_links["privatelink.servicebus.windows.net"] resource_group_name = azurerm_resource_group.core.name diff --git a/core/version.txt b/core/version.txt index 12d80d0630..670ad2c1cf 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.16.17" +__version__ = "0.16.18" diff --git a/docs/tre-templates/pipeline-templates/pipeline-schema.md b/docs/tre-templates/pipeline-templates/pipeline-schema.md index 411e5f4302..065730e192 100644 --- a/docs/tre-templates/pipeline-templates/pipeline-schema.md +++ b/docs/tre-templates/pipeline-templates/pipeline-schema.md @@ -8,7 +8,7 @@ This document will help you write a valid `pipeline: {}` block in your template. "pipeline": { "install": [ // <-- [install | upgrade | uninstall] { - "stepId": "a unique string value here", + "stepId": "a unique string value within this template", "stepTitle": "Friendly description of the step here - will be displayed in the UI", "resourceTemplateName": "name of the resource template to update", // only required for shared_service targets "resourceType": "shared-service", // [ shared-service | user-resource | workspace-service | workspace ] @@ -83,6 +83,21 @@ To do so, the `arraySubstitutionAction` field supports the following values: - `replace` - find this object in the array (using the `arrayMatchField` value), and replace it with this value - `remove` - remove this property from the array (useful for `uninstall` actions) +## Reserved Cleanup Step +Use `address-space-cleanup` as the `stepId` for a trailing workspace upgrade that releases a workspace service's address space during uninstall: + +```json +{ + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] +} +``` + +This identifier is reserved for that cleanup behavior and is only permitted as the final step in a `workspace-service` `uninstall` pipeline. Step IDs only need to be unique within a single template's pipeline. + ## Notes - Each step is executed in serial, in the order defined in the template - Theoretically any number of steps could be created diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index e57858caa2..6caa491fe0 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-azureml -version: 1.1.6 +version: 1.1.7 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/azureml/template_schema.json b/templates/workspace_services/azureml/template_schema.json index 1dfa2fdde8..6e4cdd2c69 100644 --- a/templates/workspace_services/azureml/template_schema.json +++ b/templates/workspace_services/azureml/template_schema.json @@ -412,7 +412,14 @@ }, { "stepId": "main" + }, + { + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } -} +} \ No newline at end of file diff --git a/templates/workspace_services/databricks/porter.yaml b/templates/workspace_services/databricks/porter.yaml index 9cef35d4b7..cecd9d952c 100644 --- a/templates/workspace_services/databricks/porter.yaml +++ b/templates/workspace_services/databricks/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-databricks -version: 1.0.18 +version: 1.0.19 description: "An Azure TRE service for Azure Databricks." registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/databricks/template_schema.json b/templates/workspace_services/databricks/template_schema.json index 83e9007c58..d643c05855 100644 --- a/templates/workspace_services/databricks/template_schema.json +++ b/templates/workspace_services/databricks/template_schema.json @@ -76,7 +76,9 @@ "name": "databricks", "description": "Communication with Azure Databricks dependancies.", "source_addresses": "{{ resource.properties.databricks_address_prefixes }}", - "destination_addresses": [ "AzureDatabricks"], + "destination_addresses": [ + "AzureDatabricks" + ], "destination_ports": [ "443" ], @@ -112,9 +114,15 @@ "name": "AzureAD", "description": "AAD access", "source_addresses": "{{ resource.properties.workspace_address_spaces }}", - "destination_addresses": ["AzureActiveDirectory"], - "destination_ports": ["*"], - "protocols": ["TCP"] + "destination_addresses": [ + "AzureActiveDirectory" + ], + "destination_ports": [ + "*" + ], + "protocols": [ + "TCP" + ] } ] } @@ -210,7 +218,9 @@ "name": "databricks", "description": "Communication with Azure Databricks dependancies.", "source_addresses": "{{ resource.properties.databricks_address_prefixes }}", - "destination_addresses": [ "AzureDatabricks"], + "destination_addresses": [ + "AzureDatabricks" + ], "destination_ports": [ "443" ], @@ -246,9 +256,15 @@ "name": "AzureAD", "description": "AAD access", "source_addresses": "{{ resource.properties.workspace_address_spaces }}", - "destination_addresses": ["AzureActiveDirectory"], - "destination_ports": ["*"], - "protocols": ["TCP"] + "destination_addresses": [ + "AzureActiveDirectory" + ], + "destination_ports": [ + "*" + ], + "protocols": [ + "TCP" + ] } ] } @@ -350,7 +366,14 @@ }, { "stepId": "main" + }, + { + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } -} +} \ No newline at end of file diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index 106e3f337a..0d2d9b8515 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.9 +version: 0.3.10 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/ohdsi/template_schema.json b/templates/workspace_services/ohdsi/template_schema.json index fddd1cd6d7..30f249afee 100644 --- a/templates/workspace_services/ohdsi/template_schema.json +++ b/templates/workspace_services/ohdsi/template_schema.json @@ -370,7 +370,14 @@ }, { "stepId": "main" + }, + { + "stepId": "address-space-cleanup", + "stepTitle": "Upgrade to ensure workspace is aware of address space removal", + "resourceType": "workspace", + "resourceAction": "upgrade", + "properties": [] } ] } -} +} \ No newline at end of file