From e2a60e2ef39620bdf7b02f9d8a7134aa511d48c9 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 06:20:37 +0100 Subject: [PATCH 01/37] Send an empty outputs list when Porter outputs cannot be read (#5297) If the Porter action succeeds but the output command fails, the resource processor sent "outputs": {} with a failed status. The API model reads outputs as a list, so it rejected the message as not formatted correctly. The message went to the dead-letter queue, and the operation stayed in progress. A live AML compute run hit this. A transient container registry login failure stopped the firewall step from reading its outputs, and the AML service disable stayed in pipeline_running for two hours. Return an empty list on both paths where get_porter_outputs has no outputs. --- CHANGELOG.md | 1 + resource_processor/_version.py | 2 +- resource_processor/tests_rp/test_runner.py | 33 ++++++++++++++++++++++ resource_processor/vmss_porter/runner.py | 5 ++-- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index decc945754..88c3776b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/resource_processor/_version.py b/resource_processor/_version.py index 2ad2391828..9453c00a2d 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.13.15" +__version__ = "0.13.16" diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 857204caa1..24f0f25d60 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -1086,6 +1086,39 @@ async def test_get_porter_outputs(mock_run_porter, mock_build_porter_command_for assert outputs == [{"name": "output1", "value": "value1"}] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output_result, expected_status", + [ + pytest.param( + (1, None, "ERROR: Could not connect to the registry login server 'tre.azurecr.io'."), + "updating_failed", + id="output-command-fails", + ), + pytest.param((0, "not json", None), "updated", id="invalid-json"), + ], +) +async def test_invoke_porter_action_sends_outputs_as_a_list(output_result, expected_status): + # The API model reads outputs as a list. A dict fails validation, so the + # API never records the status and the operation stays in progress. + sender = AsyncMock() + client = Mock() + client.get_queue_sender.return_value = sender + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "upgrade", "stepId": "test_step_id", "operationId": "test_operation_id"} + + with ( + patch("vmss_porter.runner.build_porter_command", return_value=([["porter", "upgrade"]], None, "params", None)), + patch("vmss_porter.runner.build_porter_command_for_outputs", return_value=[["porter", "output", "list"]]), + patch("vmss_porter.runner.run_porter", side_effect=[(0, "", None), output_result]), + ): + await invoke_porter_action(msg_body, client, config) + + final_status = json.loads(str(sender.send_messages.call_args.args[0])) + assert final_status["status"] == expected_status + assert final_status["outputs"] == [] + + @pytest.mark.asyncio @patch("asyncio.sleep", new_callable=AsyncMock) async def test_check_runners(_): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index e11154cb07..bbeabb0cce 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -480,9 +480,10 @@ async def get_porter_outputs(msg_body: dict, config: dict): error_message = "Error context message = " + " ".join(err.split("\n")) installation_id = msg_body["id"] logger.info(f"{installation_id}: Failed to get outputs with error = {error_message}") - return False, {} + # The API reads outputs as a list. It rejects a dict, so the operation never gets the failed status. + return False, [] else: - outputs_json = {} + outputs_json = [] try: outputs_json = json.loads(stdout) From 641336d4f84cb648190bb52edd9c723bb0e88049 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 24 Oct 2026 21:47:20 +0100 Subject: [PATCH 02/37] Give each resource processor worker its own Azure CLI folder (#5297) The resource processor starts several worker processes, and each worker runs az cloud set, az login and az acr login before each Porter action. All workers used the same folder, ~/.azure. In a live run, two workspace installs started 1 second apart. One worker read azureProfile.json while the other wrote it. The Azure CLI said "Failed to load or parse file /root/.azure/azureProfile.json. It will be overridden by default settings", and the next az acr login failed with "Please run 'az login' to setup account". The install failed after 3 seconds, before Porter ran. Each worker now sets AZURE_CONFIG_DIR to its own folder in the commands that it runs. The main process keeps the default folder. --- CHANGELOG.md | 1 + resource_processor/tests_rp/test_runner.py | 43 ++++++++++++++++++---- resource_processor/vmss_porter/runner.py | 9 +++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88c3776b44..94c33d482a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ BUG FIXES: * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) +* Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 24f0f25d60..bb81ebf9f1 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -8,6 +8,7 @@ check_runners, runner, run_porter, + worker_config, _cleanup_param_set, ) import json @@ -85,13 +86,17 @@ async def test_runner(mock_receive_message, mock_service_bus_client, mock_defaul mock_service_bus_client, mock_default_credential, "test_msi_id" ) - config = {"vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace"} + config = { + "vmss_msi_id": "test_msi_id", + "service_bus_namespace": "test_namespace", + "porter_env": {"HOME": "/root"}, + } await runner(0, config) mock_default_credential.assert_called_once_with("test_msi_id") mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) # Verify context manager entered and exited cleanly mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -108,13 +113,17 @@ async def test_runner_no_msi_id(mock_receive_message, mock_service_bus_client, m mock_service_bus_client, mock_default_credential, None ) - config = {"vmss_msi_id": None, "service_bus_namespace": "test_namespace"} + config = { + "vmss_msi_id": None, + "service_bus_namespace": "test_namespace", + "porter_env": {"HOME": "/root"}, + } await runner(0, config) mock_default_credential.assert_called_once_with(None) mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) # Verify context manager entered and exited cleanly mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -132,14 +141,18 @@ async def test_runner_exception(mock_receive_message, mock_service_bus_client, m ) mock_receive_message.side_effect = Exception("Test Exception") - config = {"vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace"} + config = { + "vmss_msi_id": "test_msi_id", + "service_bus_namespace": "test_namespace", + "porter_env": {"HOME": "/root"}, + } with pytest.raises(Exception, match="Test Exception"): await runner(0, config) mock_default_credential.assert_called_once_with("test_msi_id") mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) # Verify context manager entered and exited cleanly, even on exception mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -148,6 +161,18 @@ async def test_runner_exception(mock_receive_message, mock_service_bus_client, m mock_service_bus_client.return_value.__aexit__.assert_awaited_once() +def test_each_worker_has_its_own_azure_cli_folder(): + # Two workers that share one folder can read a profile file that the other one writes. + config = {"porter_env": {"HOME": "/root", "PATH": "/bin"}} + + first, second = worker_config(0, config), worker_config(1, config) + + assert first["porter_env"]["AZURE_CONFIG_DIR"] == "/root/.azure-worker-0" + assert second["porter_env"]["AZURE_CONFIG_DIR"] == "/root/.azure-worker-1" + assert first["porter_env"]["PATH"] == "/bin" + assert "AZURE_CONFIG_DIR" not in config["porter_env"] + + @pytest.mark.asyncio async def test_default_credentials_closes_real_credential_on_exception(mock_service_bus_client): mock_credential = AsyncMock() @@ -158,7 +183,11 @@ async def test_default_credentials_closes_real_credential_on_exception(mock_serv patch("vmss_porter.runner.DefaultAzureCredential", return_value=mock_credential), patch("vmss_porter.runner.receive_message", side_effect=Exception("Test Exception")), ): - config = {"vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace"} + config = { + "vmss_msi_id": "test_msi_id", + "service_bus_namespace": "test_namespace", + "porter_env": {"HOME": "/root"}, + } with pytest.raises(Exception, match="Test Exception"): await runner(0, config) diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index bbeabb0cce..a7a0ae55ce 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -499,7 +499,16 @@ async def get_porter_outputs(msg_body: dict, config: dict): return True, outputs_json +def worker_config(process_number: int, config: dict) -> dict: + # Each worker signs in with the Azure CLI before each Porter action. Workers that share one CLI folder + # read and write the same profile file at the same time, so each worker gets its own folder. + porter_env = config["porter_env"] + folder = os.path.join(porter_env["HOME"], f".azure-worker-{process_number}") + return {**config, "porter_env": {**porter_env, "AZURE_CONFIG_DIR": folder}} + + async def runner(process_number: int, config: dict): + config = worker_config(process_number, config) with tracer.start_as_current_span(process_number): async with default_credentials(config["vmss_msi_id"]) as credential: async with ServiceBusClient(config["service_bus_namespace"], credential) as service_bus_client: From 78bdfa3b7669080597b2db71248825b6cf64abed Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 24 Oct 2026 02:40:11 +0100 Subject: [PATCH 03/37] Return the trace ID of each API request in a Request-Id header (#5202) An upstream SQL run got HTTP 500 while it polled a workspace delete. The cause is unknown. Two gaps hid it: - The poll diagnostics log the request-id, x-request-id and traceparent headers of a failed response. The API sends none of them, so the test log has no ID to find the server logs. - In those environments, App Insights got no API telemetry, because Terraform linked App Insights to the private link scope before its private endpoint existed. #5281 fixed that. A small ASGI middleware now adds `Request-Id: |..` to each HTTP response. It wraps the generic error middleware, so a response for an unhandled exception also gets the header. App Insights stores the trace ID as OperationId. The E2E docs give the query. In another fork environment, App Insights has API telemetry since 2026-10-21 21:30 UTC. It shows no HTTP 5xx in about 31,000 API requests, which include many workspace deletes. So the fault did not occur here, and this change does not fix it. It makes a recurrence traceable. --- CHANGELOG.md | 1 + api_app/_version.py | 2 +- api_app/api/request_id.py | 36 +++++++++++++++ api_app/main.py | 3 ++ .../tests_ma/test_api/test_instrumentation.py | 44 +++++++++++++++++++ docs/tre-developers/end-to-end-tests.md | 10 +++++ 6 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 api_app/api/request_id.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c33d482a..edfb11ede4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ENHANCEMENTS: * Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) +* Add a `Request-Id` header with the trace ID to each API response, also to an HTTP 500. The E2E operation poll already logs this header, so a failed poll can be found in App Insights. (`api` 0.28.10) ([#5202](https://github.com/microsoft/AzureTRE/issues/5202)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/api_app/_version.py b/api_app/_version.py index 389456f828..3429bc0f86 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.9" +__version__ = "0.28.10" diff --git a/api_app/api/request_id.py b/api_app/api/request_id.py new file mode 100644 index 0000000000..d754cc1afd --- /dev/null +++ b/api_app/api/request_id.py @@ -0,0 +1,36 @@ +from opentelemetry import trace +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +HEADER = b"request-id" + + +def current_request_id() -> str: + """Return the trace of this request in the Application Insights Request-Id format. + + App Insights stores the trace ID as OperationId, so a client can find the server logs of a failed request. + """ + context = trace.get_current_span().get_span_context() + if not context.is_valid: + return "" + return f"|{context.trace_id:032x}.{context.span_id:016x}." + + +class RequestIdMiddleware: + """Add the Request-Id header to each HTTP response, also to a response for an unhandled exception.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_request_id(message: Message) -> None: + if message["type"] == "http.response.start": + request_id = current_request_id() + if request_id: + message["headers"] = [*message.get("headers", []), (HEADER, request_id.encode())] + await send(message) + + await self.app(scope, receive, send_with_request_id) diff --git a/api_app/main.py b/api_app/main.py index c5d7e9b79b..f71764686f 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -16,6 +16,7 @@ from api.errors.http_error import http_error_handler from api.errors.validation_error import http422_error_handler from api.errors.generic_error import generic_error_handler +from api.request_id import RequestIdMiddleware from core import config from db.events import bootstrap_database from services.logging import initialize_logging, logger @@ -139,6 +140,8 @@ def get_application() -> FastAPI: ) application.add_middleware(ServerErrorMiddleware, handler=generic_error_handler) + # Added after ServerErrorMiddleware, so it also wraps the response for an unhandled exception. + application.add_middleware(RequestIdMiddleware) # Allow local UI debugging with local API if config.ENABLE_LOCAL_DEBUGGING: diff --git a/api_app/tests_ma/test_api/test_instrumentation.py b/api_app/tests_ma/test_api/test_instrumentation.py index 41c4f31d20..a57500179d 100644 --- a/api_app/tests_ma/test_api/test_instrumentation.py +++ b/api_app/tests_ma/test_api/test_instrumentation.py @@ -1,3 +1,5 @@ +import re + import pytest import pytest_asyncio from fastapi import FastAPI, status @@ -49,3 +51,45 @@ async def test_instrumented_cors_preflight(instrumented_client: AsyncClient): assert response.status_code == status.HTTP_200_OK assert response.headers["access-control-allow-origin"] == "http://localhost:3000" + + +@pytest.fixture +def traced_app(): + from main import get_application + from opentelemetry.sdk.trace import TracerProvider + + app = get_application() + + async def fail(): + raise RuntimeError("synthetic failure") + + app.add_api_route("/api/test-failure", fail) + FastAPIInstrumentor.instrument_app(app, tracer_provider=TracerProvider()) + try: + yield app + finally: + FastAPIInstrumentor.uninstrument_app(app) + + +@pytest.mark.parametrize("path, expected_status", [("/api/ping", 200), ("/api/test-failure", 500)]) +async def test_each_response_has_the_request_id_of_its_trace(traced_app: FastAPI, path, expected_status): + async with AsyncClient( + transport=ASGITransport(app=traced_app, raise_app_exceptions=False), base_url="http://testserver" + ) as client: + first = await client.get(path) + second = await client.get(path) + + assert first.status_code == expected_status + assert re.fullmatch(r"\|[0-9a-f]{32}\.[0-9a-f]{16}\.", first.headers["request-id"]) + # Each request has its own trace. + assert first.headers["request-id"][:33] != second.headers["request-id"][:33] + + +async def test_a_request_without_a_trace_has_no_request_id(): + from main import get_application + + async with AsyncClient(transport=ASGITransport(app=get_application()), base_url="http://testserver") as client: + response = await client.get("/api/ping") + + assert response.status_code == status.HTTP_200_OK + assert "request-id" not in response.headers diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 306f4ca158..d807e3d36c 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -36,6 +36,16 @@ Use the "Run and Debug" panel within Visual Studio Code, select "E2E Extended", If an E2E test fails while waiting for an operation, check its test log. Use the timestamp and operation ID to find the matching API logs. +If an operation poll gets an unexpected HTTP status, the test logs the UTC time, the operation path, the status and the `request-id` header of the response. +The API puts the trace of each request in this header as `|..`, also for HTTP 500. +App Insights stores the trace ID as `OperationId`. Run this query in the Log Analytics workspace of the environment: + +```kusto +union AppRequests, AppExceptions, AppTraces +| where OperationId == "" +| order by TimeGenerated asc +``` + ## Airlock draft deletion The draft-deletion case requires AzureCloud, `ARM_SUBSCRIPTION_ID`, `TRE_ID` and the existing CI Azure login. From 5e60909885de2248eab1fd94ec672e13a568b05b Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 24 Oct 2026 02:34:49 +0100 Subject: [PATCH 04/37] Run each resource processor step once after a session lock loss (#5236) A live run on another fork showed one uninstall step that two workers ran at the same time. The session lock of the first worker expired, Service Bus gave the message to a second worker, and the API saved the result of the second run. This commit has parts 1, 2, 3 and 5 of the change for #5236. Part 1, session lock: workspacequeue keeps the lock for 5 minutes (the maximum). A task in the resource processor renews it every minute and tries again after a failure until the lock expires. It logs each failure. AutoLockRenewer stopped after one failure and logged it at debug level. Part 2, execution lease: before Porter starts, the worker gets a lease on the blob rp-executions/ in the management storage account. The resource processor identity can already write there. A worker that cannot get the lease keeps its session and waits, so the order of the steps does not change. The blob records the last step. If the same step finished, the worker sends the recorded status again and does not run Porter. A worker that cannot complete the message after a lock loss now logs a warning. If the record write fails, for example after a lease loss, the worker logs an error and still sends the status. The renewal tasks stop without hiding a cancellation of their caller. Part 3, API: each status message has an executionId. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. A message without executionId keeps the earlier rules. "Update status in DB" is now logged only after the save. Part 5: an upgrade does not install a resource that has no Porter installation. It fails with updating_failed. The next commit adds Part 4. The test setting test_skip_session_lock_renewal (RP_BUNDLE_VALUES, off by default) stops the session lock renewal while Porter runs. --- CHANGELOG.md | 2 + api_app/models/domain/operation.py | 5 + .../service_bus/deployment_status_updater.py | 34 ++- .../test_deployment_status_update.py | 180 +++++++++++++ core/terraform/servicebus.tf | 2 + core/version.txt | 2 +- docs/tre-developers/resource-processor.md | 16 ++ resource_processor/helpers/execution_lease.py | 103 ++++++++ resource_processor/helpers/lock_renewal.py | 59 +++++ resource_processor/shared/config.py | 13 + resource_processor/tests_rp/test_config.py | 50 ++++ .../tests_rp/test_execution_lease.py | 99 ++++++++ .../tests_rp/test_lock_renewal.py | 108 ++++++++ resource_processor/tests_rp/test_run_step.py | 214 ++++++++++++++++ resource_processor/tests_rp/test_runner.py | 176 ++++++++++--- .../vmss_porter/requirements.txt | 1 + resource_processor/vmss_porter/runner.py | 238 +++++++++++++----- 17 files changed, 1196 insertions(+), 106 deletions(-) create mode 100644 resource_processor/helpers/execution_lease.py create mode 100644 resource_processor/helpers/lock_renewal.py create mode 100644 resource_processor/tests_rp/test_config.py create mode 100644 resource_processor/tests_rp/test_execution_lease.py create mode 100644 resource_processor/tests_rp/test_lock_renewal.py create mode 100644 resource_processor/tests_rp/test_run_step.py diff --git a/CHANGELOG.md b/CHANGELOG.md index edfb11ede4..fc375a427d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) +* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource without a Porter installation. + (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 2c37666a26..79eda850e0 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -67,6 +67,9 @@ class OperationStep(AzureTREModel): statusUpdateComplete: bool = Field( False, description="Whether the terminal status resource writes and next-step dispatch have completed" ) + executionId: Optional[str] = Field( + default=None, description="The resource processor run whose status the step shows last" + ) # An example for this property will be if we have a step that is responsible for updating the firewall, and its origin was the guacamole workspace service, the id here will be the guacamole id sourceTemplateResourceId: Optional[str] = Field( default=None, title="sourceTemplateResourceId", description="Id of the parent of the resource to update" @@ -130,3 +133,5 @@ class DeploymentStatusUpdateMessage(AzureTREModel): status: Status = Field(title="", description="") message: str = Field(title="", description="") outputs: List[Output] = Field(title="", description="", default_factory=list) + # A resource processor before 0.13.17 does not send this field. + executionId: Optional[str] = Field(default=None, title="", description="The resource processor run") diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 08ef82c42e..316b880cd1 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -116,7 +116,6 @@ async def process_message(self, msg): logger.info(f"Received and parsed JSON for: {msg.correlation_id}") complete_message = await self.update_status_in_database(message) - logger.info(f"Update status in DB for {message.operationId} - {message.status}") except (json.JSONDecodeError, ValidationError): logger.exception(f"{strings.DEPLOYMENT_STATUS_MESSAGE_FORMAT_INCORRECT}: {msg.correlation_id}") except Exception: @@ -151,6 +150,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise f"Error finding step {message.stepId} in operation {message.operationId}" + if self.ignore_repeated_status(operation, step_to_update, message): + return True + uninstall_operation = operation.action == RequestAction.UnInstall uninstall_pending_status = Status.PipelineRunning if len(operation.steps) > 1 else Status.Deleting if uninstall_operation and self.ignore_uninstall_callback(operation, current_step_index, message): @@ -173,6 +175,10 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage step_to_update.status = message.status step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() + if message.executionId is not None: + step_to_update.executionId = message.executionId + # Repeated runs make the final status complete only after its writes and dispatch. + track_completion = uninstall_operation or message.executionId is not None # update the overall headline operation status await self.update_overall_operation_status(operation, step_to_update, is_last_step) @@ -184,6 +190,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # save the operation await self.operations_repo.update_item(operation) + logger.info(f"Update status in DB for {message.operationId} - {message.status}") # copy the step status to the resource item, for convenience resource_id = uuid.UUID(step_to_update.resourceId) @@ -203,7 +210,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. if not step_to_update.is_success(): - if uninstall_operation and step_to_update.is_failure(): + if track_completion and step_to_update.is_failure(): step_to_update.statusUpdateComplete = True await self.operations_repo.update_item(operation) return True @@ -265,7 +272,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if uninstall_operation: await self.update_primary_uninstall_status(operation) - if uninstall_operation: + if track_completion: # Save only after the resource writes and dispatch. A retry must # resume those effects if an earlier database write succeeded. step_to_update.statusUpdateComplete = True @@ -282,6 +289,27 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result + def ignore_repeated_status( + self, operation: Operation, step: OperationStep, message: DeploymentStatusUpdateMessage + ) -> bool: + """ + Service Bus can give a step to a second resource processor worker. Keep the first final status of the step. + A message without an execution ID comes from an older resource processor, so the earlier rules apply to it. + """ + if message.executionId is None or not (step.is_success() or step.is_failure()): + return False + if step.executionId != message.executionId: + logger.info(f"Ignoring a status from another run of step {operation.id}/{step.id}") + return True + if message.status != step.status: + logger.info(f"Ignoring a later status from the run that finished step {operation.id}/{step.id}") + return True + if step.statusUpdateComplete: + logger.info(f"Ignoring a repeated final status for step {operation.id}/{step.id}") + return True + # The earlier copy stopped before all of its writes. Process this copy to finish them. + return False + def ignore_uninstall_callback(self, operation: Operation, step_index: int, message: DeploymentStatusUpdateMessage): step = operation.steps[step_index] if step.is_success() or step.is_failure(): diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 479d15803e..6c45745711 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1045,3 +1045,183 @@ def fail_failure_write(item): assert stored["operation"].steps[0].statusUpdateComplete is True updater.resource_repo.update_item_dict.assert_not_awaited() send.assert_awaited_once() + + +STEP_STATUSES = { + RequestAction.Install: (Status.Deploying, Status.Deployed, Status.DeploymentFailed), + RequestAction.Upgrade: (Status.Updating, Status.Updated, Status.UpdatingFailed), + "restart": (Status.InvokingAction, Status.ActionSucceeded, Status.ActionFailed), + RequestAction.UnInstall: (Status.Deleting, Status.Deleted, Status.DeletingFailed), +} + + +@pytest.fixture(params=list(STEP_STATUSES), ids=str) +def persisted_pipeline(request): + """A main step followed by a firewall step, with operation and resource writes kept in memory.""" + action = request.param + primary = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") + firewall = create_sample_workspace_object("7af8f146-d751-4587-a22e-eb44c1c6fbe2") + operation = create_sample_operation(primary.id, action) + operation.steps[0].status = Status.AwaitingAction + operation.steps.append( + OperationStep( + id="firewall-step", + templateStepId="firewall", + resourceId=firewall.id, + resourceAction=RequestAction.Upgrade, + sourceTemplateResourceId=primary.id, + status=Status.AwaitingUpdate, + ) + ) + stored = {"operation": operation, "resources": {primary.id: primary, firewall.id: firewall}} + updater = DeploymentStatusUpdater() + updater.operations_repo = AsyncMock() + updater.resource_repo = AsyncMock() + updater.resource_template_repo = AsyncMock() + updater.resource_history_repo = AsyncMock() + updater.operations_repo.get_operation_by_id.side_effect = lambda _: stored["operation"].model_copy(deep=True) + updater.operations_repo.update_item.side_effect = lambda item: stored.update(operation=item.model_copy(deep=True)) + updater.resource_repo.get_resource_by_id.side_effect = lambda item_id: stored["resources"][str(item_id)].model_copy( + deep=True + ) + updater.resource_repo.get_resource_dict_by_id.side_effect = lambda item_id: stored["resources"][ + str(item_id) + ].model_dump() + + def save_resource(item): + stored["resources"][item.id] = item.model_copy(deep=True) + + def save_resource_dict(item): + stored["resources"][item["id"]] = stored["resources"][item["id"]].model_copy(update=item, deep=True) + + updater.resource_repo.update_item.side_effect = save_resource + updater.resource_repo.update_item_dict.side_effect = save_resource_dict + + def message(status, execution_id="run-a", step_index=0): + step = stored["operation"].steps[step_index] + return DeploymentStatusUpdateMessage( + operationId=operation.id, + stepId=step.id, + id=step.resourceId, + status=status, + message=f"{status} from {execution_id}", + executionId=execution_id, + ) + + with ( + patch("service_bus.deployment_status_updater.update_resource_for_step", return_value=firewall), + patch("service_bus.deployment_status_updater.send_deployment_message", new_callable=AsyncMock) as send, + ): + yield updater, stored, message, send, STEP_STATUSES[action] + + +@pytest.mark.parametrize("final_index", [1, 2]) +async def test_a_status_from_another_run_does_not_replace_the_final_status(persisted_pipeline, final_index): + updater, stored, message, send, step_statuses = persisted_pipeline + assert await updater.update_status_in_database(message(step_statuses[0])) is True + assert await updater.update_status_in_database(message(step_statuses[final_index])) is True + assert stored["operation"].steps[0].executionId == "run-a" + before = copy.deepcopy(stored) + + for status in step_statuses: + assert await updater.update_status_in_database(message(status, execution_id="run-b")) is True + + assert stored == before + assert send.await_count == (1 if final_index == 1 else 0) + + +async def test_a_later_status_from_the_finished_run_is_ignored(persisted_pipeline): + updater, stored, message, send, (in_progress, success, failure) = persisted_pipeline + assert await updater.update_status_in_database(message(success)) is True + before = copy.deepcopy(stored) + + assert await updater.update_status_in_database(message(in_progress)) is True + assert await updater.update_status_in_database(message(failure)) is True + + assert stored == before + send.assert_awaited_once() + + +async def test_a_repeated_final_status_does_not_dispatch_the_next_step_twice(persisted_pipeline): + updater, stored, message, send, (_, success, _) = persisted_pipeline + assert await updater.update_status_in_database(message(success)) is True + assert stored["operation"].steps[0].statusUpdateComplete is True + before = copy.deepcopy(stored) + + assert await updater.update_status_in_database(message(success)) is True + + assert stored == before + send.assert_awaited_once() + + +async def test_a_repeated_final_status_finishes_the_writes_of_the_first_copy(persisted_pipeline): + updater, stored, message, send, (_, success, _) = persisted_pipeline + save = updater.resource_repo.update_item.side_effect + writes = [] + + def fail_first_write(item): + writes.append(item.id) + if len(writes) == 1: + raise RuntimeError("Cosmos DB is not available") + save(item) + + updater.resource_repo.update_item.side_effect = fail_first_write + assert await updater.update_status_in_database(message(success)) is False + assert stored["operation"].steps[0].status == success + assert stored["operation"].steps[0].statusUpdateComplete is False + send.assert_not_awaited() + + assert await updater.update_status_in_database(message(success)) is True + + assert stored["operation"].steps[0].statusUpdateComplete is True + send.assert_awaited_once() + + +async def test_a_new_run_before_the_final_status_takes_over_the_step(persisted_pipeline): + updater, stored, message, send, (in_progress, success, _) = persisted_pipeline + assert await updater.update_status_in_database(message(in_progress, execution_id="run-a")) is True + assert await updater.update_status_in_database(message(in_progress, execution_id="run-b")) is True + assert stored["operation"].steps[0].executionId == "run-b" + + assert await updater.update_status_in_database(message(success, execution_id="run-b")) is True + assert await updater.update_status_in_database(message(success, execution_id="run-a")) is True + + assert stored["operation"].steps[0].executionId == "run-b" + assert stored["operation"].steps[0].message == f"{success} from run-b" + send.assert_awaited_once() + + +async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): + workspace = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") + operation = create_sample_operation(workspace.id, RequestAction.Install) + operation.steps[0].status = Status.Deployed + updater = DeploymentStatusUpdater() + updater.operations_repo = AsyncMock() + updater.operations_repo.get_operation_by_id.return_value = operation + updater.resource_repo = AsyncMock() + updater.resource_repo.get_resource_by_id.return_value = workspace + message = DeploymentStatusUpdateMessage( + operationId=operation.id, + stepId=operation.steps[0].id, + id=workspace.id, + status=Status.DeploymentFailed, + message="failed", + ) + + assert await updater.update_status_in_database(message) is True + + assert operation.steps[0].status == Status.DeploymentFailed + assert operation.steps[0].executionId is None + + +@patch("service_bus.deployment_status_updater.logger") +async def test_an_ignored_status_is_not_logged_as_a_database_update(logger, persisted_pipeline): + updater, _, message, _, (_, success, failure) = persisted_pipeline + assert await updater.update_status_in_database(message(success)) is True + logger.reset_mock() + + assert await updater.update_status_in_database(message(failure, execution_id="run-b")) is True + + logged = " ".join(str(call.args[0]) for call in logger.info.call_args_list) + assert "Ignoring a status from another run" in logged + assert "Update status in DB" not in logged diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index 7fbfb99af8..a8d64e4595 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -56,6 +56,8 @@ resource "azurerm_servicebus_queue" "workspacequeue" { partitioning_enabled = false requires_session = true # use sessions here to make sure updates to each resource happen in serial, in order + # The longest lock that Service Bus permits. The resource processor renews it every minute while Porter runs. + lock_duration = "PT5M" } resource "azurerm_servicebus_queue" "service_bus_deployment_status_update_queue" { diff --git a/core/version.txt b/core/version.txt index c1c747e2aa..6e7cb3b8f3 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.16" +__version__ = "0.18.17" diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 611c8512ad..5bf02764ab 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -119,6 +119,22 @@ CNAB spec defines actions that Porter [implements](https://porter.sh/author-bund Furthermore, Porter provides a set of [mixins](https://porter.sh/mixins/) - analogous to the concrete actions in GitHub workflows and tasks in Azure DevOps pipelines - which simplify and reduce the development cost when implementing deployment logic. For example, Terraform mixin installs the required tools and provides a clean step in the pipeline to execute Terraform deployments. [Exec mixin](https://porter.sh/mixins/exec/) allows running any command or script; especially useful, if no suitable mixin for a specific technology is available. Implementing custom mixins is possible too. +### Repeated messages + +Service Bus can give the same message to more than one worker. The resource processor runs each step once: + +1. The `workspacequeue` queue keeps a session lock for 5 minutes. While a worker runs a step, it renews the lock every minute. If a renewal fails, the worker tries again until the lock expires, and logs each failure. +1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. +1. The blob holds a record of the last step for the resource. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. +1. Each status message has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. +1. An upgrade does not install a resource that has no Porter installation. The resource processor sends `updating_failed`. + +For local debugging, your identity needs the Storage Blob Data Contributor role on the management storage account. + +To test a lost session lock, set the `RP_BUNDLE_VALUES` variable to `{"test_skip_session_lock_renewal": "true"}` and deploy the core. +While Porter runs, the resource processor then does not renew the session lock, so Service Bus gives a step that takes more than 5 minutes to another worker. +Do not use this setting in production. + ### Porter Azure plugin Resource Processor uses [Porter Azure plugin](https://github.com/getporter/azure-plugins) to access secrets in Azure Key Vault. diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py new file mode 100644 index 0000000000..03b8f03613 --- /dev/null +++ b/resource_processor/helpers/execution_lease.py @@ -0,0 +1,103 @@ +import asyncio +import json +import time +from contextlib import asynccontextmanager, suppress +from typing import Callable, Optional + +from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError +from azure.storage.blob.aio import BlobLeaseClient, ContainerClient + +from helpers.lock_renewal import keep_renewed, stop +from shared.logging import logger + +CONTAINER_NAME = "rp-executions" +LEASE_SECONDS = 60 +RENEW_SECONDS = 15 +WAIT_SECONDS = 30 + + +class ExecutionLease: + """One worker at a time holds this lease and runs Porter for a resource. + + The blob keeps a small record of the last step that a worker ran for the resource, + so that a worker that gets the same message again can find the result. + """ + + def __init__(self, blob, lease): + self.blob = blob + self.lease = lease + self.expires = time.monotonic() + LEASE_SECONDS + + async def renew(self): + await self.lease.renew() + self.expires = time.monotonic() + LEASE_SECONDS + + def seconds_left(self) -> float: + return self.expires - time.monotonic() + + async def read(self) -> dict: + download = await self.blob.download_blob(lease=self.lease) + content = await download.readall() + try: + record = json.loads(content or b"{}") + except ValueError: + # A damaged record does not prove a result, so the worker runs Porter. + logger.warning(f"Ignoring an invalid execution record in {self.blob.blob_name}") + return {} + return record if isinstance(record, dict) else {} + + async def write(self, record: dict): + await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) + + +async def create_blob(container: ContainerClient, blob): + for attempt in (1, 2): + try: + await blob.upload_blob(b"{}", overwrite=False) + return + except ResourceNotFoundError: + if attempt == 2: + raise + # The first lease in a deployment creates the container. + with suppress(ResourceExistsError): + await container.create_container() + except HttpResponseError as error: + # The blob exists. With an active lease, Azure can also return 412. + if error.status_code not in (409, 412): + raise + return + + +async def try_acquire(container: ContainerClient, resource_id: str) -> Optional[ExecutionLease]: + blob = container.get_blob_client(resource_id) + await create_blob(container, blob) + lease = BlobLeaseClient(blob) + try: + await lease.acquire(lease_duration=LEASE_SECONDS) + except HttpResponseError as error: + if error.status_code == 409: + return None + raise + return ExecutionLease(blob, lease) + + +@asynccontextmanager +async def execution_lease(container: ContainerClient, resource_id: str, describe: Callable[[], str]): + """Wait for the lease on this resource, keep it while the caller works, then release it.""" + while (execution := await try_acquire(container, resource_id)) is None: + # Another worker runs Porter for this resource. Keep the session and wait, + # so that the steps for the resource stay in order. + logger.info(f"Waiting for another worker to finish with {describe()}") + await asyncio.sleep(WAIT_SECONDS) + renewal = asyncio.create_task( + keep_renewed("Execution lease", execution.renew, execution.seconds_left, describe, interval=RENEW_SECONDS) + ) + try: + yield execution + finally: + await stop(renewal) + try: + await execution.lease.release() + except Exception as error: + # The lease expires by itself after LEASE_SECONDS. + logger.warning(f"Could not release the execution lease for {describe()}: {error!r}") diff --git a/resource_processor/helpers/lock_renewal.py b/resource_processor/helpers/lock_renewal.py new file mode 100644 index 0000000000..462f018bf0 --- /dev/null +++ b/resource_processor/helpers/lock_renewal.py @@ -0,0 +1,59 @@ +import asyncio +from datetime import datetime, timezone +from typing import Awaitable, Callable, Optional + +from shared.logging import logger + +RETRY_SECONDS = 5 + + +async def stop(task: asyncio.Task) -> None: + """Cancel a renewal task and wait for it, without hiding a cancellation of the caller.""" + task.cancel() + try: + await task + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise + + +def seconds_until(moment: Optional[datetime]) -> float: + return (moment - datetime.now(timezone.utc)).total_seconds() if moment else 0 + + +async def keep_renewed( + name: str, + renew: Callable[[], Awaitable[object]], + seconds_left: Callable[[], float], + describe: Callable[[], str], + *, + interval: float, + retry_seconds: float = RETRY_SECONDS, + paused: Callable[[], bool] = lambda: False, +) -> None: + """Renew a lock until the caller cancels this task. + + AutoLockRenewer stops after one failed renewal and logs it only at debug level. + This task tries again until the lock expires, and logs each failure. + """ + while True: + await asyncio.sleep(interval) + if paused(): + continue + attempt = 0 + while True: + try: + await renew() + break + except asyncio.CancelledError: + raise + except Exception as error: + attempt += 1 + remaining = seconds_left() + if remaining <= 0: + # Another worker can now get the lock and run the same step. + logger.error(f"{name} expired after {attempt} failed renewals for {describe()}: {error!r}") + return + logger.warning(f"{name} renewal {attempt} failed for {describe()}: {error!r}") + await asyncio.sleep(min(retry_seconds * attempt, remaining)) diff --git a/resource_processor/shared/config.py b/resource_processor/shared/config.py index a68883c25e..ca319695f8 100644 --- a/resource_processor/shared/config.py +++ b/resource_processor/shared/config.py @@ -4,6 +4,11 @@ VERSION = __version__ +STORAGE_SUFFIXES = { + "AzureCloud": "core.windows.net", + "AzureUSGovernment": "core.usgovcloudapi.net", + "AzureChinaCloud": "core.chinacloudapi.cn", +} def get_config() -> dict: @@ -24,6 +29,9 @@ def get_config() -> dict: config["azure_environment"] = os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") config["aad_authority_url"] = os.environ.get("AAD_AUTHORITY_URL", "https://login.microsoftonline.com") config["microsoft_graph_fqdn"] = os.environ.get("MICROSOFT_GRAPH_FQDN", "graph.microsoft.com") + # The execution leases use the Terraform state account, which the identity of the RP can already write. + suffix = STORAGE_SUFFIXES.get(config["azure_environment"], STORAGE_SUFFIXES["AzureCloud"]) + config["mgmt_blob_url"] = f"https://{config['tfstate_storage_account_name']}.blob.{suffix}" try: config["number_processes_int"] = int(config["number_processes"]) @@ -87,5 +95,10 @@ def envvar_to_key(name: str) -> str: for env_var_name in os.environ if env_var_name.startswith("RP_BUNDLE") } + # Test setting, off by default. While Porter runs, the RP does not renew the session lock, + # so that Service Bus gives the message to another worker. Never turn it on in production. + config["test_skip_session_lock_renewal"] = ( + str(config["bundle_params"].get("test_skip_session_lock_renewal", "")).lower() == "true" + ) return config diff --git a/resource_processor/tests_rp/test_config.py b/resource_processor/tests_rp/test_config.py new file mode 100644 index 0000000000..667f658043 --- /dev/null +++ b/resource_processor/tests_rp/test_config.py @@ -0,0 +1,50 @@ +import os + +import pytest + +from shared.config import get_config + +REQUIRED = { + "REGISTRY_SERVER": "acr", + "TERRAFORM_STATE_CONTAINER_NAME": "tfstate", + "MGMT_RESOURCE_GROUP_NAME": "rg-mgmt", + "MGMT_STORAGE_ACCOUNT_NAME": "stmgmt", + "SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE": "status", + "SERVICE_BUS_RESOURCE_REQUEST_QUEUE": "requests", + "SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE": "sb", + "AZURE_SUBSCRIPTION_ID": "sub", + "ARM_CLIENT_ID": "client", + "AZURE_TENANT_ID": "tenant", + "ARM_USE_MSI": "true", +} + + +@pytest.fixture +def environment(monkeypatch): + for name in list(os.environ): + if name.startswith("RP_BUNDLE") or name in ("AZURE_ENVIRONMENT", "DEVCONTAINER"): + monkeypatch.delenv(name) + for name, value in REQUIRED.items(): + monkeypatch.setenv(name, value) + return monkeypatch + + +@pytest.mark.parametrize( + "cloud, url", + [ + (None, "https://stmgmt.blob.core.windows.net"), + ("AzureUSGovernment", "https://stmgmt.blob.core.usgovcloudapi.net"), + ("AzureChinaCloud", "https://stmgmt.blob.core.chinacloudapi.cn"), + ], +) +def test_the_execution_leases_use_the_terraform_state_account(environment, cloud, url): + if cloud: + environment.setenv("AZURE_ENVIRONMENT", cloud) + assert get_config()["mgmt_blob_url"] == url + + +@pytest.mark.parametrize("value, expected", [(None, False), ("false", False), ("true", True), ("True", True)]) +def test_the_session_lock_test_setting_is_off_by_default(environment, value, expected): + if value is not None: + environment.setenv("RP_BUNDLE_test_skip_session_lock_renewal", value) + assert get_config()["test_skip_session_lock_renewal"] is expected diff --git a/resource_processor/tests_rp/test_execution_lease.py b/resource_processor/tests_rp/test_execution_lease.py new file mode 100644 index 0000000000..ea2b4a6ddf --- /dev/null +++ b/resource_processor/tests_rp/test_execution_lease.py @@ -0,0 +1,99 @@ +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError + +from helpers import execution_lease as leases + + +def http_error(error_type, status_code): + error = error_type(message=str(status_code)) + error.status_code = status_code + return error + + +def container_with(blob): + container = MagicMock() + container.get_blob_client.return_value = blob + container.create_container = AsyncMock() + return container + + +@pytest.mark.asyncio +async def test_a_lease_that_another_worker_holds_is_not_acquired(): + blob = MagicMock(upload_blob=AsyncMock(side_effect=http_error(ResourceExistsError, 409))) + lease = MagicMock(acquire=AsyncMock(side_effect=http_error(HttpResponseError, 409))) + with patch.object(leases, "BlobLeaseClient", return_value=lease): + assert await leases.try_acquire(container_with(blob), "resource-1") is None + lease.acquire.assert_awaited_once_with(lease_duration=leases.LEASE_SECONDS) + + +@pytest.mark.asyncio +async def test_the_first_lease_creates_the_container_and_the_record(): + blob = MagicMock(upload_blob=AsyncMock(side_effect=[http_error(ResourceNotFoundError, 404), None])) + container = container_with(blob) + lease = MagicMock(acquire=AsyncMock()) + with patch.object(leases, "BlobLeaseClient", return_value=lease): + execution = await leases.try_acquire(container, "resource-1") + container.get_blob_client.assert_called_once_with("resource-1") + container.create_container.assert_awaited_once() + assert blob.upload_blob.await_count == 2 + assert execution.lease is lease + + +@pytest.mark.asyncio +async def test_another_storage_error_is_raised(): + blob = MagicMock(upload_blob=AsyncMock(side_effect=http_error(HttpResponseError, 403))) + with pytest.raises(HttpResponseError): + await leases.try_acquire(container_with(blob), "resource-1") + + +@pytest.mark.asyncio +async def test_the_worker_waits_for_the_lease_and_releases_it(): + lease = MagicMock(release=AsyncMock()) + execution = leases.ExecutionLease(MagicMock(), lease) + with ( + patch.object(leases, "try_acquire", AsyncMock(side_effect=[None, None, execution])) as try_acquire, + patch.object(leases.asyncio, "sleep", AsyncMock()) as sleep, + patch.object(leases, "keep_renewed", AsyncMock()), + ): + async with leases.execution_lease(MagicMock(), "resource-1", lambda: "step") as acquired: + assert acquired is execution + lease.release.assert_not_awaited() + assert try_acquire.await_count == 3 + assert sleep.await_count == 2 + sleep.assert_awaited_with(leases.WAIT_SECONDS) + lease.release.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_the_lease_is_released_when_the_step_fails(): + lease = MagicMock(release=AsyncMock()) + execution = leases.ExecutionLease(MagicMock(), lease) + with ( + patch.object(leases, "try_acquire", AsyncMock(return_value=execution)), + patch.object(leases, "keep_renewed", AsyncMock()), + pytest.raises(RuntimeError), + ): + async with leases.execution_lease(MagicMock(), "resource-1", lambda: "step"): + raise RuntimeError("Porter failed") + lease.release.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "content, expected", [(b"", {}), (b"not json", {}), (b"[]", {}), (b'{"stepId": "s"}', {"stepId": "s"})] +) +async def test_a_record_that_is_not_an_object_reads_as_empty(content, expected): + download = MagicMock(readall=AsyncMock(return_value=content)) + blob = MagicMock(download_blob=AsyncMock(return_value=download), blob_name="resource-1") + assert await leases.ExecutionLease(blob, MagicMock()).read() == expected + + +@pytest.mark.asyncio +async def test_a_record_is_written_with_the_lease(): + blob = MagicMock(upload_blob=AsyncMock()) + lease = MagicMock() + await leases.ExecutionLease(blob, lease).write({"state": "running"}) + blob.upload_blob.assert_awaited_once_with(json.dumps({"state": "running"}), overwrite=True, lease=lease) diff --git a/resource_processor/tests_rp/test_lock_renewal.py b/resource_processor/tests_rp/test_lock_renewal.py new file mode 100644 index 0000000000..01b2fd5ab6 --- /dev/null +++ b/resource_processor/tests_rp/test_lock_renewal.py @@ -0,0 +1,108 @@ +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest + +from helpers import lock_renewal + + +async def run_until(task_condition, coroutine): + task = asyncio.create_task(coroutine) + for _ in range(200): + await asyncio.sleep(0) + if task_condition() or task.done(): + break + if not task.done(): + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return task + + +@pytest.mark.asyncio +async def test_renews_at_each_interval(): + renew = AsyncMock() + task = await run_until( + lambda: renew.await_count >= 3, + lock_renewal.keep_renewed("Session lock", renew, lambda: 300, lambda: "step", interval=0), + ) + assert renew.await_count >= 3 + assert task.cancelled() + + +@pytest.mark.asyncio +async def test_a_failed_renewal_is_tried_again_and_logged(): + renew = AsyncMock(side_effect=[RuntimeError("busy"), None, None]) + with patch.object(lock_renewal, "logger") as logger: + await run_until( + lambda: renew.await_count >= 3, + lock_renewal.keep_renewed( + "Session lock", renew, lambda: 300, lambda: "step 1", interval=0, retry_seconds=0 + ), + ) + logger.warning.assert_called_once() + assert "Session lock renewal 1 failed for step 1" in logger.warning.call_args.args[0] + logger.error.assert_not_called() + + +@pytest.mark.asyncio +async def test_an_expired_lock_is_logged_as_an_error_and_ends_the_task(): + renew = AsyncMock(side_effect=RuntimeError("lock lost")) + with patch.object(lock_renewal, "logger") as logger: + await lock_renewal.keep_renewed("Execution lease", renew, lambda: 0, lambda: "step 2", interval=0) + renew.assert_awaited_once() + assert "Execution lease expired after 1 failed renewals for step 2" in logger.error.call_args.args[0] + + +@pytest.mark.asyncio +async def test_a_paused_renewal_does_not_renew(): + renew = AsyncMock() + checks = [] + + def paused(): + checks.append(True) + return True + + await run_until( + lambda: len(checks) >= 3, + lock_renewal.keep_renewed("Session lock", renew, lambda: 300, lambda: "step", interval=0, paused=paused), + ) + renew.assert_not_awaited() + + +def test_seconds_until_without_a_time_is_zero(): + assert lock_renewal.seconds_until(None) == 0 + + +@pytest.mark.asyncio +async def test_stop_ends_the_renewal_and_returns(): + renewal = asyncio.create_task(asyncio.Event().wait()) + await asyncio.sleep(0) + await lock_renewal.stop(renewal) + assert renewal.cancelled() + + +@pytest.mark.asyncio +async def test_stop_keeps_a_cancellation_of_the_caller(): + in_stop = asyncio.Event() + + async def slow_renewal(): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + in_stop.set() + await asyncio.sleep(0.2) + raise + + async def caller(): + renewal = asyncio.create_task(slow_renewal()) + await asyncio.sleep(0) + await lock_renewal.stop(renewal) + return "finished" + + task = asyncio.create_task(caller()) + await in_stop.wait() + # The caller is cancelled while it waits for the renewal task to end. + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task diff --git a/resource_processor/tests_rp/test_run_step.py b/resource_processor/tests_rp/test_run_step.py new file mode 100644 index 0000000000..9c031d2a93 --- /dev/null +++ b/resource_processor/tests_rp/test_run_step.py @@ -0,0 +1,214 @@ +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from helpers import statuses +from vmss_porter import runner + +MESSAGE = { + "id": "resource-1", + "action": "install", + "stepId": "step-1", + "operationId": "operation-1", + "name": "bundle", + "version": "1.0.0", + "parameters": {}, +} +CONFIG = {"deployment_status_queue": "status"} + + +def service_bus(): + sender = MagicMock(send_messages=AsyncMock()) + return MagicMock(get_queue_sender=MagicMock(return_value=sender)), sender + + +def body_of(message): + return json.loads(str(message)) + + +def bodies(sender): + return [body_of(call.args[0]) for call in sender.send_messages.await_args_list] + + +@pytest.mark.asyncio +async def test_a_finished_step_sends_its_recorded_status_without_porter(): + record = { + "operationId": "operation-1", + "stepId": "step-1", + "executionId": "first-run", + "state": "finished", + "status": statuses.pass_status_string_for["install"], + "message": "install action completed successfully.", + "outputs": True, + } + execution = MagicMock(read=AsyncMock(return_value=record), write=AsyncMock()) + client, sender = service_bus() + outputs = [{"name": "url", "value": "https://example"}] + with ( + patch.object(runner, "invoke_porter_action", AsyncMock()) as invoke, + patch.object(runner, "get_porter_outputs", AsyncMock(return_value=(True, outputs))), + ): + assert await runner.run_step(MESSAGE, execution, client, CONFIG) is True + invoke.assert_not_awaited() + execution.write.assert_not_awaited() + [body] = bodies(sender) + assert body["executionId"] == "first-run" + assert body["status"] == statuses.pass_status_string_for["install"] + assert body["outputs"] == outputs + + +@pytest.mark.asyncio +async def test_a_recorded_failure_is_sent_again_without_outputs(): + record = { + "operationId": "operation-1", + "stepId": "step-1", + "executionId": "first-run", + "state": "finished", + "status": statuses.failed_status_string_for["install"], + "message": "Error message: failed", + "outputs": False, + } + execution = MagicMock(read=AsyncMock(return_value=record), write=AsyncMock()) + client, sender = service_bus() + with patch.object(runner, "get_porter_outputs", AsyncMock()) as outputs: + assert await runner.run_step(MESSAGE, execution, client, CONFIG) is False + outputs.assert_not_awaited() + [body] = bodies(sender) + assert body["status"] == statuses.failed_status_string_for["install"] + assert "outputs" not in body + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "record", + [ + {}, + {"operationId": "operation-1", "stepId": "step-1", "state": "running", "executionId": "crashed-run"}, + {"operationId": "operation-1", "stepId": "step-0", "state": "finished", "executionId": "earlier-step"}, + {"operationId": "operation-0", "stepId": "step-1", "state": "finished", "executionId": "other-operation"}, + ], +) +async def test_porter_runs_when_the_record_has_no_result_for_this_step(record): + execution = MagicMock(read=AsyncMock(return_value=record), write=AsyncMock()) + with patch.object(runner, "invoke_porter_action", AsyncMock(return_value=True)) as invoke: + assert await runner.run_step(MESSAGE, execution, MagicMock(), CONFIG) is True + execution_id = invoke.await_args.kwargs["execution_id"] + assert execution_id not in (record.get("executionId"), None) + running = execution.write.await_args.args[0] + assert running == { + "operationId": "operation-1", + "stepId": "step-1", + "action": "install", + "executionId": execution_id, + "state": "running", + } + await invoke.await_args.kwargs["finish"]("deployed", "done", True) + assert execution.write.await_args.args[0] == { + **running, + "state": "finished", + "status": "deployed", + "message": "done", + "outputs": True, + } + + +@pytest.mark.asyncio +async def test_each_status_has_the_execution_id_and_the_result_is_saved_before_the_final_status(): + client, sender = service_bus() + events = [] + sender.send_messages.side_effect = lambda message: events.append(("send", body_of(message)["status"])) + + async def finish(status, message, with_outputs): + events.append(("finish", status, with_outputs)) + + with ( + patch.object( + runner, "build_porter_command", AsyncMock(return_value=([["porter", "install"]], None, "p", None)) + ), + patch.object(runner, "run_porter", AsyncMock(return_value=(0, "", ""))), + patch.object(runner, "get_porter_outputs", AsyncMock(return_value=(True, []))), + ): + assert await runner.invoke_porter_action(MESSAGE, client, CONFIG, execution_id="run-2", finish=finish) + assert events == [ + ("send", statuses.in_progress_status_string_for["install"]), + ("finish", statuses.pass_status_string_for["install"], True), + ("send", statuses.pass_status_string_for["install"]), + ] + assert {body["executionId"] for body in bodies(sender)} == {"run-2"} + + +@pytest.mark.asyncio +async def test_a_status_without_an_execution_has_no_execution_id(): + client, sender = service_bus() + with ( + patch.object( + runner, "build_porter_command", AsyncMock(return_value=([["porter", "install"]], None, "p", None)) + ), + patch.object(runner, "run_porter", AsyncMock(return_value=(1, "", "failed"))), + ): + assert await runner.invoke_porter_action(MESSAGE, client, CONFIG) is False + assert all("executionId" not in body for body in bodies(sender)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action, required", [("upgrade", "resource-1"), ("install", None), ("uninstall", None)]) +async def test_only_an_upgrade_requires_the_installation(action, required): + client, _ = service_bus() + with ( + patch.object(runner, "build_porter_command", AsyncMock(return_value=([["porter", action]], None, "p", None))), + patch.object(runner, "run_porter", AsyncMock(return_value=(1, "", "failed"))) as run_porter, + patch.object(runner, "_confirm_porter_installation_absent", AsyncMock(return_value=False)), + ): + await runner.invoke_porter_action({**MESSAGE, "action": action}, client, CONFIG) + assert run_porter.await_args.kwargs["required_installation"] == required + + +@pytest.mark.asyncio +@pytest.mark.parametrize("absent, porter_runs", [(True, False), (False, True)]) +async def test_an_upgrade_does_not_install_a_missing_resource(absent, porter_runs): + commands = [] + + async def helper(command, config, *args, **kwargs): + commands.append(command) + return 0, "", "" + + with ( + patch.object(runner, "run_command_helper", side_effect=helper), + patch.object(runner, "azure_login_command", return_value=[]), + patch.object(runner, "azure_acr_login_command", return_value=[]), + patch.object(runner, "apply_porter_credentials_sets_command", return_value=[]), + patch.object(runner, "_confirm_porter_installation_absent", AsyncMock(return_value=absent)), + ): + code, _, error = await runner.run_porter( + [["porter", "installations", "apply"]], {}, required_installation="resource-1" + ) + assert (["porter", "installations", "apply"] in commands) is porter_runs + if absent: + assert code == 1 + assert "An upgrade does not install a missing resource" in error + else: + assert code == 0 + + +@pytest.mark.asyncio +async def test_a_failed_record_write_still_sends_the_final_status(): + client, sender = service_bus() + + async def finish(status, message, with_outputs): + raise RuntimeError("lease expired") + + with ( + patch.object( + runner, "build_porter_command", AsyncMock(return_value=([["porter", "install"]], None, "p", None)) + ), + patch.object(runner, "run_porter", AsyncMock(return_value=(0, "", ""))), + patch.object(runner, "get_porter_outputs", AsyncMock(return_value=(True, []))), + patch.object(runner, "logger") as logger, + ): + assert await runner.invoke_porter_action(MESSAGE, client, CONFIG, execution_id="run-3", finish=finish) + assert [body["status"] for body in bodies(sender)] == [ + statuses.in_progress_status_string_for["install"], + statuses.pass_status_string_for["install"], + ] + assert "Could not save the execution record" in logger.error.call_args.args[0] diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index bb81ebf9f1..2861e7d0b1 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -1,4 +1,5 @@ from azure.servicebus import ServiceBusSessionFilter +from azure.servicebus.exceptions import SessionLockLostError from azure.servicebus.aio import ServiceBusClient from vmss_porter.runner import ( set_up_config, @@ -13,7 +14,8 @@ ) import json import asyncio -from unittest.mock import patch, AsyncMock, Mock +from contextlib import asynccontextmanager +from unittest.mock import ANY, patch, AsyncMock, Mock import pytest import sys from pathlib import Path @@ -29,6 +31,15 @@ def mock_service_bus_client(): yield mock +@pytest.fixture(autouse=True) +def mock_container_client(): + with patch("vmss_porter.runner.ContainerClient") as mock: + instance = mock.return_value + instance.__aenter__ = AsyncMock(return_value=instance) + instance.__aexit__ = AsyncMock(return_value=False) + yield mock + + @pytest.fixture def mock_default_credential(): with patch("vmss_porter.runner.default_credentials") as mock: @@ -36,8 +47,25 @@ def mock_default_credential(): @pytest.fixture -def mock_auto_lock_renewer(): - with patch("vmss_porter.runner.AutoLockRenewer") as mock: +def mock_execution_lease(): + execution = AsyncMock() + execution.read.return_value = {} + + @asynccontextmanager + async def lease(container, resource_id, describe): + yield execution + + with patch("vmss_porter.runner.execution_lease", side_effect=lease) as mock: + mock.execution = execution + yield mock + + +@pytest.fixture +def mock_keep_renewed(): + async def renew_until_cancelled(*args, **kwargs): + await asyncio.Event().wait() + + with patch("vmss_porter.runner.keep_renewed", side_effect=renew_until_cancelled) as mock: yield mock @@ -90,13 +118,14 @@ async def test_runner(mock_receive_message, mock_service_bus_client, mock_defaul "vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace", "porter_env": {"HOME": "/root"}, + "mgmt_blob_url": "https://mgmt", } await runner(0, config) mock_default_credential.assert_called_once_with("test_msi_id") mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config), ANY) # Verify context manager entered and exited cleanly mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -117,13 +146,14 @@ async def test_runner_no_msi_id(mock_receive_message, mock_service_bus_client, m "vmss_msi_id": None, "service_bus_namespace": "test_namespace", "porter_env": {"HOME": "/root"}, + "mgmt_blob_url": "https://mgmt", } await runner(0, config) mock_default_credential.assert_called_once_with(None) mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config), ANY) # Verify context manager entered and exited cleanly mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -145,6 +175,7 @@ async def test_runner_exception(mock_receive_message, mock_service_bus_client, m "vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace", "porter_env": {"HOME": "/root"}, + "mgmt_blob_url": "https://mgmt", } with pytest.raises(Exception, match="Test Exception"): @@ -152,7 +183,7 @@ async def test_runner_exception(mock_receive_message, mock_service_bus_client, m mock_default_credential.assert_called_once_with("test_msi_id") mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) - mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config)) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, worker_config(0, config), ANY) # Verify context manager entered and exited cleanly, even on exception mock_default_credential.return_value.__aenter__.assert_awaited_once() @@ -187,6 +218,7 @@ async def test_default_credentials_closes_real_credential_on_exception(mock_serv "vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace", "porter_env": {"HOME": "/root"}, + "mgmt_blob_url": "https://mgmt", } with pytest.raises(Exception, match="Test Exception"): @@ -196,15 +228,12 @@ async def test_default_credentials_closes_real_credential_on_exception(mock_serv @pytest.mark.asyncio -@patch("vmss_porter.runner.invoke_porter_action", return_value=True) -async def test_receive_message(mock_invoke_porter_action, mock_service_bus_client, mock_auto_lock_renewer, mock_logger): +@patch("vmss_porter.runner.run_step", return_value=True) +async def test_receive_message( + mock_run_step, mock_service_bus_client, mock_execution_lease, mock_keep_renewed, mock_logger +): mock_service_bus_client_instance = mock_service_bus_client.return_value - # Set up the lock renewer mock correctly - mock_renewer = AsyncMock() - mock_renewer.register = Mock() # Make this a sync method to avoid await warning - mock_auto_lock_renewer.return_value.__aenter__.return_value = mock_renewer - mock_receiver = AsyncMock() mock_receiver.__aenter__.return_value = mock_receiver mock_receiver.__aexit__.return_value = None @@ -228,7 +257,7 @@ async def test_receive_message(mock_invoke_porter_action, mock_service_bus_clien config = {"resource_request_queue": "test_queue"} - await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + await receive_message(mock_service_bus_client_instance, config, AsyncMock(), keep_running=run_once) mock_receiver.complete_message.assert_awaited_once() assert "synthetic-password-not-for-logs" not in str(mock_logger.mock_calls) mock_service_bus_client_instance.get_queue_receiver.assert_called_once_with( @@ -237,14 +266,9 @@ async def test_receive_message(mock_invoke_porter_action, mock_service_bus_clien @pytest.mark.asyncio -async def test_receive_message_bad_json(mock_service_bus_client, mock_auto_lock_renewer): +async def test_receive_message_bad_json(mock_service_bus_client, mock_execution_lease, mock_keep_renewed): mock_service_bus_client_instance = mock_service_bus_client.return_value - # Set up the lock renewer mock correctly - mock_renewer = AsyncMock() - mock_renewer.register = Mock() - mock_auto_lock_renewer.return_value.__aenter__.return_value = mock_renewer - mock_receiver = AsyncMock() mock_receiver.__aenter__.return_value = mock_receiver mock_receiver.__aexit__.return_value = None @@ -257,7 +281,7 @@ async def test_receive_message_bad_json(mock_service_bus_client, mock_auto_lock_ config = {"resource_request_queue": "test_queue"} - await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + await receive_message(mock_service_bus_client_instance, config, AsyncMock(), keep_running=run_once) mock_receiver.dead_letter_message.assert_awaited_once() dead_letter_args, dead_letter_kwargs = mock_receiver.dead_letter_message.await_args assert dead_letter_args == ("invalid_json_string",) @@ -275,14 +299,10 @@ async def test_receive_message_bad_json(mock_service_bus_client, mock_auto_lock_ ], ) async def test_receive_message_dead_letter_failure_is_logged( - message, expected_log, mock_service_bus_client, mock_auto_lock_renewer, mock_logger + message, expected_log, mock_service_bus_client, mock_execution_lease, mock_keep_renewed, mock_logger ): mock_service_bus_client_instance = mock_service_bus_client.return_value - mock_renewer = AsyncMock() - mock_renewer.register = Mock() - mock_auto_lock_renewer.return_value.__aenter__.return_value = mock_renewer - mock_receiver = AsyncMock() mock_receiver.__aenter__.return_value = mock_receiver mock_receiver.__aexit__.return_value = None @@ -296,7 +316,7 @@ async def test_receive_message_dead_letter_failure_is_logged( config = {"resource_request_queue": "test_queue"} with patch("vmss_porter.runner.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: - await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + await receive_message(mock_service_bus_client_instance, config, AsyncMock(), keep_running=run_once) mock_sleep.assert_awaited_once_with(10) @@ -333,14 +353,10 @@ async def test_receive_message_dead_letter_failure_is_logged( ], ) async def test_receive_message_invalid_json_structure( - message, error_description, mock_service_bus_client, mock_auto_lock_renewer + message, error_description, mock_service_bus_client, mock_execution_lease, mock_keep_renewed ): mock_service_bus_client_instance = mock_service_bus_client.return_value - mock_renewer = AsyncMock() - mock_renewer.register = Mock() - mock_auto_lock_renewer.return_value.__aenter__.return_value = mock_renewer - mock_receiver = AsyncMock() mock_receiver.__aenter__.return_value = mock_receiver mock_receiver.__aexit__.return_value = None @@ -352,7 +368,7 @@ async def test_receive_message_invalid_json_structure( run_once = Mock(side_effect=[True, False]) config = {"resource_request_queue": "test_queue"} - await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + await receive_message(mock_service_bus_client_instance, config, AsyncMock(), keep_running=run_once) mock_receiver.dead_letter_message.assert_awaited_once_with( message, @@ -363,15 +379,12 @@ async def test_receive_message_invalid_json_structure( @pytest.mark.asyncio -async def test_receive_message_unknown_exception(mock_auto_lock_renewer, mock_service_bus_client, mock_logger): +async def test_receive_message_unknown_exception( + mock_execution_lease, mock_keep_renewed, mock_service_bus_client, mock_logger +): """Test receiving a message with an unknown exception.""" mock_service_bus_client_instance = mock_service_bus_client.return_value - # Set up the lock renewer mock correctly - mock_renewer = AsyncMock() - mock_renewer.register = Mock() # Make this a sync method to avoid await warning - mock_auto_lock_renewer.return_value.__aenter__.return_value = mock_renewer - mock_receiver = AsyncMock() mock_receiver.__aenter__.return_value = mock_receiver mock_receiver.__aexit__.return_value = None @@ -399,7 +412,7 @@ async def test_receive_message_unknown_exception(mock_auto_lock_renewer, mock_se patch("vmss_porter.runner.asyncio.sleep", new_callable=AsyncMock) as mock_sleep, patch("vmss_porter.runner.receive_message", side_effect=Exception("Test Exception")), ): - await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + await receive_message(mock_service_bus_client_instance, config, AsyncMock(), keep_running=run_once) mock_logger.exception.assert_any_call("Unknown exception. Will retry...") mock_sleep.assert_awaited_once_with(10) @@ -1162,3 +1175,86 @@ async def test_check_runners(_): await check_runners(processes, mock_httpserver, keep_running=run_once) assert mock_httpserver.kill.call_count == 1 + + +def session_receiver(mock_service_bus_client, body): + mock_service_bus_client_instance = mock_service_bus_client.return_value + mock_receiver = AsyncMock() + mock_receiver.__aenter__.return_value = mock_receiver + mock_receiver.__aexit__.return_value = None + mock_receiver.session.session_id = "test_id" + mock_receiver.__aiter__.return_value = [json.dumps(body)] + mock_service_bus_client_instance.get_queue_receiver.return_value.__aenter__.return_value = mock_receiver + return mock_service_bus_client_instance, mock_receiver + + +VALID_REQUEST = { + "id": "test_id", + "action": "install", + "stepId": "test_step_id", + "operationId": "test_operation_id", + "name": "test_bundle", + "version": "1.0.0", + "parameters": {}, +} + + +@pytest.mark.asyncio +@patch("vmss_porter.runner.run_step", return_value=True) +async def test_each_step_runs_with_the_lease_for_its_resource( + mock_run_step, mock_service_bus_client, mock_execution_lease, mock_keep_renewed +): + client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) + executions = AsyncMock() + + await receive_message( + client, {"resource_request_queue": "q"}, executions, keep_running=Mock(side_effect=[True, False]) + ) + + container, resource_id, describe = mock_execution_lease.call_args.args + assert (container, resource_id) == (executions, "test_id") + assert describe() == "resource_id=test_id, operation_id=test_operation_id, step_id=test_step_id" + assert mock_run_step.await_args.args[1] is mock_execution_lease.execution + receiver.complete_message.assert_awaited_once() + assert mock_keep_renewed.call_args.args[0] == "Session lock" + + +@pytest.mark.asyncio +@patch("vmss_porter.runner.run_step", return_value=True) +async def test_a_lost_session_lock_at_completion_is_a_warning( + mock_run_step, mock_service_bus_client, mock_execution_lease, mock_keep_renewed, mock_logger +): + client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) + receiver.complete_message.side_effect = SessionLockLostError() + + with patch("vmss_porter.runner.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await receive_message( + client, {"resource_request_queue": "q"}, AsyncMock(), keep_running=Mock(side_effect=[True, False]) + ) + + assert "The session lock expired before the message was completed" in mock_logger.warning.call_args.args[0] + mock_logger.exception.assert_not_called() + mock_sleep.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "test_setting, running, paused", [(False, True, False), (True, False, False), (True, True, True)] +) +async def test_the_test_setting_pauses_the_session_lock_only_while_porter_runs( + mock_service_bus_client, mock_execution_lease, mock_keep_renewed, test_setting, running, paused +): + client, _ = session_receiver(mock_service_bus_client, VALID_REQUEST) + seen = [] + + async def run_step(*args): + seen.append(mock_keep_renewed.call_args.kwargs["paused"]()) + return True + + config = {"resource_request_queue": "q", "test_skip_session_lock_renewal": test_setting} + with patch("vmss_porter.runner.run_step", side_effect=run_step): + await receive_message(client, config, AsyncMock(), keep_running=Mock(side_effect=[True, False])) + + if running: + assert seen == [paused] + assert mock_keep_renewed.call_args.kwargs["paused"]() is False diff --git a/resource_processor/vmss_porter/requirements.txt b/resource_processor/vmss_porter/requirements.txt index 7176c609a6..f305aa9619 100644 --- a/resource_processor/vmss_porter/requirements.txt +++ b/resource_processor/vmss_porter/requirements.txt @@ -3,4 +3,5 @@ azure-cli-core==2.68.0 azure-identity==1.25.1 azure-monitor-opentelemetry==1.8.9 azure-servicebus==7.14.3 +azure-storage-blob==12.27.1 opentelemetry-instrumentation-logging==0.64b0 diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index a7a0ae55ce..cb1833f585 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -1,10 +1,12 @@ import time -from typing import Optional +from typing import Awaitable, Callable, Optional from multiprocessing import Process import json import asyncio import os import sys + +from uuid import uuid4 from helpers.commands import ( azure_acr_login_command, azure_login_command, @@ -17,15 +19,18 @@ ) from shared.config import get_config from helpers.httpserver import start_server +from helpers.execution_lease import CONTAINER_NAME, execution_lease +from helpers.lock_renewal import keep_renewed, seconds_until, stop from shared.logging import initialize_logging, logger, tracer from shared.config import VERSION from helpers import statuses from contextlib import asynccontextmanager from azure.servicebus import ServiceBusMessage, NEXT_AVAILABLE_SESSION -from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError -from azure.servicebus.aio import ServiceBusClient, AutoLockRenewer +from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError, SessionLockLostError +from azure.servicebus.aio import ServiceBusClient from azure.identity.aio import DefaultAzureCredential +from azure.storage.blob.aio import ContainerClient RESOURCE_REQUEST_FIELD_TYPES = { @@ -38,6 +43,8 @@ "parameters": dict, } OPTIONAL_RESOURCE_REQUEST_FIELD_TYPES = {"user": dict} +# The workspace queue keeps a session lock for 5 minutes. +SESSION_RENEW_SECONDS = 60 def validate_resource_request(message: object) -> None: @@ -86,7 +93,7 @@ async def default_credentials(msi_id): await credential.close() -async def receive_message(service_bus_client, config: dict, keep_running=lambda: True): +async def receive_message(service_bus_client, config: dict, executions: ContainerClient, keep_running=lambda: True): """ This method is run per process. Each process will connect to service bus and try to establish a session. If messages are there, the process will continue to receive all the messages associated with that session. @@ -111,11 +118,20 @@ async def receive_message(service_bus_client, config: dict, keep_running=lambda: async with service_bus_client.get_queue_receiver( queue_name=q_name, max_wait_time=1, session_id=NEXT_AVAILABLE_SESSION ) as receiver: - logger.info(f"Got a session containing messages: {receiver.session.session_id}") - async with AutoLockRenewer() as renewer: - # allow a session to be auto lock renewed for up to an hour - if it's processing a message - renewer.register(receiver, receiver.session, max_lock_renewal_duration=3600) - + session = receiver.session + logger.info(f"Got a session containing messages: {session.session_id}") + step = {"text": f"session {session.session_id}", "running": False} + renewal = asyncio.create_task( + keep_renewed( + "Session lock", + session.renew_lock, + lambda: seconds_until(session.locked_until_utc), + lambda: step["text"], + interval=SESSION_RENEW_SECONDS, + paused=lambda: config.get("test_skip_session_lock_renewal", False) and step["running"], + ) + ) + try: async for msg in receiver: result = True message = "" @@ -149,23 +165,41 @@ async def receive_message(service_bus_client, config: dict, keep_running=lambda: current_span.set_attribute("action", message["action"]) current_span.set_attribute("step_id", message["stepId"]) current_span.set_attribute("operation_id", message["operationId"]) - logger.info( - f"Message received for resource_id={message['id']}, operation_id={message['operationId']}, step_id={message['stepId']}" - ) - - result = await invoke_porter_action(message, service_bus_client, config) - - if result: - logger.info(f"Resource request for {message['id']} is complete") - else: - logger.error("Message processing failed!") - - logger.info( - f"Message for resource_id={message['id']}, operation_id={message['operationId']} processed as {result} and marked complete." + step["text"] = ( + f"resource_id={message['id']}, operation_id={message['operationId']}, step_id={message['stepId']}" ) - await receiver.complete_message(msg) + logger.info(f"Message received for {step['text']}") + + # Service Bus can give the same message to a second worker. The lease lets + # one worker at a time run Porter for the resource. + async with execution_lease(executions, message["id"], lambda: step["text"]) as execution: + step["running"] = True + try: + result = await run_step(message, execution, service_bus_client, config) + finally: + step["running"] = False + + if result: + logger.info(f"Resource request for {message['id']} is complete") + else: + logger.error("Message processing failed!") + + try: + await receiver.complete_message(msg) + except SessionLockLostError: + # Another worker has the session now. It reads the execution record + # and sends the recorded status again. + logger.warning( + f"The session lock expired before the message was completed for {step['text']}" + ) + break + logger.info( + f"Message for resource_id={message['id']}, operation_id={message['operationId']} processed as {result} and marked complete." + ) - logger.info(f"Closing session: {receiver.session.session_id}") + logger.info(f"Closing session: {session.session_id}") + finally: + await stop(renewal) except OperationTimeoutError: # Timeout occurred whilst connecting to a session - this is expected and indicates no non-empty sessions are available @@ -242,7 +276,13 @@ async def check_legacy_parameter_overrides(installation_id: str, config: dict) - return None -async def run_porter(command_parts_list: list, config: dict, *, parameter_check_installation: Optional[str] = None): +async def run_porter( + command_parts_list: list, + config: dict, + *, + parameter_check_installation: Optional[str] = None, + required_installation: Optional[str] = None, +): """ Run a Porter command """ @@ -269,6 +309,15 @@ async def run_porter(command_parts_list: list, config: dict, *, parameter_check_ if error: return (1, None, error) + # `porter installations apply` installs a missing installation. An upgrade must not create the resource again. + if required_installation is not None and await _confirm_porter_installation_absent(required_installation, config): + return ( + 1, + None, + f"Cannot upgrade resource '{required_installation}': Porter has no installation for it. " + "An upgrade does not install a missing resource.", + ) + last_returncode = None last_stdout = None last_stderr = None @@ -281,7 +330,9 @@ async def run_porter(command_parts_list: list, config: dict, *, parameter_check_ return (last_returncode, last_stdout, last_stderr) -def service_bus_message_generator(sb_message: dict, status: str, deployment_message: str, outputs=None): +def service_bus_message_generator( + sb_message: dict, status: str, deployment_message: str, outputs=None, execution_id: Optional[str] = None +): """ Generate a resource request message """ @@ -296,6 +347,9 @@ def service_bus_message_generator(sb_message: dict, status: str, deployment_mess if outputs is not None: message_dict["outputs"] = outputs + if execution_id is not None: + # The API uses this ID to ignore a status from another run of the same step. + message_dict["executionId"] = execution_id resource_request_message = json.dumps(message_dict) logger.debug(f"Deployment Status Message: {resource_request_message}") @@ -353,7 +407,67 @@ async def _confirm_porter_installation_absent(installation_id: str, config: dict return False -async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, config: dict) -> bool: +async def run_step(msg_body: dict, execution, sb_client: ServiceBusClient, config: dict) -> bool: + """ + Run the step once. If the execution record shows that a worker already finished this step, send its status again. + """ + record = await execution.read() + same_step = record.get("operationId") == msg_body["operationId"] and record.get("stepId") == msg_body["stepId"] + if same_step and record.get("state") == "finished": + logger.info(f"Step {msg_body['stepId']} for {msg_body['id']} is already finished. Sending its status again.") + return await send_recorded_status(msg_body, record, sb_client, config) + if same_step: + # Porter did not finish. Run it again, as a Service Bus redelivery does. + logger.warning(f"An earlier worker stopped during step {msg_body['stepId']} for {msg_body['id']}.") + + execution_id = str(uuid4()) + record = { + "operationId": msg_body["operationId"], + "stepId": msg_body["stepId"], + "action": msg_body["action"], + "executionId": execution_id, + } + await execution.write({**record, "state": "running"}) + + async def finish(status: str, message: str, with_outputs: bool): + await execution.write( + {**record, "state": "finished", "status": status, "message": message, "outputs": with_outputs} + ) + + return await invoke_porter_action(msg_body, sb_client, config, execution_id=execution_id, finish=finish) + + +async def send_recorded_status(msg_body: dict, record: dict, sb_client: ServiceBusClient, config: dict) -> bool: + action = msg_body["action"] + status = record.get("status", statuses.failed_status_string_for[action]) + message = record.get("message", "") + outputs = None + if record.get("outputs"): + # The record does not keep outputs, because they can contain secrets. Read them again. + got_outputs, outputs = await get_porter_outputs(msg_body, config) + if not got_outputs: + status = statuses.failed_status_string_for[action] + message = f"{action} action completed successfully, but failed to get outputs." + sb_sender = sb_client.get_queue_sender(queue_name=config["deployment_status_queue"]) + await sb_sender.send_messages( + ServiceBusMessage( + body=service_bus_message_generator(msg_body, status, message, outputs, record.get("executionId")), + correlation_id=msg_body["id"], + session_id=msg_body["operationId"], + ) + ) + logger.info(f"Sent the recorded status message for {msg_body['id']}: {status}") + return status == statuses.pass_status_string_for[action] + + +async def invoke_porter_action( + msg_body: dict, + sb_client: ServiceBusClient, + config: dict, + *, + execution_id: Optional[str] = None, + finish: Optional[Callable[[str, str, bool], Awaitable[None]]] = None, +) -> bool: """ Handle resource message by invoking specified porter action (i.e. install, uninstall) """ @@ -363,15 +477,25 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf logger.info(f"{action} action starting for {installation_id}...") sb_sender = sb_client.get_queue_sender(queue_name=config["deployment_status_queue"]) - # post an update message to set the status to an 'in progress' one - resource_request_message = service_bus_message_generator( - msg_body, statuses.in_progress_status_string_for[action], "Job starting" - ) - await sb_sender.send_messages( - ServiceBusMessage( - body=resource_request_message, correlation_id=msg_body["id"], session_id=msg_body["operationId"] + async def send_status(status: str, deployment_message: str, outputs=None, final: bool = True): + if final and finish is not None: + # Save the result before the API can see it, so that another worker can send it again. + try: + await finish(status, deployment_message, outputs is not None) + except Exception as error: + # The lease can expire during a long Porter run. The API must still get the result. + logger.error(f"Could not save the execution record for {installation_id}: {error!r}") + resource_request_message = service_bus_message_generator( + msg_body, status, deployment_message, outputs, execution_id ) - ) + await sb_sender.send_messages( + ServiceBusMessage( + body=resource_request_message, correlation_id=installation_id, session_id=msg_body["operationId"] + ) + ) + + # post an update message to set the status to an 'in progress' one + await send_status(statuses.in_progress_status_string_for[action], "Job starting", final=False) logger.info( f"Sent status message for {installation_id} - {statuses.in_progress_status_string_for[action]} - Job starting" ) @@ -383,20 +507,18 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf config, msg_body, is_custom_action ) except PorterParameterDiscoveryError as error: - resource_request_message = service_bus_message_generator( - msg_body, statuses.failed_status_string_for[action], str(error) - ) - await sb_sender.send_messages( - ServiceBusMessage( - body=resource_request_message, correlation_id=installation_id, session_id=msg_body["operationId"] - ) - ) + await send_status(statuses.failed_status_string_for[action], str(error)) return False logger.debug("Starting to run porter execution command...") try: check_installation = installation_id if action not in ("install", "upgrade") else None - returncode, _, err = await run_porter(porter_command, config, parameter_check_installation=check_installation) + returncode, _, err = await run_porter( + porter_command, + config, + parameter_check_installation=check_installation, + required_installation=installation_id if action == "upgrade" else None, + ) finally: if param_set_file or installation_file: try: @@ -406,6 +528,7 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf logger.debug("Finished running porter execution command.") action_completed_without_error = returncode == 0 + outputs = None # Handle command output if ( @@ -418,9 +541,7 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf # required before completing deletion without installation outputs. action_completed_without_error = True status_for_sb_message = statuses.pass_status_string_for[action] - resource_request_message = service_bus_message_generator( - msg_body, status_for_sb_message, "Uninstall completed: Porter confirmed that the installation is absent." - ) + status_message = "Uninstall completed: Porter confirmed that the installation is absent." elif returncode != 0: # Construct a readable representation of the command(s) command_representation = "" @@ -429,16 +550,14 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf command_representation = command_representation.rstrip("; ") error_output = err or f"Porter exited with code {returncode} and no error output." - error_message = ( + status_message = ( "Error message: " + " ".join(error_output.split("\n")) + "; Porter command: " + command_representation ) # Installation lookup errors can wrap storage failures. They do not prove that resources were deleted. status_for_sb_message = statuses.failed_status_string_for[action] - resource_request_message = service_bus_message_generator(msg_body, status_for_sb_message, error_message) - # Post message on sb queue to notify receivers of action failure - logger.info(f"{installation_id}: Porter action failed with error = {error_message}") + logger.info(f"{installation_id}: Porter action failed with error = {status_message}") else: # Get the outputs @@ -452,15 +571,7 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf status_for_sb_message = statuses.failed_status_string_for[action] status_message = f"{action} action completed successfully, but failed to get outputs." - resource_request_message = service_bus_message_generator( - msg_body, status_for_sb_message, status_message, outputs - ) - - await sb_sender.send_messages( - ServiceBusMessage( - body=resource_request_message, correlation_id=msg_body["id"], session_id=msg_body["operationId"] - ) - ) + await send_status(status_for_sb_message, status_message, outputs) logger.info(f"Sent status message for {installation_id}: {status_for_sb_message}") # return true as want to continue processing the message @@ -511,8 +622,11 @@ async def runner(process_number: int, config: dict): config = worker_config(process_number, config) with tracer.start_as_current_span(process_number): async with default_credentials(config["vmss_msi_id"]) as credential: - async with ServiceBusClient(config["service_bus_namespace"], credential) as service_bus_client: - await receive_message(service_bus_client, config) + async with ( + ServiceBusClient(config["service_bus_namespace"], credential) as service_bus_client, + ContainerClient(config["mgmt_blob_url"], CONTAINER_NAME, credential) as executions, + ): + await receive_message(service_bus_client, config, executions) async def check_runners(processes: list, httpserver: Process, keep_running=lambda: True): From 5aca6d4ca139844914872ee4c93ea0c5529ee1f8 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 27 Oct 2026 13:13:11 +0000 Subject: [PATCH 05/37] Send the first step message again after an API stop (#5236) The API writes the operation to Cosmos DB and then sends the message of the first step to Service Bus. If the API stopped between these two calls, nothing sent the message. The operation and the resource stayed in the awaiting status. This is part 4 of the change for #5236: - The write that creates the operation adds a `pendingDispatch` entry to the document. The API removes the entry after the send. The Operation model does not include the entry, so the API responses do not change. - At start and every 5 minutes, the deployment status updater reads the entries that are older than 5 minutes. One API instance claims each entry with the etag of the document. If the step has a status, the updater removes the entry. If not, it builds the message again from Cosmos DB and sends it with the message ID `:`. - Cosmos DB keeps a sensitive value as `REDACTED`. If the first message had a real sensitive value, the updater fails the step and asks the user to send the request again. - If the send raises an error, the API removes the entry, because the caller already reports the error. #5236 also names the next step of a pipeline. The status message of a step stays on the queue until the API has sent the next step, so a Service Bus redelivery already covers that case. Unit tests cover the entry in the new operation, the claim, the clear, the send with the message ID, a step that has a status, sensitive values, a message that cannot be built and a failed send. The API start tracks the new task as `pending-dispatch-resender`, together with the tasks of the status updaters. The lifespan tests expect it. Version: `api` 0.29.0. --- CHANGELOG.md | 2 + api_app/db/repositories/base.py | 3 + api_app/db/repositories/operations.py | 65 +++++++- api_app/main.py | 1 + .../service_bus/deployment_status_updater.py | 98 +++++++++++- api_app/service_bus/helpers.py | 6 +- .../service_bus/resource_request_sender.py | 62 +++++--- .../test_routes/test_resource_helpers.py | 1 + .../test_operation_repository.py | 83 +++++++++- api_app/tests_ma/test_main_lifecycle.py | 8 +- .../test_service_bus/test_pending_dispatch.py | 145 ++++++++++++++++++ .../test_resource_request_sender.py | 77 ++++++++++ docs/tre-developers/resource-processor.md | 15 ++ 13 files changed, 537 insertions(+), 29 deletions(-) create mode 100644 api_app/tests_ma/test_service_bus/test_pending_dispatch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc375a427d..384cf6102b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,8 @@ BUG FIXES: * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) * Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource without a Porter installation. (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) +* Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. A request with sensitive values fails with a clear message, because the API does not store these values. + (`api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/db/repositories/base.py b/api_app/db/repositories/base.py index d0caa8a3a9..63955ebb3a 100644 --- a/api_app/db/repositories/base.py +++ b/api_app/db/repositories/base.py @@ -45,6 +45,9 @@ async def upsert_item_with_etag(self, item: BaseModel, etag: str) -> BaseModel: body=item.model_dump(), etag=etag, match_condition=MatchConditions.IfNotModified ) + async def save_item_dict(self, item_dict: dict): + await self.container.create_item(body=item_dict) + async def update_item_dict(self, item_dict: dict): await self.container.upsert_item(body=item_dict) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index d7111cfee1..28b72a03a2 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -1,7 +1,10 @@ from datetime import datetime, UTC +import re import uuid from typing import List +from azure.core import MatchConditions +from azure.cosmos.exceptions import CosmosHttpResponseError from pydantic import TypeAdapter from db.repositories.resource_templates import ResourceTemplateRepository from resources import strings @@ -15,6 +18,12 @@ from db.errors import EntityDoesNotExist from models.domain.operation import Operation, OperationStep, Status +# The operation document keeps this entry until the API sends the message of the first step. +# The Operation model leaves it out, so API responses do not show it. +PENDING_DISPATCH = "pendingDispatch" +# Operation and step IDs are UUIDs. The filter predicate below accepts only these characters. +MESSAGE_ID = re.compile(r"^[\w-]+:[\w-]+$") + class OperationRepository(BaseRepository): @classmethod @@ -35,6 +44,52 @@ def get_timestamp() -> float: def create_operation_id() -> str: return str(uuid.uuid4()) + @staticmethod + def dispatch_message_id(operation_id: str, step_id: str) -> str: + message_id = f"{operation_id}:{step_id}" + if not MESSAGE_ID.match(message_id): + raise ValueError(f"Unexpected message ID {message_id}") + return message_id + + async def get_operations_with_pending_dispatch(self, sent_before: float) -> List[dict]: + query = ( + f"SELECT * FROM c WHERE IS_DEFINED(c.{PENDING_DISPATCH}) AND c.{PENDING_DISPATCH}.createdWhen < @sentBefore" + ) + return await self.query(query=query, parameters=[{"name": "@sentBefore", "value": sent_before}]) + + async def claim_pending_dispatch(self, operation: dict) -> bool: + """Move the send time of the entry forward. Only one API instance can claim the document that it read.""" + try: + await self.container.patch_item( + item=operation["id"], + partition_key=operation["id"], + patch_operations=[ + {"op": "set", "path": f"/{PENDING_DISPATCH}/createdWhen", "value": self.get_timestamp()} + ], + etag=operation["_etag"], + match_condition=MatchConditions.IfNotModified, + ) + except CosmosHttpResponseError as error: + if error.status_code != 412: + raise + return False + return True + + async def clear_pending_dispatch(self, operation_id: str, message_id: str): + """Remove the entry if it still names this message. A full write of the operation also removes it.""" + if not MESSAGE_ID.match(message_id): + raise ValueError(f"Unexpected message ID {message_id}") + try: + await self.container.patch_item( + item=operation_id, + partition_key=operation_id, + patch_operations=[{"op": "remove", "path": f"/{PENDING_DISPATCH}"}], + filter_predicate=f"FROM c WHERE c.{PENDING_DISPATCH}.messageId = '{message_id}'", + ) + except CosmosHttpResponseError as error: + if error.status_code != 412: + raise + def create_main_step( self, resource_template: dict, action: str, resource_id: str, status: Status, message: str ) -> OperationStep: @@ -122,7 +177,15 @@ async def create_operation_item( steps=all_steps, ) - await self.save_item(operation) + # The message of the first step goes out after this write. Write the entry in the same document, + # so that the deployment status updater can send the message if the API stops before the send. + document = operation.model_dump() + document[PENDING_DISPATCH] = { + "stepId": all_steps[0].id, + "messageId": self.dispatch_message_id(operation_id, all_steps[0].id), + "createdWhen": timestamp, + } + await self.save_item_dict(document) return operation async def build_step_list( diff --git a/api_app/main.py b/api_app/main.py index f71764686f..4b287f4025 100644 --- a/api_app/main.py +++ b/api_app/main.py @@ -102,6 +102,7 @@ def _done_callback(task: asyncio.Task[Any]) -> None: task.add_done_callback(_done_callback) track(asyncio.create_task(deploymentStatusUpdater.receive_messages(), name="deployment-status-updater")) + track(asyncio.create_task(deploymentStatusUpdater.resend_pending_dispatches(), name="pending-dispatch-resender")) track(asyncio.create_task(airlockStatusUpdater.receive_messages(), name="airlock-status-updater")) try: diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 316b880cd1..bf6ba48d6b 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -14,7 +14,7 @@ from azure.servicebus import NEXT_AVAILABLE_SESSION from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError from azure.servicebus.aio import ServiceBusClient, AutoLockRenewer -from db.repositories.operations import OperationRepository +from db.repositories.operations import PENDING_DISPATCH, OperationRepository from core import config, credentials from db.errors import EntityDoesNotExist from db.repositories.resources import ResourceRepository @@ -22,6 +22,23 @@ from resources import strings from services.logging import logger, tracer +# The API sends the first step message immediately after it writes the operation. An older entry shows that +# the API stopped between the write and the send. +PENDING_DISPATCH_SECONDS = 5 * 60 +AWAITING_STATUSES = (Status.AwaitingDeployment, Status.AwaitingUpdate, Status.AwaitingDeletion, Status.AwaitingAction) +NOT_SENT_WITH_SENSITIVE_VALUES = ( + "The API stopped before it sent this request. The request has sensitive values, which the API does not store. " + "Send the request again." +) + + +def contains_redacted_value(value) -> bool: + if isinstance(value, dict): + return any(contains_redacted_value(item) for item in value.values()) + if isinstance(value, list): + return any(contains_redacted_value(item) for item in value) + return value == strings.REDACTED_SENSITIVE_VALUE + class DeploymentStatusUpdater: def __init__(self): @@ -102,6 +119,85 @@ async def receive_messages(self): logger.exception(f"Unknown exception. Will retry - {e}") await asyncio.sleep(10) + async def resend_pending_dispatches(self): + """At start and every 5 minutes, send the first step messages that the API wrote but did not send.""" + while True: + try: + sent_before = get_timestamp() - PENDING_DISPATCH_SECONDS + documents = await self.operations_repo.get_operations_with_pending_dispatch(sent_before) + except Exception: + logger.exception("Unable to read the pending first step messages") + documents = [] + for document in documents: + try: + await self.resend_pending_dispatch(document) + except Exception: + # The entry stays, so the next pass tries again. + logger.exception(f"Unable to send the pending message of operation {document.get('id')}") + await asyncio.sleep(PENDING_DISPATCH_SECONDS) + + async def resend_pending_dispatch(self, document: dict): + pending = document[PENDING_DISPATCH] + # Each API instance runs this task. The claim lets one instance send the message. + if not await self.operations_repo.claim_pending_dispatch(document): + return + operation = TypeAdapter(Operation).validate_python(document) + step = next((item for item in operation.steps if item.id == pending["stepId"]), None) + if step is None or step.status not in AWAITING_STATUSES: + # The resource processor sent a status for the step, so it has the message. + await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) + return + + try: + resource_to_send = await update_resource_for_step( + operation_step=step, + resource_repo=self.resource_repo, + resource_template_repo=self.resource_template_repo, + resource_history_repo=self.resource_history_repo, + root_resource=None, + step_resource=None, + resource_to_update_id=step.resourceId, + primary_action=operation.action, + user=operation.user, + ) + payload = resource_to_send.get_resource_request_message_payload( + operation_id=operation.id, step_id=step.id, action=step.resourceAction + ) + except Exception as e: + logger.exception(f"Unable to build the pending message of operation {operation.id}") + await self.fail_pending_step(operation, step, repr(e)) + return + + # Cosmos DB keeps a sensitive value as a placeholder. The resource processor needs the real value. + if contains_redacted_value(payload["parameters"]): + logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") + await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) + return + + logger.info(f"Sending the pending first step message of operation {operation.id}, step {step.id}") + await send_deployment_message( + content=json.dumps(payload), + correlation_id=operation.id, + session_id=resource_to_send.id, + action=step.resourceAction, + message_id=pending["messageId"], + ) + await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) + + async def fail_pending_step(self, operation: Operation, step: OperationStep, message: str): + step.status = self.get_failure_status_for_action(step.resourceAction) + step.message = message + step.updatedWhen = get_timestamp() + await self.update_overall_operation_status(operation, step, is_last_step=len(operation.steps) == 1) + # The Operation model has no pending dispatch, so this write also removes the entry. + await self.operations_repo.update_item(operation) + resource = await self.resource_repo.get_resource_by_id(uuid.UUID(step.resourceId)) + resource.deploymentStatus = step.status + await self.resource_repo.update_item(resource) + if step.resourceId != operation.resourceId: + # The root resource shows the status of the operation. + await self.update_primary_uninstall_status(operation) + async def process_message(self, msg): complete_message = False message = "" diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index 892c4ec7cd..a0fd471f4c 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -35,8 +35,10 @@ async def _send_message(message: ServiceBusMessage, queue: str): await sender.send_messages(message) -async def send_deployment_message(content, correlation_id, session_id, action): - resource_request_message = ServiceBusMessage(body=content, correlation_id=correlation_id, session_id=session_id) +async def send_deployment_message(content, correlation_id, session_id, action, message_id=None): + resource_request_message = ServiceBusMessage( + body=content, correlation_id=correlation_id, session_id=session_id, message_id=message_id + ) logger.info( f"Sending resource request message with correlation ID {resource_request_message.correlation_id}, action: {action}" ) diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index e838467e8b..f72b3c7c49 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -11,7 +11,7 @@ from models.domain.operation import Operation from db.repositories.operations import OperationRepository -from services.logging import tracer +from services.logging import logger, tracer async def send_resource_request_message( @@ -59,29 +59,45 @@ async def send_resource_request_message( # resource at this point is the original object with unmaskked values first_step = operation.steps[0] current_span.set_attribute("step_id", first_step.id) - resource_to_send = await update_resource_for_step( - operation_step=first_step, - resource_repo=resource_repo, - resource_template_repo=resource_template_repo, - resource_history_repo=resource_history_repo, - root_resource=resource, - step_resource=None, - resource_to_update_id=first_step.resourceId, - primary_action=action, - user=user, - ) + message_id = OperationRepository.dispatch_message_id(operation.id, first_step.id) + try: + resource_to_send = await update_resource_for_step( + operation_step=first_step, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + root_resource=resource, + step_resource=None, + resource_to_update_id=first_step.resourceId, + primary_action=action, + user=user, + ) - # create + send the message - content = json.dumps( - resource_to_send.get_resource_request_message_payload( - operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction + # create + send the message + content = json.dumps( + resource_to_send.get_resource_request_message_payload( + operation_id=operation.id, step_id=first_step.id, action=first_step.resourceAction + ) ) - ) - await send_deployment_message( - content=content, - correlation_id=operation.id, - session_id=first_step.resourceId, - action=first_step.resourceAction, - ) + await send_deployment_message( + content=content, + correlation_id=operation.id, + session_id=first_step.resourceId, + action=first_step.resourceAction, + message_id=message_id, + ) + except Exception: + # The caller reports this error, so the deployment status updater must not send the message later. + await clear_pending_dispatch(operations_repo, operation.id, message_id) + raise + await clear_pending_dispatch(operations_repo, operation.id, message_id) return operation + + +async def clear_pending_dispatch(operations_repo: OperationRepository, operation_id: str, message_id: str): + try: + await operations_repo.clear_pending_dispatch(operation_id, message_id) + except Exception: + # If the entry stays, the deployment status updater handles it later. A second send is safe. + logger.exception(f"Unable to clear the pending dispatch of operation {operation_id}") diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index cf9b5c8e9a..ac15a91b96 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -340,6 +340,7 @@ async def test_save_and_deploy_masks_secrets( correlation_id=operation_id, session_id=resource.id, action="install", + message_id=f"{operation_id}:{step_id}", ) # Checking that the item saved had a secret redacted diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 4111d3d78b..a392b6dbdd 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -2,6 +2,8 @@ import uuid import pytest_asyncio import pytest +from azure.core import MatchConditions +from azure.cosmos.exceptions import CosmosHttpResponseError from mock import patch from db.repositories.resource_templates import ResourceTemplateRepository from models.domain.operation import Status @@ -59,7 +61,7 @@ async def test_create_operation_steps_from_multi_step_template( expected_op.status = Status.AwaitingDeployment expected_op.message = "This resource is waiting to be deployed" - operations_repo.save_item = AsyncMock() + operations_repo.save_item_dict = AsyncMock() resource_repo.get_active_resource_by_template_name = AsyncMock(return_value=basic_shared_service) resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=multi_step_resource_template) @@ -75,3 +77,82 @@ async def test_create_operation_steps_from_multi_step_template( ) assert operation.model_dump() == expected_op.model_dump() + # The same write records the first step message until the API sends it. + document = operations_repo.save_item_dict.await_args.args[0] + assert document == { + **expected_op.model_dump(), + "pendingDispatch": { + "stepId": "random-uuid-1", + "messageId": f"{OPERATION_ID}:random-uuid-1", + "createdWhen": FAKE_CREATE_TIMESTAMP, + }, + } + + +@pytest_asyncio.fixture +async def operations_container(): + container = AsyncMock() + with patch("api.dependencies.database.Database.get_container_proxy", return_value=container): + operations_repo = await OperationRepository.create() + yield operations_repo, container + + +async def test_claim_pending_dispatch_uses_the_etag_of_the_read_document(operations_container): + operations_repo, container = operations_container + + assert await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) + + kwargs = container.patch_item.await_args.kwargs + assert kwargs["item"] == OPERATION_ID + assert kwargs["etag"] == "etag" + assert kwargs["match_condition"] == MatchConditions.IfNotModified + assert kwargs["patch_operations"][0]["path"] == "/pendingDispatch/createdWhen" + + +async def test_claim_pending_dispatch_returns_false_after_another_write(operations_container): + operations_repo, container = operations_container + container.patch_item.side_effect = CosmosHttpResponseError(status_code=412) + + assert not await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) + + +async def test_claim_pending_dispatch_raises_other_errors(operations_container): + operations_repo, container = operations_container + container.patch_item.side_effect = CosmosHttpResponseError(status_code=503) + + with pytest.raises(CosmosHttpResponseError): + await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) + + +async def test_clear_pending_dispatch_removes_only_the_named_message(operations_container): + operations_repo, container = operations_container + message_id = f"{OPERATION_ID}:step-1" + + await operations_repo.clear_pending_dispatch(OPERATION_ID, message_id) + + kwargs = container.patch_item.await_args.kwargs + assert kwargs["patch_operations"] == [{"op": "remove", "path": "/pendingDispatch"}] + assert kwargs["filter_predicate"] == f"FROM c WHERE c.pendingDispatch.messageId = '{message_id}'" + + # Another writer already removed the entry or replaced it. + container.patch_item.side_effect = CosmosHttpResponseError(status_code=412) + await operations_repo.clear_pending_dispatch(OPERATION_ID, message_id) + + +async def test_clear_pending_dispatch_rejects_other_message_ids(operations_container): + operations_repo, container = operations_container + + with pytest.raises(ValueError): + await operations_repo.clear_pending_dispatch(OPERATION_ID, "x' OR '1'='1:step") + container.patch_item.assert_not_awaited() + + +async def test_pending_dispatch_query_uses_the_send_time(operations_container): + operations_repo, _ = operations_container + operations_repo.query = AsyncMock(return_value=[]) + + await operations_repo.get_operations_with_pending_dispatch(100.0) + + kwargs = operations_repo.query.await_args.kwargs + assert "c.pendingDispatch.createdWhen < @sentBefore" in kwargs["query"] + assert kwargs["parameters"] == [{"name": "@sentBefore", "value": 100.0}] diff --git a/api_app/tests_ma/test_main_lifecycle.py b/api_app/tests_ma/test_main_lifecycle.py index facfe13af8..5a86ddc3bc 100644 --- a/api_app/tests_ma/test_main_lifecycle.py +++ b/api_app/tests_ma/test_main_lifecycle.py @@ -33,6 +33,7 @@ async def test_lifespan_lifecycle_normal(mock_airlock, mock_deployment, mock_boo mock_deployment_instance = MagicMock() mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.resend_pending_dispatches = AsyncMock() mock_deployment_instance.receive_messages = AsyncMock() mock_deployment.return_value = mock_deployment_instance @@ -47,9 +48,10 @@ async def test_lifespan_lifecycle_normal(mock_airlock, mock_deployment, mock_boo assert hasattr(app.state, "background_tasks") assert isinstance(app.state.background_tasks, BackgroundTaskManager) tasks = app.state.background_tasks.get_tasks() - assert len(tasks) == 2 + assert len(tasks) == 3 names = {t.get_name() for t in tasks} assert "deployment-status-updater" in names + assert "pending-dispatch-resender" in names assert "airlock-status-updater" in names assert app.state.background_tasks.is_shutting_down is True @@ -65,6 +67,7 @@ async def test_lifespan_task_failure_during_runtime(mock_logger, mock_airlock, m mock_deployment_instance = MagicMock() mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.resend_pending_dispatches = AsyncMock() async def fail_immediately(): raise ValueError("Runtime failure") @@ -101,6 +104,7 @@ async def test_lifespan_task_failure_during_shutdown(mock_logger, mock_airlock, mock_deployment_instance = MagicMock() mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.resend_pending_dispatches = AsyncMock() async def raise_teardown_exception(): try: @@ -146,6 +150,7 @@ async def test_lifespan_timeout_preserves_completed_task_exceptions( mock_deployment_instance = MagicMock() mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.resend_pending_dispatches = AsyncMock() # Deployment updater completes quickly during cancellation by raising an exception async def raise_teardown_exception(): @@ -226,6 +231,7 @@ async def test_lifespan_cancellation_suppression_strictly_bounded( mock_deployment_instance = MagicMock() mock_deployment_instance.init_repos = AsyncMock() + mock_deployment_instance.resend_pending_dispatches = AsyncMock() hang_event = asyncio.Event() diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py new file mode 100644 index 0000000000..e8028d067a --- /dev/null +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -0,0 +1,145 @@ +import asyncio +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from db.repositories.operations import PENDING_DISPATCH +from models.domain.operation import Status +from models.domain.workspace import Workspace +from resources import strings +from service_bus.deployment_status_updater import NOT_SENT_WITH_SENSITIVE_VALUES, DeploymentStatusUpdater +from tests_ma.test_service_bus.test_deployment_status_update import OPERATION_ID, create_sample_operation + +pytestmark = pytest.mark.asyncio + +RESOURCE_ID = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" +MESSAGE_ID = f"{OPERATION_ID}:random-uuid" + + +def pending_document(action="install", status=Status.AwaitingDeployment): + operation = create_sample_operation(RESOURCE_ID, action) + operation.steps[0].status = status + document = operation.model_dump() + document["_etag"] = "etag" + document[PENDING_DISPATCH] = {"stepId": "random-uuid", "messageId": MESSAGE_ID, "createdWhen": 1.0} + return document + + +def workspace(properties): + return Workspace( + id=RESOURCE_ID, + templateName="tre-workspace-base", + templateVersion="0.1.0", + etag="", + properties=properties, + resourcePath="test", + ) + + +@pytest.fixture +def updater(): + updater = DeploymentStatusUpdater() + updater.operations_repo = AsyncMock() + updater.resource_repo = AsyncMock() + updater.resource_template_repo = AsyncMock() + updater.resource_history_repo = AsyncMock() + updater.operations_repo.claim_pending_dispatch.return_value = True + updater.resource_repo.get_resource_by_id.return_value = workspace({}) + return updater + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_sends_the_pending_message_then_clears_the_entry(update_mock, send_mock, updater): + calls = [] + update_mock.return_value = workspace({"display_name": "name"}) + send_mock.side_effect = lambda **kwargs: calls.append("send") + updater.operations_repo.clear_pending_dispatch.side_effect = lambda *args: calls.append("clear") + + await updater.resend_pending_dispatch(pending_document()) + + kwargs = send_mock.await_args.kwargs + assert kwargs["message_id"] == MESSAGE_ID + assert kwargs["correlation_id"] == OPERATION_ID + assert kwargs["session_id"] == RESOURCE_ID + assert json.loads(kwargs["content"])["stepId"] == "random-uuid" + updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) + assert calls == ["send", "clear"] + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_an_entry_that_another_instance_claimed_is_left(update_mock, send_mock, updater): + updater.operations_repo.claim_pending_dispatch.return_value = False + + await updater.resend_pending_dispatch(pending_document()) + + update_mock.assert_not_awaited() + send_mock.assert_not_awaited() + updater.operations_repo.clear_pending_dispatch.assert_not_awaited() + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_step_with_a_status_only_clears_the_entry(update_mock, send_mock, updater): + await updater.resend_pending_dispatch(pending_document(status=Status.Deploying)) + + send_mock.assert_not_awaited() + updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) + + +@pytest.mark.parametrize( + "action, failed_status", + [ + ("install", Status.DeploymentFailed), + ("upgrade", Status.UpdatingFailed), + ("uninstall", Status.DeletingFailed), + ("start", Status.ActionFailed), + ], +) +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_sensitive_values_fail_the_step(update_mock, send_mock, updater, action, failed_status): + update_mock.return_value = workspace({"settings": {"secret": strings.REDACTED_SENSITIVE_VALUE}}) + + await updater.resend_pending_dispatch(pending_document(action=action)) + + send_mock.assert_not_awaited() + operation = updater.operations_repo.update_item.await_args.args[0] + assert operation.steps[0].status == failed_status + assert operation.steps[0].message == NOT_SENT_WITH_SENSITIVE_VALUES + assert operation.status == failed_status + assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == failed_status + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_message_that_cannot_be_built_fails_the_step(update_mock, send_mock, updater): + update_mock.side_effect = Exception("template not found") + + await updater.resend_pending_dispatch(pending_document()) + + send_mock.assert_not_awaited() + operation = updater.operations_repo.update_item.await_args.args[0] + assert operation.steps[0].status == Status.DeploymentFailed + assert "template not found" in operation.steps[0].message + + +@patch("service_bus.deployment_status_updater.asyncio.sleep", AsyncMock(side_effect=asyncio.CancelledError)) +async def test_a_failed_entry_does_not_stop_the_pass(updater): + first, second = pending_document(), pending_document() + second["id"] = "second" + updater.operations_repo.get_operations_with_pending_dispatch.return_value = [first, second] + sent = [] + + async def resend(document): + if document is first: + raise Exception("Service Bus is not available") + sent.append(document["id"]) + + updater.resend_pending_dispatch = resend + with pytest.raises(asyncio.CancelledError): + await updater.resend_pending_dispatches() + + assert sent == ["second"] diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 0f4426e408..e586544fab 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -227,3 +227,80 @@ async def test_multi_step_document_retries( # check it tried to patch and re-get the item the first time + all the retries assert len(resource_repo.patch_resource.mock_calls) == (num_retries + 1) assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) + + +async def send_first_step(resource_repo, operations_repo, resource_template_repo, resource_history_repo, template): + resource = create_test_resource() + operation = create_sample_operation(resource.id, RequestAction.Install) + operations_repo.create_operation_item.return_value = operation + resource_repo.get_resource_by_id.return_value = resource + resource_template_repo.get_template_by_name_and_version.return_value = template + resource_repo.patch_resource.return_value = (resource, template) + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo, + resource_repo=resource_repo, + user=create_test_user(), + resource_template_repo=resource_template_repo, + resource_history_repo=resource_history_repo, + action=RequestAction.Install, + ) + return operation + + +@patch("service_bus.resource_request_sender.ResourceHistoryRepository.create") +@patch("service_bus.resource_request_sender.OperationRepository.create") +@patch("service_bus.helpers.ServiceBusClient") +@patch("service_bus.resource_request_sender.ResourceRepository.create") +@patch("service_bus.resource_request_sender.ResourceTemplateRepository.create") +async def test_a_failed_send_removes_the_pending_dispatch( + resource_template_repo, + resource_repo, + service_bus_client_mock, + operations_repo_mock, + resource_history_repo_mock, + multi_step_resource_template, +): + service_bus_client_mock().get_queue_sender().send_messages = AsyncMock(side_effect=Exception("unavailable")) + + with pytest.raises(Exception, match="unavailable"): + await send_first_step( + resource_repo, + operations_repo_mock, + resource_template_repo, + resource_history_repo_mock, + multi_step_resource_template, + ) + + # The caller reports the error, so the deployment status updater must not send the message later. + operation_id = operations_repo_mock.create_operation_item.return_value.id + operations_repo_mock.clear_pending_dispatch.assert_awaited_once_with(operation_id, f"{operation_id}:random-uuid") + + +@patch("service_bus.resource_request_sender.ResourceHistoryRepository.create") +@patch("service_bus.resource_request_sender.OperationRepository.create") +@patch("service_bus.helpers.ServiceBusClient") +@patch("service_bus.resource_request_sender.ResourceRepository.create") +@patch("service_bus.resource_request_sender.ResourceTemplateRepository.create") +async def test_a_failed_clear_after_the_send_returns_the_operation( + resource_template_repo, + resource_repo, + service_bus_client_mock, + operations_repo_mock, + resource_history_repo_mock, + multi_step_resource_template, +): + service_bus_client_mock().get_queue_sender().send_messages = AsyncMock() + operations_repo_mock.clear_pending_dispatch.side_effect = Exception("Cosmos DB is not available") + + operation = await send_first_step( + resource_repo, + operations_repo_mock, + resource_template_repo, + resource_history_repo_mock, + multi_step_resource_template, + ) + + # The entry stays, and the deployment status updater sends the message again. A second send is safe. + sent_message = service_bus_client_mock().get_queue_sender().send_messages.call_args.args[0] + assert sent_message.message_id == f"{operation.id}:random-uuid" diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 5bf02764ab..8d8d40429d 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -135,6 +135,21 @@ To test a lost session lock, set the `RP_BUNDLE_VALUES` variable to `{"test_skip While Porter runs, the resource processor then does not renew the session lock, so Service Bus gives a step that takes more than 5 minutes to another worker. Do not use this setting in production. +### Messages that the API did not send + +The API sends the message of the first step of each request immediately after it writes the operation to Cosmos DB. The same write adds a `pendingDispatch` entry to the operation document, and the API removes the entry after the send. The API does not show this entry in its responses. + +If the API stops between the write and the send, the entry stays. When the API starts and every 5 minutes, the deployment status updater reads the entries that are older than 5 minutes: + +1. One API instance claims each entry with the etag of the operation document. The other instances leave the entry. +1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. +1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. +1. Cosmos DB keeps a sensitive value as `REDACTED`, so the updater cannot build a message that needs one. It sets the step and the operation to the failed status, with a message that asks the user to send the request again. + +A second copy of a message is safe, because the resource processor runs each step once. + +The status message of a step stays on the `deploymentstatus` queue until the API has sent the message of the next step. So a stop of the API during a pipeline does not need this entry. + ### Porter Azure plugin Resource Processor uses [Porter Azure plugin](https://github.com/getporter/azure-plugins) to access secrets in Azure Key Vault. From 9ecfcb7d9a676be88d67d141cc47052e7d9e0af9 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 29 Oct 2026 22:17:21 +0000 Subject: [PATCH 06/37] Refuse an upgrade after an uninstall and fail a step without storage access (#5236) A review found three defects in the resource processor part of #5236. - Porter keeps the record of an uninstalled installation. The upgrade guard refused an upgrade only if `porter installations list` was empty, so `porter installations apply` still ran the install action after an uninstall. The guard now also refuses an upgrade if the last successful action was an uninstall. A failed install has neither time, so an upgrade can still repair it. - If the management storage account refused access before Porter started, the worker sent no status. Service Bus then delivered the message again until it went to the dead-letter queue. For HTTP 401 and 403, the worker now sends the failed status and completes the message, as the Terraform backend of the bundle did before. Other storage errors keep the redelivery, and the worker logs the step. - If the execution lease expired during a storage outage, the final record write failed. The write now takes the same lease again if no other worker took it, and then writes the record. --- resource_processor/helpers/execution_lease.py | 11 +- .../tests_rp/test_execution_lease.py | 26 ++++ resource_processor/tests_rp/test_run_step.py | 30 +++- resource_processor/tests_rp/test_runner.py | 53 +++++++ resource_processor/vmss_porter/runner.py | 132 ++++++++++++++---- 5 files changed, 218 insertions(+), 34 deletions(-) diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py index 03b8f03613..a3743570fd 100644 --- a/resource_processor/helpers/execution_lease.py +++ b/resource_processor/helpers/execution_lease.py @@ -47,7 +47,16 @@ async def read(self) -> dict: return record if isinstance(record, dict) else {} async def write(self, record: dict): - await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) + try: + await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) + except HttpResponseError as error: + if error.status_code != 412: + raise + # The lease expired, for example after a storage outage that was longer than the renewal retries. + # Azure gives the same lease ID again if no other worker took the lease. If one did, this raises 409. + await self.lease.acquire(lease_duration=LEASE_SECONDS) + self.expires = time.monotonic() + LEASE_SECONDS + await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) async def create_blob(container: ContainerClient, blob): diff --git a/resource_processor/tests_rp/test_execution_lease.py b/resource_processor/tests_rp/test_execution_lease.py index ea2b4a6ddf..bbcc1a34dc 100644 --- a/resource_processor/tests_rp/test_execution_lease.py +++ b/resource_processor/tests_rp/test_execution_lease.py @@ -97,3 +97,29 @@ async def test_a_record_is_written_with_the_lease(): lease = MagicMock() await leases.ExecutionLease(blob, lease).write({"state": "running"}) blob.upload_blob.assert_awaited_once_with(json.dumps({"state": "running"}), overwrite=True, lease=lease) + + +def precondition_failed(): + error = HttpResponseError(message="lease lost") + error.status_code = 412 + return error + + +@pytest.mark.asyncio +async def test_an_expired_lease_is_taken_again_for_the_record(): + blob = MagicMock(upload_blob=AsyncMock(side_effect=[precondition_failed(), None])) + lease = MagicMock(acquire=AsyncMock()) + await leases.ExecutionLease(blob, lease).write({"state": "finished"}) + lease.acquire.assert_awaited_once_with(lease_duration=leases.LEASE_SECONDS) + assert blob.upload_blob.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_lease_that_another_worker_took_stops_the_record_write(): + taken = HttpResponseError(message="lease already present") + taken.status_code = 409 + blob = MagicMock(upload_blob=AsyncMock(side_effect=[precondition_failed(), None])) + lease = MagicMock(acquire=AsyncMock(side_effect=taken)) + with pytest.raises(HttpResponseError): + await leases.ExecutionLease(blob, lease).write({"state": "finished"}) + assert blob.upload_blob.await_count == 1 diff --git a/resource_processor/tests_rp/test_run_step.py b/resource_processor/tests_rp/test_run_step.py index 9c031d2a93..8527a01799 100644 --- a/resource_processor/tests_rp/test_run_step.py +++ b/resource_processor/tests_rp/test_run_step.py @@ -158,7 +158,7 @@ async def test_only_an_upgrade_requires_the_installation(action, required): with ( patch.object(runner, "build_porter_command", AsyncMock(return_value=([["porter", action]], None, "p", None))), patch.object(runner, "run_porter", AsyncMock(return_value=(1, "", "failed"))) as run_porter, - patch.object(runner, "_confirm_porter_installation_absent", AsyncMock(return_value=False)), + patch.object(runner, "_confirm_porter_installation_removed", AsyncMock(return_value=False)), ): await runner.invoke_porter_action({**MESSAGE, "action": action}, client, CONFIG) assert run_porter.await_args.kwargs["required_installation"] == required @@ -178,7 +178,7 @@ async def helper(command, config, *args, **kwargs): patch.object(runner, "azure_login_command", return_value=[]), patch.object(runner, "azure_acr_login_command", return_value=[]), patch.object(runner, "apply_porter_credentials_sets_command", return_value=[]), - patch.object(runner, "_confirm_porter_installation_absent", AsyncMock(return_value=absent)), + patch.object(runner, "_confirm_porter_installation_removed", AsyncMock(return_value=absent)), ): code, _, error = await runner.run_porter( [["porter", "installations", "apply"]], {}, required_installation="resource-1" @@ -191,6 +191,32 @@ async def helper(command, config, *args, **kwargs): assert code == 0 +def installation(installed=None, uninstalled=None, name="resource-1"): + return {"name": name, "status": {"installed": installed, "uninstalled": uninstalled}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output, removed", + [ + ([], True), + # Porter keeps the record after an uninstall, and `installations apply` then runs the install action. + ([installation("2026-10-29T10:00:00.123456789Z", "2026-10-29T11:00:00.5Z")], True), + ([installation(None, "2026-10-29T11:00:00Z")], True), + ([installation("2026-10-29T12:00:00Z", "2026-10-29T11:00:00Z")], False), + ([installation("2026-10-29T10:00:00Z")], False), + # A failed install has neither time. An upgrade can repair it. + ([installation()], False), + ([installation(name="other-resource")], False), + (None, False), + ], +) +async def test_an_upgrade_finds_a_removed_installation(output, removed): + result = (0, json.dumps(output), "") if output is not None else (1, "", "store not available") + with patch.object(runner, "run_command_helper", AsyncMock(return_value=result)): + assert await runner._confirm_porter_installation_removed("resource-1", {}) is removed + + @pytest.mark.asyncio async def test_a_failed_record_write_still_sends_the_final_status(): client, sender = service_bus() diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 2861e7d0b1..8aa876d63f 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -1,3 +1,4 @@ +from azure.core.exceptions import HttpResponseError from azure.servicebus import ServiceBusSessionFilter from azure.servicebus.exceptions import SessionLockLostError from azure.servicebus.aio import ServiceBusClient @@ -1258,3 +1259,55 @@ async def run_step(*args): if running: assert seen == [paused] assert mock_keep_renewed.call_args.kwargs["paused"]() is False + + +def failing_lease(status_code, reason): + error = HttpResponseError(message="storage error") + error.status_code, error.reason = status_code, reason + + @asynccontextmanager + async def lease(container, resource_id, describe): + raise error + yield + + return patch("vmss_porter.runner.execution_lease", side_effect=lease) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code, reason", [(403, "Forbidden"), (401, "Unauthorized")]) +async def test_no_access_to_the_execution_record_fails_the_step( + mock_service_bus_client, mock_keep_renewed, status_code, reason +): + client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) + client.get_queue_sender.return_value.send_messages = AsyncMock() + config = {"resource_request_queue": "q", "deployment_status_queue": "status"} + + with failing_lease(status_code, reason), patch("vmss_porter.runner.run_step") as run_step: + await receive_message(client, config, AsyncMock(), keep_running=Mock(side_effect=[True, False])) + + run_step.assert_not_called() + sent = json.loads(str(client.get_queue_sender.return_value.send_messages.await_args.args[0])) + assert sent["status"] == "deployment_failed" + assert f"HTTP {status_code} {reason}" in sent["message"] + assert "executionId" not in sent + receiver.complete_message.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_a_short_storage_failure_leaves_the_message_for_a_redelivery( + mock_service_bus_client, mock_keep_renewed, mock_logger +): + client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) + client.get_queue_sender.return_value.send_messages = AsyncMock() + + with ( + failing_lease(503, "Server Busy"), + patch("vmss_porter.runner.asyncio.sleep", new_callable=AsyncMock), + ): + await receive_message( + client, {"resource_request_queue": "q"}, AsyncMock(), keep_running=Mock(side_effect=[True, False]) + ) + + client.get_queue_sender.return_value.send_messages.assert_not_awaited() + receiver.complete_message.assert_not_awaited() + assert "with HTTP 503" in mock_logger.error.call_args.args[0] diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index cb1833f585..75763ba6f8 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -1,4 +1,6 @@ +import re import time +from datetime import datetime from typing import Awaitable, Callable, Optional from multiprocessing import Process import json @@ -26,6 +28,7 @@ from shared.config import VERSION from helpers import statuses from contextlib import asynccontextmanager +from azure.core.exceptions import HttpResponseError from azure.servicebus import ServiceBusMessage, NEXT_AVAILABLE_SESSION from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError, SessionLockLostError from azure.servicebus.aio import ServiceBusClient @@ -172,30 +175,44 @@ async def receive_message(service_bus_client, config: dict, executions: Containe # Service Bus can give the same message to a second worker. The lease lets # one worker at a time run Porter for the resource. - async with execution_lease(executions, message["id"], lambda: step["text"]) as execution: - step["running"] = True - try: - result = await run_step(message, execution, service_bus_client, config) - finally: - step["running"] = False - - if result: - logger.info(f"Resource request for {message['id']} is complete") - else: - logger.error("Message processing failed!") - - try: - await receiver.complete_message(msg) - except SessionLockLostError: - # Another worker has the session now. It reads the execution record - # and sends the recorded status again. - logger.warning( - f"The session lock expired before the message was completed for {step['text']}" + try: + async with execution_lease( + executions, message["id"], lambda: step["text"] + ) as execution: + step["running"] = True + try: + result = await run_step(message, execution, service_bus_client, config) + finally: + step["running"] = False + + if result: + logger.info(f"Resource request for {message['id']} is complete") + else: + logger.error("Message processing failed!") + + try: + await receiver.complete_message(msg) + except SessionLockLostError: + # Another worker has the session now. It reads the execution record + # and sends the recorded status again. + logger.warning( + f"The session lock expired before the message was completed for {step['text']}" + ) + break + logger.info( + f"Message for resource_id={message['id']}, operation_id={message['operationId']} processed as {result} and marked complete." ) - break - logger.info( - f"Message for resource_id={message['id']}, operation_id={message['operationId']} processed as {result} and marked complete." - ) + except HttpResponseError as error: + # The lease and the record are in the management storage account. Without access, a + # retry does not help. Fail the step, as the Terraform backend of the bundle did before. + if error.status_code not in (401, 403): + logger.error( + f"The execution record store failed for {step['text']} with HTTP {error.status_code}. " + "Service Bus delivers the message again." + ) + raise + await send_execution_store_failure(message, error, service_bus_client, config) + await receiver.complete_message(msg) logger.info(f"Closing session: {session.session_id}") finally: @@ -309,13 +326,14 @@ async def run_porter( if error: return (1, None, error) - # `porter installations apply` installs a missing installation. An upgrade must not create the resource again. - if required_installation is not None and await _confirm_porter_installation_absent(required_installation, config): + # `porter installations apply` installs a missing or uninstalled installation. An upgrade must not create the + # resource again. + if required_installation is not None and await _confirm_porter_installation_removed(required_installation, config): return ( 1, None, - f"Cannot upgrade resource '{required_installation}': Porter has no installation for it. " - "An upgrade does not install a missing resource.", + f"Cannot upgrade resource '{required_installation}': Porter has no installation for it, " + "or its last successful action was an uninstall. An upgrade does not install a missing resource.", ) last_returncode = None @@ -379,8 +397,8 @@ async def _cleanup_param_set(param_set_name: str, param_set_file: str, installat logger.debug(f"Best-effort cleanup: could not delete temp file '{path}': {e}") -async def _confirm_porter_installation_absent(installation_id: str, config: dict) -> bool: - """Confirm absence with a fresh storage query, without logging installation data.""" +async def _list_porter_installations(installation_id: str, config: dict) -> Optional[list]: + """Read the installations with this name from a fresh storage query, without logging installation data.""" try: code, output, _ = await asyncio.wait_for( run_command_helper( @@ -401,10 +419,43 @@ async def _confirm_porter_installation_absent(installation_id: str, config: dict ), timeout=30, ) - return code == 0 and json.loads(output) == [] + return json.loads(output) if code == 0 else None except Exception: - # An unreadable store, invalid response or timeout does not prove absence. + # An unreadable store, invalid response or timeout proves nothing. + return None + + +async def _confirm_porter_installation_absent(installation_id: str, config: dict) -> bool: + """Confirm absence with a fresh storage query, without logging installation data.""" + return await _list_porter_installations(installation_id, config) == [] + + +def _porter_time(value) -> Optional[datetime]: + if not value: + return None + try: + # Porter writes nanoseconds, and Python reads up to microseconds. + return datetime.fromisoformat(re.sub(r"(\.\d{6})\d+", r"\1", value).replace("Z", "+00:00")) + except ValueError: + return None + + +async def _confirm_porter_installation_removed(installation_id: str, config: dict) -> bool: + """ + Confirm that Porter has no installation, or that its last successful action was an uninstall. + Porter keeps the record of an uninstalled installation. A failed install has neither time, so an upgrade can repair it. + """ + installations = await _list_porter_installations(installation_id, config) + if installations == []: + return True + matches = [item for item in installations or [] if item.get("name") == installation_id] + if not matches: + # An unreadable store, or a list with other installations only, proves nothing. return False + status = matches[0].get("status") or {} + uninstalled = _porter_time(status.get("uninstalled")) + installed = _porter_time(status.get("installed")) + return uninstalled is not None and (installed is None or installed <= uninstalled) async def run_step(msg_body: dict, execution, sb_client: ServiceBusClient, config: dict) -> bool: @@ -437,6 +488,25 @@ async def finish(status: str, message: str, with_outputs: bool): return await invoke_porter_action(msg_body, sb_client, config, execution_id=execution_id, finish=finish) +async def send_execution_store_failure( + msg_body: dict, error: HttpResponseError, sb_client: ServiceBusClient, config: dict +): + """The resource processor cannot use the execution record, and a retry does not change that. Fail the step.""" + status = statuses.failed_status_string_for[msg_body["action"]] + message = ( + f"Cannot use the execution record in the management storage account: HTTP {error.status_code} {error.reason}" + ) + sb_sender = sb_client.get_queue_sender(queue_name=config["deployment_status_queue"]) + await sb_sender.send_messages( + ServiceBusMessage( + body=service_bus_message_generator(msg_body, status, message), + correlation_id=msg_body["id"], + session_id=msg_body["operationId"], + ) + ) + logger.error(f"Sent {status} for {msg_body['id']}, because the execution record store returned {error.status_code}") + + async def send_recorded_status(msg_body: dict, record: dict, sb_client: ServiceBusClient, config: dict) -> bool: action = msg_body["action"] status = record.get("status", statuses.failed_status_string_for[action]) From 299a63cd4cc67fe37ff5a1187cd648a2b797883c Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 29 Oct 2026 22:17:25 +0000 Subject: [PATCH 07/37] Let the first run replace a failure that the API wrote (#5236) A review found these defects in the API part of #5236. - A final status that the API writes itself has no `executionId`, so the API ignored all statuses of the real run of that step. This occurs after a failed send of the next step, or after the resender failed a step whose message the resource processor still got. The first run with an `executionId` now replaces such a status. An uninstall keeps its earlier rule. - The resender failed each request with a `REDACTED` value. Only the first message of an install has real values that Cosmos DB does not keep. Other requests come from Cosmos DB, so the resender now sends them as the first send did. - The resender did not look for a later request for the resource. It now fails the step if one exists, so that an install does not come after a delete. - Bandit B608 reported the new query, because it was an f-string. The query is now a literal string. - The comment on `executionId` named version 0.13.17. The first resource processor that sends the field is 0.13.16. The resource processor document now describes these rules, the compatibility with earlier versions and the storage access errors. The instruction for the test setting adds a key to `RP_BUNDLE_VALUES`, so that the other keys stay. --- CHANGELOG.md | 4 +- api_app/db/repositories/operations.py | 4 +- api_app/models/domain/operation.py | 2 +- .../service_bus/deployment_status_updater.py | 29 ++++++++-- .../test_deployment_status_update.py | 17 ++++++ .../test_service_bus/test_pending_dispatch.py | 58 ++++++++++++++----- docs/tre-developers/resource-processor.md | 17 ++++-- 7 files changed, 102 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384cf6102b..9f04be05fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,9 +59,9 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) -* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource without a Porter installation. +* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter does not have or that it uninstalled. (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) -* Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. A request with sensitive values fails with a clear message, because the API does not store these values. +* Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. An install with sensitive values fails with a clear message, because the API does not store these values. (`api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 28b72a03a2..b3ffd61a7f 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -52,9 +52,7 @@ def dispatch_message_id(operation_id: str, step_id: str) -> str: return message_id async def get_operations_with_pending_dispatch(self, sent_before: float) -> List[dict]: - query = ( - f"SELECT * FROM c WHERE IS_DEFINED(c.{PENDING_DISPATCH}) AND c.{PENDING_DISPATCH}.createdWhen < @sentBefore" - ) + query = "SELECT * FROM c WHERE IS_DEFINED(c.pendingDispatch) AND c.pendingDispatch.createdWhen < @sentBefore" return await self.query(query=query, parameters=[{"name": "@sentBefore", "value": sent_before}]) async def claim_pending_dispatch(self, operation: dict) -> bool: diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 79eda850e0..5ba0579c68 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -133,5 +133,5 @@ class DeploymentStatusUpdateMessage(AzureTREModel): status: Status = Field(title="", description="") message: str = Field(title="", description="") outputs: List[Output] = Field(title="", description="", default_factory=list) - # A resource processor before 0.13.17 does not send this field. + # A resource processor before 0.13.16 does not send this field. executionId: Optional[str] = Field(default=None, title="", description="The resource processor run") diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index bf6ba48d6b..0205cd1a7e 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -27,8 +27,12 @@ PENDING_DISPATCH_SECONDS = 5 * 60 AWAITING_STATUSES = (Status.AwaitingDeployment, Status.AwaitingUpdate, Status.AwaitingDeletion, Status.AwaitingAction) NOT_SENT_WITH_SENSITIVE_VALUES = ( - "The API stopped before it sent this request. The request has sensitive values, which the API does not store. " - "Send the request again." + "The API stopped before it recorded the send of this request. The request has sensitive values, which the API " + "does not store, so the API cannot send it again. If the status does not change, send the request again." +) +NOT_SENT_BEFORE_A_LATER_REQUEST = ( + "The API stopped before it recorded the send of this request, and a later request changed the resource. " + "The API does not send this request again." ) @@ -148,6 +152,17 @@ async def resend_pending_dispatch(self, document: dict): await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) return + later_requests = [ + item + for item in await self.operations_repo.get_operations_by_resource_id(operation.resourceId) + if item.createdWhen > operation.createdWhen + ] + if later_requests: + # For example, a delete of a resource whose install was not sent. The install must not come after it. + logger.warning(f"Operation {operation.id} was not sent before a later request. Failing step {step.id}") + await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST) + return + try: resource_to_send = await update_resource_for_step( operation_step=step, @@ -168,8 +183,10 @@ async def resend_pending_dispatch(self, document: dict): await self.fail_pending_step(operation, step, repr(e)) return - # Cosmos DB keeps a sensitive value as a placeholder. The resource processor needs the real value. - if contains_redacted_value(payload["parameters"]): + # The first message of an install has the real sensitive values, but Cosmos DB keeps a placeholder. Other + # requests are built from Cosmos DB, so they send the same values as the first send. + first_install = operation.action == RequestAction.Install and step.resourceId == operation.resourceId + if first_install and contains_redacted_value(payload["parameters"]): logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) return @@ -394,6 +411,10 @@ def ignore_repeated_status( """ if message.executionId is None or not (step.is_success() or step.is_failure()): return False + if step.executionId is None: + # The API or an older resource processor wrote this final status. The first run of the step replaces it, + # for example after the API failed a step whose message the resource processor still got. + return False if step.executionId != message.executionId: logger.info(f"Ignoring a status from another run of step {operation.id}/{step.id}") return True diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 6c45745711..5a6c7940e3 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1191,6 +1191,23 @@ async def test_a_new_run_before_the_final_status_takes_over_the_step(persisted_p send.assert_awaited_once() +# An uninstall keeps the earlier rule for a final step from #5253. +@pytest.mark.parametrize("persisted_pipeline", [RequestAction.Install, RequestAction.Upgrade, "restart"], indirect=True) +async def test_the_first_run_replaces_a_failure_that_the_api_wrote(persisted_pipeline): + updater, stored, message, send, (in_progress, success, failure) = persisted_pipeline + # For example, the API failed a step that it could not resend, but the resource processor got the first message. + stored["operation"].steps[0].status = failure + stored["operation"].steps[0].message = "written by the API" + + assert await updater.update_status_in_database(message(in_progress)) is True + assert stored["operation"].steps[0].status == in_progress + assert await updater.update_status_in_database(message(success)) is True + + assert stored["operation"].steps[0].status == success + assert stored["operation"].steps[0].executionId == "run-a" + send.assert_awaited_once() + + async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): workspace = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") operation = create_sample_operation(workspace.id, RequestAction.Install) diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index e8028d067a..35fc922f32 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -8,13 +8,18 @@ from models.domain.operation import Status from models.domain.workspace import Workspace from resources import strings -from service_bus.deployment_status_updater import NOT_SENT_WITH_SENSITIVE_VALUES, DeploymentStatusUpdater +from service_bus.deployment_status_updater import ( + NOT_SENT_BEFORE_A_LATER_REQUEST, + NOT_SENT_WITH_SENSITIVE_VALUES, + DeploymentStatusUpdater, +) from tests_ma.test_service_bus.test_deployment_status_update import OPERATION_ID, create_sample_operation pytestmark = pytest.mark.asyncio RESOURCE_ID = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" MESSAGE_ID = f"{OPERATION_ID}:random-uuid" +AWAITING = {"upgrade": Status.AwaitingUpdate, "uninstall": Status.AwaitingDeletion, "start": Status.AwaitingAction} def pending_document(action="install", status=Status.AwaitingDeployment): @@ -46,6 +51,7 @@ def updater(): updater.resource_history_repo = AsyncMock() updater.operations_repo.claim_pending_dispatch.return_value = True updater.resource_repo.get_resource_by_id.return_value = workspace({}) + updater.operations_repo.get_operations_by_resource_id.return_value = [] return updater @@ -89,28 +95,50 @@ async def test_a_step_with_a_status_only_clears_the_entry(update_mock, send_mock updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) -@pytest.mark.parametrize( - "action, failed_status", - [ - ("install", Status.DeploymentFailed), - ("upgrade", Status.UpdatingFailed), - ("uninstall", Status.DeletingFailed), - ("start", Status.ActionFailed), - ], -) @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") -async def test_sensitive_values_fail_the_step(update_mock, send_mock, updater, action, failed_status): +async def test_sensitive_values_of_an_install_fail_the_step(update_mock, send_mock, updater): update_mock.return_value = workspace({"settings": {"secret": strings.REDACTED_SENSITIVE_VALUE}}) - await updater.resend_pending_dispatch(pending_document(action=action)) + await updater.resend_pending_dispatch(pending_document()) send_mock.assert_not_awaited() operation = updater.operations_repo.update_item.await_args.args[0] - assert operation.steps[0].status == failed_status + assert operation.steps[0].status == Status.DeploymentFailed assert operation.steps[0].message == NOT_SENT_WITH_SENSITIVE_VALUES - assert operation.status == failed_status - assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == failed_status + assert operation.status == Status.DeploymentFailed + assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == Status.DeploymentFailed + + +@pytest.mark.parametrize("action", ["upgrade", "uninstall", "start"]) +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_other_requests_send_the_placeholder_as_the_first_send_did(update_mock, send_mock, updater, action): + # The API builds these requests from Cosmos DB, so the first send also had the placeholder. + update_mock.return_value = workspace({"settings": {"secret": strings.REDACTED_SENSITIVE_VALUE}}) + + await updater.resend_pending_dispatch(pending_document(action=action, status=AWAITING[action])) + + send_mock.assert_awaited_once() + updater.operations_repo.update_item.assert_not_awaited() + updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_later_request_for_the_resource_fails_the_step(update_mock, send_mock, updater): + document = pending_document() + later = create_sample_operation(RESOURCE_ID, "uninstall") + later.createdWhen = document["createdWhen"] + 60 + updater.operations_repo.get_operations_by_resource_id.return_value = [later] + + await updater.resend_pending_dispatch(document) + + update_mock.assert_not_awaited() + send_mock.assert_not_awaited() + operation = updater.operations_repo.update_item.await_args.args[0] + assert operation.steps[0].status == Status.DeploymentFailed + assert operation.steps[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST @patch("service_bus.deployment_status_updater.send_deployment_message") diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 8d8d40429d..c437bd3750 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -127,11 +127,16 @@ Service Bus can give the same message to more than one worker. The resource proc 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. 1. The blob holds a record of the last step for the resource. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. 1. Each status message has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. -1. An upgrade does not install a resource that has no Porter installation. The resource processor sends `updating_failed`. +1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. +1. An upgrade does not install a resource that Porter does not have. The resource processor sends `updating_failed` if Porter has no installation for the resource, or if the last successful action of the installation was an uninstall. Porter runs the install action in both cases. An upgrade can still repair a failed install. + +A status message without `executionId` comes from a resource processor before version 0.13.16, and the API keeps the earlier rules for it. An API before version 0.28.10 ignores the field. So you can deploy the API and the resource processor in any order. + +The resource processor now needs the management storage account for each step. If the account refuses access (HTTP 401 or 403), the resource processor sends the failed status for the step. A Terraform backend in the same account failed in the same way before this change. For other storage errors, Service Bus delivers the message again. For local debugging, your identity needs the Storage Blob Data Contributor role on the management storage account. -To test a lost session lock, set the `RP_BUNDLE_VALUES` variable to `{"test_skip_session_lock_renewal": "true"}` and deploy the core. +To test a lost session lock, add the key `"test_skip_session_lock_renewal": "true"` to the `RP_BUNDLE_VALUES` JSON object and deploy the core. Remove the key after the test. While Porter runs, the resource processor then does not renew the session lock, so Service Bus gives a step that takes more than 5 minutes to another worker. Do not use this setting in production. @@ -143,10 +148,14 @@ If the API stops between the write and the send, the entry stays. When the API s 1. One API instance claims each entry with the etag of the operation document. The other instances leave the entry. 1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. +1. If a later request for the same resource exists, for example a delete, the updater does not send the message. It sets the step and the operation to the failed status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. -1. Cosmos DB keeps a sensitive value as `REDACTED`, so the updater cannot build a message that needs one. It sets the step and the operation to the failed status, with a message that asks the user to send the request again. +1. The first message of an install has the real values of sensitive properties, but Cosmos DB keeps them as `REDACTED`. The updater cannot build such a message again. + It sets the step and the operation to the failed status, with a message that asks the user to send the request again if the status does not change. + Other requests come from Cosmos DB, so the updater sends them with the same values as the first send. +1. If the updater cannot build the message, it sets the step and the operation to the failed status with the error text. -A second copy of a message is safe, because the resource processor runs each step once. +A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. The status message of a step stays on the `deploymentstatus` queue until the API has sent the message of the next step. So a stop of the API during a pipeline does not need this entry. From f5f94e5d770196d870dd2bad625f6d5a4e2ebc0d Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 18:10:28 +0000 Subject: [PATCH 08/37] Store a placeholder for sensitive values when the API saves an update (#5298) At the install, the API saves a copy of the resource with REDACTED for each sensitive property, and sends the real values in the request message. An update saved the new values as they were. So the new value of a sensitive property, for example the client_secret of a workspace, was in Cosmos DB and in the resource history, and API responses with the resource properties included it. - patch_resource saves a masked copy and returns the resource with the real values, because the caller builds the request message from it. The history item is masked too. - mask_sensitive_properties moves to db.repositories.resources, so that the repositories can use it. resource_helpers imports it from there. - The operation write records in the pending dispatch entry if the first message has real sensitive values. The deployment status updater then does not send that message again, for an install and for an update. Before, the rule covered only an install. - A migration replaces the values that earlier updates stored, in the resources and in their history items. A second run changes nothing. --- CHANGELOG.md | 3 + api_app/api/routes/migrations.py | 18 +++ api_app/api/routes/resource_helpers.py | 48 +----- api_app/db/repositories/operations.py | 9 +- api_app/db/repositories/resources.py | 139 ++++++++++++++++- .../service_bus/deployment_status_updater.py | 24 +-- .../test_api/test_routes/test_migrations.py | 26 +++- .../test_routes/test_shared_services.py | 15 +- .../test_api/test_routes/test_workspaces.py | 35 ++++- .../test_operation_repository.py | 35 +++++ .../test_resource_repository.py | 142 +++++++++++++++++- .../test_service_bus/test_pending_dispatch.py | 59 ++++++-- docs/tre-developers/resource-processor.md | 5 +- 13 files changed, 461 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f04be05fc..560932f202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,9 @@ BUG FIXES: (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. An install with sensitive values fails with a clear message, because the API does not store these values. (`api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) +* Store a placeholder for each sensitive property when the API saves an update of a resource, as it already does at the install. The new value goes to the resource processor only in the request message. + Before this change, an update stored the new value of a sensitive property, for example the `client_secret` of a workspace, in Cosmos DB and in the resource history. API responses with the resource properties then included this value. + A database migration replaces the values that earlier updates stored. (`api` 0.28.10) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/api/routes/migrations.py b/api_app/api/routes/migrations.py index d9ea1cf60e..3ccb897497 100644 --- a/api_app/api/routes/migrations.py +++ b/api_app/api/routes/migrations.py @@ -2,6 +2,9 @@ from auth.rbac import require_tre_admin from resources import strings from models.schemas.migrations import Migration, MigrationOutList +from db.repositories.resource_templates import ResourceTemplateRepository +from db.repositories.resources import ResourceRepository +from db.repositories.resources_history import ResourceHistoryRepository from db.repositories.workspaces import WorkspaceRepository from services.logging import logger @@ -28,6 +31,21 @@ async def migrate_database(): Migration(issueNumber="5048", status=f"Set airlock_version=1 on {len(migrated_ids)} legacy workspace(s)") ) + # Updates stored real sensitive values before the API masked them, as it does at the install. + resource_repo = await ResourceRepository.create() + changed_resources, changed_history_items = await resource_repo.mask_stored_sensitive_values( + await ResourceTemplateRepository.create(), await ResourceHistoryRepository.create() + ) + logger.info( + f"Masked sensitive values in {changed_resources} resource(s) and {changed_history_items} history item(s)." + ) + migrations.append( + Migration( + issueNumber="5298", + status=f"Masked sensitive values in {changed_resources} resource(s) and {changed_history_items} history item(s)", + ) + ) + return MigrationOutList(migrations=migrations) except Exception as e: logger.exception("Failed to migrate database") diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 0483bc1f85..6519eb803a 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -1,14 +1,14 @@ from datetime import datetime, UTC import semantic_version from copy import deepcopy -from typing import Dict, Any, List, Optional +from typing import List, Optional from fastapi import HTTPException, status from db.repositories.user_resources import UserResourceRepository from models.domain.user_resource import UserResource from models.domain.workspace_service import WorkspaceService from models.schemas.resource import ResourcePatch -from db.repositories.resources import ResourceRepository +from db.repositories.resources import ResourceRepository, mask_sensitive_properties from db.repositories.resources_history import ResourceHistoryRepository from models.domain.resource_template import ResourceTemplate from models.domain.authentication import User @@ -126,50 +126,6 @@ async def save_and_deploy_resource( ) -def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: - updated_resource_parameters = deepcopy(properties) - - flattened_template_props = {} - - # recusrse the template and flatten all properties into a list - def flatten_template_props(template_fragment: dict): - if template_fragment is None: - return - - for prop_name, prop in template_fragment.items(): - if prop_name == "pipeline": - continue - - if prop_name == "properties" and template_fragment["properties"] is not None: - for inner_prop_name, inner_prop in template_fragment["properties"].items(): - flattened_template_props[inner_prop_name] = inner_prop - - if isinstance(prop, list) and len(prop) > 0: - for p in prop: - if isinstance(p, dict): - flatten_template_props(p) - - if isinstance(prop, dict) and prop_name != "if": - flatten_template_props(prop) - - flatten_template_props(template.model_dump()) - - def recurse_input_props(prop_dict: dict): - for prop_name, prop in prop_dict.items(): - if ( - prop_name in flattened_template_props - and "sensitive" in flattened_template_props[prop_name] - and flattened_template_props[prop_name]["sensitive"] is True - ): - prop_dict[prop_name] = strings.REDACTED_SENSITIVE_VALUE - if isinstance(prop, dict): - recurse_input_props(prop) - - recurse_input_props(updated_resource_parameters) - - return updated_resource_parameters - - def construct_location_header(operation: Operation) -> str: return f"/api{operation.resourcePath}/operations/{operation.id}" diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index b3ffd61a7f..e7617900d4 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -10,7 +10,7 @@ from resources import strings from models.domain.request_action import RequestAction from models.domain.resource import ResourceType -from db.repositories.resources import ResourceRepository +from db.repositories.resources import ResourceRepository, mask_sensitive_properties from models.domain.authentication import User from core import config from db.repositories.base import BaseRepository @@ -121,6 +121,7 @@ async def create_operation_item( # get the right "awaiting" message based on the action status, message = self.get_initial_status(action) all_steps = [] + sensitive_values = False for resource in resource_list: name = resource["templateName"] version = resource["templateVersion"] @@ -134,6 +135,11 @@ async def create_operation_item( resource_template = await resource_template_repo.get_template_by_name_and_version( name, version, resource_type, primary_parent_service_name ) + if resource["id"] == resource_id: + # The API builds the first message from this resource, which can have real sensitive values. + # Cosmos DB keeps a placeholder for each of them. + properties = resource.get("properties") or {} + sensitive_values = mask_sensitive_properties(properties, resource_template) != properties resource_template_dict = resource_template.model_dump(exclude_none=True) # if the template has a pipeline defined for this action, copy over all the steps to the ops document steps = await self.build_step_list( @@ -182,6 +188,7 @@ async def create_operation_item( "stepId": all_steps[0].id, "messageId": self.dispatch_message_id(operation_id, all_steps[0].id), "createdWhen": timestamp, + "sensitiveValues": sensitive_values, } await self.save_item_dict(document) return operation diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index e75567db66..fdbceb48a3 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -1,10 +1,10 @@ import copy import semantic_version from datetime import datetime, UTC -from typing import Optional, Tuple, List +from typing import Any, Dict, Optional, Tuple, List from azure.cosmos.exceptions import CosmosResourceNotFoundError -from resources.strings import RESOURCE_ACTION_INSTALL +from resources.strings import RESOURCE_ACTION_INSTALL, REDACTED_SENSITIVE_VALUE from core import config from db.errors import ( VersionDowngradeDenied, @@ -28,6 +28,7 @@ from models.domain.workspace_service import WorkspaceService from models.schemas.resource import ResourcePatch from pydantic import UUID4, TypeAdapter +from services.logging import logger # Keep Draft 7 keyword semantics while enforcing TRE's restriction on extra fields. @@ -36,6 +37,75 @@ ) +def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: + updated_resource_parameters = copy.deepcopy(properties) + + flattened_template_props = {} + + # recusrse the template and flatten all properties into a list + def flatten_template_props(template_fragment: dict): + if template_fragment is None: + return + + for prop_name, prop in template_fragment.items(): + if prop_name == "pipeline": + continue + + if prop_name == "properties" and template_fragment["properties"] is not None: + for inner_prop_name, inner_prop in template_fragment["properties"].items(): + flattened_template_props[inner_prop_name] = inner_prop + + if isinstance(prop, list) and len(prop) > 0: + for p in prop: + if isinstance(p, dict): + flatten_template_props(p) + + if isinstance(prop, dict) and prop_name != "if": + flatten_template_props(prop) + + flatten_template_props(template.model_dump()) + + def recurse_input_props(prop_dict: dict): + for prop_name, prop in prop_dict.items(): + if ( + prop_name in flattened_template_props + and "sensitive" in flattened_template_props[prop_name] + and flattened_template_props[prop_name]["sensitive"] is True + ): + prop_dict[prop_name] = REDACTED_SENSITIVE_VALUE + if isinstance(prop, dict): + recurse_input_props(prop) + + recurse_input_props(updated_resource_parameters) + + return updated_resource_parameters + + +def masked_property_paths(properties: dict, masked: dict, path: Optional[List[str]] = None) -> List[str]: + """Return a JSON pointer for each value that mask_sensitive_properties changed.""" + path = path or ["properties"] + paths = [] + for key, value in masked.items(): + original = properties.get(key) + if isinstance(value, dict) and isinstance(original, dict): + paths += masked_property_paths(original, value, path + [key]) + elif value != original: + paths.append("".join("/" + part.replace("~", "~0").replace("/", "~1") for part in path + [key])) + return paths + + +async def set_masked_values(container, item_id: str, partition_key: str, paths: List[str]): + # Cosmos DB accepts at most 10 operations in one patch request. + for start in range(0, len(paths), 10): + await container.patch_item( + item=item_id, + partition_key=partition_key, + patch_operations=[ + {"op": "set", "path": path, "value": REDACTED_SENSITIVE_VALUE} for path in paths[start : start + 10] + ], + ) + + class ResourceRepository(BaseRepository): @classmethod async def create(cls): @@ -198,7 +268,10 @@ async def patch_resource( resource_action: str, force_version_update: bool = False, ) -> Tuple[Resource, ResourceTemplate]: - await resource_history_repo.create_resource_history_item(resource) + # A document from before the masking of updates can still have real sensitive values. + history_resource = copy.deepcopy(resource) + history_resource.properties = mask_sensitive_properties(resource.properties, resource_template) + await resource_history_repo.create_resource_history_item(history_resource) # now update the resource props resource.resourceVersion = resource.resourceVersion + 1 resource.user = user.model_dump() if hasattr(user, "model_dump") else user @@ -219,9 +292,67 @@ async def patch_resource( # if we're here then we're valid - update the props + persist resource.properties.update(resource_patch.properties) - await self.update_item_with_etag(resource, etag) + # Cosmos DB keeps sensitive values as a placeholder, as at the install. The returned resource keeps the + # real values, because the caller builds the request message from it. + masked_resource = copy.deepcopy(resource) + masked_resource.properties = mask_sensitive_properties(resource.properties, resource_template) + await self.update_item_with_etag(masked_resource, etag) return resource, resource_template + async def mask_stored_sensitive_values( + self, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository + ) -> Tuple[int, int]: + """Replace each real sensitive value that an earlier update stored with the placeholder. + + Returns the number of resource documents and history items that changed. A second run changes nothing. + """ + resources = {resource["id"]: resource for resource in await self.query(query="SELECT * FROM c")} + templates = {} + + async def template_for(resource: dict, version: str) -> Optional[ResourceTemplate]: + resource_type = ResourceType(resource["resourceType"]) + parent_service_name = None + if resource_type == ResourceType.UserResource: + parent = resources.get(resource.get("parentWorkspaceServiceId")) + if parent is None: + logger.warning(f"No parent workspace service for user resource {resource['id']}") + return None + parent_service_name = parent["templateName"] + key = (resource["templateName"], version, resource_type, parent_service_name) + if key not in templates: + try: + templates[key] = await resource_template_repo.get_template_by_name_and_version(*key) + except EntityDoesNotExist: + logger.warning( + f"No template {resource['templateName']} {version} to mask resource {resource['id']}" + ) + templates[key] = None + return templates[key] + + async def mask(item: dict, template: Optional[ResourceTemplate], container, partition_key: str) -> bool: + if template is None: + return False + properties = item.get("properties") or {} + paths = masked_property_paths(properties, mask_sensitive_properties(properties, template)) + if paths: + await set_masked_values(container, item["id"], partition_key, paths) + return bool(paths) + + changed_resources = 0 + for resource in resources.values(): + template = await template_for(resource, resource["templateVersion"]) + changed_resources += await mask(resource, template, self.container, resource["id"]) + + changed_history_items = 0 + for item in await resource_history_repo.query(query="SELECT * FROM c"): + resource = resources.get(item.get("resourceId")) + if resource is None: + continue + template = await template_for(resource, item.get("templateVersion") or resource["templateVersion"]) + changed_history_items += await mask(item, template, resource_history_repo.container, item["resourceId"]) + + return changed_resources, changed_history_items + async def get_resource_dependency_list(self, resource: Resource) -> List: # Get the parent resource path and id parent_resource_path = resource.resourcePath diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 0205cd1a7e..0e0d998c42 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -36,14 +36,6 @@ ) -def contains_redacted_value(value) -> bool: - if isinstance(value, dict): - return any(contains_redacted_value(item) for item in value.values()) - if isinstance(value, list): - return any(contains_redacted_value(item) for item in value) - return value == strings.REDACTED_SENSITIVE_VALUE - - class DeploymentStatusUpdater: def __init__(self): pass @@ -163,6 +155,14 @@ async def resend_pending_dispatch(self, document: dict): await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST) return + # The API built the first message of the request from the resource in memory, with the real sensitive + # values. Cosmos DB keeps a placeholder for each, so the updater cannot build that message again. The API + # builds all other messages from Cosmos DB, so a resend has the same values as the first send. + if pending.get("sensitiveValues") and step.resourceId == operation.resourceId: + logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") + await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) + return + try: resource_to_send = await update_resource_for_step( operation_step=step, @@ -183,14 +183,6 @@ async def resend_pending_dispatch(self, document: dict): await self.fail_pending_step(operation, step, repr(e)) return - # The first message of an install has the real sensitive values, but Cosmos DB keeps a placeholder. Other - # requests are built from Cosmos DB, so they send the same values as the first send. - first_install = operation.action == RequestAction.Install and step.resourceId == operation.resourceId - if first_install and contains_redacted_value(payload["parameters"]): - logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") - await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) - return - logger.info(f"Sending the pending first step message of operation {operation.id}, step {step.id}") await send_deployment_message( content=json.dumps(payload), diff --git a/api_app/tests_ma/test_api/test_routes/test_migrations.py b/api_app/tests_ma/test_api/test_routes/test_migrations.py index 4aba30af02..b8e5b24e38 100644 --- a/api_app/tests_ma/test_api/test_routes/test_migrations.py +++ b/api_app/tests_ma/test_api/test_routes/test_migrations.py @@ -1,5 +1,5 @@ import pytest -from mock import patch +from mock import AsyncMock, patch from fastapi import status from auth.rbac import require_tre_admin, require_tre_user_or_admin @@ -38,6 +38,16 @@ def _prepare(cls, app, admin_user): yield app.dependency_overrides = {} + @pytest.fixture(autouse=True) + def resource_repo(self): + with ( + patch("api.routes.migrations.ResourceRepository.create") as resource_repo, + patch("api.routes.migrations.ResourceTemplateRepository.create"), + patch("api.routes.migrations.ResourceHistoryRepository.create"), + ): + resource_repo.return_value.mask_stored_sensitive_values = AsyncMock(return_value=(0, 0)) + yield resource_repo.return_value + # [POST] /migrations/ @patch("api.routes.migrations.WorkspaceRepository.create") @patch("api.routes.migrations.logger.info") @@ -60,3 +70,17 @@ async def test_post_migrations_stamps_legacy_workspaces_with_v1(self, logging, w workspace_repo.return_value.set_default_airlock_version_for_legacy_workspaces.assert_awaited_once() assert response.status_code == status.HTTP_202_ACCEPTED assert "2 legacy workspace(s)" in response.json()["migrations"][0]["status"] + + # [POST] /migrations/ + @patch("api.routes.migrations.WorkspaceRepository.create") + @patch("api.routes.migrations.logger.info") + async def test_post_migrations_masks_stored_sensitive_values( + self, logging, workspace_repo, resource_repo, client, app + ): + workspace_repo.return_value.set_default_airlock_version_for_legacy_workspaces.return_value = [] + resource_repo.mask_stored_sensitive_values.return_value = (2, 3) + response = await client.post(app.url_path_for(strings.API_MIGRATE_DATABASE)) + + resource_repo.mask_stored_sensitive_values.assert_awaited_once() + assert response.status_code == status.HTTP_202_ACCEPTED + assert "2 resource(s) and 3 history item(s)" in response.json()["migrations"][1]["status"] diff --git a/api_app/tests_ma/test_api/test_routes/test_shared_services.py b/api_app/tests_ma/test_api/test_routes/test_shared_services.py index a72ea8af89..e8685b739b 100644 --- a/api_app/tests_ma/test_api/test_routes/test_shared_services.py +++ b/api_app/tests_ma/test_api/test_routes/test_shared_services.py @@ -243,7 +243,10 @@ async def test_patch_shared_service_returns_422_if_invalid_id(self, get_shared_s "api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", return_value=sample_shared_service(SHARED_SERVICE_ID), ) - @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_shared_service(), + ) @patch( "api.routes.shared_services.SharedServiceRepository.update_item_with_etag", return_value=sample_shared_service() ) @@ -357,7 +360,10 @@ async def test_patch_shared_service_with_upgrade_major_version_and_force_update_ "api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", return_value=sample_shared_service(SHARED_SERVICE_ID), ) - @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_shared_service(), + ) @patch( "api.routes.shared_services.SharedServiceRepository.update_item_with_etag", return_value=sample_shared_service() ) @@ -392,7 +398,10 @@ async def test_patch_shared_service_with_upgrade_major_version_returns_bad_reque "api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", return_value=sample_shared_service(SHARED_SERVICE_ID), ) - @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_shared_service(), + ) @patch( "api.routes.shared_services.SharedServiceRepository.update_item_with_etag", return_value=sample_shared_service() ) diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index a0450a4fca..cd0a26e850 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -743,7 +743,10 @@ async def test_patch_workspace_rejects_v2_on_a_template_that_does_not_support_it ) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag", return_value=sample_workspace()) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspaces_patches_workspace( self, _, __, update_item_mock, ___, ____, _____, _______, app, client @@ -778,7 +781,10 @@ async def test_patch_workspaces_patches_workspace( ) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag", return_value=sample_workspace()) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspaces_with_upgrade_major_version_returns_bad_request( self, _, __, update_item_mock, ___, ____, _____, ______, app, client @@ -852,7 +858,10 @@ async def test_patch_workspaces_with_upgrade_major_version_and_force_update_retu ) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag", return_value=sample_workspace()) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspaces_with_downgrade_version_returns_bad_request( self, _, __, update_item_mock, ___, ____, _____, ______, app, client @@ -924,7 +933,10 @@ async def test_patch_workspaces_with_upgrade_minor_version_patches_workspace( @patch( "api.routes.workspaces.WorkspaceRepository.update_item_with_etag", side_effect=CosmosAccessConditionFailedError ) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspace_returns_409_if_bad_etag( self, _, __, update_item_mock, ___, ____, _____, app, client @@ -1637,7 +1649,10 @@ async def test_patch_user_resource_returns_422_if_invalid_id( "api.routes.resource_helpers.send_resource_request_message", return_value=sample_resource_operation(resource_id=USER_RESOURCE_ID, operation_id=OPERATION_ID), ) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") @patch( "api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", @@ -2081,7 +2096,10 @@ async def test_patch_workspace_service_returns_422_if_invalid_id( "api.routes.workspaces.send_resource_request_message", return_value=sample_resource_operation(resource_id=WORKSPACE_ID, operation_id=OPERATION_ID), ) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch( "api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", return_value=sample_workspace_service(), @@ -2824,7 +2842,10 @@ async def test_patch_user_resource_returns_422_if_invalid_id( "api.routes.resource_helpers.send_resource_request_message", return_value=sample_resource_operation(resource_id=USER_RESOURCE_ID, operation_id=OPERATION_ID), ) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=None) + @patch( + "api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", + return_value=sample_resource_template(), + ) @patch("api.routes.workspaces.validate_user_has_valid_role_for_user_resource") @patch( "api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index a392b6dbdd..2a501564ed 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -9,6 +9,7 @@ from models.domain.operation import Status from db.repositories.resources import ResourceRepository from db.repositories.operations import OperationRepository +from resources import strings from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP RESOURCE_ID = str(uuid.uuid4()) @@ -85,10 +86,44 @@ async def test_create_operation_steps_from_multi_step_template( "stepId": "random-uuid-1", "messageId": f"{OPERATION_ID}:random-uuid-1", "createdWhen": FAKE_CREATE_TIMESTAMP, + "sensitiveValues": False, }, } +@pytest.mark.parametrize("value, expected", [("real value", True), (strings.REDACTED_SENSITIVE_VALUE, False)]) +@patch("db.repositories.operations.OperationRepository.get_timestamp", return_value=FAKE_CREATE_TIMESTAMP) +@patch("db.repositories.operations.OperationRepository.create_operation_id", return_value=OPERATION_ID) +async def test_pending_entry_records_sensitive_values_of_the_first_message( + _, __, value, expected, operations_repo, resource_repo, resource_template_repo, test_user, basic_resource_template +): + template = basic_resource_template.model_copy(deep=True) + template.properties["client_secret"] = {"type": "string", "sensitive": True, "updateable": True} + operations_repo.save_item_dict = AsyncMock() + resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=template) + resource = { + "id": RESOURCE_ID, + "templateName": template.name, + "templateVersion": template.version, + "resourceType": "workspace", + "properties": {"display_name": "name", "client_secret": value}, + } + + await operations_repo.create_operation_item( + resource_id=RESOURCE_ID, + resource_list=[resource], + action="upgrade", + resource_path=f"/workspaces/{RESOURCE_ID}", + resource_version=1, + user=test_user, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo, + ) + + document = operations_repo.save_item_dict.await_args.args[0] + assert document["pendingDispatch"]["sensitiveValues"] is expected + + @pytest_asyncio.fixture async def operations_container(): container = AsyncMock() diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 81e860923b..3248a64ec6 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -12,7 +12,7 @@ from tests_ma.test_api.conftest import create_test_user from db.errors import EntityDoesNotExist, UserNotAuthorizedToUseTemplate -from db.repositories.resources import ResourceRepository +from db.repositories.resources import ResourceRepository, masked_property_paths, set_masked_values from azure.cosmos.exceptions import CosmosResourceNotFoundError from models.domain.resource import Resource from models.domain.resource_template import ResourceTemplate @@ -367,7 +367,9 @@ async def test_get_resource_dict_by_id_raises_entity_does_not_exist_if_no_resour @patch("db.repositories.resources_history.ResourceHistoryRepository.save_item", return_value=AsyncMock()) @patch("db.repositories.resources.ResourceRepository.validate_patch") @patch("db.repositories.resources.ResourceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) -async def test_patch_resource_preserves_property_history(_, __, ___, resource_repo, resource_history_repo): +async def test_patch_resource_preserves_property_history( + _, __, ___, resource_repo, resource_history_repo, basic_resource_template +): """ Tests that properties are copied into a history array and only certain values in the root are updated """ @@ -386,7 +388,14 @@ async def test_patch_resource_preserves_property_history(_, __, ___, resource_re expected_resource.updatedWhen = FAKE_UPDATE_TIMESTAMP await resource_repo.patch_resource( - resource, resource_patch, None, etag, None, resource_history_repo, user, strings.RESOURCE_ACTION_UPDATE + resource, + resource_patch, + basic_resource_template, + etag, + None, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE, ) resource_repo.update_item_with_etag.assert_called_once_with(expected_resource, etag) @@ -399,11 +408,136 @@ async def test_patch_resource_preserves_property_history(_, __, ___, resource_re expected_resource.user = user.model_dump() await resource_repo.patch_resource( - new_resource, new_patch, None, etag, None, resource_history_repo, user, strings.RESOURCE_ACTION_UPDATE + new_resource, + new_patch, + basic_resource_template, + etag, + None, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE, ) resource_repo.update_item_with_etag.assert_called_with(expected_resource, etag) +def template_with_secret(template: ResourceTemplate) -> ResourceTemplate: + template = template.model_copy(deep=True) + template.properties["client_secret"] = {"type": "string", "sensitive": True, "updateable": True} + return template + + +@pytest.mark.asyncio +@patch("db.repositories.resources.ResourceRepository.validate_patch") +@patch("db.repositories.resources.ResourceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +async def test_patch_resource_stores_a_placeholder_for_sensitive_values(_, __, resource_repo, basic_resource_template): + resource_repo.update_item_with_etag = AsyncMock() + history_repo = AsyncMock() + resource = sample_resource() + # A document that an update wrote before the update path masked sensitive values. + resource.properties["client_secret"] = "earlier value" + resource_patch = ResourcePatch(properties={"client_secret": "new value"}) + + returned, _ = await resource_repo.patch_resource( + resource, + resource_patch, + template_with_secret(basic_resource_template), + "etag", + None, + history_repo, + create_test_user(), + strings.RESOURCE_ACTION_UPDATE, + ) + + stored = resource_repo.update_item_with_etag.await_args.args[0] + assert stored.properties["client_secret"] == strings.REDACTED_SENSITIVE_VALUE + assert stored.properties["display_name"] == "initial display name" + # The caller builds the request message from the returned resource. + assert returned.properties["client_secret"] == "new value" + history_item = history_repo.create_resource_history_item.await_args.args[0] + assert history_item.properties["client_secret"] == strings.REDACTED_SENSITIVE_VALUE + + +def stored_resource(resource_id, template_name, resource_type, properties, **fields): + return { + "id": resource_id, + "templateName": template_name, + "templateVersion": "1.0.0", + "resourceType": resource_type, + "properties": properties, + **fields, + } + + +@pytest.mark.asyncio +async def test_mask_stored_sensitive_values_replaces_only_real_values(resource_repo, basic_resource_template): + template = template_with_secret(basic_resource_template) + resource_repo._container = AsyncMock() + resource_repo.query = AsyncMock( + return_value=[ + stored_resource("ws", "workspace-template", "workspace", {"client_secret": "real", "display_name": "n"}), + stored_resource("ws-2", "workspace-template", "workspace", {"client_secret": "REDACTED"}), + stored_resource("svc", "service-template", "workspace-service", {}), + stored_resource( + "ur", "user-template", "user-resource", {"client_secret": "real"}, parentWorkspaceServiceId="svc" + ), + stored_resource("old", "removed-template", "workspace", {"client_secret": "real"}), + ] + ) + + async def get_template(name, version, resource_type, parent_service_name=None): + if name == "removed-template": + raise EntityDoesNotExist + return template + + template_repo = AsyncMock() + template_repo.get_template_by_name_and_version.side_effect = get_template + history_repo = MagicMock() + history_repo.container = AsyncMock() + history_repo.query = AsyncMock( + return_value=[ + {"id": "h-1", "resourceId": "ws", "templateVersion": "0.9.0", "properties": {"client_secret": "older"}}, + {"id": "h-2", "resourceId": "ws", "properties": {"client_secret": "REDACTED"}}, + {"id": "h-3", "resourceId": "deleted-from-resources", "properties": {"client_secret": "older"}}, + ] + ) + + result = await resource_repo.mask_stored_sensitive_values(template_repo, history_repo) + + assert result == (2, 1) + set_secret = [{"op": "set", "path": "/properties/client_secret", "value": strings.REDACTED_SENSITIVE_VALUE}] + assert [call.kwargs for call in resource_repo.container.patch_item.await_args_list] == [ + {"item": "ws", "partition_key": "ws", "patch_operations": set_secret}, + {"item": "ur", "partition_key": "ur", "patch_operations": set_secret}, + ] + # The history container uses the resource ID as its partition key. + history_repo.container.patch_item.assert_awaited_once_with( + item="h-1", partition_key="ws", patch_operations=set_secret + ) + template_repo.get_template_by_name_and_version.assert_any_await( + "user-template", "1.0.0", ResourceType.UserResource, "service-template" + ) + template_repo.get_template_by_name_and_version.assert_any_await( + "workspace-template", "0.9.0", ResourceType.Workspace, None + ) + + +def test_masked_property_paths_escape_nested_names(): + properties = {"settings": {"a/b": "real", "c~d": "real", "plain": "x"}} + masked = {"settings": {"a/b": "REDACTED", "c~d": "REDACTED", "plain": "x"}} + + assert masked_property_paths(properties, masked) == ["/properties/settings/a~1b", "/properties/settings/c~0d"] + + +@pytest.mark.asyncio +async def test_set_masked_values_sends_at_most_ten_operations_in_each_request(): + container = AsyncMock() + paths = [f"/properties/secret_{index}" for index in range(11)] + + await set_masked_values(container, "item", "partition", paths) + + assert [len(call.kwargs["patch_operations"]) for call in container.patch_item.await_args_list] == [10, 1] + + @patch("db.repositories.resources.ResourceTemplateRepository.enrich_template") def test_validate_patch_with_good_fields_passes(template_repo, resource_repo): """ diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index 35fc922f32..554b9171c9 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -19,15 +19,25 @@ RESOURCE_ID = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" MESSAGE_ID = f"{OPERATION_ID}:random-uuid" -AWAITING = {"upgrade": Status.AwaitingUpdate, "uninstall": Status.AwaitingDeletion, "start": Status.AwaitingAction} +AWAITING = { + "install": Status.AwaitingDeployment, + "upgrade": Status.AwaitingUpdate, + "uninstall": Status.AwaitingDeletion, + "start": Status.AwaitingAction, +} -def pending_document(action="install", status=Status.AwaitingDeployment): +def pending_document(action="install", status=Status.AwaitingDeployment, sensitive_values=False): operation = create_sample_operation(RESOURCE_ID, action) operation.steps[0].status = status document = operation.model_dump() document["_etag"] = "etag" - document[PENDING_DISPATCH] = {"stepId": "random-uuid", "messageId": MESSAGE_ID, "createdWhen": 1.0} + document[PENDING_DISPATCH] = { + "stepId": "random-uuid", + "messageId": MESSAGE_ID, + "createdWhen": 1.0, + "sensitiveValues": sensitive_values, + } return document @@ -95,26 +105,49 @@ async def test_a_step_with_a_status_only_clears_the_entry(update_mock, send_mock updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) +@pytest.mark.parametrize( + "action, status, failed", + [ + ("install", Status.AwaitingDeployment, Status.DeploymentFailed), + ("upgrade", Status.AwaitingUpdate, Status.UpdatingFailed), + ], +) @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") -async def test_sensitive_values_of_an_install_fail_the_step(update_mock, send_mock, updater): - update_mock.return_value = workspace({"settings": {"secret": strings.REDACTED_SENSITIVE_VALUE}}) - - await updater.resend_pending_dispatch(pending_document()) +async def test_a_first_message_with_sensitive_values_fails_the_step( + update_mock, send_mock, updater, action, status, failed +): + # The API built that message from the resource in memory. Cosmos DB has only the placeholder. + await updater.resend_pending_dispatch(pending_document(action=action, status=status, sensitive_values=True)) + update_mock.assert_not_awaited() send_mock.assert_not_awaited() operation = updater.operations_repo.update_item.await_args.args[0] - assert operation.steps[0].status == Status.DeploymentFailed + assert operation.steps[0].status == failed assert operation.steps[0].message == NOT_SENT_WITH_SENSITIVE_VALUES - assert operation.status == Status.DeploymentFailed - assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == Status.DeploymentFailed + assert operation.status == failed + assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == failed + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_sensitive_values_of_another_resource_do_not_stop_the_send(update_mock, send_mock, updater): + # The API builds the message of a step for another resource from Cosmos DB, so a resend has the same values. + document = pending_document(sensitive_values=True) + document["steps"][0]["resourceId"] = "another-resource" + update_mock.return_value = workspace({"display_name": "name"}) + + await updater.resend_pending_dispatch(document) + + send_mock.assert_awaited_once() + updater.operations_repo.update_item.assert_not_awaited() -@pytest.mark.parametrize("action", ["upgrade", "uninstall", "start"]) +@pytest.mark.parametrize("action", ["install", "upgrade", "uninstall", "start"]) @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") -async def test_other_requests_send_the_placeholder_as_the_first_send_did(update_mock, send_mock, updater, action): - # The API builds these requests from Cosmos DB, so the first send also had the placeholder. +async def test_a_resend_without_sensitive_values_sends_the_placeholder(update_mock, send_mock, updater, action): + # The first message had no real sensitive values, so it had the same placeholder as Cosmos DB. update_mock.return_value = workspace({"settings": {"secret": strings.REDACTED_SENSITIVE_VALUE}}) await updater.resend_pending_dispatch(pending_document(action=action, status=AWAITING[action])) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index c437bd3750..1be2b00515 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -150,9 +150,10 @@ If the API stops between the write and the send, the entry stays. When the API s 1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. 1. If a later request for the same resource exists, for example a delete, the updater does not send the message. It sets the step and the operation to the failed status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. -1. The first message of an install has the real values of sensitive properties, but Cosmos DB keeps them as `REDACTED`. The updater cannot build such a message again. +1. The API builds the first message of a request from the resource in memory. After an install, or after an update that changes a sensitive property, that message has real values that Cosmos DB keeps as `REDACTED`. + The operation write records this in the entry, and the updater cannot build such a message again. It sets the step and the operation to the failed status, with a message that asks the user to send the request again if the status does not change. - Other requests come from Cosmos DB, so the updater sends them with the same values as the first send. + The API builds all other messages from Cosmos DB, so the updater sends them with the same values as the first send. 1. If the updater cannot build the message, it sets the step and the operation to the failed status with the error text. A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. From 6ec658044db00d1d0dcbaf3a817914775497fcc1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 18:10:45 +0000 Subject: [PATCH 09/37] Add an E2E case for a sensitive property after an update (#5298) test_workspace_masked_update.py creates a workspace with the manual test application, updates its client secret two times as a TRE admin, and reads the workspace and its history with the workspace owner token. The case logs only true or false values, counts and the token roles. The opt-in marker masking_validation selects it. --- docs/tre-developers/github-pr-bot-commands.md | 2 +- e2e_tests/pytest.ini | 1 + e2e_tests/test_workspace_masked_update.py | 102 ++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 e2e_tests/test_workspace_masked_update.py diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index da6f973b54..383bc9a070 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -22,7 +22,7 @@ The table describes the selected assertions. A passing command does not prove ev | `/test-airlock` | Import requests, access checks, the import review VM flow and export request storage routing | No export review VM or migration proof. | | `/test-backups` | Base workspace deployment with backups enabled and disabled | No backup restore or retention proof. | -The `workspace_services`, `linux_vm`, `airlock_validation`, `sql_validation`, `mysql_validation` and `gitea_validation` markers have no dedicated slash command. +The `workspace_services`, `linux_vm`, `airlock_validation`, `sql_validation`, `mysql_validation`, `gitea_validation` and `masking_validation` markers have no dedicated slash command. Use the branch workflow or the local test runner for these selections. The `airlock_validation` marker selects two Airlock cases. It does not select the full `airlock` suite. See [bundle selection](end-to-end-tests.md#validate-one-bundle) for exact cases and declared coverage gaps. diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index ea391b6c83..3ecb654d1d 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -19,6 +19,7 @@ markers = sql_validation: Private SQL query and configuration upgrade lifecycle mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM gitea_validation: opt-in private Gitea repository write/read across an update and an app restart + masking_validation: opt-in check that a sensitive workspace property stays masked after an update asyncio_mode = auto asyncio_default_fixture_loop_scope = session diff --git a/e2e_tests/test_workspace_masked_update.py b/e2e_tests/test_workspace_masked_update.py new file mode 100644 index 0000000000..4ae183ea61 --- /dev/null +++ b/e2e_tests/test_workspace_masked_update.py @@ -0,0 +1,102 @@ +"""A sensitive workspace property stays masked after an update. + +The case creates a workspace with the manual test application. A TRE admin then updates the client secret two +times with the same value, and the workspace token reads the workspace and its history back. The case logs only +true or false values and counts. It never logs a property value or a response text. +""" + +import base64 +import json +import logging + +import pytest +from httpx import AsyncClient + +from e2e_tests import config +from e2e_tests.conftest import clean_up_test_workspace, create_or_get_test_workspace, get_workspace_owner_token +from e2e_tests.helpers import get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.deployment import patch_done +from e2e_tests.resources.resource import wait_for + +LOGGER = logging.getLogger(__name__) +SENSITIVE = "client_secret" +MASK = "REDACTED" + + +def token_roles(provider) -> list: + """Read the role names from the token. The names are not secret.""" + payload = provider.get_token().split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + return sorted(claims.get("roles", [])) + + +async def read_workspace(client, workspace_path, token) -> dict: + response = await client.get(get_full_endpoint(f"/api{workspace_path}"), headers=get_auth_header(token)) + assert response.status_code == 200, f"The workspace GET returned {response.status_code}" + value = response.json()["workspace"]["properties"].get(SENSITIVE) + return {"masked": value == MASK, "equal_to_update": value == config.TEST_WORKSPACE_APP_SECRET} + + +async def read_history(client, workspace_path, token) -> dict: + response = await client.get(get_full_endpoint(f"/api{workspace_path}/history"), headers=get_auth_header(token)) + assert response.status_code == 200, f"The workspace history GET returned {response.status_code}" + values = [item["properties"].get(SENSITIVE) for item in response.json()["resource_history"]] + return { + "items": len(values), + "masked": sum(value == MASK for value in values), + "equal_to_update": sum(value == config.TEST_WORKSPACE_APP_SECRET for value in values), + } + + +async def update_sensitive_property(client, workspace_path, admin_token) -> str: + payload = { + "properties": { + "auth_type": "Manual", + "client_id": config.TEST_WORKSPACE_APP_ID, + "client_secret": config.TEST_WORKSPACE_APP_SECRET, + } + } + headers = {**get_auth_header(admin_token), "eTag": "*"} + response = await client.patch(get_full_endpoint(f"/api{workspace_path}"), headers=headers, json=payload) + # Do not log the response text. A validation error can repeat the request values. + assert response.status_code == 202, f"The workspace PATCH returned {response.status_code}" + return response.headers["Location"] + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.masking_validation +@pytest.mark.timeout(150 * 60) +async def test_a_sensitive_property_stays_masked_after_an_update(verify): + if not (config.TEST_WORKSPACE_APP_ID and config.TEST_WORKSPACE_APP_SECRET): + pytest.skip("The case needs the manual test workspace application") + + workspace_path, workspace_id = await create_or_get_test_workspace( + auth_type="Manual", + verify=verify, + client_id=config.TEST_WORKSPACE_APP_ID, + client_secret=config.TEST_WORKSPACE_APP_SECRET, + ) + try: + admin_token = await get_admin_token(verify) + workspace_token = await get_workspace_owner_token(workspace_id, verify) + LOGGER.info("Roles in the workspace token: %s", token_roles(workspace_token)) + + results = {} + async with AsyncClient(verify=verify, timeout=60.0) as client: + results["after_install"] = await read_workspace(client, workspace_path, workspace_token) + for update in (1, 2): + operation = await update_sensitive_property(client, workspace_path, admin_token) + # The API writes the properties before it sends the update, so read them at once. + results[f"after_update_{update}"] = await read_workspace(client, workspace_path, workspace_token) + await wait_for(patch_done, client, operation, admin_token, [strings.RESOURCE_STATUS_UPDATING_FAILED]) + results["after_operations"] = await read_workspace(client, workspace_path, workspace_token) + results["history"] = await read_history(client, workspace_path, workspace_token) + LOGGER.info("Masking of the sensitive property: %s", results) + + assert results["after_install"]["masked"], "The workspace response does not mask the property after an install" + updates = [results[name]["masked"] for name in ("after_update_1", "after_update_2", "after_operations")] + assert all(updates), "The workspace response does not mask the property after an update" + assert results["history"]["equal_to_update"] == 0, "The workspace history does not mask the property" + finally: + await clean_up_test_workspace(pre_created_workspace_id="", workspace_path=workspace_path, verify=verify) From 961eae10ea9b61249e16e79e82bf7756fb74594e Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 18:42:09 +0000 Subject: [PATCH 10/37] Give the first step of a pipeline the real sensitive values (#5298) A template with a pipeline, for example the unrestricted workspace, builds the main step message from the document in Cosmos DB, which keeps REDACTED for each sensitive value. After an update of client_secret, the message then had REDACTED, and the Terraform of the workspace kept the old secret. The install of this template already had the same problem. - try_patch puts the real value back for each placeholder when the step updates the primary resource of a new request. The other values come from the stored document, so a change by another request stays. - create_operation_item gets the properties of the resource in memory. A cascade reads its resource list from Cosmos DB, so the flag in the pending dispatch entry came from the placeholder. - The migration reads all resources one time. It reads the history and changes only the resources of templates that have a sensitive property. - The pipeline overview names the limit: put the main step first in an action that sets a sensitive property. - The E2E case also asserts that each history item has the placeholder. - The texts describe the resend rule for an install and an update. --- CHANGELOG.md | 8 ++- api_app/db/repositories/operations.py | 9 ++- api_app/db/repositories/resources.py | 64 ++++++++++++++----- .../service_bus/deployment_status_updater.py | 6 +- api_app/service_bus/helpers.py | 8 ++- .../service_bus/resource_request_sender.py | 1 + .../test_operation_repository.py | 34 ++++++++++ .../test_resource_repository.py | 28 ++++++-- .../test_resource_request_sender.py | 36 +++++++++++ docs/tre-developers/resource-processor.md | 4 +- .../pipeline-templates/overview.md | 1 + e2e_tests/test_workspace_masked_update.py | 10 ++- 12 files changed, 172 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 560932f202..0437d75709 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,11 +61,13 @@ BUG FIXES: * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) * Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter does not have or that it uninstalled. (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) -* Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. An install with sensitive values fails with a clear message, because the API does not store these values. +* Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. + If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. (`api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Store a placeholder for each sensitive property when the API saves an update of a resource, as it already does at the install. The new value goes to the resource processor only in the request message. - Before this change, an update stored the new value of a sensitive property, for example the `client_secret` of a workspace, in Cosmos DB and in the resource history. API responses with the resource properties then included this value. - A database migration replaces the values that earlier updates stored. (`api` 0.28.10) + Before this change, an update stored the new value of a sensitive property in Cosmos DB and in the resource history. An example is the `client_secret` of a workspace. API responses with the resource properties then included this value. + A database migration replaces the values that earlier updates stored. If an update set the client secret of a workspace before this version, you can create a new secret for the workspace application. + The first message of a template with a pipeline, for example the unrestricted workspace, now also gets the real sensitive values at an install. (`api` 0.28.10) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index e7617900d4..4358344102 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -1,7 +1,7 @@ from datetime import datetime, UTC import re import uuid -from typing import List +from typing import List, Optional from azure.core import MatchConditions from azure.cosmos.exceptions import CosmosHttpResponseError @@ -115,7 +115,11 @@ async def create_operation_item( user: User, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, + resource_properties: Optional[dict] = None, ) -> Operation: + """resource_properties are the properties of the resource in memory, from which the API builds the first + message. A cascade reads resource_list from Cosmos DB, which keeps a placeholder for each sensitive value. + """ operation_id = self.create_operation_id() # get the right "awaiting" message based on the action @@ -138,7 +142,8 @@ async def create_operation_item( if resource["id"] == resource_id: # The API builds the first message from this resource, which can have real sensitive values. # Cosmos DB keeps a placeholder for each of them. - properties = resource.get("properties") or {} + properties = resource_properties if resource_properties is not None else resource.get("properties") + properties = properties or {} sensitive_values = mask_sensitive_properties(properties, resource_template) != properties resource_template_dict = resource_template.model_dump(exclude_none=True) # if the template has a pipeline defined for this action, copy over all the steps to the ops document diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index fdbceb48a3..67b0f286af 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -37,12 +37,11 @@ ) -def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: - updated_resource_parameters = copy.deepcopy(properties) - +def template_properties(template: dict) -> dict: + """Return each property schema of a template by name, also from allOf, then and else branches.""" flattened_template_props = {} - # recusrse the template and flatten all properties into a list + # Recurse through the template and collect all properties in one dict. def flatten_template_props(template_fragment: dict): if template_fragment is None: return @@ -63,15 +62,25 @@ def flatten_template_props(template_fragment: dict): if isinstance(prop, dict) and prop_name != "if": flatten_template_props(prop) - flatten_template_props(template.model_dump()) + flatten_template_props(template) + return flattened_template_props + + +def sensitive_property_names(template: dict) -> set: + return { + name + for name, prop in template_properties(template).items() + if isinstance(prop, dict) and prop.get("sensitive") is True + } + + +def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: + updated_resource_parameters = copy.deepcopy(properties) + sensitive_names = sensitive_property_names(template.model_dump()) def recurse_input_props(prop_dict: dict): for prop_name, prop in prop_dict.items(): - if ( - prop_name in flattened_template_props - and "sensitive" in flattened_template_props[prop_name] - and flattened_template_props[prop_name]["sensitive"] is True - ): + if prop_name in sensitive_names: prop_dict[prop_name] = REDACTED_SENSITIVE_VALUE if isinstance(prop, dict): recurse_input_props(prop) @@ -81,6 +90,18 @@ def recurse_input_props(prop_dict: dict): return updated_resource_parameters +def restore_sensitive_values(properties: dict, source: dict) -> dict: + """Return a copy of properties with the real value from source in place of each placeholder.""" + restored = copy.deepcopy(properties) + for key, value in restored.items(): + original = source.get(key) + if isinstance(value, dict) and isinstance(original, dict): + restored[key] = restore_sensitive_values(value, original) + elif value == REDACTED_SENSITIVE_VALUE and original is not None and original != REDACTED_SENSITIVE_VALUE: + restored[key] = copy.deepcopy(original) + return restored + + def masked_property_paths(properties: dict, masked: dict, path: Optional[List[str]] = None) -> List[str]: """Return a JSON pointer for each value that mask_sensitive_properties changed.""" path = path or ["properties"] @@ -306,6 +327,12 @@ async def mask_stored_sensitive_values( Returns the number of resource documents and history items that changed. A second run changes nothing. """ + # Only resources of a template with a sensitive property, in any of its versions, can have such a value. + names_with_sensitive_values = { + template["name"] + for template in await resource_template_repo.query(query="SELECT * FROM c") + if sensitive_property_names(template) + } resources = {resource["id"]: resource for resource in await self.query(query="SELECT * FROM c")} templates = {} @@ -339,17 +366,20 @@ async def mask(item: dict, template: Optional[ResourceTemplate], container, part return bool(paths) changed_resources = 0 + changed_history_items = 0 for resource in resources.values(): + if resource["templateName"] not in names_with_sensitive_values: + continue template = await template_for(resource, resource["templateVersion"]) changed_resources += await mask(resource, template, self.container, resource["id"]) - changed_history_items = 0 - for item in await resource_history_repo.query(query="SELECT * FROM c"): - resource = resources.get(item.get("resourceId")) - if resource is None: - continue - template = await template_for(resource, item.get("templateVersion") or resource["templateVersion"]) - changed_history_items += await mask(item, template, resource_history_repo.container, item["resourceId"]) + history_items = await resource_history_repo.query( + query="SELECT * FROM c WHERE c.resourceId = @resourceId", + parameters=[{"name": "@resourceId", "value": resource["id"]}], + ) + for item in history_items: + template = await template_for(resource, item.get("templateVersion") or resource["templateVersion"]) + changed_history_items += await mask(item, template, resource_history_repo.container, item["resourceId"]) return changed_resources, changed_history_items diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 0e0d998c42..1c3c398616 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -155,9 +155,9 @@ async def resend_pending_dispatch(self, document: dict): await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST) return - # The API built the first message of the request from the resource in memory, with the real sensitive - # values. Cosmos DB keeps a placeholder for each, so the updater cannot build that message again. The API - # builds all other messages from Cosmos DB, so a resend has the same values as the first send. + # If an install or an update set a sensitive property, the first message has the real value, and Cosmos DB + # keeps a placeholder. The updater cannot build that message again. Other first messages have the same + # values as Cosmos DB, so a resend has the same values as the first send. if pending.get("sensitiveValues") and step.resourceId == operation.resourceId: logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) diff --git a/api_app/service_bus/helpers.py b/api_app/service_bus/helpers.py index a0fd471f4c..a6d3143c06 100644 --- a/api_app/service_bus/helpers.py +++ b/api_app/service_bus/helpers.py @@ -10,7 +10,7 @@ from db.repositories.resource_templates import ResourceTemplateRepository from models.domain.authentication import User from models.schemas.resource import ResourcePatch -from db.repositories.resources import ResourceRepository +from db.repositories.resources import ResourceRepository, restore_sensitive_values from core import config, credentials from services.logging import logger from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -196,6 +196,12 @@ async def try_patch( primary_parent_workspace_svc: Resource, ) -> Resource: resource_to_update = await resource_repo.get_resource_by_id(resource_to_update_id) + if resource_to_update.id == primary_resource.id: + # Cosmos DB keeps a placeholder for each sensitive value. For a new request, the primary resource is the + # resource in memory, which has the real values for the message. + resource_to_update.properties = restore_sensitive_values( + resource_to_update.properties, primary_resource.properties + ) # substitute values into new property bag for update properties = substitute_properties( diff --git a/api_app/service_bus/resource_request_sender.py b/api_app/service_bus/resource_request_sender.py index f72b3c7c49..04d29a7b7f 100644 --- a/api_app/service_bus/resource_request_sender.py +++ b/api_app/service_bus/resource_request_sender.py @@ -52,6 +52,7 @@ async def send_resource_request_message( user=user, resource_repo=resource_repo, resource_template_repo=resource_template_repo, + resource_properties=resource.properties, ) current_span.set_attribute("operation_id", operation.id) diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 2a501564ed..8e738eb4ff 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -124,6 +124,40 @@ async def test_pending_entry_records_sensitive_values_of_the_first_message( assert document["pendingDispatch"]["sensitiveValues"] is expected +@patch("db.repositories.operations.OperationRepository.get_timestamp", return_value=FAKE_CREATE_TIMESTAMP) +@patch("db.repositories.operations.OperationRepository.create_operation_id", return_value=OPERATION_ID) +async def test_pending_entry_uses_the_resource_in_memory_for_a_cascade( + _, __, operations_repo, resource_repo, resource_template_repo, test_user, basic_resource_template +): + # A cascade reads the resource list from Cosmos DB, which keeps a placeholder for each sensitive value. + template = basic_resource_template.model_copy(deep=True) + template.properties["client_secret"] = {"type": "string", "sensitive": True, "updateable": True} + operations_repo.save_item_dict = AsyncMock() + resource_template_repo.get_template_by_name_and_version = AsyncMock(return_value=template) + stored = { + "id": RESOURCE_ID, + "templateName": template.name, + "templateVersion": template.version, + "resourceType": "workspace", + "properties": {"client_secret": strings.REDACTED_SENSITIVE_VALUE}, + } + + await operations_repo.create_operation_item( + resource_id=RESOURCE_ID, + resource_list=[stored], + action="upgrade", + resource_path=f"/workspaces/{RESOURCE_ID}", + resource_version=1, + user=test_user, + resource_repo=resource_repo, + resource_template_repo=resource_template_repo, + resource_properties={"client_secret": "real value"}, + ) + + document = operations_repo.save_item_dict.await_args.args[0] + assert document["pendingDispatch"]["sensitiveValues"] is True + + @pytest_asyncio.fixture async def operations_container(): container = AsyncMock() diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 3248a64ec6..9e199bd91f 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -489,17 +489,28 @@ async def get_template(name, version, resource_type, parent_service_name=None): raise EntityDoesNotExist return template + secret = {"properties": {"client_secret": {"type": "string", "sensitive": True}}} template_repo = AsyncMock() template_repo.get_template_by_name_and_version.side_effect = get_template - history_repo = MagicMock() - history_repo.container = AsyncMock() - history_repo.query = AsyncMock( - return_value=[ + template_repo.query.return_value = [ + {"name": "workspace-template", **secret}, + {"name": "user-template", **secret}, + {"name": "removed-template", **secret}, + {"name": "service-template", "properties": {"display_name": {"type": "string"}}}, + ] + history = { + "ws": [ {"id": "h-1", "resourceId": "ws", "templateVersion": "0.9.0", "properties": {"client_secret": "older"}}, {"id": "h-2", "resourceId": "ws", "properties": {"client_secret": "REDACTED"}}, - {"id": "h-3", "resourceId": "deleted-from-resources", "properties": {"client_secret": "older"}}, - ] - ) + ], + } + + async def history_query(query, parameters): + return history.get(parameters[0]["value"], []) + + history_repo = MagicMock() + history_repo.container = AsyncMock() + history_repo.query = AsyncMock(side_effect=history_query) result = await resource_repo.mask_stored_sensitive_values(template_repo, history_repo) @@ -513,6 +524,9 @@ async def get_template(name, version, resource_type, parent_service_name=None): history_repo.container.patch_item.assert_awaited_once_with( item="h-1", partition_key="ws", patch_operations=set_secret ) + # A resource of a template without a sensitive property needs no history read. + read_ids = [call.kwargs["parameters"][0]["value"] for call in history_repo.query.await_args_list] + assert read_ids == ["ws", "ws-2", "ur", "old"] template_repo.get_template_by_name_and_version.assert_any_await( "user-template", "1.0.0", ResourceType.UserResource, "service-template" ) diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index e586544fab..8117580dad 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -16,6 +16,7 @@ ) from models.domain.workspace_service import WorkspaceService from models.domain.resource import Resource, ResourceType +from models.domain.resource_template import PipelineStep from service_bus.resource_request_sender import ( send_resource_request_message, RequestAction, @@ -229,6 +230,41 @@ async def test_multi_step_document_retries( assert len(resource_repo.get_resource_by_id.mock_calls) == (num_retries + 1) +@pytest.mark.parametrize("same_resource, expected", [(True, "real value"), (False, strings.REDACTED_SENSITIVE_VALUE)]) +async def test_a_pipeline_step_of_the_primary_resource_gets_the_real_sensitive_values( + same_resource, expected, test_user +): + # A template with a pipeline builds the main step message from the document in Cosmos DB, which keeps a + # placeholder. The primary resource of a new request is the resource in memory, with the real value. + primary = create_test_resource() + primary.properties = {"display_name": "name", "client_secret": "real value"} + stored = primary.model_copy(deep=True) + stored.properties = {"display_name": "changed later", "client_secret": strings.REDACTED_SENSITIVE_VALUE} + if not same_resource: + stored.id = str(uuid.uuid4()) + resource_repo = AsyncMock() + resource_repo.get_resource_by_id.return_value = stored + resource_repo.patch_resource.side_effect = lambda **kwargs: (kwargs["resource"], None) + + sent = await try_update_with_retries( + num_retries=0, + attempt_count=0, + resource_repo=resource_repo, + resource_template_repo=AsyncMock(), + resource_history_repo=AsyncMock(), + user=test_user, + resource_to_update_id=stored.id, + template_step=PipelineStep(stepId="main", resourceAction="upgrade", properties=[]), + primary_resource=primary, + primary_parent_workspace=None, + primary_parent_workspace_svc=None, + ) + + assert sent.properties["client_secret"] == expected + # The stored document gives the other values, so a change by another request stays. + assert sent.properties["display_name"] == "changed later" + + async def send_first_step(resource_repo, operations_repo, resource_template_repo, resource_history_repo, template): resource = create_test_resource() operation = create_sample_operation(resource.id, RequestAction.Install) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 1be2b00515..dc0424eaae 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -150,10 +150,10 @@ If the API stops between the write and the send, the entry stays. When the API s 1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. 1. If a later request for the same resource exists, for example a delete, the updater does not send the message. It sets the step and the operation to the failed status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. -1. The API builds the first message of a request from the resource in memory. After an install, or after an update that changes a sensitive property, that message has real values that Cosmos DB keeps as `REDACTED`. +1. If an install or an update sets a sensitive property, the first message has the real value, and Cosmos DB keeps `REDACTED`. The operation write records this in the entry, and the updater cannot build such a message again. It sets the step and the operation to the failed status, with a message that asks the user to send the request again if the status does not change. - The API builds all other messages from Cosmos DB, so the updater sends them with the same values as the first send. + Other first messages have the same values as Cosmos DB, so the updater sends them with the same values as the first send. 1. If the updater cannot build the message, it sets the step and the operation to the failed status with the error text. A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. diff --git a/docs/tre-templates/pipeline-templates/overview.md b/docs/tre-templates/pipeline-templates/overview.md index 9165121884..3bedae9236 100644 --- a/docs/tre-templates/pipeline-templates/overview.md +++ b/docs/tre-templates/pipeline-templates/overview.md @@ -53,3 +53,4 @@ This feature is undergoing active development, and is currently limited in the f - Only statically addressable resources can be referred to - `shared_services`, as these are singletons and can be referenced by a template name. - Only the `upgrade` action for each secondary resource is supported. Support for `install` / `uninstall` of secondary resources is planned. - No current planned support for `customActions`. +- Cosmos DB keeps the value of a sensitive property as `REDACTED`. Only the first message of a request has the real value. So put the `main` step first in each action that sets a sensitive property, as the templates in this repository do. A `main` step after another step gets `REDACTED`. diff --git a/e2e_tests/test_workspace_masked_update.py b/e2e_tests/test_workspace_masked_update.py index 4ae183ea61..1e45438841 100644 --- a/e2e_tests/test_workspace_masked_update.py +++ b/e2e_tests/test_workspace_masked_update.py @@ -2,7 +2,8 @@ The case creates a workspace with the manual test application. A TRE admin then updates the client secret two times with the same value, and the workspace token reads the workspace and its history back. The case logs only -true or false values and counts. It never logs a property value or a response text. +true or false values, counts and the role names of the token. Its own code does not log a property value or a +response text. """ import base64 @@ -97,6 +98,11 @@ async def test_a_sensitive_property_stays_masked_after_an_update(verify): assert results["after_install"]["masked"], "The workspace response does not mask the property after an install" updates = [results[name]["masked"] for name in ("after_update_1", "after_update_2", "after_operations")] assert all(updates), "The workspace response does not mask the property after an update" - assert results["history"]["equal_to_update"] == 0, "The workspace history does not mask the property" + history = results["history"] + # Two updates of a workspace without a pipeline write two history items, each with the property. + assert history["items"] >= 2 and history["masked"] == history["items"], ( + "The workspace history does not mask the property" + ) + assert history["equal_to_update"] == 0, "The workspace history has the updated value" finally: await clean_up_test_workspace(pre_created_workspace_id="", workspace_path=workspace_path, verify=verify) From 02096e1026820f476213183974fab124f10773e0 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 19:43:37 +0000 Subject: [PATCH 11/37] Describe a schema error without the submitted values (#5298) The create routes of workspaces, workspace services, user resources and shared services logged the jsonschema ValidationError with its traceback and returned str(error) in the HTTP 400 response. That text contains the submitted properties, for example a client secret. The patch routes returned error.message, which can contain the value that failed. validation_error_detail names the property and the template rule. A validator whose message names properties only, and an error that the API raises itself, keep their message. The create routes log a warning with the same text and keep the traceback for other ValueErrors. --- CHANGELOG.md | 2 + api_app/api/routes/resource_helpers.py | 24 ++++++++++ api_app/api/routes/shared_services.py | 9 +++- api_app/api/routes/workspaces.py | 25 +++++++--- .../test_routes/test_resource_helpers.py | 47 +++++++++++++++++++ .../test_api/test_routes/test_workspaces.py | 22 +++++++++ 6 files changed, 121 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0437d75709..a9b17d9d1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,8 @@ BUG FIXES: Before this change, an update stored the new value of a sensitive property in Cosmos DB and in the resource history. An example is the `client_secret` of a workspace. API responses with the resource properties then included this value. A database migration replaces the values that earlier updates stored. If an update set the client secret of a workspace before this version, you can create a new secret for the workspace application. The first message of a template with a pipeline, for example the unrestricted workspace, now also gets the real sensitive values at an install. (`api` 0.28.10) +* Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. + That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. (`api` 0.28.10) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index 6519eb803a..a3e762026d 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -4,6 +4,7 @@ from typing import List, Optional from fastapi import HTTPException, status +from jsonschema.exceptions import ValidationError from db.repositories.user_resources import UserResourceRepository from models.domain.user_resource import UserResource from models.domain.workspace_service import WorkspaceService @@ -126,6 +127,29 @@ async def save_and_deploy_resource( ) +# The messages of these validators name properties only, not the submitted values. +NAME_ONLY_VALIDATORS = {"required", "additionalProperties", "unevaluatedProperties", "dependentRequired"} +# The rule of these validators is a list of schemas, which is too long for a message. +SCHEMA_LIST_VALIDATORS = {"allOf", "anyOf", "oneOf", "not", "if", "then", "else"} + + +def validation_error_detail(error: ValidationError) -> str: + """Describe a schema error without the submitted values, because a value can be a secret. + + str(error) contains the whole input, and error.message can contain the value that failed. + """ + if not isinstance(error.validator, str) or error.validator in NAME_ONLY_VALIDATORS: + # An error that the API raises itself has no validator and a message without input values. + return error.message + location = "/".join(str(part) for part in error.absolute_path) or "properties" + if error.validator in SCHEMA_LIST_VALIDATORS: + return f"'{location}' does not match the '{error.validator}' rule of the template" + rule = repr(error.validator_value) + if len(rule) > 200: + rule = rule[:200] + "..." + return f"'{location}' does not match the '{error.validator}' rule of the template: {rule}" + + def construct_location_header(operation: Operation) -> str: return f"/api{operation.resourcePath}/operations/{operation.id}" diff --git a/api_app/api/routes/shared_services.py b/api_app/api/routes/shared_services.py index 3194e8370c..387d367157 100644 --- a/api_app/api/routes/shared_services.py +++ b/api_app/api/routes/shared_services.py @@ -29,6 +29,7 @@ from .workspaces import save_and_deploy_resource, construct_location_header from azure.cosmos.exceptions import CosmosAccessConditionFailedError from .resource_helpers import ( + validation_error_detail, enrich_resource_with_available_upgrades, enrich_resources_with_available_upgrades, send_custom_action_message, @@ -106,7 +107,11 @@ async def create_shared_service( shared_service, resource_template = await shared_services_repo.create_shared_service_item( shared_service_input, user.roles ) - except (ValidationError, ValueError) as e: + except ValidationError as e: + # Do not log or return str(e). It contains the submitted properties, which can be secrets. + logger.warning(f"Shared service input does not match the template: {validation_error_detail(e)}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(e)) + except ValueError as e: logger.exception("Failed create shared service model instance") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except DuplicateEntity as e: @@ -174,7 +179,7 @@ async def patch_shared_service( except CosmosAccessConditionFailedError: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(v)) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py index a764736604..1b0ea84e8b 100644 --- a/api_app/api/routes/workspaces.py +++ b/api_app/api/routes/workspaces.py @@ -64,6 +64,7 @@ from azure.cosmos.exceptions import CosmosAccessConditionFailedError from .resource_helpers import ( + validation_error_detail, cascaded_update_resource, delete_validation, enrich_resource_with_available_upgrades, @@ -186,7 +187,11 @@ async def create_workspace( ) # Validate the resolved (template-derived) airlock version rather than the raw request. ensure_workspace_airlock_version_supported(workspace.properties, default_version=1) - except (ValidationError, ValueError) as e: + except ValidationError as e: + # Do not log or return str(e). It contains the submitted properties, which can be secrets. + logger.warning(f"Workspace input does not match the template: {validation_error_detail(e)}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(e)) + except ValueError as e: logger.exception("Failed to create workspace model instance") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except StorageAccountNameGenerationTimeout: @@ -327,7 +332,7 @@ async def patch_workspace( if isinstance(error, CosmosAccessConditionFailedError): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) if isinstance(error, ValidationError): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=error.message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(error)) if isinstance( error, (ValueError, MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) ): @@ -534,7 +539,11 @@ async def create_workspace_service( workspace_service, resource_template = await workspace_service_repo.create_workspace_service_item( workspace_service_input, workspace.id, user.roles ) - except (ValidationError, ValueError) as e: + except ValidationError as e: + # Do not log or return str(e). It contains the submitted properties, which can be secrets. + logger.warning(f"Workspace service input does not match the template: {validation_error_detail(e)}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(e)) + except ValueError as e: logger.exception("Failed create workspace service model instance") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except UserNotAuthorizedToUseTemplate as e: @@ -641,7 +650,7 @@ async def patch_workspace_service( except CosmosAccessConditionFailedError: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(v)) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @@ -853,7 +862,11 @@ async def create_user_resource( user.roles, owner_id, ) - except (ValidationError, ValueError) as e: + except ValidationError as e: + # Do not log or return str(e). It contains the submitted properties, which can be secrets. + logger.warning(f"User resource input does not match the template: {validation_error_detail(e)}") + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(e)) + except ValueError as e: logger.exception("Failed create user resource model instance") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except UserNotAuthorizedToUseTemplate as e: @@ -955,7 +968,7 @@ async def patch_user_resource( except CosmosAccessConditionFailedError: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT) except ValidationError as v: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=v.message) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=validation_error_detail(v)) except (MajorVersionUpdateDenied, TargetTemplateVersionDoesNotExist, VersionDowngradeDenied) as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index ac15a91b96..2072d09553 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -14,7 +14,10 @@ mask_sensitive_properties, enrich_resource_with_available_upgrades, enrich_resources_with_available_upgrades, + validation_error_detail, ) +from jsonschema import validate +from jsonschema.exceptions import ValidationError from db.repositories.resources_history import ResourceHistoryRepository from tests_ma.test_api.conftest import create_test_user from resources import strings @@ -405,3 +408,47 @@ def test_sensitive_properties_get_masked(self, basic_resource_template): assert masked_resource["client_id"] == "12345" assert masked_resource["secret"] == strings.REDACTED_SENSITIVE_VALUE assert masked_resource["prop_with_nested_secret"]["nested_secret"] == strings.REDACTED_SENSITIVE_VALUE + + +def schema_error(instance, schema): + try: + validate(instance=instance, schema=schema) + except ValidationError as error: + return error + raise AssertionError("The instance matches the schema") + + +@pytest.mark.parametrize( + "instance, schema, expected", + [ + ( + {"client_secret": ["secret-value"]}, + {"properties": {"client_secret": {"type": "string"}}}, + "'client_secret' does not match the 'type' rule of the template: 'string'", + ), + ( + {"client_secret": "secret-value"}, + {"properties": {"client_secret": {"pattern": "^a"}}}, + "'client_secret' does not match the 'pattern' rule of the template: '^a'", + ), + ( + {"client_secret": "secret-value"}, + {"properties": {"client_secret": {"anyOf": [{"type": "integer"}, {"type": "boolean"}]}}}, + "'client_secret' does not match the 'anyOf' rule of the template", + ), + ({"client_secret": "secret-value"}, {"required": ["client_id"]}, "'client_id' is a required property"), + ], +) +def test_validation_error_detail_has_no_submitted_value(instance, schema, expected): + error = schema_error(instance, schema) + # This is the reason for the helper: the jsonschema text has the submitted values. + assert "secret-value" in str(error) + + detail = validation_error_detail(error) + + assert detail == expected + assert "secret-value" not in detail + + +def test_validation_error_detail_keeps_the_message_of_an_api_error(): + assert validation_error_detail(ValidationError("Version 2.0.0 is not valid")) == "Version 2.0.0 is not valid" diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index cd0a26e850..e1b6bf27b4 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -4,6 +4,8 @@ import uuid from pydantic import Field import pytest +from jsonschema import validate +from jsonschema.exceptions import ValidationError from mock import patch from fastapi import status @@ -678,6 +680,26 @@ async def test_post_workspaces_returns_503_if_storage_check_fails_with_http_erro assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE assert "Storage name availability check failed. Please try again." in response.text + @patch("api.routes.workspaces.logger") + @patch("api.routes.workspaces.WorkspaceRepository.create_workspace_item") + @patch("api.routes.workspaces.extract_auth_information") + async def test_post_workspaces_returns_400_without_the_submitted_values( + self, _, mock_create_workspace_item, logger_mock, app, client, workspace_input + ): + submitted = {"client_secret": "secret-value", "display_name": 5} + try: + validate(instance=submitted, schema={"properties": {"display_name": {"type": "string"}}}) + except ValidationError as error: + mock_create_workspace_item.side_effect = error + + response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE), json=workspace_input) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "display_name" in response.text + assert "secret-value" not in response.text + assert "secret-value" not in str(logger_mock.mock_calls) + logger_mock.exception.assert_not_called() + # [PATCH] /workspaces/{workspace_id} @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") @patch("api.routes.workspaces.WorkspaceRepository.patch_workspace", return_value=None) From b0e0e810d487d0a78107ea32b83197e160b058a1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 21:05:47 +0000 Subject: [PATCH 12/37] Check the session after the lease wait and let an upgrade repair a lost install (#5236) - A worker can lose its Service Bus session while it waits for the execution lease. Then another worker can have the message, and a later step for the resource can finish first. The record holds only the last step, so the waiting worker ran Porter for the old step again. The worker now renews the session lock when it gets the lease. If the session is lost, it does not run Porter, and the lease is released. - An install can fail before Porter writes its installation record, for example at the sign-in or in an earlier pipeline step. The upgrade guard refused an upgrade without a record, so an update could not repair such an install (#4281, fixed by #4338). The guard now refuses only an installation whose last successful action was an uninstall. --- resource_processor/tests_rp/test_run_step.py | 7 +++-- resource_processor/tests_rp/test_runner.py | 19 ++++++++++++ resource_processor/vmss_porter/runner.py | 31 +++++++++++++++----- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/resource_processor/tests_rp/test_run_step.py b/resource_processor/tests_rp/test_run_step.py index 8527a01799..a5183fe481 100644 --- a/resource_processor/tests_rp/test_run_step.py +++ b/resource_processor/tests_rp/test_run_step.py @@ -166,7 +166,7 @@ async def test_only_an_upgrade_requires_the_installation(action, required): @pytest.mark.asyncio @pytest.mark.parametrize("absent, porter_runs", [(True, False), (False, True)]) -async def test_an_upgrade_does_not_install_a_missing_resource(absent, porter_runs): +async def test_an_upgrade_does_not_install_a_deleted_resource(absent, porter_runs): commands = [] async def helper(command, config, *args, **kwargs): @@ -186,7 +186,7 @@ async def helper(command, config, *args, **kwargs): assert (["porter", "installations", "apply"] in commands) is porter_runs if absent: assert code == 1 - assert "An upgrade does not install a missing resource" in error + assert "An upgrade does not install a deleted resource" in error else: assert code == 0 @@ -199,7 +199,8 @@ def installation(installed=None, uninstalled=None, name="resource-1"): @pytest.mark.parametrize( "output, removed", [ - ([], True), + # An install can fail before Porter writes a record. An upgrade can repair it. + ([], False), # Porter keeps the record after an uninstall, and `installations apply` then runs the install action. ([installation("2026-10-29T10:00:00.123456789Z", "2026-10-29T11:00:00.5Z")], True), ([installation(None, "2026-10-29T11:00:00Z")], True), diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 8aa876d63f..534dd980d4 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -1238,6 +1238,25 @@ async def test_a_lost_session_lock_at_completion_is_a_warning( mock_sleep.assert_not_awaited() +@pytest.mark.asyncio +@patch("vmss_porter.runner.run_step", return_value=True) +async def test_a_session_lost_during_the_lease_wait_runs_no_porter( + mock_run_step, mock_service_bus_client, mock_execution_lease, mock_keep_renewed, mock_logger +): + # Another worker can have the message now, and a later step for the resource can have finished. + client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) + receiver.session.renew_lock.side_effect = SessionLockLostError() + + await receive_message( + client, {"resource_request_queue": "q"}, AsyncMock(), keep_running=Mock(side_effect=[True, False]) + ) + + mock_run_step.assert_not_awaited() + receiver.complete_message.assert_not_awaited() + assert "lost its session while it waited for the lease" in mock_logger.warning.call_args.args[0] + mock_logger.exception.assert_not_called() + + @pytest.mark.asyncio @pytest.mark.parametrize( "test_setting, running, paused", [(False, True, False), (True, False, False), (True, True, True)] diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 75763ba6f8..cdb64f0704 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -30,7 +30,12 @@ from contextlib import asynccontextmanager from azure.core.exceptions import HttpResponseError from azure.servicebus import ServiceBusMessage, NEXT_AVAILABLE_SESSION -from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError, SessionLockLostError +from azure.servicebus.exceptions import ( + OperationTimeoutError, + ServiceBusConnectionError, + ServiceBusError, + SessionLockLostError, +) from azure.servicebus.aio import ServiceBusClient from azure.identity.aio import DefaultAzureCredential from azure.storage.blob.aio import ContainerClient @@ -179,6 +184,16 @@ async def receive_message(service_bus_client, config: dict, executions: Containe async with execution_lease( executions, message["id"], lambda: step["text"] ) as execution: + # The session can end while the worker waits for the lease. Then Service Bus + # gives this message to another worker, and a later step for the resource can + # finish before this worker gets the lease. So run Porter only with the session. + try: + await session.renew_lock() + except ServiceBusError: + logger.warning( + f"The worker lost its session while it waited for the lease. Porter does not run for {step['text']}" + ) + break step["running"] = True try: result = await run_step(message, execution, service_bus_client, config) @@ -326,14 +341,14 @@ async def run_porter( if error: return (1, None, error) - # `porter installations apply` installs a missing or uninstalled installation. An upgrade must not create the + # `porter installations apply` installs an uninstalled installation. An upgrade must not create a deleted # resource again. if required_installation is not None and await _confirm_porter_installation_removed(required_installation, config): return ( 1, None, - f"Cannot upgrade resource '{required_installation}': Porter has no installation for it, " - "or its last successful action was an uninstall. An upgrade does not install a missing resource.", + f"Cannot upgrade resource '{required_installation}': the last successful action of its Porter " + "installation was an uninstall. An upgrade does not install a deleted resource.", ) last_returncode = None @@ -442,15 +457,15 @@ def _porter_time(value) -> Optional[datetime]: async def _confirm_porter_installation_removed(installation_id: str, config: dict) -> bool: """ - Confirm that Porter has no installation, or that its last successful action was an uninstall. + Confirm that the last successful action of the Porter installation was an uninstall. Porter keeps the record of an uninstalled installation. A failed install has neither time, so an upgrade can repair it. + An install can also fail before Porter writes a record, for example at the sign-in or in an earlier pipeline step. + So no record proves nothing, and an upgrade can repair that install too. """ installations = await _list_porter_installations(installation_id, config) - if installations == []: - return True matches = [item for item in installations or [] if item.get("name") == installation_id] if not matches: - # An unreadable store, or a list with other installations only, proves nothing. + # No record, an unreadable store, or a list with other installations only, proves nothing. return False status = matches[0].get("status") or {} uninstalled = _porter_time(status.get("uninstalled")) From 867049e138e5f5e3505be890640fc43b9e240f17 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 21:06:06 +0000 Subject: [PATCH 13/37] Fail an unsent first step without a change to other resources (#5236) - The check for a later request now also finds a cascade request of a parent that has a step for the resource. Before, the resender could install a child after a cascade delete of its workspace. - fail_pending_step writes the operation first, so the write removes the entry even if a later read fails. After a later request, it changes only the operation, because the later request owns the resource status. In the other cases, it sets only the resource of the request to the failed status. A first step on another resource, for example the firewall or the parent workspace, did not run. - validation_error_detail also covers a false subschema, whose message repeats the submitted value. The Airlock review VM route uses it, so its HTTP 500 response and log have no SAS URL. - The route test for a schema error now uses an error whose text has the whole input. A sender test makes sure that the operation gets the properties of the resource in memory. - api 0.29.0, because the pull request adds API features (AGENTS.md). - The resource processor document and the CHANGELOG describe the changes. --- CHANGELOG.md | 19 +++++---- api_app/_version.py | 2 +- api_app/api/routes/airlock.py | 12 +++++- api_app/api/routes/resource_helpers.py | 7 +++- api_app/db/repositories/operations.py | 13 ++++++ .../service_bus/deployment_status_updater.py | 37 +++++++++------- .../test_api/test_routes/test_airlock.py | 34 +++++++++++++++ .../test_routes/test_resource_helpers.py | 5 +++ .../test_api/test_routes/test_workspaces.py | 8 ++-- .../test_operation_repository.py | 15 +++++++ .../test_service_bus/test_pending_dispatch.py | 42 ++++++++++++++++++- .../test_resource_request_sender.py | 27 ++++++++++++ docs/tre-developers/resource-processor.md | 15 ++++--- 13 files changed, 199 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9b17d9d1b..f24d58749b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ENHANCEMENTS: * Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) -* Add a `Request-Id` header with the trace ID to each API response, also to an HTTP 500. The E2E operation poll already logs this header, so a failed poll can be found in App Insights. (`api` 0.28.10) ([#5202](https://github.com/microsoft/AzureTRE/issues/5202)) +* Add a `Request-Id` header with the trace ID to each API response, also to an HTTP 500. The E2E operation poll already logs this header, so a failed poll can be found in App Insights. (`api` 0.29.0) ([#5202](https://github.com/microsoft/AzureTRE/issues/5202)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) @@ -59,17 +59,22 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) -* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter does not have or that it uninstalled. - (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) +* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. + An upgrade no longer installs a resource that Porter uninstalled. A worker that lost its session while it waited for the lease does not run Porter. + (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. - (`api` 0.28.10) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) + The resend does not change a resource that a later request changed, and it does not change the resource of a first step on another resource. + (`api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Store a placeholder for each sensitive property when the API saves an update of a resource, as it already does at the install. The new value goes to the resource processor only in the request message. - Before this change, an update stored the new value of a sensitive property in Cosmos DB and in the resource history. An example is the `client_secret` of a workspace. API responses with the resource properties then included this value. + Before this change, an update stored the new value of a sensitive property in Cosmos DB and in the resource history. An example is the `client_secret` of a workspace. + API responses with the resource properties then included this value. A database migration replaces the values that earlier updates stored. If an update set the client secret of a workspace before this version, you can create a new secret for the workspace application. - The first message of a template with a pipeline, for example the unrestricted workspace, now also gets the real sensitive values at an install. (`api` 0.28.10) + Then update the workspace with the new secret, and delete the old secret. + For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. - That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. (`api` 0.28.10) + The create route of an Airlock review VM did the same in its HTTP 500 response. + That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. (`api` 0.29.0) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/api_app/_version.py b/api_app/_version.py index 3429bc0f86..f634f5ba5a 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.10" +__version__ = "0.29.0" diff --git a/api_app/api/routes/airlock.py b/api_app/api/routes/airlock.py index 8c00faa2c0..5817d45e3c 100644 --- a/api_app/api/routes/airlock.py +++ b/api_app/api/routes/airlock.py @@ -32,7 +32,7 @@ require_airlock_manager, ) -from .resource_helpers import construct_location_header +from .resource_helpers import construct_location_header, validation_error_detail from services.airlock import ( create_review_vm, @@ -276,7 +276,15 @@ async def create_review_user_resource( detail=f"Failed to retrieve Airlock Review configuration for workspace {workspace.id}.\ Please ask your TRE administrator to check the configuration. Details: {str(e)}", ) - except (ValidationError, ValueError) as e: + except ValidationError as e: + # Do not log or return str(e). It contains the input of the review VM, with the SAS URL of the request. + logger.warning(f"The review VM input does not match the template: {validation_error_detail(e)}") + raise HTTPException( + status_code=status_code.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Invalid configuration for creating user resource. Please contact your TRE administrator. " + f"Details: {validation_error_detail(e)}", + ) + except ValueError as e: logger.exception("Failed create user resource model instance due to validation error") raise HTTPException( status_code=status_code.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/api_app/api/routes/resource_helpers.py b/api_app/api/routes/resource_helpers.py index a3e762026d..ecdadddf37 100644 --- a/api_app/api/routes/resource_helpers.py +++ b/api_app/api/routes/resource_helpers.py @@ -136,12 +136,15 @@ async def save_and_deploy_resource( def validation_error_detail(error: ValidationError) -> str: """Describe a schema error without the submitted values, because a value can be a secret. - str(error) contains the whole input, and error.message can contain the value that failed. + str(error) can contain the whole input, and error.message can contain the value that failed. """ + location = "/".join(str(part) for part in error.absolute_path) or "properties" + if error.validator is None: + # A false subschema allows no value. Its message repeats the submitted value. + return f"The template does not allow a value at '{location}'" if not isinstance(error.validator, str) or error.validator in NAME_ONLY_VALIDATORS: # An error that the API raises itself has no validator and a message without input values. return error.message - location = "/".join(str(part) for part in error.absolute_path) or "properties" if error.validator in SCHEMA_LIST_VALIDATORS: return f"'{location}' does not match the '{error.validator}' rule of the template" rule = repr(error.validator_value) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index 4358344102..d688226d87 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -334,6 +334,19 @@ async def get_operations_by_resource_id(self, resource_id: str) -> List[Operatio operations = await self.query(query=query) return TypeAdapter(List[Operation]).validate_python(operations) + async def get_later_operations_for_resource(self, resource_id: str, created_after: float) -> List[Operation]: + """Return the operations after a time on the resource, or with a step for it, for example a cascade.""" + query = ( + "SELECT * FROM c WHERE c.createdWhen > @createdAfter AND (c.resourceId = @resourceId" + " OR EXISTS(SELECT VALUE s FROM s IN c.steps WHERE s.resourceId = @resourceId))" + ) + parameters = [ + {"name": "@createdAfter", "value": created_after}, + {"name": "@resourceId", "value": resource_id}, + ] + operations = await self.query(query=query, parameters=parameters) + return TypeAdapter(List[Operation]).validate_python(operations) + async def resource_has_deployed_operation(self, resource_id: str) -> bool: query = ( self.operations_query() diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 1c3c398616..7ffcb3c2e4 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -144,15 +144,11 @@ async def resend_pending_dispatch(self, document: dict): await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) return - later_requests = [ - item - for item in await self.operations_repo.get_operations_by_resource_id(operation.resourceId) - if item.createdWhen > operation.createdWhen - ] - if later_requests: - # For example, a delete of a resource whose install was not sent. The install must not come after it. + # A later request on the resource, or a cascade from a parent with a step for it, for example a delete of a + # resource whose install was not sent. The install must not come after it. + if await self.operations_repo.get_later_operations_for_resource(operation.resourceId, operation.createdWhen): logger.warning(f"Operation {operation.id} was not sent before a later request. Failing step {step.id}") - await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST) + await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST, write_resource=False) return # If an install or an update set a sensitive property, the first message has the real value, and Cosmos DB @@ -193,19 +189,30 @@ async def resend_pending_dispatch(self, document: dict): ) await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) - async def fail_pending_step(self, operation: Operation, step: OperationStep, message: str): + async def fail_pending_step( + self, operation: Operation, step: OperationStep, message: str, write_resource: bool = True + ): + """Fail a step that the API did not send. The step did not run, so only its request fails.""" step.status = self.get_failure_status_for_action(step.resourceAction) step.message = message step.updatedWhen = get_timestamp() - await self.update_overall_operation_status(operation, step, is_last_step=len(operation.steps) == 1) + operation.status = self.get_failure_status_for_action(operation.action) + operation.message = message + operation.updatedWhen = step.updatedWhen # The Operation model has no pending dispatch, so this write also removes the entry. await self.operations_repo.update_item(operation) - resource = await self.resource_repo.get_resource_by_id(uuid.UUID(step.resourceId)) - resource.deploymentStatus = step.status + if not write_resource: + # A later request changes the resource, so it owns the status of the resource. + return + # Only the resource of the request shows the failure. A first step on another resource, for example the + # firewall or the parent workspace, did not run, so that resource keeps its status. + try: + resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) + except EntityDoesNotExist: + logger.warning(f"Resource {operation.resourceId} of operation {operation.id} does not exist") + return + resource.deploymentStatus = operation.status await self.resource_repo.update_item(resource) - if step.resourceId != operation.resourceId: - # The root resource shows the status of the operation. - await self.update_primary_uninstall_status(operation) async def process_message(self, msg): complete_message = False diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index 373f290d06..9208e8603a 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -1,5 +1,7 @@ import time import pytest +from jsonschema import validate +from jsonschema.exceptions import ValidationError from mock import patch from fastapi import status from azure.core.exceptions import HttpResponseError @@ -776,6 +778,38 @@ async def test_post_create_review_user_resource_returns_422_if_cannot_find_works ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + @patch("api.routes.airlock.logger") + @patch("api.routes.airlock.create_review_vm") + @patch( + "services.airlock.AirlockRequestRepository.read_item_by_id", + return_value=sample_airlock_request_object(status=AirlockRequestStatus.InReview), + ) + @patch( + "api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id", return_value=sample_workspace() + ) + async def test_post_create_review_user_resource_hides_the_input_of_a_schema_error( + self, _, __, create_review_vm_mock, logger_mock, app, client + ): + try: + validate(instance={"airlock_request_sas_url": "https://sas-value"}, schema={"required": ["os_image"]}) + except ValidationError as error: + # The review VM input has the SAS URL of the request, and the text of a root error has the whole input. + assert "sas-value" in str(error) + create_review_vm_mock.side_effect = error + + response = await client.post( + app.url_path_for( + strings.API_CREATE_AIRLOCK_REVIEW_USER_RESOURCE, + workspace_id=WORKSPACE_ID, + airlock_request_id=AIRLOCK_REQUEST_ID, + ) + ) + + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert "os_image" in response.text + assert "sas-value" not in response.text + assert "sas-value" not in str(logger_mock.mock_calls) + @patch( "services.airlock.AirlockRequestRepository.read_item_by_id", return_value=sample_airlock_request_object(status=AirlockRequestStatus.Draft), diff --git a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py index 2072d09553..541e46e82e 100644 --- a/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py +++ b/api_app/tests_ma/test_api/test_routes/test_resource_helpers.py @@ -437,6 +437,11 @@ def schema_error(instance, schema): "'client_secret' does not match the 'anyOf' rule of the template", ), ({"client_secret": "secret-value"}, {"required": ["client_id"]}, "'client_id' is a required property"), + ( + {"client_secret": "secret-value"}, + {"properties": {"client_secret": False}}, + "The template does not allow a value at 'properties'", + ), ], ) def test_validation_error_detail_has_no_submitted_value(instance, schema, expected): diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index e1b6bf27b4..7b57665dc9 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -686,16 +686,18 @@ async def test_post_workspaces_returns_503_if_storage_check_fails_with_http_erro async def test_post_workspaces_returns_400_without_the_submitted_values( self, _, mock_create_workspace_item, logger_mock, app, client, workspace_input ): - submitted = {"client_secret": "secret-value", "display_name": 5} + submitted = {"client_secret": "secret-value", "display_name": "name"} try: - validate(instance=submitted, schema={"properties": {"display_name": {"type": "string"}}}) + validate(instance=submitted, schema={"required": ["client_id"]}) except ValidationError as error: + # The text of a root error has the whole input, so the old route returned and logged the secret. + assert "secret-value" in str(error) mock_create_workspace_item.side_effect = error response = await client.post(app.url_path_for(strings.API_CREATE_WORKSPACE), json=workspace_input) assert response.status_code == status.HTTP_400_BAD_REQUEST - assert "display_name" in response.text + assert "client_id" in response.text assert "secret-value" not in response.text assert "secret-value" not in str(logger_mock.mock_calls) logger_mock.exception.assert_not_called() diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 8e738eb4ff..1dcf10d111 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -158,6 +158,21 @@ async def test_pending_entry_uses_the_resource_in_memory_for_a_cascade( assert document["pendingDispatch"]["sensitiveValues"] is True +async def test_later_operations_include_a_cascade_with_a_step_for_the_resource(operations_repo): + operations_repo.query = AsyncMock(return_value=[]) + + await operations_repo.get_later_operations_for_resource(RESOURCE_ID, 10.0) + + kwargs = operations_repo.query.await_args.kwargs + assert "c.createdWhen > @createdAfter" in kwargs["query"] + assert "c.resourceId = @resourceId" in kwargs["query"] + assert "EXISTS(SELECT VALUE s FROM s IN c.steps WHERE s.resourceId = @resourceId)" in kwargs["query"] + assert kwargs["parameters"] == [ + {"name": "@createdAfter", "value": 10.0}, + {"name": "@resourceId", "value": RESOURCE_ID}, + ] + + @pytest_asyncio.fixture async def operations_container(): container = AsyncMock() diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index 554b9171c9..10ddf9600a 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -1,9 +1,11 @@ import asyncio import json +import uuid from unittest.mock import AsyncMock, patch import pytest +from db.errors import EntityDoesNotExist from db.repositories.operations import PENDING_DISPATCH from models.domain.operation import Status from models.domain.workspace import Workspace @@ -18,6 +20,7 @@ pytestmark = pytest.mark.asyncio RESOURCE_ID = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" +FIREWALL_ID = "a8b2c3d4-1111-4222-8333-944455556666" MESSAGE_ID = f"{OPERATION_ID}:random-uuid" AWAITING = { "install": Status.AwaitingDeployment, @@ -61,7 +64,7 @@ def updater(): updater.resource_history_repo = AsyncMock() updater.operations_repo.claim_pending_dispatch.return_value = True updater.resource_repo.get_resource_by_id.return_value = workspace({}) - updater.operations_repo.get_operations_by_resource_id.return_value = [] + updater.operations_repo.get_later_operations_for_resource.return_value = [] return updater @@ -163,15 +166,50 @@ async def test_a_later_request_for_the_resource_fails_the_step(update_mock, send document = pending_document() later = create_sample_operation(RESOURCE_ID, "uninstall") later.createdWhen = document["createdWhen"] + 60 - updater.operations_repo.get_operations_by_resource_id.return_value = [later] + updater.operations_repo.get_later_operations_for_resource.return_value = [later] await updater.resend_pending_dispatch(document) + updater.operations_repo.get_later_operations_for_resource.assert_awaited_once_with( + RESOURCE_ID, document["createdWhen"] + ) update_mock.assert_not_awaited() send_mock.assert_not_awaited() operation = updater.operations_repo.update_item.await_args.args[0] assert operation.steps[0].status == Status.DeploymentFailed assert operation.steps[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST + assert operation.status == Status.DeploymentFailed + # The later request owns the status of the resource, for example a resource that it deleted. + updater.resource_repo.update_item.assert_not_awaited() + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_failed_first_step_on_another_resource_changes_only_the_resource_of_the_request( + update_mock, send_mock, updater +): + # For example, the first step of an uninstall pipeline changes the firewall. That step did not run. + document = pending_document() + document["steps"][0]["resourceId"] = FIREWALL_ID + update_mock.side_effect = Exception("template not found") + + await updater.resend_pending_dispatch(document) + + updater.resource_repo.get_resource_by_id.assert_awaited_once_with(uuid.UUID(RESOURCE_ID)) + assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == Status.DeploymentFailed + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_missing_resource_still_removes_the_entry(update_mock, send_mock, updater): + update_mock.side_effect = Exception("template not found") + updater.resource_repo.get_resource_by_id.side_effect = EntityDoesNotExist + + await updater.resend_pending_dispatch(pending_document()) + + # The operation write removes the entry, so the next pass does not claim it again. + assert updater.operations_repo.update_item.await_args.args[0].status == Status.DeploymentFailed + updater.resource_repo.update_item.assert_not_awaited() @patch("service_bus.deployment_status_updater.send_deployment_message") diff --git a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py index 8117580dad..20e7f4c7ad 100644 --- a/api_app/tests_ma/test_service_bus/test_resource_request_sender.py +++ b/api_app/tests_ma/test_service_bus/test_resource_request_sender.py @@ -340,3 +340,30 @@ async def test_a_failed_clear_after_the_send_returns_the_operation( # The entry stays, and the deployment status updater sends the message again. A second send is safe. sent_message = service_bus_client_mock().get_queue_sender().send_messages.call_args.args[0] assert sent_message.message_id == f"{operation.id}:random-uuid" + + +@patch("service_bus.resource_request_sender.ResourceHistoryRepository.create") +@patch("service_bus.resource_request_sender.OperationRepository.create") +@patch("service_bus.helpers.ServiceBusClient") +@patch("service_bus.resource_request_sender.ResourceRepository.create") +@patch("service_bus.resource_request_sender.ResourceTemplateRepository.create") +async def test_the_operation_gets_the_properties_of_the_resource_in_memory( + resource_template_repo, + resource_repo, + service_bus_client_mock, + operations_repo_mock, + resource_history_repo_mock, + multi_step_resource_template, +): + # A cascade reads the resource list from Cosmos DB, which keeps a placeholder for each sensitive value. + service_bus_client_mock().get_queue_sender().send_messages = AsyncMock() + await send_first_step( + resource_repo, + operations_repo_mock, + resource_template_repo, + resource_history_repo_mock, + multi_step_resource_template, + ) + + kwargs = operations_repo_mock.create_operation_item.await_args.kwargs + assert kwargs["resource_properties"] == {"testParameter": "testValue"} diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index dc0424eaae..fa9c5a3627 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -125,12 +125,14 @@ Service Bus can give the same message to more than one worker. The resource proc 1. The `workspacequeue` queue keeps a session lock for 5 minutes. While a worker runs a step, it renews the lock every minute. If a renewal fails, the worker tries again until the lock expires, and logs each failure. 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. + When it gets the lease, it renews the session lock. If the session is lost, the worker does not run Porter, because another worker can have the message. 1. The blob holds a record of the last step for the resource. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. -1. Each status message has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. +1. Each status message of a Porter run has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. -1. An upgrade does not install a resource that Porter does not have. The resource processor sends `updating_failed` if Porter has no installation for the resource, or if the last successful action of the installation was an uninstall. Porter runs the install action in both cases. An upgrade can still repair a failed install. +1. An upgrade does not install a deleted resource again. The resource processor sends `updating_failed` if the last successful action of the Porter installation was an uninstall. Without this check, `porter installations apply` runs the install action. + An upgrade can still repair a failed install, also an install that failed before Porter wrote its record. -A status message without `executionId` comes from a resource processor before version 0.13.16, and the API keeps the earlier rules for it. An API before version 0.28.10 ignores the field. So you can deploy the API and the resource processor in any order. +A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it. An API before version 0.29.0 ignores the field. So you can deploy the API and the resource processor in any order. The resource processor now needs the management storage account for each step. If the account refuses access (HTTP 401 or 403), the resource processor sends the failed status for the step. A Terraform backend in the same account failed in the same way before this change. For other storage errors, Service Bus delivers the message again. @@ -148,7 +150,8 @@ If the API stops between the write and the send, the entry stays. When the API s 1. One API instance claims each entry with the etag of the operation document. The other instances leave the entry. 1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. -1. If a later request for the same resource exists, for example a delete, the updater does not send the message. It sets the step and the operation to the failed status. +1. If a later request for the same resource exists, for example a delete, the updater does not send the message. A cascade request of a parent with a step for the resource also counts. + The updater sets the step and the operation to the failed status. It does not change the resource, because the later request owns its status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. 1. If an install or an update sets a sensitive property, the first message has the real value, and Cosmos DB keeps `REDACTED`. The operation write records this in the entry, and the updater cannot build such a message again. @@ -156,7 +159,9 @@ If the API stops between the write and the send, the entry stays. When the API s Other first messages have the same values as Cosmos DB, so the updater sends them with the same values as the first send. 1. If the updater cannot build the message, it sets the step and the operation to the failed status with the error text. -A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. +In the last two cases, the updater also sets the resource of the request to the failed status. A first step on another resource, for example the firewall, did not run, so that resource keeps its status. + +A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. An uninstall keeps its earlier rule for a final step. The status message of a step stays on the `deploymentstatus` queue until the API has sent the message of the next step. So a stop of the API during a pipeline does not need this entry. From e8c11420512cca9f987e256255ddae5fd63b3525 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 31 Oct 2026 00:53:26 +0000 Subject: [PATCH 14/37] Count a later request on the first child of a pending cascade (#5236) - For a cascade, the first step is the main step of the deepest child. The resender looked for later requests on the parent only. So after a disable of a workspace service that the API did not send, a user could delete the disabled user resource, and the resender then sent the upgrade of that user resource after the delete. - The resender now also looks for later requests on the resource of the first step if that step belongs to a child (sourceTemplateResourceId). A first step of the request itself on another resource, for example the firewall, still does not count. - Tests: a later delete of the first child fails the cascade step, and a later request comes before the sensitive value rule. - The resource processor document describes the rule. --- .../service_bus/deployment_status_updater.py | 10 ++++- .../test_service_bus/test_pending_dispatch.py | 44 +++++++++++++++++++ docs/tre-developers/resource-processor.md | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 7ffcb3c2e4..4437f7b823 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -146,7 +146,15 @@ async def resend_pending_dispatch(self, document: dict): # A later request on the resource, or a cascade from a parent with a step for it, for example a delete of a # resource whose install was not sent. The install must not come after it. - if await self.operations_repo.get_later_operations_for_resource(operation.resourceId, operation.createdWhen): + later_operations = await self.operations_repo.get_later_operations_for_resource( + operation.resourceId, operation.createdWhen + ) + # The first step of a cascade belongs to a child, so a later request on that child also counts. + if not later_operations and step.sourceTemplateResourceId not in (None, operation.resourceId): + later_operations = await self.operations_repo.get_later_operations_for_resource( + step.sourceTemplateResourceId, operation.createdWhen + ) + if later_operations: logger.warning(f"Operation {operation.id} was not sent before a later request. Failing step {step.id}") await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST, write_resource=False) return diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index 10ddf9600a..99f0495bf4 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -21,6 +21,7 @@ RESOURCE_ID = "59b5c8e7-5c42-4fcb-a7fd-294cfc27aa76" FIREWALL_ID = "a8b2c3d4-1111-4222-8333-944455556666" +CHILD_ID = "c3d4e5f6-2222-4333-8444-a55566667777" MESSAGE_ID = f"{OPERATION_ID}:random-uuid" AWAITING = { "install": Status.AwaitingDeployment, @@ -183,6 +184,49 @@ async def test_a_later_request_for_the_resource_fails_the_step(update_mock, send updater.resource_repo.update_item.assert_not_awaited() +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_later_request_for_the_first_child_of_a_cascade_fails_the_step(update_mock, send_mock, updater): + # A disable of a workspace service starts with the main step of its user resource. A user can delete the user + # resource at once, because the cascade already disabled it. + document = pending_document(action="upgrade", status=Status.AwaitingUpdate) + document["steps"][0]["resourceId"] = CHILD_ID + document["steps"][0]["sourceTemplateResourceId"] = CHILD_ID + later = create_sample_operation(CHILD_ID, "uninstall") + later.createdWhen = document["createdWhen"] + 60 + updater.operations_repo.get_later_operations_for_resource.side_effect = lambda resource_id, _: ( + [later] if resource_id == CHILD_ID else [] + ) + update_mock.return_value = workspace({}) + + await updater.resend_pending_dispatch(document) + + queried = [call.args for call in updater.operations_repo.get_later_operations_for_resource.await_args_list] + assert queried == [(RESOURCE_ID, document["createdWhen"]), (CHILD_ID, document["createdWhen"])] + update_mock.assert_not_awaited() + send_mock.assert_not_awaited() + operation = updater.operations_repo.update_item.await_args.args[0] + assert operation.steps[0].status == Status.UpdatingFailed + assert operation.steps[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST + updater.resource_repo.update_item.assert_not_awaited() + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_later_request_comes_before_the_sensitive_values(update_mock, send_mock, updater): + # A later delete owns the status of the resource, so the sensitive value rule must not set it to failed again. + document = pending_document(sensitive_values=True) + later = create_sample_operation(RESOURCE_ID, "uninstall") + later.createdWhen = document["createdWhen"] + 60 + updater.operations_repo.get_later_operations_for_resource.return_value = [later] + + await updater.resend_pending_dispatch(document) + + send_mock.assert_not_awaited() + assert updater.operations_repo.update_item.await_args.args[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST + updater.resource_repo.update_item.assert_not_awaited() + + @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") async def test_a_failed_first_step_on_another_resource_changes_only_the_resource_of_the_request( diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index fa9c5a3627..7b4f5415e7 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -151,6 +151,7 @@ If the API stops between the write and the send, the entry stays. When the API s 1. One API instance claims each entry with the etag of the operation document. The other instances leave the entry. 1. If the step has a status from the resource processor, the resource processor has the message. The updater removes the entry. 1. If a later request for the same resource exists, for example a delete, the updater does not send the message. A cascade request of a parent with a step for the resource also counts. + The first step of a cascade is for a child resource, so a later request for that child also counts. The updater sets the step and the operation to the failed status. It does not change the resource, because the later request owns its status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. 1. If an install or an update sets a sensitive property, the first message has the real value, and Cosmos DB keeps `REDACTED`. From 68d72103edeed83ec7d1a6b002ab088c161bfc14 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 31 Oct 2026 00:53:32 +0000 Subject: [PATCH 15/37] Ask for the sensitive values again after a failed request (#5298) - The API does not keep the new value of a sensitive property. The update form shows REDACTED, and an update with REDACTED keeps the earlier value. So a repeat of a failed update from the form ends as updated with the old secret. - The failure message of an unsent request with sensitive values now asks the user to send the request again with these values. - The resource processor document and the CHANGELOG note for a new workspace client secret describe this. --- CHANGELOG.md | 2 ++ api_app/service_bus/deployment_status_updater.py | 3 ++- docs/tre-developers/resource-processor.md | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f24d58749b..2446e6cab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ BUG FIXES: API responses with the resource properties then included this value. A database migration replaces the values that earlier updates stored. If an update set the client secret of a workspace before this version, you can create a new secret for the workspace application. Then update the workspace with the new secret, and delete the old secret. + If that update fails, send it again with the new secret. The update form shows `REDACTED`, and an update with `REDACTED` keeps the earlier secret. + Delete the old secret only after an update with the new secret succeeds. For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. The create route of an Airlock review VM did the same in its HTTP 500 response. diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 4437f7b823..f682efce2b 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -28,7 +28,8 @@ AWAITING_STATUSES = (Status.AwaitingDeployment, Status.AwaitingUpdate, Status.AwaitingDeletion, Status.AwaitingAction) NOT_SENT_WITH_SENSITIVE_VALUES = ( "The API stopped before it recorded the send of this request. The request has sensitive values, which the API " - "does not store, so the API cannot send it again. If the status does not change, send the request again." + "does not store, so the API cannot send it again. If the status does not change, send the request again with " + "the sensitive values." ) NOT_SENT_BEFORE_A_LATER_REQUEST = ( "The API stopped before it recorded the send of this request, and a later request changed the resource. " diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 7b4f5415e7..2778ca652d 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -156,7 +156,8 @@ If the API stops between the write and the send, the entry stays. When the API s 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. 1. If an install or an update sets a sensitive property, the first message has the real value, and Cosmos DB keeps `REDACTED`. The operation write records this in the entry, and the updater cannot build such a message again. - It sets the step and the operation to the failed status, with a message that asks the user to send the request again if the status does not change. + It sets the step and the operation to the failed status, with a message that asks the user to send the request again with the sensitive values if the status does not change. + The API does not keep the new value of a sensitive property, and an update that sends `REDACTED` keeps the earlier value. Other first messages have the same values as Cosmos DB, so the updater sends them with the same values as the first send. 1. If the updater cannot build the message, it sets the step and the operation to the failed status with the error text. From 55fcbc3a8e70f71827b8b87c001fe1cb15b0b86c Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 31 Oct 2026 00:53:37 +0000 Subject: [PATCH 16/37] Add tests that fail if the earlier fixes go back (#5236, #5298) - Route tests send a schema error with a submitted value to the create and update routes of workspaces, workspace services, user resources and shared services. Each test fails on the code of main. The Airlock review VM test also fails if the route logs the exception. - Pending dispatch test: a first step on the firewall is an upgrade, so the resource of the request gets the status of the operation, and only later requests on the resource of the request count. - Status rule tests: a failure from the same run after an unfinished success keeps the success, and a message without an execution ID keeps the earlier rules for a step that has one. - The resource processor test for a session lost during the lease wait now fails if the worker renews the session before the wait. --- .../test_api/test_routes/test_airlock.py | 1 + .../test_routes/test_shared_services.py | 41 +++++++- .../test_api/test_routes/test_workspaces.py | 96 +++++++++++++++++++ .../test_deployment_status_update.py | 25 ++++- .../test_service_bus/test_pending_dispatch.py | 11 ++- resource_processor/tests_rp/test_runner.py | 15 ++- 6 files changed, 185 insertions(+), 4 deletions(-) diff --git a/api_app/tests_ma/test_api/test_routes/test_airlock.py b/api_app/tests_ma/test_api/test_routes/test_airlock.py index 9208e8603a..58a6f2e312 100644 --- a/api_app/tests_ma/test_api/test_routes/test_airlock.py +++ b/api_app/tests_ma/test_api/test_routes/test_airlock.py @@ -809,6 +809,7 @@ async def test_post_create_review_user_resource_hides_the_input_of_a_schema_erro assert "os_image" in response.text assert "sas-value" not in response.text assert "sas-value" not in str(logger_mock.mock_calls) + logger_mock.exception.assert_not_called() @patch( "services.airlock.AirlockRequestRepository.read_item_by_id", diff --git a/api_app/tests_ma/test_api/test_routes/test_shared_services.py b/api_app/tests_ma/test_api/test_routes/test_shared_services.py index e8685b739b..f0387880a7 100644 --- a/api_app/tests_ma/test_api/test_routes/test_shared_services.py +++ b/api_app/tests_ma/test_api/test_routes/test_shared_services.py @@ -10,7 +10,14 @@ from models.domain.resource import ResourceHistoryItem from tests_ma.test_api.conftest import create_admin_user, create_test_user -from .test_workspaces import FAKE_CREATE_TIMESTAMP, FAKE_UPDATE_TIMESTAMP, OPERATION_ID, sample_resource_operation +from .test_workspaces import ( + FAKE_CREATE_TIMESTAMP, + FAKE_UPDATE_TIMESTAMP, + OPERATION_ID, + assert_no_submitted_value, + sample_resource_operation, + schema_error_with_a_secret, +) from db.errors import EntityDoesNotExist from models.domain.shared_service import SharedService @@ -221,6 +228,38 @@ async def test_patch_shared_service_returns_409_if_bad_etag(self, _, __, app, cl assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.ETAG_CONFLICT + # [POST] /shared-services + @patch("api.routes.shared_services.logger") + @patch("api.routes.shared_services.SharedServiceRepository.create_shared_service_item") + async def test_post_shared_services_returns_400_without_the_submitted_values( + self, create_mock, logger_mock, app, client, shared_service_input + ): + create_mock.side_effect = schema_error_with_a_secret() + + response = await client.post(app.url_path_for(strings.API_CREATE_SHARED_SERVICE), json=shared_service_input) + + assert_no_submitted_value(response, logger_mock) + + # [PATCH] /shared-services/{shared_service_id} + @patch("api.routes.shared_services.logger") + @patch( + "api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", + return_value=sample_shared_service(), + ) + @patch("api.dependencies.shared_services.SharedServiceRepository.patch_shared_service") + async def test_patch_shared_service_returns_400_without_the_submitted_values( + self, patch_mock, _, logger_mock, app, client + ): + patch_mock.side_effect = schema_error_with_a_secret() + + response = await client.patch( + app.url_path_for(strings.API_UPDATE_SHARED_SERVICE, shared_service_id=SHARED_SERVICE_ID), + json={"isEnabled": True}, + headers={"etag": ETAG}, + ) + + assert_no_submitted_value(response, logger_mock) + # [PATCH] /shared-services/{shared_service_id} @patch( "api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 7b57665dc9..c118e51b00 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -87,6 +87,23 @@ def disabled_workspace() -> Workspace: return workspace +def schema_error_with_a_secret() -> ValidationError: + """A schema error of a submitted value. Its message and its text have the value, so the old routes returned it.""" + try: + validate(instance={"vm_size": "secret-value"}, schema={"properties": {"vm_size": {"enum": ["small"]}}}) + except ValidationError as error: + assert "secret-value" in error.message + return error + + +def assert_no_submitted_value(response, logger_mock): + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "vm_size" in response.text + assert "secret-value" not in response.text + assert "secret-value" not in str(logger_mock.mock_calls) + logger_mock.exception.assert_not_called() + + def sample_workspace(workspace_id=WORKSPACE_ID, auth_info: dict = {}) -> Workspace: workspace = Workspace( id=workspace_id, @@ -983,6 +1000,23 @@ async def test_patch_workspace_returns_409_if_bad_etag( assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.ETAG_CONFLICT + # [PATCH] /workspaces/{workspace_id} + @patch("api.routes.workspaces.logger") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + @patch("api.routes.workspaces.WorkspaceRepository.patch_workspace") + async def test_patch_workspace_returns_400_without_the_submitted_values( + self, patch_workspace_mock, _, logger_mock, app, client + ): + patch_workspace_mock.side_effect = schema_error_with_a_secret() + + response = await client.patch( + app.url_path_for(strings.API_UPDATE_WORKSPACE, workspace_id=WORKSPACE_ID), + json={"isEnabled": True}, + headers={"etag": "some-etag-value"}, + ) + + assert_no_submitted_value(response, logger_mock) + # [DELETE] /workspaces/{workspace_id} @patch( "api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", @@ -1284,6 +1318,26 @@ async def test_post_workspace_services_raises_400_bad_request_if_input_is_bad( assert response.status_code == status.HTTP_400_BAD_REQUEST + # [POST] /workspaces/{workspace_id}/workspace-services + @patch("api.routes.workspaces.logger") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id") + @patch("api.routes.workspaces.OperationRepository.resource_has_deployed_operation", return_value=True) + @patch("api.routes.workspaces.WorkspaceServiceRepository.create_workspace_service_item") + async def test_post_workspace_services_returns_400_without_the_submitted_values( + self, create_mock, _, get_workspace_mock, logger_mock, app, client, workspace_service_input + ): + get_workspace_mock.return_value = sample_workspace( + auth_info={"sp_id": "ab123", "roles": {"WorkspaceOwner": "ab124", "WorkspaceResearcher": "ab125"}} + ) + create_mock.side_effect = schema_error_with_a_secret() + + response = await client.post( + app.url_path_for(strings.API_CREATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID), + json=workspace_service_input, + ) + + assert_no_submitted_value(response, logger_mock) + # [DELETE] /workspaces/{workspace_id}/services/{service_id} @patch( "api.routes.resource_helpers.ResourceRepository.get_resource_dependency_list", @@ -2036,6 +2090,7 @@ async def test_patch_user_resource_400_when_invalid(self, _, __, ___, ____, ____ headers={"etag": etag}, ) assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "INVALID-DATA" not in response.text # [PATCH] /workspaces/{workspace_id}/services/{service_id} @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @@ -2085,6 +2140,27 @@ async def test_patch_workspace_service_returns_409_if_bad_etag(self, _, __, ___, assert response.status_code == status.HTTP_409_CONFLICT assert response.text == strings.ETAG_CONFLICT + # [PATCH] /workspaces/{workspace_id}/services/{service_id} + @patch("api.routes.workspaces.logger") + @patch( + "api.dependencies.workspaces.WorkspaceServiceRepository.get_workspace_service_by_id", + return_value=sample_workspace_service(), + ) + @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) + @patch("api.routes.workspaces.WorkspaceServiceRepository.patch_workspace_service") + async def test_patch_workspace_service_returns_400_without_the_submitted_values( + self, patch_mock, _, __, logger_mock, app, client + ): + patch_mock.side_effect = schema_error_with_a_secret() + + response = await client.patch( + app.url_path_for(strings.API_UPDATE_WORKSPACE_SERVICE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), + json={"isEnabled": True}, + headers={"etag": "some-etag-value"}, + ) + + assert_no_submitted_value(response, logger_mock) + # [PATCH] /workspaces/{workspace_id}/services/{service_id} @pytest.mark.parametrize( "workspace_id, workspace_service_id", @@ -2544,6 +2620,26 @@ async def test_post_user_resources_raises_400_bad_request_if_input_is_bad( assert response.status_code == status.HTTP_400_BAD_REQUEST + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources + @patch("api.routes.workspaces.logger") + @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") + @patch("api.dependencies.workspaces.WorkspaceServiceRepository.get_deployed_workspace_service_by_id") + @patch("api.routes.workspaces.UserResourceRepository.create_user_resource_item") + async def test_post_user_resources_returns_400_without_the_submitted_values( + self, create_mock, _, get_workspace_mock, logger_mock, app, client, sample_user_resource_input_data + ): + get_workspace_mock.return_value = sample_workspace( + auth_info={"sp_id": "ab123", "roles": {"WorkspaceOwner": "ab124", "WorkspaceResearcher": "ab125"}} + ) + create_mock.side_effect = schema_error_with_a_secret() + + response = await client.post( + app.url_path_for(strings.API_CREATE_USER_RESOURCE, workspace_id=WORKSPACE_ID, service_id=SERVICE_ID), + json=sample_user_resource_input_data, + ) + + assert_no_submitted_value(response, logger_mock) + # [POST] /workspaces/{workspace_id}/workspace-services/{service_id}/user-resources @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version") @patch("api.dependencies.workspaces.WorkspaceRepository.get_deployed_workspace_by_id") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index 5a6c7940e3..dffe764472 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1177,6 +1177,27 @@ def fail_first_write(item): send.assert_awaited_once() +async def test_a_failure_from_the_run_after_an_unfinished_success_keeps_the_success(persisted_pipeline): + updater, stored, message, send, (_, success, failure) = persisted_pipeline + save = updater.resource_repo.update_item.side_effect + writes = [] + + def fail_first_write(item): + writes.append(item.id) + if len(writes) == 1: + raise RuntimeError("Cosmos DB is not available") + save(item) + + updater.resource_repo.update_item.side_effect = fail_first_write + assert await updater.update_status_in_database(message(success)) is False + assert stored["operation"].steps[0].statusUpdateComplete is False + + # For example, the resource processor could not read the outputs of the run and sent a failure. + assert await updater.update_status_in_database(message(failure)) is True + + assert stored["operation"].steps[0].status == success + + async def test_a_new_run_before_the_final_status_takes_over_the_step(persisted_pipeline): updater, stored, message, send, (in_progress, success, _) = persisted_pipeline assert await updater.update_status_in_database(message(in_progress, execution_id="run-a")) is True @@ -1212,6 +1233,8 @@ async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): workspace = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") operation = create_sample_operation(workspace.id, RequestAction.Install) operation.steps[0].status = Status.Deployed + # A newer resource processor finished the step. A message from an older resource processor has no execution ID. + operation.steps[0].executionId = "run-a" updater = DeploymentStatusUpdater() updater.operations_repo = AsyncMock() updater.operations_repo.get_operation_by_id.return_value = operation @@ -1228,7 +1251,7 @@ async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): assert await updater.update_status_in_database(message) is True assert operation.steps[0].status == Status.DeploymentFailed - assert operation.steps[0].executionId is None + assert operation.steps[0].executionId == "run-a" @patch("service_bus.deployment_status_updater.logger") diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index 99f0495bf4..aa9d9ddf73 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -232,13 +232,22 @@ async def test_a_later_request_comes_before_the_sensitive_values(update_mock, se async def test_a_failed_first_step_on_another_resource_changes_only_the_resource_of_the_request( update_mock, send_mock, updater ): - # For example, the first step of an uninstall pipeline changes the firewall. That step did not run. + # For example, the first step of an install pipeline is an upgrade of the firewall. That step did not run. document = pending_document() document["steps"][0]["resourceId"] = FIREWALL_ID + document["steps"][0]["resourceAction"] = "upgrade" + document["steps"][0]["status"] = Status.AwaitingUpdate update_mock.side_effect = Exception("template not found") await updater.resend_pending_dispatch(document) + # The firewall has many requests, so only later requests on the resource of the request count. + updater.operations_repo.get_later_operations_for_resource.assert_awaited_once_with( + RESOURCE_ID, document["createdWhen"] + ) + operation = updater.operations_repo.update_item.await_args.args[0] + assert operation.steps[0].status == Status.UpdatingFailed + assert operation.status == Status.DeploymentFailed updater.resource_repo.get_resource_by_id.assert_awaited_once_with(uuid.UUID(RESOURCE_ID)) assert updater.resource_repo.update_item.await_args.args[0].deploymentStatus == Status.DeploymentFailed diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 534dd980d4..b51e6478ef 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -1245,7 +1245,20 @@ async def test_a_session_lost_during_the_lease_wait_runs_no_porter( ): # Another worker can have the message now, and a later step for the resource can have finished. client, receiver = session_receiver(mock_service_bus_client, VALID_REQUEST) - receiver.session.renew_lock.side_effect = SessionLockLostError() + leased = [] + + @asynccontextmanager + async def lease(container, resource_id, describe): + leased.append(resource_id) + yield mock_execution_lease.execution + + async def renew_lock(): + # The session ends during the wait, so only a renew after the wait finds the loss. + if leased: + raise SessionLockLostError() + + mock_execution_lease.side_effect = lease + receiver.session.renew_lock.side_effect = renew_lock await receive_message( client, {"resource_request_queue": "q"}, AsyncMock(), keep_running=Mock(side_effect=[True, False]) From 54f691439d62c2de117fc75521f39f1d92cd752f Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 31 Oct 2026 14:48:37 +0000 Subject: [PATCH 17/37] Tell operators to replace a secret from a failed create (#5298) Before this version, the API wrote the text of a schema error to its console log. The core Log Analytics workspace keeps this log for 30 days by default, and the text can contain a client secret. The CHANGELOG entry now tells operators to create a new secret for the workspace application if such a create failed. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2446e6cab3..863f274a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,8 @@ BUG FIXES: For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. The create route of an Airlock review VM did the same in its HTTP 500 response. - That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. (`api` 0.29.0) + That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. + The API also wrote this text to its console log, which the core Log Analytics workspace keeps for 30 days by default. If a create request with a client secret got a schema error before this version, create a new secret for the workspace application. (`api` 0.29.0) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) From 60ea0c18f1118a5744d60f979f0916a440b37520 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:03:06 +0000 Subject: [PATCH 18/37] Fail a step when the Porter output list is not valid JSON (#5297) get_porter_outputs returned a success with an empty list when the text of `porter installations output list` was not a JSON list. On main this path sent {}, which the API rejected. With the empty list, the API accepted the success. The next pipeline step then used an empty value for each missing output, for example the Guacamole callback URI. - The resource processor now sends the failed status with the message for outputs that it cannot read. This also applies to an empty text and to JSON that is not a list. - The conversion of each output value does not change. A value with a bracket that is not JSON stays a string, and the step succeeds. - Two older tests gave a text that is not JSON as the output list. They now give an empty JSON list. --- resource_processor/tests_rp/test_runner.py | 44 ++++++++++++++++++++-- resource_processor/vmss_porter/runner.py | 8 +++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index b51e6478ef..5293bfedc3 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -420,7 +420,7 @@ async def test_receive_message_unknown_exception( @pytest.mark.asyncio @patch("vmss_porter.runner.build_porter_command", return_value=([["porter", "install"]], None, "tre-params-test", None)) -@patch("vmss_porter.runner.run_porter", return_value=(0, "stdout", "stderr")) +@patch("vmss_porter.runner.run_porter", return_value=(0, "[]", "stderr")) @patch("vmss_porter.runner.service_bus_message_generator", return_value="test_message") async def test_invoke_porter_action( mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client @@ -591,7 +591,7 @@ async def test_invoke_porter_action_nonzero_exit_without_confirmed_absence_repor "vmss_porter.runner.build_porter_command", return_value=([["porter", "custom-action"]], None, "tre-params-test", None), ) -@patch("vmss_porter.runner.run_porter", return_value=(0, "stdout", "stderr")) +@patch("vmss_porter.runner.run_porter", return_value=(0, "[]", "stderr")) @patch("vmss_porter.runner.service_bus_message_generator", return_value="test_message") async def test_invoke_porter_action_custom_action( mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client @@ -1129,6 +1129,44 @@ async def test_get_porter_outputs(mock_run_porter, mock_build_porter_command_for assert outputs == [{"name": "output1", "value": "value1"}] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stdout", + [ + pytest.param("not json", id="invalid-json"), + pytest.param('{"name": "output1", "value": "value1"}', id="not-a-list"), + pytest.param(None, id="no-output"), + ], +) +async def test_get_porter_outputs_fails_when_the_output_list_cannot_be_read(stdout): + # A success without outputs lets the next pipeline step use an empty value, for example a callback URI. + with ( + patch("vmss_porter.runner.build_porter_command_for_outputs", return_value=[["porter", "output", "list"]]), + patch("vmss_porter.runner.run_porter", return_value=(0, stdout, None)), + ): + assert await get_porter_outputs({"id": "test_id", "action": "upgrade"}, {}) == (False, []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "value, expected", + [ + pytest.param('{"key": "value"}', {"key": "value"}, id="json-value"), + pytest.param("[not json", "[not json", id="text-with-a-bracket"), + ], +) +async def test_get_porter_outputs_keeps_a_value_that_is_not_json(value, expected): + stdout = json.dumps([{"name": "output1", "value": value}]) + with ( + patch("vmss_porter.runner.build_porter_command_for_outputs", return_value=[["porter", "output", "list"]]), + patch("vmss_porter.runner.run_porter", return_value=(0, stdout, None)), + ): + assert await get_porter_outputs({"id": "test_id", "action": "upgrade"}, {}) == ( + True, + [{"name": "output1", "value": expected}], + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "output_result, expected_status", @@ -1138,7 +1176,7 @@ async def test_get_porter_outputs(mock_run_porter, mock_build_porter_command_for "updating_failed", id="output-command-fails", ), - pytest.param((0, "not json", None), "updated", id="invalid-json"), + pytest.param((0, "not json", None), "updating_failed", id="invalid-json"), ], ) async def test_invoke_porter_action_sends_outputs_as_a_list(output_result, expected_status): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index cdb64f0704..8978dd303b 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -679,10 +679,16 @@ async def get_porter_outputs(msg_body: dict, config: dict): # The API reads outputs as a list. It rejects a dict, so the operation never gets the failed status. return False, [] else: - outputs_json = [] try: outputs_json = json.loads(stdout) + except (TypeError, ValueError): + outputs_json = None + if not isinstance(outputs_json, list): + # A success without outputs lets the API continue, and the next pipeline step gets empty values. + logger.error(f"{msg_body['id']}: Failed to get outputs, because the Porter output is not a JSON list") + return False, [] + try: # loop props individually to try to deserialise to dict/list, as all TF outputs are strings, but we want the pure value for i in range(0, len(outputs_json)): if "{" in outputs_json[i]["value"] or "[" in outputs_json[i]["value"]: From e3d8418f7b47f0033c377acd0f4bf7be6a2ca397 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:04:45 +0000 Subject: [PATCH 19/37] Fail an upgrade when the Porter installation lookup fails (#5236) The upgrade check treated a failed lookup as an installation that Porter did not uninstall. A failed lookup is a command failure, a timeout or a response that is not a JSON list. The resource processor then ran `porter installations apply`, which installs an uninstalled installation again. So a short storage failure could let an upgrade install a deleted resource. - The upgrade now fails with updating_failed and the message "Cannot verify the Porter installation of resource ''. Check Porter storage access and retry." The uninstall check and the legacy parameter check already fail in the same way. - A successful empty list still lets an upgrade repair an install that failed before Porter wrote its record. - The resource processor document describes the failure. --- docs/tre-developers/resource-processor.md | 1 + resource_processor/tests_rp/test_run_step.py | 46 +++++++++++++++++++- resource_processor/vmss_porter/runner.py | 37 +++++++++++----- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 2778ca652d..89f00510af 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -131,6 +131,7 @@ Service Bus can give the same message to more than one worker. The resource proc 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. 1. An upgrade does not install a deleted resource again. The resource processor sends `updating_failed` if the last successful action of the Porter installation was an uninstall. Without this check, `porter installations apply` runs the install action. An upgrade can still repair a failed install, also an install that failed before Porter wrote its record. + If the resource processor cannot read the Porter installation, for example because the Porter storage is not available, it also sends `updating_failed`. A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it. An API before version 0.29.0 ignores the field. So you can deploy the API and the resource processor in any order. diff --git a/resource_processor/tests_rp/test_run_step.py b/resource_processor/tests_rp/test_run_step.py index a5183fe481..8017ef8ab2 100644 --- a/resource_processor/tests_rp/test_run_step.py +++ b/resource_processor/tests_rp/test_run_step.py @@ -191,6 +191,49 @@ async def helper(command, config, *args, **kwargs): assert code == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "lookup, porter_runs", + [ + # An install can fail before Porter writes a record. An upgrade can repair it. + pytest.param((0, "[]", None), True, id="no-installation"), + pytest.param((1, None, "store not available"), False, id="command-fails"), + pytest.param((0, "not json", None), False, id="invalid-json"), + pytest.param((0, "{}", None), False, id="not-a-list"), + pytest.param(TimeoutError(), False, id="timeout"), + ], +) +async def test_an_upgrade_runs_only_after_a_successful_installation_lookup(lookup, porter_runs): + commands = [] + + async def helper(command, config, *args, **kwargs): + commands.append(command) + if command[:3] == ["porter", "installations", "list"]: + if isinstance(lookup, Exception): + raise lookup + return lookup + return 0, "", "" + + with ( + patch.object(runner, "run_command_helper", side_effect=helper), + patch.object(runner, "azure_login_command", return_value=[]), + patch.object(runner, "azure_acr_login_command", return_value=[]), + patch.object(runner, "apply_porter_credentials_sets_command", return_value=[]), + ): + code, _, error = await runner.run_porter( + [["porter", "installations", "apply"]], {}, required_installation="resource-1" + ) + assert commands[0][:3] == ["porter", "installations", "list"] + assert (["porter", "installations", "apply"] in commands) is porter_runs + if porter_runs: + assert code == 0 + else: + assert code == 1 + assert error == ( + "Cannot verify the Porter installation of resource 'resource-1'. Check Porter storage access and retry." + ) + + def installation(installed=None, uninstalled=None, name="resource-1"): return {"name": name, "status": {"installed": installed, "uninstalled": uninstalled}} @@ -209,7 +252,8 @@ def installation(installed=None, uninstalled=None, name="resource-1"): # A failed install has neither time. An upgrade can repair it. ([installation()], False), ([installation(name="other-resource")], False), - (None, False), + # A failed lookup does not show that the last successful action was not an uninstall. + (None, None), ], ) async def test_an_upgrade_finds_a_removed_installation(output, removed): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 8978dd303b..9a67fc9ca5 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -343,13 +343,22 @@ async def run_porter( # `porter installations apply` installs an uninstalled installation. An upgrade must not create a deleted # resource again. - if required_installation is not None and await _confirm_porter_installation_removed(required_installation, config): - return ( - 1, - None, - f"Cannot upgrade resource '{required_installation}': the last successful action of its Porter " - "installation was an uninstall. An upgrade does not install a deleted resource.", - ) + if required_installation is not None: + removed = await _confirm_porter_installation_removed(required_installation, config) + if removed is None: + return ( + 1, + None, + f"Cannot verify the Porter installation of resource '{required_installation}'. " + "Check Porter storage access and retry.", + ) + if removed: + return ( + 1, + None, + f"Cannot upgrade resource '{required_installation}': the last successful action of its Porter " + "installation was an uninstall. An upgrade does not install a deleted resource.", + ) last_returncode = None last_stdout = None @@ -434,10 +443,11 @@ async def _list_porter_installations(installation_id: str, config: dict) -> Opti ), timeout=30, ) - return json.loads(output) if code == 0 else None + installations = json.loads(output) if code == 0 else None except Exception: # An unreadable store, invalid response or timeout proves nothing. return None + return installations if isinstance(installations, list) else None async def _confirm_porter_installation_absent(installation_id: str, config: dict) -> bool: @@ -455,17 +465,20 @@ def _porter_time(value) -> Optional[datetime]: return None -async def _confirm_porter_installation_removed(installation_id: str, config: dict) -> bool: +async def _confirm_porter_installation_removed(installation_id: str, config: dict) -> Optional[bool]: """ - Confirm that the last successful action of the Porter installation was an uninstall. + Confirm that the last successful action of the Porter installation was an uninstall. Return None if the lookup fails. Porter keeps the record of an uninstalled installation. A failed install has neither time, so an upgrade can repair it. An install can also fail before Porter writes a record, for example at the sign-in or in an earlier pipeline step. So no record proves nothing, and an upgrade can repair that install too. """ installations = await _list_porter_installations(installation_id, config) - matches = [item for item in installations or [] if item.get("name") == installation_id] + if installations is None: + # An unreadable store does not show that the last successful action was not an uninstall. + return None + matches = [item for item in installations if item.get("name") == installation_id] if not matches: - # No record, an unreadable store, or a list with other installations only, proves nothing. + # No record, or a list with other installations only, proves nothing. return False status = matches[0].get("status") or {} uninstalled = _porter_time(status.get("uninstalled")) From 93d3ae54c721122d174e624a3f2a74efccec4901 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:06:35 +0000 Subject: [PATCH 20/37] Keep a newer execution record after the lease expired (#5236) After a lease expiry, the worker took the lease again and replaced the record without a check. In the meantime, another worker can take the lease, run a later step, write its record and release the lease. The old result then replaced the newer record. So a copy of the message of the later step did not find its result and ran Porter again. This needs a lease expiry and a session loss in the same run. - ExecutionLease keeps the execution ID of the record that the worker read or wrote last. - After it takes the lease again, the worker reads the record. It writes only if the record has that execution ID or its own execution ID. Otherwise it raises an error and keeps the record. - finish logs the error, and the worker still sends the status of the step. For the running record, the error stops the step before Porter, and Service Bus delivers the message again. - The resource processor document describes the rule. --- docs/tre-developers/resource-processor.md | 1 + resource_processor/helpers/execution_lease.py | 15 +++++++ .../tests_rp/test_execution_lease.py | 45 ++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 89f00510af..7b3d3867d2 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -127,6 +127,7 @@ Service Bus can give the same message to more than one worker. The resource proc 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. When it gets the lease, it renews the session lock. If the session is lost, the worker does not run Porter, because another worker can have the message. 1. The blob holds a record of the last step for the resource. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. + If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. After a Porter run, the worker sends the status also if it cannot write the record. 1. Each status message of a Porter run has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. 1. An upgrade does not install a deleted resource again. The resource processor sends `updating_failed` if the last successful action of the Porter installation was an uninstall. Without this check, `porter installations apply` runs the install action. diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py index a3743570fd..1bfe8d505d 100644 --- a/resource_processor/helpers/execution_lease.py +++ b/resource_processor/helpers/execution_lease.py @@ -27,6 +27,8 @@ def __init__(self, blob, lease): self.blob = blob self.lease = lease self.expires = time.monotonic() + LEASE_SECONDS + # The execution ID in the record that this worker read or wrote last. + self.execution_id = None async def renew(self): await self.lease.renew() @@ -36,6 +38,11 @@ def seconds_left(self) -> float: return self.expires - time.monotonic() async def read(self) -> dict: + record = await self._download() + self.execution_id = record.get("executionId") + return record + + async def _download(self) -> dict: download = await self.blob.download_blob(lease=self.lease) content = await download.readall() try: @@ -56,7 +63,15 @@ async def write(self, record: dict): # Azure gives the same lease ID again if no other worker took the lease. If one did, this raises 409. await self.lease.acquire(lease_duration=LEASE_SECONDS) self.expires = time.monotonic() + LEASE_SECONDS + # Another worker can have taken the lease, written its record and released the lease in the meantime. + # Keep that record, so that a copy of its message finds the result. + stored = (await self._download()).get("executionId") + if stored not in (self.execution_id, record.get("executionId")): + raise RuntimeError( + f"Another worker wrote the execution record {self.blob.blob_name} after the lease expired" + ) await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) + self.execution_id = record.get("executionId") async def create_blob(container: ContainerClient, blob): diff --git a/resource_processor/tests_rp/test_execution_lease.py b/resource_processor/tests_rp/test_execution_lease.py index bbcc1a34dc..85c7bd296d 100644 --- a/resource_processor/tests_rp/test_execution_lease.py +++ b/resource_processor/tests_rp/test_execution_lease.py @@ -105,15 +105,56 @@ def precondition_failed(): return error +def download_of(execution_id): + content = json.dumps({"executionId": execution_id, "state": "running"}).encode() + return MagicMock(readall=AsyncMock(return_value=content)) + + @pytest.mark.asyncio async def test_an_expired_lease_is_taken_again_for_the_record(): - blob = MagicMock(upload_blob=AsyncMock(side_effect=[precondition_failed(), None])) + blob = MagicMock( + upload_blob=AsyncMock(side_effect=[precondition_failed(), None]), + download_blob=AsyncMock(return_value=download_of("this-run")), + ) lease = MagicMock(acquire=AsyncMock()) - await leases.ExecutionLease(blob, lease).write({"state": "finished"}) + await leases.ExecutionLease(blob, lease).write({"executionId": "this-run", "state": "finished"}) lease.acquire.assert_awaited_once_with(lease_duration=leases.LEASE_SECONDS) assert blob.upload_blob.await_count == 2 +@pytest.mark.asyncio +async def test_a_record_that_another_worker_wrote_after_the_lease_expired_stays(): + # Another worker ran a later step, wrote its record and released the lease before this worker took it again. + blob = MagicMock( + upload_blob=AsyncMock(side_effect=[precondition_failed(), None]), + download_blob=AsyncMock(return_value=download_of("later-run")), + blob_name="resource-1", + ) + lease = MagicMock(acquire=AsyncMock()) + with pytest.raises(RuntimeError, match="Another worker wrote the execution record resource-1"): + await leases.ExecutionLease(blob, lease).write({"executionId": "this-run", "state": "finished"}) + lease.acquire.assert_awaited_once_with(lease_duration=leases.LEASE_SECONDS) + assert blob.upload_blob.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored, replaced", [("earlier-run", True), ("later-run", False)]) +async def test_the_running_record_replaces_only_the_record_that_the_worker_read(stored, replaced): + blob = MagicMock( + upload_blob=AsyncMock(side_effect=[precondition_failed(), None]), + download_blob=AsyncMock(side_effect=[download_of("earlier-run"), download_of(stored)]), + blob_name="resource-1", + ) + execution = leases.ExecutionLease(blob, MagicMock(acquire=AsyncMock())) + await execution.read() + if replaced: + await execution.write({"executionId": "this-run", "state": "running"}) + else: + with pytest.raises(RuntimeError): + await execution.write({"executionId": "this-run", "state": "running"}) + assert blob.upload_blob.await_count == (2 if replaced else 1) + + @pytest.mark.asyncio async def test_a_lease_that_another_worker_took_stops_the_record_write(): taken = HttpResponseError(message="lease already present") From 8679abed30ddd40c1f8eeb1445f39cbdd83b9bc2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:11:56 +0000 Subject: [PATCH 21/37] Keep the results of earlier steps in the execution record (#5236) The execution record kept only the last step of a resource. In the queue order A, B, A, the result of B replaced the result of A, so the copy of A ran Porter again. A copy can come when the API sends a next step again. This occurs if the API stops after the send and before it saves the completion of the earlier step. A step of another operation for the same resource, for example the firewall, can be in the queue between the two copies. - The record now also keeps the results of up to 20 earlier steps in earlierSteps. The worker finds a result by the operation ID and the step ID. The lease for each resource does not change. - When a step starts, the worker keeps the 20 most recent results and removes the other results. An earlier result keeps the first 4,000 characters of its status message, because a status message can be up to 2 MB. So a record has at most 21 results. - A record from an earlier version of this change has no earlierSteps. The worker reads its last step as before. - Tests: the order A, B, A runs Porter once for each step and sends the status of A again. The record keeps 20 earlier results with short messages, and a copy of an older step runs Porter again. - The resource processor document describes the record. --- docs/tre-developers/resource-processor.md | 4 +- resource_processor/helpers/execution_lease.py | 4 +- resource_processor/tests_rp/test_run_step.py | 109 +++++++++++++++++- resource_processor/vmss_porter/runner.py | 45 +++++++- 4 files changed, 147 insertions(+), 15 deletions(-) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 7b3d3867d2..411ef7cadb 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -126,7 +126,9 @@ Service Bus can give the same message to more than one worker. The resource proc 1. The `workspacequeue` queue keeps a session lock for 5 minutes. While a worker runs a step, it renews the lock every minute. If a renewal fails, the worker tries again until the lock expires, and logs each failure. 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. When it gets the lease, it renews the session lock. If the session is lost, the worker does not run Porter, because another worker can have the message. -1. The blob holds a record of the last step for the resource. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. +1. The blob holds a record of the last step for the resource and the results of up to 20 earlier steps. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. + This also applies to a copy of a message that arrives after other steps for the resource. For example, the API can send a step again. + When a step starts, the worker keeps the 20 most recent results as earlier results and removes the other results. An earlier result keeps the first 4,000 characters of its status message, because a status message can be up to 2 MB. If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. After a Porter run, the worker sends the status also if it cannot write the record. 1. Each status message of a Porter run has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py index 1bfe8d505d..36d5e0514e 100644 --- a/resource_processor/helpers/execution_lease.py +++ b/resource_processor/helpers/execution_lease.py @@ -19,8 +19,8 @@ class ExecutionLease: """One worker at a time holds this lease and runs Porter for a resource. - The blob keeps a small record of the last step that a worker ran for the resource, - so that a worker that gets the same message again can find the result. + The blob keeps a small record of the last step that a worker ran for the resource and the results of + earlier steps, so that a worker that gets the same message again can find the result. """ def __init__(self, blob, lease): diff --git a/resource_processor/tests_rp/test_run_step.py b/resource_processor/tests_rp/test_run_step.py index 8017ef8ab2..55cdf0ea38 100644 --- a/resource_processor/tests_rp/test_run_step.py +++ b/resource_processor/tests_rp/test_run_step.py @@ -79,17 +79,45 @@ async def test_a_recorded_failure_is_sent_again_without_outputs(): assert "outputs" not in body +def step_result(operation_id, step_id, execution_id): + return { + "operationId": operation_id, + "stepId": step_id, + "action": "install", + "executionId": execution_id, + "status": statuses.pass_status_string_for["install"], + "message": "install action completed successfully.", + "outputs": True, + } + + @pytest.mark.asyncio @pytest.mark.parametrize( - "record", + "record, earlier", [ - {}, - {"operationId": "operation-1", "stepId": "step-1", "state": "running", "executionId": "crashed-run"}, - {"operationId": "operation-1", "stepId": "step-0", "state": "finished", "executionId": "earlier-step"}, - {"operationId": "operation-0", "stepId": "step-1", "state": "finished", "executionId": "other-operation"}, + ({}, []), + ({"operationId": "operation-1", "stepId": "step-1", "state": "running", "executionId": "crashed-run"}, []), + ( + {**step_result("operation-1", "step-0", "earlier-step"), "state": "finished"}, + [step_result("operation-1", "step-0", "earlier-step")], + ), + ( + {**step_result("operation-0", "step-1", "other-operation"), "state": "finished"}, + [step_result("operation-0", "step-1", "other-operation")], + ), + ( + { + "operationId": "operation-1", + "stepId": "step-1", + "state": "running", + "executionId": "crashed-run", + "earlierSteps": [step_result("operation-0", "step-1", "other-operation")], + }, + [step_result("operation-0", "step-1", "other-operation")], + ), ], ) -async def test_porter_runs_when_the_record_has_no_result_for_this_step(record): +async def test_porter_runs_when_the_record_has_no_result_for_this_step(record, earlier): execution = MagicMock(read=AsyncMock(return_value=record), write=AsyncMock()) with patch.object(runner, "invoke_porter_action", AsyncMock(return_value=True)) as invoke: assert await runner.run_step(MESSAGE, execution, MagicMock(), CONFIG) is True @@ -102,6 +130,7 @@ async def test_porter_runs_when_the_record_has_no_result_for_this_step(record): "action": "install", "executionId": execution_id, "state": "running", + "earlierSteps": earlier, } await invoke.await_args.kwargs["finish"]("deployed", "done", True) assert execution.write.await_args.args[0] == { @@ -113,6 +142,74 @@ async def test_porter_runs_when_the_record_has_no_result_for_this_step(record): } +class RecordInMemory: + """Keep the execution record as JSON text, as the blob does.""" + + def __init__(self): + self.content = "{}" + + async def read(self): + return json.loads(self.content) + + async def write(self, record): + self.content = json.dumps(record) + + +def recording_porter(runs, message="upgrade action completed successfully."): + async def invoke(msg_body, sb_client, config, *, execution_id, finish): + runs.append((msg_body["operationId"], msg_body["stepId"], execution_id)) + await finish(statuses.pass_status_string_for[msg_body["action"]], message, False) + return True + + return patch.object(runner, "invoke_porter_action", AsyncMock(side_effect=invoke)) + + +@pytest.mark.asyncio +async def test_a_copy_of_a_step_that_arrives_after_another_step_does_not_run_porter_again(): + # The API can send a step again after a step of another operation for the same resource, for example the firewall. + step_a = {**MESSAGE, "action": "upgrade", "operationId": "operation-a"} + step_b = {**MESSAGE, "action": "upgrade", "operationId": "operation-b"} + execution = RecordInMemory() + client, sender = service_bus() + runs = [] + + with recording_porter(runs): + for message in (step_a, step_b, step_a): + assert await runner.run_step(message, execution, client, CONFIG) is True + + assert [operation for operation, _, _ in runs] == ["operation-a", "operation-b"] + [resent] = bodies(sender) + assert resent["operationId"] == "operation-a" + assert resent["executionId"] == runs[0][2] + assert resent["status"] == statuses.pass_status_string_for["upgrade"] + assert resent["message"] == "resource-1: upgrade action completed successfully." + + +@pytest.mark.asyncio +async def test_the_record_keeps_a_short_result_of_the_last_steps_only(): + steps = [{**MESSAGE, "action": "upgrade", "stepId": f"step-{number}"} for number in range(runner.EARLIER_STEPS + 2)] + long_message = "x" * (runner.EARLIER_MESSAGE_LENGTH + 100) + execution = RecordInMemory() + client, _ = service_bus() + runs = [] + + with recording_porter(runs, long_message): + for message in steps: + await runner.run_step(message, execution, client, CONFIG) + record = await execution.read() + # A copy of an earlier step in the record does not run Porter. The result of the oldest step went away. + await runner.run_step(steps[1], execution, client, CONFIG) + await runner.run_step(steps[0], execution, client, CONFIG) + + assert record["stepId"] == steps[-1]["stepId"] + assert record["message"] == long_message + assert [result["stepId"] for result in record["earlierSteps"]] == [step["stepId"] for step in steps[1:-1]] + for result in record["earlierSteps"]: + assert set(result) == {"operationId", "stepId", "action", "executionId", "status", "message", "outputs"} + assert result["message"] == long_message[: runner.EARLIER_MESSAGE_LENGTH] + assert [step for _, step, _ in runs] == [step["stepId"] for step in steps] + ["step-0"] + + @pytest.mark.asyncio async def test_each_status_has_the_execution_id_and_the_result_is_saved_before_the_final_status(): client, sender = service_bus() diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 9a67fc9ca5..86ff1ace55 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -486,19 +486,45 @@ async def _confirm_porter_installation_removed(installation_id: str, config: dic return uninstalled is not None and (installed is None or installed <= uninstalled) +# A copy of a message can arrive after a later step for the same resource, for example when the API sends a step again. +# So the execution record also keeps the results of the earlier steps. A status message can be up to 2 MB, so each +# earlier result keeps the start of its message only. +EARLIER_STEPS = 20 +EARLIER_MESSAGE_LENGTH = 4000 +RESULT_FIELDS = ("operationId", "stepId", "action", "executionId", "status", "message", "outputs") + + +def recorded_results(record: dict) -> list: + """Return the results of the finished steps in the execution record, oldest first.""" + earlier = record.get("earlierSteps") + results = [result for result in earlier if isinstance(result, dict)] if isinstance(earlier, list) else [] + if record.get("state") == "finished": + results.append(record) + return [{field: result[field] for field in RESULT_FIELDS if field in result} for result in results] + + async def run_step(msg_body: dict, execution, sb_client: ServiceBusClient, config: dict) -> bool: """ Run the step once. If the execution record shows that a worker already finished this step, send its status again. """ + + def same_step(entry: dict) -> bool: + return entry.get("operationId") == msg_body["operationId"] and entry.get("stepId") == msg_body["stepId"] + record = await execution.read() - same_step = record.get("operationId") == msg_body["operationId"] and record.get("stepId") == msg_body["stepId"] - if same_step and record.get("state") == "finished": + results = recorded_results(record) + finished = next((result for result in reversed(results) if same_step(result)), None) + if finished is not None: logger.info(f"Step {msg_body['stepId']} for {msg_body['id']} is already finished. Sending its status again.") - return await send_recorded_status(msg_body, record, sb_client, config) - if same_step: + return await send_recorded_status(msg_body, finished, sb_client, config) + if same_step(record): # Porter did not finish. Run it again, as a Service Bus redelivery does. logger.warning(f"An earlier worker stopped during step {msg_body['stepId']} for {msg_body['id']}.") + earlier = [ + {**result, "message": str(result.get("message", ""))[:EARLIER_MESSAGE_LENGTH]} + for result in results[-EARLIER_STEPS:] + ] execution_id = str(uuid4()) record = { "operationId": msg_body["operationId"], @@ -506,11 +532,18 @@ async def run_step(msg_body: dict, execution, sb_client: ServiceBusClient, confi "action": msg_body["action"], "executionId": execution_id, } - await execution.write({**record, "state": "running"}) + await execution.write({**record, "state": "running", "earlierSteps": earlier}) async def finish(status: str, message: str, with_outputs: bool): await execution.write( - {**record, "state": "finished", "status": status, "message": message, "outputs": with_outputs} + { + **record, + "state": "finished", + "status": status, + "message": message, + "outputs": with_outputs, + "earlierSteps": earlier, + } ) return await invoke_porter_action(msg_body, sb_client, config, execution_id=execution_id, finish=finish) From c8ca53448891050f635fab4d6a4d399399df07a9 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:40:25 +0000 Subject: [PATCH 22/37] Describe when a copy of a step runs Porter again (#5236) The resource processor document said that the resource processor runs each step once. For a copy of a step, the resource processor runs Porter again if the execution record does not have the result. This occurs after more than 20 other steps for the resource or after a failed record write. The API then ignores the status of the second run. - The section on messages that the API did not send now says how a second copy finds its result and when Porter runs again. The section on repeated messages says that the resource processor uses its rules to run each step once. - The list item on the execution record had 10 sentences. It is now three items: the record, the earlier results with the limit of 20 steps, and the new lease after a lease expiry. - The upgrade item says that an upgrade also fails if the lookup of the Porter installation takes more than 30 seconds. - The ExecutionLease docstring uses two short sentences. --- docs/tre-developers/resource-processor.md | 13 ++++++++----- resource_processor/helpers/execution_lease.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 411ef7cadb..7121d855a6 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -121,20 +121,21 @@ Furthermore, Porter provides a set of [mixins](https://porter.sh/mixins/) - anal ### Repeated messages -Service Bus can give the same message to more than one worker. The resource processor runs each step once: +Service Bus can give the same message to more than one worker. The resource processor uses these rules to run each step once: 1. The `workspacequeue` queue keeps a session lock for 5 minutes. While a worker runs a step, it renews the lock every minute. If a renewal fails, the worker tries again until the lock expires, and logs each failure. 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. When it gets the lease, it renews the session lock. If the session is lost, the worker does not run Porter, because another worker can have the message. 1. The blob holds a record of the last step for the resource and the results of up to 20 earlier steps. If the record shows that the same step finished, the worker does not run Porter. It reads the outputs again and sends the recorded status. This also applies to a copy of a message that arrives after other steps for the resource. For example, the API can send a step again. - When a step starts, the worker keeps the 20 most recent results as earlier results and removes the other results. An earlier result keeps the first 4,000 characters of its status message, because a status message can be up to 2 MB. - If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. After a Porter run, the worker sends the status also if it cannot write the record. +1. When a step starts, the worker keeps the 20 most recent results as earlier results and removes the other results. An earlier result keeps the first 4,000 characters of its status message, because a status message can be up to 2 MB. + A copy that arrives after more than 20 other steps for the resource does not find its result, and the worker runs Porter again. +1. If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. After a Porter run, the worker sends the status also if it cannot write the record. 1. Each status message of a Porter run has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. 1. An upgrade does not install a deleted resource again. The resource processor sends `updating_failed` if the last successful action of the Porter installation was an uninstall. Without this check, `porter installations apply` runs the install action. An upgrade can still repair a failed install, also an install that failed before Porter wrote its record. - If the resource processor cannot read the Porter installation, for example because the Porter storage is not available, it also sends `updating_failed`. + If the resource processor cannot read the Porter installation in 30 seconds, for example because the Porter storage is not available, it also sends `updating_failed`. A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it. An API before version 0.29.0 ignores the field. So you can deploy the API and the resource processor in any order. @@ -167,7 +168,9 @@ If the API stops between the write and the send, the entry stays. When the API s In the last two cases, the updater also sets the resource of the request to the failed status. A first step on another resource, for example the firewall, did not run, so that resource keeps its status. -A second copy of a message is safe, because the resource processor runs each step once. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. An uninstall keeps its earlier rule for a final step. +A second copy of a message is safe. The resource processor finds the result of the step in the execution record and sends the recorded status. +If the record does not have the result of the first run, Porter runs again, and the API ignores the status of the second run. +The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. An uninstall keeps its earlier rule for a final step. The status message of a step stays on the `deploymentstatus` queue until the API has sent the message of the next step. So a stop of the API during a pipeline does not need this entry. diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py index 36d5e0514e..0ccb3a7839 100644 --- a/resource_processor/helpers/execution_lease.py +++ b/resource_processor/helpers/execution_lease.py @@ -19,8 +19,8 @@ class ExecutionLease: """One worker at a time holds this lease and runs Porter for a resource. - The blob keeps a small record of the last step that a worker ran for the resource and the results of - earlier steps, so that a worker that gets the same message again can find the result. + The blob keeps a small record of the last step for the resource and the results of earlier steps. + A worker that gets the same message again finds the result in it. """ def __init__(self, blob, lease): From e356eaeb2194971f0e14f9e0c2d01d4cbc0eb1e2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:43:29 +0000 Subject: [PATCH 23/37] Write a result over a running record of the same step (#5236) After a lease expiry, the worker writes its record only if the stored record is the record that it read or wrote last. In the meantime, another worker can run the same step again, write its running record and stop during Porter. The first worker then could not write its result, so the worker with the next copy of the step ran Porter a third time. Before "Keep a newer execution record after the lease expired", the first worker wrote its result over that running record. This case needs a lease expiry and a session loss of the first worker, and a stop of the second worker. - A finished record now replaces a running record of the same operation and step. The written record gets the earlier results of the stored record, because a step of another operation can have finished after the first worker started. - The other cases do not change. A result does not replace a finished record or a record of another step. A running record does not replace the running record of another worker, because that worker can still run Porter. - If the other worker still runs Porter, it cannot write its result later and sends its status. The API saves the first final status of the step and ignores the other status. - Tests: the four cases of the rule, with the earlier results of the stored record. - The resource processor document describes the rule. --- docs/tre-developers/resource-processor.md | 3 +- resource_processor/helpers/execution_lease.py | 26 +++++++++++--- .../tests_rp/test_execution_lease.py | 36 +++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 7121d855a6..c7424f29f1 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -130,7 +130,8 @@ Service Bus can give the same message to more than one worker. The resource proc This also applies to a copy of a message that arrives after other steps for the resource. For example, the API can send a step again. 1. When a step starts, the worker keeps the 20 most recent results as earlier results and removes the other results. An earlier result keeps the first 4,000 characters of its status message, because a status message can be up to 2 MB. A copy that arrives after more than 20 other steps for the resource does not find its result, and the worker runs Porter again. -1. If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. After a Porter run, the worker sends the status also if it cannot write the record. +1. If the lease expired before a record write, the worker takes the lease again. It does not replace a record that another worker wrote in the meantime. + If that record shows the same step as running, the worker writes the result of its Porter run over it. After a Porter run, the worker sends the status also if it cannot write the record. 1. Each status message of a Porter run has an `executionId`. After a step has a final status, the API ignores a status from another run, a later status from the same run and a final status that it already saved. 1. If the API wrote the final status itself, for example after a failed send of the next step, the first run of the step replaces it. An uninstall keeps its earlier rule for a final step. 1. An upgrade does not install a deleted resource again. The resource processor sends `updating_failed` if the last successful action of the Porter installation was an uninstall. Without this check, `porter installations apply` runs the install action. diff --git a/resource_processor/helpers/execution_lease.py b/resource_processor/helpers/execution_lease.py index 0ccb3a7839..8f3293e4ab 100644 --- a/resource_processor/helpers/execution_lease.py +++ b/resource_processor/helpers/execution_lease.py @@ -65,15 +65,31 @@ async def write(self, record: dict): self.expires = time.monotonic() + LEASE_SECONDS # Another worker can have taken the lease, written its record and released the lease in the meantime. # Keep that record, so that a copy of its message finds the result. - stored = (await self._download()).get("executionId") - if stored not in (self.execution_id, record.get("executionId")): - raise RuntimeError( - f"Another worker wrote the execution record {self.blob.blob_name} after the lease expired" - ) + stored = await self._download() + if stored.get("executionId") not in (self.execution_id, record.get("executionId")): + if not finishes_running_step(record, stored): + raise RuntimeError( + f"Another worker wrote the execution record {self.blob.blob_name} after the lease expired" + ) + # The other worker ran the same step again and did not write a result. Write this result, so that a + # copy of the message does not run Porter again. Keep the earlier results of the other worker, because + # they can be newer. + record = {**record, "earlierSteps": stored.get("earlierSteps", record.get("earlierSteps", []))} await self.blob.upload_blob(json.dumps(record), overwrite=True, lease=self.lease) self.execution_id = record.get("executionId") +def finishes_running_step(record: dict, stored: dict) -> bool: + """Return True if the record has the result of the step that the stored record shows as running.""" + step = (record.get("operationId"), record.get("stepId")) + return ( + record.get("state") == "finished" + and stored.get("state") == "running" + and None not in step + and (stored.get("operationId"), stored.get("stepId")) == step + ) + + async def create_blob(container: ContainerClient, blob): for attempt in (1, 2): try: diff --git a/resource_processor/tests_rp/test_execution_lease.py b/resource_processor/tests_rp/test_execution_lease.py index 85c7bd296d..957cc4ea77 100644 --- a/resource_processor/tests_rp/test_execution_lease.py +++ b/resource_processor/tests_rp/test_execution_lease.py @@ -155,6 +155,42 @@ async def test_the_running_record_replaces_only_the_record_that_the_worker_read( assert blob.upload_blob.await_count == (2 if replaced else 1) +STEP = {"operationId": "operation-1", "stepId": "step-1"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stored, state, replaced", + [ + # After the lease expired, another worker ran the same step again and stopped before it wrote the result. + pytest.param({**STEP, "state": "running"}, "finished", True, id="same-step-running"), + pytest.param({**STEP, "state": "finished"}, "finished", False, id="same-step-finished"), + pytest.param({**STEP, "stepId": "step-2", "state": "running"}, "finished", False, id="other-step-running"), + # Before Porter starts, the worker does not take the step from a worker that can still run Porter. + pytest.param({**STEP, "state": "running"}, "running", False, id="running-over-running"), + ], +) +async def test_a_result_replaces_only_a_running_record_of_the_same_step(stored, state, replaced): + newer = [{**STEP, "stepId": "step-0", "executionId": "newer-result"}] + content = json.dumps({**stored, "executionId": "other-run", "earlierSteps": newer}).encode() + blob = MagicMock( + upload_blob=AsyncMock(side_effect=[precondition_failed(), None]), + download_blob=AsyncMock(return_value=MagicMock(readall=AsyncMock(return_value=content))), + blob_name="resource-1", + ) + execution = leases.ExecutionLease(blob, MagicMock(acquire=AsyncMock())) + record = {**STEP, "executionId": "this-run", "state": state, "earlierSteps": []} + if replaced: + await execution.write(record) + # The earlier results of the other worker can have a step that finished after this worker started. + assert json.loads(blob.upload_blob.await_args.args[0]) == {**record, "earlierSteps": newer} + assert execution.execution_id == "this-run" + else: + with pytest.raises(RuntimeError): + await execution.write(record) + assert blob.upload_blob.await_count == (2 if replaced else 1) + + @pytest.mark.asyncio async def test_a_lease_that_another_worker_took_stops_the_record_write(): taken = HttpResponseError(message="lease already present") From fb3fcdf51f450b24b98e5f368a403ea64363339a Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 17:58:35 +0000 Subject: [PATCH 24/37] Clear the execution ID of a step for a status without one (#5236) - After a lost session, a second worker can get the message of a step while the first worker still runs Porter. If the management storage account refuses access to the second worker (HTTP 401 or 403), that worker sends a failed status without an execution ID. - Before this change, the step kept the execution ID of the first run together with that failed status. The API then ignored the success of the first run as a later status from the run that finished the step. So a step that succeeded showed the failure. - The executionId of a step names the run whose status the step shows last. A status without an ID has no run, so the API now sets the field also when the message has none. Then a status of a Porter run replaces that status, as it replaces a final status that the API wrote. An uninstall keeps its earlier rule for a final step. - The test for a message without an execution ID now expects no ID on the step. A new test sends a status of run A, a failure without an ID and then the success of run A, for an install, an upgrade and a custom action. It fails on the earlier code. - The resource processor document describes the rule. --- .../service_bus/deployment_status_updater.py | 12 +++++----- .../test_deployment_status_update.py | 22 ++++++++++++++++++- docs/tre-developers/resource-processor.md | 6 ++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index f682efce2b..d2c3974419 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -296,8 +296,9 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage step_to_update.status = message.status step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() - if message.executionId is not None: - step_to_update.executionId = message.executionId + # A status without an execution ID also clears the ID of the step, so a run can replace that status. For + # example, a second worker sends a failure without an ID after HTTP 403, while the first worker still runs. + step_to_update.executionId = message.executionId # Repeated runs make the final status complete only after its writes and dispatch. track_completion = uninstall_operation or message.executionId is not None @@ -415,13 +416,14 @@ def ignore_repeated_status( ) -> bool: """ Service Bus can give a step to a second resource processor worker. Keep the first final status of the step. - A message without an execution ID comes from an older resource processor, so the earlier rules apply to it. + A message without an execution ID comes from an older resource processor, or from a worker after a storage + access error. The earlier rules apply to it. """ if message.executionId is None or not (step.is_success() or step.is_failure()): return False if step.executionId is None: - # The API or an older resource processor wrote this final status. The first run of the step replaces it, - # for example after the API failed a step whose message the resource processor still got. + # The API wrote this final status, or a message without an execution ID set it. The first run of the step + # replaces it, for example after the API failed a step whose message the resource processor still got. return False if step.executionId != message.executionId: logger.info(f"Ignoring a status from another run of step {operation.id}/{step.id}") diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index dffe764472..75892cf245 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -1229,6 +1229,25 @@ async def test_the_first_run_replaces_a_failure_that_the_api_wrote(persisted_pip send.assert_awaited_once() +# An uninstall keeps the earlier rule for a final step from #5253. +@pytest.mark.parametrize("persisted_pipeline", [RequestAction.Install, RequestAction.Upgrade, "restart"], indirect=True) +async def test_the_running_execution_replaces_a_failure_without_an_execution_id(persisted_pipeline): + updater, stored, message, send, (in_progress, success, failure) = persisted_pipeline + assert await updater.update_status_in_database(message(in_progress)) is True + # After a lost session, a second worker got HTTP 403 from the management storage account. It sent a failure + # without an execution ID, while the first worker still ran Porter. + assert await updater.update_status_in_database(message(failure, execution_id=None)) is True + assert stored["operation"].steps[0].status == failure + + assert await updater.update_status_in_database(message(success)) is True + + step = stored["operation"].steps[0] + assert step.status == success + assert step.executionId == "run-a" + assert stored["resources"][step.resourceId].deploymentStatus == success + send.assert_awaited_once() + + async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): workspace = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") operation = create_sample_operation(workspace.id, RequestAction.Install) @@ -1251,7 +1270,8 @@ async def test_a_message_without_an_execution_id_keeps_the_earlier_rules(): assert await updater.update_status_in_database(message) is True assert operation.steps[0].status == Status.DeploymentFailed - assert operation.steps[0].executionId == "run-a" + # The step shows a status without an execution ID, so the first run of the step can replace it. + assert operation.steps[0].executionId is None @patch("service_bus.deployment_status_updater.logger") diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index c7424f29f1..81f829ccd3 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -138,7 +138,11 @@ Service Bus can give the same message to more than one worker. The resource proc An upgrade can still repair a failed install, also an install that failed before Porter wrote its record. If the resource processor cannot read the Porter installation in 30 seconds, for example because the Porter storage is not available, it also sends `updating_failed`. -A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it. An API before version 0.29.0 ignores the field. So you can deploy the API and the resource processor in any order. +A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it and removes the `executionId` of the step. +Then a status of a Porter run replaces that status. For example, the success of a worker that ran Porter replaces the failed status of a second worker after a storage access error. +An uninstall keeps its earlier rule for a final step. + +An API before version 0.29.0 ignores the field. So you can deploy the API and the resource processor in any order. The resource processor now needs the management storage account for each step. If the account refuses access (HTTP 401 or 403), the resource processor sends the failed status for the step. A Terraform backend in the same account failed in the same way before this change. For other storage errors, Service Bus delivers the message again. From d30f21ee1711babcad66b2b023ac3d77e4a36c1a Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 18:03:21 +0000 Subject: [PATCH 25/37] Fail an unsent step only if the operation did not change after the claim (#5236) - The claim of a pending entry patched the send time with the etag of the read, but it did not keep the new etag. fail_pending_step then wrote the operation that the task read before the claim, without an etag. - If the API sent the message but stopped before it removed the entry, the resource processor can send a status while the task runs. The task then replaced that status with a failure and cleared the executionId of the step. For a final status, no later status repaired the step. - claim_pending_dispatch now returns the etag after the claim, or None after HTTP 412. fail_pending_step writes the operation with this etag. - If the operation changed after the claim, the write fails. Only a status from the resource processor, the removal of the entry after a send and a claim of another API instance change the operation, and each of these writers owns the step. So the task stops and changes neither the operation nor the resource. It does not read the operation again. If the entry stays, the next pass reads it again. - The operation write stays before the resource write, because the etag check must decide before the task changes the resource. - Tests: the claim returns the new etag or None, each failed step uses the etag of the claim, and a failed conditional write changes neither the operation nor the resource after a later request, with sensitive values and after a build error. The new tests fail on the earlier code. - The resource processor document describes the rule. --- api_app/db/repositories/operations.py | 12 ++-- .../service_bus/deployment_status_updater.py | 23 +++++--- .../test_operation_repository.py | 8 ++- .../test_service_bus/test_pending_dispatch.py | 55 +++++++++++++++---- docs/tre-developers/resource-processor.md | 3 + 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/api_app/db/repositories/operations.py b/api_app/db/repositories/operations.py index d688226d87..d6181f219d 100644 --- a/api_app/db/repositories/operations.py +++ b/api_app/db/repositories/operations.py @@ -55,10 +55,12 @@ async def get_operations_with_pending_dispatch(self, sent_before: float) -> List query = "SELECT * FROM c WHERE IS_DEFINED(c.pendingDispatch) AND c.pendingDispatch.createdWhen < @sentBefore" return await self.query(query=query, parameters=[{"name": "@sentBefore", "value": sent_before}]) - async def claim_pending_dispatch(self, operation: dict) -> bool: - """Move the send time of the entry forward. Only one API instance can claim the document that it read.""" + async def claim_pending_dispatch(self, operation: dict) -> Optional[str]: + """Move the send time of the entry forward. Only one API instance can claim the document that it read. + Return the etag after the claim, or None if another writer changed the document first. + """ try: - await self.container.patch_item( + claimed = await self.container.patch_item( item=operation["id"], partition_key=operation["id"], patch_operations=[ @@ -70,8 +72,8 @@ async def claim_pending_dispatch(self, operation: dict) -> bool: except CosmosHttpResponseError as error: if error.status_code != 412: raise - return False - return True + return None + return claimed["_etag"] async def clear_pending_dispatch(self, operation_id: str, message_id: str): """Remove the entry if it still names this message. A full write of the operation also removes it.""" diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index d2c3974419..49cb2da003 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -11,6 +11,7 @@ from models.domain.request_action import RequestAction from db.repositories.resource_templates import ResourceTemplateRepository from service_bus.helpers import send_deployment_message, update_resource_for_step +from azure.cosmos.exceptions import CosmosAccessConditionFailedError from azure.servicebus import NEXT_AVAILABLE_SESSION from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError from azure.servicebus.aio import ServiceBusClient, AutoLockRenewer @@ -136,7 +137,8 @@ async def resend_pending_dispatches(self): async def resend_pending_dispatch(self, document: dict): pending = document[PENDING_DISPATCH] # Each API instance runs this task. The claim lets one instance send the message. - if not await self.operations_repo.claim_pending_dispatch(document): + etag = await self.operations_repo.claim_pending_dispatch(document) + if etag is None: return operation = TypeAdapter(Operation).validate_python(document) step = next((item for item in operation.steps if item.id == pending["stepId"]), None) @@ -157,7 +159,7 @@ async def resend_pending_dispatch(self, document: dict): ) if later_operations: logger.warning(f"Operation {operation.id} was not sent before a later request. Failing step {step.id}") - await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST, write_resource=False) + await self.fail_pending_step(operation, step, NOT_SENT_BEFORE_A_LATER_REQUEST, etag, write_resource=False) return # If an install or an update set a sensitive property, the first message has the real value, and Cosmos DB @@ -165,7 +167,7 @@ async def resend_pending_dispatch(self, document: dict): # values as Cosmos DB, so a resend has the same values as the first send. if pending.get("sensitiveValues") and step.resourceId == operation.resourceId: logger.warning(f"Operation {operation.id} was not sent and has sensitive values. Failing step {step.id}") - await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES) + await self.fail_pending_step(operation, step, NOT_SENT_WITH_SENSITIVE_VALUES, etag) return try: @@ -185,7 +187,7 @@ async def resend_pending_dispatch(self, document: dict): ) except Exception as e: logger.exception(f"Unable to build the pending message of operation {operation.id}") - await self.fail_pending_step(operation, step, repr(e)) + await self.fail_pending_step(operation, step, repr(e), etag) return logger.info(f"Sending the pending first step message of operation {operation.id}, step {step.id}") @@ -199,7 +201,7 @@ async def resend_pending_dispatch(self, document: dict): await self.operations_repo.clear_pending_dispatch(operation.id, pending["messageId"]) async def fail_pending_step( - self, operation: Operation, step: OperationStep, message: str, write_resource: bool = True + self, operation: Operation, step: OperationStep, message: str, etag: str, write_resource: bool = True ): """Fail a step that the API did not send. The step did not run, so only its request fails.""" step.status = self.get_failure_status_for_action(step.resourceAction) @@ -208,8 +210,15 @@ async def fail_pending_step( operation.status = self.get_failure_status_for_action(operation.action) operation.message = message operation.updatedWhen = step.updatedWhen - # The Operation model has no pending dispatch, so this write also removes the entry. - await self.operations_repo.update_item(operation) + # The Operation model has no pending dispatch, so this write also removes the entry. It uses the etag of the + # claim and comes before the resource write, so a change after the claim stops both writes. + try: + await self.operations_repo.upsert_item_with_etag(operation, etag) + except CosmosAccessConditionFailedError: + # A status from the resource processor, the removal of the entry after a send or a claim of another + # instance changed the operation. That writer owns the step. If the entry stays, the next pass reads it. + logger.info(f"Operation {operation.id} changed after the claim. Step {step.id} keeps its status") + return if not write_resource: # A later request changes the resource, so it owns the status of the resource. return diff --git a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py index 1dcf10d111..7d02e305ac 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_operation_repository.py @@ -183,8 +183,10 @@ async def operations_container(): async def test_claim_pending_dispatch_uses_the_etag_of_the_read_document(operations_container): operations_repo, container = operations_container + container.patch_item.return_value = {"id": OPERATION_ID, "_etag": "claimed"} - assert await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) + # The task writes a failed step only with the etag after the claim. + assert await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) == "claimed" kwargs = container.patch_item.await_args.kwargs assert kwargs["item"] == OPERATION_ID @@ -193,11 +195,11 @@ async def test_claim_pending_dispatch_uses_the_etag_of_the_read_document(operati assert kwargs["patch_operations"][0]["path"] == "/pendingDispatch/createdWhen" -async def test_claim_pending_dispatch_returns_false_after_another_write(operations_container): +async def test_claim_pending_dispatch_returns_none_after_another_write(operations_container): operations_repo, container = operations_container container.patch_item.side_effect = CosmosHttpResponseError(status_code=412) - assert not await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) + assert await operations_repo.claim_pending_dispatch({"id": OPERATION_ID, "_etag": "etag"}) is None async def test_claim_pending_dispatch_raises_other_errors(operations_container): diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index aa9d9ddf73..5528664c24 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, patch import pytest +from azure.cosmos.exceptions import CosmosAccessConditionFailedError from db.errors import EntityDoesNotExist from db.repositories.operations import PENDING_DISPATCH @@ -23,6 +24,7 @@ FIREWALL_ID = "a8b2c3d4-1111-4222-8333-944455556666" CHILD_ID = "c3d4e5f6-2222-4333-8444-a55566667777" MESSAGE_ID = f"{OPERATION_ID}:random-uuid" +CLAIM_ETAG = "claim-etag" AWAITING = { "install": Status.AwaitingDeployment, "upgrade": Status.AwaitingUpdate, @@ -63,12 +65,19 @@ def updater(): updater.resource_repo = AsyncMock() updater.resource_template_repo = AsyncMock() updater.resource_history_repo = AsyncMock() - updater.operations_repo.claim_pending_dispatch.return_value = True + updater.operations_repo.claim_pending_dispatch.return_value = CLAIM_ETAG updater.resource_repo.get_resource_by_id.return_value = workspace({}) updater.operations_repo.get_later_operations_for_resource.return_value = [] return updater +def failed_operation(updater): + """Return the operation that the task wrote. The write uses the etag of the claim.""" + operation, etag = updater.operations_repo.upsert_item_with_etag.await_args.args + assert etag == CLAIM_ETAG + return operation + + @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") async def test_sends_the_pending_message_then_clears_the_entry(update_mock, send_mock, updater): @@ -91,7 +100,7 @@ async def test_sends_the_pending_message_then_clears_the_entry(update_mock, send @patch("service_bus.deployment_status_updater.send_deployment_message") @patch("service_bus.deployment_status_updater.update_resource_for_step") async def test_an_entry_that_another_instance_claimed_is_left(update_mock, send_mock, updater): - updater.operations_repo.claim_pending_dispatch.return_value = False + updater.operations_repo.claim_pending_dispatch.return_value = None await updater.resend_pending_dispatch(pending_document()) @@ -126,7 +135,7 @@ async def test_a_first_message_with_sensitive_values_fails_the_step( update_mock.assert_not_awaited() send_mock.assert_not_awaited() - operation = updater.operations_repo.update_item.await_args.args[0] + operation = failed_operation(updater) assert operation.steps[0].status == failed assert operation.steps[0].message == NOT_SENT_WITH_SENSITIVE_VALUES assert operation.status == failed @@ -144,7 +153,7 @@ async def test_sensitive_values_of_another_resource_do_not_stop_the_send(update_ await updater.resend_pending_dispatch(document) send_mock.assert_awaited_once() - updater.operations_repo.update_item.assert_not_awaited() + updater.operations_repo.upsert_item_with_etag.assert_not_awaited() @pytest.mark.parametrize("action", ["install", "upgrade", "uninstall", "start"]) @@ -157,7 +166,7 @@ async def test_a_resend_without_sensitive_values_sends_the_placeholder(update_mo await updater.resend_pending_dispatch(pending_document(action=action, status=AWAITING[action])) send_mock.assert_awaited_once() - updater.operations_repo.update_item.assert_not_awaited() + updater.operations_repo.upsert_item_with_etag.assert_not_awaited() updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) @@ -176,7 +185,7 @@ async def test_a_later_request_for_the_resource_fails_the_step(update_mock, send ) update_mock.assert_not_awaited() send_mock.assert_not_awaited() - operation = updater.operations_repo.update_item.await_args.args[0] + operation = failed_operation(updater) assert operation.steps[0].status == Status.DeploymentFailed assert operation.steps[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST assert operation.status == Status.DeploymentFailed @@ -205,7 +214,7 @@ async def test_a_later_request_for_the_first_child_of_a_cascade_fails_the_step(u assert queried == [(RESOURCE_ID, document["createdWhen"]), (CHILD_ID, document["createdWhen"])] update_mock.assert_not_awaited() send_mock.assert_not_awaited() - operation = updater.operations_repo.update_item.await_args.args[0] + operation = failed_operation(updater) assert operation.steps[0].status == Status.UpdatingFailed assert operation.steps[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST updater.resource_repo.update_item.assert_not_awaited() @@ -223,7 +232,7 @@ async def test_a_later_request_comes_before_the_sensitive_values(update_mock, se await updater.resend_pending_dispatch(document) send_mock.assert_not_awaited() - assert updater.operations_repo.update_item.await_args.args[0].message == NOT_SENT_BEFORE_A_LATER_REQUEST + assert failed_operation(updater).message == NOT_SENT_BEFORE_A_LATER_REQUEST updater.resource_repo.update_item.assert_not_awaited() @@ -245,7 +254,7 @@ async def test_a_failed_first_step_on_another_resource_changes_only_the_resource updater.operations_repo.get_later_operations_for_resource.assert_awaited_once_with( RESOURCE_ID, document["createdWhen"] ) - operation = updater.operations_repo.update_item.await_args.args[0] + operation = failed_operation(updater) assert operation.steps[0].status == Status.UpdatingFailed assert operation.status == Status.DeploymentFailed updater.resource_repo.get_resource_by_id.assert_awaited_once_with(uuid.UUID(RESOURCE_ID)) @@ -261,7 +270,7 @@ async def test_a_missing_resource_still_removes_the_entry(update_mock, send_mock await updater.resend_pending_dispatch(pending_document()) # The operation write removes the entry, so the next pass does not claim it again. - assert updater.operations_repo.update_item.await_args.args[0].status == Status.DeploymentFailed + assert failed_operation(updater).status == Status.DeploymentFailed updater.resource_repo.update_item.assert_not_awaited() @@ -273,11 +282,35 @@ async def test_a_message_that_cannot_be_built_fails_the_step(update_mock, send_m await updater.resend_pending_dispatch(pending_document()) send_mock.assert_not_awaited() - operation = updater.operations_repo.update_item.await_args.args[0] + operation = failed_operation(updater) assert operation.steps[0].status == Status.DeploymentFailed assert "template not found" in operation.steps[0].message +@pytest.mark.parametrize("reason", ["later_request", "sensitive_values", "build_error"]) +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +async def test_a_status_after_the_claim_is_not_replaced(update_mock, send_mock, updater, reason): + document = pending_document(sensitive_values=reason == "sensitive_values") + if reason == "later_request": + later = create_sample_operation(RESOURCE_ID, "uninstall") + later.createdWhen = document["createdWhen"] + 60 + updater.operations_repo.get_later_operations_for_resource.return_value = [later] + update_mock.side_effect = Exception("template not found") + # The API sent the message but stopped before it removed the entry. A status from the resource processor changed + # the operation after the claim, so the write with the etag of the claim fails. + updater.operations_repo.upsert_item_with_etag.side_effect = CosmosAccessConditionFailedError(status_code=412) + + await updater.resend_pending_dispatch(document) + + # The task changes neither the operation nor the resource. + updater.operations_repo.update_item.assert_not_awaited() + updater.resource_repo.get_resource_by_id.assert_not_awaited() + updater.resource_repo.update_item.assert_not_awaited() + send_mock.assert_not_awaited() + failed_operation(updater) + + @patch("service_bus.deployment_status_updater.asyncio.sleep", AsyncMock(side_effect=asyncio.CancelledError)) async def test_a_failed_entry_does_not_stop_the_pass(updater): first, second = pending_document(), pending_document() diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 81f829ccd3..73262167cc 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -173,6 +173,9 @@ If the API stops between the write and the send, the entry stays. When the API s In the last two cases, the updater also sets the resource of the request to the failed status. A first step on another resource, for example the firewall, did not run, so that resource keeps its status. +The updater writes the failed status with the etag of its claim. If the operation changed after the claim, for example because the resource processor sent a status, this write fails. +Then the updater changes neither the operation nor the resource. + A second copy of a message is safe. The resource processor finds the result of the step in the execution record and sends the recorded status. If the record does not have the result of the first run, Porter runs again, and the API ignores the status of the second run. The entry stays if the API sent the message but stopped before it removed the entry. If the resource processor then sends a status for a step that the updater set to failed, the API saves that status. An uninstall keeps its earlier rule for a final step. From a73d2d6116ca7e3d23edab15bb92cb340f092c66 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 19:05:08 +0000 Subject: [PATCH 26/37] Keep a property sensitive if one of its declarations is sensitive (#5298) - A template can declare a property name more than once, for example at the top level and again in an allOf, anyOf, oneOf, then or else branch. sensitive_property_names kept only the last declaration of each name. So a later branch that adds a rule to the property, for example a pattern, removed the sensitive flag. - Then an update stored the real value, and the migration skipped the template. The install on main has the same behaviour, so this change also applies to the install. - A name is now sensitive if one or more of its declarations has sensitive set to true. The install, the update, the history item, the sensitiveValues flag of the operation and the migration all use this function. - Tests: then, else, anyOf and oneOf branches that declare a sensitive property again without the flag, and a later allOf branch after a sensitive one. The migration finds such a template. The new tests fail on the earlier code. - The template authoring document and the CHANGELOG describe the rule. --- CHANGELOG.md | 1 + api_app/db/repositories/resources.py | 55 ++++++-------- .../test_resource_repository.py | 74 ++++++++++++++++++- .../authoring-workspace-templates.md | 2 + 4 files changed, 99 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863f274a28..8a6f2be958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ BUG FIXES: Then update the workspace with the new secret, and delete the old secret. If that update fails, send it again with the new secret. The update form shows `REDACTED`, and an update with `REDACTED` keeps the earlier secret. Delete the old secret only after an update with the new secret succeeds. + If a template declares a property in more than one branch, one declaration with `sensitive` is enough. For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. The create route of an Airlock review VM did the same in its HTTP 500 response. diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index 67b0f286af..769d97b814 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -37,41 +37,32 @@ ) -def template_properties(template: dict) -> dict: - """Return each property schema of a template by name, also from allOf, then and else branches.""" - flattened_template_props = {} +def sensitive_property_names(template: dict) -> set: + """Return the name of each property that has sensitive set to true in one or more of its declarations. - # Recurse through the template and collect all properties in one dict. - def flatten_template_props(template_fragment: dict): - if template_fragment is None: - return + The template can declare a name more than once, for example at the top level and in an allOf, anyOf, oneOf, + then or else branch. A declaration without the flag does not remove it. Nested objects and array items count, + but if conditions and the pipeline do not. + """ + names = set() - for prop_name, prop in template_fragment.items(): - if prop_name == "pipeline": + def collect(template_fragment: dict): + for key, value in template_fragment.items(): + if key in ("pipeline", "if"): continue - - if prop_name == "properties" and template_fragment["properties"] is not None: - for inner_prop_name, inner_prop in template_fragment["properties"].items(): - flattened_template_props[inner_prop_name] = inner_prop - - if isinstance(prop, list) and len(prop) > 0: - for p in prop: - if isinstance(p, dict): - flatten_template_props(p) - - if isinstance(prop, dict) and prop_name != "if": - flatten_template_props(prop) - - flatten_template_props(template) - return flattened_template_props - - -def sensitive_property_names(template: dict) -> set: - return { - name - for name, prop in template_properties(template).items() - if isinstance(prop, dict) and prop.get("sensitive") is True - } + if key == "properties" and isinstance(value, dict): + names.update( + name for name, prop in value.items() if isinstance(prop, dict) and prop.get("sensitive") is True + ) + if isinstance(value, dict): + collect(value) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + collect(item) + + collect(template) + return names def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 9e199bd91f..f751edfc17 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -12,7 +12,12 @@ from tests_ma.test_api.conftest import create_test_user from db.errors import EntityDoesNotExist, UserNotAuthorizedToUseTemplate -from db.repositories.resources import ResourceRepository, masked_property_paths, set_masked_values +from db.repositories.resources import ( + ResourceRepository, + mask_sensitive_properties, + masked_property_paths, + set_masked_values, +) from azure.cosmos.exceptions import CosmosResourceNotFoundError from models.domain.resource import Resource from models.domain.resource_template import ResourceTemplate @@ -535,6 +540,19 @@ async def history_query(query, parameters): ) +def masking_template(properties, all_of=None) -> ResourceTemplate: + return ResourceTemplate( + id="masking-template-id", + name="masking-template", + version="1.0.0", + resourceType=ResourceType.Workspace, + current=True, + required=[], + properties=properties, + allOf=all_of, + ) + + def test_masked_property_paths_escape_nested_names(): properties = {"settings": {"a/b": "real", "c~d": "real", "plain": "x"}} masked = {"settings": {"a/b": "REDACTED", "c~d": "REDACTED", "plain": "x"}} @@ -552,6 +570,60 @@ async def test_set_masked_values_sends_at_most_ten_operations_in_each_request(): assert [len(call.kwargs["patch_operations"]) for call in container.patch_item.await_args_list] == [10, 1] +PASSWORD = {"type": "string", "sensitive": True, "updateable": True} +PASSWORD_RULE = {"properties": {"password": {"type": "string", "pattern": "^.{8,}$"}}} +IF_MODE_A = {"properties": {"mode": {"const": "a"}}} + + +@pytest.mark.parametrize( + "properties, all_of", + [ + ({"password": PASSWORD}, [{"if": IF_MODE_A, "then": PASSWORD_RULE}]), + ({"password": PASSWORD}, [{"if": IF_MODE_A, "then": {}, "else": PASSWORD_RULE}]), + ({"password": PASSWORD}, [{"anyOf": [PASSWORD_RULE]}]), + ({"password": PASSWORD}, [{"oneOf": [PASSWORD_RULE, {}]}]), + ( + {"password": {"type": "string"}}, + [ + {"if": IF_MODE_A, "then": {"properties": {"password": PASSWORD}}}, + {"if": IF_MODE_A, "then": PASSWORD_RULE}, + ], + ), + ], +) +def test_a_property_stays_sensitive_if_one_of_its_declarations_is_sensitive(properties, all_of): + # A later declaration of the same name without the flag must not remove it. + masked = mask_sensitive_properties({"password": "real value", "mode": "a"}, masking_template(properties, all_of)) + + assert masked == {"password": strings.REDACTED_SENSITIVE_VALUE, "mode": "a"} + + +def history_without_items(): + history_repo = MagicMock() + history_repo.container = AsyncMock() + history_repo.query = AsyncMock(return_value=[]) + return history_repo + + +@pytest.mark.asyncio +async def test_mask_stored_sensitive_values_finds_a_property_that_a_branch_declares_again(resource_repo): + template = masking_template({"password": PASSWORD}, [{"if": IF_MODE_A, "then": PASSWORD_RULE}]) + resource_repo._container = AsyncMock() + resource_repo.query = AsyncMock( + return_value=[stored_resource("ws", "masking-template", "workspace", {"password": "real"})] + ) + template_repo = AsyncMock() + template_repo.query.return_value = [template.model_dump()] + template_repo.get_template_by_name_and_version.return_value = template + + assert await resource_repo.mask_stored_sensitive_values(template_repo, history_without_items()) == (1, 0) + resource_repo.container.patch_item.assert_awaited_once_with( + item="ws", + partition_key="ws", + patch_operations=[{"op": "set", "path": "/properties/password", "value": strings.REDACTED_SENSITIVE_VALUE}], + ) + + @patch("db.repositories.resources.ResourceTemplateRepository.enrich_template") def test_validate_patch_with_good_fields_passes(template_repo, resource_repo): """ diff --git a/docs/tre-workspace-authors/authoring-workspace-templates.md b/docs/tre-workspace-authors/authoring-workspace-templates.md index 2df9f75327..7185a72456 100644 --- a/docs/tre-workspace-authors/authoring-workspace-templates.md +++ b/docs/tre-workspace-authors/authoring-workspace-templates.md @@ -105,6 +105,8 @@ Put conditional rules inside `allOf`, rather than using root `if`, `then` or `el Keep Azure TRE annotations such as `updateable`, `sensitive` and `readOnly` on the property itself when using a `$ref`. A reusable definition does not supply these annotations to Azure TRE's resource processing. +The API stores `REDACTED` in place of each sensitive value, at the install and at an update. Only the first message of a request has the real value. The API finds a sensitive value by its property name, also in a nested object. If one declaration of a name has `sensitive: true`, for example in an `allOf` branch, the name is sensitive. + An API upgrade does not rewrite stored templates or add missing metadata. To adopt Draft 2020-12, review the custom template's keywords, increment its bundle version and register the updated bundle. Changing only `$schema` can change validation semantics. Test creation and updates before upgrading existing resources. !!! note From 275b33a4235b9479a25d8b5e7e1edeefe367e1aa Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 19:05:55 +0000 Subject: [PATCH 27/37] Mask sensitive values in array items (#5298) - mask_sensitive_properties went into nested objects but not into lists. For a template with a sensitive property in the items of an array, the install and the update stored the real value of each item. The migration did not find these values either. The install on main has the same behaviour. - sensitive_values now gives the path of each sensitive value, also in array items. mask_sensitive_properties sets the placeholder at these paths. masked_property_paths gives the JSON pointers of the same paths for the migration. A pointer has an index for each list item, so the migration does not replace a whole list. So the install, the update and the migration mask the same values. - restore_sensitive_values pairs the items of two lists of the same length by index. So a pipeline step of the primary resource still gets the real values of its array items. A list with a different length keeps its placeholders, because its items can be different items. - In a later action that does not give a new value for such a property, the bundle now gets REDACTED, as for other sensitive properties. - Tests: the masked values and the migration paths for array items, nested lists and nested objects, the update with its history item, the migration of a resource and a history item, and the restore of array items. The new tests fail on the earlier code. - The template authoring document and the CHANGELOG describe the change. --- CHANGELOG.md | 1 + api_app/db/repositories/resources.py | 89 +++++++++------ .../test_resource_repository.py | 106 +++++++++++++++++- .../authoring-workspace-templates.md | 4 +- 4 files changed, 163 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a6f2be958..9b87dda4e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ BUG FIXES: Then update the workspace with the new secret, and delete the old secret. If that update fails, send it again with the new secret. The update form shows `REDACTED`, and an update with `REDACTED` keeps the earlier secret. Delete the old secret only after an update with the new secret succeeds. + The placeholder also covers a sensitive property in an array item, at the install too. In a later action that does not give a new value for such a property, the bundle gets `REDACTED`, as for other sensitive properties. If a template declares a property in more than one branch, one declaration with `sensitive` is enough. For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index 769d97b814..0d133f72cc 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -65,45 +65,67 @@ def collect(template_fragment: dict): return names -def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: - updated_resource_parameters = copy.deepcopy(properties) - sensitive_names = sensitive_property_names(template.model_dump()) +def sensitive_values(properties: Dict[str, Any], template: ResourceTemplate) -> List[Tuple[tuple, Any]]: + """Return the path and the value of each sensitive value in properties. + + A path is a tuple of keys and list indexes. A value is sensitive if its key is a sensitive property name of the + template, also in a nested object or in an array item. The install, the update and the migration all use this + function, so they mask the same values. + """ + names = sensitive_property_names(template.model_dump()) + found = [] - def recurse_input_props(prop_dict: dict): - for prop_name, prop in prop_dict.items(): - if prop_name in sensitive_names: - prop_dict[prop_name] = REDACTED_SENSITIVE_VALUE - if isinstance(prop, dict): - recurse_input_props(prop) + def collect(value: Any, path: tuple): + if isinstance(value, dict): + for key, item in value.items(): + if key in names: + found.append((path + (key,), item)) + else: + collect(item, path + (key,)) + elif isinstance(value, list): + for index, item in enumerate(value): + collect(item, path + (index,)) - recurse_input_props(updated_resource_parameters) + collect(properties, ()) + return found - return updated_resource_parameters + +def mask_sensitive_properties(properties: Dict[str, Any], template: ResourceTemplate) -> dict: + masked = copy.deepcopy(properties) + for path, _ in sensitive_values(masked, template): + parent = masked + for part in path[:-1]: + parent = parent[part] + parent[path[-1]] = REDACTED_SENSITIVE_VALUE + return masked def restore_sensitive_values(properties: dict, source: dict) -> dict: - """Return a copy of properties with the real value from source in place of each placeholder.""" - restored = copy.deepcopy(properties) - for key, value in restored.items(): - original = source.get(key) - if isinstance(value, dict) and isinstance(original, dict): - restored[key] = restore_sensitive_values(value, original) - elif value == REDACTED_SENSITIVE_VALUE and original is not None and original != REDACTED_SENSITIVE_VALUE: - restored[key] = copy.deepcopy(original) - return restored - - -def masked_property_paths(properties: dict, masked: dict, path: Optional[List[str]] = None) -> List[str]: - """Return a JSON pointer for each value that mask_sensitive_properties changed.""" - path = path or ["properties"] - paths = [] - for key, value in masked.items(): - original = properties.get(key) + """Return a copy of properties with the real value from source in place of each placeholder. + + Two lists of the same length pair their items by index. A list with a different length keeps its placeholders, + because its items can be different items. + """ + + def restore(value: Any, original: Any) -> Any: if isinstance(value, dict) and isinstance(original, dict): - paths += masked_property_paths(original, value, path + [key]) - elif value != original: - paths.append("".join("/" + part.replace("~", "~0").replace("/", "~1") for part in path + [key])) - return paths + return {key: restore(item, original.get(key)) for key, item in value.items()} + if isinstance(value, list) and isinstance(original, list) and len(value) == len(original): + return [restore(item, original_item) for item, original_item in zip(value, original)] + if value == REDACTED_SENSITIVE_VALUE and original is not None and original != REDACTED_SENSITIVE_VALUE: + return copy.deepcopy(original) + return copy.deepcopy(value) + + return restore(properties, source) + + +def masked_property_paths(properties: dict, template: ResourceTemplate) -> List[str]: + """Return a JSON pointer for each sensitive value in properties that is not the placeholder.""" + return [ + "".join("/" + str(part).replace("~", "~0").replace("/", "~1") for part in ("properties",) + path) + for path, value in sensitive_values(properties, template) + if value != REDACTED_SENSITIVE_VALUE + ] async def set_masked_values(container, item_id: str, partition_key: str, paths: List[str]): @@ -350,8 +372,7 @@ async def template_for(resource: dict, version: str) -> Optional[ResourceTemplat async def mask(item: dict, template: Optional[ResourceTemplate], container, partition_key: str) -> bool: if template is None: return False - properties = item.get("properties") or {} - paths = masked_property_paths(properties, mask_sensitive_properties(properties, template)) + paths = masked_property_paths(item.get("properties") or {}, template) if paths: await set_masked_values(container, item["id"], partition_key, paths) return bool(paths) diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index f751edfc17..0e84adf98f 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -16,6 +16,7 @@ ResourceRepository, mask_sensitive_properties, masked_property_paths, + restore_sensitive_values, set_masked_values, ) from azure.cosmos.exceptions import CosmosResourceNotFoundError @@ -540,6 +541,16 @@ async def history_query(query, parameters): ) +SECRET = {"type": "string", "sensitive": True} +ACCOUNTS = { + "accounts": { + "type": "array", + "updateable": True, + "items": {"type": "object", "properties": {"name": {"type": "string"}, "token": SECRET}}, + } +} + + def masking_template(properties, all_of=None) -> ResourceTemplate: return ResourceTemplate( id="masking-template-id", @@ -554,10 +565,10 @@ def masking_template(properties, all_of=None) -> ResourceTemplate: def test_masked_property_paths_escape_nested_names(): + template = masking_template({"settings": {"type": "object", "properties": {"a/b": SECRET, "c~d": SECRET}}}) properties = {"settings": {"a/b": "real", "c~d": "real", "plain": "x"}} - masked = {"settings": {"a/b": "REDACTED", "c~d": "REDACTED", "plain": "x"}} - assert masked_property_paths(properties, masked) == ["/properties/settings/a~1b", "/properties/settings/c~0d"] + assert masked_property_paths(properties, template) == ["/properties/settings/a~1b", "/properties/settings/c~0d"] @pytest.mark.asyncio @@ -624,6 +635,97 @@ async def test_mask_stored_sensitive_values_finds_a_property_that_a_branch_decla ) +def test_the_migration_finds_the_values_that_an_install_or_an_update_masks(): + template = masking_template({**ACCOUNTS, "settings": {"type": "object", "properties": {"password": SECRET}}}) + properties = { + "accounts": [{"name": "a", "token": "first"}, {"name": "b", "token": strings.REDACTED_SENSITIVE_VALUE}], + "groups": [[{"token": "nested"}]], + "settings": {"password": "real", "name": "token"}, + } + + assert mask_sensitive_properties(properties, template) == { + "accounts": [ + {"name": "a", "token": strings.REDACTED_SENSITIVE_VALUE}, + {"name": "b", "token": strings.REDACTED_SENSITIVE_VALUE}, + ], + "groups": [[{"token": strings.REDACTED_SENSITIVE_VALUE}]], + "settings": {"password": strings.REDACTED_SENSITIVE_VALUE, "name": "token"}, + } + # The migration changes only the values that are not the placeholder yet, and not a whole list. + assert masked_property_paths(properties, template) == [ + "/properties/accounts/0/token", + "/properties/groups/0/0/token", + "/properties/settings/password", + ] + + +@pytest.mark.asyncio +@patch("db.repositories.resources.ResourceRepository.validate_patch") +@patch("db.repositories.resources.ResourceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +async def test_patch_resource_stores_a_placeholder_for_sensitive_values_in_array_items(_, __, resource_repo): + resource_repo.update_item_with_etag = AsyncMock() + history_repo = AsyncMock() + resource = sample_resource() + resource.properties["accounts"] = [{"name": "a", "token": "earlier value"}] + resource_patch = ResourcePatch(properties={"accounts": [{"name": "a", "token": "new value"}]}) + + returned, _ = await resource_repo.patch_resource( + resource, + resource_patch, + masking_template(ACCOUNTS), + "etag", + None, + history_repo, + create_test_user(), + strings.RESOURCE_ACTION_UPDATE, + ) + + masked_accounts = [{"name": "a", "token": strings.REDACTED_SENSITIVE_VALUE}] + assert resource_repo.update_item_with_etag.await_args.args[0].properties["accounts"] == masked_accounts + assert returned.properties["accounts"] == [{"name": "a", "token": "new value"}] + assert history_repo.create_resource_history_item.await_args.args[0].properties["accounts"] == masked_accounts + + +@pytest.mark.asyncio +async def test_mask_stored_sensitive_values_masks_values_in_array_items(resource_repo): + template = masking_template(ACCOUNTS) + resource_repo._container = AsyncMock() + accounts = [{"name": "a", "token": "real"}, {"name": "b", "token": strings.REDACTED_SENSITIVE_VALUE}] + resource_repo.query = AsyncMock( + return_value=[stored_resource("ws", "masking-template", "workspace", {"accounts": accounts})] + ) + template_repo = AsyncMock() + template_repo.query.return_value = [template.model_dump()] + template_repo.get_template_by_name_and_version.return_value = template + history_repo = history_without_items() + history_repo.query.return_value = [{"id": "h-1", "resourceId": "ws", "properties": {"accounts": accounts[:1]}}] + + assert await resource_repo.mask_stored_sensitive_values(template_repo, history_repo) == (1, 1) + set_token = [{"op": "set", "path": "/properties/accounts/0/token", "value": strings.REDACTED_SENSITIVE_VALUE}] + resource_repo.container.patch_item.assert_awaited_once_with( + item="ws", partition_key="ws", patch_operations=set_token + ) + history_repo.container.patch_item.assert_awaited_once_with( + item="h-1", partition_key="ws", patch_operations=set_token + ) + + +def test_restore_sensitive_values_restores_values_in_array_items(): + # A pipeline step of the primary resource reads the stored document and gets the real values from memory. + placeholder = {"token": strings.REDACTED_SENSITIVE_VALUE} + stored = {"accounts": [{"name": "a", **placeholder}, {"name": "b", **placeholder}], "groups": [placeholder]} + source = { + "accounts": [{"name": "a", "token": "first"}, {"name": "b", "token": "second"}], + "groups": [{"token": "x"}, {"token": "y"}], + } + + restored = restore_sensitive_values(stored, source) + + assert restored["accounts"] == source["accounts"] + # The items of two lists with different lengths can be different items, so the placeholder stays. + assert restored["groups"] == [placeholder] + + @patch("db.repositories.resources.ResourceTemplateRepository.enrich_template") def test_validate_patch_with_good_fields_passes(template_repo, resource_repo): """ diff --git a/docs/tre-workspace-authors/authoring-workspace-templates.md b/docs/tre-workspace-authors/authoring-workspace-templates.md index 7185a72456..63a77eb9a2 100644 --- a/docs/tre-workspace-authors/authoring-workspace-templates.md +++ b/docs/tre-workspace-authors/authoring-workspace-templates.md @@ -105,7 +105,9 @@ Put conditional rules inside `allOf`, rather than using root `if`, `then` or `el Keep Azure TRE annotations such as `updateable`, `sensitive` and `readOnly` on the property itself when using a `$ref`. A reusable definition does not supply these annotations to Azure TRE's resource processing. -The API stores `REDACTED` in place of each sensitive value, at the install and at an update. Only the first message of a request has the real value. The API finds a sensitive value by its property name, also in a nested object. If one declaration of a name has `sensitive: true`, for example in an `allOf` branch, the name is sensitive. +The API stores `REDACTED` in place of each sensitive value, at the install and at an update. Only the first message of a request has the real value. The API finds a sensitive value by its property name, also in a nested object or in an array item. If one declaration of a name has `sensitive: true`, for example in an `allOf` branch, the name is sensitive. + +If a later action does not give a new value for a sensitive property, the bundle gets `REDACTED`. The bundle must then use the earlier value. For example, the base workspace reads its client secret from the workspace Key Vault. An API upgrade does not rewrite stored templates or add missing metadata. To adopt Draft 2020-12, review the custom template's keywords, increment its bundle version and register the updated bundle. Changing only `$schema` can change validation semantics. Test creation and updates before upgrading existing resources. From d2b92c4c1e2eab80b872bfe162d0c67fe7680b3a Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 19:06:31 +0000 Subject: [PATCH 28/37] Mask an update with the template of the target version (#5298) - patch_resource masked the stored copy with the template of the current version. The routes read that template before a PATCH that also changes templateVersion. If only the target version marks a property sensitive, such a PATCH stored the new value of the property. API responses then included it until the migration of the next core deployment. - validate_template_version_patch already reads the target template, and it now returns it. patch_resource masks the stored copy with this template, because the stored document has the target version. - The masking does not use both versions. If the target version no longer marks a property sensitive, the stored copy keeps the new value and not the placeholder. Later messages then have the real value. - The history item keeps the template of the earlier version, because it stores the earlier document. The sensitiveValues flag of the operation already uses the target version. - Tests: a PATCH with a new template version and a new value stores the placeholder if only the target version marks the property sensitive, and the real value if only the earlier version does. The new tests fail on the earlier code. - The template authoring document and the CHANGELOG describe the rule. --- CHANGELOG.md | 2 +- api_app/db/repositories/resources.py | 13 +++--- .../test_resource_repository.py | 40 ++++++++++++++++++- .../authoring-workspace-templates.md | 2 +- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b87dda4e3..42c614f047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,7 +74,7 @@ BUG FIXES: If that update fails, send it again with the new secret. The update form shows `REDACTED`, and an update with `REDACTED` keeps the earlier secret. Delete the old secret only after an update with the new secret succeeds. The placeholder also covers a sensitive property in an array item, at the install too. In a later action that does not give a new value for such a property, the bundle gets `REDACTED`, as for other sensitive properties. - If a template declares a property in more than one branch, one declaration with `sensitive` is enough. + If a template declares a property in more than one branch, one declaration with `sensitive` is enough. At an update that changes the template version, the new version decides which properties are sensitive. For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. The create route of an Airlock review VM did the same in its HTTP 500 response. diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index 0d133f72cc..c83fd1663c 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -314,8 +314,9 @@ async def patch_resource( if resource_patch.isEnabled is not None: resource.isEnabled = resource_patch.isEnabled + target_template = resource_template if resource_patch.templateVersion is not None: - await self.validate_template_version_patch( + target_template = await self.validate_template_version_patch( resource, resource_patch, resource_template_repo, resource_template, force_version_update ) resource.templateVersion = resource_patch.templateVersion @@ -327,9 +328,10 @@ async def patch_resource( resource.properties.update(resource_patch.properties) # Cosmos DB keeps sensitive values as a placeholder, as at the install. The returned resource keeps the - # real values, because the caller builds the request message from it. + # real values, because the caller builds the request message from it. The stored document has the target + # version, so the template of that version decides which values are sensitive. masked_resource = copy.deepcopy(resource) - masked_resource.properties = mask_sensitive_properties(resource.properties, resource_template) + masked_resource.properties = mask_sensitive_properties(resource.properties, target_template) await self.update_item_with_etag(masked_resource, etag) return resource, resource_template @@ -424,7 +426,8 @@ async def validate_template_version_patch( resource_template_repo: ResourceTemplateRepository, resource_template: ResourceTemplate, force_version_update: bool = False, - ): + ) -> ResourceTemplate: + """Return the template of the target version.""" parent_service_template_name = None if resource.resourceType == ResourceType.UserResource: try: @@ -455,7 +458,7 @@ async def validate_template_version_patch( # validate if target template with desired version is registered try: - await resource_template_repo.get_template_by_name_and_version( + return await resource_template_repo.get_template_by_name_and_version( resource.templateName, resource_patch.templateVersion, resource_template.resourceType, diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 0e84adf98f..65c04392c7 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -551,11 +551,11 @@ async def history_query(query, parameters): } -def masking_template(properties, all_of=None) -> ResourceTemplate: +def masking_template(properties, all_of=None, version="1.0.0") -> ResourceTemplate: return ResourceTemplate( id="masking-template-id", name="masking-template", - version="1.0.0", + version=version, resourceType=ResourceType.Workspace, current=True, required=[], @@ -726,6 +726,42 @@ def test_restore_sensitive_values_restores_values_in_array_items(): assert restored["groups"] == [placeholder] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sensitive_in_target, expected", [(True, strings.REDACTED_SENSITIVE_VALUE), (False, "new value")] +) +@patch("db.repositories.resources.ResourceRepository.validate_patch") +@patch("db.repositories.resources.ResourceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) +async def test_patch_resource_masks_with_the_template_of_the_target_version( + _, __, resource_repo, sensitive_in_target, expected +): + client_secret = {"type": "string", "updateable": True} + source = masking_template({"client_secret": {**client_secret, "sensitive": not sensitive_in_target}}) + target = masking_template({"client_secret": {**client_secret, "sensitive": sensitive_in_target}}, version="1.1.0") + template_repo = AsyncMock() + template_repo.get_template_by_name_and_version.return_value = target + resource_repo.update_item_with_etag = AsyncMock() + resource = sample_resource() + resource.templateVersion = "1.0.0" + resource_patch = ResourcePatch(templateVersion="1.1.0", properties={"client_secret": "new value"}) + + returned, _ = await resource_repo.patch_resource( + resource, + resource_patch, + source, + "etag", + template_repo, + AsyncMock(), + create_test_user(), + strings.RESOURCE_ACTION_UPDATE, + ) + + stored = resource_repo.update_item_with_etag.await_args.args[0] + assert stored.templateVersion == "1.1.0" + assert stored.properties["client_secret"] == expected + assert returned.properties["client_secret"] == "new value" + + @patch("db.repositories.resources.ResourceTemplateRepository.enrich_template") def test_validate_patch_with_good_fields_passes(template_repo, resource_repo): """ diff --git a/docs/tre-workspace-authors/authoring-workspace-templates.md b/docs/tre-workspace-authors/authoring-workspace-templates.md index 63a77eb9a2..00ea57c0d1 100644 --- a/docs/tre-workspace-authors/authoring-workspace-templates.md +++ b/docs/tre-workspace-authors/authoring-workspace-templates.md @@ -107,7 +107,7 @@ Keep Azure TRE annotations such as `updateable`, `sensitive` and `readOnly` on t The API stores `REDACTED` in place of each sensitive value, at the install and at an update. Only the first message of a request has the real value. The API finds a sensitive value by its property name, also in a nested object or in an array item. If one declaration of a name has `sensitive: true`, for example in an `allOf` branch, the name is sensitive. -If a later action does not give a new value for a sensitive property, the bundle gets `REDACTED`. The bundle must then use the earlier value. For example, the base workspace reads its client secret from the workspace Key Vault. +At an update that changes the template version, the new version decides which values are sensitive. If a later action does not give a new value for a sensitive property, the bundle gets `REDACTED`. The bundle must then use the earlier value. For example, the base workspace reads its client secret from the workspace Key Vault. An API upgrade does not rewrite stored templates or add missing metadata. To adopt Draft 2020-12, review the custom template's keywords, increment its bundle version and register the updated bundle. Changing only `$schema` can change validation semantics. Test creation and updates before upgrading existing resources. From adfdc97c388324039ecab08d9075f609f8f77fd5 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 19:42:30 +0000 Subject: [PATCH 29/37] Describe the known limits of the execution lease and the resend (#5236) The resource processor document said that the resource processor uses its rules to run each step once. An expired execution lease does not stop a Porter action that runs, because Porter 1.4.0 cannot pass a cancel request to its Docker container. If the worker also loses its session, a second worker runs the same step at the same time. - A new section, Known limits, describes two runs of one step at the same time, with the log messages and the steps for an operator. The API keeps the first final status of the two runs, also if it comes from the first run. - The section also describes the short time between the check for a later request and the send of a first step that the API did not send. A new request in this time can run before that step. - It also describes an API stop between the operation write and the resource write of a failed first step. The resource then keeps its earlier status, and an operator can set it in Cosmos DB. - The section on repeated messages and the CHANGELOG entry no longer say that the resource processor runs each step once. --- CHANGELOG.md | 3 +- docs/tre-developers/resource-processor.md | 51 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c614f047..7a235081be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,9 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) -* Run each resource processor step once when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease stops a second Porter run. The API ignores a status from another run of a finished step. +* Prevent a second run of a resource processor step when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease lets one worker at a time run Porter for a resource. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter uninstalled. A worker that lost its session while it waited for the lease does not run Porter. + Two runs can still overlap if a worker loses the lease and its session during one Porter run. The resource processor document gives the [known limits](docs/tre-developers/resource-processor.md#known-limits). (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 73262167cc..ebdd221840 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -121,7 +121,7 @@ Furthermore, Porter provides a set of [mixins](https://porter.sh/mixins/) - anal ### Repeated messages -Service Bus can give the same message to more than one worker. The resource processor uses these rules to run each step once: +Service Bus can give the same message to more than one worker. The resource processor uses these rules to prevent a second run of a step: 1. The `workspacequeue` queue keeps a session lock for 5 minutes. While a worker runs a step, it renews the lock every minute. If a renewal fails, the worker tries again until the lock expires, and logs each failure. 1. Before Porter starts, the worker gets a 60-second lease on the blob `rp-executions/` in the management storage account. It renews the lease every 15 seconds. If another worker holds the lease, the worker keeps its session and tries again every 30 seconds. @@ -138,6 +138,8 @@ Service Bus can give the same message to more than one worker. The resource proc An upgrade can still repair a failed install, also an install that failed before Porter wrote its record. If the resource processor cannot read the Porter installation in 30 seconds, for example because the Porter storage is not available, it also sends `updating_failed`. +These rules do not prevent all second runs. [Known limits](#known-limits) describes two runs of a step at the same time. + A status message without `executionId` comes from a resource processor before version 0.13.16, or it is the failed status after a storage access error. The API keeps the earlier rules for it and removes the `executionId` of the step. Then a status of a Porter run replaces that status. For example, the success of a worker that ran Porter replaces the failed status of a second worker after a storage access error. An uninstall keeps its earlier rule for a final step. @@ -182,6 +184,53 @@ The entry stays if the API sent the message but stopped before it removed the en The status message of a step stays on the `deploymentstatus` queue until the API has sent the message of the next step. So a stop of the API during a pipeline does not need this entry. +### Known limits + +#### Two runs of one step at the same time + +If a worker cannot renew the execution lease for 60 seconds, for example because it cannot connect to the management storage account, the lease expires. +The Porter action of the worker continues, because the resource processor cannot stop it. Porter 1.4.0 cannot pass a cancel request to the Docker container of its action. +If the worker also loses its Service Bus session, another worker gets the message and the lease. The record shows the step as running, so the second worker runs Porter again. + +The second worker cannot know if the first worker stopped or only lost access to the storage account. After a worker stops during a Porter run, the next worker must run the step again in the same way. +So the resource processor does not block the second run. Before version 0.13.16, a lost session alone caused a second run. +For a bundle that uses Terraform, the Terraform state lock often stops the second run, and that run fails. + +The API keeps the first final status that it gets for the step, and it ignores the final status of the other run. This can be the status of the first run, also after the second run started. +For example, the step can show the state lock failure of the second run, while the first run changed the resources. + +If two runs overlap, the logs show these messages: + +- The first worker logs `Execution lease expired after failed renewals` for the step. It can also log `Session lock expired` or `The session lock expired before the message was completed`. + After its Porter run, it can log `Could not save the execution record`. +- The second worker logs `An earlier worker stopped during step for .` +- The API logs `Ignoring a status from another run of step /` for the final status that comes second. + +If you find these messages, do these steps: + +1. Wait until both workers log `Sent status message for : `. Then both Porter runs are complete. +1. Examine the Azure resources of the step. +1. If the status of the step is not correct for these resources, send the request again. For example, update the resource or send the delete again. + +#### A new request while the updater sends a first step + +The updater first checks for a later request for the resource, and then it builds and sends the message. These are two separate actions, and the API does not stop a new request between them. +The time between them is short: one or two Cosmos DB queries, the build of the message and the send. The API can accept a new request for the resource in this time. +Then the first step of the earlier request can run after the first step of the new request. For example, an install can run after a delete and deploy the resource again. + +If a deleted resource shows `deploying` or `deployed` again, delete it again. The API log then contains `Sending the pending first step message of operation ` for the install. + +#### The resource status after the updater writes a failed status + +If the updater also sets the resource to the failed status, it writes the operation first and then the resource. The operation write comes first, so that the etag check of the claim can stop both writes. +The operation write also removes the `pendingDispatch` entry. So if the API stops between the two writes, or if it cannot write the resource, the updater does not find the entry again. + +The operation then shows the failed status, but the resource keeps the status that it had before the request. A new resource keeps an empty `deploymentStatus`. +If the API could not write the resource, it logs `Unable to send the pending message of operation `. + +To repair the status, set `deploymentStatus` of the resource to the `status` of the operation in Cosmos DB. [Manually editing resources in Cosmos DB](../troubleshooting-faq/manually-editing-resources.md) gives the steps. +A later request for the resource also sets its status. + ### Porter Azure plugin Resource Processor uses [Porter Azure plugin](https://github.com/getporter/azure-plugins) to access secrets in Azure Key Vault. From ffb3016268d35718fa3c1c2b2c857732549a49d9 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 20:10:47 +0000 Subject: [PATCH 30/37] Describe two runs after a worker stops during a Porter action (#5236) A Porter action runs in its own Docker container on the Docker daemon of the VM. If a worker process or the resource processor container stops while the VM runs, this container can continue. The next worker then runs the step again at the same time. The known limits named only a worker that lost the lease and its session during a Porter run. - The section "Two runs of one step at the same time" describes this second cause. A worker that stopped logs none of the listed messages, and it sends no final status. - The first operator step no longer waits until both workers log "Sent status message". A worker that stopped does not log it. A worker that cannot send its final status, or that cannot read the parameters of the bundle, also does not log it. The step now tells the operator to find the container of the Porter action with docker ps. - The CHANGELOG entry names both causes. - The docstring of run_step no longer says that it runs the step once. --- CHANGELOG.md | 2 +- docs/tre-developers/resource-processor.md | 14 +++++++++++--- resource_processor/vmss_porter/runner.py | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a235081be..dfde7d0c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ BUG FIXES: * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) * Prevent a second run of a resource processor step when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease lets one worker at a time run Porter for a resource. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter uninstalled. A worker that lost its session while it waited for the lease does not run Porter. - Two runs can still overlap if a worker loses the lease and its session during one Porter run. The resource processor document gives the [known limits](docs/tre-developers/resource-processor.md#known-limits). + Two runs can still overlap if a worker loses the lease and its session during one Porter run. They can also overlap if a worker stops and its Porter action continues. The resource processor document gives the [known limits](docs/tre-developers/resource-processor.md#known-limits). (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index ebdd221840..e99073f832 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -192,7 +192,12 @@ If a worker cannot renew the execution lease for 60 seconds, for example because The Porter action of the worker continues, because the resource processor cannot stop it. Porter 1.4.0 cannot pass a cancel request to the Docker container of its action. If the worker also loses its Service Bus session, another worker gets the message and the lease. The record shows the step as running, so the second worker runs Porter again. -The second worker cannot know if the first worker stopped or only lost access to the storage account. After a worker stops during a Porter run, the next worker must run the step again in the same way. +A stop of a worker also does not always stop its Porter action. The Docker container of the action runs on the Docker daemon of the VM, next to the resource processor container. +If a worker process or the resource processor container stops while the VM runs, the Docker container of the action can continue. +For example, a worker process can stop when the VM has no free memory, or an operator can restart the resource processor container. +After such a stop, another worker gets the message and the lease. That worker runs Porter for the step again, also if the earlier action still runs. + +The second worker cannot know if the Porter action of the first worker still runs. After a worker and its Porter action stop, for example when the VM restarts, the next worker must run the step again. So the resource processor does not block the second run. Before version 0.13.16, a lost session alone caused a second run. For a bundle that uses Terraform, the Terraform state lock often stops the second run, and that run fails. @@ -202,13 +207,16 @@ For example, the step can show the state lock failure of the second run, while t If two runs overlap, the logs show these messages: - The first worker logs `Execution lease expired after failed renewals` for the step. It can also log `Session lock expired` or `The session lock expired before the message was completed`. - After its Porter run, it can log `Could not save the execution record`. + After its Porter run, it can log `Could not save the execution record`. A worker that stopped does not log these messages. - The second worker logs `An earlier worker stopped during step for .` + If only the second worker logs a message of this list, examine the VM of the first worker. A Porter action of a stopped worker can still run there. - The API logs `Ignoring a status from another run of step /` for the final status that comes second. + A worker that stopped does not send a final status, so the API does not log this message. If you find these messages, do these steps: -1. Wait until both workers log `Sent status message for : `. Then both Porter runs are complete. +1. Wait until both Porter runs end. A worker logs `Sent status message for : ` after its run. + If a worker does not log this message, [connect to its VM](#connect-to-the-resource-processor-terminal). Use `docker ps` to find the container of the Porter action for the step. Wait until this container stops. 1. Examine the Azure resources of the step. 1. If the status of the step is not correct for these resources, send the request again. For example, update the resource or send the delete again. diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 86ff1ace55..23663d7682 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -505,7 +505,7 @@ def recorded_results(record: dict) -> list: async def run_step(msg_body: dict, execution, sb_client: ServiceBusClient, config: dict) -> bool: """ - Run the step once. If the execution record shows that a worker already finished this step, send its status again. + Run the step. If the execution record shows that a worker already finished this step, send its status again. """ def same_step(entry: dict) -> bool: From 9590aad396e78f1f08fcc8c526ff9ffe84b09dca Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 20:11:29 +0000 Subject: [PATCH 31/37] Name the status that a workspace keeps after a failed resend (#5236) If the API stops between the operation write and the resource write of a failed first step, the resource keeps its status. The known limits said that this is the status from before the request. A workspace update that changes the Airlock version is different. The API writes awaiting_update to the workspace before it writes the operation, so that workspace keeps awaiting_update. The resource processor document now names this case. --- docs/tre-developers/resource-processor.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index e99073f832..3e57c89c4c 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -234,6 +234,7 @@ If the updater also sets the resource to the failed status, it writes the operat The operation write also removes the `pendingDispatch` entry. So if the API stops between the two writes, or if it cannot write the resource, the updater does not find the entry again. The operation then shows the failed status, but the resource keeps the status that it had before the request. A new resource keeps an empty `deploymentStatus`. +If a workspace update changes the Airlock version, the API writes `awaiting_update` to the workspace before it writes the operation. Then the workspace keeps `awaiting_update`. If the API could not write the resource, it logs `Unable to send the pending message of operation `. To repair the status, set `deploymentStatus` of the resource to the `status` of the operation in Cosmos DB. [Manually editing resources in Cosmos DB](../troubleshooting-faq/manually-editing-resources.md) gives the steps. From 59b0758fffbac2b19d8d767fb656b080bdcd69d5 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 20:42:13 +0000 Subject: [PATCH 32/37] Describe the new failures and the masked bundle values in the CHANGELOG - BREAKING CHANGES has a new entry for custom bundles. A bundle gets REDACTED in a later action that does not give a new value. This now also applies after an update, to a sensitive property in an array item and to a property with more than one declaration. - The entry for the empty outputs list says that the step also fails if the output text of Porter is not a JSON list. - The entry for repeated messages says that an upgrade fails with updating_failed if the resource processor cannot read the Porter installation in 30 seconds. - The versions do not change. --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfde7d0c1d..f58ab4c4b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) * `tre-workspace-service-gitea` 2.0.0 keeps the Git repositories on its file share. The upgrade restarts the web app, so the repositories on the container disk are lost. Mirror the repositories before the upgrade, and rotate the workspace client secret after it. See the [Gitea upgrade steps](docs/tre-templates/workspace-services/gitea.md#upgrading-to-version-200). The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* A bundle gets `REDACTED` for a sensitive property in a later action that does not give a new value for it. This now also applies after an update that set the property. + It also applies to a sensitive property in an array item. It also applies to a property with more than one declaration in the template, if one declaration has `sensitive`. + If a custom bundle needs such a value in a later action, make the bundle keep it, for example in Key Vault. The base workspace does this for its client secret. (`api` 0.29.0) ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) @@ -57,10 +60,12 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) -* Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. (`resource_processor` 0.13.16) +* Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. + The resource processor also sends the failed status for the step if the output text of Porter is not a JSON list. (`resource_processor` 0.13.16) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) * Prevent a second run of a resource processor step when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease lets one worker at a time run Porter for a resource. The API ignores a status from another run of a finished step. - An upgrade no longer installs a resource that Porter uninstalled. A worker that lost its session while it waited for the lease does not run Porter. + An upgrade no longer installs a resource that Porter uninstalled. An upgrade also fails with `updating_failed` if the resource processor cannot read the Porter installation in 30 seconds. + A worker that lost its session while it waited for the lease does not run Porter. Two runs can still overlap if a worker loses the lease and its session during one Porter run. They can also overlap if a worker stops and its Porter action continues. The resource processor document gives the [known limits](docs/tre-developers/resource-processor.md#known-limits). (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. From 65fe1f17e2f8bcc064b2bb6ca760634fcfcf181e Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 11:43:34 +0000 Subject: [PATCH 33/37] Add the issue links to the new CHANGELOG entries (#5297, #5298) The resource processor entries link #5297, and the sensitive value entries link #5298. --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f58ab4c4b9..67a47b0304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Setting to `false` will delete existing airlock storage accounts and must only b Mirror the repositories before the upgrade, and rotate the workspace client secret after it. See the [Gitea upgrade steps](docs/tre-templates/workspace-services/gitea.md#upgrading-to-version-200). The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * A bundle gets `REDACTED` for a sensitive property in a later action that does not give a new value for it. This now also applies after an update that set the property. It also applies to a sensitive property in an array item. It also applies to a property with more than one declaration in the template, if one declaration has `sensitive`. - If a custom bundle needs such a value in a later action, make the bundle keep it, for example in Key Vault. The base workspace does this for its client secret. (`api` 0.29.0) + If a custom bundle needs such a value in a later action, make the bundle keep it, for example in Key Vault. The base workspace does this for its client secret. (`api` 0.29.0) ([#5298](https://github.com/microsoft/AzureTRE/issues/5298)) ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) @@ -61,8 +61,9 @@ BUG FIXES: * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Send an empty outputs list when the resource processor cannot read the Porter outputs. The API rejected the empty dict, so a pipeline step that failed to read its outputs, for example after a transient container registry login failure, stayed in progress until the caller stopped waiting. - The resource processor also sends the failed status for the step if the output text of Porter is not a JSON list. (`resource_processor` 0.13.16) + The resource processor also sends the failed status for the step if the output text of Porter is not a JSON list. (`resource_processor` 0.13.16) ([#5297](https://github.com/microsoft/AzureTRE/issues/5297)) * Give each resource processor worker its own Azure CLI folder. Each worker signs in before each Porter action. When two workers did this at the same time, one worker read the shared profile file while the other wrote it, and the next `az acr login` failed with `Please run 'az login'`. The action then failed after 3 seconds. (`resource_processor` 0.13.16) + ([#5297](https://github.com/microsoft/AzureTRE/issues/5297)) * Prevent a second run of a resource processor step when Service Bus gives its message to a second worker. The resource processor renews the session lock with retries, and a blob lease lets one worker at a time run Porter for a resource. The API ignores a status from another run of a finished step. An upgrade no longer installs a resource that Porter uninstalled. An upgrade also fails with `updating_failed` if the resource processor cannot read the Porter installation in 30 seconds. A worker that lost its session while it waited for the lease does not run Porter. @@ -81,11 +82,11 @@ BUG FIXES: Delete the old secret only after an update with the new secret succeeds. The placeholder also covers a sensitive property in an array item, at the install too. In a later action that does not give a new value for such a property, the bundle gets `REDACTED`, as for other sensitive properties. If a template declares a property in more than one branch, one declaration with `sensitive` is enough. At an update that changes the template version, the new version decides which properties are sensitive. - For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) + For a template whose action has more steps than `main`, for example the unrestricted workspace, the first message now also gets the real sensitive values at an install. (`api` 0.29.0) ([#5298](https://github.com/microsoft/AzureTRE/issues/5298)) * Describe a schema error of a resource request without the submitted values. The API logged the full jsonschema error text and returned it in the HTTP 400 response of a create request. The create route of an Airlock review VM did the same in its HTTP 500 response. That text contains the submitted properties, for example a client secret. The response now names the property and the template rule. - The API also wrote this text to its console log, which the core Log Analytics workspace keeps for 30 days by default. If a create request with a client secret got a schema error before this version, create a new secret for the workspace application. (`api` 0.29.0) + The API also wrote this text to its console log, which the core Log Analytics workspace keeps for 30 days by default. If a create request with a client secret got a schema error before this version, create a new secret for the workspace application. (`api` 0.29.0) ([#5298](https://github.com/microsoft/AzureTRE/issues/5298)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) From 8c471e8f14119e8aaddf108d38156ad3bac62407 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 11:43:40 +0000 Subject: [PATCH 34/37] Use the next core version after the VM size change #5290 sets core 0.19.0. This change uses core 0.19.1, so that the two changes can merge in either order with one more bump at most. --- CHANGELOG.md | 2 +- core/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67a47b0304..6829140310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,7 @@ BUG FIXES: An upgrade no longer installs a resource that Porter uninstalled. An upgrade also fails with `updating_failed` if the resource processor cannot read the Porter installation in 30 seconds. A worker that lost its session while it waited for the lease does not run Porter. Two runs can still overlap if a worker loses the lease and its session during one Porter run. They can also overlap if a worker stops and its Porter action continues. The resource processor document gives the [known limits](docs/tre-developers/resource-processor.md#known-limits). - (`core` 0.18.17, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) + (`core` 0.19.1, `resource_processor` 0.13.16, `api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. The resend does not change a resource that a later request changed, and it does not change the resource of a first step on another resource. diff --git a/core/version.txt b/core/version.txt index 6e7cb3b8f3..4c1ca3c809 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.17" +__version__ = "0.19.1" From bf72548764b3991215b7d63f157b0a09071fb6d0 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 12:33:45 +0000 Subject: [PATCH 35/37] Add an appended array item only once if the API prepares a step again (#5236) If the API stops after the patch of the resource of a first step and before the send, the resend task prepares the step again. The `append` array action then added the same item a second time, for example the same firewall rule collection. The same occurs on `main` if Service Bus delivers a status message again and the API prepares the next step again. Now `append` does not add an item that the array already has. The other array actions already give the same result when they run again. The new test stops the API between the patch and the send, and then runs the resend. Before this change, the firewall had the rule collection two times. --- CHANGELOG.md | 1 + api_app/service_bus/substitutions.py | 4 +- .../test_service_bus/test_pending_dispatch.py | 143 +++++++++++++++++- .../test_service_bus/test_substitutions.py | 15 ++ docs/tre-developers/resource-processor.md | 1 + .../pipeline-templates/pipeline-schema.md | 2 +- 6 files changed, 163 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6829140310..1d13f4acd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ BUG FIXES: * Send the first step message of a request again if the API stops after it writes the operation and before it sends the message. The API keeps a `pendingDispatch` entry in the operation until the send, and the deployment status updater handles older entries every 5 minutes. If that message has sensitive values from an install or an update, the request fails with a clear message, because the API does not store these values. The resend does not change a resource that a later request changed, and it does not change the resource of a first step on another resource. + The `append` array action of a pipeline step does not add an item that the array already has. So if the API prepares a step again, for example at a resend after the patch of the resource, the array does not get the same item two times. (`api` 0.29.0) ([#5236](https://github.com/microsoft/AzureTRE/issues/5236)) * Store a placeholder for each sensitive property when the API saves an update of a resource, as it already does at the install. The new value goes to the resource processor only in the request message. Before this change, an update stored the new value of a sensitive property in Cosmos DB and in the resource history. An example is the `client_secret` of a workspace. diff --git a/api_app/service_bus/substitutions.py b/api_app/service_bus/substitutions.py index 2aef1e34a1..8700f50319 100644 --- a/api_app/service_bus/substitutions.py +++ b/api_app/service_bus/substitutions.py @@ -37,7 +37,9 @@ def substitute_properties( if prop.arraySubstitutionAction == "overwrite": existing_arr = [val] - if prop.arraySubstitutionAction == "append": + # The API can prepare a step again, for example at the resend of a first step after an API stop. So + # "append" does not add an item that the array already has. + if prop.arraySubstitutionAction == "append" and val not in existing_arr: existing_arr.append(val) if prop.arraySubstitutionAction == "remove": diff --git a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py index 5528664c24..0d4bdc9082 100644 --- a/api_app/tests_ma/test_service_bus/test_pending_dispatch.py +++ b/api_app/tests_ma/test_service_bus/test_pending_dispatch.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import uuid from unittest.mock import AsyncMock, patch @@ -8,7 +9,11 @@ from db.errors import EntityDoesNotExist from db.repositories.operations import PENDING_DISPATCH -from models.domain.operation import Status +from models.domain.operation import Operation, OperationStep, Status +from models.domain.request_action import RequestAction +from models.domain.resource import ResourceType +from models.domain.resource_template import Pipeline, PipelineStep, PipelineStepProperty, ResourceTemplate +from models.domain.shared_service import SharedService from models.domain.workspace import Workspace from resources import strings from service_bus.deployment_status_updater import ( @@ -16,6 +21,7 @@ NOT_SENT_WITH_SENSITIVE_VALUES, DeploymentStatusUpdater, ) +from service_bus.resource_request_sender import send_resource_request_message from tests_ma.test_service_bus.test_deployment_status_update import OPERATION_ID, create_sample_operation pytestmark = pytest.mark.asyncio @@ -328,3 +334,138 @@ async def resend(document): await updater.resend_pending_dispatches() assert sent == ["second"] + + +class ApiStopped(BaseException): + """The API process stops. The error handler of the sender does not run.""" + + +def template(name: str, resource_type: ResourceType, pipeline: Pipeline = None) -> ResourceTemplate: + return ResourceTemplate( + id=f"{name}-id", + name=name, + version="0.1.0", + resourceType=resource_type, + current=True, + required=[], + properties={}, + pipeline=pipeline, + ) + + +def firewall_append_template() -> ResourceTemplate: + """The first step of the install adds a rule collection to the firewall with the "append" action.""" + return template( + "tre-workspace-base", + ResourceType.Workspace, + Pipeline( + install=[ + PipelineStep( + stepId="firewall-step", + resourceTemplateName="tre-shared-service-firewall", + resourceType=ResourceType.SharedService, + resourceAction="upgrade", + properties=[ + PipelineStepProperty( + name="rule_collections", + type="array", + arraySubstitutionAction="append", + value={"name": "rc_{{ resource.id }}", "action": "Allow"}, + ) + ], + ), + PipelineStep(stepId="main"), + ] + ), + ) + + +def resource_store(*resources): + """Keep the resources as Cosmos DB does, so that a second preparation reads the patch of the first one.""" + stored = {resource.id: resource for resource in resources} + repo = AsyncMock() + repo.get_resource_by_id.side_effect = lambda resource_id: copy.deepcopy(stored[str(resource_id)]) + + def patch_resource(resource, resource_patch, resource_template, **_): + resource.properties.update(resource_patch.properties) + stored[resource.id] = copy.deepcopy(resource) + return resource, resource_template + + repo.patch_resource.side_effect = patch_resource + return repo, stored + + +@patch("service_bus.deployment_status_updater.send_deployment_message") +@patch("service_bus.resource_request_sender.send_deployment_message") +async def test_a_resend_after_a_stop_between_the_patch_and_the_send_appends_the_item_once( + first_send_mock, resend_mock, updater, test_user +): + templates = { + "tre-workspace-base": firewall_append_template(), + "tre-shared-service-firewall": template("tre-shared-service-firewall", ResourceType.SharedService), + } + resource = workspace({}) + firewall = SharedService( + id=FIREWALL_ID, templateName="tre-shared-service-firewall", templateVersion="0.1.0", etag="", properties={} + ) + resource_repo, stored = resource_store(resource, firewall) + template_repo = AsyncMock() + template_repo.get_template_by_name_and_version.side_effect = lambda name, *_: templates[name] + operation = Operation( + id=OPERATION_ID, + resourceId=RESOURCE_ID, + resourcePath="test", + action=RequestAction.Install, + createdWhen=1.0, + user=test_user, + steps=[ + OperationStep( + id="random-uuid", + templateStepId="firewall-step", + resourceId=FIREWALL_ID, + resourceTemplateName="tre-shared-service-firewall", + resourceType=ResourceType.SharedService, + resourceAction="upgrade", + status=Status.AwaitingUpdate, + sourceTemplateResourceId=RESOURCE_ID, + ), + OperationStep( + id="main-step", + templateStepId="main", + resourceId=RESOURCE_ID, + resourceAction="install", + status=Status.AwaitingDeployment, + sourceTemplateResourceId=RESOURCE_ID, + ), + ], + ) + operations_repo = AsyncMock() + operations_repo.create_operation_item.return_value = operation + # The API patches the firewall, then stops before the send. The entry stays. + first_send_mock.side_effect = ApiStopped + + with pytest.raises(ApiStopped): + await send_resource_request_message( + resource=resource, + operations_repo=operations_repo, + resource_repo=resource_repo, + user=test_user, + resource_template_repo=template_repo, + resource_history_repo=AsyncMock(), + action=RequestAction.Install, + ) + operations_repo.clear_pending_dispatch.assert_not_awaited() + assert len(stored[FIREWALL_ID].properties["rule_collections"]) == 1 + + # The task prepares the first step again from Cosmos DB and sends it. + updater.resource_repo = resource_repo + updater.resource_template_repo = template_repo + document = operation.model_dump() + document["_etag"] = "etag" + document[PENDING_DISPATCH] = {"stepId": "random-uuid", "messageId": MESSAGE_ID, "createdWhen": 1.0} + await updater.resend_pending_dispatch(document) + + expected = [{"name": f"rc_{RESOURCE_ID}", "action": "Allow"}] + assert stored[FIREWALL_ID].properties["rule_collections"] == expected + assert json.loads(resend_mock.await_args.kwargs["content"])["parameters"]["rule_collections"] == expected + updater.operations_repo.clear_pending_dispatch.assert_awaited_once_with(OPERATION_ID, MESSAGE_ID) diff --git a/api_app/tests_ma/test_service_bus/test_substitutions.py b/api_app/tests_ma/test_service_bus/test_substitutions.py index 8442a7be5e..526b1d396b 100644 --- a/api_app/tests_ma/test_service_bus/test_substitutions.py +++ b/api_app/tests_ma/test_service_bus/test_substitutions.py @@ -328,6 +328,21 @@ def test_substitution_array_append_remove(pipeline_step, primary_resource, resou assert len(obj["rule_collections"]) == 0 +def test_substitution_array_append_does_not_add_an_equal_item(pipeline_step, primary_resource, resource_to_update): + # The API can prepare a step again, for example after it stopped between the patch and the send. + step = copy.deepcopy(pipeline_step) + step.properties[0].arraySubstitutionAction = "append" + resource_to_update.properties = substitute_properties( + copy.deepcopy(step), primary_resource, None, None, resource_to_update + ) + first = copy.deepcopy(resource_to_update.properties["rule_collections"]) + + obj = substitute_properties(copy.deepcopy(step), primary_resource, None, None, resource_to_update) + + assert len(first) == 1 + assert obj["rule_collections"] == first + + def test_substitution_array_append_replace(pipeline_step, primary_resource, resource_to_update): # add object 1 step = copy.deepcopy(pipeline_step) diff --git a/docs/tre-developers/resource-processor.md b/docs/tre-developers/resource-processor.md index 3e57c89c4c..706258ab31 100644 --- a/docs/tre-developers/resource-processor.md +++ b/docs/tre-developers/resource-processor.md @@ -166,6 +166,7 @@ If the API stops between the write and the send, the entry stays. When the API s The first step of a cascade is for a child resource, so a later request for that child also counts. The updater sets the step and the operation to the failed status. It does not change the resource, because the later request owns its status. 1. If the step has no status, the updater builds the message again from Cosmos DB and sends it with the message ID `:`. Then it removes the entry. + The build applies the properties of the pipeline step again. If the API changed the resource of the step before it stopped, the `append` array action finds its item and does not add it again. 1. If an install or an update sets a sensitive property, the first message has the real value, and Cosmos DB keeps `REDACTED`. The operation write records this in the entry, and the updater cannot build such a message again. It sets the step and the operation to the failed status, with a message that asks the user to send the request again with the sensitive values if the status does not change. diff --git a/docs/tre-templates/pipeline-templates/pipeline-schema.md b/docs/tre-templates/pipeline-templates/pipeline-schema.md index e638684e08..fb305aa294 100644 --- a/docs/tre-templates/pipeline-templates/pipeline-schema.md +++ b/docs/tre-templates/pipeline-templates/pipeline-schema.md @@ -79,7 +79,7 @@ The below example references 2 properties from the primary resource to be used i It's possible that a resource property would actually be an array. As an example, the firewall shared service has the `rule_collections` property. This single property contains an array of objects. Since the values inside this array may have been sourced from different resources, it's important to leave other values in tact when modifying the property. To do so, the `arraySubstitutionAction` field supports the following values: -- `append` - just append this object into the array +- `append` - add this object to the end of the array. If the array already has an equal object, the array does not change. So if the API prepares a step again, for example after an API stop, the array does not get the same object two times. - `replace` - find this object in the array (using the `arrayMatchField` value), and replace it with this value - `remove` - remove this property from the array (useful for `uninstall` actions) From 204e74db4de1203a614930e269d201ddc4cb37f8 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 12:56:25 +0000 Subject: [PATCH 36/37] Give a null sensitive value to the first step of a pipeline The install and the update store REDACTED for each sensitive property, also if the request sets it to null. For a template with a pipeline, restore_sensitive_values puts the real values back for the first step of the primary resource. It used a null in the request as a missing key, so the first message had REDACTED in place of null. This also occurred in array items. Now only a key that the request does not have keeps its placeholder. A template without a pipeline already sends null in the first message. The new test gives a null in a nested object and in an array item. Before this change, both values stayed REDACTED. --- api_app/db/repositories/resources.py | 8 +++++--- .../test_repositories/test_resource_repository.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index c83fd1663c..66ad61e421 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -104,15 +104,17 @@ def restore_sensitive_values(properties: dict, source: dict) -> dict: """Return a copy of properties with the real value from source in place of each placeholder. Two lists of the same length pair their items by index. A list with a different length keeps its placeholders, - because its items can be different items. + because its items can be different items. A null in source is a real value. Only a key that source does not have + keeps its placeholder. """ + missing = object() def restore(value: Any, original: Any) -> Any: if isinstance(value, dict) and isinstance(original, dict): - return {key: restore(item, original.get(key)) for key, item in value.items()} + return {key: restore(item, original.get(key, missing)) for key, item in value.items()} if isinstance(value, list) and isinstance(original, list) and len(value) == len(original): return [restore(item, original_item) for item, original_item in zip(value, original)] - if value == REDACTED_SENSITIVE_VALUE and original is not None and original != REDACTED_SENSITIVE_VALUE: + if value == REDACTED_SENSITIVE_VALUE and original is not missing and original != REDACTED_SENSITIVE_VALUE: return copy.deepcopy(original) return copy.deepcopy(value) diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 65c04392c7..32be108e37 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -726,6 +726,17 @@ def test_restore_sensitive_values_restores_values_in_array_items(): assert restored["groups"] == [placeholder] +def test_restore_sensitive_values_restores_an_explicit_null(): + # The install stores a placeholder also for a null value. Only a key that the request does not have keeps it. + placeholder = strings.REDACTED_SENSITIVE_VALUE + stored = {"settings": {"token": placeholder, "password": placeholder}, "accounts": [{"token": placeholder}]} + source = {"settings": {"token": None}, "accounts": [{"token": None}]} + + restored = restore_sensitive_values(stored, source) + + assert restored == {"settings": {"token": None, "password": placeholder}, "accounts": [{"token": None}]} + + @pytest.mark.asyncio @pytest.mark.parametrize( "sensitive_in_target, expected", [(True, strings.REDACTED_SENSITIVE_VALUE), (False, "new value")] From 32bbf9f81a6c36be8010501a9df7be2c0edf7714 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 13:00:10 +0000 Subject: [PATCH 37/37] Patch a document in the migration only if it did not change after the read The migration of sensitive values reads all resources one time. Then it patches each resource from that read. make tre-deploy runs the migration after the deploy of the API, so a request can change a resource between the read and the patch. Then the patch can be wrong: - An update can set a new value and a template version that does not mark the property sensitive. The patch replaced the new value with REDACTED. - An update can remove an array item. The patch then names an index that is not in the document. Cosmos DB refuses the request, and the migration stopped with the other resources not masked. Now each patch request has the etag of the document that gave its paths. The next request of the same document has the etag of the patch result. If Cosmos DB refuses a request because the etag changed, the migration reads the document again, gets the template of its version and finds the paths again. After 3 refusals, it logs a warning and continues with the next document. The next migration masks that document. Each of the 4 new tests failed before this change. --- api_app/db/repositories/resources.py | 63 ++++--- .../test_resource_repository.py | 160 ++++++++++++++++-- 2 files changed, 184 insertions(+), 39 deletions(-) diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index 66ad61e421..a86b2ecdd8 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -3,7 +3,8 @@ from datetime import datetime, UTC from typing import Any, Dict, Optional, Tuple, List -from azure.cosmos.exceptions import CosmosResourceNotFoundError +from azure.core import MatchConditions +from azure.cosmos.exceptions import CosmosAccessConditionFailedError, CosmosResourceNotFoundError from resources.strings import RESOURCE_ACTION_INSTALL, REDACTED_SENSITIVE_VALUE from core import config from db.errors import ( @@ -130,18 +131,6 @@ def masked_property_paths(properties: dict, template: ResourceTemplate) -> List[ ] -async def set_masked_values(container, item_id: str, partition_key: str, paths: List[str]): - # Cosmos DB accepts at most 10 operations in one patch request. - for start in range(0, len(paths), 10): - await container.patch_item( - item=item_id, - partition_key=partition_key, - patch_operations=[ - {"op": "set", "path": path, "value": REDACTED_SENSITIVE_VALUE} for path in paths[start : start + 10] - ], - ) - - class ResourceRepository(BaseRepository): @classmethod async def create(cls): @@ -373,29 +362,55 @@ async def template_for(resource: dict, version: str) -> Optional[ResourceTemplat templates[key] = None return templates[key] - async def mask(item: dict, template: Optional[ResourceTemplate], container, partition_key: str) -> bool: - if template is None: - return False - paths = masked_property_paths(item.get("properties") or {}, template) - if paths: - await set_masked_values(container, item["id"], partition_key, paths) - return bool(paths) + async def mask(resource: dict, item: dict, container, partition_key: str) -> bool: + """Replace each real sensitive value of item, which is the resource or one of its history items. + + Each patch request has the etag of the document that gave its paths. If another request changed the + document after that read, Cosmos DB refuses the patch. Then the migration reads the document again, gets + the template of its version and finds the paths again. After 3 refusals, the next run masks the document. + """ + changed = False + for _ in range(3): + template = await template_for(resource, item.get("templateVersion") or resource["templateVersion"]) + if template is None: + return changed + paths = masked_property_paths(item.get("properties") or {}, template) + try: + # Cosmos DB accepts at most 10 operations in one patch request. The patched document has the etag + # for the next request. + for start in range(0, len(paths), 10): + item = await container.patch_item( + item=item["id"], + partition_key=partition_key, + patch_operations=[ + {"op": "set", "path": path, "value": REDACTED_SENSITIVE_VALUE} + for path in paths[start : start + 10] + ], + etag=item["_etag"], + match_condition=MatchConditions.IfNotModified, + ) + changed = True + return changed + except CosmosAccessConditionFailedError: + item = await container.read_item(item=item["id"], partition_key=partition_key) + logger.warning( + f"Document {item['id']} changed during each attempt to mask it. The next migration masks it." + ) + return changed changed_resources = 0 changed_history_items = 0 for resource in resources.values(): if resource["templateName"] not in names_with_sensitive_values: continue - template = await template_for(resource, resource["templateVersion"]) - changed_resources += await mask(resource, template, self.container, resource["id"]) + changed_resources += await mask(resource, resource, self.container, resource["id"]) history_items = await resource_history_repo.query( query="SELECT * FROM c WHERE c.resourceId = @resourceId", parameters=[{"name": "@resourceId", "value": resource["id"]}], ) for item in history_items: - template = await template_for(resource, item.get("templateVersion") or resource["templateVersion"]) - changed_history_items += await mask(item, template, resource_history_repo.container, item["resourceId"]) + changed_history_items += await mask(resource, item, resource_history_repo.container, item["resourceId"]) return changed_resources, changed_history_items diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 32be108e37..b86351f936 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -17,9 +17,13 @@ mask_sensitive_properties, masked_property_paths, restore_sensitive_values, - set_masked_values, ) -from azure.cosmos.exceptions import CosmosResourceNotFoundError +from azure.core import MatchConditions +from azure.cosmos.exceptions import ( + CosmosAccessConditionFailedError, + CosmosHttpResponseError, + CosmosResourceNotFoundError, +) from models.domain.resource import Resource from models.domain.resource_template import ResourceTemplate from models.domain.user_resource_template import UserResourceTemplate @@ -470,6 +474,7 @@ def stored_resource(resource_id, template_name, resource_type, properties, **fie "templateVersion": "1.0.0", "resourceType": resource_type, "properties": properties, + "_etag": "1", **fields, } @@ -506,8 +511,14 @@ async def get_template(name, version, resource_type, parent_service_name=None): ] history = { "ws": [ - {"id": "h-1", "resourceId": "ws", "templateVersion": "0.9.0", "properties": {"client_secret": "older"}}, - {"id": "h-2", "resourceId": "ws", "properties": {"client_secret": "REDACTED"}}, + { + "id": "h-1", + "resourceId": "ws", + "templateVersion": "0.9.0", + "properties": {"client_secret": "older"}, + "_etag": "1", + }, + {"id": "h-2", "resourceId": "ws", "properties": {"client_secret": "REDACTED"}, "_etag": "1"}, ], } @@ -522,13 +533,14 @@ async def history_query(query, parameters): assert result == (2, 1) set_secret = [{"op": "set", "path": "/properties/client_secret", "value": strings.REDACTED_SENSITIVE_VALUE}] + if_not_modified = {"etag": "1", "match_condition": MatchConditions.IfNotModified} assert [call.kwargs for call in resource_repo.container.patch_item.await_args_list] == [ - {"item": "ws", "partition_key": "ws", "patch_operations": set_secret}, - {"item": "ur", "partition_key": "ur", "patch_operations": set_secret}, + {"item": "ws", "partition_key": "ws", "patch_operations": set_secret, **if_not_modified}, + {"item": "ur", "partition_key": "ur", "patch_operations": set_secret, **if_not_modified}, ] # The history container uses the resource ID as its partition key. history_repo.container.patch_item.assert_awaited_once_with( - item="h-1", partition_key="ws", patch_operations=set_secret + item="h-1", partition_key="ws", patch_operations=set_secret, **if_not_modified ) # A resource of a template without a sensitive property needs no history read. read_ids = [call.kwargs["parameters"][0]["value"] for call in history_repo.query.await_args_list] @@ -571,14 +583,127 @@ def test_masked_property_paths_escape_nested_names(): assert masked_property_paths(properties, template) == ["/properties/settings/a~1b", "/properties/settings/c~0d"] +class FakeContainer: + """Apply a patch with an etag only to the document that has this etag, as Cosmos DB does.""" + + def __init__(self, *documents): + self.documents = {document["id"]: copy.deepcopy(document) for document in documents} + self.requests = [] + + async def read_item(self, item, partition_key): + return copy.deepcopy(self.documents[item]) + + async def patch_item(self, item, partition_key, patch_operations, etag=None, match_condition=None): + self.requests.append((item, len(patch_operations), etag)) + document = copy.deepcopy(self.documents[item]) + if match_condition == MatchConditions.IfNotModified and etag != document["_etag"]: + raise CosmosAccessConditionFailedError(status_code=412) + for operation in patch_operations: + *parents, name = [part.replace("~1", "/").replace("~0", "~") for part in operation["path"].split("/")[1:]] + target = document + try: + for part in parents: + target = target[int(part)] if isinstance(target, list) else target[part] + target[int(name) if isinstance(target, list) else name] = operation["value"] + except (IndexError, KeyError): + raise CosmosHttpResponseError(status_code=400, message=f"The path {operation['path']} does not exist") + document["_etag"] = str(int(document["_etag"]) + 1) + self.documents[item] = document + return copy.deepcopy(document) + + +def migration_template_repo(*templates): + template_repo = AsyncMock() + template_repo.query.return_value = [template.model_dump() for template in templates] + by_version = {template.version: template for template in templates} + template_repo.get_template_by_name_and_version.side_effect = lambda name, version, *_: by_version[version] + return template_repo + + @pytest.mark.asyncio -async def test_set_masked_values_sends_at_most_ten_operations_in_each_request(): - container = AsyncMock() - paths = [f"/properties/secret_{index}" for index in range(11)] +async def test_the_migration_sends_at_most_ten_operations_with_the_etag_of_each_result(resource_repo): + secrets = {f"secret_{index}": SECRET for index in range(11)} + snapshot = stored_resource("ws", "masking-template", "workspace", {name: "real" for name in secrets}) + resource_repo._container = FakeContainer(snapshot) + resource_repo.query = AsyncMock(return_value=[snapshot]) + + result = await resource_repo.mask_stored_sensitive_values( + migration_template_repo(masking_template(secrets)), history_without_items() + ) - await set_masked_values(container, "item", "partition", paths) + assert result == (1, 0) + # Cosmos DB accepts at most 10 operations in one request. The second request has the etag of the first result. + assert resource_repo.container.requests == [("ws", 10, "1"), ("ws", 1, "2")] + assert set(resource_repo.container.documents["ws"]["properties"].values()) == {strings.REDACTED_SENSITIVE_VALUE} - assert [len(call.kwargs["patch_operations"]) for call in container.patch_item.await_args_list] == [10, 1] + +@pytest.mark.asyncio +async def test_the_migration_keeps_a_new_value_that_the_new_version_does_not_mark_sensitive(resource_repo): + first = masking_template({"token": SECRET}) + second = masking_template({"token": {"type": "string"}}, version="2.0.0") + snapshot = stored_resource("ws", "masking-template", "workspace", {"token": "earlier value"}) + # After the read of the migration, an update set a new value and a version that does not mark it sensitive. + updated = {**snapshot, "templateVersion": "2.0.0", "properties": {"token": "new value"}, "_etag": "2"} + resource_repo._container = FakeContainer(updated) + resource_repo.query = AsyncMock(return_value=[snapshot]) + + result = await resource_repo.mask_stored_sensitive_values( + migration_template_repo(first, second), history_without_items() + ) + + assert result == (0, 0) + assert resource_repo.container.documents["ws"]["properties"] == {"token": "new value"} + + +@pytest.mark.asyncio +async def test_the_migration_computes_the_paths_again_after_a_change_and_continues(resource_repo): + accounts = [{"name": "a", "token": "real"}, {"name": "b", "token": "real"}] + snapshot = [ + stored_resource("ws", "masking-template", "workspace", {"accounts": accounts}), + stored_resource("ws-2", "masking-template", "workspace", {"accounts": accounts[:1]}), + ] + # After the read of the migration, an update removed the second account. The path with index 1 is not valid now. + updated = {**snapshot[0], "properties": {"accounts": accounts[:1]}, "_etag": "2"} + resource_repo._container = FakeContainer(updated, snapshot[1]) + resource_repo.query = AsyncMock(return_value=snapshot) + + result = await resource_repo.mask_stored_sensitive_values( + migration_template_repo(masking_template(ACCOUNTS)), history_without_items() + ) + + assert result == (2, 0) + assert resource_repo.container.requests == [("ws", 2, "1"), ("ws", 1, "2"), ("ws-2", 1, "1")] + masked = [{"name": "a", "token": strings.REDACTED_SENSITIVE_VALUE}] + documents = resource_repo.container.documents + assert [documents[resource_id]["properties"]["accounts"] for resource_id in ("ws", "ws-2")] == [masked] * 2 + + +@pytest.mark.asyncio +async def test_the_migration_continues_if_a_document_changes_at_each_attempt(resource_repo): + class BusyContainer(FakeContainer): + async def patch_item(self, item, **kwargs): + if item == "ws": + # Another request changes the document before each patch of the migration. + self.documents[item]["_etag"] = str(int(self.documents[item]["_etag"]) + 1) + return await super().patch_item(item, **kwargs) + + snapshot = [ + stored_resource("ws", "masking-template", "workspace", {"token": "real"}), + stored_resource("ws-2", "masking-template", "workspace", {"token": "real"}), + ] + resource_repo._container = BusyContainer(*snapshot) + resource_repo.query = AsyncMock(return_value=snapshot) + + with patch("db.repositories.resources.logger") as logger: + result = await resource_repo.mask_stored_sensitive_values( + migration_template_repo(masking_template({"token": SECRET})), history_without_items() + ) + + assert result == (1, 0) + assert [request[0] for request in resource_repo.container.requests] == ["ws", "ws", "ws", "ws-2"] + assert resource_repo.container.documents["ws"]["properties"] == {"token": "real"} + assert resource_repo.container.documents["ws-2"]["properties"] == {"token": strings.REDACTED_SENSITIVE_VALUE} + logger.warning.assert_called_once() PASSWORD = {"type": "string", "sensitive": True, "updateable": True} @@ -632,6 +757,8 @@ async def test_mask_stored_sensitive_values_finds_a_property_that_a_branch_decla item="ws", partition_key="ws", patch_operations=[{"op": "set", "path": "/properties/password", "value": strings.REDACTED_SENSITIVE_VALUE}], + etag="1", + match_condition=MatchConditions.IfNotModified, ) @@ -698,15 +825,18 @@ async def test_mask_stored_sensitive_values_masks_values_in_array_items(resource template_repo.query.return_value = [template.model_dump()] template_repo.get_template_by_name_and_version.return_value = template history_repo = history_without_items() - history_repo.query.return_value = [{"id": "h-1", "resourceId": "ws", "properties": {"accounts": accounts[:1]}}] + history_repo.query.return_value = [ + {"id": "h-1", "resourceId": "ws", "properties": {"accounts": accounts[:1]}, "_etag": "1"} + ] assert await resource_repo.mask_stored_sensitive_values(template_repo, history_repo) == (1, 1) set_token = [{"op": "set", "path": "/properties/accounts/0/token", "value": strings.REDACTED_SENSITIVE_VALUE}] + if_not_modified = {"etag": "1", "match_condition": MatchConditions.IfNotModified} resource_repo.container.patch_item.assert_awaited_once_with( - item="ws", partition_key="ws", patch_operations=set_token + item="ws", partition_key="ws", patch_operations=set_token, **if_not_modified ) history_repo.container.patch_item.assert_awaited_once_with( - item="h-1", partition_key="ws", patch_operations=set_token + item="h-1", partition_key="ws", patch_operations=set_token, **if_not_modified )