From a0fbd131e77d57bc69b3e4f058933c055f540aa2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 13:57:12 +0100 Subject: [PATCH 01/19] Clean up backup test workspaces after failure (#5135) Source PRs: #5171 --- CHANGELOG.md | 1 + e2e_tests/test_backups.py | 54 ++---- .../test_backup_workspace_cleanup.py | 158 ++++++++++++++++++ 3 files changed, 177 insertions(+), 36 deletions(-) create mode 100644 e2e_tests/unit_tests/test_backup_workspace_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f49a503145..6f8b5acda4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ BUG FIXES: (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) +* Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/test_backups.py b/e2e_tests/test_backups.py index 77504ba5da..986511110d 100644 --- a/e2e_tests/test_backups.py +++ b/e2e_tests/test_backups.py @@ -1,14 +1,11 @@ import uuid import pytest -from httpx import AsyncClient from starlette import status -from e2e_tests.conftest import clean_up_test_workspace -from e2e_tests.helpers import assert_status, get_admin_token, get_auth_header, get_template +from e2e_tests.helpers import assert_status, get_admin_token, get_template from e2e_tests.resources import strings -from e2e_tests.resources.resource import post_resource -from e2e_tests.resources.workspace import get_workspace +from e2e_tests.resources.resource import get_resource, temporary_resource pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -30,33 +27,22 @@ async def test_create_base_workspace_with_backup_setting(enable_backup, expected template_properties = response.json().get("properties", {}) assert "enable_backup" in template_properties - workspace_path = "" - workspace_id = "" - - try: - properties = { - "display_name": f"E2E Backup Workspace {uuid.uuid4().hex[:8]}", - "description": "Base workspace for backup E2E tests", - "auth_type": "Automatic", - "address_space_size": "small", - "enable_backup": enable_backup, - } - if enable_backup: - properties["delete_backups_on_uninstall"] = True - payload = { - "templateName": strings.BASE_WORKSPACE, - "properties": properties, - } - - workspace_path, workspace_id = await post_resource( - payload, - strings.API_WORKSPACES, - access_token=admin_token, - verify=verify, - ) - - async with AsyncClient(verify=verify) as client: - workspace = await get_workspace(client, workspace_id, get_auth_header(admin_token)) + properties = { + "display_name": f"E2E Backup Workspace {uuid.uuid4().hex[:8]}", + "description": "Base workspace for backup E2E tests", + "auth_type": "Automatic", + "address_space_size": "small", + "enable_backup": enable_backup, + } + if enable_backup: + properties["delete_backups_on_uninstall"] = True + payload = { + "templateName": strings.BASE_WORKSPACE, + "properties": properties, + } + + async with temporary_resource(payload, strings.API_WORKSPACES, admin_token, verify) as workspace_path: + workspace = (await get_resource(f"/api{workspace_path}", admin_token, verify))["workspace"] assert workspace["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED @@ -73,7 +59,3 @@ async def test_create_base_workspace_with_backup_setting(enable_backup, expected assert value, f"Expected non-empty workspace property '{prop}' when backup is enabled" else: assert not value, f"Expected empty workspace property '{prop}' when backup is disabled" - - finally: - if workspace_path: - await clean_up_test_workspace(pre_created_workspace_id="", workspace_path=workspace_path, verify=verify) diff --git a/e2e_tests/unit_tests/test_backup_workspace_cleanup.py b/e2e_tests/unit_tests/test_backup_workspace_cleanup.py new file mode 100644 index 0000000000..5a3eaa0809 --- /dev/null +++ b/e2e_tests/unit_tests/test_backup_workspace_cleanup.py @@ -0,0 +1,158 @@ +"""Exercise backup workspace cleanup through the real E2E test and API helpers.""" + +import json +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import AsyncClient, MockTransport, Response + +from e2e_tests import helpers +from e2e_tests import test_backups as backups +from e2e_tests.resources import resource, strings + + +class BackupWorkspaceCleanupTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + self.enterContext(patch.object(backups, "get_admin_token", new=AsyncMock(return_value="admin-token"))) + self.resource_path = "/workspaces/backup-workspace" + self.requests = [] + + def mock_api( + self, + *, + enabled=True, + install=strings.RESOURCE_STATUS_DEPLOYED, + disable=strings.RESOURCE_STATUS_UPDATED, + delete=strings.RESOURCE_STATUS_DELETED, + post_status=202, + missing_output=False, + ): + def handle(request): + if request.url.path == "/api/workspace-templates/tre-workspace-base": + return Response(200, json={"properties": {"enable_backup": {"type": "boolean"}}}) + self.requests.append(request) + phase = request.url.path.rsplit("/", 1)[-1] + response_status = 200 + if request.method == "POST": + self.assertEqual(request.url.path, "/api/workspaces") + properties = json.loads(request.content)["properties"] + self.assertEqual(properties["enable_backup"], enabled) + if enabled: + self.assertIs(properties["delete_backups_on_uninstall"], True) + else: + self.assertNotIn("delete_backups_on_uninstall", properties) + phase, response_status, state = "install", post_status, "deploying" + elif request.method == "PATCH": + self.assertEqual(request.url.path, f"/api{self.resource_path}") + self.assertEqual(json.loads(request.content), {"isEnabled": False}) + phase, response_status, state = "disable", 202, "updating" + elif request.method == "DELETE": + self.assertEqual(request.url.path, f"/api{self.resource_path}") + phase, state = "delete", "deleting" + elif request.url.path == f"/api{self.resource_path}": + properties = { + key: f"backup-{key}" if enabled else "" + for key in ["backup_vault_name", "vm_backup_policy_id", "fileshare_backup_policy_id"] + } + if missing_output: + properties.pop("fileshare_backup_policy_id") + return Response( + 200, + json={"workspace": {"deploymentStatus": "deployed", "isEnabled": False, "properties": properties}}, + ) + else: + self.assertEqual(request.method, "GET") + state = {"install": install, "disable": disable, "delete": delete}[phase] + if isinstance(state, list): + state = state.pop(0) if len(state) > 1 else state[0] + if isinstance(state, Exception): + raise state + return Response( + response_status, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourcePath": self.resource_path, + "resourceId": "backup-workspace", + "status": state, + "message": "Mock operation outcome", + "steps": [], + } + }, + ) + + def create_client(**_): + return AsyncClient(transport=MockTransport(handle)) + + self.enterContext(patch.object(helpers, "AsyncClient", side_effect=create_client)) + self.enterContext(patch.object(resource, "AsyncClient", side_effect=create_client)) + + def assert_deleted(self): + self.assertEqual([r.method for r in self.requests if r.method != "GET"], ["POST", "PATCH", "DELETE"]) + self.assertEqual(self.requests[-1].url.path, "/api/operations/delete") + self.assertEqual(self.requests[-2].url.path, f"/api{self.resource_path}") + + async def test_success_cleans_up_both_backup_settings(self): + for enabled in [True, False]: + with self.subTest(enabled=enabled): + self.requests = [] + self.mock_api(enabled=enabled) + await backups.test_create_base_workspace_with_backup_setting(enabled, enabled, True) + self.assert_deleted() + + async def test_failed_deployment_cleans_up_even_if_disable_fails(self): + self.mock_api( + install=strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, disable=strings.RESOURCE_STATUS_UPDATING_FAILED + ) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + self.assertEqual( + [r.method for r in self.requests], ["POST", "GET", "GET", "PATCH", "GET", "GET", "DELETE", "GET"] + ) + + async def test_output_assertion_failure_still_cleans_up(self): + self.mock_api(missing_output=True) + with self.assertRaisesRegex(AssertionError, "fileshare_backup_policy_id"): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + + async def test_cleanup_failure_preserves_original_polling_error(self): + failure = RuntimeError("backup registration polling failed") + self.mock_api( + install=[failure, strings.RESOURCE_STATUS_DEPLOYMENT_FAILED], delete=strings.RESOURCE_STATUS_DELETING_FAILED + ) + with self.assertRaises(RuntimeError) as caught: + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertIs(caught.exception, failure) + self.assertIn( + "Cleanup of /workspaces/backup-workspace after /api/operations/install also failed", + caught.exception.__notes__[0], + ) + self.assert_deleted() + + async def test_unreadable_install_operation_preserves_resource_and_original_error(self): + failure = RuntimeError("backup registration polling failed") + self.mock_api(install=failure) + with self.assertRaises(RuntimeError) as caught: + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertIs(caught.exception, failure) + self.assertIn("also failed", caught.exception.__notes__[0]) + self.assertEqual([r.method for r in self.requests], ["POST", "GET", "GET"]) + + async def test_cleanup_failure_fails_a_successful_deployment(self): + self.mock_api(delete=strings.RESOURCE_STATUS_DELETING_FAILED) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + + async def test_rejected_creation_does_not_attempt_cleanup(self): + self.mock_api(post_status=403) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertEqual([r.method for r in self.requests], ["POST"]) + + +if __name__ == "__main__": + unittest.main() From 71bf455322834a52ec49fa4547d980a6efe67ca2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 22 Oct 2026 03:00:14 +0100 Subject: [PATCH 02/19] Make one Azure Backup registration attempt before Terraform registers (#5135) Every backup-enabled workspace failed to deploy. Azure Backup returned CloudInternalError when Terraform registered the workspace storage account with the new vault. A scratch reproduction in switzerlandnorth showed that the first registration in a new vault usually fails, whatever the storage settings: 4 of 5 new vaults failed it, as did 3 of 3 TRE deployments. Every later registration in the same vault succeeded (9 of 9). A failed attempt leaves no container behind. A new step runs before the Terraform registration: - If the storage account is already registered, it does nothing. - Otherwise it makes one registration attempt and waits for the result. - If that attempt succeeds, it removes the registration again, so that Terraform creates and owns it. A scratch Terraform apply covered a new vault and a re-run on a registered account. Unit tests run the step against a fake Azure CLI. --- CHANGELOG.md | 1 + .../unit_tests/test_backup_registration.py | 102 ++++++++++++++++++ .../airlock-import-review/porter.yaml | 2 +- templates/workspaces/base/porter.yaml | 2 +- .../workspaces/base/terraform/storage.tf | 53 +++++++++ templates/workspaces/unrestricted/porter.yaml | 2 +- 6 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 e2e_tests/unit_tests/test_backup_registration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8b5acda4..9d97e36789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ BUG FIXES: (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) +* Make one storage registration attempt before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py new file mode 100644 index 0000000000..994024787e --- /dev/null +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -0,0 +1,102 @@ +"""Run the backup registration warm-up from the base workspace Terraform against a fake Azure CLI.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +STORAGE = ROOT / "templates/workspaces/base/terraform/storage.tf" +CONTAINER = ( + "https://management.azure.com/vault/backupFabrics/Azure/protectionContainers/container?api-version=2023-04-01" +) +OPERATION = "https://management.azure.com/vault/operations/1" + +FAKE_AZ = """import json,os,sys +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps(args)+'\\n') +state=json.load(open(os.environ['STATE'])) +def save():json.dump(state,open(os.environ['STATE'],'w')) +if args[0]!='rest':sys.exit(0) +method=args[args.index('--method')+1];url=args[args.index('--url')+1] +if url==os.environ['OPERATION']: + print(state['outcome']) + state['registered']=state['outcome']=='Succeeded';save();sys.exit(0) +if method=='get':sys.exit(0 if state['registered'] else 1) +if method=='put': + if state['header']:print("INFO: cli.azure.cli.core.util: 'Azure-AsyncOperation': '"+os.environ['OPERATION']+"'",file=sys.stderr) + sys.exit(0) +if method=='delete':state['registered']=False;save();sys.exit(0) +""" + + +def warm_up_command(): + source = STORAGE.read_text() + block = source.split('resource "terraform_data" "backup_registration_warm_up" {', 1)[1] + command = block.split("</dev/null; then + echo "The storage account is already registered with the vault" + exit 0 +fi +body='{"properties":{"backupManagementType":"AzureStorage","containerType":"StorageContainer","sourceResourceId":"${self.input.storage_account_id}"}}' +operation=$(az rest --method put --url "$url" --body "$body" --debug 2>&1 | sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/p" | head -n 1 || true) +status=Unknown +if [ -n "$operation" ]; then + for attempt in $(seq 1 60); do + status=$(az rest --method get --url "$operation" --query status --output tsv 2>/dev/null || echo Unknown) + [ "$status" = "InProgress" ] || break + sleep 10 + done +else + sleep 120 +fi +echo "The first registration attempt ended with status $status" +if az rest --method get --url "$url" --output none 2>/dev/null; then + # Remove a successful first registration, so that Terraform creates and owns the registration. + az rest --method delete --url "$url" --output none + for attempt in $(seq 1 60); do + az rest --method get --url "$url" --output none 2>/dev/null || exit 0 + sleep 10 + done + echo "The storage account is still registered with the vault after 10 minutes" >&2 + exit 1 +fi +EOT + } +} + resource "azurerm_backup_container_storage_account" "storage_account" { count = var.enable_backup ? 1 : 0 resource_group_name = azurerm_resource_group.ws.name recovery_vault_name = module.backup[0].vault_name storage_account_id = azurerm_storage_account.stg.id + + depends_on = [ + terraform_data.backup_registration_warm_up + ] } resource "terraform_data" "wait_for_backup_cleanup" { diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index 5f37520600..805be3a171 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.15.2 +version: 0.15.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre From bd131e1c0ce4b569e3e3e8bf57310451f199e77c Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 25 Oct 2026 04:46:36 +0000 Subject: [PATCH 03/19] Try the backup registration warm-up up to five times (#5135) In two live runs, the warm-up attempt failed with CloudInternalError, and the Terraform registration right after it failed with the same error. In two earlier runs, the Terraform registration passed after a failed warm-up attempt. The warm-up now tries up to five times, with 60 seconds between the attempts. It removes the first registration that succeeds, so that Terraform creates and owns the registration. After five failed attempts, Terraform makes its own attempt, as before. --- CHANGELOG.md | 2 +- .../unit_tests/test_backup_registration.py | 32 ++++++----- .../workspaces/base/terraform/storage.tf | 54 ++++++++++--------- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d97e36789..33439c2446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ BUG FIXES: (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) -* Make one storage registration attempt before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) +* Try the storage registration up to five times before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. Terraform registers after an attempt succeeds, or after five failed attempts. (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 994024787e..e35a17d1ad 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -23,8 +23,9 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) if args[0]!='rest':sys.exit(0) method=args[args.index('--method')+1];url=args[args.index('--url')+1] if url==os.environ['OPERATION']: - print(state['outcome']) - state['registered']=state['outcome']=='Succeeded';save();sys.exit(0) + outcome=state['outcomes'].pop(0) if len(state['outcomes'])>1 else state['outcomes'][0] + print(outcome) + state['registered']=outcome=='Succeeded';save();sys.exit(0) if method=='get':sys.exit(0 if state['registered'] else 1) if method=='put': if state['header']:print("INFO: cli.azure.cli.core.util: 'Azure-AsyncOperation': '"+os.environ['OPERATION']+"'",file=sys.stderr) @@ -54,9 +55,9 @@ def setUp(self): path.write_text("#!" + sys.executable + "\n" + body) path.chmod(0o755) - def run_warm_up(self, *, registered=False, outcome="Failed", header=True): + def run_warm_up(self, *, registered=False, outcomes=("Failed",), header=True): state = self.folder / "state.json" - state.write_text(json.dumps({"registered": registered, "outcome": outcome, "header": header})) + state.write_text(json.dumps({"registered": registered, "outcomes": list(outcomes), "header": header})) env = os.environ | { "PATH": str(self.folder) + os.pathsep + os.environ["PATH"], "CALLS": str(self.folder / "calls.jsonl"), @@ -75,24 +76,31 @@ def test_existing_registration_is_not_changed(self): self.assertEqual(methods, ["get"]) self.assertTrue(state["registered"]) - def test_failed_first_attempt_leaves_registration_to_terraform(self): - result, methods, state = self.run_warm_up(outcome="Failed") + def test_five_failed_attempts_leave_registration_to_terraform(self): + result, methods, state = self.run_warm_up(outcomes=["Failed"]) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get"]) + self.assertEqual(methods, ["get"] + ["put", "get"] * 5) self.assertFalse(state["registered"]) - self.assertIn("ended with status Failed", result.stdout) + self.assertIn("Registration attempt 5 ended with status Failed", result.stdout) + + def test_attempts_continue_until_one_succeeds(self): + result, methods, state = self.run_warm_up(outcomes=["Failed", "Failed", "Succeeded"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get"] + ["put", "get"] * 3 + ["delete", "get"]) + self.assertFalse(state["registered"]) + self.assertIn("Registration attempt 3 ended with status Succeeded", result.stdout) def test_successful_first_attempt_is_removed_for_terraform(self): - result, methods, state = self.run_warm_up(outcome="Succeeded") + result, methods, state = self.run_warm_up(outcomes=["Succeeded"]) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) self.assertFalse(state["registered"]) - def test_missing_operation_header_waits_before_terraform(self): + def test_missing_operation_header_waits_before_each_check(self): result, methods, _ = self.run_warm_up(header=False) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get"]) - self.assertIn("ended with status Unknown", result.stdout) + self.assertEqual(methods, ["get"] + ["put", "get"] * 5) + self.assertIn("Registration attempt 1 ended with status Unknown", result.stdout) def test_terraform_registration_waits_for_the_warm_up(self): block = STORAGE.read_text().split('resource "azurerm_backup_container_storage_account" "storage_account" {', 1)[ diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index b552f58ae9..c8746b5b01 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -166,8 +166,9 @@ resource "azurerm_private_endpoint" "stgdfspe" { } } -# Azure Backup fails the first storage registration in a new vault with CloudInternalError. -# This step makes that first attempt, so that the registration below is not the first one. +# Azure Backup can fail the first storage registrations in a new vault with CloudInternalError. +# This step tries up to five times and removes the first registration that succeeds. Then the +# registration below is not the first one, and Terraform creates and owns the registration. resource "terraform_data" "backup_registration_warm_up" { count = var.enable_backup ? 1 : 0 @@ -189,28 +190,33 @@ if az rest --method get --url "$url" --output none 2>/dev/null; then exit 0 fi body='{"properties":{"backupManagementType":"AzureStorage","containerType":"StorageContainer","sourceResourceId":"${self.input.storage_account_id}"}}' -operation=$(az rest --method put --url "$url" --body "$body" --debug 2>&1 | sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/p" | head -n 1 || true) -status=Unknown -if [ -n "$operation" ]; then - for attempt in $(seq 1 60); do - status=$(az rest --method get --url "$operation" --query status --output tsv 2>/dev/null || echo Unknown) - [ "$status" = "InProgress" ] || break - sleep 10 - done -else - sleep 120 -fi -echo "The first registration attempt ended with status $status" -if az rest --method get --url "$url" --output none 2>/dev/null; then - # Remove a successful first registration, so that Terraform creates and owns the registration. - az rest --method delete --url "$url" --output none - for attempt in $(seq 1 60); do - az rest --method get --url "$url" --output none 2>/dev/null || exit 0 - sleep 10 - done - echo "The storage account is still registered with the vault after 10 minutes" >&2 - exit 1 -fi +for try in 1 2 3 4 5; do + if [ "$try" -gt 1 ]; then + sleep 60 + fi + operation=$(az rest --method put --url "$url" --body "$body" --debug 2>&1 | sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/p" | head -n 1 || true) + status=Unknown + if [ -n "$operation" ]; then + for attempt in $(seq 1 60); do + status=$(az rest --method get --url "$operation" --query status --output tsv 2>/dev/null || echo Unknown) + [ "$status" = "InProgress" ] || break + sleep 10 + done + else + sleep 120 + fi + echo "Registration attempt $try ended with status $status" + if az rest --method get --url "$url" --output none 2>/dev/null; then + # Remove the registration, so that Terraform creates and owns the registration. + az rest --method delete --url "$url" --output none + for attempt in $(seq 1 60); do + az rest --method get --url "$url" --output none 2>/dev/null || exit 0 + sleep 10 + done + echo "The storage account is still registered with the vault after 10 minutes" >&2 + exit 1 + fi +done EOT } } From 8b21541ee066533252e150c9d47e9a29d0886b5c Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 25 Oct 2026 22:43:38 +0000 Subject: [PATCH 04/19] Wrap a long CHANGELOG entry (#5135) The Markdown lint rejects a line with more than 400 characters. The entry now continues on lines that start with two spaces, as an earlier entry in the file does. The rendered text does not change. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33439c2446..3faa259704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,8 @@ BUG FIXES: (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) -* Try the storage registration up to five times before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. Terraform registers after an attempt succeeds, or after five failed attempts. (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) +* Try the storage registration up to five times before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. Terraform registers after an attempt succeeds, or after five failed attempts. + (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) From 82083d0e44fa7d55ba74599967e2d130b66c8cd0 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 19:20:06 +0100 Subject: [PATCH 05/19] Validate OpenAI model prerequisites and private lifecycle (#5150) --- .github/workflows/deploy_tre_reusable.yml | 2 + CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 19 + .../workspace-services/openai.md | 24 +- e2e_tests/bundle_coverage.json | 22 +- e2e_tests/pytest.ini | 1 + e2e_tests/resources/openai.py | 114 ++++++ e2e_tests/resources/sql_probe.py | 4 +- e2e_tests/test_openai.py | 93 +++++ e2e_tests/test_workspace_services.py | 1 - e2e_tests/unit_tests/test_bundle_selection.py | 9 + .../unit_tests/test_openai_validation.py | 349 ++++++++++++++++++ .../workspace_services/openai/porter.yaml | 2 +- .../openai/template_schema.json | 7 +- .../openai/terraform/openai.tf | 3 +- 15 files changed, 636 insertions(+), 15 deletions(-) create mode 100644 e2e_tests/resources/openai.py create mode 100644 e2e_tests/test_openai.py create mode 100644 e2e_tests/unit_tests/test_openai_validation.py diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 6849c357d8..859ab83f71 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -718,6 +718,8 @@ jobs: BUNDLE_DIR: "./templates/workspace_services/ohdsi" - BUNDLE_TYPE: "workspace_service" BUNDLE_DIR: "./templates/workspace_services/azuresql" + - BUNDLE_TYPE: "workspace_service" + BUNDLE_DIR: "./templates/workspace_services/openai" environment: ${{ inputs.environmentName }} steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3faa259704..3419ca449f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: +* Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index c62d54490e..b0894e3853 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -82,6 +82,25 @@ That case requires an owned workspace so cascade cleanup removes its new service Some shared-service cases replace existing services, so use an isolated validation environment. Check the selected entry's prerequisites and limitations before running it. +### OpenAI selection + +Select `tre-workspace-service-openai` to run `test_openai.py::test_private_openai_lifecycle`. +The branch workflow registers the OpenAI bundle. The test also replaces its generic case in the `workspace_services` group. +The test requests `gpt-5.1 | 2025-11-13`, private access and one regional `Standard` capacity unit. +It does not switch models or deployment types when Azure cannot satisfy those inputs. + +The prerequisite check uses the workspace resource group's actual region and the current subscription model and usage APIs. +It requires a generally available chat model, a current Standard tier and at least one free capacity unit. +It rejects retired model/tier dates, fine-tuning quotas and incomplete or blocked quota data. +The CI identity needs permission to read model and quota data at subscription scope. +The check does not grant roles or reserve capacity. Azure deployment can still fail after it passes. + +Prerequisite failures fail the selected test before service creation. They do not count as skipped or passing coverage. +Service creation has a 45-minute deadline. Verification has a separate five-minute deadline. +Failed creation and teardown use the existing helper's separate cleanup deadline. +A configured workspace is retained. A fixture-created workspace uses the fixture's cleanup. +Private inference, access control, upgrades and complete physical cleanup remain separate checks. + ### Certificate and Nexus selection Select `tre-shared-service-certs` to install, uninstall and reinstall the certificate shared service with the same hostname and certificate name. diff --git a/docs/tre-templates/workspace-services/openai.md b/docs/tre-templates/workspace-services/openai.md index 5ad0b57395..1ea4bf8077 100644 --- a/docs/tre-templates/workspace-services/openai.md +++ b/docs/tre-templates/workspace-services/openai.md @@ -17,6 +17,28 @@ See: [Azure OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/ ## Properties - `is_exposed_externally` - If `True`, the OpenAI workspace is accessible from outside of the workspace virtual network. -- `openai_model` - The model to use for the OpenAI deployment ` | `. The default is `gpt-35-turbo | 0301`. +- `openai_model` - The model to use for the OpenAI deployment ` | `. The default is `gpt-5.1 | 2025-11-13`. - Important note: Models are subject to different quota and region availability and the deployment may fail if you don't have the correct quota. Please review this link on current limits and how to request increases: [Open AI Quotas](https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits) + + +## Model prerequisites + +The bundle uses the regional `Standard` deployment type with one capacity unit. +Check availability and quota for the exact model version in the workspace region before deployment. +The older model values remain in the schema for existing resource records. Their presence does not mean Azure can deploy them. +Changing the template default does not change the model value stored on an existing service. +An explicit model change can replace the deployment and needs separate upgrade validation. + +The OpenAI bundle currently requires the workspace and core in the same subscription. +Private network access remains the default. The bundle does not add researcher RBAC roles or change key authentication. + +See [model availability](https://learn.microsoft.com/en-us/rest/api/aiservices/accountmanagement/models/list?view=rest-aiservices-accountmanagement-2024-10-01) +and [subscription quota](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/quota). + +## Independent validation + +Select `tre-workspace-service-openai` through the [bundle validation procedure](../../tre-developers/end-to-end-tests.md#validate-one-bundle). +The test checks current model and quota prerequisites, deploys a private service, checks its model and network settings, and uninstalls it. +It confirms removal of the API record and active Azure account. +Private inference, researcher access, model upgrades, secret removal and soft-deleted account purge require separate evidence. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index d443823a66..2e61202415 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -411,15 +411,25 @@ "prerequisites": [ "tre-workspace-base" ], - "tests": [], - "related_tests": [ - "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-openai]" + "tests": [ + "test_openai.py::test_private_openai_lifecycle" + ], + "related_tests": [], + "coverage": [ + "model-prerequisites", + "install", + "deployment-status", + "private-network-setting", + "uninstall", + "api-removal", + "active-account-removal" ], - "coverage": [], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, a registered 1.1.0 template, regional Standard model availability and subscription quota read access. Preflight does not reserve Azure capacity.", + "Uses the workspace subscription and actual resource-group region. A workspace in a different subscription from ARM_SUBSCRIPTION_ID is unsupported by this bundle.", + "Does not prove private inference, researcher access, model upgrades, private endpoint connectivity, secret removal or soft-deleted account purge." ], - "unavailable_reason": "The branch deployment does not register this bundle (#5150)." + "unavailable_reason": "" }, "tre-workspace-airlock-import-review": { "manifest": "templates/workspaces/airlock-import-review/porter.yaml", diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index b810211fe8..8ce1ae4975 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -15,6 +15,7 @@ markers = airlock_validation: draft-container deletion and review-VM flow, including Nexus prerequisites and cleanup workspace_services workspace_validation: opt-in base-workspace creation, scoped API access and deletion + openai_validation: opt-in private OpenAI deployment with model and quota preflight sql_validation: Private SQL query and configuration upgrade lifecycle asyncio_mode = auto diff --git a/e2e_tests/resources/openai.py b/e2e_tests/resources/openai.py new file mode 100644 index 0000000000..3465f142c8 --- /dev/null +++ b/e2e_tests/resources/openai.py @@ -0,0 +1,114 @@ +"""Check the prerequisites for the private OpenAI lifecycle case.""" + +from datetime import datetime, timezone +import math +from urllib.parse import parse_qsl, urlsplit + +COGNITIVE_API = "2024-10-01" +MODEL = "gpt-5.1 | 2025-11-13" +CAPACITY = 1 + + +async def list_values(arm, resource_id): + """Read all ARM pages, keeping credentials on the original collection.""" + values, seen, query = [], set(), {} + for _ in range(100): + page = await arm.request("GET", resource_id, COGNITIVE_API, query=query) + if not isinstance(page.get("value"), list): + raise ValueError("Azure returned an invalid model or quota list") + values.extend(page["value"]) + link = page.get("nextLink") + if not link: + return values + parsed = urlsplit(link) + pairs = parse_qsl(parsed.query, keep_blank_values=True) + query = dict(pairs) + if ( + parsed.scheme != "https" + or parsed.netloc != "management.azure.com" + or parsed.path != resource_id + or parsed.fragment + or len(query) != len(pairs) + or query.get("api-version") != COGNITIVE_API + or link in seen + ): + raise ValueError("Azure returned an invalid model or quota continuation link") + seen.add(link) + query.pop("api-version") + raise ValueError("Azure model or quota list exceeded the page limit") + + +def unexpired(value, now): + if value is None: + return True + try: + deadline = datetime.fromisoformat(value.replace("Z", "+00:00")) + return deadline.tzinfo is not None and deadline > now + except (ValueError, TypeError, AttributeError): + return False + + +def supported_sku(models, now): + """Match the exact approved model and regional Standard deployment type.""" + name, version = (part.strip() for part in MODEL.split("|")) + for entry in models: + model = entry.get("model", {}) + if ( + entry.get("kind") != "OpenAI" + or entry.get("skuName") != "S0" + or model.get("format") != "OpenAI" + or model.get("name") != name + or model.get("version") != version + or model.get("lifecycleStatus") != "GenerallyAvailable" + or model.get("capabilities", {}).get("chatCompletion") != "true" + or not unexpired(model.get("deprecation", {}).get("inference"), now) + ): + continue + for sku in model.get("skus", []): + usage = sku.get("usageName", "") + if ( + sku.get("name") == "Standard" + and usage + and "finetune" not in usage.lower() + and unexpired(sku.get("deprecationDate"), now) + ): + return sku + raise ValueError(f"OpenAI prerequisite failed: {MODEL} has no current regional Standard chat deployment") + + +def check_capacity(sku, usages): + capacity = sku.get("capacity", {}) + minimum, maximum, step = capacity.get("minimum", 1), capacity.get("maximum", CAPACITY), capacity.get("step", 1) + if ( + any(type(value) is not int for value in (minimum, maximum, step)) + or step < 1 + or not minimum <= CAPACITY <= maximum + or (CAPACITY - minimum) % step + or ("allowedValues" in capacity and CAPACITY not in capacity["allowedValues"]) + ): + raise ValueError("OpenAI prerequisite failed: Standard does not support one capacity unit") + for usage in usages: + if usage.get("name", {}).get("value") != sku["usageName"]: + continue + current, limit = usage.get("currentValue"), usage.get("limit") + if ( + type(current) not in (int, float) + or type(limit) not in (int, float) + or not math.isfinite(current) + or not math.isfinite(limit) + or current < 0 + or limit < 0 + or usage.get("unit") != "Count" + or usage.get("status") in ("Blocked", "InOverage", "Unknown") + ): + raise ValueError("OpenAI prerequisite failed: quota values are invalid or blocked") + if limit - current < CAPACITY: + raise ValueError("OpenAI prerequisite failed: Standard model quota has no free capacity unit") + return + raise ValueError("OpenAI prerequisite failed: the Standard model quota was not returned") + + +async def check_prerequisites(arm, subscription_id, location): + collection = f"/subscriptions/{subscription_id}/providers/Microsoft.CognitiveServices/locations/{location}" + sku = supported_sku(await list_values(arm, collection + "/models"), datetime.now(timezone.utc)) + check_capacity(sku, await list_values(arm, collection + "/usages")) diff --git a/e2e_tests/resources/sql_probe.py b/e2e_tests/resources/sql_probe.py index e72333139f..81b3e527f5 100644 --- a/e2e_tests/resources/sql_probe.py +++ b/e2e_tests/resources/sql_probe.py @@ -30,11 +30,11 @@ def __init__(self, client, credential): self.client = client self.credential = credential - async def request(self, method, resource_id, api_version, *, body=None, missing_ok=False, expand=False): + async def request(self, method, resource_id, api_version, *, body=None, missing_ok=False, expand=False, query=None): if not resource_id.startswith("/subscriptions/") or any(c in resource_id for c in "?#"): raise ValueError("Expected an ARM resource ID") token = await asyncio.to_thread(self.credential.get_token, f"{ARM}/.default") - params = {"api-version": api_version} + params = {**(query or {}), "api-version": api_version} if expand: params["$expand"] = "instanceView" response = await self.client.request( diff --git a/e2e_tests/test_openai.py b/e2e_tests/test_openai.py new file mode 100644 index 0000000000..e046e53159 --- /dev/null +++ b/e2e_tests/test_openai.py @@ -0,0 +1,93 @@ +"""Validate private OpenAI deployment after checking model and quota prerequisites.""" + +import asyncio +import os +import re + +import pytest +from httpx import AsyncClient + +from e2e_tests import config +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import TIMEOUT, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.openai import CAPACITY, COGNITIVE_API, MODEL, check_prerequisites +from e2e_tests.resources.resource import get_resource, temporary_resource +from e2e_tests.resources.sql_probe import arm_client + +CREATE_TIMEOUT_SECONDS = 45 * 60 +VERIFY_TIMEOUT_SECONDS = 5 * 60 + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.workspace_services +@pytest.mark.openai_validation +async def test_private_openai_lifecycle(verify, setup_test_workspace): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("The OpenAI prerequisite check currently supports AzureCloud only") + workspace_path, workspace_id = setup_test_workspace + token = await get_workspace_owner_token(workspace_id, verify) + async with arm_client() as arm: + async with asyncio.timeout(5 * 60): + workspace = (await get_resource(f"/api{workspace_path}", token, verify))["workspace"] + subscription = os.environ["ARM_SUBSCRIPTION_ID"] + if ( + workspace["properties"].get("workspace_subscription_id") or subscription + ).lower() != subscription.lower(): + raise ValueError("This OpenAI bundle requires the workspace and core in the same subscription") + if not re.fullmatch(r"[a-fA-F0-9-]{36}", subscription) or not re.fullmatch(r"[a-z0-9-]+", config.TRE_ID): + raise ValueError("Invalid subscription or TRE identifier for the OpenAI prerequisite check") + group = f"/subscriptions/{subscription}/resourceGroups/rg-{config.TRE_ID}-ws-{workspace_id[-4:]}" + region = (await arm.request("GET", group, "2022-09-01"))["location"] + if not re.fullmatch(r"[a-z0-9]+", region): + raise ValueError("Invalid workspace region for the OpenAI prerequisite check") + await check_prerequisites(arm, subscription, region) + payload = { + "templateName": strings.OPENAI_SERVICE, + "properties": { + "display_name": "Private OpenAI lifecycle test", + "description": "OpenAI deployment and cleanup validation", + "openai_model": MODEL, + "is_exposed_externally": False, + }, + } + # Cancel the creation deadline after deployment. Verification and cleanup + # each have their own budget, even if creation uses its full allowance. + async with asyncio.timeout(CREATE_TIMEOUT_SECONDS) as creation_deadline: + async with temporary_resource( + payload, f"/api{workspace_path}/workspace-services", token, verify + ) as service_path: + creation_deadline.reschedule(None) + async with asyncio.timeout(VERIFY_TIMEOUT_SECONDS): + service = (await get_resource(f"/api{service_path}", token, verify))["workspaceService"] + assert service["templateName"] == strings.OPENAI_SERVICE + assert service["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED + properties = service["properties"] + assert properties["openai_model"] == MODEL + assert properties["is_exposed_externally"] is False + account_name = f"openai-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{service['id'][-4:]}" + account_id = f"{group}/providers/Microsoft.CognitiveServices/accounts/{account_name}" + account = await arm.request("GET", account_id, COGNITIVE_API) + assert account["properties"]["provisioningState"] == "Succeeded" + assert account["properties"]["publicNetworkAccess"] == "Disabled" + assert properties["openai_fqdn"] == account["properties"]["endpoint"] + name, version = (part.strip() for part in MODEL.split("|")) + deployment_name = ( + f"openai-{name}-{version}-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{service['id'][-4:]}" + ) + assert properties["openai_deployment_id"] == deployment_name + deployment = await arm.request("GET", f"{account_id}/deployments/{deployment_name}", COGNITIVE_API) + assert deployment["properties"]["provisioningState"] == "Succeeded" + model = deployment["properties"]["model"] + assert (model["format"], model["name"], model["version"]) == ("OpenAI", name, version) + assert deployment["sku"]["name"] == "Standard" + assert deployment["sku"]["capacity"] == CAPACITY + async with asyncio.timeout(5 * 60): + async with AsyncClient(verify=verify) as client: + response = await client.get( + get_full_endpoint(f"/api{service_path}"), headers=get_auth_header(token), timeout=TIMEOUT + ) + assert response.status_code == 404, f"OpenAI service remains visible: HTTP {response.status_code}" + assert await arm.request("GET", account_id, COGNITIVE_API, missing_ok=True) is None, ( + "OpenAI account remains active after uninstall" + ) diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index fbb4704278..3ce22c5614 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -13,7 +13,6 @@ strings.MYSQL_SERVICE, strings.HEALTH_SERVICE, strings.AZURESQL_SERVICE, - strings.OPENAI_SERVICE, ] diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index fdca790ea5..5fc60fc50e 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -210,6 +210,15 @@ def test_unavailable_bundle_fails_with_its_coverage_gap(self): self.assertIn("#5127", report["error"]) self.assertEqual(report["collected_tests"], []) + def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): + result, report = self.run_selector("--bundle", "tre-workspace-service-openai", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_openai.py::test_private_openai_lifecycle"]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["source_bundle_version"], "1.1.0") + self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + def test_marker_and_parallel_worker_inputs_are_rejected(self): for extra in ({"TEST_BUNDLE_MARKER": "workspace_services"}, {"E2E_TESTS_NUMBER_PROCESSES": "2"}): with self.subTest(environment=extra): diff --git a/e2e_tests/unit_tests/test_openai_validation.py b/e2e_tests/unit_tests/test_openai_validation.py new file mode 100644 index 0000000000..ffc7c4cf2f --- /dev/null +++ b/e2e_tests/unit_tests/test_openai_validation.py @@ -0,0 +1,349 @@ +"""Check OpenAI prerequisite failures, evidence and owned-resource cleanup offline.""" + +import asyncio +from contextlib import asynccontextmanager +from copy import deepcopy +from datetime import datetime, timezone +import json +import os +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from httpx import AsyncClient, MockTransport, Response +from jsonschema import validate +import yaml + +from e2e_tests import test_openai as lifecycle +from e2e_tests.resources import openai, resource +from e2e_tests.resources.sql_probe import ArmClient + +NOW = datetime(2026, 10, 10, tzinfo=timezone.utc) +ROOT = Path(__file__).resolve().parents[2] +SUBSCRIPTION = "00000000-0000-4000-8000-000000000000" +WORKSPACE = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +PATH = f"/workspaces/{WORKSPACE}/workspace-services/{SERVICE}" +GROUP = f"/subscriptions/{SUBSCRIPTION}/resourceGroups/rg-tretest-ws-1111" +ACCOUNT = GROUP + "/providers/Microsoft.CognitiveServices/accounts/openai-tretest-ws-1111-svc-2222" +DEPLOYMENT = "openai-gpt-5.1-2025-11-13-tretest-ws-1111-svc-2222" + + +def model_entry(): + return { + "kind": "OpenAI", + "skuName": "S0", + "model": { + "format": "OpenAI", + "name": "gpt-5.1", + "version": "2025-11-13", + "lifecycleStatus": "GenerallyAvailable", + "capabilities": {"chatCompletion": "true"}, + "skus": [{"name": "Standard", "usageName": "OpenAI.Standard.gpt-5.1"}], + }, + } + + +def usage_entry(): + return {"name": {"value": "OpenAI.Standard.gpt-5.1"}, "currentValue": 9, "limit": 10, "unit": "Count"} + + +class ModelPrerequisiteTests(unittest.TestCase): + def test_current_exact_model_and_last_capacity_unit_are_accepted(self): + sku = openai.supported_sku([model_entry()], NOW) + openai.check_capacity(sku, [usage_entry()]) + + def test_wrong_version_preview_and_non_chat_models_are_rejected(self): + for field, value in ( + ("version", "wrong"), + ("lifecycleStatus", "Preview"), + ("capabilities", {}), + ("format", "other"), + ): + entry = model_entry() + entry["model"][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "no current regional"): + openai.supported_sku([entry], NOW) + + def test_global_fine_tuning_and_expired_standard_tiers_are_rejected(self): + cases = ( + ("name", "GlobalStandard"), + ("usageName", "OpenAI.Standard.gpt-5.1-finetune"), + ("usageName", ""), + ("deprecationDate", "2026-10-10T00:00:00Z"), + ("deprecationDate", "invalid"), + ("deprecationDate", "2027-01-01"), + ) + for key, value in cases: + entry = model_entry() + entry["model"]["skus"][0][key] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + openai.supported_sku([entry], NOW) + entry = model_entry() + entry["model"]["deprecation"] = {"inference": "2020-01-01T00:00:00Z"} + with self.assertRaises(ValueError): + openai.supported_sku([entry], NOW) + + def test_missing_exhausted_and_invalid_quota_never_pass(self): + sku = openai.supported_sku([model_entry()], NOW) + cases = [[], [dict(usage_entry(), name={"value": "GlobalStandard"})]] + for key, value in ( + ("currentValue", 10), + ("currentValue", -1), + ("limit", None), + ("limit", float("nan")), + ("limit", True), + ("status", "Blocked"), + ("status", "Unknown"), + ): + cases.append([dict(usage_entry(), **{key: value})]) + for usages in cases: + with self.subTest(usages=usages), self.assertRaises(ValueError): + openai.check_capacity(sku, usages) + + def test_model_capacity_minimum_and_step_can_prevent_deployment(self): + for capacity in ({"minimum": 10}, {"allowedValues": [2, 4]}, {"step": 0}, {"minimum": 0, "step": 2}): + sku = dict(openai.supported_sku([model_entry()], NOW), capacity=capacity) + with self.subTest(capacity=capacity), self.assertRaisesRegex(ValueError, "capacity unit"): + openai.check_capacity(sku, [usage_entry()]) + + def test_bundle_schema_and_ci_registration_agree_with_selection(self): + folder = ROOT / "templates/workspace_services/openai" + schema = json.loads((folder / "template_schema.json").read_text()) + validate({"openai_model": openai.MODEL, "is_exposed_externally": False}, schema) + self.assertEqual(schema["properties"]["openai_model"]["default"], openai.MODEL) + manifest = yaml.safe_load((folder / "porter.yaml").read_text()) + self.assertEqual(manifest["version"], "1.1.0") + workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) + registrations = workflow["jobs"]["register_bundles"]["strategy"]["matrix"]["include"] + self.assertIn( + {"BUNDLE_TYPE": "workspace_service", "BUNDLE_DIR": "./templates/workspace_services/openai"}, registrations + ) + + +class PaginationTests(unittest.IsolatedAsyncioTestCase): + async def test_pagination_retains_api_version_and_renews_credentials(self): + collection = f"/subscriptions/{SUBSCRIPTION}/providers/Microsoft.CognitiveServices/locations/westeurope/models" + requests = [] + + def handle(request): + requests.append(request) + self.assertEqual(request.url.params["api-version"], openai.COGNITIVE_API) + self.assertEqual(request.headers["Authorization"], "Bearer token") + if len(requests) == 1: + return Response( + 200, + json={ + "value": [1], + "nextLink": f"https://management.azure.com{collection}?api-version={openai.COGNITIVE_API}&$skiptoken=a%2Bb", + }, + ) + self.assertEqual(request.url.params["$skiptoken"], "a+b") + return Response(200, json={"value": [2]}) + + credential = Mock() + credential.get_token.return_value = SimpleNamespace(token="token") + async with AsyncClient(transport=MockTransport(handle)) as client: + self.assertEqual(await openai.list_values(ArmClient(client, credential), collection), [1, 2]) + self.assertEqual(credential.get_token.call_count, 2) + + async def test_cross_host_path_and_version_links_fail_before_next_request(self): + collection = f"/subscriptions/{SUBSCRIPTION}/models" + for link in ( + "https://example.test" + collection + "?api-version=" + openai.COGNITIVE_API, + "https://management.azure.com/another?api-version=" + openai.COGNITIVE_API, + "https://management.azure.com" + collection + "?api-version=wrong", + ): + arm = SimpleNamespace(request=AsyncMock(return_value={"value": [], "nextLink": link})) + with self.subTest(link=link), self.assertRaises(ValueError): + await openai.list_values(arm, collection) + self.assertEqual(arm.request.await_count, 1) + + async def test_repeated_continuation_is_rejected(self): + collection = f"/subscriptions/{SUBSCRIPTION}/models" + arm = SimpleNamespace( + request=AsyncMock( + return_value={ + "value": [], + "nextLink": f"https://management.azure.com{collection}?api-version={openai.COGNITIVE_API}&$skiptoken=same", + } + ) + ) + with self.assertRaises(ValueError): + await openai.list_values(arm, collection) + self.assertEqual(arm.request.await_count, 2) + + +class OpenAILifecycleTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext( + patch.dict(os.environ, {"ARM_SUBSCRIPTION_ID": SUBSCRIPTION, "AZURE_ENVIRONMENT": "AzureCloud"}) + ) + self.enterContext(patch.object(lifecycle.config, "TRE_ID", "tretest")) + self.enterContext(patch.object(lifecycle, "get_workspace_owner_token", AsyncMock(return_value="token"))) + self.preflight = self.enterContext(patch.object(lifecycle, "check_prerequisites", AsyncMock())) + self.post = self.enterContext(patch.object(resource, "post_resource", AsyncMock(return_value=(PATH, SERVICE)))) + self.cleanup = self.enterContext(patch.object(resource, "disable_and_delete_resource", AsyncMock())) + self.lookup_status = 404 + self.workspace = {"workspace": {"properties": {}}} + self.service = { + "workspaceService": { + "id": SERVICE, + "templateName": "tre-workspace-service-openai", + "deploymentStatus": "deployed", + "properties": { + "openai_model": openai.MODEL, + "is_exposed_externally": False, + "openai_fqdn": "https://openai.example.test/", + "openai_deployment_id": DEPLOYMENT, + }, + } + } + self.get = self.enterContext( + patch.object( + lifecycle, + "get_resource", + AsyncMock( + side_effect=lambda endpoint, *_: deepcopy( + self.service if "workspace-services" in endpoint else self.workspace + ) + ), + ) + ) + self.account = { + "properties": { + "provisioningState": "Succeeded", + "publicNetworkAccess": "Disabled", + "endpoint": "https://openai.example.test/", + } + } + self.deployment = { + "properties": { + "provisioningState": "Succeeded", + "model": {"format": "OpenAI", "name": "gpt-5.1", "version": "2025-11-13"}, + }, + "sku": {"name": "Standard", "capacity": 1}, + } + self.retained = False + self.group = GROUP + self.account_id = ACCOUNT + self.deployment_name = DEPLOYMENT + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if identifier == self.group: + return {"location": "westeurope"} + if identifier == self.account_id: + return self.account if not kwargs.get("missing_ok") or self.retained else None + if identifier == self.account_id + "/deployments/" + self.deployment_name: + return self.deployment + raise AssertionError("Unexpected ARM resource") + + self.arm = SimpleNamespace(request=AsyncMock(side_effect=request)) + + @asynccontextmanager + async def arm(): + yield self.arm + + self.enterContext(patch.object(lifecycle, "arm_client", arm)) + + def lookup(request): + self.assertEqual(request.url.path, "/api" + PATH) + return Response(self.lookup_status) + + self.enterContext( + patch.object(lifecycle, "AsyncClient", lambda **_: AsyncClient(transport=MockTransport(lookup))) + ) + self.enterContext( + patch.object(lifecycle, "get_full_endpoint", lambda endpoint: "https://tre.example.test" + endpoint) + ) + + async def run_case(self): + await lifecycle.test_private_openai_lifecycle(True, (f"/workspaces/{WORKSPACE}", WORKSPACE)) + + async def test_success_checks_actual_region_and_removes_only_the_created_service(self): + await self.run_case() + self.preflight.assert_awaited_once_with(self.arm, SUBSCRIPTION, "westeurope") + self.assertTrue(self.post.await_args.kwargs["cleanup_failed_create"]) + payload = self.post.await_args.args[0] + self.assertEqual(payload["properties"]["openai_model"], openai.MODEL) + self.assertIs(payload["properties"]["is_exposed_externally"], False) + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True) + + async def test_hyphenated_tre_id_reaches_validation_and_cleanup(self): + self.group = GROUP.replace("tretest", "tre-test") + self.account_id = ACCOUNT.replace("tretest", "tre-test") + self.deployment_name = DEPLOYMENT.replace("tretest", "tre-test") + self.service["workspaceService"]["properties"]["openai_deployment_id"] = self.deployment_name + with patch.object(lifecycle.config, "TRE_ID", "tre-test"): + await self.run_case() + self.preflight.assert_awaited_once_with(self.arm, SUBSCRIPTION, "westeurope") + self.post.assert_awaited_once() + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True) + + async def test_path_and_query_characters_are_rejected_before_arm_access(self): + for tre_id in ("tre/test", "tre?test", "tre#test", "tre%2ftest"): + with self.subTest(tre_id=tre_id), patch.object(lifecycle.config, "TRE_ID", tre_id): + with self.assertRaisesRegex(ValueError, "Invalid subscription or TRE identifier"): + await self.run_case() + self.arm.request.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_model_or_quota_failure_prevents_service_creation(self): + self.preflight.side_effect = ValueError("quota unavailable") + with self.assertRaisesRegex(ValueError, "quota unavailable"): + await self.run_case() + self.post.assert_not_awaited() + self.cleanup.assert_not_awaited() + + async def test_other_subscription_fails_before_arm_access_or_service_creation(self): + self.workspace["workspace"]["properties"]["workspace_subscription_id"] = "another" + with self.assertRaisesRegex(ValueError, "same subscription"): + await self.run_case() + self.arm.request.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_public_access_failure_still_cleans_up(self): + self.account["properties"]["publicNetworkAccess"] = "Enabled" + with self.assertRaises(AssertionError): + await self.run_case() + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True, allow_failed_disable=True) + + async def test_wrong_model_deployment_still_cleans_up(self): + self.deployment["properties"]["model"]["version"] = "other" + with self.assertRaises(AssertionError): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_normal_cleanup_can_outlast_the_creation_deadline(self): + async def cleanup(*args, **kwargs): + await asyncio.sleep(0.1) + + self.cleanup.side_effect = cleanup + with patch.object(lifecycle, "CREATE_TIMEOUT_SECONDS", 0.05): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_verification_timeout_still_cleans_up(self): + async def block_account(method, identifier, api, **kwargs): + if identifier == GROUP: + return {"location": "westeurope"} + await asyncio.Event().wait() + raise AssertionError("The blocking ARM request unexpectedly resumed") + + self.arm.request.side_effect = block_account + with patch.object(lifecycle, "VERIFY_TIMEOUT_SECONDS", 0.01), self.assertRaises(TimeoutError): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_cleanup_failure_is_not_success(self): + self.cleanup.side_effect = RuntimeError("delete failed") + with self.assertRaisesRegex(RuntimeError, "delete failed"): + await self.run_case() + + async def test_retained_api_record_or_account_is_not_success(self): + for api_status, retained in ((200, False), (404, True)): + self.lookup_status, self.retained = api_status, retained + with self.subTest(api=api_status, retained=retained), self.assertRaises(AssertionError): + await self.run_case() diff --git a/templates/workspace_services/openai/porter.yaml b/templates/workspace_services/openai/porter.yaml index 0fc2de4256..2a598f58fe 100644 --- a/templates/workspace_services/openai/porter.yaml +++ b/templates/workspace_services/openai/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-openai -version: 1.0.11 +version: 1.1.0 description: "An OpenAI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/openai/template_schema.json b/templates/workspace_services/openai/template_schema.json index bc0b640ec3..be77422c97 100644 --- a/templates/workspace_services/openai/template_schema.json +++ b/templates/workspace_services/openai/template_schema.json @@ -37,7 +37,7 @@ "openai_model": { "type": "string", "title": "OpenAI Model", - "description": "Which OpenAI Model should be used? (be mindful of subscription limits)", + "description": "Select a model available with the Standard deployment type in the workspace region. Check subscription quota before deployment.", "enum": [ "gpt-35-turbo | 0301", "gpt-35-turbo | 1106", @@ -45,9 +45,10 @@ "gpt-35-turbo | 0125", "gpt-4 | 0613", "gpt-4 | turbo-2024-04-09", - "gpt-4o | 2024-05-13" + "gpt-4o | 2024-05-13", + "gpt-5.1 | 2025-11-13" ], - "default": "gpt-35-turbo | 0301", + "default": "gpt-5.1 | 2025-11-13", "updateable": true } } diff --git a/templates/workspace_services/openai/terraform/openai.tf b/templates/workspace_services/openai/terraform/openai.tf index 44a0061c9c..34077a81f3 100644 --- a/templates/workspace_services/openai/terraform/openai.tf +++ b/templates/workspace_services/openai/terraform/openai.tf @@ -24,7 +24,8 @@ resource "azurerm_cognitive_deployment" "openai" { } scale { - type = "Standard" + type = "Standard" + capacity = 1 } } From 32448c719cfdd436011a5fb4b4c35ab29e20ac1a Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 21 Oct 2026 19:46:12 +0100 Subject: [PATCH 06/19] Create the OpenAI private endpoint after the model deployment (#5249) The private endpoint and the model deployment depended only on the account, so Terraform created them at the same time. Azure rejected the endpoint with AccountProvisioningStateInvalid while the deployment updated the account. Make the endpoint depend on the deployment, and check this order in the bundle contract test. --- CHANGELOG.md | 1 + e2e_tests/unit_tests/test_openai_validation.py | 6 ++++++ templates/workspace_services/openai/terraform/openai.tf | 3 +++ 3 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3419ca449f..7441aa7962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ BUG FIXES: * Try the storage registration up to five times before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. Terraform registers after an attempt succeeds, or after five failed attempts. (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) +* Create the OpenAI private endpoint after the model deployment, so that Azure does not reject it while the deployment updates the account. (`tre-workspace-service-openai` 1.1.0) ([#5249](https://github.com/microsoft/AzureTRE/issues/5249)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_openai_validation.py b/e2e_tests/unit_tests/test_openai_validation.py index ffc7c4cf2f..619477292c 100644 --- a/e2e_tests/unit_tests/test_openai_validation.py +++ b/e2e_tests/unit_tests/test_openai_validation.py @@ -121,6 +121,12 @@ def test_bundle_schema_and_ci_registration_agree_with_selection(self): {"BUNDLE_TYPE": "workspace_service", "BUNDLE_DIR": "./templates/workspace_services/openai"}, registrations ) + def test_private_endpoint_waits_for_the_model_deployment(self): + source = (ROOT / "templates/workspace_services/openai/terraform/openai.tf").read_text() + endpoint = source.split('resource "azurerm_private_endpoint" "openai_private_endpoint" {', 1)[1] + endpoint = endpoint.split("\nresource ", 1)[0] + self.assertRegex(endpoint, r"depends_on\s*=\s*\[\s*azurerm_cognitive_deployment\.openai\s*\]") + class PaginationTests(unittest.IsolatedAsyncioTestCase): async def test_pagination_retains_api_version_and_renews_credentials(self): diff --git a/templates/workspace_services/openai/terraform/openai.tf b/templates/workspace_services/openai/terraform/openai.tf index 34077a81f3..3926deb93c 100644 --- a/templates/workspace_services/openai/terraform/openai.tf +++ b/templates/workspace_services/openai/terraform/openai.tf @@ -48,6 +48,9 @@ resource "azurerm_private_endpoint" "openai_private_endpoint" { private_dns_zone_ids = [data.azurerm_private_dns_zone.openai.id] } + # Azure rejects the endpoint with AccountProvisioningStateInvalid while a model deployment updates the account. + depends_on = [azurerm_cognitive_deployment.openai] + lifecycle { ignore_changes = [tags] } } From 8b04660ea301bbd8ed078ab7157e7f41e68ed572 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 14:06:21 +0100 Subject: [PATCH 07/19] Disable public network access on the MySQL server (#5148) The MySQL flexible server had public network access enabled. It had no firewall rules, so no public client could connect, but the server kept a public endpoint. A live MySQL run in Switzerland North showed publicNetworkAccess Enabled. The Azure SQL bundle disables public access, and the SQL test checks it. The server now sets public_network_access to Disabled, so that clients can connect only through the private endpoint. azurerm 3.117.0 cannot set this value, so the bundle moves to azurerm 4.27.0, as the Gitea and OHDSI bundles use. In 4.27.0 a change of this value is an in-place update, so an upgrade keeps the server and its data. --- CHANGELOG.md | 1 + .../tre-templates/workspace-services/mysql.md | 4 +++ .../unit_tests/test_mysql_bundle_contract.py | 24 ++++++++++++++++ .../workspace_services/mysql/porter.yaml | 2 +- .../mysql/terraform/.terraform.lock.hcl | 28 +++++++++---------- .../mysql/terraform/main.tf | 2 +- .../mysql/terraform/mysql.tf | 4 ++- 7 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 e2e_tests/unit_tests/test_mysql_bundle_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7441aa7962..bcb93eb3d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ BUG FIXES: (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) * Create the OpenAI private endpoint after the model deployment, so that Azure does not reject it while the deployment updates the account. (`tre-workspace-service-openai` 1.1.0) ([#5249](https://github.com/microsoft/AzureTRE/issues/5249)) +* Disable public network access on the MySQL flexible server, so that clients can connect only through the private endpoint. Move the MySQL bundle to azurerm 4.27.0, which has this setting. An upgrade changes the setting in place. (`tre-workspace-service-mysql` 1.0.16) ([#5148](https://github.com/microsoft/AzureTRE/issues/5148)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/docs/tre-templates/workspace-services/mysql.md b/docs/tre-templates/workspace-services/mysql.md index c7174cf292..0c979d9a21 100644 --- a/docs/tre-templates/workspace-services/mysql.md +++ b/docs/tre-templates/workspace-services/mysql.md @@ -10,6 +10,10 @@ See: [MySQL Azure](https://learn.microsoft.com/en-GB/azure/mysql/) `make workspace_service_bundle BUNDLE=mysql` +## Network access + +The MySQL server has public network access disabled. Clients connect through its private endpoint, for example from a VM in the workspace. + ## Authenticating to MySQL 1. Navigate to the MySQL workspace service using the `Mysql fqdn` from the details tab. diff --git a/e2e_tests/unit_tests/test_mysql_bundle_contract.py b/e2e_tests/unit_tests/test_mysql_bundle_contract.py new file mode 100644 index 0000000000..73b88897ec --- /dev/null +++ b/e2e_tests/unit_tests/test_mysql_bundle_contract.py @@ -0,0 +1,24 @@ +"""Check that the MySQL server accepts private endpoint connections only.""" + +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/workspace_services/mysql/terraform" + + +class MySQLBundleContractTests(unittest.TestCase): + def test_server_disables_public_network_access(self): + source = (BUNDLE / "mysql.tf").read_text() + server = source.split('resource "azurerm_mysql_flexible_server" "mysql" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(server, r'\n public_network_access\s*=\s*"Disabled"\n') + + def test_provider_supports_the_public_network_access_argument(self): + # azurerm added public_network_access for MySQL flexible servers in the 4.x releases. + main = (BUNDLE / "main.tf").read_text() + version = re.search(r'source\s*=\s*"hashicorp/azurerm"\s*\n\s*version\s*=\s*"=?\s*(\d+)\.', main) + self.assertIsNotNone(version) + self.assertGreaterEqual(int(version.group(1)), 4) + lock = (BUNDLE / ".terraform.lock.hcl").read_text() + self.assertRegex(lock, r'provider "registry\.terraform\.io/hashicorp/azurerm" \{\n version = "4\.') diff --git a/templates/workspace_services/mysql/porter.yaml b/templates/workspace_services/mysql/porter.yaml index cc39df9638..fab5984d88 100644 --- a/templates/workspace_services/mysql/porter.yaml +++ b/templates/workspace_services/mysql/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-mysql -version: 1.0.15 +version: 1.0.16 description: "A MySQL workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/mysql/terraform/.terraform.lock.hcl b/templates/workspace_services/mysql/terraform/.terraform.lock.hcl index 904ea2b772..86a0086e86 100644 --- a/templates/workspace_services/mysql/terraform/.terraform.lock.hcl +++ b/templates/workspace_services/mysql/terraform/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.117.0" - constraints = "3.117.0" + version = "4.27.0" + constraints = "4.27.0" hashes = [ - "h1:Ynfg+Iy7x6K8M6W1AhqXCe3wkoiqIQhROlca7C3KC3w=", - "zh:2e25f47492366821a786762369f0e0921cc9452d64bfd5075f6fdfcf1a9c6d70", - "zh:41eb34f2f7469bf3eb1019dfb0e7fc28256f809824016f4f8b9d691bf473b2ac", - "zh:48bb9c87b3d928da1abc1d3db75453c9725de4674c612daf3800160cc7145d30", - "zh:5d6b0de0bbd78943fcc65c53944ef4496329e247f434c6eab86ed051c5cea67b", - "zh:78c9f6fdb1206a89cf0e6706b4f46178169a93b6c964a4cad8a321058ccbd9b4", - "zh:793b702c352589d4360b580d4a1cf654a7439d2ad6bdb7bfea91de07bc4b0fac", - "zh:7ed687ff0a5509463a592f97431863574fe5cc80a34e395be06766215b8c6285", - "zh:955ba18789bd15592824eb426a8d0f38595bd09fffc6939c1c58933489c1a71e", - "zh:bf5949a55be0714cd9c8815d472eae4baa48ba06d0f6bf2b96775869acda8a54", - "zh:da5d31f635abd2c645ffc76d6176d73f646128e73720cc368247cc424975c127", - "zh:eed5a66d59883c9c56729b0a964a2b60d758ea7489ef3e920a6fbd48518ce5f5", + "h1:2fs47aLDaEm93ANXXVRdTjlbUBmFBZRsFjyshKoPE3o=", + "zh:0c69edea1995bd3bd9e61980757169c35bf22281b660b5c755b6cb13d08d29d2", + "zh:25b86bf7b9678371d8573983954c571696f3e64a3967133be3b835da36307106", + "zh:49921cff4f26a49bafada60cd07dabb52c5eb35231059ed928a4f4722e269c82", + "zh:4b986166531f9fd1289f01d8220519443e74888a21da512c1b841b006dad6215", + "zh:53fb65b2ca4df637f03e4748a100a7d7fc77249e307c03e294d6259cec0310f6", + "zh:5c0d021a387ca4e2a5a01da009746a08c45f08e971c10d9bda54539d7264d671", + "zh:600043f2b20dc5a45275e43f175c19fe8b6e8e9557a0c884aef018f1f63de90e", + "zh:a0284f6f38912f67bb4cb7829fda3fa75be81fea6a9b21119965c2a839430092", + "zh:a7ac0576e2069ef77557042c6b5157ded364fbd355b2f9bf7f5441622424086e", + "zh:c5db0bcafe986868e28cc6225b68b2d1cf4bf631939d260ca845f17a9aa1677d", + "zh:ce620c0eb71b1fdd925828b30cf232a869abccf1c459180f2f991c4166315251", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } diff --git a/templates/workspace_services/mysql/terraform/main.tf b/templates/workspace_services/mysql/terraform/main.tf index d5761a8686..bc80c2d3ee 100644 --- a/templates/workspace_services/mysql/terraform/main.tf +++ b/templates/workspace_services/mysql/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.117.0" + version = "= 4.27.0" } random = { source = "hashicorp/random" diff --git a/templates/workspace_services/mysql/terraform/mysql.tf b/templates/workspace_services/mysql/terraform/mysql.tf index 0b878e6541..091c0e1e8e 100644 --- a/templates/workspace_services/mysql/terraform/mysql.tf +++ b/templates/workspace_services/mysql/terraform/mysql.tf @@ -16,7 +16,9 @@ resource "azurerm_mysql_flexible_server" "mysql" { version = "8.0.21" backup_retention_days = 7 geo_redundant_backup_enabled = false - tags = local.workspace_service_tags + # Clients connect through the private endpoint only. + public_network_access = "Disabled" + tags = local.workspace_service_tags storage { size_gb = local.max_storage_gb From c786c7a290c09434d988ab54983c3700c5283eaf Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 14:06:28 +0100 Subject: [PATCH 08/19] Add a private MySQL lifecycle check (#5148) The MySQL bundle selection ran only the generic install test. It did not show that a workspace client can use the database. The new case creates a workspace, a Guacamole service, a Linux VM and the MySQL service. The Linux VM installs the MySQL client through Nexus, reads the administrator password from the workspace Key Vault through a temporary role assignment on that secret, and writes a row through private DNS with TLS. The case then changes the SKU from Burstable to General Purpose and reads the row again. The SQL lifecycle helper now accepts its deadline variable, enters Nexus as a prerequisite that leaves after the VM, and can get a new token for clean-up, because this case can outlive the create token. --- .github/workflows/deploy_tre_reusable.yml | 5 + CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 10 + docs/tre-developers/github-pr-bot-commands.md | 2 +- e2e_tests/bundle_coverage.json | 15 +- e2e_tests/pytest.ini | 1 + e2e_tests/resources/mysql_probe.py | 86 ++++++++ e2e_tests/resources/mysql_probe.sh | 72 +++++++ e2e_tests/resources/sql_lifecycle.py | 20 +- e2e_tests/test_mysql.py | 109 ++++++++++ e2e_tests/unit_tests/test_mysql_probe.py | 193 ++++++++++++++++++ 11 files changed, 504 insertions(+), 10 deletions(-) create mode 100644 e2e_tests/resources/mysql_probe.py create mode 100644 e2e_tests/resources/mysql_probe.sh create mode 100644 e2e_tests/test_mysql.py create mode 100644 e2e_tests/unit_tests/test_mysql_probe.py diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 859ab83f71..ffc65f879a 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -1073,6 +1073,10 @@ jobs: if: inputs.e2eBundle == 'tre-workspace-service-azuresql' run: echo "SQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve MySQL cleanup and reporting time + if: inputs.e2eBundle == 'tre-workspace-service-mysql' + run: echo "MYSQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -1100,6 +1104,7 @@ jobs: COMMAND: >- TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=${{ inputs.runCertificateTestsOnWeekends }} SQL_VALIDATION_DEADLINE=${{ env.SQL_VALIDATION_DEADLINE }} + MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index bcb93eb3d0..7f71d52999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Setting to `false` will delete existing airlock storage accounts and must only b ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) +* Add a private MySQL lifecycle check. A Nexus-backed Linux VM writes a row through private DNS, the SKU changes from Burstable to General Purpose, and the VM reads the row again. ([#5148](https://github.com/microsoft/AzureTRE/issues/5148)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index b0894e3853..b645840a9a 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -168,6 +168,16 @@ Failed-create recovery and temporary probe cleanup also count towards the lifecy The final 30 minutes are reserved for results and job overhead. Cleanup failures still fail the test. For a local run, the 270-minute lifecycle budget starts when the SQL case begins. +### MySQL selection + +Select `tre-workspace-service-mysql` to run `test_mysql.py::test_private_mysql_query_survives_sku_change`. +The case creates its own workspace, Guacamole service, Linux VM and MySQL service. +The Linux VM installs the MySQL client through Nexus. Thus the run needs a suitable Nexus instance or `acceptNexusEula=true`. +The case writes a row through private DNS with the Burstable SKU. It then changes the SKU to General Purpose and reads the row again. +The VM reads the administrator password from the workspace Key Vault through a temporary role assignment on that secret only. +The MySQL job uses the SQL time limits above with its own `MYSQL_VALIDATION_DEADLINE`. If the case creates Nexus, it removes Nexus after the VM. +These checks do not prove previous-version upgrades or researcher access. + ### Run the branch workflow before merge 1. Push the reviewed changes to a branch in `microsoft/AzureTRE`. diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index d6513b3ae3..9722956fef 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -22,7 +22,7 @@ The table describes the selected assertions. A passing command does not prove ev | `/test-airlock` | Import requests, access checks, the import review VM flow and export request storage routing | No export review VM or migration proof. | | `/test-backups` | Base workspace deployment with backups enabled and disabled | No backup restore or retention proof. | -The `workspace_services`, `linux_vm`, `airlock_validation` and `sql_validation` markers have no dedicated slash command. +The `workspace_services`, `linux_vm`, `airlock_validation`, `sql_validation` and `mysql_validation` markers have no dedicated slash command. Use the branch workflow or the local test runner for these selections. The `airlock_validation` marker selects two Airlock cases. It does not select the full `airlock` suite. See [bundle selection](end-to-end-tests.md#validate-one-bundle) for exact cases and declared coverage gaps. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 2e61202415..efd816964a 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -376,18 +376,27 @@ "manifest": "templates/workspace_services/mysql/porter.yaml", "family": "workspace_service", "prerequisites": [ - "tre-workspace-base" + "tre-workspace-base", + "tre-service-guacamole", + "tre-service-guacamole-linuxvm", + "tre-shared-service-certs", + "tre-shared-service-sonatype-nexus" ], "tests": [ + "test_mysql.py::test_private_mysql_query_survives_sku_change" + ], + "related_tests": [ "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-mysql]" ], - "related_tests": [], "coverage": [ "install", + "private MySQL write/read", + "Burstable-to-General Purpose SKU change with retained data", "uninstall" ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "AzureCloud only. Requires a suitable Nexus instance or explicit Nexus EULA consent, permission to run commands on a test-created workspace VM and a temporary secret-scoped role assignment.", + "The SKU change uses the current bundle version. Previous-version migration remains separate (#5128). The MySQL bundle has no custom actions." ], "unavailable_reason": "" }, diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 8ce1ae4975..6613a900a9 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -17,6 +17,7 @@ markers = workspace_validation: opt-in base-workspace creation, scoped API access and deletion openai_validation: opt-in private OpenAI deployment with model and quota preflight sql_validation: Private SQL query and configuration upgrade lifecycle + mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM asyncio_mode = auto asyncio_default_fixture_loop_scope = session diff --git a/e2e_tests/resources/mysql_probe.py b/e2e_tests/resources/mysql_probe.py new file mode 100644 index 0000000000..df86b7ee5c --- /dev/null +++ b/e2e_tests/resources/mysql_probe.py @@ -0,0 +1,86 @@ +"""Run a private MySQL probe from a test-created workspace Linux VM.""" + +import asyncio +import json +from pathlib import Path +import re +from uuid import uuid4 + +from e2e_tests.resources.sql_probe import COMPUTE_API, delete_after + +SCRIPT = Path(__file__).with_name("mysql_probe.sh") +RESULT = "MYSQL_PROBE_RESULT=" +FAILED = "MYSQL_PROBE_FAILED=" +PATTERNS = { + "SERVER": r"[a-z0-9-]+\.mysql\.database\.azure\.com", + "VAULT_NAME": r"[a-z0-9-]+", + "SECRET_NAME": r"[a-z0-9-]+", + "DATABASE": r"[a-z0-9_]+", + "PROBE_ID": r"[a-f0-9]{32}", + "PHASE": r"write|read", +} + + +def probe_script(**values): + """Set the validated values in the script text. They contain no secrets.""" + if set(values) != set(PATTERNS): + raise ValueError("Unexpected MySQL probe values") + for name, value in values.items(): + if not re.fullmatch(PATTERNS[name], value): + raise ValueError(f"Invalid MySQL probe value for {name}") + header = "".join(f"{name}='{value}'\n" for name, value in values.items()) + return "#!/usr/bin/env bash\n" + header + SCRIPT.read_text() + + +def probe_result(resource, phase, probe_id): + properties = resource.get("properties", {}) + view = properties.get("instanceView", {}) + state = view.get("executionState", "") + lines = view.get("output", "").splitlines() + if properties.get("provisioningState") in ("Failed", "Canceled") or state in ("Failed", "Canceled", "TimedOut"): + stages = [line[len(FAILED) :] for line in lines if line.startswith(FAILED)] + raise AssertionError( + f"MySQL probe failed: state={state}, exitCode={view.get('exitCode')}, stage={','.join(stages) or 'unknown'}" + ) + if state != "Succeeded": + return False + if view.get("exitCode") != 0: + raise AssertionError("MySQL probe returned a non-zero exit code") + results = [line[len(RESULT) :] for line in lines if line.startswith(RESULT)] + expected = {"phase": phase, "probe": probe_id, "row_count": 1, "private_dns": True} + if len(results) != 1 or json.loads(results[0]) != expected: + raise AssertionError("MySQL probe did not confirm the expected data and private DNS") + return True + + +async def run_probe(arm, vm_id, location, *, server, vault_name, secret_name, database, probe_id, phase): + script = probe_script( + SERVER=server, + VAULT_NAME=vault_name, + SECRET_NAME=secret_name, + DATABASE=database, + PROBE_ID=probe_id, + PHASE=phase, + ) + command = f"{vm_id}/runCommands/mysql-probe-{uuid4().hex}" + # Removing our command also terminates it if its deadline expired. + async with delete_after(arm, command, COMPUTE_API): + async with asyncio.timeout(20 * 60): + await arm.request( + "PUT", + command, + COMPUTE_API, + body={ + "location": location, + "properties": { + "source": {"script": script}, + "timeoutInSeconds": 900, + "treatFailureAsDeploymentFailure": True, + }, + }, + ) + while True: + result = await arm.request("GET", command, COMPUTE_API, expand=True) + if probe_result(result, phase, probe_id): + return + await asyncio.sleep(10) diff --git a/e2e_tests/resources/mysql_probe.sh b/e2e_tests/resources/mysql_probe.sh new file mode 100644 index 0000000000..a09a6fb8cb --- /dev/null +++ b/e2e_tests/resources/mysql_probe.sh @@ -0,0 +1,72 @@ +# The test validates SERVER, VAULT_NAME, SECRET_NAME, DATABASE, PROBE_ID and PHASE and sets them above this line. +set -euo pipefail +stage=input +fail() { + # Do not print command output, tokens or the password. + echo "MYSQL_PROBE_FAILED=$stage" + exit 1 +} +[[ "$SERVER" =~ ^[a-z0-9-]+\.mysql\.database\.azure\.com$ ]] || fail +[[ "$VAULT_NAME" =~ ^[a-z0-9-]+$ ]] || fail +[[ "$SECRET_NAME" =~ ^[a-z0-9-]+$ ]] || fail +[[ "$DATABASE" =~ ^[a-z0-9_]+$ ]] || fail +[[ "$PROBE_ID" =~ ^[a-f0-9]{32}$ ]] || fail +[[ "$PHASE" =~ ^(write|read)$ ]] || fail + +stage=private-dns +addresses=$(getent ahostsv4 "$SERVER" | awk '{print $1}' | sort -u) || fail +[ -n "$addresses" ] || fail +for address in $addresses; do + case "$address" in + 10.* | 192.168.* | 172.1[6-9].* | 172.2[0-9].* | 172.3[01].*) ;; + *) fail ;; + esac +done + +stage=mysql-client +# The VM bootstrap points apt at Nexus. Wait for it before the install. +cloud-init status --wait >/dev/null 2>&1 || true +attempt=0 +until command -v mysql >/dev/null; do + attempt=$((attempt + 1)) + [ "$attempt" -le 3 ] || fail + export DEBIAN_FRONTEND=noninteractive + if ! { apt-get -o DPkg::Lock::Timeout=300 update -qq && apt-get -o DPkg::Lock::Timeout=300 install -y -qq --no-install-recommends mysql-client-core-8.0; } >/dev/null 2>&1; then + sleep 30 + fi +done + +stage=secret-access +deadline=$(($(date +%s) + 300)) +password="" +until [ -n "$password" ]; do + token=$(curl -sf --max-time 15 -H Metadata:true "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net" | + python3 -c 'import json, sys; print(json.load(sys.stdin)["access_token"])' 2>/dev/null) || token="" + if [ -n "$token" ]; then + password=$(curl -sf --max-time 15 -H "Authorization: Bearer $token" "https://$VAULT_NAME.vault.azure.net/secrets/$SECRET_NAME?api-version=7.4" | + python3 -c 'import json, sys; print(json.load(sys.stdin)["value"])' 2>/dev/null) || password="" + fi + if [ -z "$password" ]; then + [ "$(date +%s)" -lt "$deadline" ] || fail + sleep 10 + fi +done +token="" + +query() { + MYSQL_PWD="$password" mysql --host="$SERVER" --user=mysqladmin --database="$DATABASE" \ + --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/certs/ca-certificates.crt \ + --connect-timeout=30 --batch --skip-column-names --execute="$1" 2>/dev/null +} + +stage=mysql-connect +query "SELECT 1" >/dev/null || fail + +stage=mysql-data +if [ "$PHASE" = write ]; then + query "CREATE TABLE tre_e2e_probe (probe CHAR(32) NOT NULL PRIMARY KEY); INSERT INTO tre_e2e_probe (probe) VALUES ('$PROBE_ID');" >/dev/null || fail +fi +count=$(query "SELECT COUNT(*) FROM tre_e2e_probe WHERE probe = '$PROBE_ID'") || fail +password="" +[ "$count" = 1 ] || fail +echo "MYSQL_PROBE_RESULT={\"phase\": \"$PHASE\", \"probe\": \"$PROBE_ID\", \"row_count\": $count, \"private_dns\": true}" diff --git a/e2e_tests/resources/sql_lifecycle.py b/e2e_tests/resources/sql_lifecycle.py index b574849fca..466e81dab2 100644 --- a/e2e_tests/resources/sql_lifecycle.py +++ b/e2e_tests/resources/sql_lifecycle.py @@ -37,10 +37,10 @@ } -def lifecycle_deadlines(): +def lifecycle_deadlines(variable="SQL_VALIDATION_DEADLINE"): """Use the job's absolute deadline, including time spent starting the container.""" loop = asyncio.get_running_loop() - configured = os.environ.get("SQL_VALIDATION_DEADLINE", "") + configured = os.environ.get(variable, "") deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS if not math.isfinite(deadline): raise ValueError("SQL validation deadline must be finite") @@ -60,6 +60,11 @@ def __init__(self, verify, finish): def own(self, name, callback, *args, **kwargs): self.finalisers.append((name, partial(callback, *args, **kwargs))) + async def enter(self, name, context): + """Enter a prerequisite context now and exit it in order with the owned resources.""" + await context.__aenter__() + self.own(name, context.__aexit__, None, None, None) + async def setup(self): # Keep these resources inside the SQL budget rather than session fixtures. workspace_path, workspace_id = await create_or_get_test_workspace( @@ -79,15 +84,18 @@ async def setup(self): self.own(guacamole_path, clean_up_test_workspace_service, "", guacamole_path, workspace_id, self.verify) return workspace_path, workspace_id, guacamole_path, guacamole_id - async def create(self, payload, endpoint, token): + async def create(self, payload, endpoint, token, cleanup_token=None): async with asyncio.timeout(60 * 60): resource_path, resource_id = await post_resource( payload, endpoint, token, self.verify, cleanup_failed_create=True ) - self.own(resource_path, self.delete, resource_path, token) + self.own(resource_path, self.delete, resource_path, cleanup_token or token) return resource_path, resource_id async def delete(self, resource_path, token): + # A long case can outlive the create token, so cleanup can ask for a new one. + if callable(token): + token = await token() while True: operations = (await get_resource(f"/api{resource_path}/operations", token, self.verify))["operations"] if all(op["status"] in TERMINAL_STATES for op in operations): @@ -113,8 +121,8 @@ async def close(self): @asynccontextmanager -async def sql_lifecycle(verify): - work_finish, finish = lifecycle_deadlines() +async def sql_lifecycle(verify, deadline_variable="SQL_VALIDATION_DEADLINE"): + work_finish, finish = lifecycle_deadlines(deadline_variable) resources = SQLResources(verify, finish) original = None # Accepted-create recovery inherits the same absolute limit. It cannot add diff --git a/e2e_tests/test_mysql.py b/e2e_tests/test_mysql.py new file mode 100644 index 0000000000..948960a190 --- /dev/null +++ b/e2e_tests/test_mysql.py @@ -0,0 +1,109 @@ +"""Exercise MySQL data access and a supported SKU change privately.""" + +import asyncio +from functools import partial +import logging +import os +from uuid import uuid4 + +import pytest + +from e2e_tests import config +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.resources import strings +from e2e_tests.resources.mysql_probe import run_probe +from e2e_tests.resources.nexus import nexus_prerequisites +from e2e_tests.resources.resource import get_resource, post_resource +from e2e_tests.resources.sql_lifecycle import sql_lifecycle +from e2e_tests.resources.sql_probe import arm_client, COMPUTE_API, secret_access, validate_vm + +LOGGER = logging.getLogger(__name__) +MYSQL_API = "2023-12-30" +DATABASE = "tredb" +MYSQL_PAYLOAD = { + "templateName": strings.MYSQL_SERVICE, + "properties": { + "display_name": "MySQL lifecycle test", + "description": "Private MySQL query and SKU change validation", + "sql_sku": "B | 4GB 2vCores", + "db_name": DATABASE, + }, +} +VM_PAYLOAD = { + "templateName": strings.GUACAMOLE_LINUX_USER_RESOURCE, + "properties": { + "display_name": "MySQL test client", + "description": "Temporary private MySQL validation client", + "os_image": "Ubuntu 22.04 LTS", + "vm_size": "2 CPU | 8GB RAM", + # CI uses a service principal, so the bundle cannot derive a username from its owner. + "admin_username": "researcher", + }, +} +SKUS = {"B | 4GB 2vCores": "Standard_B2s", "GP | 8GB 2vCores": "Standard_D2ds_v4"} + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.mysql_validation +@pytest.mark.timeout(285 * 60) +async def test_private_mysql_query_survives_sku_change(verify): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("The MySQL probe currently supports AzureCloud only") + async with sql_lifecycle(verify, deadline_variable="MYSQL_VALIDATION_DEADLINE") as resources: + workspace_path, workspace_id, guacamole_path, guacamole_id = await resources.setup() + # The Linux VM installs its packages through Nexus. + async with asyncio.timeout(60 * 60): + await resources.enter("Nexus prerequisites", nexus_prerequisites(verify)) + new_token = partial(get_workspace_owner_token, workspace_id, verify) + token = await new_token() + mysql_path, mysql_id = await resources.create( + MYSQL_PAYLOAD, f"/api{workspace_path}/workspace-services", token, cleanup_token=new_token + ) + token = await new_token() + vm_path, vm_id = await resources.create( + VM_PAYLOAD, f"/api{guacamole_path}/user-resources", token, cleanup_token=new_token + ) + mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] + vm_properties = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"] + subscription_id = vm_properties.get("workspace_subscription_id") or os.environ["ARM_SUBSCRIPTION_ID"] + server = f"mysql-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{mysql_id[-4:]}" + assert mysql["properties"]["mysql_fqdn"] == f"{server}.mysql.database.azure.com" + vault = f"kv-{f'{config.TRE_ID}-ws-{workspace_id[-4:]}'[-20:]}".lower() + azure_vm_id = vm_properties["azure_resource_id"] + async with arm_client() as arm: + vm = await arm.request("GET", azure_vm_id, COMPUTE_API) + group = validate_vm(vm, azure_vm_id, subscription_id, config.TRE_ID, workspace_id, guacamole_id, vm_id) + server_id = f"{group}/providers/Microsoft.DBforMySQL/flexibleServers/{server}" + server_record = await arm.request("GET", server_id, MYSQL_API) + assert server_record["sku"]["name"] == SKUS["B | 4GB 2vCores"] + assert server_record["properties"]["network"]["publicNetworkAccess"] == "Disabled" + secure = await arm.request("GET", f"{server_id}/configurations/require_secure_transport", MYSQL_API) + assert secure["properties"]["value"].upper() == "ON" + vault_id = f"{group}/providers/Microsoft.KeyVault/vaults/{vault}" + secret_name = f"{server}-administrator-password" + probe_id = uuid4().hex + async with secret_access(arm, vault_id, secret_name, vm["identity"]["principalId"], subscription_id): + probe = dict( + server=f"{server}.mysql.database.azure.com", + vault_name=vault, + secret_name=secret_name, + database=DATABASE, + probe_id=probe_id, + ) + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="write") + LOGGER.info("MySQL query passed through private DNS on the Burstable SKU") + token = await new_token() + mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] + async with asyncio.timeout(60 * 60): + await post_resource( + {"properties": {"sql_sku": "GP | 8GB 2vCores"}}, + f"/api{mysql_path}", + token, + verify, + method="PATCH", + etag=mysql["_etag"], + ) + server_record = await arm.request("GET", server_id, MYSQL_API) + assert server_record["sku"]["name"] == SKUS["GP | 8GB 2vCores"] + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="read") + LOGGER.info("MySQL data survived the change to the General Purpose SKU") diff --git a/e2e_tests/unit_tests/test_mysql_probe.py b/e2e_tests/unit_tests/test_mysql_probe.py new file mode 100644 index 0000000000..429f163a9a --- /dev/null +++ b/e2e_tests/unit_tests/test_mysql_probe.py @@ -0,0 +1,193 @@ +"""Check the MySQL probe's evidence, input handling and cleanup boundaries.""" + +import json +import os +from pathlib import Path +import re +import subprocess +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from jsonschema import validate +import yaml + +from api_app.services.schema_service import enrich_user_resource_template, enrich_workspace_service_template +from e2e_tests import test_mysql as mysql +from e2e_tests.resources import mysql_probe as probe +from e2e_tests.resources import sql_lifecycle as lifecycle + + +ROOT = Path(__file__).resolve().parents[2] +VM_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.Compute/virtualMachines/linuxvm1234" +PROBE_ID = "0123456789abcdef0123456789abcdef" +VALUES = { + "SERVER": "mysql-tretest-ws-abcd-svc-1234.mysql.database.azure.com", + "VAULT_NAME": "kv-tretest-ws-abcd", + "SECRET_NAME": "mysql-tretest-ws-abcd-svc-1234-administrator-password", + "DATABASE": "tredb", + "PROBE_ID": PROBE_ID, + "PHASE": "read", +} + + +def result(output=None, state="Succeeded", code=0, **overrides): + evidence = {"phase": "read", "probe": PROBE_ID, "row_count": 1, "private_dns": True, **overrides} + return { + "properties": { + "provisioningState": "Succeeded", + "instanceView": { + "executionState": state, + "exitCode": code, + "output": output if output is not None else "MYSQL_PROBE_RESULT=" + json.dumps(evidence), + }, + } + } + + +class ProbeScriptTests(unittest.TestCase): + def test_script_sets_only_validated_values(self): + script = probe.probe_script(**VALUES) + self.assertTrue(script.startswith("#!/usr/bin/env bash\n")) + for name, value in VALUES.items(): + self.assertIn(f"{name}='{value}'\n", script) + self.assertNotIn("password=", script.split("set -euo pipefail")[0]) + + def test_unexpected_or_unsafe_values_are_rejected(self): + for name, value in ( + ("SERVER", "evil.example.com"), + ("SERVER", "a'; reboot; '.mysql.database.azure.com"), + ("DATABASE", "tredb; DROP DATABASE mysql"), + ("PROBE_ID", "abc"), + ("PHASE", "delete"), + ): + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + probe.probe_script(**{**VALUES, name: value}) + with self.assertRaises(ValueError): + probe.probe_script(**{k: v for k, v in VALUES.items() if k != "PHASE"}) + + def test_script_is_valid_bash(self): + completed = subprocess.run( + ["bash", "-n"], input=probe.probe_script(**VALUES), text=True, capture_output=True, check=False + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_script_rejects_values_that_skip_python_validation(self): + script = probe.probe_script(**VALUES).replace("PHASE='read'", "PHASE='drop'") + completed = subprocess.run(["bash", "-c", script], text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 1) + self.assertEqual(completed.stdout.strip(), "MYSQL_PROBE_FAILED=input") + + +class ProbeEvidenceTests(unittest.TestCase): + def test_success_requires_expected_data_and_private_dns(self): + self.assertTrue(probe.probe_result(result(), "read", PROBE_ID)) + for change in ({"phase": "write"}, {"probe": "other"}, {"row_count": 0}, {"private_dns": False}): + with self.subTest(change=change), self.assertRaises(AssertionError): + probe.probe_result(result(**change), "read", PROBE_ID) + + def test_running_command_is_not_a_pass(self): + resource = result() + resource["properties"]["instanceView"] = {} + self.assertFalse(probe.probe_result(resource, "read", PROBE_ID)) + + def test_failure_reports_only_the_failed_stage(self): + failed = result(output="MYSQL_PROBE_FAILED=mysql-connect\n", state="Failed", code=1) + with self.assertRaisesRegex(AssertionError, "stage=mysql-connect"): + probe.probe_result(failed, "read", PROBE_ID) + with self.assertRaisesRegex(AssertionError, "non-zero"): + probe.probe_result(result(code=1), "read", PROBE_ID) + + def test_payloads_match_enriched_api_schemas(self): + cases = ( + (mysql.MYSQL_PAYLOAD, "mysql", enrich_workspace_service_template), + (mysql.VM_PAYLOAD, "guacamole/user_resources/guacamole-azure-linuxvm", enrich_user_resource_template), + ) + for payload, folder, enrich in cases: + with self.subTest(bundle=payload["templateName"]): + schema = json.loads( + (ROOT / "templates/workspace_services" / folder / "template_schema.json").read_text() + ) + validate(payload["properties"], enrich(SimpleNamespace(model_dump=lambda **_: schema))) + + def test_expected_skus_match_the_bundle(self): + # Terraform uses a tier prefix, for example B_Standard_B2s. ARM returns the SKU name without it. + locals_tf = (ROOT / "templates/workspace_services/mysql/terraform/locals.tf").read_text() + for option, sku in mysql.SKUS.items(): + with self.subTest(option=option): + self.assertRegex(locals_tf, rf'"{re.escape(option)}"\s*=\s*{{ value = "[A-Z]+_{sku}" }}') + + def test_deadline_precedes_container(self): + jobs = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text())["jobs"] + job = jobs["e2e_tests_bundle"] + reserve = next(s for s in job["steps"] if "MYSQL_VALIDATION_DEADLINE" in s.get("run", "")) + run = next(s for s in job["steps"] if s["name"] == "Run E2E Tests") + self.assertLess(job["steps"].index(reserve), job["steps"].index(run)) + self.assertEqual(reserve["if"], "inputs.e2eBundle == 'tre-workspace-service-mysql'") + self.assertIn("MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }}", run["with"]["COMMAND"]) + + +class ProbeRecoveryTests(unittest.IsolatedAsyncioTestCase): + async def test_failed_command_is_removed_and_never_counts_as_pass(self): + failed = result(output="MYSQL_PROBE_FAILED=secret-access\n", state="Failed", code=1) + arm = SimpleNamespace(request=AsyncMock(side_effect=[{}, failed]), delete=AsyncMock()) + with self.assertRaisesRegex(AssertionError, "secret-access"): + await probe.run_probe( + arm, + VM_ID, + "switzerlandnorth", + server=VALUES["SERVER"], + vault_name=VALUES["VAULT_NAME"], + secret_name=VALUES["SECRET_NAME"], + database="tredb", + probe_id=PROBE_ID, + phase="read", + ) + command = arm.request.call_args_list[0].args[1] + self.assertTrue(command.startswith(VM_ID + "/runCommands/mysql-probe-")) + arm.delete.assert_awaited_once_with(command, probe.COMPUTE_API) + body = arm.request.call_args_list[0].kwargs["body"] + self.assertNotIn("parameters", body["properties"]) + self.assertNotIn("protectedParameters", body["properties"]) + + async def test_prerequisite_context_exits_after_later_resources(self): + events = [] + + class Context: + async def __aenter__(self): + events.append("enter") + + async def __aexit__(self, *args): + events.append(("exit", args)) + + async def delete(*_, **__): + events.append("delete") + + with ( + patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE=""), + patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/mysql", "id"))), + patch.object(lifecycle, "get_resource", AsyncMock(return_value={"operations": []})), + patch.object(lifecycle, "disable_and_delete_resource", side_effect=delete), + ): + async with lifecycle.sql_lifecycle(True, deadline_variable="MYSQL_VALIDATION_DEADLINE") as owned: + await owned.enter("Nexus", Context()) + await owned.create({}, "/services", "token") + self.assertEqual(events, ["enter", "delete", ("exit", (None, None, None))]) + + async def test_cleanup_can_request_a_new_token(self): + tokens = AsyncMock(return_value="new-token") + delete = AsyncMock() + with ( + patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE=""), + patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/mysql", "id"))), + patch.object(lifecycle, "get_resource", AsyncMock(return_value={"operations": []})), + patch.object(lifecycle, "disable_and_delete_resource", delete), + ): + async with lifecycle.sql_lifecycle(True, deadline_variable="MYSQL_VALIDATION_DEADLINE") as owned: + await owned.create({}, "/services", "old-token", cleanup_token=tokens) + tokens.assert_awaited_once() + self.assertEqual(delete.await_args.args[1], "new-token") + + async def test_mysql_deadline_variable_is_read(self): + with patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE="nan"), self.assertRaisesRegex(ValueError, "finite"): + lifecycle.lifecycle_deadlines("MYSQL_VALIDATION_DEADLINE") From a3f3b993fdc1b3a36efcc1b89667f448b7884d56 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 20:02:49 +0100 Subject: [PATCH 09/19] Give the MySQL probe script a shebang (#5148) The Lint job of the MySQL pull request failed on e2e_tests/resources/mysql_probe.sh. Shellcheck reported SC2148, because the file had no shebang, and bash-exec reported that the file was not executable. The probe code added the shebang only to the generated script. The file now starts with the shebang and is executable. The probe code puts the validated values after the first line, so the generated script does not change. --- e2e_tests/resources/mysql_probe.py | 3 ++- e2e_tests/resources/mysql_probe.sh | 1 + e2e_tests/unit_tests/test_mysql_probe.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 e2e_tests/resources/mysql_probe.sh diff --git a/e2e_tests/resources/mysql_probe.py b/e2e_tests/resources/mysql_probe.py index df86b7ee5c..64247da849 100644 --- a/e2e_tests/resources/mysql_probe.py +++ b/e2e_tests/resources/mysql_probe.py @@ -29,7 +29,8 @@ def probe_script(**values): if not re.fullmatch(PATTERNS[name], value): raise ValueError(f"Invalid MySQL probe value for {name}") header = "".join(f"{name}='{value}'\n" for name, value in values.items()) - return "#!/usr/bin/env bash\n" + header + SCRIPT.read_text() + shebang, body = SCRIPT.read_text().split("\n", 1) + return f"{shebang}\n{header}{body}" def probe_result(resource, phase, probe_id): diff --git a/e2e_tests/resources/mysql_probe.sh b/e2e_tests/resources/mysql_probe.sh old mode 100644 new mode 100755 index a09a6fb8cb..c02a5ff581 --- a/e2e_tests/resources/mysql_probe.sh +++ b/e2e_tests/resources/mysql_probe.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # The test validates SERVER, VAULT_NAME, SECRET_NAME, DATABASE, PROBE_ID and PHASE and sets them above this line. set -euo pipefail stage=input diff --git a/e2e_tests/unit_tests/test_mysql_probe.py b/e2e_tests/unit_tests/test_mysql_probe.py index 429f163a9a..1d65b130c5 100644 --- a/e2e_tests/unit_tests/test_mysql_probe.py +++ b/e2e_tests/unit_tests/test_mysql_probe.py @@ -49,6 +49,7 @@ class ProbeScriptTests(unittest.TestCase): def test_script_sets_only_validated_values(self): script = probe.probe_script(**VALUES) self.assertTrue(script.startswith("#!/usr/bin/env bash\n")) + self.assertEqual(script.count("#!"), 1) for name, value in VALUES.items(): self.assertIn(f"{name}='{value}'\n", script) self.assertNotIn("password=", script.split("set -euo pipefail")[0]) From 503733be08f03b0cc5a9fe5773b5ba7550c959e4 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 14:42:43 +0100 Subject: [PATCH 10/19] Run the Linux VM actions in the Guacamole Linux VM check (#5145) The Linux VM check proved the install, the Nexus-backed bootstrap and the delete. It did not run the stop, start and reset_password actions. After the bootstrap, the check now runs stop and start and reads the VM power state from Azure after each action. It then runs reset_password and makes sure that the Key Vault secret with the VM credentials has a new version. The check reads only the secret metadata through Azure Resource Manager, not the secret value. The check now requests the workspace owner token after Nexus, because the VM delete uses it and Nexus can take a long time. --- CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 8 ++ e2e_tests/bundle_coverage.json | 5 +- e2e_tests/resources/vm_actions.py | 55 ++++++++++++++ e2e_tests/test_guacamole_linuxvm.py | 33 +++++++-- .../test_guacamole_linuxvm_cleanup.py | 10 +++ e2e_tests/unit_tests/test_vm_actions.py | 74 +++++++++++++++++++ 7 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 e2e_tests/resources/vm_actions.py create mode 100644 e2e_tests/unit_tests/test_vm_actions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f71d52999..5e263cf715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Setting to `false` will delete existing airlock storage accounts and must only b ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) * Add a private MySQL lifecycle check. A Nexus-backed Linux VM writes a row through private DNS, the SKU changes from Burstable to General Purpose, and the VM reads the row again. ([#5148](https://github.com/microsoft/AzureTRE/issues/5148)) +* Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index b645840a9a..e4f727a0c6 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -143,6 +143,14 @@ Select `e2eBundle=tre-shared-service-certs` in `deploy_tre_branch.yml` and leave PR slash commands use workflow definitions from `main`. `/test` runs smoke tests and `/test-shared-services` excludes this certificate-only case. A test-only PR without the new certificate bundle cannot validate reuse. Run Nexus and Airlock only after this case and its cleanup pass. +### Linux VM selection + +Select `tre-service-guacamole-linuxvm` to run `test_guacamole_linuxvm.py::test_create_guacamole_linux_vm`. +The case creates a Linux VM and waits for its Nexus-backed bootstrap. Thus the run needs a suitable Nexus instance or `acceptNexusEula=true`. +The case then runs the `stop`, `start` and `reset_password` actions. +After `stop` and `start`, the case reads the VM power state from Azure. After `reset_password`, it makes sure that the Key Vault secret has a new version. +The case does not sign in to the VM, so it does not prove interactive access or the new password. + ### Base workspace lifecycle Select `tre-workspace-base` to run `test_workspace_base.py::test_base_workspace_lifecycle` independently. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index efd816964a..53d209e75f 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -325,10 +325,13 @@ "coverage": [ "install", "bootstrap-completion", + "stop action with deallocated power state", + "start action with running power state", + "reset_password action with a new Key Vault secret version", "uninstall" ], "limitations": [ - "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites. VM availability and interactive access need separate validation." + "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites. Interactive access, sign-in with the new password and other VM sizes need separate validation." ], "unavailable_reason": "" }, diff --git a/e2e_tests/resources/vm_actions.py b/e2e_tests/resources/vm_actions.py new file mode 100644 index 0000000000..43ae2ec99d --- /dev/null +++ b/e2e_tests/resources/vm_actions.py @@ -0,0 +1,55 @@ +"""Invoke user resource actions and read the VM state that they change.""" + +from httpx import AsyncClient +from starlette import status + +from e2e_tests.bundle_evidence import record_resource +from e2e_tests.helpers import assert_status, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.deployment import check_deployment +from e2e_tests.resources.resource import TIMEOUT, wait_for +from e2e_tests.resources.sql_probe import COMPUTE_API + +KEY_VAULT_API = "2023-07-01" +ACTION_TERMINAL = (strings.RESOURCE_ACTION_STATUS_SUCCEEDED, strings.RESOURCE_ACTION_STATUS_FAILED) + + +async def action_done(client, operation_endpoint, access_token): + action_status, message, steps = await check_deployment(client, operation_endpoint, access_token) + return action_status in ACTION_TERMINAL, action_status, message, steps + + +async def invoke_action(resource_path, action, access_token, verify): + """Run one custom action and fail unless it succeeds.""" + async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: + response = await client.post( + get_full_endpoint(f"/api{resource_path}/invoke-action"), + headers=get_auth_header(access_token), + params={"action": action}, + ) + assert_status(response, [status.HTTP_202_ACCEPTED], f"The {action} action could not be sent") + operation = response.json()["operation"] + if operation["resourcePath"] != resource_path: + raise ValueError("The action returned an operation for a different resource") + record_resource(None, operation, "POST") + await wait_for( + action_done, + client, + response.headers["Location"], + access_token, + [strings.RESOURCE_ACTION_STATUS_FAILED], + ) + + +async def power_state(arm, vm_id): + view = await arm.request("GET", f"{vm_id}/instanceView", COMPUTE_API) + states = [s["code"] for s in view.get("statuses", []) if s.get("code", "").startswith("PowerState/")] + if len(states) != 1: + raise AssertionError("The VM reported no single power state") + return states[0] + + +async def secret_version(arm, vault_id, secret_name): + """Read the current secret version through ARM. The response contains no secret value.""" + secret = await arm.request("GET", f"{vault_id}/secrets/{secret_name}", KEY_VAULT_API) + return secret["properties"]["secretUriWithVersion"].rsplit("/", 1)[1] diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index 1cc4ada88f..2f6d2dcff1 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -1,11 +1,14 @@ -"""Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap.""" +"""Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap and runs its actions.""" import pytest +from e2e_tests import config from e2e_tests.conftest import get_workspace_owner_token from e2e_tests.resources import strings -from e2e_tests.resources.resource import temporary_resource +from e2e_tests.resources.resource import get_resource, temporary_resource from e2e_tests.resources.nexus import nexus_prerequisites +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.resources.vm_actions import invoke_action, power_state, secret_version pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -15,7 +18,6 @@ @pytest.mark.timeout(150 * 60) async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_service, verify): _, workspace_id, workspace_service_path, _ = setup_test_workspace_and_guacamole_service - workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) payload = { "templateName": strings.GUACAMOLE_LINUX_USER_RESOURCE, "properties": { @@ -29,7 +31,28 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv } async with nexus_prerequisites(verify): + # Nexus can take a long time. The VM delete uses this token, so request it after Nexus. + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) async with temporary_resource( payload, f"/api{workspace_service_path}/{strings.API_USER_RESOURCES}", workspace_owner_token, verify - ): - pass + ) as vm_path: + await check_vm_actions(vm_path, workspace_id, verify) + + +async def check_vm_actions(vm_path, workspace_id, verify): + """Run the VM actions and read their effect from Azure.""" + token = await get_workspace_owner_token(workspace_id, verify) + vm_id = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"]["azure_resource_id"] + group = vm_id.split("/providers/", 1)[0] + vault = f"kv-{f'{config.TRE_ID}-ws-{workspace_id[-4:]}'[-20:]}".lower() + vault_id = f"{group}/providers/Microsoft.KeyVault/vaults/{vault}" + secret_name = f"{vm_id.rsplit('/', 1)[1]}-admin-credentials" + async with arm_client() as arm: + assert await power_state(arm, vm_id) == "PowerState/running" + await invoke_action(vm_path, "stop", await get_workspace_owner_token(workspace_id, verify), verify) + assert await power_state(arm, vm_id) == "PowerState/deallocated" + await invoke_action(vm_path, "start", await get_workspace_owner_token(workspace_id, verify), verify) + assert await power_state(arm, vm_id) == "PowerState/running" + version = await secret_version(arm, vault_id, secret_name) + await invoke_action(vm_path, "reset_password", await get_workspace_owner_token(workspace_id, verify), verify) + assert await secret_version(arm, vault_id, secret_name) != version, "reset_password kept the secret" diff --git a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py index e615f7a047..3e1ee11ca4 100644 --- a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py +++ b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py @@ -18,6 +18,7 @@ def setUp(self): self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) self.enterContext(patch.object(linuxvm, "get_workspace_owner_token", new=AsyncMock(return_value="token"))) self.enterContext(patch.object(nexus, "get_admin_token", new=AsyncMock(return_value="admin-token"))) + self.actions = self.enterContext(patch.object(linuxvm, "check_vm_actions", new=AsyncMock())) self.fixture = ("/workspaces/ws", "ws", "/workspaces/ws/workspace-services/service", "service") self.resource_path = self.fixture[2] + "/user-resources/linux" self.requests = [] @@ -112,6 +113,14 @@ async def test_success_uses_explicit_username_and_cleans_up_vm(self): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) payload = json.loads(self.requests[0].content) self.assertEqual(payload["properties"]["admin_username"], "researcher") + self.actions.assert_awaited_once_with(self.resource_path, "ws", True) + self.assert_deleted() + + async def test_failed_action_still_cleans_up_vm(self): + self.mock_api(strings.RESOURCE_STATUS_DEPLOYED) + self.actions.side_effect = AssertionError("stop failed") + with self.assertRaisesRegex(AssertionError, "stop failed"): + await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assert_deleted() async def test_failed_bootstrap_cleans_up_after_failed_disable(self): @@ -121,6 +130,7 @@ async def test_failed_bootstrap_cleans_up_after_failed_disable(self): with self.assertRaises(AssertionError): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assert_deleted(failed_disable=True, failed_create=True) + self.actions.assert_not_awaited() async def test_failed_disable_does_not_delete_an_enabled_resource(self): self.mock_api( diff --git a/e2e_tests/unit_tests/test_vm_actions.py b/e2e_tests/unit_tests/test_vm_actions.py new file mode 100644 index 0000000000..22d413555f --- /dev/null +++ b/e2e_tests/unit_tests/test_vm_actions.py @@ -0,0 +1,74 @@ +"""Check the user resource action helper and the VM state readers.""" + +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import Response + +from e2e_tests.resources import strings +from e2e_tests.resources import vm_actions as actions + +VM_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.Compute/virtualMachines/linuxvm1234" +VAULT_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.KeyVault/vaults/kv-tretest-ws-abcd" +PATH = "/workspaces/ws/workspace-services/svc/user-resources/vm" + + +class ActionTests(unittest.IsolatedAsyncioTestCase): + async def test_action_success_and_failure_are_terminal(self): + for state, done in ( + (strings.RESOURCE_ACTION_STATUS_SUCCEEDED, True), + (strings.RESOURCE_ACTION_STATUS_FAILED, True), + (strings.RESOURCE_ACTION_STATUS_INVOKING, False), + (strings.RESOURCE_STATUS_DEPLOYED, False), + ): + with ( + self.subTest(state=state), + patch.object(actions, "check_deployment", AsyncMock(return_value=(state, "", []))), + ): + self.assertEqual((await actions.action_done(None, "/op", "token"))[:2], (done, state)) + + async def test_invoke_waits_for_the_action_and_rejects_another_resource(self): + def response(path): + return Response( + 202, + json={"operation": {"id": "op", "resourcePath": path, "resourceId": "vm"}}, + headers={"Location": "/api/op"}, + ) + + for path, error in ((PATH, None), ("/workspaces/ws/other", ValueError)): + post = AsyncMock(return_value=response(path)) + client = SimpleNamespace(post=post) + client_context = AsyncMock() + client_context.__aenter__.return_value = client + with ( + self.subTest(path=path), + patch.object(actions, "AsyncClient", return_value=client_context), + patch.object(actions, "record_resource"), + patch.object(actions, "wait_for", AsyncMock()) as wait, + ): + if error: + with self.assertRaises(error): + await actions.invoke_action(PATH, "stop", "token", True) + wait.assert_not_awaited() + else: + await actions.invoke_action(PATH, "stop", "token", True) + self.assertEqual(post.await_args.kwargs["params"], {"action": "stop"}) + self.assertTrue(post.await_args.args[0].endswith(f"/api{PATH}/invoke-action")) + self.assertEqual(wait.await_args.args[2], "/api/op") + self.assertEqual(wait.await_args.args[4], [strings.RESOURCE_ACTION_STATUS_FAILED]) + + async def test_power_state_needs_exactly_one_value(self): + view = {"statuses": [{"code": "ProvisioningState/succeeded"}, {"code": "PowerState/deallocated"}]} + arm = SimpleNamespace(request=AsyncMock(return_value=view)) + self.assertEqual(await actions.power_state(arm, VM_ID), "PowerState/deallocated") + self.assertEqual(arm.request.await_args.args[1], VM_ID + "/instanceView") + arm.request = AsyncMock(return_value={"statuses": [{"code": "ProvisioningState/succeeded"}]}) + with self.assertRaises(AssertionError): + await actions.power_state(arm, VM_ID) + + async def test_secret_version_comes_from_arm_metadata(self): + secret = {"properties": {"secretUriWithVersion": "https://kv.vault.azure.net/secrets/name/abc123"}} + arm = SimpleNamespace(request=AsyncMock(return_value=secret)) + self.assertEqual(await actions.secret_version(arm, VAULT_ID, "linuxvm1234-admin-credentials"), "abc123") + self.assertEqual(arm.request.await_args.args[1], VAULT_ID + "/secrets/linuxvm1234-admin-credentials") From f0f9045acb67bafbc1e1c9f3e77b998903e2bf22 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 19:34:27 +0000 Subject: [PATCH 11/19] Retry the backup registration reads and the unregister of the warm-up (#5135) The warm-up step read any error of a registration read as "not registered". On an upgrade, one throttled read could start a PUT and a DELETE on the registration that Terraform owns. The step now treats only a not-found answer as "not registered". It reads two more times after another error, and then it fails. The step sent the unregister one time. Azure can answer it with a short CloudInternalError, and the step then failed and left a registration that Terraform cannot create. The step now sends the unregister again each minute while the registration exists. The registration URL has no host, so az rest uses the Resource Manager endpoint of the current cloud. --- .../unit_tests/test_backup_registration.py | 48 ++++++++++++++++--- .../workspaces/base/terraform/storage.tf | 41 ++++++++++++++-- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index e35a17d1ad..64ee85dddf 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -10,9 +10,7 @@ ROOT = Path(__file__).resolve().parents[2] STORAGE = ROOT / "templates/workspaces/base/terraform/storage.tf" -CONTAINER = ( - "https://management.azure.com/vault/backupFabrics/Azure/protectionContainers/container?api-version=2023-04-01" -) +CONTAINER = "/vault/backupFabrics/Azure/protectionContainers/container?api-version=2023-04-01" OPERATION = "https://management.azure.com/vault/operations/1" FAKE_AZ = """import json,os,sys @@ -26,11 +24,18 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) outcome=state['outcomes'].pop(0) if len(state['outcomes'])>1 else state['outcomes'][0] print(outcome) state['registered']=outcome=='Succeeded';save();sys.exit(0) -if method=='get':sys.exit(0 if state['registered'] else 1) +if method=='get': + if state['get_errors']: + state['get_errors']-=1;save();print('ERROR: (TooManyRequests) The request was throttled.',file=sys.stderr);sys.exit(1) + if state['registered']:sys.exit(0) + print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) if method=='put': if state['header']:print("INFO: cli.azure.cli.core.util: 'Azure-AsyncOperation': '"+os.environ['OPERATION']+"'",file=sys.stderr) sys.exit(0) -if method=='delete':state['registered']=False;save();sys.exit(0) +if method=='delete': + if state['delete_failures']: + state['delete_failures']-=1;save();print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) + state['registered']=False;save();sys.exit(0) """ @@ -55,9 +60,19 @@ def setUp(self): path.write_text("#!" + sys.executable + "\n" + body) path.chmod(0o755) - def run_warm_up(self, *, registered=False, outcomes=("Failed",), header=True): + def run_warm_up(self, *, registered=False, outcomes=("Failed",), header=True, get_errors=0, delete_failures=0): state = self.folder / "state.json" - state.write_text(json.dumps({"registered": registered, "outcomes": list(outcomes), "header": header})) + state.write_text( + json.dumps( + { + "registered": registered, + "outcomes": list(outcomes), + "header": header, + "get_errors": get_errors, + "delete_failures": delete_failures, + } + ) + ) env = os.environ | { "PATH": str(self.folder) + os.pathsep + os.environ["PATH"], "CALLS": str(self.folder / "calls.jsonl"), @@ -96,6 +111,25 @@ def test_successful_first_attempt_is_removed_for_terraform(self): self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) self.assertFalse(state["registered"]) + def test_failed_unregister_is_sent_again(self): + # Azure can answer the unregister with a short CloudInternalError. + result, methods, state = self.run_warm_up(outcomes=["Succeeded"], delete_failures=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "put", "get", "delete"] + ["get"] * 6 + ["delete", "get"]) + self.assertFalse(state["registered"]) + + def test_read_error_does_not_change_an_existing_registration(self): + result, methods, state = self.run_warm_up(registered=True, get_errors=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "get"]) + self.assertTrue(state["registered"]) + + def test_lasting_read_error_stops_the_step(self): + result, methods, state = self.run_warm_up(get_errors=3) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(methods, ["get"] * 3) + self.assertIn("Could not read the backup registration", result.stderr) + def test_missing_operation_header_waits_before_each_check(self): result, methods, _ = self.run_warm_up(header=False) self.assertEqual(result.returncode, 0, result.stderr) diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index c8746b5b01..988c2aeefd 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -173,7 +173,7 @@ resource "terraform_data" "backup_registration_warm_up" { count = var.enable_backup ? 1 : 0 input = { - container_url = "https://management.azure.com${module.backup[0].vault_id}/backupFabrics/Azure/protectionContainers/StorageContainer;storage;${azurerm_resource_group.ws.name};${azurerm_storage_account.stg.name}?api-version=2023-04-01" + container_url = "${module.backup[0].vault_id}/backupFabrics/Azure/protectionContainers/StorageContainer;storage;${azurerm_resource_group.ws.name};${azurerm_storage_account.stg.name}?api-version=2023-04-01" storage_account_id = azurerm_storage_account.stg.id subscription_id = data.azurerm_client_config.current.subscription_id } @@ -184,8 +184,32 @@ resource "terraform_data" "backup_registration_warm_up" { set -euo pipefail az login --identity --output none az account set --subscription "${self.input.subscription_id}" +# A URL without a host uses the Resource Manager endpoint of the current cloud. url="${self.input.container_url}" -if az rest --method get --url "$url" --output none 2>/dev/null; then + +# Print "registered" or "missing". Only a not-found answer means "missing". +# Other errors, for example throttling, get two more reads. Then the step fails. +container_state() { + local output + for read in 1 2 3; do + if output=$(az rest --method get --url "$url" --output none 2>&1); then + echo registered + return 0 + fi + if grep -qiE 'NotFound|Not Found|was not found|does not exist' <<< "$output"; then + echo missing + return 0 + fi + if [ "$read" -lt 3 ]; then + sleep 10 + fi + done + echo "Could not read the backup registration of the storage account: $output" >&2 + return 1 +} + +state=$(container_state) +if [ "$state" = registered ]; then echo "The storage account is already registered with the vault" exit 0 fi @@ -206,12 +230,19 @@ for try in 1 2 3 4 5; do sleep 120 fi echo "Registration attempt $try ended with status $status" - if az rest --method get --url "$url" --output none 2>/dev/null; then + state=$(container_state) + if [ "$state" = registered ]; then # Remove the registration, so that Terraform creates and owns the registration. - az rest --method delete --url "$url" --output none + # Azure can answer the unregister with a short CloudInternalError, so send it again each minute. for attempt in $(seq 1 60); do - az rest --method get --url "$url" --output none 2>/dev/null || exit 0 + if [ $((attempt % 6)) -eq 1 ]; then + az rest --method delete --url "$url" --output none 2>/dev/null || true + fi sleep 10 + state=$(container_state) + if [ "$state" = missing ]; then + exit 0 + fi done echo "The storage account is still registered with the vault after 10 minutes" >&2 exit 1 From 73e5b3e6ff1ed78091931225c6eef708a928ab4a Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 19:40:14 +0000 Subject: [PATCH 12/19] Use one workspace owner token for the MySQL and Linux VM cleanup (#5148) The MySQL test asked for a new owner token before each step and gave the cleanup a function that asked for another one. The stated reason was that a long case can outlive the token. But the owner token is a TokenProvider, and get_auth_header gets a current token from it for each request. So one token stays valid for the whole case and its cleanup. The cleanup token parameter, its unit test and the comment in the Linux VM test are removed. --- e2e_tests/resources/sql_lifecycle.py | 7 ++----- e2e_tests/test_guacamole_linuxvm.py | 1 - e2e_tests/test_mysql.py | 14 +++----------- e2e_tests/unit_tests/test_mysql_probe.py | 14 -------------- 4 files changed, 5 insertions(+), 31 deletions(-) diff --git a/e2e_tests/resources/sql_lifecycle.py b/e2e_tests/resources/sql_lifecycle.py index 466e81dab2..8b19ae9986 100644 --- a/e2e_tests/resources/sql_lifecycle.py +++ b/e2e_tests/resources/sql_lifecycle.py @@ -84,18 +84,15 @@ async def setup(self): self.own(guacamole_path, clean_up_test_workspace_service, "", guacamole_path, workspace_id, self.verify) return workspace_path, workspace_id, guacamole_path, guacamole_id - async def create(self, payload, endpoint, token, cleanup_token=None): + async def create(self, payload, endpoint, token): async with asyncio.timeout(60 * 60): resource_path, resource_id = await post_resource( payload, endpoint, token, self.verify, cleanup_failed_create=True ) - self.own(resource_path, self.delete, resource_path, cleanup_token or token) + self.own(resource_path, self.delete, resource_path, token) return resource_path, resource_id async def delete(self, resource_path, token): - # A long case can outlive the create token, so cleanup can ask for a new one. - if callable(token): - token = await token() while True: operations = (await get_resource(f"/api{resource_path}/operations", token, self.verify))["operations"] if all(op["status"] in TERMINAL_STATES for op in operations): diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index 2f6d2dcff1..0470b15733 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -31,7 +31,6 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv } async with nexus_prerequisites(verify): - # Nexus can take a long time. The VM delete uses this token, so request it after Nexus. workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) async with temporary_resource( payload, f"/api{workspace_service_path}/{strings.API_USER_RESOURCES}", workspace_owner_token, verify diff --git a/e2e_tests/test_mysql.py b/e2e_tests/test_mysql.py index 948960a190..72b7b8551b 100644 --- a/e2e_tests/test_mysql.py +++ b/e2e_tests/test_mysql.py @@ -1,7 +1,6 @@ """Exercise MySQL data access and a supported SKU change privately.""" import asyncio -from functools import partial import logging import os from uuid import uuid4 @@ -54,15 +53,9 @@ async def test_private_mysql_query_survives_sku_change(verify): # The Linux VM installs its packages through Nexus. async with asyncio.timeout(60 * 60): await resources.enter("Nexus prerequisites", nexus_prerequisites(verify)) - new_token = partial(get_workspace_owner_token, workspace_id, verify) - token = await new_token() - mysql_path, mysql_id = await resources.create( - MYSQL_PAYLOAD, f"/api{workspace_path}/workspace-services", token, cleanup_token=new_token - ) - token = await new_token() - vm_path, vm_id = await resources.create( - VM_PAYLOAD, f"/api{guacamole_path}/user-resources", token, cleanup_token=new_token - ) + token = await get_workspace_owner_token(workspace_id, verify) + mysql_path, mysql_id = await resources.create(MYSQL_PAYLOAD, f"/api{workspace_path}/workspace-services", token) + vm_path, vm_id = await resources.create(VM_PAYLOAD, f"/api{guacamole_path}/user-resources", token) mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] vm_properties = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"] subscription_id = vm_properties.get("workspace_subscription_id") or os.environ["ARM_SUBSCRIPTION_ID"] @@ -92,7 +85,6 @@ async def test_private_mysql_query_survives_sku_change(verify): ) await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="write") LOGGER.info("MySQL query passed through private DNS on the Burstable SKU") - token = await new_token() mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] async with asyncio.timeout(60 * 60): await post_resource( diff --git a/e2e_tests/unit_tests/test_mysql_probe.py b/e2e_tests/unit_tests/test_mysql_probe.py index 1d65b130c5..b555ef6d99 100644 --- a/e2e_tests/unit_tests/test_mysql_probe.py +++ b/e2e_tests/unit_tests/test_mysql_probe.py @@ -175,20 +175,6 @@ async def delete(*_, **__): await owned.create({}, "/services", "token") self.assertEqual(events, ["enter", "delete", ("exit", (None, None, None))]) - async def test_cleanup_can_request_a_new_token(self): - tokens = AsyncMock(return_value="new-token") - delete = AsyncMock() - with ( - patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE=""), - patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/mysql", "id"))), - patch.object(lifecycle, "get_resource", AsyncMock(return_value={"operations": []})), - patch.object(lifecycle, "disable_and_delete_resource", delete), - ): - async with lifecycle.sql_lifecycle(True, deadline_variable="MYSQL_VALIDATION_DEADLINE") as owned: - await owned.create({}, "/services", "old-token", cleanup_token=tokens) - tokens.assert_awaited_once() - self.assertEqual(delete.await_args.args[1], "new-token") - async def test_mysql_deadline_variable_is_read(self): with patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE="nan"), self.assertRaisesRegex(ValueError, "finite"): lifecycle.lifecycle_deadlines("MYSQL_VALIDATION_DEADLINE") From 48fe038be4967fefed846e597cb88be4e877082d Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 19:42:04 +0000 Subject: [PATCH 13/19] Stop pinning the OpenAI bundle version in the unit tests (#5150) Two unit tests compared the OpenAI bundle version with "1.1.0". The E2E helper tests run on each change to a porter.yaml file, so the next version bump of the bundle would fail them. The schema test now accepts 1.1.0 or later, because 1.1.0 is the first version with the new default model. The selection test compares the report with the version in porter.yaml. The coverage text says "1.1.0 or later". --- e2e_tests/bundle_coverage.json | 2 +- e2e_tests/unit_tests/test_bundle_selection.py | 5 ++++- e2e_tests/unit_tests/test_openai_validation.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 53d209e75f..927319df11 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -437,7 +437,7 @@ "active-account-removal" ], "limitations": [ - "Requires AzureCloud, a registered 1.1.0 template, regional Standard model availability and subscription quota read access. Preflight does not reserve Azure capacity.", + "Requires AzureCloud, a registered 1.1.0 or later template, regional Standard model availability and subscription quota read access. Preflight does not reserve Azure capacity.", "Uses the workspace subscription and actual resource-group region. A workspace in a different subscription from ARM_SUBSCRIPTION_ID is unsupported by this bundle.", "Does not prove private inference, researcher access, model upgrades, private endpoint connectivity, secret removal or soft-deleted account purge." ], diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index 5fc60fc50e..2adf50baa1 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -3,6 +3,7 @@ import json import os from pathlib import Path +import re import subprocess import sys import tempfile @@ -216,7 +217,9 @@ def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): self.assertEqual(report["collected_tests"], ["test_openai.py::test_private_openai_lifecycle"]) self.assertEqual(report["status"], "not_run") self.assertFalse(report["full_lifecycle_proven"]) - self.assertEqual(report["source_bundle_version"], "1.1.0") + manifest = (ROOT / "templates/workspace_services/openai/porter.yaml").read_text() + version = re.search(r"^version: (\S+)$", manifest, re.MULTILINE).group(1) + self.assertEqual(report["source_bundle_version"], version) self.assertIn("tre-workspace-base", report["declared_prerequisites"]) def test_marker_and_parallel_worker_inputs_are_rejected(self): diff --git a/e2e_tests/unit_tests/test_openai_validation.py b/e2e_tests/unit_tests/test_openai_validation.py index 619477292c..0f53f96472 100644 --- a/e2e_tests/unit_tests/test_openai_validation.py +++ b/e2e_tests/unit_tests/test_openai_validation.py @@ -114,7 +114,8 @@ def test_bundle_schema_and_ci_registration_agree_with_selection(self): validate({"openai_model": openai.MODEL, "is_exposed_externally": False}, schema) self.assertEqual(schema["properties"]["openai_model"]["default"], openai.MODEL) manifest = yaml.safe_load((folder / "porter.yaml").read_text()) - self.assertEqual(manifest["version"], "1.1.0") + # 1.1.0 is the first version with this default model. Later versions keep it. + self.assertGreaterEqual(tuple(int(part) for part in manifest["version"].split(".")), (1, 1, 0)) workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) registrations = workflow["jobs"]["register_bundles"]["strategy"]["matrix"]["include"] self.assertIn( From 26e0c8bb3ebe5a66b9cd6a0f5b109de4844f4d46 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 20:08:08 +0000 Subject: [PATCH 14/19] Use the bundle cloud and login mode in the backup warm-up and wait for a final status (#5135) The warm-up logged in with the managed identity of the default cloud. It now selects the cloud from AZURE_ENVIRONMENT and logs in with the service principal when ARM_USE_MSI is false, as the resource processor does. The defaults keep the managed identity in AzureCloud. A failed read of the registration status ended the wait, and a status that stayed InProgress let the next attempt start. Another PUT could then overlap a running registration. The step now reads the status again after a failed read, and it fails if no final status comes within 10 minutes. --- .../unit_tests/test_backup_registration.py | 77 +++++++++++++++++-- .../workspaces/base/terraform/storage.tf | 20 ++++- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 64ee85dddf..28a1ff8203 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -22,6 +22,7 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) method=args[args.index('--method')+1];url=args[args.index('--url')+1] if url==os.environ['OPERATION']: outcome=state['outcomes'].pop(0) if len(state['outcomes'])>1 else state['outcomes'][0] + if outcome=='ERROR':save();sys.exit(1) print(outcome) state['registered']=outcome=='Succeeded';save();sys.exit(0) if method=='get': @@ -44,7 +45,9 @@ def warm_up_command(): block = source.split('resource "terraform_data" "backup_registration_warm_up" {', 1)[1] command = block.split("<&1 | sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/p" | head -n 1 || true) status=Unknown if [ -n "$operation" ]; then + # A failed status read gives Unknown, and the step reads again. Another PUT must not overlap a running attempt. for attempt in $(seq 1 60); do status=$(az rest --method get --url "$operation" --query status --output tsv 2>/dev/null || echo Unknown) - [ "$status" = "InProgress" ] || break + case "$status" in + Succeeded | Failed | Canceled | Cancelled) break ;; + esac sleep 10 done + case "$status" in + Succeeded | Failed | Canceled | Cancelled) ;; + *) + echo "Registration attempt $try did not end within 10 minutes. The last status was $status." >&2 + exit 1 + ;; + esac else sleep 120 fi From 96661f44e0ac35c46634c3233a7a85b9e8ea76ce Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 20:08:09 +0000 Subject: [PATCH 15/19] Check the MySQL settings after the SKU change and keep the VM actions outside AzureCloud (#5148, #5145) The MySQL case checked public network access and secure transport only before the SKU change. It now checks both again after the change, so the case fails if the update turns either setting back. The Linux VM checks read the power state and the secret version through a Resource Manager client that supports AzureCloud only. In other clouds, the case now runs the three actions without these reads. --- e2e_tests/test_guacamole_linuxvm.py | 10 ++++++++++ e2e_tests/test_mysql.py | 4 ++++ .../test_guacamole_linuxvm_cleanup.py | 18 +++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index 0470b15733..99141ddfb5 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -1,5 +1,8 @@ """Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap and runs its actions.""" +import logging +import os + import pytest from e2e_tests import config @@ -12,6 +15,7 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") +LOGGER = logging.getLogger(__name__) @pytest.mark.linux_vm @@ -40,6 +44,12 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv async def check_vm_actions(vm_path, workspace_id, verify): """Run the VM actions and read their effect from Azure.""" + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + # The Resource Manager client of these checks supports AzureCloud only. Run the actions without the reads. + LOGGER.warning("Not reading the VM power state and secret version outside AzureCloud") + for action in ("stop", "start", "reset_password"): + await invoke_action(vm_path, action, await get_workspace_owner_token(workspace_id, verify), verify) + return token = await get_workspace_owner_token(workspace_id, verify) vm_id = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"]["azure_resource_id"] group = vm_id.split("/providers/", 1)[0] diff --git a/e2e_tests/test_mysql.py b/e2e_tests/test_mysql.py index 72b7b8551b..901fd5b1cc 100644 --- a/e2e_tests/test_mysql.py +++ b/e2e_tests/test_mysql.py @@ -97,5 +97,9 @@ async def test_private_mysql_query_survives_sku_change(verify): ) server_record = await arm.request("GET", server_id, MYSQL_API) assert server_record["sku"]["name"] == SKUS["GP | 8GB 2vCores"] + # The SKU change must keep the network and TLS settings. + assert server_record["properties"]["network"]["publicNetworkAccess"] == "Disabled" + secure = await arm.request("GET", f"{server_id}/configurations/require_secure_transport", MYSQL_API) + assert secure["properties"]["value"].upper() == "ON" await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="read") LOGGER.info("MySQL data survived the change to the General Purpose SKU") diff --git a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py index 3e1ee11ca4..7f99369cd7 100644 --- a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py +++ b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py @@ -1,9 +1,10 @@ """Exercise Linux VM cleanup through the real polling and deletion helpers.""" import json +import os import unittest from pathlib import Path -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch from httpx import AsyncClient, MockTransport, Response from jsonschema import validate @@ -187,5 +188,20 @@ async def test_successful_bootstrap_still_requires_successful_disable(self): self.assertEqual([request.method for request in self.requests], ["POST", "GET", "PATCH", "GET"]) +class LinuxVmCloudTests(unittest.IsolatedAsyncioTestCase): + async def test_other_clouds_run_the_actions_without_azure_reads(self): + invoke = AsyncMock() + arm = Mock(side_effect=AssertionError("The public-cloud client must not run outside AzureCloud")) + with ( + patch.dict(os.environ, {"AZURE_ENVIRONMENT": "AzureUSGovernment"}), + patch.object(linuxvm, "invoke_action", invoke), + patch.object(linuxvm, "arm_client", arm), + patch.object(linuxvm, "get_workspace_owner_token", new=AsyncMock(return_value="token")), + ): + await linuxvm.check_vm_actions("/workspaces/ws/vm", "ws", True) + self.assertEqual([call.args[1] for call in invoke.await_args_list], ["stop", "start", "reset_password"]) + arm.assert_not_called() + + if __name__ == "__main__": unittest.main() From c8b8aae3ef357ea46300fb0e06aa260aec713ae1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 21:01:31 +0000 Subject: [PATCH 16/19] Stop the backup warm-up when an accepted registration cannot be followed (#5135) If Azure accepted the registration PUT but the step found no operation header, the step waited 120 seconds and went on. A missing registration does not prove that the operation ended, so a later PUT or the Terraform registration could overlap it. The step now keeps the result of the PUT and reads the operation header in any case. After a rejected PUT, it waits and tries again, as before. After an accepted PUT without an operation, it goes on only if the registration exists, and it fails otherwise. --- .../unit_tests/test_backup_registration.py | 34 +++++++++++++++++-- .../workspaces/base/terraform/storage.tf | 17 +++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 28a1ff8203..84d265b811 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -31,7 +31,10 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) if state['registered']:sys.exit(0) print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) if method=='put': - if state['header']:print("INFO: cli.azure.cli.core.util: 'Azure-AsyncOperation': '"+os.environ['OPERATION']+"'",file=sys.stderr) + if state['put_failures']: + state['put_failures']-=1;save();print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) + if state['put_registers']:state['registered']=True;save() + if state['header']:print("INFO: cli.azure.cli.core.util: '"+state['header_name']+"': '"+os.environ['OPERATION']+"'",file=sys.stderr) sys.exit(0) if method=='delete': if state['delete_failures']: @@ -72,6 +75,9 @@ def run_warm_up( get_errors=0, delete_failures=0, environment=None, + put_failures=0, + put_registers=False, + header_name="Azure-AsyncOperation", ): state = self.folder / "state.json" state.write_text( @@ -82,6 +88,9 @@ def run_warm_up( "header": header, "get_errors": get_errors, "delete_failures": delete_failures, + "put_failures": put_failures, + "put_registers": put_registers, + "header_name": header_name, } ) ) @@ -191,11 +200,30 @@ def test_login_uses_the_service_principal_without_a_managed_identity(self): ], ) - def test_missing_operation_header_waits_before_each_check(self): + def test_accepted_put_without_an_operation_stops_the_step(self): result, methods, _ = self.run_warm_up(header=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(methods, ["get", "put", "get"]) + self.assertIn("has no operation to follow", result.stderr) + + def test_accepted_put_without_an_operation_continues_when_registered(self): + result, methods, state = self.run_warm_up(header=False, put_registers=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "put", "get", "get", "delete", "get"]) + self.assertFalse(state["registered"]) + + def test_rejected_puts_wait_and_leave_registration_to_terraform(self): + result, methods, state = self.run_warm_up(put_failures=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(methods, ["get"] + ["put", "get"] * 5) - self.assertIn("Registration attempt 1 ended with status Unknown", result.stdout) + self.assertFalse(state["registered"]) + self.assertIn("Registration attempt 5 ended with status Rejected", result.stdout) + + def test_operation_header_is_read_in_any_case(self): + result, methods, state = self.run_warm_up(outcomes=["Succeeded"], header_name="azure-asyncoperation") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) + self.assertFalse(state["registered"]) def test_terraform_registration_waits_for_the_warm_up(self): block = STORAGE.read_text().split('resource "azurerm_backup_container_storage_account" "storage_account" {', 1)[ diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index 5e6fb37fb9..29841f4152 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -224,7 +224,13 @@ for try in 1 2 3 4 5; do if [ "$try" -gt 1 ]; then sleep 60 fi - operation=$(az rest --method put --url "$url" --body "$body" --debug 2>&1 | sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/p" | head -n 1 || true) + # Keep the result of the PUT. A rejected PUT starts no registration, but an accepted PUT can. + if put_output=$(az rest --method put --url "$url" --body "$body" --debug 2>&1); then + put_accepted=true + else + put_accepted=false + fi + operation=$(sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/Ip" <<< "$put_output" | head -n 1 || true) status=Unknown if [ -n "$operation" ]; then # A failed status read gives Unknown, and the step reads again. Another PUT must not overlap a running attempt. @@ -242,7 +248,16 @@ for try in 1 2 3 4 5; do exit 1 ;; esac + elif [ "$put_accepted" = true ]; then + # Azure accepted the PUT, but the step has no operation to follow. Continue only if the registration exists. + state=$(container_state) + if [ "$state" != registered ]; then + echo "Registration attempt $try has no operation to follow, so the step cannot make sure that it ended." >&2 + exit 1 + fi + status=Succeeded else + status=Rejected sleep 120 fi echo "Registration attempt $try ended with status $status" From c210b54963bdc1d4c93bdd8d7e1629003aa8bf60 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 29 Oct 2026 13:45:54 +0000 Subject: [PATCH 17/19] Wait for the backup registration status when no operation is returned (#5135) Azure can accept the registration PUT without an Azure-AsyncOperation header. Then the warm-up had no operation to follow, and it treated an existing container as a completed registration. The container can exist before the registration ends. So the warm-up now waits until the container shows the status Registered, for 10 minutes at most. If the status does not come, the step fails and does not send the unregister. If the unregister still fails after 10 minutes, the error now tells the operator to unregister the storage account in the vault and to update the workspace again. A comment at the early return explains why the step does not remove a registration that exists. --- .../unit_tests/test_backup_registration.py | 32 ++++++++++++++++++- .../workspaces/base/terraform/storage.tf | 21 +++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 84d265b811..326fbab637 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -28,6 +28,8 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) if method=='get': if state['get_errors']: state['get_errors']-=1;save();print('ERROR: (TooManyRequests) The request was throttled.',file=sys.stderr);sys.exit(1) + if state['registered'] and '--query' in args: + statuses=state['registration_statuses'];status=statuses.pop(0) if len(statuses)>1 else statuses[0];save();print(status);sys.exit(0) if state['registered']:sys.exit(0) print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) if method=='put': @@ -78,6 +80,7 @@ def run_warm_up( put_failures=0, put_registers=False, header_name="Azure-AsyncOperation", + registration_statuses=("Registered",), ): state = self.folder / "state.json" state.write_text( @@ -91,6 +94,7 @@ def run_warm_up( "put_failures": put_failures, "put_registers": put_registers, "header_name": header_name, + "registration_statuses": list(registration_statuses), } ) ) @@ -144,6 +148,13 @@ def test_failed_unregister_is_sent_again(self): self.assertEqual(methods, ["get", "put", "get", "delete"] + ["get"] * 6 + ["delete", "get"]) self.assertFalse(state["registered"]) + def test_lasting_unregister_failure_names_the_recovery(self): + result, methods, state = self.run_warm_up(outcomes=["Succeeded"], delete_failures=100) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(methods.count("delete"), 10) + self.assertTrue(state["registered"]) + self.assertIn("Unregister the storage account in the vault, then update the workspace again", result.stderr) + def test_read_error_does_not_change_an_existing_registration(self): result, methods, state = self.run_warm_up(registered=True, get_errors=1) self.assertEqual(result.returncode, 0, result.stderr) @@ -203,8 +214,27 @@ def test_login_uses_the_service_principal_without_a_managed_identity(self): def test_accepted_put_without_an_operation_stops_the_step(self): result, methods, _ = self.run_warm_up(header=False) self.assertNotEqual(result.returncode, 0) - self.assertEqual(methods, ["get", "put", "get"]) + self.assertEqual(methods, ["get", "put"] + ["get"] * 60) self.assertIn("has no operation to follow", result.stderr) + self.assertIn("The last status was Unknown", result.stderr) + + def test_accepted_put_without_an_operation_waits_for_the_registration(self): + # The container can exist before its registration ends. The step must not unregister it then. + result, methods, state = self.run_warm_up( + header=False, put_registers=True, registration_statuses=["Registering", "Registering", "Registered"] + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "put", "get", "get", "get", "get", "delete", "get"]) + self.assertFalse(state["registered"]) + + def test_accepted_put_without_an_operation_stops_while_the_registration_runs(self): + result, methods, state = self.run_warm_up( + header=False, put_registers=True, registration_statuses=["Registering"] + ) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("delete", methods) + self.assertTrue(state["registered"]) + self.assertIn("The last status was Registering", result.stderr) def test_accepted_put_without_an_operation_continues_when_registered(self): result, methods, state = self.run_warm_up(header=False, put_registers=True) diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index 29841f4152..3182a3e1f3 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -216,6 +216,8 @@ container_state() { state=$(container_state) if [ "$state" = registered ]; then + # Terraform owns a registration that exists, for example when a workspace upgrades to this version. + # The step cannot tell it from a registration of an earlier warm-up, so it does not remove it. echo "The storage account is already registered with the vault" exit 0 fi @@ -249,10 +251,18 @@ for try in 1 2 3 4 5; do ;; esac elif [ "$put_accepted" = true ]; then - # Azure accepted the PUT, but the step has no operation to follow. Continue only if the registration exists. - state=$(container_state) - if [ "$state" != registered ]; then - echo "Registration attempt $try has no operation to follow, so the step cannot make sure that it ended." >&2 + # Azure accepted the PUT, but the step has no operation to follow. The container can exist before + # the registration ends, so continue only when the container shows the status Registered. + registration=Unknown + for attempt in $(seq 1 60); do + registration=$(az rest --method get --url "$url" --query properties.registrationStatus --output tsv 2>/dev/null || echo Unknown) + if [ "$registration" = Registered ]; then + break + fi + sleep 10 + done + if [ "$registration" != Registered ]; then + echo "Registration attempt $try has no operation to follow, and the container did not show the status Registered within 10 minutes. The last status was $registration." >&2 exit 1 fi status=Succeeded @@ -275,7 +285,8 @@ for try in 1 2 3 4 5; do exit 0 fi done - echo "The storage account is still registered with the vault after 10 minutes" >&2 + echo "The storage account is still registered with the vault after 10 minutes." >&2 + echo "Terraform cannot create a registration that exists. Unregister the storage account in the vault, then update the workspace again." >&2 exit 1 fi done From c1ebf9e3017d38590c37e53f0e76693907f9bb0a Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 29 Oct 2026 14:01:01 +0000 Subject: [PATCH 18/19] Unregister a warm-up registration only after it ended (#5135) An error from the registration PUT does not prove that Azure rejected it. For example, Azure can accept the PUT and the response can get lost. Then the warm-up treated the attempt as rejected, and after 120 seconds it sent the unregister if the container existed, also when the registration still ran. Now one check runs before each unregister, for all attempt results: the container must show the status Registered within 10 minutes. If it does not, the step fails and does not send the unregister. The attempt without an operation to follow uses the same check. --- .../unit_tests/test_backup_registration.py | 33 +++++++++++++---- .../workspaces/base/terraform/storage.tf | 35 +++++++++++++------ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 326fbab637..3e95423224 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -33,6 +33,8 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) if state['registered']:sys.exit(0) print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) if method=='put': + if state['put_lost']: + state['registered']=True;save();print('ERROR: (ConnectionResetError) Connection aborted.',file=sys.stderr);sys.exit(1) if state['put_failures']: state['put_failures']-=1;save();print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) if state['put_registers']:state['registered']=True;save() @@ -81,6 +83,7 @@ def run_warm_up( put_registers=False, header_name="Azure-AsyncOperation", registration_statuses=("Registered",), + put_lost=False, ): state = self.folder / "state.json" state.write_text( @@ -95,6 +98,7 @@ def run_warm_up( "put_registers": put_registers, "header_name": header_name, "registration_statuses": list(registration_statuses), + "put_lost": put_lost, } ) ) @@ -131,22 +135,37 @@ def test_five_failed_attempts_leave_registration_to_terraform(self): def test_attempts_continue_until_one_succeeds(self): result, methods, state = self.run_warm_up(outcomes=["Failed", "Failed", "Succeeded"]) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get"] + ["put", "get"] * 3 + ["delete", "get"]) + self.assertEqual(methods, ["get"] + ["put", "get"] * 3 + ["get", "delete", "get"]) self.assertFalse(state["registered"]) self.assertIn("Registration attempt 3 ended with status Succeeded", result.stdout) def test_successful_first_attempt_is_removed_for_terraform(self): result, methods, state = self.run_warm_up(outcomes=["Succeeded"]) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) def test_failed_unregister_is_sent_again(self): # Azure can answer the unregister with a short CloudInternalError. result, methods, state = self.run_warm_up(outcomes=["Succeeded"], delete_failures=1) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "delete"] + ["get"] * 6 + ["delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "delete"] + ["get"] * 6 + ["delete", "get"]) + self.assertFalse(state["registered"]) + + def test_lost_put_response_waits_for_the_registration_before_the_unregister(self): + # Azure accepted the PUT, but the response was lost, so the step sees an error. + result, methods, state = self.run_warm_up(put_lost=True, registration_statuses=["Registering", "Registered"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(methods, ["get", "put", "get", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) + self.assertIn("Registration attempt 1 ended with status Rejected", result.stdout) + + def test_lost_put_response_stops_the_step_while_the_registration_runs(self): + result, methods, state = self.run_warm_up(put_lost=True, registration_statuses=["Registering"]) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("delete", methods) + self.assertTrue(state["registered"]) + self.assertIn("does not unregister a registration that has not ended", result.stderr) def test_lasting_unregister_failure_names_the_recovery(self): result, methods, state = self.run_warm_up(outcomes=["Succeeded"], delete_failures=100) @@ -176,7 +195,7 @@ def test_attempt_without_a_final_status_stops_the_step(self): def test_failed_status_read_is_read_again(self): result, methods, state = self.run_warm_up(outcomes=["ERROR", "InProgress", "Succeeded"]) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) def test_login_uses_the_cloud_and_the_managed_identity_by_default(self): @@ -224,7 +243,7 @@ def test_accepted_put_without_an_operation_waits_for_the_registration(self): header=False, put_registers=True, registration_statuses=["Registering", "Registering", "Registered"] ) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "get", "get", "get", "delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "get", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) def test_accepted_put_without_an_operation_stops_while_the_registration_runs(self): @@ -239,7 +258,7 @@ def test_accepted_put_without_an_operation_stops_while_the_registration_runs(sel def test_accepted_put_without_an_operation_continues_when_registered(self): result, methods, state = self.run_warm_up(header=False, put_registers=True) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "get", "delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) def test_rejected_puts_wait_and_leave_registration_to_terraform(self): @@ -252,7 +271,7 @@ def test_rejected_puts_wait_and_leave_registration_to_terraform(self): def test_operation_header_is_read_in_any_case(self): result, methods, state = self.run_warm_up(outcomes=["Succeeded"], header_name="azure-asyncoperation") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(methods, ["get", "put", "get", "delete", "get"]) + self.assertEqual(methods, ["get", "put", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) def test_terraform_registration_waits_for_the_warm_up(self): diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index 3182a3e1f3..0dab81995a 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -214,6 +214,20 @@ container_state() { return 1 } +# Print the registration status of the container. Read it for up to 10 minutes, until it is Registered. +# The container can exist before its registration ends, and the step must not unregister it then. +registration_status() { + local registration=Unknown + for _ in $(seq 1 60); do + registration=$(az rest --method get --url "$url" --query properties.registrationStatus --output tsv 2>/dev/null || echo Unknown) + if [ "$registration" = Registered ]; then + break + fi + sleep 10 + done + echo "$registration" +} + state=$(container_state) if [ "$state" = registered ]; then # Terraform owns a registration that exists, for example when a workspace upgrades to this version. @@ -227,6 +241,7 @@ for try in 1 2 3 4 5; do sleep 60 fi # Keep the result of the PUT. A rejected PUT starts no registration, but an accepted PUT can. + # An error can also hide an accepted PUT, for example when the response is lost. if put_output=$(az rest --method put --url "$url" --body "$body" --debug 2>&1); then put_accepted=true else @@ -251,16 +266,9 @@ for try in 1 2 3 4 5; do ;; esac elif [ "$put_accepted" = true ]; then - # Azure accepted the PUT, but the step has no operation to follow. The container can exist before - # the registration ends, so continue only when the container shows the status Registered. - registration=Unknown - for attempt in $(seq 1 60); do - registration=$(az rest --method get --url "$url" --query properties.registrationStatus --output tsv 2>/dev/null || echo Unknown) - if [ "$registration" = Registered ]; then - break - fi - sleep 10 - done + # Azure accepted the PUT, but the step has no operation to follow. Continue only when the container + # shows the status Registered. + registration=$(registration_status) if [ "$registration" != Registered ]; then echo "Registration attempt $try has no operation to follow, and the container did not show the status Registered within 10 minutes. The last status was $registration." >&2 exit 1 @@ -273,6 +281,13 @@ for try in 1 2 3 4 5; do echo "Registration attempt $try ended with status $status" state=$(container_state) if [ "$state" = registered ]; then + # Unregister only a registration that ended. After any attempt, for example one with a lost + # response, the registration can still run. + registration=$(registration_status) + if [ "$registration" != Registered ]; then + echo "After registration attempt $try, the container did not show the status Registered within 10 minutes. The last status was $registration. The step does not unregister a registration that has not ended." >&2 + exit 1 + fi # Remove the registration, so that Terraform creates and owns the registration. # Azure can answer the unregister with a short CloudInternalError, so send it again each minute. for attempt in $(seq 1 60); do From 7e0d78b25dde01ab2923ff163d2dfd5437e0c4ba Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 29 Oct 2026 14:24:56 +0000 Subject: [PATCH 19/19] Stop the backup warm-up when a PUT gets no answer from Azure (#5135) The warm-up treated each failed registration PUT as a rejection. But Azure can accept a PUT whose response gets lost, and the container of that registration can show some minutes later. Then the warm-up sent the next PUT while the first registration still ran. After five such attempts, it ended with success, and Terraform started its own registration over the running one. With --debug, az rest prints the response status when Azure answers. Now the warm-up treats a failed PUT as a rejection only when Azure answered with a 4xx or 5xx status. Without an answer, the result is unknown, so the step stops, and no other registration overlaps it. --- .../unit_tests/test_backup_registration.py | 33 ++++++++++++++----- .../workspaces/base/terraform/storage.tf | 6 ++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py index 3e95423224..d3d2e13779 100644 --- a/e2e_tests/unit_tests/test_backup_registration.py +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -34,9 +34,10 @@ def save():json.dump(state,open(os.environ['STATE'],'w')) print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) if method=='put': if state['put_lost']: - state['registered']=True;save();print('ERROR: (ConnectionResetError) Connection aborted.',file=sys.stderr);sys.exit(1) + if state['put_registers']:state['registered']=True + save();print('ERROR: (ConnectionResetError) Connection aborted.',file=sys.stderr);sys.exit(1) if state['put_failures']: - state['put_failures']-=1;save();print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) + state['put_failures']-=1;save();print('INFO: cli.azure.cli.core.util: Response status: 500',file=sys.stderr);print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) if state['put_registers']:state['registered']=True;save() if state['header']:print("INFO: cli.azure.cli.core.util: '"+state['header_name']+"': '"+os.environ['OPERATION']+"'",file=sys.stderr) sys.exit(0) @@ -152,16 +153,32 @@ def test_failed_unregister_is_sent_again(self): self.assertEqual(methods, ["get", "put", "get", "get", "delete"] + ["get"] * 6 + ["delete", "get"]) self.assertFalse(state["registered"]) - def test_lost_put_response_waits_for_the_registration_before_the_unregister(self): - # Azure accepted the PUT, but the response was lost, so the step sees an error. - result, methods, state = self.run_warm_up(put_lost=True, registration_statuses=["Registering", "Registered"]) + def test_put_without_an_answer_stops_the_step(self): + # Azure accepted the PUT, but the response was lost. The registration can still run. + result, methods, state = self.run_warm_up(put_lost=True, put_registers=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(methods, ["get", "put"]) + self.assertTrue(state["registered"]) + self.assertIn("got no answer from Azure", result.stderr) + + def test_put_without_an_answer_stops_the_step_before_the_container_shows(self): + # The container of an accepted PUT can show some minutes later. Another PUT must not overlap it. + result, methods, state = self.run_warm_up(put_lost=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(methods, ["get", "put"]) + self.assertFalse(state["registered"]) + self.assertIn("got no answer from Azure", result.stderr) + + def test_unregister_waits_until_the_registration_ended(self): + result, methods, state = self.run_warm_up( + outcomes=["Succeeded"], registration_statuses=["Registering", "Registered"] + ) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(methods, ["get", "put", "get", "get", "get", "delete", "get"]) self.assertFalse(state["registered"]) - self.assertIn("Registration attempt 1 ended with status Rejected", result.stdout) - def test_lost_put_response_stops_the_step_while_the_registration_runs(self): - result, methods, state = self.run_warm_up(put_lost=True, registration_statuses=["Registering"]) + def test_registration_that_does_not_end_is_not_unregistered(self): + result, methods, state = self.run_warm_up(outcomes=["Succeeded"], registration_statuses=["Registering"]) self.assertNotEqual(result.returncode, 0) self.assertNotIn("delete", methods) self.assertTrue(state["registered"]) diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index 0dab81995a..53c45fd79e 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -275,6 +275,12 @@ for try in 1 2 3 4 5; do fi status=Succeeded else + # Only an error answer from Azure proves a rejection. Without an answer, Azure can still run the + # registration, so the step stops, and no other registration overlaps it. + if ! grep -qE 'Response status: [45][0-9]{2}' <<< "$put_output"; then + echo "Registration attempt $try got no answer from Azure, so its result is unknown. The step stops, so that no other registration overlaps it." >&2 + exit 1 + fi status=Rejected sleep 120 fi