From d5d845c84e9bba9bac3bd171a402335b319a562d Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 23 Oct 2026 08:39:05 +0100 Subject: [PATCH 01/24] Record the deployed template version in bundle evidence Bundle tests that read a deployed resource now record its template name and template version in the evidence report, with the resource ID and path. The record does not copy the resource properties, so secrets in the properties do not go into the report. The CycleCloud and OHDSI lifecycle tests use this to show which bundle version they tested. --- CHANGELOG.md | 1 + e2e_tests/bundle_evidence.py | 15 +++++++++++ e2e_tests/unit_tests/test_bundle_selection.py | 26 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index decc94575..0c5ed517b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ENHANCEMENTS: * Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) +* Record the deployed template version of each resource that a bundle test reads in the bundle evidence, without the resource properties. * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/e2e_tests/bundle_evidence.py b/e2e_tests/bundle_evidence.py index 27f6a2deb..bb6f243d1 100644 --- a/e2e_tests/bundle_evidence.py +++ b/e2e_tests/bundle_evidence.py @@ -26,6 +26,21 @@ def record_resource(payload, operation, method): _ACTIVE_REPORT.write() +def record_deployed_resource(resource, resource_path): + """Record the observed bundle version without copying resource properties.""" + if _ACTIVE_REPORT is None: + return + record = { + "method": "GET", + "resourceId": resource["id"], + "resourcePath": resource_path, + "bundle": resource["templateName"], + "deployed_template_version": resource["templateVersion"], + } + _ACTIVE_REPORT.data["resources"].append(record) + _ACTIVE_REPORT.write() + + def record_reused_resource(resource): """Record only the selected prerequisite's identity and deployed version.""" if _ACTIVE_REPORT is None: diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index 2adf50baa..86a40e230 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -222,6 +222,32 @@ def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): self.assertEqual(report["source_bundle_version"], version) self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + def test_observed_deployment_evidence_excludes_properties(self): + with tempfile.TemporaryDirectory() as directory: + report = bundle_evidence.BundleReport(Path(directory) / "report.json", {}) + with patch.object(bundle_evidence, "_ACTIVE_REPORT", report): + bundle_evidence.record_deployed_resource( + { + "id": "vm", + "templateName": "review", + "templateVersion": "2.0.9", + "properties": {"secret": "excluded"}, + }, + "/workspaces/ws/workspace-services/service/user-resources/vm", + ) + self.assertEqual( + report.data["resources"], + [ + { + "method": "GET", + "resourceId": "vm", + "resourcePath": "/workspaces/ws/workspace-services/service/user-resources/vm", + "bundle": "review", + "deployed_template_version": "2.0.9", + } + ], + ) + def test_marker_and_parallel_worker_inputs_are_rejected(self): for extra in ({"TEST_BUNDLE_MARKER": "workspace_services"}, {"E2E_TESTS_NUMBER_PROCESSES": "2"}): with self.subTest(environment=extra): From ddf81ce41910684cc38120bb8cbf1ae7139314a2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 21:01:07 +0100 Subject: [PATCH 02/24] Fix CycleCloud VM action outputs (#5159) --- CHANGELOG.md | 1 + .../test_cyclecloud_bundle_contract.py | 32 +++++++++++++++++++ .../shared_services/cyclecloud/porter.yaml | 11 ++++++- .../cyclecloud/terraform/outputs.tf | 4 +++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c5ed517b..f3e616cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py new file mode 100644 index 000000000..1127f5b3b --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -0,0 +1,32 @@ +"""Check that CycleCloud VM actions can resolve the Terraform output.""" + +from pathlib import Path +import re +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/shared_services/cyclecloud" + + +class CycleCloudBundleContractTests(unittest.TestCase): + def test_vm_actions_have_a_declared_and_emitted_resource_id(self): + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + output = next(o for o in manifest["outputs"] if o["name"] == "azure_resource_id") + self.assertEqual(output["type"], "string") + for action in ("install", "upgrade", "start", "stop"): + with self.subTest(action=action): + self.assertIn(action, output["applyTo"]) + terraform = next(step["terraform"] for step in manifest[action] if "terraform" in step) + self.assertIn({"name": "azure_resource_id"}, terraform["outputs"]) + for action, command in (("start", "start"), ("stop", "deallocate")): + with self.subTest(action=action): + step = next(s["az"] for s in manifest[action] if s.get("az", {}).get("arguments") == ["vm", command]) + self.assertEqual(step["flags"]["ids"], "${ bundle.outputs.azure_resource_id }") + output_source = (BUNDLE / "terraform/outputs.tf").read_text() + self.assertRegex( + output_source, r'output "azure_resource_id"\s*\{\s*value\s*=\s*azurerm_virtual_machine\.cyclecloud\.id\s*\}' + ) + vm_source = (BUNDLE / "terraform/cyclecloud.tf").read_text() + self.assertIsNotNone(re.search(r'resource "azurerm_virtual_machine" "cyclecloud"', vm_source)) diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index 1cd58af1d..537dd523c 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-cyclecloud -version: 0.7.9 +version: 0.7.10 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl @@ -55,6 +55,13 @@ parameters: default: "" outputs: + - name: azure_resource_id + type: string + applyTo: + - install + - upgrade + - start + - stop - name: connection_uri type: string applyTo: @@ -90,6 +97,7 @@ install: container_name: ${ bundle.parameters.tfstate_container_name } key: ${ bundle.parameters.id } outputs: + - name: azure_resource_id - name: connection_uri - name: shared_subnet_address_prefixes @@ -110,6 +118,7 @@ upgrade: container_name: ${ bundle.parameters.tfstate_container_name } key: ${ bundle.parameters.id } outputs: + - name: azure_resource_id - name: connection_uri - name: shared_subnet_address_prefixes diff --git a/templates/shared_services/cyclecloud/terraform/outputs.tf b/templates/shared_services/cyclecloud/terraform/outputs.tf index d6b7b5c87..6220f4370 100644 --- a/templates/shared_services/cyclecloud/terraform/outputs.tf +++ b/templates/shared_services/cyclecloud/terraform/outputs.tf @@ -1,3 +1,7 @@ +output "azure_resource_id" { + value = azurerm_virtual_machine.cyclecloud.id +} + output "connection_uri" { value = "https://${azurerm_private_dns_zone.cyclecloud.name}" } From 0c7ae5d49a1985167748915a65bce25334a95894 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 21:10:04 +0100 Subject: [PATCH 03/24] Validate isolated CycleCloud server lifecycle (#5159) Keep the cleanup error message in the note that the CycleCloud lifecycle helper adds to an earlier test failure. The open review comment on #5223 reported the same defect in the OHDSI helper. --- .github/workflows/deploy_tre_reusable.yml | 5 + CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 27 ++ e2e_tests/bundle_coverage.json | 25 +- e2e_tests/pytest.ini | 1 + e2e_tests/resources/cyclecloud.py | 297 ++++++++++++++++ e2e_tests/resources/cyclecloud_lifecycle.py | 211 ++++++++++++ e2e_tests/test_cyclecloud.py | 120 +++++++ e2e_tests/test_shared_services.py | 3 +- e2e_tests/unit_tests/test_bundle_selection.py | 9 + .../test_cyclecloud_configuration.py | 221 ++++++++++++ .../unit_tests/test_cyclecloud_lifecycle.py | 320 ++++++++++++++++++ 12 files changed, 1233 insertions(+), 7 deletions(-) create mode 100644 e2e_tests/resources/cyclecloud.py create mode 100644 e2e_tests/resources/cyclecloud_lifecycle.py create mode 100644 e2e_tests/test_cyclecloud.py create mode 100644 e2e_tests/unit_tests/test_cyclecloud_configuration.py create mode 100644 e2e_tests/unit_tests/test_cyclecloud_lifecycle.py diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 65ebe1539..5b8a4b41c 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -1072,6 +1072,10 @@ jobs: # The job timeout is 300 minutes. Each step below sets an absolute deadline 270 minutes # (270 * 60 seconds) after the job start. The lifecycle helper keeps the last 120 minutes # before the deadline for the clean-up. The 30 minutes after the deadline are for the report. + - name: Reserve CycleCloud cleanup and reporting time + if: inputs.e2eBundle == 'tre-shared-service-cyclecloud' + run: echo "CYCLECLOUD_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve SQL cleanup and reporting time if: inputs.e2eBundle == 'tre-workspace-service-azuresql' run: echo "SQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" @@ -1113,6 +1117,7 @@ jobs: SQL_VALIDATION_DEADLINE=${{ env.SQL_VALIDATION_DEADLINE }} MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }} GITEA_VALIDATION_DEADLINE=${{ env.GITEA_VALIDATION_DEADLINE }} + CYCLECLOUD_VALIDATION_DEADLINE=${{ env.CYCLECLOUD_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e616cf3..39308a30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ENHANCEMENTS: * Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) +* Add an independent CycleCloud server lifecycle with marketplace checks, VM power actions and bounded cleanup. ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Record the deployed template version of each resource that a bundle test reads in the bundle evidence, without the resource properties. * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 306f4ca15..cd332411e 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -237,3 +237,30 @@ Source versions describe the checked-out manifests; verify deployed versions sep An entirely skipped selection exits unsuccessfully. Individual skipped cases retain their reasons, and a mixed pass/skip result is marked `partial`. Unavailable bundles fail with the gap recorded in the map. Adding a selector does not implement missing tests or fix bundle publication and registration gaps. + +### CycleCloud server validation + +Select `tre-shared-service-cyclecloud` with the branch workflow's bundle input, or use: + +```bash +TEST_BUNDLE=tre-shared-service-cyclecloud make test-e2e-bundle +``` + +Use a dedicated AzureCloud environment with one writer and no existing CycleCloud service or DNS zone. +Publish and register the checkout firewall and CycleCloud bundles. Deploy the firewall before validation. +Register the repaired CycleCloud bundle, version 0.7.10 or later, with `start` and `stop` actions. +Accept the CycleCloud server image terms separately before the run. The test reads the agreement and regional image list. It does not accept terms. +Check `Standard_DS3_v2` capacity and quota in the core region before deployment. Image availability does not prove VM capacity. + +The case creates one server and checks its private VM address, DNS configuration, blob private endpoint, managed identity and firewall rules. +It invokes stop/start through TRE and verifies the VM power states. It changes the overview to exercise a same-version Terraform upgrade and checks that the VM identity remains stable. +The test records the existing firewall as reused. Smoke completes before the bundle job starts. + +The job sets `CYCLECLOUD_VALIDATION_DEADLINE` before container setup. The test reserves 120 minutes for cleanup within its 270-minute deadline. +It owns an accepted service before polling. Cleanup waits for terminal operations, verifies ownership and then uses TRE deletion. +It verifies removal of the VM, disk, NIC, storage, blob private endpoint, DNS zone and captured identity's roles. It also checks that the original firewall rules are restored. +Failed cleanup fails the test. Core Key Vault credentials can remain soft-deleted. The test does not retrieve or purge them. + +This case does not configure the CycleCloud application or create an HPC cluster. Browser access, application TLS trust, private DNS reachability and prior-release upgrades remain unproven. +The existing bundle grants subscription Contributor to its VM and permits SharedSubnet access to ARM and GitHub. It does not disable the storage public endpoint. +A private endpoint and a successful server lifecycle do not prove workspace isolation or storage access restrictions. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 49194451c..6fe874c72 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -73,14 +73,29 @@ "tre-shared-service-cyclecloud": { "manifest": "templates/shared_services/cyclecloud/porter.yaml", "family": "shared_service", - "prerequisites": [], - "tests": [], + "prerequisites": [ + "tre-shared-service-firewall" + ], + "tests": [ + "test_cyclecloud.py::test_cyclecloud_lifecycle" + ], "related_tests": [], - "coverage": [], + "coverage": [ + "create", + "update", + "custom_actions", + "delete", + "configuration" + ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, accepted CycleCloud marketplace image terms, the regional image and Standard_DS3_v2 capacity. The test does not accept terms or change quotas.", + "Requires an empty CycleCloud slot, current firewall and one writer in a dedicated environment. The fixed DNS zone prevents parallel CycleCloud services.", + "Checks VM power actions, ARM configuration, TRE firewall rules, same-version metadata upgrade and bounded cleanup. Previous-version upgrades remain unproven.", + "Does not configure the CycleCloud application, create HPC clusters or prove browser access, private DNS reachability or application TLS trust.", + "The existing bundle grants its VM subscription Contributor and allows SharedSubnet access to ARM and GitHub. Storage public access is not disabled by this bundle. These controls do not prove workspace isolation.", + "Cleanup verifies the VM, disk, NIC, storage, private endpoint, DNS zone, captured identity roles and firewall rules. Key Vault credentials can remain soft-deleted; secret purge is not attempted." ], - "unavailable_reason": "The existing shared-service case is disabled pending repair (#5154)." + "unavailable_reason": "" }, "tre-shared-service-databricks-private-auth": { "manifest": "templates/shared_services/databricks-auth/porter.yaml", diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index ea391b6c8..285d13993 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -16,6 +16,7 @@ markers = workspace_services workspace_validation: opt-in base-workspace creation, scoped API access and deletion openai_validation: opt-in private OpenAI deployment with model and quota preflight + cyclecloud_validation: opt-in isolated CycleCloud server lifecycle and VM power actions sql_validation: Private SQL query and configuration upgrade lifecycle mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM gitea_validation: opt-in private Gitea repository write/read across an update and an app restart diff --git a/e2e_tests/resources/cyclecloud.py b/e2e_tests/resources/cyclecloud.py new file mode 100644 index 000000000..436a6dedb --- /dev/null +++ b/e2e_tests/resources/cyclecloud.py @@ -0,0 +1,297 @@ +"""Inspect the owned CycleCloud infrastructure without reading credentials.""" + +import asyncio +from copy import deepcopy +from dataclasses import dataclass +from ipaddress import ip_address, ip_network +import os +import re +from urllib.parse import parse_qsl, urlsplit +from uuid import UUID + +from e2e_tests import config + +COMPUTE_API = "2023-03-01" +DISK_API = "2023-10-02" +NETWORK_API = "2024-05-01" +DNS_API = "2024-06-01" +STORAGE_API = "2023-05-01" +ROLE_API = "2022-04-01" +CONTRIBUTOR = "b24988ac-6180-42a0-ab88-20f7382dd24c" +TEMPLATE = "tre-shared-service-cyclecloud" +FIREWALL = "tre-shared-service-firewall" + + +def uuid_value(value): + try: + parsed = UUID(value) + if not parsed.int or str(parsed) != value: + raise ValueError + except (ValueError, TypeError, AttributeError): + raise ValueError("CycleCloud requires canonical, non-zero resource UUIDs") from None + return value + + +@dataclass(frozen=True) +class Settings: + subscription: str + tre_id: str + + def __post_init__(self): + uuid_value(self.subscription) + if not isinstance(self.tre_id, str) or not re.fullmatch(r"[a-z0-9-]{1,11}", self.tre_id): + raise ValueError("CycleCloud requires a valid TRE_ID shorter than 12 characters") + + @property + def group(self): + return f"/subscriptions/{self.subscription}/resourceGroups/rg-{self.tre_id}" + + +def require_settings(): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("CycleCloud validation currently supports AzureCloud only") + if uuid_value(os.environ.get("ARM_TENANT_ID", "")) != uuid_value(config.AAD_TENANT_ID): + raise ValueError("CycleCloud requires the deployment and authentication tenants to match") + return Settings(os.environ.get("ARM_SUBSCRIPTION_ID", ""), config.TRE_ID) + + +def same_id(actual, expected): + assert isinstance(actual, str) and actual.lower() == expected.lower(), "Unexpected CycleCloud ARM identity" + + +def owned(record, identifier, tags): + same_id(record.get("id"), identifier) + assert all(record.get("tags", {}).get(k) == v for k, v in tags.items()), "CycleCloud ownership tags differ" + + +async def preflight(arm, settings): + group = await arm.request("GET", settings.group, "2021-04-01") + owned(group, settings.group, {"tre_id": settings.tre_id}) + location = group.get("location", "").lower().replace(" ", "") + assert re.fullmatch(r"[a-z0-9]+", location), "Core region is missing" + vnet = f"{settings.group}/providers/Microsoft.Network/virtualNetworks/vnet-{settings.tre_id}" + subnet = vnet + "/subnets/SharedSubnet" + subnet_record = await arm.request("GET", subnet, NETWORK_API) + same_id(subnet_record.get("id"), subnet) + prefixes = subnet_record["properties"].get("addressPrefixes") or [subnet_record["properties"]["addressPrefix"]] + assert prefixes and all(ip_network(v).is_private for v in prefixes), "SharedSubnet must use private addresses" + pip = f"{settings.group}/providers/Microsoft.Network/publicIPAddresses/pip-agw-{settings.tre_id}" + fqdn = (await arm.request("GET", pip, NETWORK_API))["properties"]["dnsSettings"]["fqdn"] + assert re.fullmatch(r"[a-z0-9.-]+\.cloudapp\.azure\.com", fqdn), "Unexpected core public DNS name" + zone = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/cyclecloud-{fqdn}" + if await arm.request("GET", zone, DNS_API, missing_ok=True) is not None: + raise ValueError("An existing CycleCloud DNS zone prevents isolated validation") + blob_zone = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net" + same_id((await arm.request("GET", blob_zone, DNS_API)).get("id"), blob_zone) + subscription = f"/subscriptions/{settings.subscription}" + terms = ( + subscription + + "/providers/Microsoft.MarketplaceOrdering/offerTypes/virtualmachine/publishers/azurecyclecloud/offers/azure-cyclecloud/plans/cyclecloud8/agreements/current" + ) + agreement = await arm.request("GET", terms, "2021-01-01") + if agreement.get("properties", {}).get("accepted") is not True: + raise ValueError( + "Accept the CycleCloud marketplace image terms before validation; this test does not accept terms" + ) + images = await arm.request( + "GET", + subscription + + f"/providers/Microsoft.Compute/locations/{location}/publishers/azurecyclecloud/artifacttypes/vmimage/offers/azure-cyclecloud/skus/cyclecloud8/versions", + "2024-07-01", + ) + if not isinstance(images, list) or not images or not all(i.get("name") for i in images): + raise ValueError("The CycleCloud marketplace image is unavailable in the core region") + return { + "location": location, + "vnet": vnet, + "subnet": subnet, + "prefixes": prefixes, + "zone": zone, + "blob_zone": blob_zone, + } + + +def resource_ids(settings, core, identifier): + suffix = uuid_value(identifier)[-4:] + vm = f"cyclecloud-{suffix}" + storage = f"stgcc{settings.tre_id}{suffix}".replace("-", "") + group = settings.group + return { + "vm": f"{group}/providers/Microsoft.Compute/virtualMachines/{vm}", + "disk": f"{group}/providers/Microsoft.Compute/disks/{vm}-osdisk", + "nic": f"{group}/providers/Microsoft.Network/networkInterfaces/nic-cyclecloud-{settings.tre_id}-{suffix}", + "storage": f"{group}/providers/Microsoft.Storage/storageAccounts/{storage}", + "endpoint": f"{group}/providers/Microsoft.Network/privateEndpoints/pe-{storage}", + "zone": core["zone"], + } + + +def resource_types(ids): + return [ + (ids[k], v) + for k, v in ( + ("vm", COMPUTE_API), + ("disk", DISK_API), + ("nic", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ) + ] + + +async def guard_owned(arm, ids, tags): + # The disk is deleted through its VM. Its generated tags are not guaranteed. + for key, version in ( + ("vm", COMPUTE_API), + ("nic", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ): + record = await arm.request("GET", ids[key], version, missing_ok=True) + if record is not None: + owned(record, ids[key], tags) + + +async def principal_roles(arm, subscription, principal): + uuid_value(principal) + endpoint = f"/subscriptions/{subscription}/providers/Microsoft.Authorization/roleAssignments" + query = {"$filter": f"principalId eq '{principal}'"} + records = [] + for _ in range(20): + page = await arm.request("GET", endpoint, ROLE_API, query=query) + records.extend(page["value"]) + link = page.get("nextLink") + if not link: + return records + parsed = urlsplit(link) + if ( + parsed.scheme != "https" + or parsed.netloc != "management.azure.com" + or parsed.path.lower() != endpoint.lower() + ): + raise ValueError("Unexpected role-assignment continuation URL") + query = dict(parse_qsl(parsed.query)) + query.pop("api-version", None) + raise RuntimeError("CycleCloud role-assignment pagination exceeded its bound") + + +async def validate_resources(arm, settings, core, ids, identifier): + tags = {"tre_id": settings.tre_id, "tre_shared_service_id": identifier} + records = {} + for key, version in ( + ("vm", COMPUTE_API), + ("nic", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ): + records[key] = await arm.request("GET", ids[key], version) + owned(records[key], ids[key], tags) + vm = records["vm"] + properties = vm["properties"] + assert properties["provisioningState"] == "Succeeded" + assert properties["hardwareProfile"]["vmSize"] == "Standard_DS3_v2" + image = properties["storageProfile"]["imageReference"] + assert (image["publisher"], image["offer"], image["sku"]) == ("azurecyclecloud", "azure-cyclecloud", "cyclecloud8") + assert all( + vm["plan"].get(k) == v + for k, v in {"name": "cyclecloud8", "product": "azure-cyclecloud", "publisher": "azurecyclecloud"}.items() + ) + assert vm["identity"]["type"] == "SystemAssigned" + principal = uuid_value(vm["identity"]["principalId"]) + same_id(properties["storageProfile"]["osDisk"]["managedDisk"]["id"], ids["disk"]) + assert len(properties["networkProfile"]["networkInterfaces"]) == 1 + same_id(properties["networkProfile"]["networkInterfaces"][0]["id"], ids["nic"]) + configs = records["nic"]["properties"]["ipConfigurations"] + assert len(configs) == 1 + ip = configs[0]["properties"] + same_id(ip["subnet"]["id"], core["subnet"]) + assert not ip.get("publicIPAddress"), "CycleCloud VM must not have a public IP" + assert any(ip_address(ip["privateIPAddress"]) in ip_network(cidr) for cidr in core["prefixes"]) + dns = await arm.request("GET", ids["zone"] + "/A/@", DNS_API) + assert dns["properties"]["aRecords"] == [{"ipv4Address": ip["privateIPAddress"]}] + link = await arm.request("GET", ids["zone"] + "/virtualNetworkLinks/cyclecloudlink-core", DNS_API) + same_id(link["properties"]["virtualNetwork"]["id"], core["vnet"]) + storage = records["storage"] + assert storage["sku"]["name"] == "Standard_GRS" + assert storage["properties"]["encryption"]["requireInfrastructureEncryption"] is True + assert storage["properties"]["allowCrossTenantReplication"] is False + pe = records["endpoint"]["properties"] + same_id(pe["subnet"]["id"], core["subnet"]) + connections = pe["privateLinkServiceConnections"] + assert len(connections) == 1 + connection = connections[0]["properties"] + same_id(connection["privateLinkServiceId"], ids["storage"]) + assert [v.lower() for v in connection["groupIds"]] == ["blob"] + assert connection["privateLinkServiceConnectionState"]["status"] == "Approved" + zone_group = await arm.request("GET", ids["endpoint"] + "/privateDnsZoneGroups/private-dns-zone-group", NETWORK_API) + zones = zone_group["properties"]["privateDnsZoneConfigs"] + assert len(zones) == 1 + same_id(zones[0]["properties"]["privateDnsZoneId"], core["blob_zone"]) + roles = await principal_roles(arm, settings.subscription, principal) + assert all(r["properties"]["principalId"].lower() == principal for r in roles), ( + "Role query returned a different principal" + ) + assert any( + r["properties"]["scope"].lower() == f"/subscriptions/{settings.subscription}" + and r["properties"]["roleDefinitionId"].lower().endswith("/" + CONTRIBUTOR) + for r in roles + ), "CycleCloud subscription Contributor assignment is missing" + return principal + + +async def wait_power(arm, vm_id, expected): + async with asyncio.timeout(10 * 60): + while True: + vm = await arm.request("GET", vm_id, COMPUTE_API, expand=True) + codes = {s["code"] for s in vm["properties"].get("instanceView", {}).get("statuses", [])} + if f"PowerState/{expected}" in codes: + return + await asyncio.sleep(10) + + +async def assert_removed(arm, ids, subscription, principal=None): + async with asyncio.timeout(10 * 60): + while True: + remaining = [ + identifier + for identifier, version in resource_types(ids) + if await arm.request("GET", identifier, version, missing_ok=True) is not None + ] + roles = await principal_roles(arm, subscription, principal) if principal else [] + if not remaining and not roles: + return + await asyncio.sleep(10) + + +def firewall_rules(resource): + result = {} + for key in ("rule_collections", "network_rule_collections"): + values = resource["properties"].get(key, []) + result[key] = {v["name"]: deepcopy(v) for v in values} + assert len(result[key]) == len(values), "Duplicate firewall collection names" + return result + + +def validate_firewall(resource, baseline, identifier, prefixes): + rules = firewall_rules(resource) + for key, prefix in (("rule_collections", "arc"), ("network_rule_collections", "nrc")): + name = f"{prefix}_svc_{identifier}_cyclecloud" + collection = rules[key].pop(name) + assert collection["action"] == "Allow" and len(collection["rules"]) == 1 + rule = collection["rules"][0] + assert set(rule["source_addresses"]) == set(prefixes) + if prefix == "arc": + assert rule["protocols"] == [{"port": "443", "type": "Https"}] + assert set(rule["target_fqdns"]) == { + "github.com", + "api.github.com", + "codeload.github.com", + "objects.githubusercontent.com", + } + else: + assert rule["destination_addresses"] == ["AzureResourceManager"] + assert rule["destination_ports"] == ["443"] and rule["protocols"] == ["TCP"] + assert rules == baseline, "Unrelated firewall rules changed" diff --git a/e2e_tests/resources/cyclecloud_lifecycle.py b/e2e_tests/resources/cyclecloud_lifecycle.py new file mode 100644 index 000000000..81135d0d6 --- /dev/null +++ b/e2e_tests/resources/cyclecloud_lifecycle.py @@ -0,0 +1,211 @@ +"""Own a CycleCloud service before polling and reserve time for its deletion.""" + +import asyncio +from builtins import BaseExceptionGroup +from contextlib import asynccontextmanager +import math +import os +import time + +from httpx import AsyncClient + +from e2e_tests.bundle_evidence import record_operation, record_resource +from e2e_tests.helpers import TIMEOUT, get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources.cyclecloud import TEMPLATE, uuid_value +from e2e_tests.resources.resource import disable_and_delete_resource +from e2e_tests.timeouts import cleanup_deadline + +LIFECYCLE_SECONDS = 270 * 60 +CLEANUP_RESERVE_SECONDS = 120 * 60 +TERMINAL = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} +FAILED = {"deployment_failed", "updating_failed", "deleting_failed", "action_failed", "pipeline_failed"} + + +def lifecycle_deadlines(): + configured = os.environ.get("CYCLECLOUD_VALIDATION_DEADLINE", "") + deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS + if not math.isfinite(deadline): + raise ValueError("CycleCloud validation deadline must be finite") + remaining = min(LIFECYCLE_SECONDS, deadline - time.time()) + if remaining <= CLEANUP_RESERVE_SECONDS: + raise TimeoutError("Insufficient CycleCloud time remains after reserving cleanup") + finish = asyncio.get_running_loop().time() + remaining + return finish - CLEANUP_RESERVE_SECONDS, finish + + +class CycleCloudService: + def __init__(self, verify, finish, payload, before_remove, after_remove): + self.verify = verify + self.finish = finish + self.payload = payload + self.path = None + self.identifier = None + self.before_remove = before_remove + self.after_remove = after_remove + + def validate_owner(self, body): + resource = body["sharedService"] + if ( + resource["id"] != self.identifier + or resource["templateName"] != TEMPLATE + or any( + resource["properties"].get(k) != self.payload["properties"][k] for k in ("display_name", "description") + ) + ): + raise ValueError("CycleCloud resource ownership changed") + return resource + + async def headers(self): + return get_auth_header(await get_admin_token(self.verify)) + + async def create(self): + if self.payload.get("templateName") != TEMPLATE or any( + not self.payload.get("properties", {}).get(k) for k in ("display_name", "description") + ): + raise ValueError("CycleCloud creation needs the expected template and ownership properties") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.post( + get_full_endpoint("/api/shared-services"), headers=await self.headers(), json=self.payload + ) + if response.status_code != 202: + raise RuntimeError(f"CycleCloud creation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + identifier = uuid_value(operation["resourceId"]) + path = operation["resourcePath"] + if path != f"/shared-services/{identifier}": + raise ValueError("Accepted CycleCloud resource has an unexpected path") + self.path, self.identifier = path, identifier + # Own the accepted resource before evidence writes or client closure. + record_resource(self.payload, operation, "POST") + resource, operations = await self.wait_terminal() + if resource["deploymentStatus"] != "deployed" or any(o["status"] in FAILED for o in operations): + raise RuntimeError("CycleCloud deployment or firewall pipeline failed") + return resource + + async def wait_terminal(self, *, allow_absent=False, operation_id=None): + endpoint = get_full_endpoint(f"/api{self.path}") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get(endpoint, headers=await self.headers()) + if response.status_code == 404 and allow_absent: + return None + if response.status_code != 200: + raise RuntimeError(f"Could not inspect CycleCloud resource: HTTP {response.status_code}") + resource = self.validate_owner(response.json()) + response = await client.get(endpoint + "/operations", headers=await self.headers()) + if response.status_code != 200: + raise RuntimeError(f"Could not inspect CycleCloud operations: HTTP {response.status_code}") + operations = response.json()["operations"] + if not isinstance(operations, list): + raise ValueError("CycleCloud operations must be a list") + for operation in operations: + uuid_value(operation["id"]) + record_operation( + f"/api{self.path}/operations/{operation['id']}", + operation["status"], + operation["status"] in TERMINAL, + ) + expected_seen = operation_id is None or any(o["id"] == operation_id for o in operations) + if ( + expected_seen + and operations + and resource["deploymentStatus"] in TERMINAL + and all(o["status"] in TERMINAL for o in operations) + ): + return resource, operations + await asyncio.sleep(15) + + async def change(self, *, action=None, overview=None): + if action not in (None, "start", "stop") or (action is not None) == (overview is not None): + raise ValueError("Select one supported CycleCloud action or a metadata update") + resource, _ = await self.wait_terminal() + endpoint = get_full_endpoint(f"/api{self.path}") + headers = await self.headers() + headers["etag"] = resource["_etag"] + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + if action: + response = await client.post(endpoint + "/invoke-action", headers=headers, params={"action": action}) + else: + response = await client.patch(endpoint, headers=headers, json={"properties": {"overview": overview}}) + if response.status_code != 202: + raise RuntimeError(f"CycleCloud mutation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + if operation["resourcePath"] != self.path or operation["resourceId"] != self.identifier: + raise ValueError("CycleCloud mutation returned a different resource") + operation_id = uuid_value(operation["id"]) + record_resource(None, operation, "POST" if action else "PATCH") + result, operations = await self.wait_terminal(operation_id=operation_id) + observed = next(o for o in operations if o["id"] == operation_id) + if observed["status"] in FAILED or result["deploymentStatus"] in FAILED: + raise RuntimeError("CycleCloud mutation or its pipeline failed") + if overview is not None and result["properties"].get("overview") != overview: + raise AssertionError("CycleCloud metadata update was not preserved") + return result + + async def close(self): + if self.path is None: + return [] + try: + async with cleanup_deadline(self.finish - asyncio.get_running_loop().time()): + if await self.wait_terminal(allow_absent=True) is not None: + await self.before_remove(self.identifier) + await disable_and_delete_resource( + f"/api{self.path}", await get_admin_token(self.verify), self.verify, allow_failed_disable=True + ) + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get( + get_full_endpoint(f"/api{self.path}"), headers=await self.headers() + ) + if response.status_code == 404: + break + if response.status_code != 200: + raise RuntimeError(f"Could not verify CycleCloud deletion: HTTP {response.status_code}") + self.validate_owner(response.json()) + await asyncio.sleep(10) + await self.after_remove(self.identifier) + except (Exception, asyncio.CancelledError) as error: + error.add_note(f"CycleCloud cleanup failed for {self.path}") + return [error] + return [] + + +@asynccontextmanager +async def cyclecloud_lifecycle(verify, payload, before_remove, after_remove): + work_finish, finish = lifecycle_deadlines() + resource = CycleCloudService(verify, finish, payload, before_remove, after_remove) + original = None + try: + async with asyncio.timeout_at(work_finish): + yield resource + except BaseException as error: + original = error + raise + finally: + cleanup = asyncio.create_task(resource.close()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + failures = cleanup.result() + failure = original if original is not None else cancelled + if failure is not None: + for error in failures: + failure.add_note(f"CycleCloud cleanup also failed: {error!r}; {error.__notes__}") + elif failures: + raise BaseExceptionGroup("CycleCloud cleanup failed", failures) + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/test_cyclecloud.py b/e2e_tests/test_cyclecloud.py new file mode 100644 index 000000000..b1b047969 --- /dev/null +++ b/e2e_tests/test_cyclecloud.py @@ -0,0 +1,120 @@ +"""Validate one CycleCloud server without configuring or creating HPC clusters.""" + +from uuid import uuid4 + +import pytest + +from e2e_tests.bundle_evidence import record_deployed_resource, record_reused_resource +from e2e_tests.helpers import get_admin_token +from e2e_tests.resources.cyclecloud import ( + COMPUTE_API, + FIREWALL, + TEMPLATE, + assert_removed, + firewall_rules, + guard_owned, + preflight, + require_settings, + resource_ids, + uuid_value, + validate_firewall, + validate_resources, + wait_power, +) +from e2e_tests.resources.cyclecloud_lifecycle import cyclecloud_lifecycle +from e2e_tests.resources.resource import get_resource +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.run_bundle import load_catalog + + +async def prerequisites(verify): + catalog = load_catalog() + token = await get_admin_token(verify) + for name in (TEMPLATE, FIREWALL): + template = await get_resource(f"/api/shared-service-templates/{name}", token, verify) + if template["name"] != name or template["version"] != catalog[name]["source_version"]: + raise ValueError(f"Register the checkout version of {name} before CycleCloud validation") + if name == TEMPLATE and not {"start", "stop"} <= {a["name"] for a in template.get("customActions", [])}: + raise ValueError("The registered CycleCloud template must expose start and stop") + services = (await get_resource("/api/shared-services", token, verify))["sharedServices"] + if any(s["templateName"] == TEMPLATE for s in services): + raise ValueError("An existing CycleCloud service prevents isolated validation") + firewalls = [s for s in services if s["templateName"] == FIREWALL] + if len(firewalls) != 1: + raise ValueError("CycleCloud requires exactly one deployed firewall") + firewall = firewalls[0] + if ( + firewall["isEnabled"] is not True + or firewall["deploymentStatus"] not in ("deployed", "updated") + or firewall["templateVersion"] != catalog[FIREWALL]["source_version"] + ): + raise ValueError("Deploy and enable the checkout firewall version before CycleCloud validation") + record_reused_resource(firewall) + return firewall + + +async def validate_lifecycle(verify): + settings = require_settings() + payload = { + "templateName": TEMPLATE, + "properties": { + "display_name": f"CycleCloud validation {uuid4()}", + "description": "Owned CycleCloud lifecycle validation", + }, + } + core = None + baseline = None + firewall = None + principal = None + + async with arm_client() as arm: + + async def current_firewall(): + record = ( + await get_resource(f"/api/shared-services/{firewall['id']}", await get_admin_token(verify), verify) + )["sharedService"] + if record["id"] != firewall["id"] or record["templateName"] != FIREWALL: + raise ValueError("CycleCloud firewall prerequisite identity changed") + return record + + async def before_remove(identifier): + nonlocal principal + ids = resource_ids(settings, core, identifier) + await guard_owned(arm, ids, {"tre_id": settings.tre_id, "tre_shared_service_id": identifier}) + vm = await arm.request("GET", ids["vm"], COMPUTE_API, missing_ok=True) + if vm is not None and vm.get("identity", {}).get("principalId"): + principal = uuid_value(vm["identity"]["principalId"]) + + async def after_remove(identifier): + await assert_removed(arm, resource_ids(settings, core, identifier), settings.subscription, principal) + assert firewall_rules(await current_firewall()) == baseline, "CycleCloud firewall rules were not restored" + + async with cyclecloud_lifecycle(verify, payload, before_remove, after_remove) as service: + firewall = await prerequisites(verify) + baseline = firewall_rules(firewall) + core = await preflight(arm, settings) + record = await service.create() + ids = resource_ids(settings, core, service.identifier) + + async def inspect(record): + assert record["templateVersion"] == load_catalog()[TEMPLATE]["source_version"] + assert record["properties"]["connection_uri"] == "https://" + core["zone"].rsplit("/", 1)[1] + record_deployed_resource(record, service.path) + identity = await validate_resources(arm, settings, core, ids, service.identifier) + validate_firewall(await current_firewall(), baseline, service.identifier, core["prefixes"]) + return identity + + principal = await inspect(record) + await wait_power(arm, ids["vm"], "running") + await service.change(action="stop") + await wait_power(arm, ids["vm"], "deallocated") + await service.change(action="start") + await wait_power(arm, ids["vm"], "running") + updated = await service.change(overview="CycleCloud lifecycle metadata upgrade") + assert await inspect(updated) == principal, "CycleCloud upgrade replaced the VM identity" + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.cyclecloud_validation +async def test_cyclecloud_lifecycle(verify): + await validate_lifecycle(verify) diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index c38fd4dc8..ef7bd04ea 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -209,8 +209,7 @@ async def test_patch_firewall(verify): # Tested in test_create_certs_nexus_shared_service # strings.NEXUS_SHARED_SERVICE, strings.AIRLOCK_NOTIFIER_SHARED_SERVICE, - # TODO: fix cyclecloud and enable this - # strings.CYCLECLOUD_SHARED_SERVICE, + # CycleCloud has a separate opt-in lifecycle in test_cyclecloud.py. ] create_airlock_notifier_properties = { diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index 86a40e230..9659e573b 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -151,6 +151,15 @@ def test_base_workspace_collects_its_own_case_without_reusing_a_workspace(self): self.assertEqual(report["resources"], []) self.assertEqual(report["declared_prerequisites"], {}) + def test_cyclecloud_selects_only_its_server_case_with_firewall_dependency(self): + result, report = self.run_selector("--bundle", "tre-shared-service-cyclecloud", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_cyclecloud.py::test_cyclecloud_lifecycle"]) + self.assertEqual(set(report["declared_prerequisites"]), {"tre-shared-service-firewall"}) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + def test_unknown_partial_and_shell_like_identifiers_fail_before_collection(self): for identifier in ("azuresql", "tre-nonexistent", SQL + "; echo unexpected", SQL + "$(echo unexpected)"): with self.subTest(identifier=identifier): diff --git a/e2e_tests/unit_tests/test_cyclecloud_configuration.py b/e2e_tests/unit_tests/test_cyclecloud_configuration.py new file mode 100644 index 000000000..86f2703b1 --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_configuration.py @@ -0,0 +1,221 @@ +"""Check CycleCloud evidence against private infrastructure and prerequisite cases.""" + +from copy import deepcopy +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +import yaml + +from e2e_tests.resources import cyclecloud as cc + +SUB = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +PRINCIPAL = "33333333-3333-4333-8333-333333333333" +ROOT = Path(__file__).resolve().parents[2] + + +def fixture(): + settings = cc.Settings(SUB, "tre-test") + net = settings.group + "/providers/Microsoft.Network/virtualNetworks/vnet-tre-test" + core = { + "vnet": net, + "subnet": net + "/subnets/SharedSubnet", + "prefixes": ["10.0.1.0/24"], + "zone": settings.group + + "/providers/Microsoft.Network/privateDnsZones/cyclecloud-tre-test.switzerlandnorth.cloudapp.azure.com", + "blob_zone": settings.group + "/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net", + "location": "switzerlandnorth", + } + ids = cc.resource_ids(settings, core, SERVICE) + records = { + value: {"id": value, "tags": {"tre_id": settings.tre_id, "tre_shared_service_id": SERVICE}} + for value in ids.values() + } + records[ids["vm"]].update( + identity={"type": "SystemAssigned", "principalId": PRINCIPAL}, + plan={ + "name": "cyclecloud8", + "product": "azure-cyclecloud", + "publisher": "azurecyclecloud", + "promotionCode": None, + }, + properties={ + "provisioningState": "Succeeded", + "hardwareProfile": {"vmSize": "Standard_DS3_v2"}, + "storageProfile": { + "imageReference": { + "publisher": "azurecyclecloud", + "offer": "azure-cyclecloud", + "sku": "cyclecloud8", + "version": "8.8.0", + }, + "osDisk": {"managedDisk": {"id": ids["disk"]}}, + }, + "networkProfile": {"networkInterfaces": [{"id": ids["nic"]}]}, + }, + ) + records[ids["nic"]]["properties"] = { + "ipConfigurations": [{"properties": {"subnet": {"id": core["subnet"]}, "privateIPAddress": "10.0.1.5"}}] + } + records[ids["zone"] + "/A/@"] = {"properties": {"aRecords": [{"ipv4Address": "10.0.1.5"}]}} + records[ids["zone"] + "/virtualNetworkLinks/cyclecloudlink-core"] = {"properties": {"virtualNetwork": {"id": net}}} + records[ids["storage"]].update( + sku={"name": "Standard_GRS"}, + properties={"encryption": {"requireInfrastructureEncryption": True}, "allowCrossTenantReplication": False}, + ) + records[ids["endpoint"]]["properties"] = { + "subnet": {"id": core["subnet"]}, + "privateLinkServiceConnections": [ + { + "properties": { + "privateLinkServiceId": ids["storage"], + "groupIds": ["blob"], + "privateLinkServiceConnectionState": {"status": "Approved"}, + } + } + ], + } + records[ids["endpoint"] + "/privateDnsZoneGroups/private-dns-zone-group"] = { + "properties": {"privateDnsZoneConfigs": [{"properties": {"privateDnsZoneId": core["blob_zone"]}}]} + } + records[f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments"] = { + "value": [ + { + "properties": { + "principalId": PRINCIPAL, + "scope": f"/subscriptions/{SUB}", + "roleDefinitionId": f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleDefinitions/{cc.CONTRIBUTOR}", + } + } + ] + } + return settings, core, ids, records + + +class ConfigurationTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.settings, self.core, self.ids, self.records = fixture() + + async def request(method, identifier, version, **kwargs): + self.assertEqual(method, "GET") + if kwargs.get("missing_ok"): + return self.records.get(identifier) + return self.records[identifier] + + self.arm = SimpleNamespace(request=AsyncMock(side_effect=request)) + + async def test_private_resources_and_subscription_role_match_the_bundle(self): + self.assertEqual(await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE), PRINCIPAL) + self.assertEqual(self.ids["storage"].rsplit("/", 1)[1], "stgcctretest2222") + + async def test_public_ip_wrong_subnet_dns_target_or_role_is_rejected(self): + for defect in ("public_ip", "subnet", "dns", "role", "tags", "endpoint"): + with self.subTest(defect=defect): + _, _, _, self.records = fixture() + ip = self.records[self.ids["nic"]]["properties"]["ipConfigurations"][0]["properties"] + if defect == "public_ip": + ip["publicIPAddress"] = {"id": "public"} + if defect == "subnet": + ip["subnet"]["id"] = "other" + if defect == "dns": + self.records[self.ids["zone"] + "/A/@"]["properties"]["aRecords"] = [] + if defect == "tags": + self.records[self.ids["vm"]]["tags"]["tre_shared_service_id"] = "other" + if defect == "endpoint": + self.records[self.ids["endpoint"]]["properties"]["privateLinkServiceConnections"][0]["properties"][ + "privateLinkServiceId" + ] = "other" + if defect == "role": + self.records[f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments"][ + "value" + ] = [] + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE) + + async def test_cleanup_refuses_foreign_tags_before_any_write(self): + self.records[self.ids["storage"]]["tags"]["tre_shared_service_id"] = "other" + with self.assertRaisesRegex(AssertionError, "ownership"): + await cc.guard_owned(self.arm, self.ids, {"tre_id": self.settings.tre_id, "tre_shared_service_id": SERVICE}) + + async def test_disk_cleanup_uses_the_disk_api_and_role_removal_is_checked(self): + self.records = {} + role_endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" + self.records[role_endpoint] = {"value": [{"id": "role"}]} + + async def replicate_removal(seconds): + self.records[role_endpoint] = {"value": []} + + with patch.object(cc.asyncio, "sleep", AsyncMock(side_effect=replicate_removal)) as sleep: + await cc.assert_removed(self.arm, self.ids, SUB, PRINCIPAL) + sleep.assert_awaited_once() + self.arm.request.assert_any_await("GET", self.ids["disk"], cc.DISK_API, missing_ok=True) + + async def test_role_pagination_is_complete_and_cannot_change_authority(self): + endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" + self.arm.request.side_effect = [ + { + "value": [1], + "nextLink": f"https://management.azure.com{endpoint}?api-version=2022-04-01&$skiptoken=second", + }, + {"value": [2]}, + ] + self.assertEqual(await cc.principal_roles(self.arm, SUB, PRINCIPAL), [1, 2]) + self.arm.request.side_effect = [{"value": [], "nextLink": f"https://other.example{endpoint}"}] + with self.assertRaisesRegex(ValueError, "continuation"): + await cc.principal_roles(self.arm, SUB, PRINCIPAL) + + async def test_terms_and_image_failures_happen_without_writes(self): + self.records[self.settings.group] = { + "id": self.settings.group, + "tags": {"tre_id": self.settings.tre_id}, + "location": self.core["location"], + } + self.records[self.core["subnet"]] = {"id": self.core["subnet"], "properties": {"addressPrefix": "10.0.1.0/24"}} + self.records[self.settings.group + "/providers/Microsoft.Network/publicIPAddresses/pip-agw-tre-test"] = { + "properties": {"dnsSettings": {"fqdn": "tre-test.switzerlandnorth.cloudapp.azure.com"}} + } + self.records[self.core["blob_zone"]] = {"id": self.core["blob_zone"]} + del self.records[self.core["zone"]] + agreement = f"/subscriptions/{SUB}/providers/Microsoft.MarketplaceOrdering/offerTypes/virtualmachine/publishers/azurecyclecloud/offers/azure-cyclecloud/plans/cyclecloud8/agreements/current" + images = f"/subscriptions/{SUB}/providers/Microsoft.Compute/locations/switzerlandnorth/publishers/azurecyclecloud/artifacttypes/vmimage/offers/azure-cyclecloud/skus/cyclecloud8/versions" + self.records[agreement] = {"properties": {"accepted": False}} + with self.assertRaisesRegex(ValueError, "terms"): + await cc.preflight(self.arm, self.settings) + self.records[agreement]["properties"]["accepted"] = True + self.records[images] = [] + with self.assertRaisesRegex(ValueError, "image is unavailable"): + await cc.preflight(self.arm, self.settings) + self.records[images] = [{"name": "8.8.0"}] + self.assertEqual(await cc.preflight(self.arm, self.settings), self.core) + + +class FirewallTests(unittest.TestCase): + def test_only_owned_rules_are_added_and_unrelated_rules_are_preserved(self): + baseline = {"rule_collections": {"keep": {"name": "keep"}}, "network_rule_collections": {}} + schema = json.loads((ROOT / "templates/shared_services/cyclecloud/template_schema.json").read_text()) + properties = {key: list(value.values()) for key, value in deepcopy(baseline).items()} + for prop in schema["pipeline"]["install"][1]["properties"]: + value = prop["value"] + value["name"] = value["name"].replace("{{ resource.id }}", SERVICE) + value["rules"][0]["source_addresses"] = ["10.0.1.0/24"] + properties[prop["name"]].append(value) + cc.validate_firewall({"properties": properties}, baseline, SERVICE, ["10.0.1.0/24"]) + properties["rule_collections"][0]["changed"] = True + with self.assertRaisesRegex(AssertionError, "Unrelated"): + cc.validate_firewall({"properties": properties}, baseline, SERVICE, ["10.0.1.0/24"]) + + def test_deadline_precedes_container_and_smoke_is_a_dependency(self): + jobs = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text())["jobs"] + job = jobs["e2e_tests_bundle"] + self.assertIn("e2e_tests_smoke", job["needs"]) + reserve = next(s for s in job["steps"] if "CYCLECLOUD_VALIDATION_DEADLINE" in s.get("run", "")) + run = next(s for s in job["steps"] if s["name"] == "Run E2E Tests") + self.assertLess(job["steps"].index(reserve), job["steps"].index(run)) + self.assertEqual(reserve["if"], "inputs.e2eBundle == 'tre-shared-service-cyclecloud'") + self.assertIn( + "CYCLECLOUD_VALIDATION_DEADLINE=${{ env.CYCLECLOUD_VALIDATION_DEADLINE }}", run["with"]["COMMAND"] + ) + self.assertGreater(job["timeout-minutes"], 270) diff --git a/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py new file mode 100644 index 000000000..fe9e49759 --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py @@ -0,0 +1,320 @@ +"""Exercise accepted ownership, pipeline waits, deadlines and cleanup failures.""" + +import asyncio +from builtins import ExceptionGroup +from copy import deepcopy +import os +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests.resources import cyclecloud_lifecycle as lc +from e2e_tests import test_cyclecloud as case + +SERVICE = "22222222-2222-4222-8222-222222222222" +OPERATION = "33333333-3333-4333-8333-333333333333" +OLD_OPERATION = "44444444-4444-4444-8444-444444444444" +PATH = f"/shared-services/{SERVICE}" +PAYLOAD = {"templateName": lc.TEMPLATE, "properties": {"display_name": "unique test", "description": "owned test"}} + + +def response(body=None, code=200): + return SimpleNamespace(status_code=code, json=lambda: body) + + +def accepted(**changes): + return response({"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH, **changes}}, 202) + + +def resource(state="deployed", **properties): + return { + "id": SERVICE, + "templateName": lc.TEMPLATE, + "deploymentStatus": state, + "_etag": "etag", + "properties": PAYLOAD["properties"] | properties, + } + + +def operations(state="deployed", identifier=OPERATION): + return response({"operations": [{"id": identifier, "status": state}]}) + + +class LifecycleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.before, self.after = AsyncMock(), AsyncMock() + self.service = lc.CycleCloudService( + True, asyncio.get_running_loop().time() + 2, deepcopy(PAYLOAD), self.before, self.after + ) + self.client = AsyncMock() + self.client.__aenter__.return_value = self.client + self.client.post.return_value = accepted() + self.enterContext(patch.object(lc, "AsyncClient", return_value=self.client)) + self.enterContext(patch.object(lc, "get_admin_token", AsyncMock(return_value="token"))) + self.evidence = self.enterContext(patch.object(lc, "record_resource")) + self.enterContext(patch.object(lc, "record_operation")) + self.enterContext(patch.object(lc, "get_full_endpoint", side_effect=lambda v: "https://tre" + v)) + self.remove = self.enterContext(patch.object(lc, "disable_and_delete_resource", AsyncMock())) + + def own(self): + self.service.path, self.service.identifier = PATH, SERVICE + + async def test_accepted_resource_is_owned_before_client_exit_cancellation(self): + self.client.__aexit__.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await self.service.create() + self.assertEqual((self.service.path, self.service.identifier), (PATH, SERVICE)) + self.evidence.assert_called_once() + self.client.__aexit__.side_effect = None + self.client.get.return_value = response(code=404) + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=(resource(), []))): + self.assertEqual(await self.service.close(), []) + self.remove.assert_awaited_once() + self.before.assert_awaited_once_with(SERVICE) + self.after.assert_awaited_once_with(SERVICE) + + async def test_evidence_failure_cannot_lose_accepted_identity(self): + self.evidence.side_effect = OSError("evidence disk full") + with self.assertRaises(OSError): + await self.service.create() + self.assertEqual(self.service.path, PATH) + + async def test_invalid_accepted_identity_is_not_owned(self): + for changes in ({"resourcePath": "/other"}, {"resourceId": "not-uuid"}): + self.client.post.return_value = accepted(**changes) + with self.assertRaises(ValueError): + await self.service.create() + self.assertIsNone(self.service.path) + self.evidence.assert_not_called() + + async def test_failed_firewall_pipeline_remains_owned(self): + with patch.object( + self.service, "wait_terminal", AsyncMock(return_value=(resource(), [{"status": "pipeline_failed"}])) + ): + with self.assertRaisesRegex(RuntimeError, "pipeline"): + await self.service.create() + self.assertEqual(self.service.path, PATH) + + async def test_terminal_wait_requires_current_operation_and_pipeline_completion(self): + self.own() + self.client.get.side_effect = [ + response({"sharedService": resource()}), + operations(identifier=OLD_OPERATION), + response({"sharedService": resource()}), + operations("updating"), + response({"sharedService": resource("updated")}), + operations("pipeline_succeeded"), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + result, _ = await self.service.wait_terminal(operation_id=OPERATION) + self.assertEqual(result["deploymentStatus"], "updated") + self.assertEqual(sleep.await_count, 2) + + async def test_wait_rejects_changed_ownership(self): + self.own() + self.client.get.return_value = response({"sharedService": resource(display_name="another test")}) + with self.assertRaisesRegex(ValueError, "ownership"): + await self.service.wait_terminal() + self.remove.assert_not_awaited() + + async def test_action_uses_etag_and_waits_for_returned_operation(self): + self.own() + with patch.object( + self.service, + "wait_terminal", + AsyncMock( + side_effect=[ + (resource(), []), + (resource("action_succeeded"), [{"id": OPERATION, "status": "action_succeeded"}]), + ] + ), + ) as wait: + await self.service.change(action="stop") + self.client.post.assert_awaited_once_with( + "https://tre/api" + PATH + "/invoke-action", + headers={"Authorization": "Bearer token", "etag": "etag"}, + params={"action": "stop"}, + ) + self.assertEqual(wait.await_args.kwargs, {"operation_id": OPERATION}) + + async def test_metadata_update_must_be_preserved(self): + self.own() + self.client.patch.return_value = accepted() + with patch.object( + self.service, + "wait_terminal", + AsyncMock(side_effect=[(resource(), []), (resource("updated"), [{"id": OPERATION, "status": "updated"}])]), + ): + with self.assertRaisesRegex(AssertionError, "not preserved"): + await self.service.change(overview="new text") + self.assertEqual(self.client.patch.await_args.kwargs["json"], {"properties": {"overview": "new text"}}) + + async def test_cleanup_waits_then_guards_then_deletes_and_verifies(self): + self.own() + order = [] + + async def wait(**kwargs): + order.append("wait") + self.assertTrue(kwargs["allow_absent"]) + return resource("deployment_failed"), [] + + async def before(identifier): + order.append("guard") + + async def remove(*args, **kwargs): + order.append("delete") + + async def after(identifier): + order.append("verify") + + self.before.side_effect, self.remove.side_effect, self.after.side_effect = before, remove, after + self.client.get.return_value = response(code=404) + with patch.object(self.service, "wait_terminal", AsyncMock(side_effect=wait)): + self.assertEqual(await self.service.close(), []) + self.assertEqual(order, ["wait", "guard", "delete", "verify"]) + self.assertTrue(self.remove.await_args.kwargs["allow_failed_disable"]) + + async def test_cleanup_guard_failure_prevents_delete(self): + self.own() + self.before.side_effect = ValueError("foreign tag") + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=(resource(), []))): + failures = await self.service.close() + self.assertIsInstance(failures[0], ValueError) + self.remove.assert_not_awaited() + self.after.assert_not_awaited() + + async def test_cleanup_after_api_absence_still_checks_azure(self): + self.own() + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=None)): + self.assertEqual(await self.service.close(), []) + self.remove.assert_not_awaited() + self.after.assert_awaited_once_with(SERVICE) + + async def test_cleanup_deadline_bounds_stalled_pipeline_without_delete(self): + self.own() + self.service.finish = asyncio.get_running_loop().time() + 0.02 + + async def wait(**kwargs): + await asyncio.sleep(10) + + with patch.object(self.service, "wait_terminal", side_effect=wait): + failures = await self.service.close() + self.assertIsInstance(failures[0], TimeoutError) + self.remove.assert_not_awaited() + + +class DeadlineTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE="")) + + async def test_absolute_deadline_counts_container_setup_and_caps_extension(self): + for deadline, seconds in (("10000", 9000), ("1000000", lc.LIFECYCLE_SECONDS)): + with ( + patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE=deadline), + patch.object(lc.time, "time", return_value=1000), + ): + work, finish = lc.lifecycle_deadlines() + self.assertAlmostEqual(finish - asyncio.get_running_loop().time(), seconds, delta=1) + self.assertEqual(finish - work, lc.CLEANUP_RESERVE_SECONDS) + + async def test_invalid_or_exhausted_deadline_prevents_provisioning(self): + for value in ("nan", "inf", "-inf", "invalid", "0"): + with self.subTest(value=value), patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE=value): + with self.assertRaises((ValueError, TimeoutError)): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + self.fail("Invalid deadline reached provisioning") + + async def test_work_timeout_still_runs_cleanup(self): + now = asyncio.get_running_loop().time() + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 0.02, now + 1)), + patch.object(lc.CycleCloudService, "close", AsyncMock(return_value=[])) as close, + ): + with self.assertRaises(TimeoutError): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + await asyncio.sleep(10) + close.assert_awaited_once() + + async def test_cleanup_failure_preserves_original_or_fails_successful_case(self): + cleanup_error = RuntimeError("cleanup failed") + cleanup_error.add_note(PATH) + original = ValueError("validation failed") + with patch.object(lc.CycleCloudService, "close", AsyncMock(return_value=[cleanup_error])): + with self.assertRaises(ValueError) as raised: + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + raise original + self.assertIs(raised.exception, original) + self.assertIn(PATH, str(original.__notes__)) + self.assertIn("cleanup failed", str(original.__notes__)) + with self.assertRaises(ExceptionGroup): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + pass + + async def test_repeated_cancellation_waits_for_cleanup(self): + entered, cleaning, release = (asyncio.Event() for _ in range(3)) + + async def close(): + cleaning.set() + await release.wait() + return [] + + async def run(): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + entered.set() + await asyncio.sleep(10) + + with patch.object(lc.CycleCloudService, "close", side_effect=close): + task = asyncio.create_task(run()) + await entered.wait() + task.cancel() + await cleaning.wait() + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await task + + +class PrerequisiteTests(unittest.IsolatedAsyncioTestCase): + async def test_wrong_version_missing_actions_existing_service_or_disabled_firewall_prevents_creation(self): + catalog = case.load_catalog() + template = { + "name": lc.TEMPLATE, + "version": catalog[lc.TEMPLATE]["source_version"], + "customActions": [{"name": "start"}, {"name": "stop"}], + } + firewall_template = {"name": case.FIREWALL, "version": catalog[case.FIREWALL]["source_version"]} + firewall = { + "id": SERVICE, + "templateName": case.FIREWALL, + "templateVersion": firewall_template["version"], + "isEnabled": True, + "deploymentStatus": "deployed", + } + for defect in ("none", "version", "actions", "existing", "disabled"): + with self.subTest(defect=defect): + first, deployed = deepcopy(template), deepcopy(firewall) + services = [deployed] + if defect == "version": + first["version"] = "0.0.0" + if defect == "actions": + first["customActions"] = [] + if defect == "existing": + services.append({"templateName": lc.TEMPLATE}) + if defect == "disabled": + deployed["isEnabled"] = False + with ( + patch.object(case, "get_admin_token", AsyncMock()), + patch.object(case, "record_reused_resource"), + patch.object( + case, + "get_resource", + AsyncMock(side_effect=[first, firewall_template, {"sharedServices": services}]), + ), + ): + if defect == "none": + self.assertEqual(await case.prerequisites(True), deployed) + else: + with self.assertRaises(ValueError): + await case.prerequisites(True) From d7bd64ade2ea98f7b1ca3007cc064a0cf47256f1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 22 Oct 2026 22:52:43 +0100 Subject: [PATCH 04/24] Deploy new CycleCloud servers with the Standard_D4s_v5 VM size The CycleCloud bundle set the server VM size to Standard_DS3_v2. Azure does not offer this size to all subscriptions in all regions. In a live run in another fork, the server failed to deploy in Switzerland North with SkuNotAvailable, and the SKU list shows NotAvailableForSubscription for this size in that region. New servers now use Standard_D4s_v5. This size also has 4 vCPUs, it supports premium disks, and it accepts the Generation 1 CycleCloud image. Terraform ignores changes to the VM size, so an upgrade does not resize a server that an earlier version made. --- CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 2 +- docs/tre-templates/shared-services/cyclecloud.md | 2 ++ e2e_tests/bundle_coverage.json | 2 +- e2e_tests/resources/cyclecloud.py | 2 +- e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py | 8 ++++++++ e2e_tests/unit_tests/test_cyclecloud_configuration.py | 2 +- .../shared_services/cyclecloud/terraform/cyclecloud.tf | 5 +++-- 8 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39308a30b..54315fc1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ BUG FIXES: * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index cd332411e..f338ce3e2 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -250,7 +250,7 @@ Use a dedicated AzureCloud environment with one writer and no existing CycleClou Publish and register the checkout firewall and CycleCloud bundles. Deploy the firewall before validation. Register the repaired CycleCloud bundle, version 0.7.10 or later, with `start` and `stop` actions. Accept the CycleCloud server image terms separately before the run. The test reads the agreement and regional image list. It does not accept terms. -Check `Standard_DS3_v2` capacity and quota in the core region before deployment. Image availability does not prove VM capacity. +Check `Standard_D4s_v5` capacity and quota in the core region before deployment. Image availability does not prove VM capacity. The case creates one server and checks its private VM address, DNS configuration, blob private endpoint, managed identity and firewall rules. It invokes stop/start through TRE and verifies the VM power states. It changes the overview to exercise a same-version Terraform upgrade and checks that the VM identity remains stable. diff --git a/docs/tre-templates/shared-services/cyclecloud.md b/docs/tre-templates/shared-services/cyclecloud.md index 147d13635..2b1048a4e 100644 --- a/docs/tre-templates/shared-services/cyclecloud.md +++ b/docs/tre-templates/shared-services/cyclecloud.md @@ -23,6 +23,8 @@ az vm image terms accept --urn almalinux:almalinux-hpc:8_5-hpc:latest Deploy the CycleCloud server using UI or API. +The CycleCloud server uses the `Standard_D4s_v5` VM size. Before you deploy, make sure that your subscription can use this size in the TRE region. An upgrade from a version before 0.7.10 keeps the size of the existing server. + To connect to the CycleCloud server, the TRE Administrator must connect to the CycleCloud server from the administration jumpbox. Use Azure Bastion to connect to the jumpbox a with the username `admin` and the select the password located in your core KeyVault. Connect to the CycleCloud server at the URL: `https://cyclecloud-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`. - Provide a name for the cyclecloud server instance. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 6fe874c72..0c2940e11 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -88,7 +88,7 @@ "configuration" ], "limitations": [ - "Requires AzureCloud, accepted CycleCloud marketplace image terms, the regional image and Standard_DS3_v2 capacity. The test does not accept terms or change quotas.", + "Requires AzureCloud, accepted CycleCloud marketplace image terms, the regional image and Standard_D4s_v5 capacity. The test does not accept terms or change quotas.", "Requires an empty CycleCloud slot, current firewall and one writer in a dedicated environment. The fixed DNS zone prevents parallel CycleCloud services.", "Checks VM power actions, ARM configuration, TRE firewall rules, same-version metadata upgrade and bounded cleanup. Previous-version upgrades remain unproven.", "Does not configure the CycleCloud application, create HPC clusters or prove browser access, private DNS reachability or application TLS trust.", diff --git a/e2e_tests/resources/cyclecloud.py b/e2e_tests/resources/cyclecloud.py index 436a6dedb..0b3ed30aa 100644 --- a/e2e_tests/resources/cyclecloud.py +++ b/e2e_tests/resources/cyclecloud.py @@ -192,7 +192,7 @@ async def validate_resources(arm, settings, core, ids, identifier): vm = records["vm"] properties = vm["properties"] assert properties["provisioningState"] == "Succeeded" - assert properties["hardwareProfile"]["vmSize"] == "Standard_DS3_v2" + assert properties["hardwareProfile"]["vmSize"] == "Standard_D4s_v5" image = properties["storageProfile"]["imageReference"] assert (image["publisher"], image["offer"], image["sku"]) == ("azurecyclecloud", "azure-cyclecloud", "cyclecloud8") assert all( diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py index 1127f5b3b..0f686908a 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -30,3 +30,11 @@ def test_vm_actions_have_a_declared_and_emitted_resource_id(self): ) vm_source = (BUNDLE / "terraform/cyclecloud.tf").read_text() self.assertIsNotNone(re.search(r'resource "azurerm_virtual_machine" "cyclecloud"', vm_source)) + + def test_new_servers_use_an_offered_size_and_upgrades_keep_the_size(self): + source = (BUNDLE / "terraform/cyclecloud.tf").read_text() + vm = source.split('resource "azurerm_virtual_machine" "cyclecloud" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(vm, r'\n vm_size\s*=\s*"Standard_D4s_v5"\n') + ignored = re.search(r"lifecycle\s*\{\s*ignore_changes\s*=\s*\[([^\]]*)\]", vm) + self.assertIsNotNone(ignored) + self.assertIn("vm_size", [name.strip() for name in ignored.group(1).split(",")]) diff --git a/e2e_tests/unit_tests/test_cyclecloud_configuration.py b/e2e_tests/unit_tests/test_cyclecloud_configuration.py index 86f2703b1..104e3352a 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_configuration.py +++ b/e2e_tests/unit_tests/test_cyclecloud_configuration.py @@ -44,7 +44,7 @@ def fixture(): }, properties={ "provisioningState": "Succeeded", - "hardwareProfile": {"vmSize": "Standard_DS3_v2"}, + "hardwareProfile": {"vmSize": "Standard_D4s_v5"}, "storageProfile": { "imageReference": { "publisher": "azurecyclecloud", diff --git a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf index 7d58b39a6..c5c12b469 100644 --- a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf +++ b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf @@ -29,7 +29,7 @@ resource "azurerm_virtual_machine" "cyclecloud" { location = data.azurerm_resource_group.rg.location resource_group_name = data.azurerm_resource_group.rg.name network_interface_ids = [azurerm_network_interface.cyclecloud.id] - vm_size = "Standard_DS3_v2" + vm_size = "Standard_D4s_v5" delete_os_disk_on_termination = true delete_data_disks_on_termination = true @@ -70,7 +70,8 @@ resource "azurerm_virtual_machine" "cyclecloud" { tags = local.tre_shared_service_tags - lifecycle { ignore_changes = [tags] } + # Azure does not offer Standard_DS3_v2 to all subscriptions in all regions. Keep the size of servers made before 0.7.11. + lifecycle { ignore_changes = [tags, vm_size] } } From 70e5590b35f24d7216983f06e3d245de07272d28 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 26 Oct 2026 02:25:17 +0000 Subject: [PATCH 05/24] Ignore the firewall collections of other services in the CycleCloud check (#5159) The CycleCloud case compared the full firewall with a baseline. Other services that run at the same time add and remove their own "nrc_svc_" and "arc_svc_" collections, so the case can fail in a run with other cases in parallel. The OHDSI case, which uses the same check, failed for this reason. The check now leaves out the collections of other services. All other collections must stay equal to the baseline, and the CycleCloud collections must go away after the delete. --- e2e_tests/resources/cyclecloud.py | 14 ++++++++++++-- e2e_tests/test_cyclecloud.py | 4 +++- .../unit_tests/test_cyclecloud_configuration.py | 10 ++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/e2e_tests/resources/cyclecloud.py b/e2e_tests/resources/cyclecloud.py index 0b3ed30aa..6a6d8554b 100644 --- a/e2e_tests/resources/cyclecloud.py +++ b/e2e_tests/resources/cyclecloud.py @@ -266,17 +266,27 @@ async def assert_removed(arm, ids, subscription, principal=None): await asyncio.sleep(10) -def firewall_rules(resource): +# Other services add and remove their own collections while this case runs. +SERVICE_COLLECTION = re.compile(r"(?:nrc|arc)_svc_([0-9a-f-]{36})") + + +def other_service(name, identifier): + match = SERVICE_COLLECTION.match(name) + return match is not None and match.group(1) != identifier + + +def firewall_rules(resource, identifier=None): result = {} for key in ("rule_collections", "network_rule_collections"): values = resource["properties"].get(key, []) result[key] = {v["name"]: deepcopy(v) for v in values} assert len(result[key]) == len(values), "Duplicate firewall collection names" + result[key] = {name: v for name, v in result[key].items() if not other_service(name, identifier)} return result def validate_firewall(resource, baseline, identifier, prefixes): - rules = firewall_rules(resource) + rules = firewall_rules(resource, identifier) for key, prefix in (("rule_collections", "arc"), ("network_rule_collections", "nrc")): name = f"{prefix}_svc_{identifier}_cyclecloud" collection = rules[key].pop(name) diff --git a/e2e_tests/test_cyclecloud.py b/e2e_tests/test_cyclecloud.py index b1b047969..e160f4dbd 100644 --- a/e2e_tests/test_cyclecloud.py +++ b/e2e_tests/test_cyclecloud.py @@ -87,7 +87,9 @@ async def before_remove(identifier): async def after_remove(identifier): await assert_removed(arm, resource_ids(settings, core, identifier), settings.subscription, principal) - assert firewall_rules(await current_firewall()) == baseline, "CycleCloud firewall rules were not restored" + assert firewall_rules(await current_firewall(), identifier) == baseline, ( + "CycleCloud firewall rules were not restored" + ) async with cyclecloud_lifecycle(verify, payload, before_remove, after_remove) as service: firewall = await prerequisites(verify) diff --git a/e2e_tests/unit_tests/test_cyclecloud_configuration.py b/e2e_tests/unit_tests/test_cyclecloud_configuration.py index 104e3352a..0d64c2ec0 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_configuration.py +++ b/e2e_tests/unit_tests/test_cyclecloud_configuration.py @@ -207,6 +207,16 @@ def test_only_owned_rules_are_added_and_unrelated_rules_are_preserved(self): with self.assertRaisesRegex(AssertionError, "Unrelated"): cc.validate_firewall({"properties": properties}, baseline, SERVICE, ["10.0.1.0/24"]) + def test_other_services_can_change_their_own_rules(self): + other = "55555555-5555-4555-8555-555555555555" + first = {"rule_collections": [{"name": "keep"}, {"name": f"arc_svc_{other}_health_services"}]} + baseline = cc.firewall_rules({"properties": first}) + self.assertEqual(baseline, {"rule_collections": {"keep": {"name": "keep"}}, "network_rule_collections": {}}) + later = {"rule_collections": [{"name": "keep"}], "network_rule_collections": [{"name": f"nrc_svc_{other}"}]} + self.assertEqual(cc.firewall_rules({"properties": later}, SERVICE), baseline) + later["rule_collections"].append({"name": f"arc_svc_{SERVICE}_cyclecloud"}) + self.assertNotEqual(cc.firewall_rules({"properties": later}, SERVICE), baseline) + def test_deadline_precedes_container_and_smoke_is_a_dependency(self): jobs = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text())["jobs"] job = jobs["e2e_tests_bundle"] From 0f60ce4f56935379e2e45f01f654e6716651f2fe Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 21:37:21 +0100 Subject: [PATCH 06/24] Repair OHDSI private bootstrap and firewall cleanup (#5149) --- CHANGELOG.md | 1 + .../unit_tests/test_ohdsi_bundle_contract.py | 216 ++++++++++++++++++ .../workspace_services/ohdsi/porter.yaml | 2 +- .../ohdsi/scripts/atlas_db_init.sh | 27 ++- .../ohdsi/scripts/atlas_security.sh | 71 ++++-- .../ohdsi/sql/atlas_create_roles_users.sql | 46 ++-- .../ohdsi/sql/atlas_create_schema.sql | 24 +- .../ohdsi/template_schema.json | 27 +++ .../ohdsi/terraform/atlas_database.tf | 25 +- .../ohdsi/terraform/atlas_security.tf | 1 + .../ohdsi/terraform/atlas_ui.tf | 9 +- .../ohdsi/terraform/ohdsi_web_api.tf | 9 +- 12 files changed, 368 insertions(+), 90 deletions(-) create mode 100644 e2e_tests/unit_tests/test_ohdsi_bundle_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 54315fc1b..14f10f093 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) +* Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py new file mode 100644 index 000000000..7e8be7d6c --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -0,0 +1,216 @@ +"""Check OHDSI bootstrap inputs, private access and pipeline cleanup contracts.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/workspace_services/ohdsi" + + +class BootstrapTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.folder = Path(self.tmp.name) + self.env = os.environ | { + "PATH": str(self.folder) + os.pathsep + os.environ["PATH"], + "CALLS": str(self.folder / "calls.jsonl"), + "OHDSI_ADMIN_PASSWORD": "admin'password", + "OHDSI_APP_PASSWORD": "app'password", + "OHDSI_ADMIN_USERNAME": "ohdsi_admin_user", + "OHDSI_APP_USERNAME": "ohdsi_app_user", + "OHDSI_ADMIN_ROLE": "ohdsi_admin", + "OHDSI_APP_ROLE": "ohdsi_app", + "DATABASE_NAME": "atlas_webapi_db", + "SCHEMA_NAME": "webapi", + "MAIN_CONNECTION_STRING": "main", + "OHDSI_ADMIN_CONNECTION_STRING": "admin", + "ATLAS_USERS": "admin,secret", + "WEB_API_URL": "https://webapi.example/WebAPI/", + "WEB_API_VERSION": "2.12.1", + } + self.fake( + "psql", + """import json,os,sys +from pathlib import Path +args=sys.argv[1:] +body=Path(args[args.index('-f')+1]).read_text() if '-f' in args else sys.stdin.read() +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'args':args,'body':body})+'\\n') +sys.exit(int(os.environ.get('PSQL_EXIT','0'))) +""", + ) + self.fake("envsubst", "import os,sys\nprint(os.path.expandvars(sys.stdin.read()))\n") + self.fake("md5sum", "import hashlib,sys\nprint(hashlib.md5(sys.stdin.buffer.read()).hexdigest())\n") + + def fake(self, name, body): + p = self.folder / name + p.write_text("#!" + sys.executable + "\n" + body) + p.chmod(0o755) + + def run_script(self, name, **env): + result = subprocess.run( + ["bash", str(BUNDLE / "scripts" / name)], + cwd=BUNDLE / "terraform", + env=self.env | env, + capture_output=True, + text=True, + timeout=10, + ) + calls = ( + [json.loads(line) for line in Path(self.env["CALLS"]).read_text().splitlines()] + if Path(self.env["CALLS"]).exists() + else [] + ) + return result, calls + + def security_commands(self): + self.fake( + "curl", + """import json,os,sys +from pathlib import Path +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'curl':args})+'\\n') +if args[0].endswith('/info'): + sequence=os.environ.get('INFO_VERSIONS','2.12.1').split(',') + counter=Path(os.environ['CALLS']+'.count') + count=int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(count+1)) + Path(args[args.index('--output')+1]).write_text(json.dumps({'version':sequence[min(count,len(sequence)-1)]})) + sys.exit(int(os.environ.get('CURL_EXIT','0'))) +if '--output' not in args:print('authentication-token') +""", + ) + self.fake( + "jq", + """import json,sys +from pathlib import Path +args=sys.argv[1:] +expected=args[args.index('--arg')+2] +sys.exit(0 if json.loads(Path(args[-1]).read_text()).get('version')==expected else 1) +""", + ) + self.fake("sleep", "import time\ntime.sleep(0.01)\n") + self.fake( + "timeout", + """import json,os,signal,subprocess,sys +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'timeout':args[:1]})+'\\n') +# Use a shorter test deadline while exercising the real shell loop. +process=subprocess.Popen(args[1:],start_new_session=True) +try: + sys.exit(process.wait(timeout=2)) +except subprocess.TimeoutExpired: + os.killpg(process.pid,signal.SIGKILL) + process.wait() + sys.exit(124) +""", + ) + self.fake("htpasswd", "print(':test-password-hash')\n") + + def test_security_waits_without_repeated_logins_or_token_output(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh") + self.assertEqual(result.returncode, 0, result.stderr) + requests = [c["curl"] for c in calls if "curl" in c] + self.assertEqual(len(requests), 2) + self.assertEqual(requests[0][0], "https://webapi.example/WebAPI/info") + self.assertEqual([c["timeout"] for c in calls if "timeout" in c], [["600"]]) + self.assertIn("--max-time", requests[0]) + self.assertFalse(Path(requests[0][requests[0].index("--output") + 1]).exists()) + self.assertNotIn("--retry", requests[1]) + self.assertEqual(requests[1][requests[1].index("--output") + 1], "/dev/null") + self.assertNotIn("authentication-token", result.stdout + result.stderr) + sql = [c for c in calls if "args" in c] + self.assertTrue(all("-e" not in c["args"] for c in sql)) + self.assertIn(":'password_hash'", sql[2]["body"]) + self.assertIn("NOT EXISTS", sql[3]["body"]) + self.assertIn(r"\if :role_assigned", sql[3]["body"]) + self.assertIn("RAISE EXCEPTION", sql[3]["body"]) + + def test_readiness_failure_prevents_database_changes_and_login(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", CURL_EXIT="22") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.returncode, 124) + self.assertTrue(all("args" not in c for c in calls)) + self.assertTrue(all(c["curl"][0].endswith("/info") for c in calls if "curl" in c)) + + def test_unexpected_version_prevents_database_changes_and_login(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", INFO_VERSIONS="2.11.0") + self.assertEqual(result.returncode, 124) + self.assertTrue(all("args" not in c for c in calls)) + self.assertTrue(all(c["curl"][0].endswith("/info") for c in calls if "curl" in c)) + + def test_stale_http_success_is_retried_before_security_initialisation(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", INFO_VERSIONS="2.11.0,2.12.1") + self.assertEqual(result.returncode, 0, result.stderr) + requests = [c["curl"] for c in calls if "curl" in c] + self.assertEqual([r[0].rsplit("/", 1)[1] for r in requests], ["info", "info", "db"]) + first_sql = next(i for i, c in enumerate(calls) if "args" in c) + self.assertEqual(sum("curl" in c for c in calls[:first_sql]), 2) + self.assertNotIn("--retry", requests[-1]) + + def test_passwords_use_psql_quoted_variables_without_sql_echo(self): + result, calls = self.run_script("atlas_db_init.sh") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(calls), 2) + self.assertIn("admin_password=admin'password", calls[0]["args"]) + self.assertIn("app_password=app'password", calls[0]["args"]) + self.assertIn(":'admin_password'", calls[0]["body"]) + self.assertIn(":'app_password'", calls[0]["body"]) + for call in calls: + self.assertIn("ON_ERROR_STOP=1", call["args"]) + self.assertNotIn("-e", call["args"]) + self.assertNotIn("password", result.stdout + result.stderr) + + def test_role_creation_failure_stops_before_schema_creation(self): + result, calls = self.run_script("atlas_db_init.sh", PSQL_EXIT="7") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(calls), 1) + + def test_roles_and_schema_are_safe_to_repeat(self): + roles = (BUNDLE / "sql/atlas_create_roles_users.sql").read_text() + self.assertEqual(roles.count("WHERE NOT EXISTS"), 4) + self.assertEqual(roles.count("\\gexec"), 4) + self.assertIn("CREATE SCHEMA IF NOT EXISTS", (BUNDLE / "sql/atlas_create_schema.sql").read_text()) + + +class BundleContractTests(unittest.TestCase): + def test_database_and_apps_disable_public_access(self): + for filename in ("atlas_database.tf", "atlas_ui.tf", "ohdsi_web_api.tf"): + with self.subTest(filename=filename): + source = (BUNDLE / "terraform" / filename).read_text() + self.assertRegex(source, r"public_network_access_enabled\s*=\s*false") + + def test_uninstall_removes_only_owned_firewall_collections(self): + schema = json.loads((BUNDLE / "template_schema.json").read_text()) + uninstall = schema["pipeline"]["uninstall"] + cleanup = [s for s in uninstall if s.get("resourceTemplateName") == "tre-shared-service-firewall"] + self.assertEqual(len(cleanup), 1) + self.assertEqual(cleanup[0]["resourceAction"], "upgrade") + self.assertEqual( + {p["name"]: p["value"] for p in cleanup[0]["properties"]}, + { + "network_rule_collections": {"name": "nrc_svc_{{ resource.id }}"}, + "rule_collections": {"name": "arc_svc_{{ resource.id }}"}, + }, + ) + for prop in cleanup[0]["properties"]: + self.assertEqual(prop["arraySubstitutionAction"], "remove") + self.assertEqual(prop["arrayMatchField"], "name") + self.assertLess(uninstall.index(cleanup[0]), next(i for i, s in enumerate(uninstall) if s["stepId"] == "main")) + + def test_bootstrap_version_matches_pinned_image(self): + source = (BUNDLE / "terraform/atlas_security.tf").read_text() + self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + self.assertEqual(manifest["version"], "0.3.14") diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index dba38adcd..a18290240 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.13 +version: 0.3.14 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh b/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh index b50f073fb..723f38716 100755 --- a/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh +++ b/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh @@ -3,19 +3,18 @@ set -o errexit set -o pipefail set -o nounset -admin_user_password="${OHDSI_ADMIN_PASSWORD}${OHDSI_ADMIN_USERNAME}" -app_user_password="${OHDSI_APP_PASSWORD}${OHDSI_APP_USERNAME}" -admin_md5=("md5$(echo -n "$admin_user_password" | md5sum | awk '{ print $1 }')") -export admin_md5 -app_md5=("md5$(echo -n "$app_user_password" | md5sum | awk '{ print $1 }')'") -export app_md5 +export PGCONNECT_TIMEOUT=30 +export PGOPTIONS='-c statement_timeout=600000 -c lock_timeout=30000' -printf 'Creating roles and users' -envsubst < ../sql/atlas_create_roles_users.sql | psql -v ON_ERROR_STOP=0 -e "$MAIN_CONNECTION_STRING" -printf 'Creating roles and users: done.' +printf 'Creating roles and users\n' +psql -X -v ON_ERROR_STOP=1 "$MAIN_CONNECTION_STRING" \ + -v "admin_role=$OHDSI_ADMIN_ROLE" -v "app_role=$OHDSI_APP_ROLE" \ + -v "admin_username=$OHDSI_ADMIN_USERNAME" -v "app_username=$OHDSI_APP_USERNAME" \ + -v "admin_password=$OHDSI_ADMIN_PASSWORD" -v "app_password=$OHDSI_APP_PASSWORD" \ + -v "database_name=$DATABASE_NAME" -f ../sql/atlas_create_roles_users.sql -printf 'Creating schema' -envsubst < ../sql/atlas_create_schema.sql | psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -printf 'Creating schema: done.' - -printf 'Done' +printf 'Creating schema\n' +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "schema_name=$SCHEMA_NAME" -v "admin_role=$OHDSI_ADMIN_ROLE" \ + -v "app_role=$OHDSI_APP_ROLE" -f ../sql/atlas_create_schema.sql +printf 'Database initialisation complete\n' diff --git a/templates/workspace_services/ohdsi/scripts/atlas_security.sh b/templates/workspace_services/ohdsi/scripts/atlas_security.sh index 0d15b4bc6..ebcf8c81f 100755 --- a/templates/workspace_services/ohdsi/scripts/atlas_security.sh +++ b/templates/workspace_services/ohdsi/scripts/atlas_security.sh @@ -3,27 +3,58 @@ set -o errexit set -o pipefail set -o nounset -psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -f "../sql/atlas_create_security.sql" +export PGCONNECT_TIMEOUT=30 +export PGOPTIONS='-c statement_timeout=600000 -c lock_timeout=30000' -psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -f "../sql/atlas_default_roles.sql" +# Retry only the unauthenticated readiness request. Failed logins can lock accounts. +info_file=$(mktemp) +trap 'rm -f "$info_file"' EXIT +# Expand readiness variables inside the bounded child shell. +# shellcheck disable=SC2016 +WEB_API_INFO_FILE="$info_file" timeout 600 bash -eu -o pipefail -c ' + until curl "${WEB_API_URL%/}/info" --fail --silent --show-error \ + --connect-timeout 5 --max-time 15 --output "$WEB_API_INFO_FILE" && + jq -e --arg version "$WEB_API_VERSION" ".version == \$version" "$WEB_API_INFO_FILE" > /dev/null; do + sleep 10 + done +' -count=1 -for i in ${ATLAS_USERS//,/ } -do - if [ "$(("$count" % 2))" -eq "1" ]; then - username=$i - else - # shellcheck disable=SC2016 - atlaspw=$(htpasswd -bnBC 4 "" "$i" | tr -d ':\n' | sed 's/$2y/$2a/') - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi_security.security (email,password) values ('$username', E'$atlaspw');" - # this step adds some required rows/ids in the db - curl "$WEB_API_URL/user/login/db" --data-urlencode "login=$username" --data-urlencode "password=$i" --fail +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" -f ../sql/atlas_create_security.sql +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" -f ../sql/atlas_default_roles.sql - if [ "$count" = "2" ]; then - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi.sec_user_role (user_id, role_id) values ((select id from webapi.sec_user where login='$username'),2);" #admin role - else - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi.sec_user_role (user_id, role_id) values ((select id from webapi.sec_user where login='$username'),10);" #atlas user role - fi - fi - ((count++)) +IFS=',' read -r -a users <<< "$ATLAS_USERS" +if (( ${#users[@]} == 0 || ${#users[@]} % 2 != 0 )); then + printf 'ATLAS_USERS must contain username and password pairs\n' >&2 + exit 1 +fi +for ((index=0; index<${#users[@]}; index+=2)); do + username=${users[index]} + password=${users[index+1]} + # shellcheck disable=SC2016 + atlaspw=$(htpasswd -bnBC 4 "" "$password" | tr -d ':\n' | sed 's/$2y/$2a/') + psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "username=$username" -v "password_hash=$atlaspw" <<'SQL' +INSERT INTO webapi_security.security (email, password) VALUES (:'username', :'password_hash'); +SQL + # The response can contain an authentication token. Do not write it to the log. + curl "${WEB_API_URL%/}/user/login/db" --fail --silent --show-error \ + --connect-timeout 5 --max-time 30 --output /dev/null \ + --data-urlencode "login=$username" --data-urlencode "password=$password" + + role_id=10 + if (( index == 0 )); then role_id=2; fi + psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "username=$username" -v "role_id=$role_id" <<'SQL' +INSERT INTO webapi.sec_user_role (user_id, role_id) +SELECT id, :'role_id'::integer FROM webapi.sec_user u WHERE login = :'username' +AND NOT EXISTS (SELECT FROM webapi.sec_user_role r WHERE r.user_id = u.id AND r.role_id = :'role_id'::integer); +SELECT EXISTS ( + SELECT FROM webapi.sec_user u JOIN webapi.sec_user_role r ON r.user_id = u.id + WHERE u.login = :'username' AND r.role_id = :'role_id'::integer +) AS role_assigned \gset +\if :role_assigned +\else + DO $$ BEGIN RAISE EXCEPTION 'Application user role assignment failed'; END $$; +\endif +SQL done diff --git a/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql b/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql index f77ffa515..e817e645a 100644 --- a/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql +++ b/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql @@ -1,23 +1,23 @@ -CREATE ROLE ${OHDSI_ADMIN_ROLE} CREATEDB REPLICATION VALID UNTIL 'infinity'; - -COMMENT ON ROLE ${OHDSI_ADMIN_ROLE} IS 'Administration group for OHDSI applications'; - -CREATE ROLE ${OHDSI_APP_ROLE} VALID UNTIL 'infinity'; - -COMMENT ON ROLE ${OHDSI_APP_ROLE} IS 'Application groupfor OHDSI applications'; - -CREATE ROLE ${OHDSI_ADMIN_USERNAME} LOGIN ENCRYPTED PASSWORD ${admin_md5} VALID UNTIL 'infinity'; - -GRANT ${OHDSI_ADMIN_ROLE} TO ${OHDSI_ADMIN_USERNAME}; - -COMMENT ON ROLE ${OHDSI_ADMIN_USERNAME} IS 'Admin user account for OHDSI applications'; - -CREATE ROLE ${OHDSI_APP_USERNAME} LOGIN ENCRYPTED PASSWORD ${app_md5} VALID UNTIL 'infinity'; - -GRANT ${OHDSI_APP_ROLE} TO ${OHDSI_APP_USERNAME}; - -COMMENT ON ROLE ${OHDSI_APP_USERNAME} IS 'Application user account for OHDSI applications'; - -GRANT ALL ON DATABASE ${DATABASE_NAME} TO GROUP ${OHDSI_ADMIN_ROLE}; - -GRANT CONNECT, TEMPORARY ON DATABASE ${DATABASE_NAME} TO GROUP ${OHDSI_APP_ROLE}; +-- Quote identifiers and values with psql. A retry can reuse roles already created. +SELECT format('CREATE ROLE %I CREATEDB REPLICATION', :'admin_role') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'admin_role') \gexec +COMMENT ON ROLE :"admin_role" IS 'Administration group for OHDSI applications'; + +SELECT format('CREATE ROLE %I', :'app_role') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_role') \gexec +COMMENT ON ROLE :"app_role" IS 'Application group for OHDSI applications'; + +SELECT format('CREATE ROLE %I LOGIN', :'admin_username') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'admin_username') \gexec +ALTER ROLE :"admin_username" PASSWORD :'admin_password' VALID UNTIL 'infinity'; +GRANT :"admin_role" TO :"admin_username"; +COMMENT ON ROLE :"admin_username" IS 'Admin user account for OHDSI applications'; + +SELECT format('CREATE ROLE %I LOGIN', :'app_username') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_username') \gexec +ALTER ROLE :"app_username" PASSWORD :'app_password' VALID UNTIL 'infinity'; +GRANT :"app_role" TO :"app_username"; +COMMENT ON ROLE :"app_username" IS 'Application user account for OHDSI applications'; + +GRANT ALL ON DATABASE :"database_name" TO GROUP :"admin_role"; +GRANT CONNECT, TEMPORARY ON DATABASE :"database_name" TO GROUP :"app_role"; diff --git a/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql b/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql index f6bd62d23..42fb049f6 100644 --- a/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql +++ b/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql @@ -1,24 +1,24 @@ -CREATE SCHEMA ${SCHEMA_NAME} -AUTHORIZATION ${OHDSI_ADMIN_ROLE}; +CREATE SCHEMA IF NOT EXISTS :"schema_name" +AUTHORIZATION :"admin_role"; -COMMENT ON SCHEMA ${SCHEMA_NAME} IS 'Schema containing tables to support WebAPI functionality'; +COMMENT ON SCHEMA :"schema_name" IS 'Schema containing tables to support WebAPI functionality'; -GRANT USAGE ON SCHEMA ${SCHEMA_NAME} TO PUBLIC; +GRANT USAGE ON SCHEMA :"schema_name" TO PUBLIC; -GRANT ALL ON SCHEMA ${SCHEMA_NAME} TO GROUP ${OHDSI_ADMIN_ROLE}; +GRANT ALL ON SCHEMA :"schema_name" TO GROUP :"admin_role"; -GRANT USAGE ON SCHEMA ${SCHEMA_NAME} TO GROUP ${OHDSI_APP_ROLE}; +GRANT USAGE ON SCHEMA :"schema_name" TO GROUP :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT INSERT, SELECT, UPDATE, DELETE, REFERENCES, - TRIGGER ON TABLES TO ${OHDSI_APP_ROLE}; + TRIGGER ON TABLES TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT -SELECT, USAGE ON SEQUENCES TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT +SELECT, USAGE ON SEQUENCES TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT EXECUTE ON FUNCTIONS TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT EXECUTE ON FUNCTIONS TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT USAGE ON TYPES TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT USAGE ON TYPES TO :"app_role"; diff --git a/templates/workspace_services/ohdsi/template_schema.json b/templates/workspace_services/ohdsi/template_schema.json index 0794edb11..913c5dc05 100644 --- a/templates/workspace_services/ohdsi/template_schema.json +++ b/templates/workspace_services/ohdsi/template_schema.json @@ -368,6 +368,33 @@ } ] }, + { + "stepId": "83a8b148-d455-416c-9779-320fb7ae279c", + "stepTitle": "Remove OHDSI firewall rules", + "resourceTemplateName": "tre-shared-service-firewall", + "resourceType": "shared-service", + "resourceAction": "upgrade", + "properties": [ + { + "name": "network_rule_collections", + "type": "array", + "arraySubstitutionAction": "remove", + "arrayMatchField": "name", + "value": { + "name": "nrc_svc_{{ resource.id }}" + } + }, + { + "name": "rule_collections", + "type": "array", + "arraySubstitutionAction": "remove", + "arrayMatchField": "name", + "value": { + "name": "arc_svc_{{ resource.id }}" + } + } + ] + }, { "stepId": "main" } diff --git a/templates/workspace_services/ohdsi/terraform/atlas_database.tf b/templates/workspace_services/ohdsi/terraform/atlas_database.tf index 7d2691870..4bba2d071 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_database.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_database.tf @@ -147,18 +147,19 @@ resource "terraform_data" "postgres_subnet_wait" { } resource "azurerm_postgresql_flexible_server" "postgres" { - name = "psql-server-${local.service_suffix}" - resource_group_name = data.azurerm_resource_group.ws.name - location = data.azurerm_resource_group.ws.location - delegated_subnet_id = azurerm_subnet.postgres.id - private_dns_zone_id = data.azurerm_private_dns_zone.postgres.id - sku_name = var.postgres_sku - version = local.postgres_version - administrator_login = local.postgres_admin_username - administrator_password = azurerm_key_vault_secret.postgres_admin_password.value - storage_mb = var.postgres_storage_size_in_mb - zone = "1" - tags = local.tre_workspace_service_tags + public_network_access_enabled = false + name = "psql-server-${local.service_suffix}" + resource_group_name = data.azurerm_resource_group.ws.name + location = data.azurerm_resource_group.ws.location + delegated_subnet_id = azurerm_subnet.postgres.id + private_dns_zone_id = data.azurerm_private_dns_zone.postgres.id + sku_name = var.postgres_sku + version = local.postgres_version + administrator_login = local.postgres_admin_username + administrator_password = azurerm_key_vault_secret.postgres_admin_password.value + storage_mb = var.postgres_storage_size_in_mb + zone = "1" + tags = local.tre_workspace_service_tags timeouts { # If this doesn't complete in a realistic time, no point in waiting the full/default 60m diff --git a/templates/workspace_services/ohdsi/terraform/atlas_security.tf b/templates/workspace_services/ohdsi/terraform/atlas_security.tf index 0dd1fc308..901bef75d 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_security.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_security.tf @@ -23,6 +23,7 @@ resource "terraform_data" "deployment_atlas_security" { ATLAS_SECURITY_ADMIN_PASSWORD = azurerm_key_vault_secret.atlas_security_admin_password.value ATLAS_USERS = "admin,${azurerm_key_vault_secret.atlas_security_admin_password.value}" WEB_API_URL = local.ohdsi_webapi_url + WEB_API_VERSION = local.ohdsi_api_docker_image_tag } command = "../scripts/atlas_security.sh" diff --git a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf index 24879a80d..dab2236af 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf @@ -20,10 +20,11 @@ resource "azurerm_storage_share_file" "config_local" { } resource "azurerm_linux_web_app" "atlas_ui" { - name = local.atlas_ui_name - location = data.azurerm_resource_group.ws.location - resource_group_name = data.azurerm_resource_group.ws.name - virtual_network_subnet_id = data.azurerm_subnet.web_app.id + public_network_access_enabled = false + name = local.atlas_ui_name + location = data.azurerm_resource_group.ws.location + resource_group_name = data.azurerm_resource_group.ws.name + virtual_network_subnet_id = data.azurerm_subnet.web_app.id service_plan_id = data.azurerm_service_plan.workspace.id https_only = true diff --git a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf index 2606dbc5c..328498ef1 100644 --- a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf +++ b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf @@ -23,10 +23,11 @@ resource "azurerm_role_assignment" "keyvault_ohdsi_ws_role" { } resource "azurerm_linux_web_app" "ohdsi_webapi" { - name = local.ohdsi_webapi_name - location = data.azurerm_resource_group.ws.location - resource_group_name = data.azurerm_resource_group.ws.name - virtual_network_subnet_id = data.azurerm_subnet.web_app.id + public_network_access_enabled = false + name = local.ohdsi_webapi_name + location = data.azurerm_resource_group.ws.location + resource_group_name = data.azurerm_resource_group.ws.name + virtual_network_subnet_id = data.azurerm_subnet.web_app.id service_plan_id = data.azurerm_service_plan.workspace.id https_only = true From f76a7f35626666eb86f207c845c764fffbf5f47f Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 10 Oct 2026 21:42:41 +0100 Subject: [PATCH 07/24] Validate private OHDSI lifecycle and bounded cleanup (#5149) Add a bundle selection for the OHDSI workspace service. The case creates the service privately, checks the callback, firewall and shared DNS configuration, upgrades it, checks it again and deletes it with bounded clean-up. The API stores a new resource without deploymentStatus, and the first deployment status message sets it. The wait helper accepts a missing status, so a read in the first seconds after the POST keeps polling. The OHDSI bundle sets docker_image_name with the registry host in it and sets no docker_registry_url. azurerm 4.27.0 builds linuxFxVersion as "DOCKER|/", so the test expects "DOCKER|/index.docker.io/ohdsi/:2.12.1". --- .github/workflows/deploy_tre_reusable.yml | 4 + CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 26 ++ e2e_tests/bundle_coverage.json | 26 +- e2e_tests/pytest.ini | 1 + e2e_tests/resources/ohdsi.py | 272 ++++++++++++++ e2e_tests/resources/ohdsi_lifecycle.py | 289 ++++++++++++++ e2e_tests/test_ohdsi.py | 224 +++++++++++ e2e_tests/unit_tests/test_bundle_selection.py | 10 + .../unit_tests/test_ohdsi_configuration.py | 354 ++++++++++++++++++ e2e_tests/unit_tests/test_ohdsi_lifecycle.py | 234 ++++++++++++ 11 files changed, 1436 insertions(+), 5 deletions(-) create mode 100644 e2e_tests/resources/ohdsi.py create mode 100644 e2e_tests/resources/ohdsi_lifecycle.py create mode 100644 e2e_tests/test_ohdsi.py create mode 100644 e2e_tests/unit_tests/test_ohdsi_configuration.py create mode 100644 e2e_tests/unit_tests/test_ohdsi_lifecycle.py diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 5b8a4b41c..cf3aa887b 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -1075,6 +1075,9 @@ jobs: - name: Reserve CycleCloud cleanup and reporting time if: inputs.e2eBundle == 'tre-shared-service-cyclecloud' run: echo "CYCLECLOUD_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve OHDSI cleanup and reporting time + if: inputs.e2eBundle == 'tre-workspace-service-ohdsi' + run: echo "OHDSI_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" - name: Reserve SQL cleanup and reporting time if: inputs.e2eBundle == 'tre-workspace-service-azuresql' @@ -1118,6 +1121,7 @@ jobs: MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }} GITEA_VALIDATION_DEADLINE=${{ env.GITEA_VALIDATION_DEADLINE }} CYCLECLOUD_VALIDATION_DEADLINE=${{ env.CYCLECLOUD_VALIDATION_DEADLINE }} + OHDSI_VALIDATION_DEADLINE=${{ env.OHDSI_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f10f093..8bd8d9aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ENHANCEMENTS: * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) * Add an independent CycleCloud server lifecycle with marketplace checks, VM power actions and bounded cleanup. ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Add independent private OHDSI lifecycle validation with callback, firewall, shared DNS preservation and bounded cleanup checks. ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Record the deployed template version of each resource that a bundle test reads in the bundle evidence, without the resource properties. * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index f338ce3e2..f5b0da48b 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -264,3 +264,29 @@ Failed cleanup fails the test. Core Key Vault credentials can remain soft-delete This case does not configure the CycleCloud application or create an HPC cluster. Browser access, application TLS trust, private DNS reachability and prior-release upgrades remain unproven. The existing bundle grants subscription Contributor to its VM and permits SharedSubnet access to ARM and GitHub. It does not disable the storage public endpoint. A private endpoint and a successful server lifecycle do not prove workspace isolation or storage access restrictions. +### Private OHDSI validation + +Select the bundle from a branch that contains its repaired bootstrap and validation case: + +```bash +TEST_BUNDLE=tre-workspace-service-ohdsi make test-e2e-bundle +``` + +Use a dedicated AzureCloud environment with one writer and matching deployment and authentication tenants. +Publish and register the checkout base-workspace, firewall and OHDSI bundles. Deploy the firewall before the test. +Check PostgreSQL 14 capacity for `B_Standard_B1ms`, 32 GiB storage and availability zone 1 in the selected region. +The case creates a fresh Automatic workspace. It disables backups and sets `configure_data_source=false` without data-source properties. + +The case checks both private web apps, their pinned images, PostgreSQL networking and the WebAPI identity's vault role. +Successful installation includes the bundle's private WebAPI readiness check and local administrator setup. +A metadata update exercises a same-version upgrade. The test checks the callback and firewall collections after each operation. +It waits for terminal operations before updates or deletion. It owns accepted resources before evidence writes or client closure. + +The job sets `OHDSI_VALIDATION_DEADLINE` before container setup. The case reserves 120 minutes for cleanup within 270 minutes. +Cleanup removes the service before its workspace. It checks Azure resource removal, app DNS records, callback removal and firewall restoration. +If service cleanup fails, the case retains the workspace and fails. The shared core PostgreSQL DNS link remains in place. +Key Vault items can remain soft-deleted under the existing retention policy. + +Browser sign-in, independent private HTTP and DNS probes, database data persistence and previous-version upgrades remain unproven. +Synapse provisioning, CDM queries and clinical workflows need separate cases. Retained non-empty data-source properties are outside this case. +The Synapse script does not currently inspect `configure_data_source`; empty properties avoid that path. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 0c2940e11..f717f9dd3 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -435,15 +435,31 @@ "manifest": "templates/workspace_services/ohdsi/porter.yaml", "family": "workspace_service", "prerequisites": [ - "tre-workspace-base" + "tre-workspace-base", + "tre-shared-service-firewall" + ], + "tests": [ + "test_ohdsi.py::test_private_ohdsi_lifecycle" ], - "tests": [], "related_tests": [], - "coverage": [], + "coverage": [ + "install", + "private-arm-configuration", + "managed-identity", + "same-version-upgrade", + "callback-registration", + "firewall-restoration", + "uninstall" + ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, matching Azure and authentication tenants, the checkout firewall and a fresh Automatic workspace. Use one writer in a dedicated environment.", + "Requires PostgreSQL 14 capacity in availability zone 1. Check regional SKU and capacity before the run.", + "Sets configure_data_source=false and omits data-source properties. Synapse provisioning, retained data-source properties, CDM queries and clinical workflows remain untested.", + "Checks private ARM configuration and successful bundle bootstrap. Browser sign-in, independent private HTTP and DNS probes, database data persistence and previous-version upgrades remain unproven.", + "Reserves 120 minutes for cleanup within a 270-minute job deadline. A failed service cleanup retains its workspace and fails validation.", + "Removes owned service resources, callbacks, app DNS records and firewall collections. The shared core PostgreSQL DNS link is retained; Key Vault items can remain soft-deleted." ], - "unavailable_reason": "Missing workspace-service case and parametrisation (#5126)." + "unavailable_reason": "" }, "tre-workspace-service-openai": { "manifest": "templates/workspace_services/openai/porter.yaml", diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 285d13993..902a33018 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -17,6 +17,7 @@ markers = workspace_validation: opt-in base-workspace creation, scoped API access and deletion openai_validation: opt-in private OpenAI deployment with model and quota preflight cyclecloud_validation: opt-in isolated CycleCloud server lifecycle and VM power actions + ohdsi_validation: opt-in private OHDSI service lifecycle without an external data source sql_validation: Private SQL query and configuration upgrade lifecycle mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM gitea_validation: opt-in private Gitea repository write/read across an update and an app restart diff --git a/e2e_tests/resources/ohdsi.py b/e2e_tests/resources/ohdsi.py new file mode 100644 index 000000000..2d62f1ab7 --- /dev/null +++ b/e2e_tests/resources/ohdsi.py @@ -0,0 +1,272 @@ +"""Read OHDSI infrastructure without requesting application settings or secrets.""" + +import asyncio +from copy import deepcopy +from dataclasses import dataclass +import os +import re + +from e2e_tests import config +from e2e_tests.resources.ohdsi_lifecycle import require_uuid + +TEMPLATE = "tre-workspace-service-ohdsi" +FIREWALL = "tre-shared-service-firewall" +BASE = "tre-workspace-base" +WEB_API = "2024-04-01" +POSTGRES_API = "2024-08-01" +NETWORK_API = "2024-05-01" +DNS_API = "2024-06-01" +GROUP_API = "2021-04-01" +IDENTITY_API = "2023-01-31" +STORAGE_API = "2023-05-01" +ROLE_API = "2022-04-01" +SECRETS_USER = "4633458b-17de-408a-b874-0445c86b69e6" + + +@dataclass(frozen=True) +class Settings: + subscription: str + tenant: str + tre_id: str + + def __post_init__(self): + require_uuid(self.subscription) + require_uuid(self.tenant) + if not isinstance(self.tre_id, str) or not re.fullmatch(r"[a-z0-9-]{1,11}", self.tre_id): + raise ValueError("OHDSI requires a valid TRE_ID shorter than 12 characters") + + @property + def group(self): + return f"/subscriptions/{self.subscription}/resourceGroups/rg-{self.tre_id}" + + +def require_settings(): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("OHDSI validation currently supports AzureCloud only") + settings = Settings(os.environ.get("ARM_SUBSCRIPTION_ID", ""), os.environ.get("ARM_TENANT_ID", ""), config.TRE_ID) + if require_uuid(config.AAD_TENANT_ID) != settings.tenant: + raise ValueError("OHDSI requires matching deployment and authentication tenants") + return settings + + +def resource_ids(settings, workspace_id, service_id): + require_uuid(workspace_id) + require_uuid(service_id) + ws = f"{settings.tre_id}-ws-{workspace_id[-4:]}" + suffix = f"{ws}-svc-{service_id[-4:]}" + group = f"/subscriptions/{settings.subscription}/resourceGroups/rg-{ws}" + vnet = f"{group}/providers/Microsoft.Network/virtualNetworks/vnet-{ws}" + apps = {key: f"{group}/providers/Microsoft.Web/sites/app-ohdsi-{key}-{suffix}" for key in ("atlas", "webapi")} + endpoints = { + key: f"{group}/providers/Microsoft.Network/privateEndpoints/pe-{app.rsplit('/', 1)[1]}" + for key, app in apps.items() + } + storage = f"{group}/providers/Microsoft.Storage/storageAccounts/{('stg' + ws[-8:]).replace('-', '').lower()}" + ids = { + "group": group, + "vnet": vnet, + "apps": apps, + "endpoints": endpoints, + "postgres": f"{group}/providers/Microsoft.DBforPostgreSQL/flexibleServers/psql-server-{suffix}", + "postgres_subnet": f"{vnet}/subnets/PostgreSQLSubnet{service_id[-4:]}", + "nsg": f"{group}/providers/Microsoft.Network/networkSecurityGroups/nsg-psql-{suffix}", + "identity": f"{group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ohdsi-webapi-{suffix}", + "share": f"{storage}/fileServices/default/shares/atlas-{suffix}", + "vault": f"{group}/providers/Microsoft.KeyVault/vaults/kv-{ws[-20:].lower()}", + "web_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.azurewebsites.net", + "pg_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.postgres.database.azure.com", + } + ids["removal"] = [(value, WEB_API) for value in apps.values()] + [ + (value, NETWORK_API) for value in endpoints.values() + ] + ids["removal"] += [ + (ids[key], version) + for key, version in ( + ("postgres", POSTGRES_API), + ("postgres_subnet", NETWORK_API), + ("nsg", NETWORK_API), + ("identity", IDENTITY_API), + ("share", STORAGE_API), + ) + ] + return ids + + +def same_id(actual, expected): + assert isinstance(actual, str) and actual.lower() == expected.lower(), "Unexpected OHDSI ARM resource ID" + + +def owned(resource, identifier, tags): + same_id(resource.get("id"), identifier) + assert all(resource.get("tags", {}).get(key) == value for key, value in tags.items()), "OHDSI ownership tags differ" + + +async def guard_owned(arm, ids, tags): + for identifier, api in ( + [(v, WEB_API) for v in ids["apps"].values()] + + [(ids["postgres"], POSTGRES_API), (ids["identity"], IDENTITY_API), (ids["nsg"], NETWORK_API)] + + [(v, NETWORK_API) for v in ids["endpoints"].values()] + ): + resource = await arm.request("GET", identifier, api, missing_ok=True) + if resource is not None: + owned(resource, identifier, tags) + + +async def preflight(arm, settings): + group = await arm.request("GET", settings.group, GROUP_API) + owned(group, settings.group, {"tre_id": settings.tre_id}) + for zone in ("privatelink.azurewebsites.net", "privatelink.postgres.database.azure.com"): + identifier = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/{zone}" + owned(await arm.request("GET", identifier, DNS_API), identifier, {"tre_id": settings.tre_id}) + + +async def postgres_core_dns_link(arm, settings, *, missing_ok=False): + identifier = ( + f"{settings.group}/providers/Microsoft.Network/privateDnsZones/" + "privatelink.postgres.database.azure.com/virtualNetworkLinks/core" + ) + link = await arm.request("GET", identifier, DNS_API, missing_ok=missing_ok) + if link is None: + assert missing_ok, "Shared PostgreSQL core DNS link is missing" + return None + same_id(link.get("id"), identifier) + properties = link["properties"] + same_id( + properties.get("virtualNetwork", {}).get("id"), + f"{settings.group}/providers/Microsoft.Network/virtualNetworks/vnet-{settings.tre_id}", + ) + assert properties.get("registrationEnabled") is False, "Shared PostgreSQL DNS registration changed" + assert properties.get("provisioningState") == "Succeeded", "Shared PostgreSQL core DNS link is not ready" + return link + + +def validate_app(app, configuration, identifier, ids, tags, key): + owned(app, identifier, tags) + props = app["properties"] + assert props.get("state") == "Running", "OHDSI web app is not running" + assert props.get("publicNetworkAccess") == "Disabled", "OHDSI web app permits public access" + assert props.get("httpsOnly") is True, "OHDSI web app permits HTTP" + same_id(props.get("virtualNetworkSubnetId"), ids["vnet"] + "/subnets/WebAppsSubnet") + site = configuration["properties"] + assert site.get("ftpsState") == "Disabled", "OHDSI web app permits FTP" + assert site.get("minTlsVersion") == "1.3", "OHDSI web app TLS setting differs" + # The bundle sets no docker_registry_url, so azurerm writes an empty registry host before the image. + assert site.get("linuxFxVersion") == f"DOCKER|/index.docker.io/ohdsi/{key}:2.12.1", "OHDSI image differs" + if key == "webapi": + identity = app.get("identity", {}) + assert identity.get("type") == "UserAssigned", "Unexpected OHDSI application identity type" + assert {k.lower() for k in identity.get("userAssignedIdentities", {})} == {ids["identity"].lower()} + same_id(props.get("keyVaultReferenceIdentity"), ids["identity"]) + + +async def validate_resources(arm, settings, ids, tags): + async with asyncio.timeout(10 * 60): + postgres = await arm.request("GET", ids["postgres"], POSTGRES_API) + owned(postgres, ids["postgres"], tags) + props = postgres["properties"] + assert props.get("state") == "Ready", "OHDSI PostgreSQL is not ready" + assert props.get("version") == "14", "OHDSI PostgreSQL version differs" + assert props.get("network", {}).get("publicNetworkAccess") == "Disabled", "PostgreSQL permits public access" + same_id(props["network"].get("delegatedSubnetResourceId"), ids["postgres_subnet"]) + same_id(props["network"].get("privateDnsZoneArmResourceId"), ids["pg_zone"]) + subnet = await arm.request("GET", ids["postgres_subnet"], NETWORK_API) + same_id(subnet.get("id"), ids["postgres_subnet"]) + same_id(subnet["properties"].get("networkSecurityGroup", {}).get("id"), ids["nsg"]) + assert subnet["properties"].get("defaultOutboundAccess") is False + assert any( + d.get("properties", {}).get("serviceName") == "Microsoft.DBforPostgreSQL/flexibleServers" + for d in subnet["properties"].get("delegations", []) + ) + for key, identifier in ids["apps"].items(): + app = await arm.request("GET", identifier, WEB_API) + configuration = await arm.request("GET", identifier + "/config/web", WEB_API) + validate_app(app, configuration, identifier, ids, tags, key) + endpoint = await arm.request("GET", ids["endpoints"][key], NETWORK_API) + owned(endpoint, ids["endpoints"][key], tags) + properties = endpoint["properties"] + assert properties.get("provisioningState") == "Succeeded" + same_id(properties.get("subnet", {}).get("id"), ids["vnet"] + "/subnets/ServicesSubnet") + assert not properties.get("manualPrivateLinkServiceConnections") + connections = properties.get("privateLinkServiceConnections", []) + assert len(connections) == 1 + connection = connections[0]["properties"] + same_id(connection.get("privateLinkServiceId"), identifier) + assert connection.get("groupIds") == ["sites"] + assert connection.get("privateLinkServiceConnectionState", {}).get("status") == "Approved" + zone_group_id = ids["endpoints"][key] + "/privateDnsZoneGroups/privatelink.azurewebsites.net" + zone_group = await arm.request("GET", zone_group_id, NETWORK_API) + same_id(zone_group.get("id"), zone_group_id) + zones = zone_group["properties"].get("privateDnsZoneConfigs", []) + assert len(zones) == 1 + same_id(zones[0]["properties"].get("privateDnsZoneId"), ids["web_zone"]) + identity = await arm.request("GET", ids["identity"], IDENTITY_API) + owned(identity, ids["identity"], tags) + assert require_uuid(identity["properties"].get("tenantId")) == settings.tenant + principal = require_uuid(identity["properties"].get("principalId")) + assignments = await arm.request( + "GET", ids["vault"] + "/providers/Microsoft.Authorization/roleAssignments", ROLE_API + ) + assert isinstance(assignments.get("value"), list) and not assignments.get("nextLink"), ( + "Incomplete OHDSI role inventory" + ) + roles = [ + r["properties"] for r in assignments["value"] if r["properties"].get("principalId", "").lower() == principal + ] + assert len(roles) == 1, "Unexpected OHDSI vault role count" + same_id(roles[0].get("scope"), ids["vault"]) + assert roles[0].get("roleDefinitionId", "").rsplit("/", 1)[-1].lower() == SECRETS_USER + return principal + + +def firewall_rules(resource): + return { + key: deepcopy(resource["properties"].get(key, [])) for key in ("network_rule_collections", "rule_collections") + } + + +def validate_firewall(resource, baseline, service_id, prefixes): + rules = firewall_rules(resource) + for key, prefix in (("network_rule_collections", "nrc"), ("rule_collections", "arc")): + name = f"{prefix}_svc_{service_id}" + matches = [c for c in rules[key] if c.get("name") == name] + assert len(matches) == 1, "OHDSI firewall collection is missing or duplicated" + collection = matches[0] + rules[key].remove(collection) + assert collection["action"] == "Allow" and len(collection["rules"]) == 1 + rule = collection["rules"][0] + assert set(rule["source_addresses"]) == set(prefixes) + if prefix == "nrc": + assert rule["destination_addresses"] == ["AzureActiveDirectory"] + assert rule["destination_ports"] == ["*"] and rule["protocols"] == ["TCP"] + else: + assert rule["protocols"] == [{"port": "80", "type": "Http"}, {"port": "443", "type": "Https"}] + assert set(rule["target_fqdns"]) == { + "*.msftauth.net", + "*.msauth.net", + "login.microsoftonline.com", + "aadcdn.msftauthimages.net", + "aadcdn.msauthimages.net", + "*.login.live.com", + "*.microsoftonline-p.com", + "msft.sts.microsoft.com", + } + assert rules == baseline, "Unrelated firewall collections changed" + + +async def assert_removed(arm, identifiers): + async with asyncio.timeout(5 * 60): + for identifier, api in identifiers: + while await arm.request("GET", identifier, api, missing_ok=True) is not None: + await asyncio.sleep(10) + + +async def assert_roles_removed(arm, vault, principal): + async with asyncio.timeout(5 * 60): + while True: + roles = await arm.request("GET", vault + "/providers/Microsoft.Authorization/roleAssignments", ROLE_API) + assert isinstance(roles.get("value"), list) and not roles.get("nextLink"), ( + "Incomplete OHDSI cleanup role inventory" + ) + if not any(r["properties"].get("principalId", "").lower() == principal for r in roles["value"]): + return + await asyncio.sleep(10) diff --git a/e2e_tests/resources/ohdsi_lifecycle.py b/e2e_tests/resources/ohdsi_lifecycle.py new file mode 100644 index 000000000..363e06ca3 --- /dev/null +++ b/e2e_tests/resources/ohdsi_lifecycle.py @@ -0,0 +1,289 @@ +"""Own OHDSI prerequisites until their dependent resources are removed.""" + +import asyncio +from builtins import BaseExceptionGroup +from contextlib import asynccontextmanager +from dataclasses import dataclass +import math +import os +import time +from uuid import UUID + +from httpx import AsyncClient + +from e2e_tests.bundle_evidence import record_operation, record_resource +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import TIMEOUT, get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources.resource import disable_and_delete_resource +from e2e_tests.timeouts import cleanup_deadline + +LIFECYCLE_SECONDS = 270 * 60 +CLEANUP_RESERVE_SECONDS = 120 * 60 +CREATE_SECONDS = 90 * 60 +TERMINAL_STATES = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} +FAILED_STATES = {"deployment_failed", "updating_failed", "deleting_failed", "action_failed", "pipeline_failed"} + + +def lifecycle_deadlines(): + configured = os.environ.get("OHDSI_VALIDATION_DEADLINE", "") + deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS + if not math.isfinite(deadline): + raise ValueError("OHDSI validation deadline must be finite") + remaining = min(LIFECYCLE_SECONDS, deadline - time.time()) + if remaining <= CLEANUP_RESERVE_SECONDS: + raise TimeoutError("Insufficient OHDSI validation time remains after reserving cleanup") + finish = asyncio.get_running_loop().time() + remaining + return finish - CLEANUP_RESERVE_SECONDS, finish + + +def require_uuid(value): + if not isinstance(value, str): + raise ValueError("Resource identity must be a canonical, non-zero UUID") + try: + parsed = UUID(value) + except ValueError: + raise ValueError("Resource identity must be a canonical, non-zero UUID") from None + if not parsed.int or str(parsed) != value: + raise ValueError("Resource identity must be a canonical, non-zero UUID") + return value + + +@dataclass +class OwnedResource: + path: str + identifier: str + wrapper: str + template: str + display_name: str + description: str + workspace_id: str | None + before_remove: object = None + after_remove: object = None + + def validate(self, body): + resource = body.get(self.wrapper, {}) + properties = resource.get("properties", {}) + if ( + resource.get("id") != self.identifier + or resource.get("templateName") != self.template + or properties.get("display_name") != self.display_name + or properties.get("description") != self.description + # The API stores no status until the first deployment status message arrives. + or not isinstance(resource.get("deploymentStatus"), str | None) + ): + raise ValueError("OHDSI resource ownership or deployment status changed") + return resource + + +def creation_contract(payload, endpoint, workspace_id): + if endpoint == "/api/workspaces" and workspace_id is None: + wrapper, template = "workspace", "tre-workspace-base" + if payload.get("properties", {}).get("auth_type") != "Automatic": + raise ValueError("OHDSI validation needs a fresh Automatic workspace") + elif workspace_id is not None and endpoint == f"/api/workspaces/{require_uuid(workspace_id)}/workspace-services": + wrapper, template = "workspaceService", "tre-workspace-service-ohdsi" + properties = payload.get("properties", {}) + if properties.get("configure_data_source") is not False or any( + key in properties for key in ("data_source_config", "data_source_daimons") + ): + raise ValueError("OHDSI validation requires no external data source") + else: + raise ValueError("Unsupported OHDSI resource creation endpoint or parent") + properties = payload.get("properties", {}) + if payload.get("templateName") != template or any( + not isinstance(properties.get(name), str) or not properties[name].strip() + for name in ("display_name", "description") + ): + raise ValueError("OHDSI creation needs the expected template and ownership properties") + return wrapper, template + + +class OHDSIResources: + def __init__(self, verify, work_finish, finish): + self.verify = verify + self.work_finish = work_finish + self.finish = finish + self.owned = [] + + async def token(self, resource): + if resource.workspace_id is not None: + return await get_workspace_owner_token(resource.workspace_id, self.verify) + return await get_admin_token(self.verify) + + async def create(self, payload, endpoint, workspace_id=None, *, after_remove=None, before_remove=None): + wrapper, template = creation_contract(payload, endpoint, workspace_id) + resource = OwnedResource( + "", + "", + wrapper, + template, + payload["properties"]["display_name"], + payload["properties"]["description"], + workspace_id, + before_remove, + after_remove, + ) + async with asyncio.timeout_at(min(self.work_finish, asyncio.get_running_loop().time() + CREATE_SECONDS)): + token = await self.token(resource) + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.post(get_full_endpoint(endpoint), headers=get_auth_header(token), json=payload) + if response.status_code != 202: + raise RuntimeError(f"OHDSI creation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + resource_id = require_uuid(operation["resourceId"]) + resource_path = operation["resourcePath"] + if resource_path != f"{endpoint.removeprefix('/api')}/{resource_id}": + raise ValueError("Accepted OHDSI resource has an unexpected path or parent") + resource.path, resource.identifier = resource_path, resource_id + # Own the accepted resource before evidence writes or client closure. + self.owned.append(resource) + operation_id = require_uuid(operation["id"]) + record_resource(payload, operation, "POST") + result = await self.wait_terminal(resource, allow_absent=False, operation_id=operation_id) + deployed, operations = result + if deployed["deploymentStatus"] != "deployed" or any( + operation["status"] in FAILED_STATES for operation in operations + ): + raise RuntimeError("OHDSI resource deployment or its pipeline failed") + return resource.path, resource.identifier + + async def wait_terminal(self, resource, *, allow_absent=False, operation_id=None): + endpoint = get_full_endpoint(f"/api{resource.path}") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get(endpoint, headers=get_auth_header(await self.token(resource))) + if response.status_code == 404 and allow_absent: + return None + if response.status_code != 200: + raise RuntimeError(f"Could not inspect owned OHDSI resource: HTTP {response.status_code}") + deployed = resource.validate(response.json()) + response = await client.get( + endpoint + "/operations", headers=get_auth_header(await self.token(resource)) + ) + if response.status_code != 200: + raise RuntimeError(f"Could not inspect OHDSI operations: HTTP {response.status_code}") + operations = response.json()["operations"] + if not isinstance(operations, list): + raise ValueError("OHDSI operations response must contain a list") + for operation in operations: + observed_id = require_uuid(operation["id"]) + record_operation( + f"/api{resource.path}/operations/{observed_id}", + operation["status"], + operation["status"] in TERMINAL_STATES, + ) + if ( + (operation_id is None or any(o["id"] == operation_id for o in operations)) + and deployed["deploymentStatus"] in TERMINAL_STATES + and operations + and all(operation["status"] in TERMINAL_STATES for operation in operations) + ): + return deployed, operations + await asyncio.sleep(30) + + async def upgrade(self, resource, overview): + if resource.workspace_id is None: + raise ValueError("Only the owned OHDSI service can be upgraded") + deployed, _ = await self.wait_terminal(resource) + headers = get_auth_header(await self.token(resource)) + headers["etag"] = deployed["_etag"] + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.patch( + get_full_endpoint(f"/api{resource.path}"), headers=headers, json={"properties": {"overview": overview}} + ) + if response.status_code != 202: + raise RuntimeError(f"OHDSI upgrade failed: HTTP {response.status_code}") + operation = response.json()["operation"] + if operation["resourcePath"] != resource.path or operation["resourceId"] != resource.identifier: + raise ValueError("OHDSI upgrade returned a different resource") + operation_id = require_uuid(operation["id"]) + record_resource(None, operation, "PATCH") + deployed, operations = await self.wait_terminal(resource, operation_id=operation_id) + if deployed["deploymentStatus"] != "updated" or any(o["status"] in FAILED_STATES for o in operations): + raise RuntimeError("OHDSI upgrade or its pipeline failed") + assert deployed["properties"].get("overview") == overview, "OHDSI overview update was not preserved" + return deployed + + async def assert_api_removed(self, resource): + async with asyncio.timeout(5 * 60): + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get( + get_full_endpoint(f"/api{resource.path}"), + headers=get_auth_header(await self.token(resource)), + ) + if response.status_code == 404: + return + if response.status_code != 200: + raise RuntimeError(f"Could not verify OHDSI removal: HTTP {response.status_code}") + resource.validate(response.json()) + await asyncio.sleep(10) + + async def remove(self, resource): + present = await self.wait_terminal(resource, allow_absent=True) + if present is not None: + if resource.before_remove is not None: + await resource.before_remove(resource.path, resource.identifier) + await disable_and_delete_resource( + f"/api{resource.path}", await self.token(resource), self.verify, allow_failed_disable=True + ) + await self.assert_api_removed(resource) + if resource.after_remove is not None: + await resource.after_remove(resource.path, resource.identifier) + + async def close(self): + failures = [] + while self.owned: + seconds = (self.finish - asyncio.get_running_loop().time()) / len(self.owned) + resource = self.owned.pop() + try: + if resource.workspace_id is None and failures: + raise RuntimeError("OHDSI workspace retained because service cleanup failed") + async with cleanup_deadline(seconds): + await self.remove(resource) + except (Exception, asyncio.CancelledError) as error: + error.add_note(f"OHDSI validation cleanup failed for {resource.path}") + failures.append(error) + return failures + + +@asynccontextmanager +async def ohdsi_lifecycle(verify): + work_finish, finish = lifecycle_deadlines() + resources = OHDSIResources(verify, work_finish, finish) + original = None + async with cleanup_deadline(finish - asyncio.get_running_loop().time()): + try: + async with asyncio.timeout_at(work_finish): + yield resources + except BaseException as error: + original = error + raise + finally: + cleanup = asyncio.create_task(resources.close()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + failures = cleanup.result() + failure = original if original is not None else cancelled + if failure is not None: + for error in failures: + failure.add_note(f"OHDSI cleanup also failed: {error!r}; {error.__notes__}") + elif failures: + raise BaseExceptionGroup("OHDSI resource cleanup failed", failures) + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/test_ohdsi.py b/e2e_tests/test_ohdsi.py new file mode 100644 index 000000000..b938df115 --- /dev/null +++ b/e2e_tests/test_ohdsi.py @@ -0,0 +1,224 @@ +"""Validate a private OHDSI deployment without an external clinical data source.""" + +from uuid import uuid4 + +import pytest + +from e2e_tests import config +from e2e_tests.bundle_evidence import record_deployed_resource, record_reused_resource +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import get_admin_token +from e2e_tests.resources.ohdsi import ( + BASE, + FIREWALL, + TEMPLATE, + GROUP_API, + IDENTITY_API, + DNS_API, + assert_removed, + assert_roles_removed, + firewall_rules, + guard_owned, + owned, + postgres_core_dns_link, + preflight, + require_settings, + resource_ids, + validate_firewall, + validate_resources, +) +from e2e_tests.resources.ohdsi_lifecycle import ohdsi_lifecycle, require_uuid +from e2e_tests.resources.resource import get_resource +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.run_bundle import load_catalog + + +async def prerequisites(verify): + catalog = load_catalog() + token = await get_admin_token(verify) + for name, kind in ((TEMPLATE, "workspace-service"), (BASE, "workspace"), (FIREWALL, "shared-service")): + template = await get_resource(f"/api/{kind}-templates/{name}", token, verify) + if template["name"] != name or template["version"] != catalog[name]["source_version"]: + raise ValueError(f"Register the checkout version of {name} before OHDSI validation") + services = (await get_resource("/api/shared-services", token, verify))["sharedServices"] + firewalls = [s for s in services if s["templateName"] == FIREWALL] + if len(firewalls) != 1: + raise ValueError("OHDSI requires exactly one deployed firewall") + firewall = firewalls[0] + if ( + firewall["isEnabled"] is not True + or firewall["deploymentStatus"] not in ("deployed", "updated") + or firewall["templateVersion"] != catalog[FIREWALL]["source_version"] + ): + raise ValueError("Deploy and enable the checkout firewall version before OHDSI validation") + record_reused_resource(firewall) + return firewall + + +def payload(template, label, **properties): + return { + "templateName": template, + "properties": { + "display_name": f"OHDSI validation {label} {uuid4()}", + "description": "Owned OHDSI lifecycle validation", + **properties, + }, + } + + +async def validate_lifecycle(verify): + settings = require_settings() + async with arm_client() as arm: + async with ohdsi_lifecycle(verify) as resources: + firewall = await prerequisites(verify) + baseline = firewall_rules(firewall) + await preflight(arm, settings) + core_link = await postgres_core_dns_link(arm, settings, missing_ok=True) + workspace_path = None + workspace_id = None + callbacks = None + principal = None + + async def check_core_link(*, required=False): + nonlocal core_link + # A failed install might not reach DNS creation. Preserve any link + # observed before install, during validation or before removal. + core_link = await postgres_core_dns_link(arm, settings, missing_ok=not required and core_link is None) + + async def current_firewall(): + result = ( + await get_resource(f"/api/shared-services/{firewall['id']}", await get_admin_token(verify), verify) + )["sharedService"] + assert result["id"] == firewall["id"] and result["templateName"] == FIREWALL + return result + + async def workspace_record(): + return (await get_resource(f"/api{workspace_path}", await get_admin_token(verify), verify))["workspace"] + + def group_id(identifier): + return ( + f"/subscriptions/{settings.subscription}/resourceGroups/rg-{settings.tre_id}-ws-{identifier[-4:]}" + ) + + async def guard_workspace(_path, identifier): + group = await arm.request("GET", group_id(identifier), GROUP_API, missing_ok=True) + if group is not None: + owned(group, group_id(identifier), {"tre_id": settings.tre_id, "tre_workspace_id": identifier}) + + async def removed_workspace(_path, identifier): + await assert_removed(arm, [(group_id(identifier), GROUP_API)]) + + properties = {"auth_type": "Automatic", "address_space_size": "small", "enable_backup": False} + if config.TEST_WORKSPACE_APP_PLAN: + properties["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN + workspace_path, workspace_id = await resources.create( + payload(BASE, "workspace", **properties), + "/api/workspaces", + before_remove=guard_workspace, + after_remove=removed_workspace, + ) + workspace = await workspace_record() + assert workspace["templateVersion"] == load_catalog()[BASE]["source_version"] + assert workspace["properties"]["auth_type"] == "Automatic" + assert ( + workspace["properties"].get("workspace_subscription_id") or settings.subscription + ).lower() == settings.subscription + record_deployed_resource(workspace, workspace_path) + callbacks = list(workspace["properties"].get("aad_redirect_uris", [])) + await guard_workspace(workspace_path, workspace_id) + + async def guard_service(_path, identifier): + nonlocal principal + ids = resource_ids(settings, workspace_id, identifier) + await guard_workspace(workspace_path, workspace_id) + await guard_owned( + arm, + ids, + { + "tre_id": settings.tre_id, + "tre_workspace_id": workspace_id, + "tre_workspace_service_id": identifier, + }, + ) + identity = await arm.request("GET", ids["identity"], IDENTITY_API, missing_ok=True) + if identity is not None: + principal = require_uuid(identity["properties"]["principalId"]) + await check_core_link() + + async def removed_service(_path, identifier): + ids = resource_ids(settings, workspace_id, identifier) + dns_records = [ + (f"{ids['web_zone']}/A/{app.rsplit('/', 1)[1]}{suffix}", DNS_API) + for app in ids["apps"].values() + for suffix in ("", ".scm") + ] + await assert_removed(arm, ids["removal"] + dns_records) + await check_core_link() + if principal: + await assert_roles_removed(arm, ids["vault"], principal) + assert firewall_rules(await current_firewall()) == baseline, "OHDSI firewall rules were not restored" + assert (await workspace_record())["properties"].get("aad_redirect_uris", []) == callbacks, ( + "OHDSI callback was not removed" + ) + + service_path, service_id = await resources.create( + payload(TEMPLATE, "service", configure_data_source=False), + f"/api{workspace_path}/workspace-services", + workspace_id, + before_remove=guard_service, + after_remove=removed_service, + ) + ids = resource_ids(settings, workspace_id, service_id) + + async def inspect(record): + await check_core_link(required=True) + assert ( + record["id"] == service_id + and record["templateVersion"] == load_catalog()[TEMPLATE]["source_version"] + ) + record_deployed_resource(record, service_path) + props = record["properties"] + assert props["configure_data_source"] is False + assert props["is_exposed_externally"] is False + assert ( + props["connection_uri"] + == "https://" + ids["apps"]["atlas"].rsplit("/", 1)[1] + ".azurewebsites.net/atlas" + ) + assert ( + props["webapi_uri"] + == "https://" + ids["apps"]["webapi"].rsplit("/", 1)[1] + ".azurewebsites.net/WebAPI/" + ) + expected_callback = props["webapi_uri"] + "user/oauth/callback?client_name=OidcClient" + assert props["authentication_callback_uri"] == expected_callback + parent = await workspace_record() + entries = list(parent["properties"].get("aad_redirect_uris", [])) + expected = {"name": service_id, "value": expected_callback} + assert entries.count(expected) == 1 + entries.remove(expected) + assert entries == callbacks, "Unrelated workspace callbacks changed" + validate_firewall( + await current_firewall(), baseline, service_id, parent["properties"]["address_spaces"] + ) + return await validate_resources( + arm, + settings, + ids, + { + "tre_id": settings.tre_id, + "tre_workspace_id": workspace_id, + "tre_workspace_service_id": service_id, + }, + ) + + record = ( + await get_resource(f"/api{service_path}", await get_workspace_owner_token(workspace_id, verify), verify) + )["workspaceService"] + principal = await inspect(record) + updated = await resources.upgrade(resources.owned[-1], "OHDSI lifecycle metadata upgrade") + assert await inspect(updated) == principal, "OHDSI upgrade replaced the managed identity" + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.ohdsi_validation +async def test_private_ohdsi_lifecycle(verify): + await validate_lifecycle(verify) diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index 9659e573b..7a24f3de0 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -231,6 +231,16 @@ def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): self.assertEqual(report["source_bundle_version"], version) self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + def test_ohdsi_selects_only_its_private_lifecycle_without_provisioning(self): + result, report = self.run_selector("--bundle", "tre-workspace-service-ohdsi", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_ohdsi.py::test_private_ohdsi_lifecycle"]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + self.assertEqual(set(report["declared_prerequisites"]), {"tre-workspace-base", "tre-shared-service-firewall"}) + self.assertTrue(any("Synapse" in limit for limit in report["limitations"])) + def test_observed_deployment_evidence_excludes_properties(self): with tempfile.TemporaryDirectory() as directory: report = bundle_evidence.BundleReport(Path(directory) / "report.json", {}) diff --git a/e2e_tests/unit_tests/test_ohdsi_configuration.py b/e2e_tests/unit_tests/test_ohdsi_configuration.py new file mode 100644 index 000000000..e6db92ba0 --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_configuration.py @@ -0,0 +1,354 @@ +"""Check OHDSI private configuration, prerequisite rejection and workflow wiring.""" + +from copy import deepcopy +from contextlib import asynccontextmanager +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +import yaml + +from e2e_tests import test_ohdsi as case +from e2e_tests.resources import ohdsi as h + +ROOT = Path(__file__).resolve().parents[2] +SUB = "11111111-1111-4111-8111-111111111111" +TENANT = "22222222-2222-4222-8222-222222222222" +WS = "33333333-3333-4333-8333-333333333333" +SERVICE = "44444444-4444-4444-8444-444444444444" +PRINCIPAL = "55555555-5555-4555-8555-555555555555" +SETTINGS = h.Settings(SUB, TENANT, "testtre") +IDS = h.resource_ids(SETTINGS, WS, SERVICE) +TAGS = {"tre_id": "testtre", "tre_workspace_id": WS, "tre_workspace_service_id": SERVICE} +CORE_LINK = IDS["pg_zone"] + "/virtualNetworkLinks/core" + + +def core_link(): + return { + "id": CORE_LINK, + "properties": { + "virtualNetwork": { + "id": f"{SETTINGS.group}/providers/Microsoft.Network/virtualNetworks/vnet-{SETTINGS.tre_id}" + }, + "registrationEnabled": False, + "provisioningState": "Succeeded", + }, + } + + +def arm_records(): + def resource(identifier, properties): + return {"id": identifier, "tags": TAGS.copy(), "properties": properties} + + records = { + IDS["postgres"]: resource( + IDS["postgres"], + { + "state": "Ready", + "version": "14", + "network": { + "publicNetworkAccess": "Disabled", + "delegatedSubnetResourceId": IDS["postgres_subnet"], + "privateDnsZoneArmResourceId": IDS["pg_zone"], + }, + }, + ), + IDS["postgres_subnet"]: resource( + IDS["postgres_subnet"], + { + "networkSecurityGroup": {"id": IDS["nsg"]}, + "defaultOutboundAccess": False, + "delegations": [{"properties": {"serviceName": "Microsoft.DBforPostgreSQL/flexibleServers"}}], + }, + ), + IDS["identity"]: resource(IDS["identity"], {"tenantId": TENANT, "principalId": PRINCIPAL}), + IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments": { + "value": [ + { + "properties": { + "principalId": PRINCIPAL, + "scope": IDS["vault"], + "roleDefinitionId": "/roles/" + h.SECRETS_USER, + } + } + ] + }, + } + for key, identifier in IDS["apps"].items(): + records[identifier] = resource( + identifier, + { + "state": "Running", + "publicNetworkAccess": "Disabled", + "httpsOnly": True, + "virtualNetworkSubnetId": IDS["vnet"] + "/subnets/WebAppsSubnet", + "keyVaultReferenceIdentity": IDS["identity"], + }, + ) + records[identifier]["identity"] = {"type": "UserAssigned", "userAssignedIdentities": {IDS["identity"]: {}}} + records[identifier + "/config/web"] = { + "properties": { + "ftpsState": "Disabled", + "minTlsVersion": "1.3", + "linuxFxVersion": f"DOCKER|/index.docker.io/ohdsi/{key}:2.12.1", + } + } + endpoint = IDS["endpoints"][key] + records[endpoint] = resource( + endpoint, + { + "provisioningState": "Succeeded", + "subnet": {"id": IDS["vnet"] + "/subnets/ServicesSubnet"}, + "privateLinkServiceConnections": [ + { + "properties": { + "privateLinkServiceId": identifier, + "groupIds": ["sites"], + "privateLinkServiceConnectionState": {"status": "Approved"}, + } + } + ], + }, + ) + zone_group = endpoint + "/privateDnsZoneGroups/privatelink.azurewebsites.net" + records[zone_group] = resource( + zone_group, {"privateDnsZoneConfigs": [{"properties": {"privateDnsZoneId": IDS["web_zone"]}}]} + ) + return records + + +class ConfigurationTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.records = arm_records() + self.arm = AsyncMock() + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if kwargs.get("missing_ok"): + return self.records.get(identifier) + return self.records[identifier] + + self.arm.request.side_effect = request + + async def test_private_configuration_reads_no_secrets_or_app_settings(self): + self.assertEqual(await h.validate_resources(self.arm, SETTINGS, IDS, TAGS), PRINCIPAL) + for call in self.arm.request.await_args_list: + self.assertNotIn("/secrets/", call.args[1]) + self.assertNotIn("/appsettings", call.args[1]) + + async def test_role_cleanup_waits_for_rbac_convergence(self): + self.arm.request.side_effect = [ + {"value": [{"properties": {"principalId": PRINCIPAL}}]}, + {"value": []}, + ] + with patch.object(h.asyncio, "sleep", AsyncMock()) as sleep: + await h.assert_roles_removed(self.arm, IDS["vault"], PRINCIPAL) + self.assertEqual(self.arm.request.await_count, 2) + sleep.assert_awaited_once_with(10) + + async def test_public_web_app_fails_configuration(self): + self.records[IDS["apps"]["atlas"]]["properties"]["publicNetworkAccess"] = "Enabled" + with self.assertRaisesRegex(AssertionError, "public access"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_public_postgres_fails_configuration(self): + self.records[IDS["postgres"]]["properties"]["network"]["publicNetworkAccess"] = "Enabled" + with self.assertRaisesRegex(AssertionError, "public access"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_wrong_tenant_or_vault_role_is_rejected(self): + self.records[IDS["identity"]]["properties"]["tenantId"] = SUB + with self.assertRaises(AssertionError): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.records = arm_records() + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"][0]["properties"][ + "scope" + ] = SETTINGS.group + with self.assertRaises(AssertionError): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_paginated_roles_are_not_treated_as_complete(self): + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["nextLink"] = "next-page" + with self.assertRaisesRegex(AssertionError, "Incomplete"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_wrong_owner_blocks_cleanup_guard(self): + self.records[IDS["postgres"]]["tags"]["tre_workspace_service_id"] = WS + with self.assertRaisesRegex(AssertionError, "ownership"): + await h.guard_owned(self.arm, IDS, TAGS) + + async def test_removal_does_not_treat_api_failure_as_absence(self): + self.arm.request.side_effect = RuntimeError("HTTP 403") + with self.assertRaisesRegex(RuntimeError, "HTTP 403"): + await h.assert_removed(self.arm, [(IDS["postgres"], h.POSTGRES_API)]) + + def test_ids_remain_in_owned_workspace(self): + self.assertTrue(IDS["apps"]["atlas"].endswith("/app-ohdsi-atlas-testtre-ws-3333-svc-4444")) + self.assertTrue(IDS["postgres_subnet"].endswith("/PostgreSQLSubnet4444")) + self.assertTrue(all(identifier.startswith(IDS["group"] + "/") for identifier, _ in IDS["removal"])) + for value in ("", "../other", "00000000-0000-0000-0000-000000000000"): + with self.assertRaises(ValueError): + h.resource_ids(SETTINGS, value, SERVICE) + + def test_firewall_restores_unrelated_collections(self): + schema = json.loads((ROOT / "templates/workspace_services/ohdsi/template_schema.json").read_text()) + props = schema["pipeline"]["install"][-1]["properties"] + baseline = {"network_rule_collections": [{"name": "unrelated", "rules": []}], "rule_collections": []} + actual = {"properties": deepcopy(baseline)} + for prop in props: + collection = deepcopy(prop["value"]) + collection["name"] = collection["name"].replace("{{ resource.id }}", SERVICE) + collection["rules"][0]["source_addresses"] = ["10.0.1.0/24"] + actual["properties"][prop["name"]].append(collection) + h.validate_firewall(actual, baseline, SERVICE, ["10.0.1.0/24"]) + actual["properties"]["network_rule_collections"][0]["rules"] = ["changed"] + with self.assertRaisesRegex(AssertionError, "Unrelated"): + h.validate_firewall(actual, baseline, SERVICE, ["10.0.1.0/24"]) + + def test_deadline_starts_before_container_setup_and_is_forwarded(self): + workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) + jobs = [ + j + for j in workflow["jobs"].values() + if any(s.get("name") == "Reserve OHDSI cleanup and reporting time" for s in j.get("steps", [])) + ] + self.assertEqual(len(jobs), 1) + steps = jobs[0]["steps"] + deadline = next(i for i, s in enumerate(steps) if s.get("name") == "Reserve OHDSI cleanup and reporting time") + run = next(i for i, s in enumerate(steps) if s.get("name") == "Run E2E Tests") + self.assertLess(deadline, run) + self.assertIn("270 * 60", steps[deadline]["run"]) + self.assertIn("inputs.e2eBundle == 'tre-workspace-service-ohdsi'", steps[deadline]["if"]) + self.assertIn("OHDSI_VALIDATION_DEADLINE=${{ env.OHDSI_VALIDATION_DEADLINE }}", steps[run]["with"]["COMMAND"]) + self.assertEqual(jobs[0]["timeout-minutes"], 300) + self.assertIn("e2e_tests_smoke", jobs[0]["needs"]) + + +class PrerequisiteTests(unittest.IsolatedAsyncioTestCase): + async def test_missing_firewall_is_rejected(self): + async def get(endpoint, *args): + if endpoint == "/api/shared-services": + return {"sharedServices": []} + name = endpoint.rsplit("/", 1)[1] + return {"name": name, "version": case.load_catalog()[name]["source_version"]} + + with ( + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "get_resource", side_effect=get), + ): + with self.assertRaisesRegex(ValueError, "exactly one"): + await case.prerequisites(True) + + async def test_stale_template_is_rejected(self): + with ( + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "get_resource", AsyncMock(return_value={"name": h.TEMPLATE, "version": "0.0.0"})), + ): + with self.assertRaisesRegex(ValueError, "checkout version"): + await case.prerequisites(True) + + +class SharedDnsTests(unittest.IsolatedAsyncioTestCase): + async def test_core_link_requires_expected_target_and_settings(self): + arm = AsyncMock() + arm.request.return_value = core_link() + self.assertEqual(await h.postgres_core_dns_link(arm, SETTINGS), core_link()) + for key, value in ( + ("virtualNetwork", {"id": IDS["vnet"]}), + ("registrationEnabled", True), + ("provisioningState", "Failed"), + ): + with self.subTest(key=key): + link = core_link() + link["properties"][key] = value + arm.request.return_value = link + with self.assertRaises(AssertionError): + await h.postgres_core_dns_link(arm, SETTINGS) + + async def test_core_link_does_not_treat_read_failure_as_absence(self): + arm = AsyncMock() + arm.request.side_effect = RuntimeError("HTTP 403") + with self.assertRaisesRegex(RuntimeError, "HTTP 403"): + await h.postgres_core_dns_link(arm, SETTINGS, missing_ok=True) + + async def exercise_cleanup(self, *, existing=True, create_link=False, damage=None): + records = {CORE_LINK: core_link()} if existing else {} + arm = AsyncMock() + arm.__aenter__.return_value = arm + workspace_path = f"/workspaces/{WS}" + workspace = { + "templateVersion": case.load_catalog()[h.BASE]["source_version"], + "properties": {"auth_type": "Automatic", "aad_redirect_uris": []}, + } + firewall = {"id": "firewall", "templateName": h.FIREWALL, "properties": {}} + events = [] + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if identifier == CORE_LINK: + events.append("read link") + value = records.get(identifier) + if value is None and not kwargs.get("missing_ok"): + raise AssertionError("Shared PostgreSQL core DNS link is missing") + return value + if identifier == IDS["group"]: + return {"id": identifier, "tags": {"tre_id": SETTINGS.tre_id, "tre_workspace_id": WS}} + self.assertEqual(identifier, IDS["identity"]) + return None + + async def create(payload, endpoint, *args, **kwargs): + if endpoint == "/api/workspaces": + self.assertEqual(events, ["read link"]) + return workspace_path, WS + # Exercise cleanup after a partial install, before normal inspection. + if create_link: + records[CORE_LINK] = core_link() + await kwargs["before_remove"]("service", SERVICE) + events.append("remove service") + if damage == "delete": + records.pop(CORE_LINK) + elif damage == "retarget": + records[CORE_LINK]["properties"]["virtualNetwork"]["id"] = IDS["vnet"] + await kwargs["after_remove"]("service", SERVICE) + raise RuntimeError("simulated partial install") + + @asynccontextmanager + async def lifecycle(_verify): + yield SimpleNamespace(create=create) + + arm.request.side_effect = request + with ( + patch.object(case, "arm_client", return_value=arm), + patch.object(case, "ohdsi_lifecycle", lifecycle), + patch.object(case, "require_settings", return_value=SETTINGS), + patch.object(case, "prerequisites", AsyncMock(return_value=firewall)), + patch.object(case, "preflight", AsyncMock()), + patch.object(case, "guard_owned", AsyncMock()), + patch.object(case, "assert_removed", AsyncMock()), + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "record_deployed_resource"), + patch.object( + case, "get_resource", AsyncMock(return_value={"workspace": workspace, "sharedService": firewall}) + ), + ): + await case.validate_lifecycle(True) + + async def test_existing_link_survives_cleanup(self): + with self.assertRaisesRegex(RuntimeError, "simulated partial install"): + await self.exercise_cleanup() + + async def test_cleanup_detects_deleted_existing_or_new_link(self): + for existing in (True, False): + with self.subTest(existing=existing): + with self.assertRaisesRegex(AssertionError, "core DNS link is missing"): + await self.exercise_cleanup(existing=existing, create_link=True, damage="delete") + + async def test_cleanup_detects_retargeted_core_link(self): + with self.assertRaisesRegex(AssertionError, "ARM resource ID"): + await self.exercise_cleanup(damage="retarget") + + async def test_failed_install_before_link_creation_allows_absence(self): + with self.assertRaisesRegex(RuntimeError, "simulated partial install"): + await self.exercise_cleanup(existing=False) diff --git a/e2e_tests/unit_tests/test_ohdsi_lifecycle.py b/e2e_tests/unit_tests/test_ohdsi_lifecycle.py new file mode 100644 index 000000000..a40ebad9b --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_lifecycle.py @@ -0,0 +1,234 @@ +"""Exercise OHDSI ownership, pipeline completion and bounded cleanup failures.""" + +import asyncio +from builtins import ExceptionGroup +import os +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from e2e_tests.resources import ohdsi_lifecycle as lc + +WORKSPACE = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +OPERATION = "33333333-3333-4333-8333-333333333333" +OLD = "44444444-4444-4444-8444-444444444444" +ENDPOINT = f"/api/workspaces/{WORKSPACE}/workspace-services" +PATH = ENDPOINT.removeprefix("/api") + "/" + SERVICE +PAYLOAD = { + "templateName": "tre-workspace-service-ohdsi", + "properties": {"display_name": "unique name", "description": "owned resource", "configure_data_source": False}, +} + + +def response(body=None, code=200): + return SimpleNamespace(status_code=code, json=lambda: body) + + +def owner(): + return lc.OwnedResource( + PATH, SERVICE, "workspaceService", PAYLOAD["templateName"], "unique name", "owned resource", WORKSPACE + ) + + +def record(state="deployed", **properties): + return { + "id": SERVICE, + "templateName": PAYLOAD["templateName"], + "properties": PAYLOAD["properties"] | properties, + "deploymentStatus": state, + "_etag": "tag", + } + + +def operations(state="deployed", identifier=OPERATION): + return response({"operations": [{"id": identifier, "status": state}]}) + + +class LifecycleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + now = asyncio.get_running_loop().time() + self.resources = lc.OHDSIResources(True, now + 5, now + 10) + self.client = AsyncMock() + self.client.__aenter__.return_value = self.client + self.client.post.return_value = response( + {"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH}}, 202 + ) + self.enterContext(patch.object(lc, "AsyncClient", return_value=self.client)) + self.token = self.enterContext(patch.object(lc, "get_workspace_owner_token", AsyncMock(return_value="owner"))) + self.admin = self.enterContext(patch.object(lc, "get_admin_token", AsyncMock(return_value="admin"))) + self.evidence = self.enterContext(patch.object(lc, "record_resource")) + self.enterContext(patch.object(lc, "record_operation")) + self.enterContext(patch.object(lc, "get_full_endpoint", side_effect=lambda p: "https://tre" + p)) + self.remove = self.enterContext(patch.object(lc, "disable_and_delete_resource", AsyncMock())) + + async def test_accepted_resource_is_owned_before_client_exit_cancellation(self): + self.client.__aexit__.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].path, PATH) + self.evidence.assert_called_once() + self.token.assert_awaited_once_with(WORKSPACE, True) + self.admin.assert_not_awaited() + + async def test_evidence_error_does_not_lose_accepted_identity(self): + self.evidence.side_effect = OSError("disk full") + with self.assertRaises(OSError): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].identifier, SERVICE) + + async def test_external_source_or_wrong_parent_is_rejected_before_post(self): + for properties in ({"configure_data_source": True}, {"data_source_config": {}}, {"data_source_daimons": {}}): + with self.assertRaises(ValueError): + await self.resources.create( + PAYLOAD | {"properties": PAYLOAD["properties"] | properties}, ENDPOINT, WORKSPACE + ) + with self.assertRaises(ValueError): + await self.resources.create(PAYLOAD, "/api/shared-services", WORKSPACE) + self.client.post.assert_not_awaited() + + async def test_failed_pipeline_remains_owned(self): + with patch.object( + self.resources, "wait_terminal", AsyncMock(return_value=(record(), [{"status": "pipeline_failed"}])) + ): + with self.assertRaisesRegex(RuntimeError, "pipeline"): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].path, PATH) + + async def test_wait_requires_current_operation_and_terminal_pipeline(self): + self.client.get.side_effect = [ + response({"workspaceService": record()}), + operations(identifier=OLD), + response({"workspaceService": record()}), + operations("pipeline_running"), + response({"workspaceService": record()}), + operations("pipeline_succeeded"), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + await self.resources.wait_terminal(owner(), operation_id=OPERATION) + self.assertEqual(sleep.await_count, 2) + + async def test_new_resource_without_a_status_keeps_polling(self): + self.client.get.side_effect = [ + response({"workspaceService": record(None)}), + operations("awaiting_deployment"), + response({"workspaceService": record()}), + operations(), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + deployed, _ = await self.resources.wait_terminal(owner()) + self.assertEqual(deployed["deploymentStatus"], "deployed") + sleep.assert_awaited_once_with(30) + + async def test_ownership_change_prevents_deletion(self): + self.resources.owned = [owner()] + self.client.get.return_value = response({"workspaceService": record(description="different owner")}) + errors = await self.resources.close() + self.assertEqual(len(errors), 1) + self.remove.assert_not_awaited() + + async def test_reverse_cleanup_reserves_parent_after_child(self): + workspace = lc.OwnedResource( + "/workspaces/" + WORKSPACE, WORKSPACE, "workspace", "tre-workspace-base", "test", "owned", None + ) + self.resources.owned = [workspace, owner()] + with patch.object(self.resources, "remove", AsyncMock()) as remove: + self.assertEqual(await self.resources.close(), []) + self.assertEqual([c.args[0].identifier for c in remove.await_args_list], [SERVICE, WORKSPACE]) + + async def test_child_failure_retains_parent_and_reports_both(self): + workspace = lc.OwnedResource( + "/workspaces/" + WORKSPACE, WORKSPACE, "workspace", "tre-workspace-base", "test", "owned", None + ) + self.resources.owned = [workspace, owner()] + with patch.object(self.resources, "remove", AsyncMock(side_effect=RuntimeError("delete failed"))) as remove: + errors = await self.resources.close() + self.assertEqual(remove.await_count, 1) + self.assertEqual(len(errors), 2) + self.assertIn("retained", str(errors[1])) + + async def test_absent_resource_still_checks_azure_and_pipeline_cleanup(self): + resource = owner() + resource.after_remove = AsyncMock() + with patch.object(self.resources, "wait_terminal", AsyncMock(return_value=None)): + await self.resources.remove(resource) + self.remove.assert_not_awaited() + resource.after_remove.assert_awaited_once_with(PATH, SERVICE) + + async def test_same_version_upgrade_waits_for_returned_operation(self): + self.client.patch.return_value = response( + {"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH}}, 202 + ) + wait = AsyncMock( + side_effect=[ + (record(), []), + (record("updated", overview="new"), [{"id": OPERATION, "status": "pipeline_succeeded"}]), + ] + ) + with patch.object(self.resources, "wait_terminal", wait): + await self.resources.upgrade(owner(), "new") + self.assertEqual(wait.await_args.kwargs, {"operation_id": OPERATION}) + self.assertEqual(self.client.patch.call_args.kwargs["headers"]["etag"], "tag") + self.assertEqual(self.client.patch.call_args.kwargs["json"], {"properties": {"overview": "new"}}) + + async def test_deadline_rejects_nonfinite_or_exhausted_budget(self): + for value in ("nan", "inf", "0"): + with patch.dict(os.environ, OHDSI_VALIDATION_DEADLINE=value): + with self.assertRaises((ValueError, TimeoutError)): + lc.lifecycle_deadlines() + + async def test_cleanup_failure_fails_an_otherwise_successful_case(self): + now = asyncio.get_running_loop().time() + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 2, now + 3)), + patch.object(lc.OHDSIResources, "close", AsyncMock(return_value=[RuntimeError("cleanup failed")])), + ): + with self.assertRaises(ExceptionGroup): + async with lc.ohdsi_lifecycle(True): + pass + + async def test_cleanup_failure_note_keeps_the_cleanup_error_message(self): + now = asyncio.get_running_loop().time() + cleanup_error = RuntimeError("delete returned HTTP 409") + cleanup_error.add_note("OHDSI validation cleanup failed for /workspaces/w/workspace-services/s") + original = ValueError("validation failed") + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 2, now + 3)), + patch.object(lc.OHDSIResources, "close", AsyncMock(return_value=[cleanup_error])), + ): + with self.assertRaises(ValueError) as raised: + async with lc.ohdsi_lifecycle(True): + raise original + self.assertIs(raised.exception, original) + notes = " ".join(original.__notes__) + self.assertIn("delete returned HTTP 409", notes) + self.assertIn("/workspaces/w/workspace-services/s", notes) + + async def test_cancellation_waits_for_cleanup(self): + started = asyncio.Event() + release = asyncio.Event() + done = Mock() + + async def close(): + started.set() + await release.wait() + done() + return [] + + async def run(): + async with lc.ohdsi_lifecycle(True): + pass + + with ( + patch.object(lc.OHDSIResources, "close", side_effect=close), + patch.dict(os.environ, OHDSI_VALIDATION_DEADLINE=""), + ): + task = asyncio.create_task(run()) + await started.wait() + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await task + done.assert_called_once() From 293137740149bf8f21d307f272d4bdfb60c4dd99 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 22 Oct 2026 09:04:31 +0100 Subject: [PATCH 08/24] Upload the OHDSI Atlas UI config with the storage account key Since azurerm 4, azurerm_storage_share_file uses Azure AD when storage_use_azuread is true. The resource processor has no file data role, so the upload failed with HTTP 403 and OHDSI did not deploy. The share create and the web app mount already use the storage account key, so the upload now uses a provider alias with the key as well. This also keeps upgrades of older OHDSI services working: their refresh can read the file, so Terraform does not try to create it again. --- CHANGELOG.md | 1 + e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 8 ++++++++ templates/workspace_services/ohdsi/terraform/atlas_ui.tf | 1 + templates/workspace_services/ohdsi/terraform/providers.tf | 8 ++++++++ 4 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bd8d9aba..14c284130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ BUG FIXES: * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) +* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index 7e8be7d6c..4cd97dd83 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -209,6 +209,14 @@ def test_uninstall_removes_only_owned_firewall_collections(self): self.assertEqual(prop["arrayMatchField"], "name") self.assertLess(uninstall.index(cleanup[0]), next(i for i, s in enumerate(uninstall) if s["stepId"] == "main")) + def test_atlas_config_upload_uses_the_storage_account_key(self): + providers = (BUNDLE / "terraform/providers.tf").read_text() + alias = providers.split('alias = "shared_key"', 1)[1].split("\n}", 1)[0] + self.assertRegex(alias, r"storage_use_azuread\s*=\s*false") + source = (BUNDLE / "terraform/atlas_ui.tf").read_text() + upload = source.split('resource "azurerm_storage_share_file" "config_local" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(upload, r"provider\s*=\s*azurerm\.shared_key") + def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") diff --git a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf index dab2236af..115f4eb05 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf @@ -10,6 +10,7 @@ resource "local_file" "config_local" { } resource "azurerm_storage_share_file" "config_local" { + provider = azurerm.shared_key name = "config-local.js" storage_share_id = azurerm_storage_share.atlas_ui.id source = local.config_local_file_path diff --git a/templates/workspace_services/ohdsi/terraform/providers.tf b/templates/workspace_services/ohdsi/terraform/providers.tf index e2f5f9849..11b59e057 100644 --- a/templates/workspace_services/ohdsi/terraform/providers.tf +++ b/templates/workspace_services/ohdsi/terraform/providers.tf @@ -35,6 +35,14 @@ provider "azurerm" { storage_use_azuread = true } +# Uploads the Atlas UI config with the storage account key. The share and its web app mount already use the key, +# and Azure AD access to file data needs a role that the resource processor does not have. +provider "azurerm" { + alias = "shared_key" + features {} + storage_use_azuread = false +} + module "terraform_azurerm_environment_configuration" { source = "git::https://github.com/microsoft/terraform-azurerm-environment-configuration.git?ref=0.7.0" arm_environment = var.arm_environment From 094aa8659ad23cf651340f99342b77d3748449fa Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 22 Oct 2026 14:35:07 +0100 Subject: [PATCH 09/24] Create the OHDSI web apps after the PostgreSQL subnet writes (#5149) Azure rejects a web app subnet link while another subnet write runs in the same VNet. In a live run in another fork, the Atlas UI web app started its link to WebAppsSubnet while Terraform wrote the NSG association to the new PostgreSQL subnet. Azure returned ReferencedResourceNotProvisioned and OHDSI did not deploy. An earlier run passed only because the link ended 0.7 seconds before that write started. The PostgreSQL service also writes its subnet during the server create, so the Atlas UI web app now waits for the server. The WebAPI web app waits for the Atlas UI web app, so that the two subnet links do not overlap. The WebAPI web app already waited for the database initialisation, so the deployment time does not change much. --- CHANGELOG.md | 1 + e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 10 ++++++++++ .../workspace_services/ohdsi/terraform/atlas_ui.tf | 3 +++ .../ohdsi/terraform/ohdsi_web_api.tf | 4 +++- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c284130..faa16b6de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ BUG FIXES: * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) +* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index 4cd97dd83..66d91a0ab 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -217,6 +217,16 @@ def test_atlas_config_upload_uses_the_storage_account_key(self): upload = source.split('resource "azurerm_storage_share_file" "config_local" {', 1)[1].split("\n}", 1)[0] self.assertRegex(upload, r"provider\s*=\s*azurerm\.shared_key") + def test_web_apps_join_the_subnet_after_the_postgres_subnet_writes(self): + terraform = BUNDLE / "terraform" + + def depends_on(path, address): + block = (terraform / path).read_text().split(f'resource "azurerm_linux_web_app" "{address}" {{', 1)[1] + return block.split("depends_on = [", 1)[1].split("]", 1)[0] + + self.assertIn("azurerm_postgresql_flexible_server.postgres", depends_on("atlas_ui.tf", "atlas_ui")) + self.assertIn("azurerm_linux_web_app.atlas_ui", depends_on("ohdsi_web_api.tf", "ohdsi_webapi")) + def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") diff --git a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf index 115f4eb05..8b8649e54 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf @@ -70,6 +70,9 @@ resource "azurerm_linux_web_app" "atlas_ui" { depends_on = [ azurerm_storage_share_file.config_local, + # Azure rejects the web app subnet link while another subnet write runs in the same VNet. + # The PostgreSQL server create makes the last write to the PostgreSQL subnet. + azurerm_postgresql_flexible_server.postgres, ] tags = local.tre_workspace_service_tags diff --git a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf index 328498ef1..85b91eaea 100644 --- a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf +++ b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf @@ -111,7 +111,9 @@ resource "azurerm_linux_web_app" "ohdsi_webapi" { tags = local.tre_workspace_service_tags depends_on = [ - terraform_data.deployment_ohdsi_webapi_init + terraform_data.deployment_ohdsi_webapi_init, + # Link the two web apps to the web app subnet one after the other. + azurerm_linux_web_app.atlas_ui, ] lifecycle { ignore_changes = [tags] } From d7f28fc1760209bfac7a7b9cecffae50ee7f0562 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 22 Oct 2026 18:17:23 +0100 Subject: [PATCH 10/24] Declare the Microsoft.Storage endpoint on the OHDSI PostgreSQL subnet During the server create, the PostgreSQL service adds the Microsoft.Storage service endpoint to its delegated subnet. Terraform did not declare it, so each upgrade removed it again. In a live run in another fork, the service added it at 16:00:06 and the upgrade removed it at 16:18:27. The subnet now declares the endpoint. New subnets get it at create, and an upgrade of an existing service makes no change to the subnet. --- CHANGELOG.md | 1 + e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 5 +++++ .../workspace_services/ohdsi/terraform/atlas_database.tf | 2 ++ 3 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faa16b6de..dccb93bf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ BUG FIXES: * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) * Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) +* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index 66d91a0ab..2ef8d2cb1 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -227,6 +227,11 @@ def depends_on(path, address): self.assertIn("azurerm_postgresql_flexible_server.postgres", depends_on("atlas_ui.tf", "atlas_ui")) self.assertIn("azurerm_linux_web_app.atlas_ui", depends_on("ohdsi_web_api.tf", "ohdsi_webapi")) + def test_postgres_subnet_declares_the_endpoint_that_the_service_adds(self): + source = (BUNDLE / "terraform/atlas_database.tf").read_text() + subnet = source.split('resource "azurerm_subnet" "postgres" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(subnet, r'service_endpoints\s*=\s*\["Microsoft\.Storage"\]') + def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") diff --git a/templates/workspace_services/ohdsi/terraform/atlas_database.tf b/templates/workspace_services/ohdsi/terraform/atlas_database.tf index 4bba2d071..ee677c64c 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_database.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_database.tf @@ -105,6 +105,8 @@ resource "azurerm_subnet" "postgres" { resource_group_name = data.azurerm_resource_group.ws.name address_prefixes = [var.address_space] default_outbound_access_enabled = false + # The PostgreSQL service adds this endpoint during the server create, so declare it to stop upgrades removing it. + service_endpoints = ["Microsoft.Storage"] delegation { name = "psql-delegation" From 916e8b14a62130419a499a4a9d88cbe9d6f9d8c4 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 26 Oct 2026 02:25:17 +0000 Subject: [PATCH 11/24] Ignore the firewall collections of other services in the OHDSI check (#5149) The OHDSI case compared the full firewall with a baseline. Other services that run at the same time add and remove their own "nrc_svc_" and "arc_svc_" collections, so the case failed in a run with the Health case in parallel. The check now leaves out the collections of other services. All other collections must stay equal to the baseline, and the OHDSI collections must go away after the delete. --- e2e_tests/resources/ohdsi.py | 16 +++++++++++++--- e2e_tests/test_ohdsi.py | 4 +++- .../unit_tests/test_ohdsi_configuration.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/e2e_tests/resources/ohdsi.py b/e2e_tests/resources/ohdsi.py index 2d62f1ab7..97f66d9d8 100644 --- a/e2e_tests/resources/ohdsi.py +++ b/e2e_tests/resources/ohdsi.py @@ -218,14 +218,24 @@ async def validate_resources(arm, settings, ids, tags): return principal -def firewall_rules(resource): +# Other services add and remove their own collections while this case runs. +SERVICE_COLLECTION = re.compile(r"(?:nrc|arc)_svc_([0-9a-f-]{36})") + + +def other_service(collection, service_id): + match = SERVICE_COLLECTION.match(collection.get("name", "")) + return match is not None and match.group(1) != service_id + + +def firewall_rules(resource, service_id=None): return { - key: deepcopy(resource["properties"].get(key, [])) for key in ("network_rule_collections", "rule_collections") + key: [deepcopy(c) for c in resource["properties"].get(key, []) if not other_service(c, service_id)] + for key in ("network_rule_collections", "rule_collections") } def validate_firewall(resource, baseline, service_id, prefixes): - rules = firewall_rules(resource) + rules = firewall_rules(resource, service_id) for key, prefix in (("network_rule_collections", "nrc"), ("rule_collections", "arc")): name = f"{prefix}_svc_{service_id}" matches = [c for c in rules[key] if c.get("name") == name] diff --git a/e2e_tests/test_ohdsi.py b/e2e_tests/test_ohdsi.py index b938df115..4a80b911f 100644 --- a/e2e_tests/test_ohdsi.py +++ b/e2e_tests/test_ohdsi.py @@ -156,7 +156,9 @@ async def removed_service(_path, identifier): await check_core_link() if principal: await assert_roles_removed(arm, ids["vault"], principal) - assert firewall_rules(await current_firewall()) == baseline, "OHDSI firewall rules were not restored" + assert firewall_rules(await current_firewall(), identifier) == baseline, ( + "OHDSI firewall rules were not restored" + ) assert (await workspace_record())["properties"].get("aad_redirect_uris", []) == callbacks, ( "OHDSI callback was not removed" ) diff --git a/e2e_tests/unit_tests/test_ohdsi_configuration.py b/e2e_tests/unit_tests/test_ohdsi_configuration.py index e6db92ba0..b9558e74c 100644 --- a/e2e_tests/unit_tests/test_ohdsi_configuration.py +++ b/e2e_tests/unit_tests/test_ohdsi_configuration.py @@ -207,6 +207,23 @@ def test_firewall_restores_unrelated_collections(self): with self.assertRaisesRegex(AssertionError, "Unrelated"): h.validate_firewall(actual, baseline, SERVICE, ["10.0.1.0/24"]) + def test_firewall_ignores_the_collections_of_other_services(self): + other = "55555555-5555-4555-8555-555555555555" + first = { + "properties": { + "network_rule_collections": [{"name": "core"}, {"name": f"nrc_svc_{other}_health_services"}], + "rule_collections": [], + } + } + baseline = h.firewall_rules(first) + self.assertEqual(baseline, {"network_rule_collections": [{"name": "core"}], "rule_collections": []}) + later = deepcopy(first) + later["properties"]["network_rule_collections"].pop() + later["properties"]["rule_collections"].append({"name": f"arc_svc_{other}_azureml_client"}) + self.assertEqual(h.firewall_rules(later, SERVICE), baseline) + later["properties"]["rule_collections"].append({"name": f"arc_svc_{SERVICE}"}) + self.assertNotEqual(h.firewall_rules(later, SERVICE), baseline) + def test_deadline_starts_before_container_setup_and_is_forwarded(self): workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) jobs = [ From 16e2de49fb884c63f14e89a61b20837400f16870 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 25 Oct 2026 22:18:25 +0000 Subject: [PATCH 12/24] Give the OHDSI Web API identity a role on its own secrets only The OHDSI Web API identity had the role "Key Vault Secrets User" on the whole workspace Key Vault. The web app reads only five secrets through its Key Vault references: the JDBC connection string, the two Web API database passwords, and the workspace client ID and secret. The identity now has the role on each of these five secrets, and the web app waits for these roles. A contract test makes sure that the roles match the Key Vault references of the web app. Version: `tre-workspace-service-ohdsi` 0.3.15. --- CHANGELOG.md | 1 + e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 14 +++++++++++++- templates/workspace_services/ohdsi/porter.yaml | 2 +- .../ohdsi/terraform/ohdsi_web_api.tf | 14 ++++++++++++-- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccb93bf1..9354bb44e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) +* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.15) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) * Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index 2ef8d2cb1..e87d22907 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -2,6 +2,7 @@ import json import os +import re from pathlib import Path import subprocess import sys @@ -227,6 +228,17 @@ def depends_on(path, address): self.assertIn("azurerm_postgresql_flexible_server.postgres", depends_on("atlas_ui.tf", "atlas_ui")) self.assertIn("azurerm_linux_web_app.atlas_ui", depends_on("ohdsi_web_api.tf", "ohdsi_webapi")) + def test_web_api_identity_reads_only_its_key_vault_references(self): + # The workspace Key Vault also has the secrets of the other workspace services. + source = (BUNDLE / "terraform/ohdsi_web_api.tf").read_text() + self.assertNotRegex(source, r"\n scope\s*=\s*data\.azurerm_key_vault\.ws\.id\n") + roles = source.split('resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" {', 1)[1] + roles = roles.split("\n}", 1)[0] + scoped = set(re.findall(r"=\s*((?:data\.)?azurerm_key_vault_secret\.\w+)\.resource_versionless_id", roles)) + referenced = set(re.findall(r"SecretName=\$\{((?:data\.)?azurerm_key_vault_secret\.\w+)\.name\}", source)) + self.assertTrue(referenced) + self.assertEqual(scoped, referenced) + def test_postgres_subnet_declares_the_endpoint_that_the_service_adds(self): source = (BUNDLE / "terraform/atlas_database.tf").read_text() subnet = source.split('resource "azurerm_subnet" "postgres" {', 1)[1].split("\n}", 1)[0] @@ -236,4 +248,4 @@ def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) - self.assertEqual(manifest["version"], "0.3.14") + self.assertEqual(manifest["version"], "0.3.15") diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index a18290240..f8ad51eb3 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.14 +version: 0.3.15 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf index 85b91eaea..a1d732bd9 100644 --- a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf +++ b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf @@ -16,8 +16,17 @@ resource "azurerm_user_assigned_identity" "ohdsi_webapi_id" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_role_assignment" "keyvault_ohdsi_ws_role" { - scope = data.azurerm_key_vault.ws.id +# The web app reads only the secrets of its Key Vault references, so its identity gets a role on each of them. +resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" { + for_each = { + jdbc_connection_string = azurerm_key_vault_secret.jdbc_connection_string_webapi_admin.resource_versionless_id + webapi_admin_password = azurerm_key_vault_secret.postgres_webapi_admin_password.resource_versionless_id + webapi_app_password = azurerm_key_vault_secret.postgres_webapi_app_password.resource_versionless_id + workspace_client_id = data.azurerm_key_vault_secret.workspace_client_id.resource_versionless_id + workspace_client_secret = data.azurerm_key_vault_secret.workspace_client_secret.resource_versionless_id + } + + scope = each.value role_definition_name = "Key Vault Secrets User" principal_id = azurerm_user_assigned_identity.ohdsi_webapi_id.principal_id } @@ -112,6 +121,7 @@ resource "azurerm_linux_web_app" "ohdsi_webapi" { depends_on = [ terraform_data.deployment_ohdsi_webapi_init, + azurerm_role_assignment.keyvault_ohdsi_secret_role, # Link the two web apps to the web app subnet one after the other. azurerm_linux_web_app.atlas_ui, ] From f9b3e95926fb0304fc2c0b39dafaea8d6ceb38a9 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 26 Oct 2026 11:17:10 +0000 Subject: [PATCH 13/24] Expect the OHDSI roles on the secrets in the E2E test The OHDSI Web API identity now has a role on each secret of its Key Vault references, and no role on the vault. The E2E test still expected one role on the vault, so the lifecycle test failed with "Unexpected OHDSI vault role count". The test now reads the roles of the identity with the principalId filter. With this filter, Azure returns the roles at, above and below the vault. The identity must have the role "Key Vault Secrets User" on each of the five secrets, and no other role. The clean-up wait uses the same filter, so it also waits for the roles on the secrets. A unit test makes sure that the secret list of the E2E test matches the secret roles of the bundle. --- e2e_tests/resources/ohdsi.py | 36 +++++++++++--- .../unit_tests/test_ohdsi_configuration.py | 48 +++++++++++++++++-- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/e2e_tests/resources/ohdsi.py b/e2e_tests/resources/ohdsi.py index 97f66d9d8..e7f9e43c0 100644 --- a/e2e_tests/resources/ohdsi.py +++ b/e2e_tests/resources/ohdsi.py @@ -62,6 +62,15 @@ def resource_ids(settings, workspace_id, service_id): for key, app in apps.items() } storage = f"{group}/providers/Microsoft.Storage/storageAccounts/{('stg' + ws[-8:]).replace('-', '').lower()}" + vault = f"{group}/providers/Microsoft.KeyVault/vaults/kv-{ws[-20:].lower()}" + # The Web API reads these secrets through its Key Vault references. + secrets = ( + f"jdbc-connectionstring-{service_id[-4:]}", + f"ohdsi-admin-password-{service_id[-4:]}", + f"ohdsi-app-password-{service_id[-4:]}", + "workspace-client-id", + "workspace-client-secret", + ) ids = { "group": group, "vnet": vnet, @@ -72,7 +81,8 @@ def resource_ids(settings, workspace_id, service_id): "nsg": f"{group}/providers/Microsoft.Network/networkSecurityGroups/nsg-psql-{suffix}", "identity": f"{group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ohdsi-webapi-{suffix}", "share": f"{storage}/fileServices/default/shares/atlas-{suffix}", - "vault": f"{group}/providers/Microsoft.KeyVault/vaults/kv-{ws[-20:].lower()}", + "vault": vault, + "secrets": [f"{vault}/secrets/{name}" for name in secrets], "web_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.azurewebsites.net", "pg_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.postgres.database.azure.com", } @@ -204,7 +214,10 @@ async def validate_resources(arm, settings, ids, tags): assert require_uuid(identity["properties"].get("tenantId")) == settings.tenant principal = require_uuid(identity["properties"].get("principalId")) assignments = await arm.request( - "GET", ids["vault"] + "/providers/Microsoft.Authorization/roleAssignments", ROLE_API + "GET", + ids["vault"] + "/providers/Microsoft.Authorization/roleAssignments", + ROLE_API, + query=principal_filter(principal), ) assert isinstance(assignments.get("value"), list) and not assignments.get("nextLink"), ( "Incomplete OHDSI role inventory" @@ -212,9 +225,10 @@ async def validate_resources(arm, settings, ids, tags): roles = [ r["properties"] for r in assignments["value"] if r["properties"].get("principalId", "").lower() == principal ] - assert len(roles) == 1, "Unexpected OHDSI vault role count" - same_id(roles[0].get("scope"), ids["vault"]) - assert roles[0].get("roleDefinitionId", "").rsplit("/", 1)[-1].lower() == SECRETS_USER + # One role on each secret, and no role on the vault or above it. + scopes = sorted(r.get("scope", "").lower() for r in roles) + assert scopes == sorted(s.lower() for s in ids["secrets"]), "Unexpected OHDSI secret role scopes" + assert all(r.get("roleDefinitionId", "").rsplit("/", 1)[-1].lower() == SECRETS_USER for r in roles) return principal @@ -270,10 +284,20 @@ async def assert_removed(arm, identifiers): await asyncio.sleep(10) +def principal_filter(principal): + # With this filter, Azure returns the roles of the principal at, above and below the scope. + return {"$filter": f"principalId eq '{principal}'"} + + async def assert_roles_removed(arm, vault, principal): async with asyncio.timeout(5 * 60): while True: - roles = await arm.request("GET", vault + "/providers/Microsoft.Authorization/roleAssignments", ROLE_API) + roles = await arm.request( + "GET", + vault + "/providers/Microsoft.Authorization/roleAssignments", + ROLE_API, + query=principal_filter(principal), + ) assert isinstance(roles.get("value"), list) and not roles.get("nextLink"), ( "Incomplete OHDSI cleanup role inventory" ) diff --git a/e2e_tests/unit_tests/test_ohdsi_configuration.py b/e2e_tests/unit_tests/test_ohdsi_configuration.py index b9558e74c..46d91e97a 100644 --- a/e2e_tests/unit_tests/test_ohdsi_configuration.py +++ b/e2e_tests/unit_tests/test_ohdsi_configuration.py @@ -4,6 +4,7 @@ from contextlib import asynccontextmanager import json from pathlib import Path +import re from types import SimpleNamespace import unittest from unittest.mock import AsyncMock, patch @@ -69,10 +70,11 @@ def resource(identifier, properties): { "properties": { "principalId": PRINCIPAL, - "scope": IDS["vault"], + "scope": secret, "roleDefinitionId": "/roles/" + h.SECRETS_USER, } } + for secret in IDS["secrets"] ] }, } @@ -138,6 +140,28 @@ async def test_private_configuration_reads_no_secrets_or_app_settings(self): self.assertNotIn("/secrets/", call.args[1]) self.assertNotIn("/appsettings", call.args[1]) + async def test_roles_are_read_for_the_identity_also_below_the_vault(self): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + call = next(c for c in self.arm.request.await_args_list if c.args[1].endswith("/roleAssignments")) + self.assertEqual(call.kwargs["query"], {"$filter": f"principalId eq '{PRINCIPAL}'"}) + self.assertTrue(all(secret.startswith(IDS["vault"] + "/secrets/") for secret in IDS["secrets"])) + + def test_secrets_match_the_secret_roles_of_the_bundle(self): + terraform = ROOT / "templates/workspace_services/ohdsi/terraform" + source = "".join(path.read_text() for path in sorted(terraform.glob("*.tf"))) + roles = source.split('resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" {', 1)[1] + roles = roles.split("\n}", 1)[0] + names = set() + for data, resource in re.findall( + r"=\s*(data\.)?azurerm_key_vault_secret\.(\w+)\.resource_versionless_id", roles + ): + header = f'{"data" if data else "resource"} "azurerm_key_vault_secret" "{resource}" {{' + block = source.split(header, 1)[1].split("\n}", 1)[0] + name = re.search(r'\n name\s*=\s*"([^"]+)"', block).group(1) + names.add(name.replace("${local.short_service_id}", SERVICE[-4:])) + self.assertEqual(len(names), 5) + self.assertEqual({secret.rsplit("/", 1)[1] for secret in IDS["secrets"]}, names) + async def test_role_cleanup_waits_for_rbac_convergence(self): self.arm.request.side_effect = [ {"value": [{"properties": {"principalId": PRINCIPAL}}]}, @@ -146,6 +170,7 @@ async def test_role_cleanup_waits_for_rbac_convergence(self): with patch.object(h.asyncio, "sleep", AsyncMock()) as sleep: await h.assert_roles_removed(self.arm, IDS["vault"], PRINCIPAL) self.assertEqual(self.arm.request.await_count, 2) + self.assertEqual(self.arm.request.await_args.kwargs["query"], {"$filter": f"principalId eq '{PRINCIPAL}'"}) sleep.assert_awaited_once_with(10) async def test_public_web_app_fails_configuration(self): @@ -162,11 +187,24 @@ async def test_wrong_tenant_or_vault_role_is_rejected(self): self.records[IDS["identity"]]["properties"]["tenantId"] = SUB with self.assertRaises(AssertionError): await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + for scope in (SETTINGS.group, IDS["vault"]): + with self.subTest(scope=scope): + self.records = arm_records() + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"][0][ + "properties" + ]["scope"] = scope + with self.assertRaisesRegex(AssertionError, "secret role scopes"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_missing_or_extra_secret_role_is_rejected(self): + roles = self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"] + extra = deepcopy(roles[0]) + roles.pop() + with self.assertRaisesRegex(AssertionError, "secret role scopes"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) self.records = arm_records() - self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"][0]["properties"][ - "scope" - ] = SETTINGS.group - with self.assertRaises(AssertionError): + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"].append(extra) + with self.assertRaisesRegex(AssertionError, "secret role scopes"): await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) async def test_paginated_roles_are_not_treated_as_complete(self): From ec75b50bbe20c79297ca22bdd5dd838109bc483a Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 26 Oct 2026 13:18:58 +0000 Subject: [PATCH 14/24] Make sure that the OHDSI Web API resolves its Key Vault references The OHDSI E2E test read the roles of the Web API identity, but not the result of these roles. If the web app cannot read a secret, the state of the web app stays "Running". The test now reads the Key Vault references of the Web API. Azure returns the status and the secret name of each reference, not the value. Each of the five secrets must have a reference, and each reference must have the status "Resolved". The web app resolves its references when it starts, so the test reads them again for up to 4.5 minutes. The unit tests still make sure that the test reads no secret and no other app settings path. --- e2e_tests/resources/ohdsi.py | 20 ++++++++++- .../unit_tests/test_ohdsi_configuration.py | 33 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/e2e_tests/resources/ohdsi.py b/e2e_tests/resources/ohdsi.py index e7f9e43c0..32262e565 100644 --- a/e2e_tests/resources/ohdsi.py +++ b/e2e_tests/resources/ohdsi.py @@ -1,4 +1,4 @@ -"""Read OHDSI infrastructure without requesting application settings or secrets.""" +"""Read OHDSI infrastructure without requesting the values of application settings or secrets.""" import asyncio from copy import deepcopy @@ -229,9 +229,27 @@ async def validate_resources(arm, settings, ids, tags): scopes = sorted(r.get("scope", "").lower() for r in roles) assert scopes == sorted(s.lower() for s in ids["secrets"]), "Unexpected OHDSI secret role scopes" assert all(r.get("roleDefinitionId", "").rsplit("/", 1)[-1].lower() == SECRETS_USER for r in roles) + # The roles on the secrets must let the Web API resolve each of its Key Vault references. + references = await key_vault_references(arm, ids["apps"]["webapi"]) + names = {r.get("secretName", "").lower() for r in references} + assert names == {s.rsplit("/", 1)[1].lower() for s in ids["secrets"]}, "Unexpected OHDSI Key Vault references" + statuses = sorted({str(r.get("status")) for r in references}) + assert statuses == ["Resolved"], f"OHDSI Key Vault references are not resolved: {statuses}" return principal +async def key_vault_references(arm, site_id, attempts=10): + """Read the Key Vault references of a web app. Azure returns the status of each reference, not the value.""" + for attempt in range(attempts): + page = await arm.request("GET", site_id + "/config/configreferences/appsettings", WEB_API) + assert isinstance(page.get("value"), list) and not page.get("nextLink"), "Incomplete OHDSI Key Vault references" + references = [r.get("properties", {}) for r in page["value"]] + # The web app resolves its references when it starts, so a new web app can show an earlier status. + if (references and all(r.get("status") == "Resolved" for r in references)) or attempt == attempts - 1: + return references + await asyncio.sleep(30) + + # Other services add and remove their own collections while this case runs. SERVICE_COLLECTION = re.compile(r"(?:nrc|arc)_svc_([0-9a-f-]{36})") diff --git a/e2e_tests/unit_tests/test_ohdsi_configuration.py b/e2e_tests/unit_tests/test_ohdsi_configuration.py index 46d91e97a..0d46508ca 100644 --- a/e2e_tests/unit_tests/test_ohdsi_configuration.py +++ b/e2e_tests/unit_tests/test_ohdsi_configuration.py @@ -24,6 +24,7 @@ IDS = h.resource_ids(SETTINGS, WS, SERVICE) TAGS = {"tre_id": "testtre", "tre_workspace_id": WS, "tre_workspace_service_id": SERVICE} CORE_LINK = IDS["pg_zone"] + "/virtualNetworkLinks/core" +REFERENCES = IDS["apps"]["webapi"] + "/config/configreferences/appsettings" def core_link(): @@ -118,6 +119,11 @@ def resource(identifier, properties): records[zone_group] = resource( zone_group, {"privateDnsZoneConfigs": [{"properties": {"privateDnsZoneId": IDS["web_zone"]}}]} ) + records[REFERENCES] = { + "value": [ + {"properties": {"secretName": secret.rsplit("/", 1)[1], "status": "Resolved"}} for secret in IDS["secrets"] + ] + } return records @@ -138,7 +144,32 @@ async def test_private_configuration_reads_no_secrets_or_app_settings(self): self.assertEqual(await h.validate_resources(self.arm, SETTINGS, IDS, TAGS), PRINCIPAL) for call in self.arm.request.await_args_list: self.assertNotIn("/secrets/", call.args[1]) - self.assertNotIn("/appsettings", call.args[1]) + # Azure returns the status of a Key Vault reference, not the value of the app setting. + self.assertNotIn("/appsettings", call.args[1].removesuffix("/config/configreferences/appsettings")) + self.assertIn(REFERENCES, [call.args[1] for call in self.arm.request.await_args_list]) + + async def test_key_vault_references_wait_for_the_web_app_start(self): + resolved = self.records[REFERENCES] + self.records[REFERENCES] = deepcopy(resolved) + self.records[REFERENCES]["value"][0]["properties"]["status"] = "Initialized" + with patch.object( + h.asyncio, "sleep", AsyncMock(side_effect=lambda _: self.records.update({REFERENCES: resolved})) + ): + self.assertEqual(await h.validate_resources(self.arm, SETTINGS, IDS, TAGS), PRINCIPAL) + + async def test_unresolved_or_missing_key_vault_reference_is_rejected(self): + self.records[REFERENCES]["value"][0]["properties"]["status"] = "AccessToKeyVaultDenied" + with patch.object(h.asyncio, "sleep", AsyncMock()) as sleep: + with self.assertRaisesRegex(AssertionError, "AccessToKeyVaultDenied"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.assertEqual(sleep.await_count, 9) + self.records = arm_records() + self.records[REFERENCES]["value"].pop() + with self.assertRaisesRegex(AssertionError, "Unexpected OHDSI Key Vault references"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.records[REFERENCES]["nextLink"] = "next-page" + with self.assertRaisesRegex(AssertionError, "Incomplete OHDSI Key Vault references"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) async def test_roles_are_read_for_the_identity_also_below_the_vault(self): await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) From 1a10c551ff66dd860741c17afd817985c664e764 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 30 Oct 2026 23:14:55 +0000 Subject: [PATCH 15/24] Use one OHDSI version bump Earlier commits bumped the OHDSI bundle two times on an earlier main. Set one bump from the current main, 0.3.14, in the bundle, the CHANGELOG and the contract test. --- CHANGELOG.md | 2 +- e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 2 +- templates/workspace_services/ohdsi/porter.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9354bb44e..b5fc84138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) -* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.15) +* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) * Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index e87d22907..1c60594a6 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -248,4 +248,4 @@ def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) - self.assertEqual(manifest["version"], "0.3.15") + self.assertEqual(manifest["version"], "0.3.14") diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index f8ad51eb3..a18290240 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.15 +version: 0.3.14 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl From 23ec913f00a36501de4ff42912b78ebdc3b08d98 Mon Sep 17 00:00:00 2001 From: Jay W Date: Sat, 31 Oct 2026 02:09:21 +0000 Subject: [PATCH 16/24] Fix review points of the CycleCloud and OHDSI change - The CycleCloud comment named 0.7.11, a version that this change does not use. The rule keeps the size of each existing server, so the comment now names no version. - The E2E document described the old vault role check of OHDSI. It now names the role on each of the five secrets and the Key Vault references. - The OHDSI contract test for the image tag also compared the bundle version with 0.3.14, so each version change broke it. The test now checks only the image tag. - The CycleCloud and OHDSI CHANGELOG entries get their issue links (#5159, #5149). --- CHANGELOG.md | 10 +++++----- docs/tre-developers/end-to-end-tests.md | 3 ++- e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 3 --- .../shared_services/cyclecloud/terraform/cyclecloud.tf | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5fc84138..c2bc1d552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,12 +60,12 @@ BUG FIXES: * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) -* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) -* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) +* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) -* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) -* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) +* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) +* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) +* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index f5b0da48b..562ace12c 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -277,7 +277,8 @@ Publish and register the checkout base-workspace, firewall and OHDSI bundles. De Check PostgreSQL 14 capacity for `B_Standard_B1ms`, 32 GiB storage and availability zone 1 in the selected region. The case creates a fresh Automatic workspace. It disables backups and sets `configure_data_source=false` without data-source properties. -The case checks both private web apps, their pinned images, PostgreSQL networking and the WebAPI identity's vault role. +The case checks both private web apps, their pinned images and PostgreSQL networking. +It also checks that the WebAPI identity has a role on each of its five Key Vault secrets and no role on the vault, and that each Key Vault reference resolves. Successful installation includes the bundle's private WebAPI readiness check and local administrator setup. A metadata update exercises a same-version upgrade. The test checks the callback and firewall collections after each operation. It waits for terminal operations before updates or deletion. It owns accepted resources before evidence writes or client closure. diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index 1c60594a6..ea16a05b7 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -9,7 +9,6 @@ import tempfile import unittest -import yaml ROOT = Path(__file__).resolve().parents[2] BUNDLE = ROOT / "templates/workspace_services/ohdsi" @@ -247,5 +246,3 @@ def test_postgres_subnet_declares_the_endpoint_that_the_service_adds(self): def test_bootstrap_version_matches_pinned_image(self): source = (BUNDLE / "terraform/atlas_security.tf").read_text() self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") - manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) - self.assertEqual(manifest["version"], "0.3.14") diff --git a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf index c5c12b469..02b6082fb 100644 --- a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf +++ b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf @@ -70,7 +70,7 @@ resource "azurerm_virtual_machine" "cyclecloud" { tags = local.tre_shared_service_tags - # Azure does not offer Standard_DS3_v2 to all subscriptions in all regions. Keep the size of servers made before 0.7.11. + # Azure does not offer Standard_DS3_v2 to all subscriptions in all regions. Keep the size of existing servers. lifecycle { ignore_changes = [tags, vm_size] } } From 365f248acc81931ccba899ec2a60002e12f9bfca Mon Sep 17 00:00:00 2001 From: Jay W Date: Sun, 1 Nov 2026 01:27:53 +0000 Subject: [PATCH 17/24] Make the OHDSI role test fail on the old script and correct two texts (#5149) The fake psql now returns the error only when ON_ERROR_STOP is on, as a real psql does. So the role test fails on a script that continues after a failed CREATE ROLE. The test no longer makes envsubst and md5sum fakes that no script calls. The CHANGELOG line of the evidence helper gives its real scope, and the OHDSI lines write WebAPI. The E2E document names the deployed version record and has a blank line before the OHDSI heading. --- CHANGELOG.md | 4 ++-- docs/tre-developers/end-to-end-tests.md | 2 ++ e2e_tests/unit_tests/test_ohdsi_bundle_contract.py | 5 ++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bc1d552..fdb43cec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ ENHANCEMENTS: The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) * Add an independent CycleCloud server lifecycle with marketplace checks, VM power actions and bounded cleanup. ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Add independent private OHDSI lifecycle validation with callback, firewall, shared DNS preservation and bounded cleanup checks. ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Record the deployed template version of each resource that a bundle test reads in the bundle evidence, without the resource properties. +* Add a bundle evidence helper that records the deployed template version of a resource that a test gives to it, without the resource properties. * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) @@ -61,7 +61,7 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) -* Give the OHDSI Web API identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) +* Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 562ace12c..0696aa32c 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -233,6 +233,7 @@ The **E2E Test Results (Bundle)** artefact contains: The JSON report copies only selected identifiers and operation states. It excludes resource properties, tokens and failure tracebacks. Source versions describe the checked-out manifests; verify deployed versions separately when reusing an environment. +A test can call `record_deployed_resource` for a resource that it reads. The report then has a `deployed_template_version` record for that resource. `passed_selected_cases` means the selected assertions and their teardown passed. The report always leaves `full_lifecycle_proven` false for a separate evidence review. An entirely skipped selection exits unsuccessfully. Individual skipped cases retain their reasons, and a mixed pass/skip result is marked `partial`. @@ -264,6 +265,7 @@ Failed cleanup fails the test. Core Key Vault credentials can remain soft-delete This case does not configure the CycleCloud application or create an HPC cluster. Browser access, application TLS trust, private DNS reachability and prior-release upgrades remain unproven. The existing bundle grants subscription Contributor to its VM and permits SharedSubnet access to ARM and GitHub. It does not disable the storage public endpoint. A private endpoint and a successful server lifecycle do not prove workspace isolation or storage access restrictions. + ### Private OHDSI validation Select the bundle from a branch that contains its repaired bootstrap and validation case: diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py index ea16a05b7..5d24634ac 100644 --- a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -43,11 +43,10 @@ def setUp(self): args=sys.argv[1:] body=Path(args[args.index('-f')+1]).read_text() if '-f' in args else sys.stdin.read() with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'args':args,'body':body})+'\\n') -sys.exit(int(os.environ.get('PSQL_EXIT','0'))) +# A real psql returns 0 after a failed statement unless ON_ERROR_STOP is on. +sys.exit(int(os.environ.get('PSQL_EXIT','0')) if 'ON_ERROR_STOP=1' in args else 0) """, ) - self.fake("envsubst", "import os,sys\nprint(os.path.expandvars(sys.stdin.read()))\n") - self.fake("md5sum", "import hashlib,sys\nprint(hashlib.md5(sys.stdin.buffer.read()).hexdigest())\n") def fake(self, name, body): p = self.folder / name From 32b0d55aaa44d98f83b9228a59018ee8e8fce7f7 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 12:34:31 +0000 Subject: [PATCH 18/24] Add network rules to the CycleCloud server The network card of the CycleCloud server had no network security group, and the SharedSubnet group has only the Azure default rules. A new group on the network card sets these inbound rules: - Allow TCP 443 and 22 from the core address space, which has the admin jumpbox and the Azure Bastion subnet. - Allow TCP 9443 from the address spaces in the new property cluster_address_spaces. Cluster nodes without a return proxy use this port. The rule exists only when the list is not empty. - Allow the AzureLoadBalancer service tag. - Deny other traffic from the VirtualNetwork service tag, with the lowest priority. The outbound rules do not change. The property is an updateable list that is empty by default. Porter passes it to Terraform as base 64 JSON, as the base workspace does for address_spaces. Terraform accepts only IPv4 network addresses in CIDR format. The VM starts after the association, so a new server never runs without the rules. An upgrade removes the access of existing workspace clusters to port 9443 until an admin sets the property, so the bundle version is 1.0.0. Contract tests check the association, the rules and the property. --- .../test_cyclecloud_bundle_contract.py | 99 ++++++++++++++++++- .../shared_services/cyclecloud/porter.yaml | 9 +- .../cyclecloud/template_schema.json | 12 +++ .../cyclecloud/terraform/cyclecloud.tf | 3 + .../cyclecloud/terraform/locals.tf | 1 + .../cyclecloud/terraform/nsg.tf | 68 +++++++++++++ .../cyclecloud/terraform/variables.tf | 12 +++ 7 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 templates/shared_services/cyclecloud/terraform/nsg.tf diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py index 0f686908a..a5663dc28 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -1,5 +1,7 @@ -"""Check that CycleCloud VM actions can resolve the Terraform output.""" +"""Check the CycleCloud bundle outputs, VM size and network rules.""" +import base64 +import json from pathlib import Path import re import unittest @@ -10,6 +12,24 @@ BUNDLE = ROOT / "templates/shared_services/cyclecloud" +def blocks(source, opener): + """Return the body of each block that starts with the opener, for example 'security_rule {'.""" + result = [] + for match in re.finditer(re.escape(opener), source): + depth = 0 + for index in range(match.end() - 1, len(source)): + depth += {"{": 1, "}": -1}.get(source[index], 0) + if depth == 0: + result.append(source[match.end() : index]) + break + return result + + +def setting(block, name): + match = re.search(rf"^\s*{name}\s*=\s*(.+)$", block, re.MULTILINE) + return match.group(1).strip() if match else None + + class CycleCloudBundleContractTests(unittest.TestCase): def test_vm_actions_have_a_declared_and_emitted_resource_id(self): manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) @@ -38,3 +58,80 @@ def test_new_servers_use_an_offered_size_and_upgrades_keep_the_size(self): ignored = re.search(r"lifecycle\s*\{\s*ignore_changes\s*=\s*\[([^\]]*)\]", vm) self.assertIsNotNone(ignored) self.assertIn("vm_size", [name.strip() for name in ignored.group(1).split(",")]) + + def test_network_card_has_inbound_rules_for_the_core_network(self): + source = (BUNDLE / "terraform/nsg.tf").read_text() + (association,) = blocks( + source, 'resource "azurerm_network_interface_security_group_association" "cyclecloud" {' + ) + self.assertEqual(setting(association, "network_interface_id"), "azurerm_network_interface.cyclecloud.id") + self.assertEqual( + setting(association, "network_security_group_id"), "azurerm_network_security_group.cyclecloud.id" + ) + (nsg,) = blocks(source, 'resource "azurerm_network_security_group" "cyclecloud" {') + self.assertEqual(setting(nsg, "tags"), "local.tre_shared_service_tags") + self.assertEqual(setting(nsg, "resource_group_name"), "local.core_resource_group_name") + rules = blocks(nsg, "security_rule {") + self.assertEqual(len(rules), 3) + self.assertTrue(all(setting(rule, "direction") == '"Inbound"' for rule in rules)) + core = [ + r + for r in rules + if setting(r, "source_address_prefixes") == "data.azurerm_virtual_network.core.address_space" + ] + self.assertEqual(len(core), 1) + self.assertEqual(setting(core[0], "access"), '"Allow"') + self.assertEqual(setting(core[0], "protocol"), '"Tcp"') + self.assertEqual(setting(core[0], "destination_port_ranges"), '["22", "443"]') + balancer = [r for r in rules if setting(r, "source_address_prefix") == '"AzureLoadBalancer"'] + self.assertEqual([setting(r, "access") for r in balancer], ['"Allow"']) + deny = [r for r in rules if setting(r, "access") == '"Deny"'] + self.assertEqual(len(deny), 1) + self.assertEqual(setting(deny[0], "source_address_prefix"), '"VirtualNetwork"') + self.assertEqual(setting(deny[0], "destination_port_range"), '"*"') + self.assertEqual(setting(deny[0], "protocol"), '"*"') + priorities = [int(setting(rule, "priority")) for rule in rules] + self.assertEqual(max(priorities), 4096) + self.assertEqual(int(setting(deny[0], "priority")), 4096) + self.assertEqual(len(set(priorities)), len(priorities)) + vm = (BUNDLE / "terraform/cyclecloud.tf").read_text() + self.assertRegex(vm, r"depends_on\s*=\s*\[azurerm_network_interface_security_group_association\.cyclecloud\]") + + def test_cluster_port_is_open_only_for_the_configured_ranges(self): + source = (BUNDLE / "terraform/nsg.tf").read_text() + (nsg,) = blocks(source, 'resource "azurerm_network_security_group" "cyclecloud" {') + self.assertFalse(any("9443" in rule for rule in blocks(nsg, "security_rule {"))) + (dynamic,) = blocks(nsg, 'dynamic "security_rule" {') + self.assertEqual( + setting(dynamic, "for_each"), + "length(local.cluster_address_spaces) > 0 ? [local.cluster_address_spaces] : []", + ) + (content,) = blocks(dynamic, "content {") + self.assertEqual(setting(content, "access"), '"Allow"') + self.assertEqual(setting(content, "destination_port_range"), '"9443"') + self.assertEqual(setting(content, "source_address_prefixes"), "security_rule.value") + + def test_cluster_address_spaces_is_an_updateable_list_that_is_empty_by_default(self): + schema = json.loads((BUNDLE / "template_schema.json").read_text()) + prop = schema["properties"]["cluster_address_spaces"] + self.assertEqual((prop["type"], prop["default"], prop["updateable"]), ("array", [], True)) + self.assertEqual(prop["items"]["type"], "string") + pattern = re.compile(prop["items"]["pattern"]) + self.assertTrue(pattern.fullmatch("10.1.4.0/24")) + for value in ("10.1.4.0", "10.1.4.0/33", "256.1.4.0/24", "10.1.4/24"): + self.assertIsNone(pattern.fullmatch(value), value) + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + parameter = next(p for p in manifest["parameters"] if p["name"] == "cluster_address_spaces") + self.assertEqual(parameter["type"], "string") + self.assertEqual(json.loads(base64.b64decode(parameter["default"])), []) + for action in ("install", "upgrade", "uninstall"): + with self.subTest(action=action): + terraform = next(step["terraform"] for step in manifest[action] if "terraform" in step) + self.assertEqual( + terraform["vars"]["cluster_address_spaces"], "${ bundle.parameters.cluster_address_spaces }" + ) + self.assertIn('variable "cluster_address_spaces"', (BUNDLE / "terraform/variables.tf").read_text()) + self.assertRegex( + (BUNDLE / "terraform/locals.tf").read_text(), + r"cluster_address_spaces\s*=\s*jsondecode\(base64decode\(var\.cluster_address_spaces\)\)", + ) diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index 537dd523c..272abd1d1 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-cyclecloud -version: 0.7.10 +version: 1.0.0 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl @@ -53,6 +53,10 @@ parameters: - name: key_store_id type: string default: "" + - name: cluster_address_spaces + type: string + description: "Address spaces of cluster nodes that connect to the CycleCloud server on port 9443" + default: "W10=" # b64 for [] outputs: - name: azure_resource_id @@ -89,6 +93,7 @@ install: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -110,6 +115,7 @@ upgrade: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -131,6 +137,7 @@ uninstall: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" diff --git a/templates/shared_services/cyclecloud/template_schema.json b/templates/shared_services/cyclecloud/template_schema.json index 425fe4bd9..d6048e27d 100644 --- a/templates/shared_services/cyclecloud/template_schema.json +++ b/templates/shared_services/cyclecloud/template_schema.json @@ -26,6 +26,18 @@ "description": "Long form description of the shared service, in markdown syntax", "default": "Azure CycleCloud is an enterprise-friendly tool for orchestrating and managing High Performance Computing (HPC) environments on Azure.", "updateable": true + }, + "cluster_address_spaces": { + "type": "array", + "title": "Cluster address spaces", + "description": "Address spaces in CIDR format, for example 10.1.4.0/24. Cluster nodes in these address spaces can connect to the CycleCloud server on TCP port 9443. Add the address space of each workspace that has a CycleCloud cluster.", + "default": [], + "updateable": true, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])/(3[0-2]|[12]?[0-9])$" + } } }, "pipeline": { diff --git a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf index 02b6082fb..a3e156b3e 100644 --- a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf +++ b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf @@ -70,6 +70,9 @@ resource "azurerm_virtual_machine" "cyclecloud" { tags = local.tre_shared_service_tags + # Start the server only after its network card has the inbound rules. + depends_on = [azurerm_network_interface_security_group_association.cyclecloud] + # Azure does not offer Standard_DS3_v2 to all subscriptions in all regions. Keep the size of existing servers. lifecycle { ignore_changes = [tags, vm_size] } } diff --git a/templates/shared_services/cyclecloud/terraform/locals.tf b/templates/shared_services/cyclecloud/terraform/locals.tf index 9d75e9f53..cb6c2ac64 100644 --- a/templates/shared_services/cyclecloud/terraform/locals.tf +++ b/templates/shared_services/cyclecloud/terraform/locals.tf @@ -2,6 +2,7 @@ locals { core_resource_group_name = "rg-${var.tre_id}" core_vnet = "vnet-${var.tre_id}" short_service_id = substr(var.tre_resource_id, -4, -1) + cluster_address_spaces = jsondecode(base64decode(var.cluster_address_spaces)) vm_name = "cyclecloud-${local.short_service_id}" storage_name = lower(replace("stgcc${var.tre_id}${local.short_service_id}", "-", "")) tre_shared_service_tags = { diff --git a/templates/shared_services/cyclecloud/terraform/nsg.tf b/templates/shared_services/cyclecloud/terraform/nsg.tf new file mode 100644 index 000000000..cd6dd076c --- /dev/null +++ b/templates/shared_services/cyclecloud/terraform/nsg.tf @@ -0,0 +1,68 @@ +# Inbound rules for the CycleCloud server. The outbound rules stay at the Azure defaults. +resource "azurerm_network_security_group" "cyclecloud" { + name = "nsg-cyclecloud-${var.tre_id}-${local.short_service_id}" + location = data.azurerm_resource_group.rg.location + resource_group_name = local.core_resource_group_name + tags = local.tre_shared_service_tags + + # The admin jumpbox and Azure Bastion are in the core virtual network. + security_rule { + name = "AllowCoreHttpsSshInbound" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_ranges = ["22", "443"] + source_address_prefixes = data.azurerm_virtual_network.core.address_space + destination_address_prefix = "*" + } + + # Cluster nodes without a return proxy connect to the CycleCloud server on port 9443. + dynamic "security_rule" { + for_each = length(local.cluster_address_spaces) > 0 ? [local.cluster_address_spaces] : [] + content { + name = "AllowClusterNodesInbound" + priority = 110 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9443" + source_address_prefixes = security_rule.value + destination_address_prefix = "*" + } + } + + security_rule { + name = "AllowAzureLoadBalancerInbound" + priority = 120 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "*" + } + + # Peered workspace networks are part of the VirtualNetwork service tag. + security_rule { + name = "DenyVirtualNetworkInbound" + priority = 4096 + direction = "Inbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + } + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_network_interface_security_group_association" "cyclecloud" { + network_interface_id = azurerm_network_interface.cyclecloud.id + network_security_group_id = azurerm_network_security_group.cyclecloud.id +} diff --git a/templates/shared_services/cyclecloud/terraform/variables.tf b/templates/shared_services/cyclecloud/terraform/variables.tf index 48a64a194..1451ab8a1 100644 --- a/templates/shared_services/cyclecloud/terraform/variables.tf +++ b/templates/shared_services/cyclecloud/terraform/variables.tf @@ -14,3 +14,15 @@ variable "enable_cmk_encryption" { variable "key_store_id" { type = string } +variable "cluster_address_spaces" { + type = string + description = "Address spaces of cluster nodes that connect to the CycleCloud server on port 9443 (base 64 JSON array)" + + validation { + condition = try(alltrue([ + for cidr in jsondecode(base64decode(var.cluster_address_spaces)) : + can(cidrnetmask(cidr)) && cidrhost(cidr, 0) == split("/", cidr)[0] + ]), false) + error_message = "cluster_address_spaces must be a list of IPv4 network addresses in CIDR format, for example 10.1.4.0/24." + } +} From e3fb24ff2ab1549f27b2e19bc874a7d359961a0d Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 12:34:45 +0000 Subject: [PATCH 19/24] Check the CycleCloud network rules in the E2E test After the create, the test reads the network card and its network security group from Azure Resource Manager. The card must have the group of the service. The inbound allow rules must be exactly TCP 443 and 22 from the core address space and the AzureLoadBalancer tag. One deny rule for the VirtualNetwork tag must have the lowest priority. The metadata update now also sets cluster_address_spaces to one test range outside the core address space. The test makes sure that the TCP 9443 rule allows exactly that range. The update helper takes a dict of properties and checks that the API keeps each value. The ownership guard and the removal check include the group. The preflight reads the core address space from the core VNet. The test fails before the create if the test range overlaps it. The E2E document and the coverage map name the new checks and the limit: the test reads the rules, but sends no traffic from a workspace. --- docs/tre-developers/end-to-end-tests.md | 7 +- e2e_tests/bundle_coverage.json | 4 +- e2e_tests/resources/cyclecloud.py | 52 ++++++++++- e2e_tests/resources/cyclecloud_lifecycle.py | 12 +-- e2e_tests/test_cyclecloud.py | 22 ++++- .../test_cyclecloud_configuration.py | 88 ++++++++++++++++++- .../unit_tests/test_cyclecloud_lifecycle.py | 28 +++++- 7 files changed, 196 insertions(+), 17 deletions(-) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 0696aa32c..2935e4940 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -249,20 +249,23 @@ TEST_BUNDLE=tre-shared-service-cyclecloud make test-e2e-bundle Use a dedicated AzureCloud environment with one writer and no existing CycleCloud service or DNS zone. Publish and register the checkout firewall and CycleCloud bundles. Deploy the firewall before validation. -Register the repaired CycleCloud bundle, version 0.7.10 or later, with `start` and `stop` actions. +Register the CycleCloud bundle, version 1.0.0 or later, with `start` and `stop` actions. Accept the CycleCloud server image terms separately before the run. The test reads the agreement and regional image list. It does not accept terms. Check `Standard_D4s_v5` capacity and quota in the core region before deployment. Image availability does not prove VM capacity. The case creates one server and checks its private VM address, DNS configuration, blob private endpoint, managed identity and firewall rules. +It also checks the network security group of the VM network card. By default, the inbound allow rules must be for the core address space and the `AzureLoadBalancer` service tag only, and the lowest-priority rule must deny other `VirtualNetwork` traffic. It invokes stop/start through TRE and verifies the VM power states. It changes the overview to exercise a same-version Terraform upgrade and checks that the VM identity remains stable. +The same update sets `cluster_address_spaces` to one test range outside the core address space. The test then checks that the TCP 9443 rule allows exactly that range. The test records the existing firewall as reused. Smoke completes before the bundle job starts. The job sets `CYCLECLOUD_VALIDATION_DEADLINE` before container setup. The test reserves 120 minutes for cleanup within its 270-minute deadline. It owns an accepted service before polling. Cleanup waits for terminal operations, verifies ownership and then uses TRE deletion. -It verifies removal of the VM, disk, NIC, storage, blob private endpoint, DNS zone and captured identity's roles. It also checks that the original firewall rules are restored. +It verifies removal of the VM, disk, NIC, network security group, storage, blob private endpoint, DNS zone and captured identity's roles. It also checks that the original firewall rules are restored. Failed cleanup fails the test. Core Key Vault credentials can remain soft-deleted. The test does not retrieve or purge them. This case does not configure the CycleCloud application or create an HPC cluster. Browser access, application TLS trust, private DNS reachability and prior-release upgrades remain unproven. +The case reads the network rules from Azure Resource Manager. It does not send traffic from a workspace to the server. The existing bundle grants subscription Contributor to its VM and permits SharedSubnet access to ARM and GitHub. It does not disable the storage public endpoint. A private endpoint and a successful server lifecycle do not prove workspace isolation or storage access restrictions. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index f717f9dd3..3046cf74f 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -90,10 +90,10 @@ "limitations": [ "Requires AzureCloud, accepted CycleCloud marketplace image terms, the regional image and Standard_D4s_v5 capacity. The test does not accept terms or change quotas.", "Requires an empty CycleCloud slot, current firewall and one writer in a dedicated environment. The fixed DNS zone prevents parallel CycleCloud services.", - "Checks VM power actions, ARM configuration, TRE firewall rules, same-version metadata upgrade and bounded cleanup. Previous-version upgrades remain unproven.", + "Checks VM power actions, ARM configuration, TRE firewall rules, the network rules of the VM network card, a same-version update of the metadata and cluster address spaces, and bounded cleanup. Reads the network rules from ARM and does not send traffic from a workspace. Previous-version upgrades remain unproven.", "Does not configure the CycleCloud application, create HPC clusters or prove browser access, private DNS reachability or application TLS trust.", "The existing bundle grants its VM subscription Contributor and allows SharedSubnet access to ARM and GitHub. Storage public access is not disabled by this bundle. These controls do not prove workspace isolation.", - "Cleanup verifies the VM, disk, NIC, storage, private endpoint, DNS zone, captured identity roles and firewall rules. Key Vault credentials can remain soft-deleted; secret purge is not attempted." + "Cleanup verifies the VM, disk, NIC, network security group, storage, private endpoint, DNS zone, captured identity roles and firewall rules. Key Vault credentials can remain soft-deleted; secret purge is not attempted." ], "unavailable_reason": "" }, diff --git a/e2e_tests/resources/cyclecloud.py b/e2e_tests/resources/cyclecloud.py index 6a6d8554b..e369ef2a5 100644 --- a/e2e_tests/resources/cyclecloud.py +++ b/e2e_tests/resources/cyclecloud.py @@ -70,6 +70,12 @@ async def preflight(arm, settings): location = group.get("location", "").lower().replace(" ", "") assert re.fullmatch(r"[a-z0-9]+", location), "Core region is missing" vnet = f"{settings.group}/providers/Microsoft.Network/virtualNetworks/vnet-{settings.tre_id}" + vnet_record = await arm.request("GET", vnet, NETWORK_API) + same_id(vnet_record.get("id"), vnet) + address_space = vnet_record["properties"]["addressSpace"]["addressPrefixes"] + assert address_space and all(ip_network(v).is_private for v in address_space), ( + "Core VNet must use private addresses" + ) subnet = vnet + "/subnets/SharedSubnet" subnet_record = await arm.request("GET", subnet, NETWORK_API) same_id(subnet_record.get("id"), subnet) @@ -104,6 +110,7 @@ async def preflight(arm, settings): return { "location": location, "vnet": vnet, + "address_space": address_space, "subnet": subnet, "prefixes": prefixes, "zone": zone, @@ -120,6 +127,7 @@ def resource_ids(settings, core, identifier): "vm": f"{group}/providers/Microsoft.Compute/virtualMachines/{vm}", "disk": f"{group}/providers/Microsoft.Compute/disks/{vm}-osdisk", "nic": f"{group}/providers/Microsoft.Network/networkInterfaces/nic-cyclecloud-{settings.tre_id}-{suffix}", + "nsg": f"{group}/providers/Microsoft.Network/networkSecurityGroups/nsg-cyclecloud-{settings.tre_id}-{suffix}", "storage": f"{group}/providers/Microsoft.Storage/storageAccounts/{storage}", "endpoint": f"{group}/providers/Microsoft.Network/privateEndpoints/pe-{storage}", "zone": core["zone"], @@ -133,6 +141,7 @@ def resource_types(ids): ("vm", COMPUTE_API), ("disk", DISK_API), ("nic", NETWORK_API), + ("nsg", NETWORK_API), ("storage", STORAGE_API), ("endpoint", NETWORK_API), ("zone", DNS_API), @@ -145,6 +154,7 @@ async def guard_owned(arm, ids, tags): for key, version in ( ("vm", COMPUTE_API), ("nic", NETWORK_API), + ("nsg", NETWORK_API), ("storage", STORAGE_API), ("endpoint", NETWORK_API), ("zone", DNS_API), @@ -177,12 +187,49 @@ async def principal_roles(arm, subscription, principal): raise RuntimeError("CycleCloud role-assignment pagination exceeded its bound") -async def validate_resources(arm, settings, core, ids, identifier): +def rule_values(properties, single, plural): + values = list(properties.get(plural) or []) + if properties.get(single): + values.append(properties[single]) + return tuple(sorted(values)) + + +def validate_network_rules(nsg, address_space, clusters): + """Allow the core network, the cluster ranges and the load balancer. Deny other virtual network sources.""" + inbound = [r["properties"] for r in nsg["properties"]["securityRules"] if r["properties"]["direction"] == "Inbound"] + allowed = sorted( + ( + p["protocol"].lower(), + rule_values(p, "destinationPortRange", "destinationPortRanges"), + rule_values(p, "sourceAddressPrefix", "sourceAddressPrefixes"), + ) + for p in inbound + if p["access"] == "Allow" + ) + expected = [("*", ("*",), ("AzureLoadBalancer",)), ("tcp", ("22", "443"), tuple(sorted(address_space)))] + if clusters: + expected.append(("tcp", ("9443",), tuple(sorted(clusters)))) + assert allowed == sorted(expected), "CycleCloud inbound allow rules differ from the core and cluster ranges" + denied = [p for p in inbound if p["access"] == "Deny"] + assert len(denied) == 1, "CycleCloud needs one inbound deny rule" + deny = denied[0] + assert ( + deny["protocol"], + rule_values(deny, "destinationPortRange", "destinationPortRanges"), + rule_values(deny, "sourceAddressPrefix", "sourceAddressPrefixes"), + ) == ("*", ("*",), ("VirtualNetwork",)), "CycleCloud must deny other inbound virtual network traffic" + assert all(p["priority"] < deny["priority"] for p in inbound if p is not deny), ( + "The CycleCloud deny rule must have the lowest priority" + ) + + +async def validate_resources(arm, settings, core, ids, identifier, clusters=()): tags = {"tre_id": settings.tre_id, "tre_shared_service_id": identifier} records = {} for key, version in ( ("vm", COMPUTE_API), ("nic", NETWORK_API), + ("nsg", NETWORK_API), ("storage", STORAGE_API), ("endpoint", NETWORK_API), ("zone", DNS_API), @@ -204,6 +251,9 @@ async def validate_resources(arm, settings, core, ids, identifier): same_id(properties["storageProfile"]["osDisk"]["managedDisk"]["id"], ids["disk"]) assert len(properties["networkProfile"]["networkInterfaces"]) == 1 same_id(properties["networkProfile"]["networkInterfaces"][0]["id"], ids["nic"]) + nsg = records["nic"]["properties"].get("networkSecurityGroup") or {} + same_id(nsg.get("id"), ids["nsg"]) + validate_network_rules(records["nsg"], core["address_space"], clusters) configs = records["nic"]["properties"]["ipConfigurations"] assert len(configs) == 1 ip = configs[0]["properties"] diff --git a/e2e_tests/resources/cyclecloud_lifecycle.py b/e2e_tests/resources/cyclecloud_lifecycle.py index 81135d0d6..99456a3ca 100644 --- a/e2e_tests/resources/cyclecloud_lifecycle.py +++ b/e2e_tests/resources/cyclecloud_lifecycle.py @@ -126,9 +126,9 @@ async def wait_terminal(self, *, allow_absent=False, operation_id=None): return resource, operations await asyncio.sleep(15) - async def change(self, *, action=None, overview=None): - if action not in (None, "start", "stop") or (action is not None) == (overview is not None): - raise ValueError("Select one supported CycleCloud action or a metadata update") + async def change(self, *, action=None, properties=None): + if action not in (None, "start", "stop") or (action is not None) == bool(properties): + raise ValueError("Select one supported CycleCloud action or a property update") resource, _ = await self.wait_terminal() endpoint = get_full_endpoint(f"/api{self.path}") headers = await self.headers() @@ -137,7 +137,7 @@ async def change(self, *, action=None, overview=None): if action: response = await client.post(endpoint + "/invoke-action", headers=headers, params={"action": action}) else: - response = await client.patch(endpoint, headers=headers, json={"properties": {"overview": overview}}) + response = await client.patch(endpoint, headers=headers, json={"properties": properties}) if response.status_code != 202: raise RuntimeError(f"CycleCloud mutation failed: HTTP {response.status_code}") operation = response.json()["operation"] @@ -149,8 +149,8 @@ async def change(self, *, action=None, overview=None): observed = next(o for o in operations if o["id"] == operation_id) if observed["status"] in FAILED or result["deploymentStatus"] in FAILED: raise RuntimeError("CycleCloud mutation or its pipeline failed") - if overview is not None and result["properties"].get("overview") != overview: - raise AssertionError("CycleCloud metadata update was not preserved") + if properties and any(result["properties"].get(k) != v for k, v in properties.items()): + raise AssertionError("CycleCloud property update was not preserved") return result async def close(self): diff --git a/e2e_tests/test_cyclecloud.py b/e2e_tests/test_cyclecloud.py index e160f4dbd..5792a0246 100644 --- a/e2e_tests/test_cyclecloud.py +++ b/e2e_tests/test_cyclecloud.py @@ -1,5 +1,6 @@ """Validate one CycleCloud server without configuring or creating HPC clusters.""" +from ipaddress import ip_network from uuid import uuid4 import pytest @@ -26,6 +27,9 @@ from e2e_tests.resources.sql_probe import arm_client from e2e_tests.run_bundle import load_catalog +# A private range outside the core address space. The update allows it to reach port 9443. +CLUSTER_TEST_RANGE = "172.31.254.0/24" + async def prerequisites(verify): catalog = load_catalog() @@ -95,25 +99,35 @@ async def after_remove(identifier): firewall = await prerequisites(verify) baseline = firewall_rules(firewall) core = await preflight(arm, settings) + if any(ip_network(CLUSTER_TEST_RANGE).overlaps(ip_network(v)) for v in core["address_space"]): + raise ValueError("The CycleCloud cluster test range must be outside the core address space") record = await service.create() ids = resource_ids(settings, core, service.identifier) - async def inspect(record): + async def inspect(record, clusters=()): assert record["templateVersion"] == load_catalog()[TEMPLATE]["source_version"] assert record["properties"]["connection_uri"] == "https://" + core["zone"].rsplit("/", 1)[1] record_deployed_resource(record, service.path) - identity = await validate_resources(arm, settings, core, ids, service.identifier) + identity = await validate_resources(arm, settings, core, ids, service.identifier, clusters) validate_firewall(await current_firewall(), baseline, service.identifier, core["prefixes"]) return identity + # By default, only the core network and the load balancer can connect to the server. principal = await inspect(record) await wait_power(arm, ids["vm"], "running") await service.change(action="stop") await wait_power(arm, ids["vm"], "deallocated") await service.change(action="start") await wait_power(arm, ids["vm"], "running") - updated = await service.change(overview="CycleCloud lifecycle metadata upgrade") - assert await inspect(updated) == principal, "CycleCloud upgrade replaced the VM identity" + updated = await service.change( + properties={ + "overview": "CycleCloud lifecycle metadata upgrade", + "cluster_address_spaces": [CLUSTER_TEST_RANGE], + } + ) + assert await inspect(updated, [CLUSTER_TEST_RANGE]) == principal, ( + "CycleCloud upgrade replaced the VM identity" + ) @pytest.mark.asyncio(loop_scope="session") diff --git a/e2e_tests/unit_tests/test_cyclecloud_configuration.py b/e2e_tests/unit_tests/test_cyclecloud_configuration.py index 0d64c2ec0..2ea09e82a 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_configuration.py +++ b/e2e_tests/unit_tests/test_cyclecloud_configuration.py @@ -15,6 +15,41 @@ SERVICE = "22222222-2222-4222-8222-222222222222" PRINCIPAL = "33333333-3333-4333-8333-333333333333" ROOT = Path(__file__).resolve().parents[2] +CLUSTER = "172.31.254.0/24" + + +def nsg_rule(name, priority, access, protocol, ports, sources): + properties = { + "priority": priority, + "direction": "Inbound", + "access": access, + "protocol": protocol, + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRanges": [], + "sourceAddressPrefixes": [], + } + if len(ports) == 1: + properties["destinationPortRange"] = ports[0] + else: + properties["destinationPortRanges"] = ports + if sources in ("AzureLoadBalancer", "VirtualNetwork"): + properties["sourceAddressPrefix"] = sources + else: + properties["sourceAddressPrefixes"] = sources + return {"name": name, "properties": properties} + + +def default_rules(): + return [ + nsg_rule("AllowCoreHttpsSshInbound", 100, "Allow", "Tcp", ["22", "443"], ["10.0.0.0/22"]), + nsg_rule("AllowAzureLoadBalancerInbound", 120, "Allow", "*", ["*"], "AzureLoadBalancer"), + nsg_rule("DenyVirtualNetworkInbound", 4096, "Deny", "*", ["*"], "VirtualNetwork"), + ] + + +def cluster_rule(sources): + return nsg_rule("AllowClusterNodesInbound", 110, "Allow", "Tcp", ["9443"], sources) def fixture(): @@ -22,6 +57,7 @@ def fixture(): net = settings.group + "/providers/Microsoft.Network/virtualNetworks/vnet-tre-test" core = { "vnet": net, + "address_space": ["10.0.0.0/22"], "subnet": net + "/subnets/SharedSubnet", "prefixes": ["10.0.1.0/24"], "zone": settings.group @@ -58,8 +94,10 @@ def fixture(): }, ) records[ids["nic"]]["properties"] = { - "ipConfigurations": [{"properties": {"subnet": {"id": core["subnet"]}, "privateIPAddress": "10.0.1.5"}}] + "networkSecurityGroup": {"id": ids["nsg"]}, + "ipConfigurations": [{"properties": {"subnet": {"id": core["subnet"]}, "privateIPAddress": "10.0.1.5"}}], } + records[ids["nsg"]]["properties"] = {"securityRules": default_rules()} records[ids["zone"] + "/A/@"] = {"properties": {"aRecords": [{"ipv4Address": "10.0.1.5"}]}} records[ids["zone"] + "/virtualNetworkLinks/cyclecloudlink-core"] = {"properties": {"virtualNetwork": {"id": net}}} records[ids["storage"]].update( @@ -135,11 +173,54 @@ async def test_public_ip_wrong_subnet_dns_target_or_role_is_rejected(self): with self.assertRaises(AssertionError): await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE) + async def test_network_card_allows_only_the_core_network_by_default(self): + rules = self.records[self.ids["nsg"]]["properties"]["securityRules"] + for defect in ("association", "workspace", "cluster", "core", "deny", "priority", "outbound_only"): + with self.subTest(defect=defect): + rules[:] = default_rules() + self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] = {"id": self.ids["nsg"]} + if defect == "association": + del self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] + if defect == "workspace": + rules.append(nsg_rule("AllowWorkspace", 130, "Allow", "Tcp", ["443"], ["10.2.0.0/24"])) + if defect == "cluster": + rules.append(cluster_rule([CLUSTER])) + if defect == "core": + rules[0]["properties"]["sourceAddressPrefixes"] = ["10.0.0.0/8"] + if defect == "deny": + del rules[2] + if defect == "priority": + rules[2]["properties"]["priority"] = 105 + if defect == "outbound_only": + rules[2]["properties"]["direction"] = "Outbound" + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE) + rules[:] = default_rules() + self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] = {"id": self.ids["nsg"]} + self.assertEqual(await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE), PRINCIPAL) + + async def test_cluster_rule_allows_exactly_the_requested_ranges(self): + rules = self.records[self.ids["nsg"]]["properties"]["securityRules"] + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]) + rules.append(cluster_rule([CLUSTER])) + self.assertEqual( + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]), PRINCIPAL + ) + rules[-1] = cluster_rule([CLUSTER, "10.2.0.0/24"]) + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]) + async def test_cleanup_refuses_foreign_tags_before_any_write(self): self.records[self.ids["storage"]]["tags"]["tre_shared_service_id"] = "other" with self.assertRaisesRegex(AssertionError, "ownership"): await cc.guard_owned(self.arm, self.ids, {"tre_id": self.settings.tre_id, "tre_shared_service_id": SERVICE}) + async def test_cleanup_refuses_a_foreign_network_security_group(self): + self.records[self.ids["nsg"]]["tags"]["tre_shared_service_id"] = "other" + with self.assertRaisesRegex(AssertionError, "ownership"): + await cc.guard_owned(self.arm, self.ids, {"tre_id": self.settings.tre_id, "tre_shared_service_id": SERVICE}) + async def test_disk_cleanup_uses_the_disk_api_and_role_removal_is_checked(self): self.records = {} role_endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" @@ -152,6 +233,7 @@ async def replicate_removal(seconds): await cc.assert_removed(self.arm, self.ids, SUB, PRINCIPAL) sleep.assert_awaited_once() self.arm.request.assert_any_await("GET", self.ids["disk"], cc.DISK_API, missing_ok=True) + self.arm.request.assert_any_await("GET", self.ids["nsg"], cc.NETWORK_API, missing_ok=True) async def test_role_pagination_is_complete_and_cannot_change_authority(self): endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" @@ -173,6 +255,10 @@ async def test_terms_and_image_failures_happen_without_writes(self): "tags": {"tre_id": self.settings.tre_id}, "location": self.core["location"], } + self.records[self.core["vnet"]] = { + "id": self.core["vnet"], + "properties": {"addressSpace": {"addressPrefixes": ["10.0.0.0/22"]}}, + } self.records[self.core["subnet"]] = {"id": self.core["subnet"], "properties": {"addressPrefix": "10.0.1.0/24"}} self.records[self.settings.group + "/providers/Microsoft.Network/publicIPAddresses/pip-agw-tre-test"] = { "properties": {"dnsSettings": {"fqdn": "tre-test.switzerlandnorth.cloudapp.azure.com"}} diff --git a/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py index fe9e49759..41c8292ea 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py +++ b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py @@ -146,9 +146,35 @@ async def test_metadata_update_must_be_preserved(self): AsyncMock(side_effect=[(resource(), []), (resource("updated"), [{"id": OPERATION, "status": "updated"}])]), ): with self.assertRaisesRegex(AssertionError, "not preserved"): - await self.service.change(overview="new text") + await self.service.change(properties={"overview": "new text"}) self.assertEqual(self.client.patch.await_args.kwargs["json"], {"properties": {"overview": "new text"}}) + async def test_cluster_address_spaces_update_is_sent_and_preserved(self): + self.own() + self.client.patch.return_value = accepted() + update = {"overview": "new text", "cluster_address_spaces": ["172.31.254.0/24"]} + with patch.object( + self.service, + "wait_terminal", + AsyncMock( + side_effect=[ + (resource(), []), + (resource("updated", **update), [{"id": OPERATION, "status": "updated"}]), + ] + ), + ): + result = await self.service.change(properties=update) + self.assertEqual(result["properties"]["cluster_address_spaces"], ["172.31.254.0/24"]) + self.assertEqual(self.client.patch.await_args.kwargs["json"], {"properties": update}) + + async def test_change_needs_one_action_or_a_property_update(self): + self.own() + for arguments in ({}, {"properties": {}}, {"action": "restart"}, {"action": "stop", "properties": {"a": 1}}): + with self.subTest(arguments=arguments), self.assertRaises(ValueError): + await self.service.change(**arguments) + self.client.patch.assert_not_awaited() + self.client.post.assert_not_awaited() + async def test_cleanup_waits_then_guards_then_deletes_and_verifies(self): self.own() order = [] From c2277774ad2733a0ece3730ff3eaac57cf2304b1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 12:34:56 +0000 Subject: [PATCH 20/24] Describe the CycleCloud network rules and the upgrade step The CycleCloud document gets a Network rules section with the rule table, the cluster_address_spaces property and the source of port 9443. It gives the upgrade steps for version 1.0.0: a force version update, then an update that sets cluster_address_spaces. The API checks the properties of a PATCH against the current template version, so the property update is a separate step. Create a Cluster gets a step that adds the address space of the workspace before the cluster create. The CHANGELOG gets a BREAKING CHANGES entry with the upgrade step and a BUG FIXES entry. Both link a placeholder for the issue. --- CHANGELOG.md | 5 ++++ .../shared-services/cyclecloud.md | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb43cec8..3959e08d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) * `tre-workspace-service-gitea` 2.0.0 keeps the Git repositories on its file share. The upgrade restarts the web app, so the repositories on the container disk are lost. Mirror the repositories before the upgrade, and rotate the workspace client secret after it. See the [Gitea upgrade steps](docs/tre-templates/workspace-services/gitea.md#upgrading-to-version-200). The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* `tre-shared-service-cyclecloud` 1.0.0 adds network rules to the CycleCloud server. After the upgrade, cluster nodes in workspaces cannot connect to the server on TCP port 9443 until you set the new `cluster_address_spaces` property. + The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). After the upgrade, update the CycleCloud shared service and add the address space of each workspace that has a cluster to `cluster_address_spaces`. + See the [CycleCloud upgrade steps](docs/tre-templates/shared-services/cyclecloud.md#upgrade-to-version-100). ([#{ISSUE_CC_NETWORK}](https://github.com/microsoft/AzureTRE/issues/{ISSUE_CC_NETWORK})) ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) @@ -61,6 +64,8 @@ BUG FIXES: * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Add network rules to the network card of the CycleCloud server. The rules allow TCP ports 443 and 22 from the core virtual network, and TCP port 9443 from the address spaces in the new `cluster_address_spaces` property, which is empty by default. + The rules deny other inbound traffic from virtual networks. Before this change, the server had the default access of the Azure network rules, which allow inbound traffic from all peered virtual networks. (`tre-shared-service-cyclecloud` 1.0.0) ([#{ISSUE_CC_NETWORK}](https://github.com/microsoft/AzureTRE/issues/{ISSUE_CC_NETWORK})) * Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) * Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) diff --git a/docs/tre-templates/shared-services/cyclecloud.md b/docs/tre-templates/shared-services/cyclecloud.md index 2b1048a4e..1e0277d06 100644 --- a/docs/tre-templates/shared-services/cyclecloud.md +++ b/docs/tre-templates/shared-services/cyclecloud.md @@ -39,9 +39,39 @@ To connect to the CycleCloud server, the TRE Administrator must connect to the C - Hit Save, and then "Back to Clusters" +## Network rules + +The CycleCloud server has a network security group on its network card. The network security group has these inbound rules: + +| Priority | Source | Ports | Access | +| --- | --- | --- | --- | +| 100 | The address space of the core virtual network | TCP 443 and 22 | Allow | +| 110 | The address spaces in the `cluster_address_spaces` property | TCP 9443 | Allow | +| 120 | The `AzureLoadBalancer` service tag | All | Allow | +| 4096 | The `VirtualNetwork` service tag | All | Deny | + +- The admin jumpbox is in the core virtual network. Administrators connect to the CycleCloud server from the jumpbox, as the steps above describe. +- The `VirtualNetwork` service tag includes the peered workspace virtual networks. By default, workspaces cannot connect to the CycleCloud server. +- The bundle creates the rule for TCP port 9443 only when `cluster_address_spaces` has one or more address spaces. The default is an empty list. +- The outbound rules are the Azure default rules. + +Cluster nodes connect to the CycleCloud server on TCP port 9443 when the cluster does not use a return proxy. For the ports that CycleCloud uses, see [CycleCloud required ports and traffic](https://learn.microsoft.com/azure/cyclecloud/how-to/network-security?view=cyclecloud-8#required-ports-and-traffic). + +To let the nodes of a cluster connect, update the CycleCloud shared service and add the address space of the workspace to `cluster_address_spaces`. Use the CIDR format, for example `10.1.4.0/24`. When you delete the cluster, remove the address space from the list. + +### Upgrade to version 1.0.0 + +Version 1.0.0 adds the network rules. After the upgrade, cluster nodes in workspaces cannot connect to the CycleCloud server until you set `cluster_address_spaces`. + +1. Find the address space of each workspace that has a CycleCloud cluster. The `address_spaces` property of the workspace has this value. +1. Upgrade the CycleCloud shared service to version 1.0.0. The upgrade needs a [force version update](../../tre-admins/upgrading-resources.md#force-version-update). +1. Update the CycleCloud shared service. Set `cluster_address_spaces` to the address spaces from step 1. + ## Create a Cluster - Before you start creating the cluster retrieve the last 4 digits of the workspace ID that you want to deploy the cluster into. +- Update the CycleCloud shared service. Add the address space of the workspace to `cluster_address_spaces`, so that the cluster nodes can connect to the CycleCloud server. See [Network rules](#network-rules). + - Create a user in CycleCloud as per . The SSH key for the user will need to be created within the workspace and public key exporting. We suggest using the 4 digits retrieved in step 1 as part of the user account. - Select your cluster type, we have tested Slurm and Grid Engine using the methods documented here. From b08d7bd6c69ca92f7d2bc23b42763f924b5b84e8 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 15:31:47 +0000 Subject: [PATCH 21/24] Use the network address of each cluster range and keep a range until its last cluster The template schema accepts a range with host bits, for example 10.1.4.5/24, but the Terraform validation refused it. So the API stored the value, and the update failed later at the plan. Terraform now uses the network address of each range, for example 10.1.4.0/24, and removes duplicate ranges. The validation accepts each IPv4 range in CIDR format. A contract test checks the conversion. All clusters in one workspace use the same address space. The document now says to keep the address space of a workspace until the last cluster in that workspace is deleted. --- docs/tre-templates/shared-services/cyclecloud.md | 4 +++- .../unit_tests/test_cyclecloud_bundle_contract.py | 9 ++++++++- .../shared_services/cyclecloud/terraform/locals.tf | 10 +++++++--- .../shared_services/cyclecloud/terraform/variables.tf | 4 ++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/tre-templates/shared-services/cyclecloud.md b/docs/tre-templates/shared-services/cyclecloud.md index 1e0277d06..f25bcfa3c 100644 --- a/docs/tre-templates/shared-services/cyclecloud.md +++ b/docs/tre-templates/shared-services/cyclecloud.md @@ -57,7 +57,9 @@ The CycleCloud server has a network security group on its network card. The netw Cluster nodes connect to the CycleCloud server on TCP port 9443 when the cluster does not use a return proxy. For the ports that CycleCloud uses, see [CycleCloud required ports and traffic](https://learn.microsoft.com/azure/cyclecloud/how-to/network-security?view=cyclecloud-8#required-ports-and-traffic). -To let the nodes of a cluster connect, update the CycleCloud shared service and add the address space of the workspace to `cluster_address_spaces`. Use the CIDR format, for example `10.1.4.0/24`. When you delete the cluster, remove the address space from the list. +To let the nodes of a cluster connect, update the CycleCloud shared service and add the address space of the workspace to `cluster_address_spaces`. Use the CIDR format, for example `10.1.4.0/24`. If a range has host bits, for example `10.1.4.5/24`, the bundle uses its network address, `10.1.4.0/24`. + +All clusters in one workspace use the same address space. Keep the address space of a workspace in the list until you delete the last cluster in that workspace. ### Upgrade to version 1.0.0 diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py index a5663dc28..dff849458 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -133,5 +133,12 @@ def test_cluster_address_spaces_is_an_updateable_list_that_is_empty_by_default(s self.assertIn('variable "cluster_address_spaces"', (BUNDLE / "terraform/variables.tf").read_text()) self.assertRegex( (BUNDLE / "terraform/locals.tf").read_text(), - r"cluster_address_spaces\s*=\s*jsondecode\(base64decode\(var\.cluster_address_spaces\)\)", + r"for cidr in jsondecode\(base64decode\(var\.cluster_address_spaces\)\)", ) + + def test_cluster_ranges_use_their_network_address(self): + # The schema accepts a range with host bits, so Terraform must not refuse it after the API stored it. + locals_tf = (BUNDLE / "terraform/locals.tf").read_text() + self.assertRegex(locals_tf, r"cluster_address_spaces\s*=\s*distinct\(\[") + self.assertIn('"${cidrhost(cidr, 0)}/${split("/", cidr)[1]}"', locals_tf) + self.assertNotIn("cidrhost", (BUNDLE / "terraform/variables.tf").read_text()) diff --git a/templates/shared_services/cyclecloud/terraform/locals.tf b/templates/shared_services/cyclecloud/terraform/locals.tf index cb6c2ac64..8e93664bc 100644 --- a/templates/shared_services/cyclecloud/terraform/locals.tf +++ b/templates/shared_services/cyclecloud/terraform/locals.tf @@ -2,9 +2,13 @@ locals { core_resource_group_name = "rg-${var.tre_id}" core_vnet = "vnet-${var.tre_id}" short_service_id = substr(var.tre_resource_id, -4, -1) - cluster_address_spaces = jsondecode(base64decode(var.cluster_address_spaces)) - vm_name = "cyclecloud-${local.short_service_id}" - storage_name = lower(replace("stgcc${var.tre_id}${local.short_service_id}", "-", "")) + # A range with host bits, for example 10.1.4.5/24, becomes its network address, 10.1.4.0/24. + cluster_address_spaces = distinct([ + for cidr in jsondecode(base64decode(var.cluster_address_spaces)) : + "${cidrhost(cidr, 0)}/${split("/", cidr)[1]}" + ]) + vm_name = "cyclecloud-${local.short_service_id}" + storage_name = lower(replace("stgcc${var.tre_id}${local.short_service_id}", "-", "")) tre_shared_service_tags = { tre_id = var.tre_id tre_shared_service_id = var.tre_resource_id diff --git a/templates/shared_services/cyclecloud/terraform/variables.tf b/templates/shared_services/cyclecloud/terraform/variables.tf index 1451ab8a1..62d44c519 100644 --- a/templates/shared_services/cyclecloud/terraform/variables.tf +++ b/templates/shared_services/cyclecloud/terraform/variables.tf @@ -21,8 +21,8 @@ variable "cluster_address_spaces" { validation { condition = try(alltrue([ for cidr in jsondecode(base64decode(var.cluster_address_spaces)) : - can(cidrnetmask(cidr)) && cidrhost(cidr, 0) == split("/", cidr)[0] + can(cidrnetmask(cidr)) ]), false) - error_message = "cluster_address_spaces must be a list of IPv4 network addresses in CIDR format, for example 10.1.4.0/24." + error_message = "cluster_address_spaces must be a list of IPv4 address ranges in CIDR format, for example 10.1.4.0/24." } } From ffc589b36f28d0093e4748e9a765b8c8919ff774 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 15:32:00 +0000 Subject: [PATCH 22/24] Name CycleCloud 1.0.0 in place of 0.7.10 This change takes the CycleCloud bundle from 0.7.9 to 1.0.0, so no release has version 0.7.10. The two CHANGELOG entries of the earlier CycleCloud fixes now name 1.0.0. The CycleCloud document names 1.0.0 as the first version with the new VM size. --- CHANGELOG.md | 4 ++-- docs/tre-templates/shared-services/cyclecloud.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3959e08d4..854a09ade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,8 +62,8 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) -* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) -* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 0.7.10) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 1.0.0) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 1.0.0) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Add network rules to the network card of the CycleCloud server. The rules allow TCP ports 443 and 22 from the core virtual network, and TCP port 9443 from the address spaces in the new `cluster_address_spaces` property, which is empty by default. The rules deny other inbound traffic from virtual networks. Before this change, the server had the default access of the Azure network rules, which allow inbound traffic from all peered virtual networks. (`tre-shared-service-cyclecloud` 1.0.0) ([#{ISSUE_CC_NETWORK}](https://github.com/microsoft/AzureTRE/issues/{ISSUE_CC_NETWORK})) * Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) diff --git a/docs/tre-templates/shared-services/cyclecloud.md b/docs/tre-templates/shared-services/cyclecloud.md index f25bcfa3c..fb3117b11 100644 --- a/docs/tre-templates/shared-services/cyclecloud.md +++ b/docs/tre-templates/shared-services/cyclecloud.md @@ -23,7 +23,7 @@ az vm image terms accept --urn almalinux:almalinux-hpc:8_5-hpc:latest Deploy the CycleCloud server using UI or API. -The CycleCloud server uses the `Standard_D4s_v5` VM size. Before you deploy, make sure that your subscription can use this size in the TRE region. An upgrade from a version before 0.7.10 keeps the size of the existing server. +The CycleCloud server uses the `Standard_D4s_v5` VM size. Before you deploy, make sure that your subscription can use this size in the TRE region. An upgrade from a version before 1.0.0 keeps the size of the existing server. To connect to the CycleCloud server, the TRE Administrator must connect to the CycleCloud server from the administration jumpbox. Use Azure Bastion to connect to the jumpbox a with the username `admin` and the select the password located in your core KeyVault. Connect to the CycleCloud server at the URL: `https://cyclecloud-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`. From 6dfc56f249ed6213b5af5e4ebe9c4e8a67083b7c Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 16:11:46 +0000 Subject: [PATCH 23/24] Add the issue and pull request links to the new CHANGELOG entries The CycleCloud network rule entries link #5301. The CycleCloud and OHDSI bug fix entries link #5300. The evidence helper entry links #5302. --- CHANGELOG.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 854a09ade..30b0a8a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Setting to `false` will delete existing airlock storage accounts and must only b Mirror the repositories before the upgrade, and rotate the workspace client secret after it. See the [Gitea upgrade steps](docs/tre-templates/workspace-services/gitea.md#upgrading-to-version-200). The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * `tre-shared-service-cyclecloud` 1.0.0 adds network rules to the CycleCloud server. After the upgrade, cluster nodes in workspaces cannot connect to the server on TCP port 9443 until you set the new `cluster_address_spaces` property. The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). After the upgrade, update the CycleCloud shared service and add the address space of each workspace that has a cluster to `cluster_address_spaces`. - See the [CycleCloud upgrade steps](docs/tre-templates/shared-services/cyclecloud.md#upgrade-to-version-100). ([#{ISSUE_CC_NETWORK}](https://github.com/microsoft/AzureTRE/issues/{ISSUE_CC_NETWORK})) + See the [CycleCloud upgrade steps](docs/tre-templates/shared-services/cyclecloud.md#upgrade-to-version-100). ([#5301](https://github.com/microsoft/AzureTRE/issues/5301)) ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) @@ -18,7 +18,7 @@ ENHANCEMENTS: The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) * Add an independent CycleCloud server lifecycle with marketplace checks, VM power actions and bounded cleanup. ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) * Add independent private OHDSI lifecycle validation with callback, firewall, shared DNS preservation and bounded cleanup checks. ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Add a bundle evidence helper that records the deployed template version of a resource that a test gives to it, without the resource properties. +* Add a bundle evidence helper that records the deployed template version of a resource that a test gives to it, without the resource properties. ([#5302](https://github.com/microsoft/AzureTRE/pull/5302)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) @@ -62,15 +62,15 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) -* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 1.0.0) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) -* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 1.0.0) ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 1.0.0) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 1.0.0) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) * Add network rules to the network card of the CycleCloud server. The rules allow TCP ports 443 and 22 from the core virtual network, and TCP port 9443 from the address spaces in the new `cluster_address_spaces` property, which is empty by default. - The rules deny other inbound traffic from virtual networks. Before this change, the server had the default access of the Azure network rules, which allow inbound traffic from all peered virtual networks. (`tre-shared-service-cyclecloud` 1.0.0) ([#{ISSUE_CC_NETWORK}](https://github.com/microsoft/AzureTRE/issues/{ISSUE_CC_NETWORK})) -* Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) -* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) + The rules deny other inbound traffic from virtual networks. Before this change, the server had the default access of the Azure network rules, which allow inbound traffic from all peered virtual networks. (`tre-shared-service-cyclecloud` 1.0.0) ([#5301](https://github.com/microsoft/AzureTRE/issues/5301)) +* Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) From 36396b3c34b3f6e6baf25b69f365cb312efc5a77 Mon Sep 17 00:00:00 2001 From: Jay W Date: Mon, 2 Nov 2026 16:35:55 +0000 Subject: [PATCH 24/24] Add cluster_address_spaces to the CycleCloud parameter set make bundle-check-params compares the Porter parameters of a bundle with its parameters.json file, and make bundle-install needs this check. The new parameter cluster_address_spaces was not in the file, so the check failed. The file now reads it from CLUSTER_ADDRESS_SPACES. A contract test makes sure that the parameter set lists each Porter parameter of the bundle, apart from arm_use_msi, as the check does. --- e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py | 9 +++++++++ templates/shared_services/cyclecloud/parameters.json | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py index dff849458..3c1bf9909 100644 --- a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -136,6 +136,15 @@ def test_cluster_address_spaces_is_an_updateable_list_that_is_empty_by_default(s r"for cidr in jsondecode\(base64decode\(var\.cluster_address_spaces\)\)", ) + def test_parameter_set_lists_each_porter_parameter(self): + # make bundle-check-params compares these names, and make bundle-install needs the check to pass. + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + parameter_set = json.loads((BUNDLE / "parameters.json").read_text()) + self.assertEqual( + {p["name"] for p in manifest["parameters"]} - {"arm_use_msi"}, + {p["name"] for p in parameter_set["parameters"]} - {"arm_use_msi"}, + ) + def test_cluster_ranges_use_their_network_address(self): # The schema accepts a range with host bits, so Terraform must not refuse it after the API stored it. locals_tf = (BUNDLE / "terraform/locals.tf").read_text() diff --git a/templates/shared_services/cyclecloud/parameters.json b/templates/shared_services/cyclecloud/parameters.json index c39909257..8e0d126ec 100755 --- a/templates/shared_services/cyclecloud/parameters.json +++ b/templates/shared_services/cyclecloud/parameters.json @@ -57,6 +57,12 @@ "source": { "env": "KEY_STORE_ID" } + }, + { + "name": "cluster_address_spaces", + "source": { + "env": "CLUSTER_ADDRESS_SPACES" + } } ] }