diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 65ebe15392..cf3aa887bd 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -1072,6 +1072,13 @@ jobs: # The job timeout is 300 minutes. Each step below sets an absolute deadline 270 minutes # (270 * 60 seconds) after the job start. The lifecycle helper keeps the last 120 minutes # before the deadline for the clean-up. The 30 minutes after the deadline are for the report. + - name: Reserve CycleCloud cleanup and reporting time + if: inputs.e2eBundle == 'tre-shared-service-cyclecloud' + run: echo "CYCLECLOUD_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve OHDSI cleanup and reporting time + if: inputs.e2eBundle == 'tre-workspace-service-ohdsi' + run: echo "OHDSI_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve SQL cleanup and reporting time if: inputs.e2eBundle == 'tre-workspace-service-azuresql' run: echo "SQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" @@ -1113,6 +1120,8 @@ jobs: SQL_VALIDATION_DEADLINE=${{ env.SQL_VALIDATION_DEADLINE }} MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }} GITEA_VALIDATION_DEADLINE=${{ env.GITEA_VALIDATION_DEADLINE }} + CYCLECLOUD_VALIDATION_DEADLINE=${{ env.CYCLECLOUD_VALIDATION_DEADLINE }} + OHDSI_VALIDATION_DEADLINE=${{ env.OHDSI_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index decc945754..30b0a8a9f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) * `tre-workspace-service-gitea` 2.0.0 keeps the Git repositories on its file share. The upgrade restarts the web app, so the repositories on the container disk are lost. Mirror the repositories before the upgrade, and rotate the workspace client secret after it. See the [Gitea upgrade steps](docs/tre-templates/workspace-services/gitea.md#upgrading-to-version-200). The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* `tre-shared-service-cyclecloud` 1.0.0 adds network rules to the CycleCloud server. After the upgrade, cluster nodes in workspaces cannot connect to the server on TCP port 9443 until you set the new `cluster_address_spaces` property. + The upgrade needs a [force version update](docs/tre-admins/upgrading-resources.md#force-version-update). After the upgrade, update the CycleCloud shared service and add the address space of each workspace that has a cluster to `cluster_address_spaces`. + See the [CycleCloud upgrade steps](docs/tre-templates/shared-services/cyclecloud.md#upgrade-to-version-100). ([#5301](https://github.com/microsoft/AzureTRE/issues/5301)) ENHANCEMENTS: * Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) @@ -13,6 +16,9 @@ ENHANCEMENTS: * Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Add a private Gitea lifecycle check. A workspace VM uses the Gitea API through private DNS, gets HTTP 401 without credentials and with a wrong password, and commits a file to a private repository. The VM reads the file again after an update, a change of the OIDC client ID, a stop and a start of the app. The check also makes sure that the OIDC change stops Gitea one time and that the console log of the web app shows no secret. ([#5141](https://github.com/microsoft/AzureTRE/issues/5141)) +* Add an independent CycleCloud server lifecycle with marketplace checks, VM power actions and bounded cleanup. ([#5159](https://github.com/microsoft/AzureTRE/issues/5159)) +* Add independent private OHDSI lifecycle validation with callback, firewall, shared DNS preservation and bounded cleanup checks. ([#5149](https://github.com/microsoft/AzureTRE/issues/5149)) +* Add a bundle evidence helper that records the deployed template version of a resource that a test gives to it, without the resource properties. ([#5302](https://github.com/microsoft/AzureTRE/pull/5302)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) @@ -56,6 +62,15 @@ BUG FIXES: The workspace start script gives the OIDC source the current client secret at each start, and it stops Gitea one time when the OIDC settings change. The web app reads the client secret through a Key Vault reference. (`tre-workspace-service-gitea` 2.0.0, `tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Gitea shared service identity the Key Vault role only on the two secrets that its web app reads. It had the role on the whole core Key Vault. (`tre-shared-service-gitea` 1.2.6) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) * Give the Nexus VM identity the Key Vault role only on the Nexus admin password and the Nexus TLS certificate. It had the role on the whole core Key Vault. (`tre-shared-service-sonatype-nexus` 3.11.4) ([#5283](https://github.com/microsoft/AzureTRE/issues/5283)) +* Export the CycleCloud VM ID for start and stop actions. (`tre-shared-service-cyclecloud` 1.0.0) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Deploy new CycleCloud servers with the `Standard_D4s_v5` VM size. Azure does not offer `Standard_DS3_v2` to all subscriptions in all regions, for example Switzerland North, so the CycleCloud server failed to deploy there. An upgrade keeps the size of an existing server. (`tre-shared-service-cyclecloud` 1.0.0) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Add network rules to the network card of the CycleCloud server. The rules allow TCP ports 443 and 22 from the core virtual network, and TCP port 9443 from the address spaces in the new `cluster_address_spaces` property, which is empty by default. + The rules deny other inbound traffic from virtual networks. Before this change, the server had the default access of the Azure network rules, which allow inbound traffic from all peered virtual networks. (`tre-shared-service-cyclecloud` 1.0.0) ([#5301](https://github.com/microsoft/AzureTRE/issues/5301)) +* Give the OHDSI WebAPI identity the Key Vault role only on the five secrets that its web app reads. It had the role on the whole workspace Key Vault. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Fix OHDSI database credentials, bound WebAPI startup checks, disable public access and remove owned firewall rules on uninstall. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Upload the OHDSI Atlas UI config with the storage account key, as the share and its web app mount already do. Since azurerm 4, the upload used Azure AD and failed with HTTP 403, because the resource processor has no file data role. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Create the OHDSI Atlas UI web app after the PostgreSQL server, and the WebAPI web app after the Atlas UI web app. Azure rejected the web app subnet link with ReferencedResourceNotProvisioned while a PostgreSQL subnet write ran in the same VNet. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) +* Declare the `Microsoft.Storage` service endpoint on the OHDSI PostgreSQL subnet. The PostgreSQL service adds it during the server create, and each upgrade removed it again. (`tre-workspace-service-ohdsi` 0.3.14) ([#5300](https://github.com/microsoft/AzureTRE/issues/5300)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 306f4ca158..2935e4940b 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -233,7 +233,66 @@ The **E2E Test Results (Bundle)** artefact contains: The JSON report copies only selected identifiers and operation states. It excludes resource properties, tokens and failure tracebacks. Source versions describe the checked-out manifests; verify deployed versions separately when reusing an environment. +A test can call `record_deployed_resource` for a resource that it reads. The report then has a `deployed_template_version` record for that resource. `passed_selected_cases` means the selected assertions and their teardown passed. The report always leaves `full_lifecycle_proven` false for a separate evidence review. An entirely skipped selection exits unsuccessfully. Individual skipped cases retain their reasons, and a mixed pass/skip result is marked `partial`. Unavailable bundles fail with the gap recorded in the map. Adding a selector does not implement missing tests or fix bundle publication and registration gaps. + +### CycleCloud server validation + +Select `tre-shared-service-cyclecloud` with the branch workflow's bundle input, or use: + +```bash +TEST_BUNDLE=tre-shared-service-cyclecloud make test-e2e-bundle +``` + +Use a dedicated AzureCloud environment with one writer and no existing CycleCloud service or DNS zone. +Publish and register the checkout firewall and CycleCloud bundles. Deploy the firewall before validation. +Register the CycleCloud bundle, version 1.0.0 or later, with `start` and `stop` actions. +Accept the CycleCloud server image terms separately before the run. The test reads the agreement and regional image list. It does not accept terms. +Check `Standard_D4s_v5` capacity and quota in the core region before deployment. Image availability does not prove VM capacity. + +The case creates one server and checks its private VM address, DNS configuration, blob private endpoint, managed identity and firewall rules. +It also checks the network security group of the VM network card. By default, the inbound allow rules must be for the core address space and the `AzureLoadBalancer` service tag only, and the lowest-priority rule must deny other `VirtualNetwork` traffic. +It invokes stop/start through TRE and verifies the VM power states. It changes the overview to exercise a same-version Terraform upgrade and checks that the VM identity remains stable. +The same update sets `cluster_address_spaces` to one test range outside the core address space. The test then checks that the TCP 9443 rule allows exactly that range. +The test records the existing firewall as reused. Smoke completes before the bundle job starts. + +The job sets `CYCLECLOUD_VALIDATION_DEADLINE` before container setup. The test reserves 120 minutes for cleanup within its 270-minute deadline. +It owns an accepted service before polling. Cleanup waits for terminal operations, verifies ownership and then uses TRE deletion. +It verifies removal of the VM, disk, NIC, network security group, storage, blob private endpoint, DNS zone and captured identity's roles. It also checks that the original firewall rules are restored. +Failed cleanup fails the test. Core Key Vault credentials can remain soft-deleted. The test does not retrieve or purge them. + +This case does not configure the CycleCloud application or create an HPC cluster. Browser access, application TLS trust, private DNS reachability and prior-release upgrades remain unproven. +The case reads the network rules from Azure Resource Manager. It does not send traffic from a workspace to the server. +The existing bundle grants subscription Contributor to its VM and permits SharedSubnet access to ARM and GitHub. It does not disable the storage public endpoint. +A private endpoint and a successful server lifecycle do not prove workspace isolation or storage access restrictions. + +### Private OHDSI validation + +Select the bundle from a branch that contains its repaired bootstrap and validation case: + +```bash +TEST_BUNDLE=tre-workspace-service-ohdsi make test-e2e-bundle +``` + +Use a dedicated AzureCloud environment with one writer and matching deployment and authentication tenants. +Publish and register the checkout base-workspace, firewall and OHDSI bundles. Deploy the firewall before the test. +Check PostgreSQL 14 capacity for `B_Standard_B1ms`, 32 GiB storage and availability zone 1 in the selected region. +The case creates a fresh Automatic workspace. It disables backups and sets `configure_data_source=false` without data-source properties. + +The case checks both private web apps, their pinned images and PostgreSQL networking. +It also checks that the WebAPI identity has a role on each of its five Key Vault secrets and no role on the vault, and that each Key Vault reference resolves. +Successful installation includes the bundle's private WebAPI readiness check and local administrator setup. +A metadata update exercises a same-version upgrade. The test checks the callback and firewall collections after each operation. +It waits for terminal operations before updates or deletion. It owns accepted resources before evidence writes or client closure. + +The job sets `OHDSI_VALIDATION_DEADLINE` before container setup. The case reserves 120 minutes for cleanup within 270 minutes. +Cleanup removes the service before its workspace. It checks Azure resource removal, app DNS records, callback removal and firewall restoration. +If service cleanup fails, the case retains the workspace and fails. The shared core PostgreSQL DNS link remains in place. +Key Vault items can remain soft-deleted under the existing retention policy. + +Browser sign-in, independent private HTTP and DNS probes, database data persistence and previous-version upgrades remain unproven. +Synapse provisioning, CDM queries and clinical workflows need separate cases. Retained non-empty data-source properties are outside this case. +The Synapse script does not currently inspect `configure_data_source`; empty properties avoid that path. diff --git a/docs/tre-templates/shared-services/cyclecloud.md b/docs/tre-templates/shared-services/cyclecloud.md index 147d136354..fb3117b11b 100644 --- a/docs/tre-templates/shared-services/cyclecloud.md +++ b/docs/tre-templates/shared-services/cyclecloud.md @@ -23,6 +23,8 @@ az vm image terms accept --urn almalinux:almalinux-hpc:8_5-hpc:latest Deploy the CycleCloud server using UI or API. +The CycleCloud server uses the `Standard_D4s_v5` VM size. Before you deploy, make sure that your subscription can use this size in the TRE region. An upgrade from a version before 1.0.0 keeps the size of the existing server. + To connect to the CycleCloud server, the TRE Administrator must connect to the CycleCloud server from the administration jumpbox. Use Azure Bastion to connect to the jumpbox a with the username `admin` and the select the password located in your core KeyVault. Connect to the CycleCloud server at the URL: `https://cyclecloud-{TRE_ID}.{LOCATION}.cloudapp.azure.com/`. - Provide a name for the cyclecloud server instance. @@ -37,9 +39,41 @@ To connect to the CycleCloud server, the TRE Administrator must connect to the C - Hit Save, and then "Back to Clusters" +## Network rules + +The CycleCloud server has a network security group on its network card. The network security group has these inbound rules: + +| Priority | Source | Ports | Access | +| --- | --- | --- | --- | +| 100 | The address space of the core virtual network | TCP 443 and 22 | Allow | +| 110 | The address spaces in the `cluster_address_spaces` property | TCP 9443 | Allow | +| 120 | The `AzureLoadBalancer` service tag | All | Allow | +| 4096 | The `VirtualNetwork` service tag | All | Deny | + +- The admin jumpbox is in the core virtual network. Administrators connect to the CycleCloud server from the jumpbox, as the steps above describe. +- The `VirtualNetwork` service tag includes the peered workspace virtual networks. By default, workspaces cannot connect to the CycleCloud server. +- The bundle creates the rule for TCP port 9443 only when `cluster_address_spaces` has one or more address spaces. The default is an empty list. +- The outbound rules are the Azure default rules. + +Cluster nodes connect to the CycleCloud server on TCP port 9443 when the cluster does not use a return proxy. For the ports that CycleCloud uses, see [CycleCloud required ports and traffic](https://learn.microsoft.com/azure/cyclecloud/how-to/network-security?view=cyclecloud-8#required-ports-and-traffic). + +To let the nodes of a cluster connect, update the CycleCloud shared service and add the address space of the workspace to `cluster_address_spaces`. Use the CIDR format, for example `10.1.4.0/24`. If a range has host bits, for example `10.1.4.5/24`, the bundle uses its network address, `10.1.4.0/24`. + +All clusters in one workspace use the same address space. Keep the address space of a workspace in the list until you delete the last cluster in that workspace. + +### Upgrade to version 1.0.0 + +Version 1.0.0 adds the network rules. After the upgrade, cluster nodes in workspaces cannot connect to the CycleCloud server until you set `cluster_address_spaces`. + +1. Find the address space of each workspace that has a CycleCloud cluster. The `address_spaces` property of the workspace has this value. +1. Upgrade the CycleCloud shared service to version 1.0.0. The upgrade needs a [force version update](../../tre-admins/upgrading-resources.md#force-version-update). +1. Update the CycleCloud shared service. Set `cluster_address_spaces` to the address spaces from step 1. + ## Create a Cluster - Before you start creating the cluster retrieve the last 4 digits of the workspace ID that you want to deploy the cluster into. +- Update the CycleCloud shared service. Add the address space of the workspace to `cluster_address_spaces`, so that the cluster nodes can connect to the CycleCloud server. See [Network rules](#network-rules). + - Create a user in CycleCloud as per . The SSH key for the user will need to be created within the workspace and public key exporting. We suggest using the 4 digits retrieved in step 1 as part of the user account. - Select your cluster type, we have tested Slurm and Grid Engine using the methods documented here. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 49194451cd..3046cf74f6 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -73,14 +73,29 @@ "tre-shared-service-cyclecloud": { "manifest": "templates/shared_services/cyclecloud/porter.yaml", "family": "shared_service", - "prerequisites": [], - "tests": [], + "prerequisites": [ + "tre-shared-service-firewall" + ], + "tests": [ + "test_cyclecloud.py::test_cyclecloud_lifecycle" + ], "related_tests": [], - "coverage": [], + "coverage": [ + "create", + "update", + "custom_actions", + "delete", + "configuration" + ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, accepted CycleCloud marketplace image terms, the regional image and Standard_D4s_v5 capacity. The test does not accept terms or change quotas.", + "Requires an empty CycleCloud slot, current firewall and one writer in a dedicated environment. The fixed DNS zone prevents parallel CycleCloud services.", + "Checks VM power actions, ARM configuration, TRE firewall rules, the network rules of the VM network card, a same-version update of the metadata and cluster address spaces, and bounded cleanup. Reads the network rules from ARM and does not send traffic from a workspace. Previous-version upgrades remain unproven.", + "Does not configure the CycleCloud application, create HPC clusters or prove browser access, private DNS reachability or application TLS trust.", + "The existing bundle grants its VM subscription Contributor and allows SharedSubnet access to ARM and GitHub. Storage public access is not disabled by this bundle. These controls do not prove workspace isolation.", + "Cleanup verifies the VM, disk, NIC, network security group, storage, private endpoint, DNS zone, captured identity roles and firewall rules. Key Vault credentials can remain soft-deleted; secret purge is not attempted." ], - "unavailable_reason": "The existing shared-service case is disabled pending repair (#5154)." + "unavailable_reason": "" }, "tre-shared-service-databricks-private-auth": { "manifest": "templates/shared_services/databricks-auth/porter.yaml", @@ -420,15 +435,31 @@ "manifest": "templates/workspace_services/ohdsi/porter.yaml", "family": "workspace_service", "prerequisites": [ - "tre-workspace-base" + "tre-workspace-base", + "tre-shared-service-firewall" + ], + "tests": [ + "test_ohdsi.py::test_private_ohdsi_lifecycle" ], - "tests": [], "related_tests": [], - "coverage": [], + "coverage": [ + "install", + "private-arm-configuration", + "managed-identity", + "same-version-upgrade", + "callback-registration", + "firewall-restoration", + "uninstall" + ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, matching Azure and authentication tenants, the checkout firewall and a fresh Automatic workspace. Use one writer in a dedicated environment.", + "Requires PostgreSQL 14 capacity in availability zone 1. Check regional SKU and capacity before the run.", + "Sets configure_data_source=false and omits data-source properties. Synapse provisioning, retained data-source properties, CDM queries and clinical workflows remain untested.", + "Checks private ARM configuration and successful bundle bootstrap. Browser sign-in, independent private HTTP and DNS probes, database data persistence and previous-version upgrades remain unproven.", + "Reserves 120 minutes for cleanup within a 270-minute job deadline. A failed service cleanup retains its workspace and fails validation.", + "Removes owned service resources, callbacks, app DNS records and firewall collections. The shared core PostgreSQL DNS link is retained; Key Vault items can remain soft-deleted." ], - "unavailable_reason": "Missing workspace-service case and parametrisation (#5126)." + "unavailable_reason": "" }, "tre-workspace-service-openai": { "manifest": "templates/workspace_services/openai/porter.yaml", diff --git a/e2e_tests/bundle_evidence.py b/e2e_tests/bundle_evidence.py index 27f6a2deb5..bb6f243d1e 100644 --- a/e2e_tests/bundle_evidence.py +++ b/e2e_tests/bundle_evidence.py @@ -26,6 +26,21 @@ def record_resource(payload, operation, method): _ACTIVE_REPORT.write() +def record_deployed_resource(resource, resource_path): + """Record the observed bundle version without copying resource properties.""" + if _ACTIVE_REPORT is None: + return + record = { + "method": "GET", + "resourceId": resource["id"], + "resourcePath": resource_path, + "bundle": resource["templateName"], + "deployed_template_version": resource["templateVersion"], + } + _ACTIVE_REPORT.data["resources"].append(record) + _ACTIVE_REPORT.write() + + def record_reused_resource(resource): """Record only the selected prerequisite's identity and deployed version.""" if _ACTIVE_REPORT is None: diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index ea391b6c83..902a33018c 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -16,6 +16,8 @@ markers = workspace_services workspace_validation: opt-in base-workspace creation, scoped API access and deletion openai_validation: opt-in private OpenAI deployment with model and quota preflight + cyclecloud_validation: opt-in isolated CycleCloud server lifecycle and VM power actions + ohdsi_validation: opt-in private OHDSI service lifecycle without an external data source sql_validation: Private SQL query and configuration upgrade lifecycle mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM gitea_validation: opt-in private Gitea repository write/read across an update and an app restart diff --git a/e2e_tests/resources/cyclecloud.py b/e2e_tests/resources/cyclecloud.py new file mode 100644 index 0000000000..e369ef2a5d --- /dev/null +++ b/e2e_tests/resources/cyclecloud.py @@ -0,0 +1,357 @@ +"""Inspect the owned CycleCloud infrastructure without reading credentials.""" + +import asyncio +from copy import deepcopy +from dataclasses import dataclass +from ipaddress import ip_address, ip_network +import os +import re +from urllib.parse import parse_qsl, urlsplit +from uuid import UUID + +from e2e_tests import config + +COMPUTE_API = "2023-03-01" +DISK_API = "2023-10-02" +NETWORK_API = "2024-05-01" +DNS_API = "2024-06-01" +STORAGE_API = "2023-05-01" +ROLE_API = "2022-04-01" +CONTRIBUTOR = "b24988ac-6180-42a0-ab88-20f7382dd24c" +TEMPLATE = "tre-shared-service-cyclecloud" +FIREWALL = "tre-shared-service-firewall" + + +def uuid_value(value): + try: + parsed = UUID(value) + if not parsed.int or str(parsed) != value: + raise ValueError + except (ValueError, TypeError, AttributeError): + raise ValueError("CycleCloud requires canonical, non-zero resource UUIDs") from None + return value + + +@dataclass(frozen=True) +class Settings: + subscription: str + tre_id: str + + def __post_init__(self): + uuid_value(self.subscription) + if not isinstance(self.tre_id, str) or not re.fullmatch(r"[a-z0-9-]{1,11}", self.tre_id): + raise ValueError("CycleCloud requires a valid TRE_ID shorter than 12 characters") + + @property + def group(self): + return f"/subscriptions/{self.subscription}/resourceGroups/rg-{self.tre_id}" + + +def require_settings(): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("CycleCloud validation currently supports AzureCloud only") + if uuid_value(os.environ.get("ARM_TENANT_ID", "")) != uuid_value(config.AAD_TENANT_ID): + raise ValueError("CycleCloud requires the deployment and authentication tenants to match") + return Settings(os.environ.get("ARM_SUBSCRIPTION_ID", ""), config.TRE_ID) + + +def same_id(actual, expected): + assert isinstance(actual, str) and actual.lower() == expected.lower(), "Unexpected CycleCloud ARM identity" + + +def owned(record, identifier, tags): + same_id(record.get("id"), identifier) + assert all(record.get("tags", {}).get(k) == v for k, v in tags.items()), "CycleCloud ownership tags differ" + + +async def preflight(arm, settings): + group = await arm.request("GET", settings.group, "2021-04-01") + owned(group, settings.group, {"tre_id": settings.tre_id}) + location = group.get("location", "").lower().replace(" ", "") + assert re.fullmatch(r"[a-z0-9]+", location), "Core region is missing" + vnet = f"{settings.group}/providers/Microsoft.Network/virtualNetworks/vnet-{settings.tre_id}" + vnet_record = await arm.request("GET", vnet, NETWORK_API) + same_id(vnet_record.get("id"), vnet) + address_space = vnet_record["properties"]["addressSpace"]["addressPrefixes"] + assert address_space and all(ip_network(v).is_private for v in address_space), ( + "Core VNet must use private addresses" + ) + subnet = vnet + "/subnets/SharedSubnet" + subnet_record = await arm.request("GET", subnet, NETWORK_API) + same_id(subnet_record.get("id"), subnet) + prefixes = subnet_record["properties"].get("addressPrefixes") or [subnet_record["properties"]["addressPrefix"]] + assert prefixes and all(ip_network(v).is_private for v in prefixes), "SharedSubnet must use private addresses" + pip = f"{settings.group}/providers/Microsoft.Network/publicIPAddresses/pip-agw-{settings.tre_id}" + fqdn = (await arm.request("GET", pip, NETWORK_API))["properties"]["dnsSettings"]["fqdn"] + assert re.fullmatch(r"[a-z0-9.-]+\.cloudapp\.azure\.com", fqdn), "Unexpected core public DNS name" + zone = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/cyclecloud-{fqdn}" + if await arm.request("GET", zone, DNS_API, missing_ok=True) is not None: + raise ValueError("An existing CycleCloud DNS zone prevents isolated validation") + blob_zone = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net" + same_id((await arm.request("GET", blob_zone, DNS_API)).get("id"), blob_zone) + subscription = f"/subscriptions/{settings.subscription}" + terms = ( + subscription + + "/providers/Microsoft.MarketplaceOrdering/offerTypes/virtualmachine/publishers/azurecyclecloud/offers/azure-cyclecloud/plans/cyclecloud8/agreements/current" + ) + agreement = await arm.request("GET", terms, "2021-01-01") + if agreement.get("properties", {}).get("accepted") is not True: + raise ValueError( + "Accept the CycleCloud marketplace image terms before validation; this test does not accept terms" + ) + images = await arm.request( + "GET", + subscription + + f"/providers/Microsoft.Compute/locations/{location}/publishers/azurecyclecloud/artifacttypes/vmimage/offers/azure-cyclecloud/skus/cyclecloud8/versions", + "2024-07-01", + ) + if not isinstance(images, list) or not images or not all(i.get("name") for i in images): + raise ValueError("The CycleCloud marketplace image is unavailable in the core region") + return { + "location": location, + "vnet": vnet, + "address_space": address_space, + "subnet": subnet, + "prefixes": prefixes, + "zone": zone, + "blob_zone": blob_zone, + } + + +def resource_ids(settings, core, identifier): + suffix = uuid_value(identifier)[-4:] + vm = f"cyclecloud-{suffix}" + storage = f"stgcc{settings.tre_id}{suffix}".replace("-", "") + group = settings.group + return { + "vm": f"{group}/providers/Microsoft.Compute/virtualMachines/{vm}", + "disk": f"{group}/providers/Microsoft.Compute/disks/{vm}-osdisk", + "nic": f"{group}/providers/Microsoft.Network/networkInterfaces/nic-cyclecloud-{settings.tre_id}-{suffix}", + "nsg": f"{group}/providers/Microsoft.Network/networkSecurityGroups/nsg-cyclecloud-{settings.tre_id}-{suffix}", + "storage": f"{group}/providers/Microsoft.Storage/storageAccounts/{storage}", + "endpoint": f"{group}/providers/Microsoft.Network/privateEndpoints/pe-{storage}", + "zone": core["zone"], + } + + +def resource_types(ids): + return [ + (ids[k], v) + for k, v in ( + ("vm", COMPUTE_API), + ("disk", DISK_API), + ("nic", NETWORK_API), + ("nsg", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ) + ] + + +async def guard_owned(arm, ids, tags): + # The disk is deleted through its VM. Its generated tags are not guaranteed. + for key, version in ( + ("vm", COMPUTE_API), + ("nic", NETWORK_API), + ("nsg", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ): + record = await arm.request("GET", ids[key], version, missing_ok=True) + if record is not None: + owned(record, ids[key], tags) + + +async def principal_roles(arm, subscription, principal): + uuid_value(principal) + endpoint = f"/subscriptions/{subscription}/providers/Microsoft.Authorization/roleAssignments" + query = {"$filter": f"principalId eq '{principal}'"} + records = [] + for _ in range(20): + page = await arm.request("GET", endpoint, ROLE_API, query=query) + records.extend(page["value"]) + link = page.get("nextLink") + if not link: + return records + parsed = urlsplit(link) + if ( + parsed.scheme != "https" + or parsed.netloc != "management.azure.com" + or parsed.path.lower() != endpoint.lower() + ): + raise ValueError("Unexpected role-assignment continuation URL") + query = dict(parse_qsl(parsed.query)) + query.pop("api-version", None) + raise RuntimeError("CycleCloud role-assignment pagination exceeded its bound") + + +def rule_values(properties, single, plural): + values = list(properties.get(plural) or []) + if properties.get(single): + values.append(properties[single]) + return tuple(sorted(values)) + + +def validate_network_rules(nsg, address_space, clusters): + """Allow the core network, the cluster ranges and the load balancer. Deny other virtual network sources.""" + inbound = [r["properties"] for r in nsg["properties"]["securityRules"] if r["properties"]["direction"] == "Inbound"] + allowed = sorted( + ( + p["protocol"].lower(), + rule_values(p, "destinationPortRange", "destinationPortRanges"), + rule_values(p, "sourceAddressPrefix", "sourceAddressPrefixes"), + ) + for p in inbound + if p["access"] == "Allow" + ) + expected = [("*", ("*",), ("AzureLoadBalancer",)), ("tcp", ("22", "443"), tuple(sorted(address_space)))] + if clusters: + expected.append(("tcp", ("9443",), tuple(sorted(clusters)))) + assert allowed == sorted(expected), "CycleCloud inbound allow rules differ from the core and cluster ranges" + denied = [p for p in inbound if p["access"] == "Deny"] + assert len(denied) == 1, "CycleCloud needs one inbound deny rule" + deny = denied[0] + assert ( + deny["protocol"], + rule_values(deny, "destinationPortRange", "destinationPortRanges"), + rule_values(deny, "sourceAddressPrefix", "sourceAddressPrefixes"), + ) == ("*", ("*",), ("VirtualNetwork",)), "CycleCloud must deny other inbound virtual network traffic" + assert all(p["priority"] < deny["priority"] for p in inbound if p is not deny), ( + "The CycleCloud deny rule must have the lowest priority" + ) + + +async def validate_resources(arm, settings, core, ids, identifier, clusters=()): + tags = {"tre_id": settings.tre_id, "tre_shared_service_id": identifier} + records = {} + for key, version in ( + ("vm", COMPUTE_API), + ("nic", NETWORK_API), + ("nsg", NETWORK_API), + ("storage", STORAGE_API), + ("endpoint", NETWORK_API), + ("zone", DNS_API), + ): + records[key] = await arm.request("GET", ids[key], version) + owned(records[key], ids[key], tags) + vm = records["vm"] + properties = vm["properties"] + assert properties["provisioningState"] == "Succeeded" + assert properties["hardwareProfile"]["vmSize"] == "Standard_D4s_v5" + image = properties["storageProfile"]["imageReference"] + assert (image["publisher"], image["offer"], image["sku"]) == ("azurecyclecloud", "azure-cyclecloud", "cyclecloud8") + assert all( + vm["plan"].get(k) == v + for k, v in {"name": "cyclecloud8", "product": "azure-cyclecloud", "publisher": "azurecyclecloud"}.items() + ) + assert vm["identity"]["type"] == "SystemAssigned" + principal = uuid_value(vm["identity"]["principalId"]) + same_id(properties["storageProfile"]["osDisk"]["managedDisk"]["id"], ids["disk"]) + assert len(properties["networkProfile"]["networkInterfaces"]) == 1 + same_id(properties["networkProfile"]["networkInterfaces"][0]["id"], ids["nic"]) + nsg = records["nic"]["properties"].get("networkSecurityGroup") or {} + same_id(nsg.get("id"), ids["nsg"]) + validate_network_rules(records["nsg"], core["address_space"], clusters) + configs = records["nic"]["properties"]["ipConfigurations"] + assert len(configs) == 1 + ip = configs[0]["properties"] + same_id(ip["subnet"]["id"], core["subnet"]) + assert not ip.get("publicIPAddress"), "CycleCloud VM must not have a public IP" + assert any(ip_address(ip["privateIPAddress"]) in ip_network(cidr) for cidr in core["prefixes"]) + dns = await arm.request("GET", ids["zone"] + "/A/@", DNS_API) + assert dns["properties"]["aRecords"] == [{"ipv4Address": ip["privateIPAddress"]}] + link = await arm.request("GET", ids["zone"] + "/virtualNetworkLinks/cyclecloudlink-core", DNS_API) + same_id(link["properties"]["virtualNetwork"]["id"], core["vnet"]) + storage = records["storage"] + assert storage["sku"]["name"] == "Standard_GRS" + assert storage["properties"]["encryption"]["requireInfrastructureEncryption"] is True + assert storage["properties"]["allowCrossTenantReplication"] is False + pe = records["endpoint"]["properties"] + same_id(pe["subnet"]["id"], core["subnet"]) + connections = pe["privateLinkServiceConnections"] + assert len(connections) == 1 + connection = connections[0]["properties"] + same_id(connection["privateLinkServiceId"], ids["storage"]) + assert [v.lower() for v in connection["groupIds"]] == ["blob"] + assert connection["privateLinkServiceConnectionState"]["status"] == "Approved" + zone_group = await arm.request("GET", ids["endpoint"] + "/privateDnsZoneGroups/private-dns-zone-group", NETWORK_API) + zones = zone_group["properties"]["privateDnsZoneConfigs"] + assert len(zones) == 1 + same_id(zones[0]["properties"]["privateDnsZoneId"], core["blob_zone"]) + roles = await principal_roles(arm, settings.subscription, principal) + assert all(r["properties"]["principalId"].lower() == principal for r in roles), ( + "Role query returned a different principal" + ) + assert any( + r["properties"]["scope"].lower() == f"/subscriptions/{settings.subscription}" + and r["properties"]["roleDefinitionId"].lower().endswith("/" + CONTRIBUTOR) + for r in roles + ), "CycleCloud subscription Contributor assignment is missing" + return principal + + +async def wait_power(arm, vm_id, expected): + async with asyncio.timeout(10 * 60): + while True: + vm = await arm.request("GET", vm_id, COMPUTE_API, expand=True) + codes = {s["code"] for s in vm["properties"].get("instanceView", {}).get("statuses", [])} + if f"PowerState/{expected}" in codes: + return + await asyncio.sleep(10) + + +async def assert_removed(arm, ids, subscription, principal=None): + async with asyncio.timeout(10 * 60): + while True: + remaining = [ + identifier + for identifier, version in resource_types(ids) + if await arm.request("GET", identifier, version, missing_ok=True) is not None + ] + roles = await principal_roles(arm, subscription, principal) if principal else [] + if not remaining and not roles: + return + await asyncio.sleep(10) + + +# Other services add and remove their own collections while this case runs. +SERVICE_COLLECTION = re.compile(r"(?:nrc|arc)_svc_([0-9a-f-]{36})") + + +def other_service(name, identifier): + match = SERVICE_COLLECTION.match(name) + return match is not None and match.group(1) != identifier + + +def firewall_rules(resource, identifier=None): + result = {} + for key in ("rule_collections", "network_rule_collections"): + values = resource["properties"].get(key, []) + result[key] = {v["name"]: deepcopy(v) for v in values} + assert len(result[key]) == len(values), "Duplicate firewall collection names" + result[key] = {name: v for name, v in result[key].items() if not other_service(name, identifier)} + return result + + +def validate_firewall(resource, baseline, identifier, prefixes): + rules = firewall_rules(resource, identifier) + for key, prefix in (("rule_collections", "arc"), ("network_rule_collections", "nrc")): + name = f"{prefix}_svc_{identifier}_cyclecloud" + collection = rules[key].pop(name) + assert collection["action"] == "Allow" and len(collection["rules"]) == 1 + rule = collection["rules"][0] + assert set(rule["source_addresses"]) == set(prefixes) + if prefix == "arc": + assert rule["protocols"] == [{"port": "443", "type": "Https"}] + assert set(rule["target_fqdns"]) == { + "github.com", + "api.github.com", + "codeload.github.com", + "objects.githubusercontent.com", + } + else: + assert rule["destination_addresses"] == ["AzureResourceManager"] + assert rule["destination_ports"] == ["443"] and rule["protocols"] == ["TCP"] + assert rules == baseline, "Unrelated firewall rules changed" diff --git a/e2e_tests/resources/cyclecloud_lifecycle.py b/e2e_tests/resources/cyclecloud_lifecycle.py new file mode 100644 index 0000000000..99456a3caa --- /dev/null +++ b/e2e_tests/resources/cyclecloud_lifecycle.py @@ -0,0 +1,211 @@ +"""Own a CycleCloud service before polling and reserve time for its deletion.""" + +import asyncio +from builtins import BaseExceptionGroup +from contextlib import asynccontextmanager +import math +import os +import time + +from httpx import AsyncClient + +from e2e_tests.bundle_evidence import record_operation, record_resource +from e2e_tests.helpers import TIMEOUT, get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources.cyclecloud import TEMPLATE, uuid_value +from e2e_tests.resources.resource import disable_and_delete_resource +from e2e_tests.timeouts import cleanup_deadline + +LIFECYCLE_SECONDS = 270 * 60 +CLEANUP_RESERVE_SECONDS = 120 * 60 +TERMINAL = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} +FAILED = {"deployment_failed", "updating_failed", "deleting_failed", "action_failed", "pipeline_failed"} + + +def lifecycle_deadlines(): + configured = os.environ.get("CYCLECLOUD_VALIDATION_DEADLINE", "") + deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS + if not math.isfinite(deadline): + raise ValueError("CycleCloud validation deadline must be finite") + remaining = min(LIFECYCLE_SECONDS, deadline - time.time()) + if remaining <= CLEANUP_RESERVE_SECONDS: + raise TimeoutError("Insufficient CycleCloud time remains after reserving cleanup") + finish = asyncio.get_running_loop().time() + remaining + return finish - CLEANUP_RESERVE_SECONDS, finish + + +class CycleCloudService: + def __init__(self, verify, finish, payload, before_remove, after_remove): + self.verify = verify + self.finish = finish + self.payload = payload + self.path = None + self.identifier = None + self.before_remove = before_remove + self.after_remove = after_remove + + def validate_owner(self, body): + resource = body["sharedService"] + if ( + resource["id"] != self.identifier + or resource["templateName"] != TEMPLATE + or any( + resource["properties"].get(k) != self.payload["properties"][k] for k in ("display_name", "description") + ) + ): + raise ValueError("CycleCloud resource ownership changed") + return resource + + async def headers(self): + return get_auth_header(await get_admin_token(self.verify)) + + async def create(self): + if self.payload.get("templateName") != TEMPLATE or any( + not self.payload.get("properties", {}).get(k) for k in ("display_name", "description") + ): + raise ValueError("CycleCloud creation needs the expected template and ownership properties") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.post( + get_full_endpoint("/api/shared-services"), headers=await self.headers(), json=self.payload + ) + if response.status_code != 202: + raise RuntimeError(f"CycleCloud creation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + identifier = uuid_value(operation["resourceId"]) + path = operation["resourcePath"] + if path != f"/shared-services/{identifier}": + raise ValueError("Accepted CycleCloud resource has an unexpected path") + self.path, self.identifier = path, identifier + # Own the accepted resource before evidence writes or client closure. + record_resource(self.payload, operation, "POST") + resource, operations = await self.wait_terminal() + if resource["deploymentStatus"] != "deployed" or any(o["status"] in FAILED for o in operations): + raise RuntimeError("CycleCloud deployment or firewall pipeline failed") + return resource + + async def wait_terminal(self, *, allow_absent=False, operation_id=None): + endpoint = get_full_endpoint(f"/api{self.path}") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get(endpoint, headers=await self.headers()) + if response.status_code == 404 and allow_absent: + return None + if response.status_code != 200: + raise RuntimeError(f"Could not inspect CycleCloud resource: HTTP {response.status_code}") + resource = self.validate_owner(response.json()) + response = await client.get(endpoint + "/operations", headers=await self.headers()) + if response.status_code != 200: + raise RuntimeError(f"Could not inspect CycleCloud operations: HTTP {response.status_code}") + operations = response.json()["operations"] + if not isinstance(operations, list): + raise ValueError("CycleCloud operations must be a list") + for operation in operations: + uuid_value(operation["id"]) + record_operation( + f"/api{self.path}/operations/{operation['id']}", + operation["status"], + operation["status"] in TERMINAL, + ) + expected_seen = operation_id is None or any(o["id"] == operation_id for o in operations) + if ( + expected_seen + and operations + and resource["deploymentStatus"] in TERMINAL + and all(o["status"] in TERMINAL for o in operations) + ): + return resource, operations + await asyncio.sleep(15) + + async def change(self, *, action=None, properties=None): + if action not in (None, "start", "stop") or (action is not None) == bool(properties): + raise ValueError("Select one supported CycleCloud action or a property update") + resource, _ = await self.wait_terminal() + endpoint = get_full_endpoint(f"/api{self.path}") + headers = await self.headers() + headers["etag"] = resource["_etag"] + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + if action: + response = await client.post(endpoint + "/invoke-action", headers=headers, params={"action": action}) + else: + response = await client.patch(endpoint, headers=headers, json={"properties": properties}) + if response.status_code != 202: + raise RuntimeError(f"CycleCloud mutation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + if operation["resourcePath"] != self.path or operation["resourceId"] != self.identifier: + raise ValueError("CycleCloud mutation returned a different resource") + operation_id = uuid_value(operation["id"]) + record_resource(None, operation, "POST" if action else "PATCH") + result, operations = await self.wait_terminal(operation_id=operation_id) + observed = next(o for o in operations if o["id"] == operation_id) + if observed["status"] in FAILED or result["deploymentStatus"] in FAILED: + raise RuntimeError("CycleCloud mutation or its pipeline failed") + if properties and any(result["properties"].get(k) != v for k, v in properties.items()): + raise AssertionError("CycleCloud property update was not preserved") + return result + + async def close(self): + if self.path is None: + return [] + try: + async with cleanup_deadline(self.finish - asyncio.get_running_loop().time()): + if await self.wait_terminal(allow_absent=True) is not None: + await self.before_remove(self.identifier) + await disable_and_delete_resource( + f"/api{self.path}", await get_admin_token(self.verify), self.verify, allow_failed_disable=True + ) + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get( + get_full_endpoint(f"/api{self.path}"), headers=await self.headers() + ) + if response.status_code == 404: + break + if response.status_code != 200: + raise RuntimeError(f"Could not verify CycleCloud deletion: HTTP {response.status_code}") + self.validate_owner(response.json()) + await asyncio.sleep(10) + await self.after_remove(self.identifier) + except (Exception, asyncio.CancelledError) as error: + error.add_note(f"CycleCloud cleanup failed for {self.path}") + return [error] + return [] + + +@asynccontextmanager +async def cyclecloud_lifecycle(verify, payload, before_remove, after_remove): + work_finish, finish = lifecycle_deadlines() + resource = CycleCloudService(verify, finish, payload, before_remove, after_remove) + original = None + try: + async with asyncio.timeout_at(work_finish): + yield resource + except BaseException as error: + original = error + raise + finally: + cleanup = asyncio.create_task(resource.close()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + failures = cleanup.result() + failure = original if original is not None else cancelled + if failure is not None: + for error in failures: + failure.add_note(f"CycleCloud cleanup also failed: {error!r}; {error.__notes__}") + elif failures: + raise BaseExceptionGroup("CycleCloud cleanup failed", failures) + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/resources/ohdsi.py b/e2e_tests/resources/ohdsi.py new file mode 100644 index 0000000000..32262e5659 --- /dev/null +++ b/e2e_tests/resources/ohdsi.py @@ -0,0 +1,324 @@ +"""Read OHDSI infrastructure without requesting the values of application settings or secrets.""" + +import asyncio +from copy import deepcopy +from dataclasses import dataclass +import os +import re + +from e2e_tests import config +from e2e_tests.resources.ohdsi_lifecycle import require_uuid + +TEMPLATE = "tre-workspace-service-ohdsi" +FIREWALL = "tre-shared-service-firewall" +BASE = "tre-workspace-base" +WEB_API = "2024-04-01" +POSTGRES_API = "2024-08-01" +NETWORK_API = "2024-05-01" +DNS_API = "2024-06-01" +GROUP_API = "2021-04-01" +IDENTITY_API = "2023-01-31" +STORAGE_API = "2023-05-01" +ROLE_API = "2022-04-01" +SECRETS_USER = "4633458b-17de-408a-b874-0445c86b69e6" + + +@dataclass(frozen=True) +class Settings: + subscription: str + tenant: str + tre_id: str + + def __post_init__(self): + require_uuid(self.subscription) + require_uuid(self.tenant) + if not isinstance(self.tre_id, str) or not re.fullmatch(r"[a-z0-9-]{1,11}", self.tre_id): + raise ValueError("OHDSI requires a valid TRE_ID shorter than 12 characters") + + @property + def group(self): + return f"/subscriptions/{self.subscription}/resourceGroups/rg-{self.tre_id}" + + +def require_settings(): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("OHDSI validation currently supports AzureCloud only") + settings = Settings(os.environ.get("ARM_SUBSCRIPTION_ID", ""), os.environ.get("ARM_TENANT_ID", ""), config.TRE_ID) + if require_uuid(config.AAD_TENANT_ID) != settings.tenant: + raise ValueError("OHDSI requires matching deployment and authentication tenants") + return settings + + +def resource_ids(settings, workspace_id, service_id): + require_uuid(workspace_id) + require_uuid(service_id) + ws = f"{settings.tre_id}-ws-{workspace_id[-4:]}" + suffix = f"{ws}-svc-{service_id[-4:]}" + group = f"/subscriptions/{settings.subscription}/resourceGroups/rg-{ws}" + vnet = f"{group}/providers/Microsoft.Network/virtualNetworks/vnet-{ws}" + apps = {key: f"{group}/providers/Microsoft.Web/sites/app-ohdsi-{key}-{suffix}" for key in ("atlas", "webapi")} + endpoints = { + key: f"{group}/providers/Microsoft.Network/privateEndpoints/pe-{app.rsplit('/', 1)[1]}" + for key, app in apps.items() + } + storage = f"{group}/providers/Microsoft.Storage/storageAccounts/{('stg' + ws[-8:]).replace('-', '').lower()}" + vault = f"{group}/providers/Microsoft.KeyVault/vaults/kv-{ws[-20:].lower()}" + # The Web API reads these secrets through its Key Vault references. + secrets = ( + f"jdbc-connectionstring-{service_id[-4:]}", + f"ohdsi-admin-password-{service_id[-4:]}", + f"ohdsi-app-password-{service_id[-4:]}", + "workspace-client-id", + "workspace-client-secret", + ) + ids = { + "group": group, + "vnet": vnet, + "apps": apps, + "endpoints": endpoints, + "postgres": f"{group}/providers/Microsoft.DBforPostgreSQL/flexibleServers/psql-server-{suffix}", + "postgres_subnet": f"{vnet}/subnets/PostgreSQLSubnet{service_id[-4:]}", + "nsg": f"{group}/providers/Microsoft.Network/networkSecurityGroups/nsg-psql-{suffix}", + "identity": f"{group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-ohdsi-webapi-{suffix}", + "share": f"{storage}/fileServices/default/shares/atlas-{suffix}", + "vault": vault, + "secrets": [f"{vault}/secrets/{name}" for name in secrets], + "web_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.azurewebsites.net", + "pg_zone": f"{settings.group}/providers/Microsoft.Network/privateDnsZones/privatelink.postgres.database.azure.com", + } + ids["removal"] = [(value, WEB_API) for value in apps.values()] + [ + (value, NETWORK_API) for value in endpoints.values() + ] + ids["removal"] += [ + (ids[key], version) + for key, version in ( + ("postgres", POSTGRES_API), + ("postgres_subnet", NETWORK_API), + ("nsg", NETWORK_API), + ("identity", IDENTITY_API), + ("share", STORAGE_API), + ) + ] + return ids + + +def same_id(actual, expected): + assert isinstance(actual, str) and actual.lower() == expected.lower(), "Unexpected OHDSI ARM resource ID" + + +def owned(resource, identifier, tags): + same_id(resource.get("id"), identifier) + assert all(resource.get("tags", {}).get(key) == value for key, value in tags.items()), "OHDSI ownership tags differ" + + +async def guard_owned(arm, ids, tags): + for identifier, api in ( + [(v, WEB_API) for v in ids["apps"].values()] + + [(ids["postgres"], POSTGRES_API), (ids["identity"], IDENTITY_API), (ids["nsg"], NETWORK_API)] + + [(v, NETWORK_API) for v in ids["endpoints"].values()] + ): + resource = await arm.request("GET", identifier, api, missing_ok=True) + if resource is not None: + owned(resource, identifier, tags) + + +async def preflight(arm, settings): + group = await arm.request("GET", settings.group, GROUP_API) + owned(group, settings.group, {"tre_id": settings.tre_id}) + for zone in ("privatelink.azurewebsites.net", "privatelink.postgres.database.azure.com"): + identifier = f"{settings.group}/providers/Microsoft.Network/privateDnsZones/{zone}" + owned(await arm.request("GET", identifier, DNS_API), identifier, {"tre_id": settings.tre_id}) + + +async def postgres_core_dns_link(arm, settings, *, missing_ok=False): + identifier = ( + f"{settings.group}/providers/Microsoft.Network/privateDnsZones/" + "privatelink.postgres.database.azure.com/virtualNetworkLinks/core" + ) + link = await arm.request("GET", identifier, DNS_API, missing_ok=missing_ok) + if link is None: + assert missing_ok, "Shared PostgreSQL core DNS link is missing" + return None + same_id(link.get("id"), identifier) + properties = link["properties"] + same_id( + properties.get("virtualNetwork", {}).get("id"), + f"{settings.group}/providers/Microsoft.Network/virtualNetworks/vnet-{settings.tre_id}", + ) + assert properties.get("registrationEnabled") is False, "Shared PostgreSQL DNS registration changed" + assert properties.get("provisioningState") == "Succeeded", "Shared PostgreSQL core DNS link is not ready" + return link + + +def validate_app(app, configuration, identifier, ids, tags, key): + owned(app, identifier, tags) + props = app["properties"] + assert props.get("state") == "Running", "OHDSI web app is not running" + assert props.get("publicNetworkAccess") == "Disabled", "OHDSI web app permits public access" + assert props.get("httpsOnly") is True, "OHDSI web app permits HTTP" + same_id(props.get("virtualNetworkSubnetId"), ids["vnet"] + "/subnets/WebAppsSubnet") + site = configuration["properties"] + assert site.get("ftpsState") == "Disabled", "OHDSI web app permits FTP" + assert site.get("minTlsVersion") == "1.3", "OHDSI web app TLS setting differs" + # The bundle sets no docker_registry_url, so azurerm writes an empty registry host before the image. + assert site.get("linuxFxVersion") == f"DOCKER|/index.docker.io/ohdsi/{key}:2.12.1", "OHDSI image differs" + if key == "webapi": + identity = app.get("identity", {}) + assert identity.get("type") == "UserAssigned", "Unexpected OHDSI application identity type" + assert {k.lower() for k in identity.get("userAssignedIdentities", {})} == {ids["identity"].lower()} + same_id(props.get("keyVaultReferenceIdentity"), ids["identity"]) + + +async def validate_resources(arm, settings, ids, tags): + async with asyncio.timeout(10 * 60): + postgres = await arm.request("GET", ids["postgres"], POSTGRES_API) + owned(postgres, ids["postgres"], tags) + props = postgres["properties"] + assert props.get("state") == "Ready", "OHDSI PostgreSQL is not ready" + assert props.get("version") == "14", "OHDSI PostgreSQL version differs" + assert props.get("network", {}).get("publicNetworkAccess") == "Disabled", "PostgreSQL permits public access" + same_id(props["network"].get("delegatedSubnetResourceId"), ids["postgres_subnet"]) + same_id(props["network"].get("privateDnsZoneArmResourceId"), ids["pg_zone"]) + subnet = await arm.request("GET", ids["postgres_subnet"], NETWORK_API) + same_id(subnet.get("id"), ids["postgres_subnet"]) + same_id(subnet["properties"].get("networkSecurityGroup", {}).get("id"), ids["nsg"]) + assert subnet["properties"].get("defaultOutboundAccess") is False + assert any( + d.get("properties", {}).get("serviceName") == "Microsoft.DBforPostgreSQL/flexibleServers" + for d in subnet["properties"].get("delegations", []) + ) + for key, identifier in ids["apps"].items(): + app = await arm.request("GET", identifier, WEB_API) + configuration = await arm.request("GET", identifier + "/config/web", WEB_API) + validate_app(app, configuration, identifier, ids, tags, key) + endpoint = await arm.request("GET", ids["endpoints"][key], NETWORK_API) + owned(endpoint, ids["endpoints"][key], tags) + properties = endpoint["properties"] + assert properties.get("provisioningState") == "Succeeded" + same_id(properties.get("subnet", {}).get("id"), ids["vnet"] + "/subnets/ServicesSubnet") + assert not properties.get("manualPrivateLinkServiceConnections") + connections = properties.get("privateLinkServiceConnections", []) + assert len(connections) == 1 + connection = connections[0]["properties"] + same_id(connection.get("privateLinkServiceId"), identifier) + assert connection.get("groupIds") == ["sites"] + assert connection.get("privateLinkServiceConnectionState", {}).get("status") == "Approved" + zone_group_id = ids["endpoints"][key] + "/privateDnsZoneGroups/privatelink.azurewebsites.net" + zone_group = await arm.request("GET", zone_group_id, NETWORK_API) + same_id(zone_group.get("id"), zone_group_id) + zones = zone_group["properties"].get("privateDnsZoneConfigs", []) + assert len(zones) == 1 + same_id(zones[0]["properties"].get("privateDnsZoneId"), ids["web_zone"]) + identity = await arm.request("GET", ids["identity"], IDENTITY_API) + owned(identity, ids["identity"], tags) + assert require_uuid(identity["properties"].get("tenantId")) == settings.tenant + principal = require_uuid(identity["properties"].get("principalId")) + assignments = await arm.request( + "GET", + ids["vault"] + "/providers/Microsoft.Authorization/roleAssignments", + ROLE_API, + query=principal_filter(principal), + ) + assert isinstance(assignments.get("value"), list) and not assignments.get("nextLink"), ( + "Incomplete OHDSI role inventory" + ) + roles = [ + r["properties"] for r in assignments["value"] if r["properties"].get("principalId", "").lower() == principal + ] + # One role on each secret, and no role on the vault or above it. + scopes = sorted(r.get("scope", "").lower() for r in roles) + assert scopes == sorted(s.lower() for s in ids["secrets"]), "Unexpected OHDSI secret role scopes" + assert all(r.get("roleDefinitionId", "").rsplit("/", 1)[-1].lower() == SECRETS_USER for r in roles) + # The roles on the secrets must let the Web API resolve each of its Key Vault references. + references = await key_vault_references(arm, ids["apps"]["webapi"]) + names = {r.get("secretName", "").lower() for r in references} + assert names == {s.rsplit("/", 1)[1].lower() for s in ids["secrets"]}, "Unexpected OHDSI Key Vault references" + statuses = sorted({str(r.get("status")) for r in references}) + assert statuses == ["Resolved"], f"OHDSI Key Vault references are not resolved: {statuses}" + return principal + + +async def key_vault_references(arm, site_id, attempts=10): + """Read the Key Vault references of a web app. Azure returns the status of each reference, not the value.""" + for attempt in range(attempts): + page = await arm.request("GET", site_id + "/config/configreferences/appsettings", WEB_API) + assert isinstance(page.get("value"), list) and not page.get("nextLink"), "Incomplete OHDSI Key Vault references" + references = [r.get("properties", {}) for r in page["value"]] + # The web app resolves its references when it starts, so a new web app can show an earlier status. + if (references and all(r.get("status") == "Resolved" for r in references)) or attempt == attempts - 1: + return references + await asyncio.sleep(30) + + +# Other services add and remove their own collections while this case runs. +SERVICE_COLLECTION = re.compile(r"(?:nrc|arc)_svc_([0-9a-f-]{36})") + + +def other_service(collection, service_id): + match = SERVICE_COLLECTION.match(collection.get("name", "")) + return match is not None and match.group(1) != service_id + + +def firewall_rules(resource, service_id=None): + return { + key: [deepcopy(c) for c in resource["properties"].get(key, []) if not other_service(c, service_id)] + for key in ("network_rule_collections", "rule_collections") + } + + +def validate_firewall(resource, baseline, service_id, prefixes): + rules = firewall_rules(resource, service_id) + for key, prefix in (("network_rule_collections", "nrc"), ("rule_collections", "arc")): + name = f"{prefix}_svc_{service_id}" + matches = [c for c in rules[key] if c.get("name") == name] + assert len(matches) == 1, "OHDSI firewall collection is missing or duplicated" + collection = matches[0] + rules[key].remove(collection) + assert collection["action"] == "Allow" and len(collection["rules"]) == 1 + rule = collection["rules"][0] + assert set(rule["source_addresses"]) == set(prefixes) + if prefix == "nrc": + assert rule["destination_addresses"] == ["AzureActiveDirectory"] + assert rule["destination_ports"] == ["*"] and rule["protocols"] == ["TCP"] + else: + assert rule["protocols"] == [{"port": "80", "type": "Http"}, {"port": "443", "type": "Https"}] + assert set(rule["target_fqdns"]) == { + "*.msftauth.net", + "*.msauth.net", + "login.microsoftonline.com", + "aadcdn.msftauthimages.net", + "aadcdn.msauthimages.net", + "*.login.live.com", + "*.microsoftonline-p.com", + "msft.sts.microsoft.com", + } + assert rules == baseline, "Unrelated firewall collections changed" + + +async def assert_removed(arm, identifiers): + async with asyncio.timeout(5 * 60): + for identifier, api in identifiers: + while await arm.request("GET", identifier, api, missing_ok=True) is not None: + await asyncio.sleep(10) + + +def principal_filter(principal): + # With this filter, Azure returns the roles of the principal at, above and below the scope. + return {"$filter": f"principalId eq '{principal}'"} + + +async def assert_roles_removed(arm, vault, principal): + async with asyncio.timeout(5 * 60): + while True: + roles = await arm.request( + "GET", + vault + "/providers/Microsoft.Authorization/roleAssignments", + ROLE_API, + query=principal_filter(principal), + ) + assert isinstance(roles.get("value"), list) and not roles.get("nextLink"), ( + "Incomplete OHDSI cleanup role inventory" + ) + if not any(r["properties"].get("principalId", "").lower() == principal for r in roles["value"]): + return + await asyncio.sleep(10) diff --git a/e2e_tests/resources/ohdsi_lifecycle.py b/e2e_tests/resources/ohdsi_lifecycle.py new file mode 100644 index 0000000000..363e06ca3c --- /dev/null +++ b/e2e_tests/resources/ohdsi_lifecycle.py @@ -0,0 +1,289 @@ +"""Own OHDSI prerequisites until their dependent resources are removed.""" + +import asyncio +from builtins import BaseExceptionGroup +from contextlib import asynccontextmanager +from dataclasses import dataclass +import math +import os +import time +from uuid import UUID + +from httpx import AsyncClient + +from e2e_tests.bundle_evidence import record_operation, record_resource +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import TIMEOUT, get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources.resource import disable_and_delete_resource +from e2e_tests.timeouts import cleanup_deadline + +LIFECYCLE_SECONDS = 270 * 60 +CLEANUP_RESERVE_SECONDS = 120 * 60 +CREATE_SECONDS = 90 * 60 +TERMINAL_STATES = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} +FAILED_STATES = {"deployment_failed", "updating_failed", "deleting_failed", "action_failed", "pipeline_failed"} + + +def lifecycle_deadlines(): + configured = os.environ.get("OHDSI_VALIDATION_DEADLINE", "") + deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS + if not math.isfinite(deadline): + raise ValueError("OHDSI validation deadline must be finite") + remaining = min(LIFECYCLE_SECONDS, deadline - time.time()) + if remaining <= CLEANUP_RESERVE_SECONDS: + raise TimeoutError("Insufficient OHDSI validation time remains after reserving cleanup") + finish = asyncio.get_running_loop().time() + remaining + return finish - CLEANUP_RESERVE_SECONDS, finish + + +def require_uuid(value): + if not isinstance(value, str): + raise ValueError("Resource identity must be a canonical, non-zero UUID") + try: + parsed = UUID(value) + except ValueError: + raise ValueError("Resource identity must be a canonical, non-zero UUID") from None + if not parsed.int or str(parsed) != value: + raise ValueError("Resource identity must be a canonical, non-zero UUID") + return value + + +@dataclass +class OwnedResource: + path: str + identifier: str + wrapper: str + template: str + display_name: str + description: str + workspace_id: str | None + before_remove: object = None + after_remove: object = None + + def validate(self, body): + resource = body.get(self.wrapper, {}) + properties = resource.get("properties", {}) + if ( + resource.get("id") != self.identifier + or resource.get("templateName") != self.template + or properties.get("display_name") != self.display_name + or properties.get("description") != self.description + # The API stores no status until the first deployment status message arrives. + or not isinstance(resource.get("deploymentStatus"), str | None) + ): + raise ValueError("OHDSI resource ownership or deployment status changed") + return resource + + +def creation_contract(payload, endpoint, workspace_id): + if endpoint == "/api/workspaces" and workspace_id is None: + wrapper, template = "workspace", "tre-workspace-base" + if payload.get("properties", {}).get("auth_type") != "Automatic": + raise ValueError("OHDSI validation needs a fresh Automatic workspace") + elif workspace_id is not None and endpoint == f"/api/workspaces/{require_uuid(workspace_id)}/workspace-services": + wrapper, template = "workspaceService", "tre-workspace-service-ohdsi" + properties = payload.get("properties", {}) + if properties.get("configure_data_source") is not False or any( + key in properties for key in ("data_source_config", "data_source_daimons") + ): + raise ValueError("OHDSI validation requires no external data source") + else: + raise ValueError("Unsupported OHDSI resource creation endpoint or parent") + properties = payload.get("properties", {}) + if payload.get("templateName") != template or any( + not isinstance(properties.get(name), str) or not properties[name].strip() + for name in ("display_name", "description") + ): + raise ValueError("OHDSI creation needs the expected template and ownership properties") + return wrapper, template + + +class OHDSIResources: + def __init__(self, verify, work_finish, finish): + self.verify = verify + self.work_finish = work_finish + self.finish = finish + self.owned = [] + + async def token(self, resource): + if resource.workspace_id is not None: + return await get_workspace_owner_token(resource.workspace_id, self.verify) + return await get_admin_token(self.verify) + + async def create(self, payload, endpoint, workspace_id=None, *, after_remove=None, before_remove=None): + wrapper, template = creation_contract(payload, endpoint, workspace_id) + resource = OwnedResource( + "", + "", + wrapper, + template, + payload["properties"]["display_name"], + payload["properties"]["description"], + workspace_id, + before_remove, + after_remove, + ) + async with asyncio.timeout_at(min(self.work_finish, asyncio.get_running_loop().time() + CREATE_SECONDS)): + token = await self.token(resource) + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.post(get_full_endpoint(endpoint), headers=get_auth_header(token), json=payload) + if response.status_code != 202: + raise RuntimeError(f"OHDSI creation failed: HTTP {response.status_code}") + operation = response.json()["operation"] + resource_id = require_uuid(operation["resourceId"]) + resource_path = operation["resourcePath"] + if resource_path != f"{endpoint.removeprefix('/api')}/{resource_id}": + raise ValueError("Accepted OHDSI resource has an unexpected path or parent") + resource.path, resource.identifier = resource_path, resource_id + # Own the accepted resource before evidence writes or client closure. + self.owned.append(resource) + operation_id = require_uuid(operation["id"]) + record_resource(payload, operation, "POST") + result = await self.wait_terminal(resource, allow_absent=False, operation_id=operation_id) + deployed, operations = result + if deployed["deploymentStatus"] != "deployed" or any( + operation["status"] in FAILED_STATES for operation in operations + ): + raise RuntimeError("OHDSI resource deployment or its pipeline failed") + return resource.path, resource.identifier + + async def wait_terminal(self, resource, *, allow_absent=False, operation_id=None): + endpoint = get_full_endpoint(f"/api{resource.path}") + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get(endpoint, headers=get_auth_header(await self.token(resource))) + if response.status_code == 404 and allow_absent: + return None + if response.status_code != 200: + raise RuntimeError(f"Could not inspect owned OHDSI resource: HTTP {response.status_code}") + deployed = resource.validate(response.json()) + response = await client.get( + endpoint + "/operations", headers=get_auth_header(await self.token(resource)) + ) + if response.status_code != 200: + raise RuntimeError(f"Could not inspect OHDSI operations: HTTP {response.status_code}") + operations = response.json()["operations"] + if not isinstance(operations, list): + raise ValueError("OHDSI operations response must contain a list") + for operation in operations: + observed_id = require_uuid(operation["id"]) + record_operation( + f"/api{resource.path}/operations/{observed_id}", + operation["status"], + operation["status"] in TERMINAL_STATES, + ) + if ( + (operation_id is None or any(o["id"] == operation_id for o in operations)) + and deployed["deploymentStatus"] in TERMINAL_STATES + and operations + and all(operation["status"] in TERMINAL_STATES for operation in operations) + ): + return deployed, operations + await asyncio.sleep(30) + + async def upgrade(self, resource, overview): + if resource.workspace_id is None: + raise ValueError("Only the owned OHDSI service can be upgraded") + deployed, _ = await self.wait_terminal(resource) + headers = get_auth_header(await self.token(resource)) + headers["etag"] = deployed["_etag"] + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + response = await client.patch( + get_full_endpoint(f"/api{resource.path}"), headers=headers, json={"properties": {"overview": overview}} + ) + if response.status_code != 202: + raise RuntimeError(f"OHDSI upgrade failed: HTTP {response.status_code}") + operation = response.json()["operation"] + if operation["resourcePath"] != resource.path or operation["resourceId"] != resource.identifier: + raise ValueError("OHDSI upgrade returned a different resource") + operation_id = require_uuid(operation["id"]) + record_resource(None, operation, "PATCH") + deployed, operations = await self.wait_terminal(resource, operation_id=operation_id) + if deployed["deploymentStatus"] != "updated" or any(o["status"] in FAILED_STATES for o in operations): + raise RuntimeError("OHDSI upgrade or its pipeline failed") + assert deployed["properties"].get("overview") == overview, "OHDSI overview update was not preserved" + return deployed + + async def assert_api_removed(self, resource): + async with asyncio.timeout(5 * 60): + async with AsyncClient(verify=self.verify, timeout=TIMEOUT) as client: + while True: + response = await client.get( + get_full_endpoint(f"/api{resource.path}"), + headers=get_auth_header(await self.token(resource)), + ) + if response.status_code == 404: + return + if response.status_code != 200: + raise RuntimeError(f"Could not verify OHDSI removal: HTTP {response.status_code}") + resource.validate(response.json()) + await asyncio.sleep(10) + + async def remove(self, resource): + present = await self.wait_terminal(resource, allow_absent=True) + if present is not None: + if resource.before_remove is not None: + await resource.before_remove(resource.path, resource.identifier) + await disable_and_delete_resource( + f"/api{resource.path}", await self.token(resource), self.verify, allow_failed_disable=True + ) + await self.assert_api_removed(resource) + if resource.after_remove is not None: + await resource.after_remove(resource.path, resource.identifier) + + async def close(self): + failures = [] + while self.owned: + seconds = (self.finish - asyncio.get_running_loop().time()) / len(self.owned) + resource = self.owned.pop() + try: + if resource.workspace_id is None and failures: + raise RuntimeError("OHDSI workspace retained because service cleanup failed") + async with cleanup_deadline(seconds): + await self.remove(resource) + except (Exception, asyncio.CancelledError) as error: + error.add_note(f"OHDSI validation cleanup failed for {resource.path}") + failures.append(error) + return failures + + +@asynccontextmanager +async def ohdsi_lifecycle(verify): + work_finish, finish = lifecycle_deadlines() + resources = OHDSIResources(verify, work_finish, finish) + original = None + async with cleanup_deadline(finish - asyncio.get_running_loop().time()): + try: + async with asyncio.timeout_at(work_finish): + yield resources + except BaseException as error: + original = error + raise + finally: + cleanup = asyncio.create_task(resources.close()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + failures = cleanup.result() + failure = original if original is not None else cancelled + if failure is not None: + for error in failures: + failure.add_note(f"OHDSI cleanup also failed: {error!r}; {error.__notes__}") + elif failures: + raise BaseExceptionGroup("OHDSI resource cleanup failed", failures) + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/test_cyclecloud.py b/e2e_tests/test_cyclecloud.py new file mode 100644 index 0000000000..5792a02466 --- /dev/null +++ b/e2e_tests/test_cyclecloud.py @@ -0,0 +1,136 @@ +"""Validate one CycleCloud server without configuring or creating HPC clusters.""" + +from ipaddress import ip_network +from uuid import uuid4 + +import pytest + +from e2e_tests.bundle_evidence import record_deployed_resource, record_reused_resource +from e2e_tests.helpers import get_admin_token +from e2e_tests.resources.cyclecloud import ( + COMPUTE_API, + FIREWALL, + TEMPLATE, + assert_removed, + firewall_rules, + guard_owned, + preflight, + require_settings, + resource_ids, + uuid_value, + validate_firewall, + validate_resources, + wait_power, +) +from e2e_tests.resources.cyclecloud_lifecycle import cyclecloud_lifecycle +from e2e_tests.resources.resource import get_resource +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.run_bundle import load_catalog + +# A private range outside the core address space. The update allows it to reach port 9443. +CLUSTER_TEST_RANGE = "172.31.254.0/24" + + +async def prerequisites(verify): + catalog = load_catalog() + token = await get_admin_token(verify) + for name in (TEMPLATE, FIREWALL): + template = await get_resource(f"/api/shared-service-templates/{name}", token, verify) + if template["name"] != name or template["version"] != catalog[name]["source_version"]: + raise ValueError(f"Register the checkout version of {name} before CycleCloud validation") + if name == TEMPLATE and not {"start", "stop"} <= {a["name"] for a in template.get("customActions", [])}: + raise ValueError("The registered CycleCloud template must expose start and stop") + services = (await get_resource("/api/shared-services", token, verify))["sharedServices"] + if any(s["templateName"] == TEMPLATE for s in services): + raise ValueError("An existing CycleCloud service prevents isolated validation") + firewalls = [s for s in services if s["templateName"] == FIREWALL] + if len(firewalls) != 1: + raise ValueError("CycleCloud requires exactly one deployed firewall") + firewall = firewalls[0] + if ( + firewall["isEnabled"] is not True + or firewall["deploymentStatus"] not in ("deployed", "updated") + or firewall["templateVersion"] != catalog[FIREWALL]["source_version"] + ): + raise ValueError("Deploy and enable the checkout firewall version before CycleCloud validation") + record_reused_resource(firewall) + return firewall + + +async def validate_lifecycle(verify): + settings = require_settings() + payload = { + "templateName": TEMPLATE, + "properties": { + "display_name": f"CycleCloud validation {uuid4()}", + "description": "Owned CycleCloud lifecycle validation", + }, + } + core = None + baseline = None + firewall = None + principal = None + + async with arm_client() as arm: + + async def current_firewall(): + record = ( + await get_resource(f"/api/shared-services/{firewall['id']}", await get_admin_token(verify), verify) + )["sharedService"] + if record["id"] != firewall["id"] or record["templateName"] != FIREWALL: + raise ValueError("CycleCloud firewall prerequisite identity changed") + return record + + async def before_remove(identifier): + nonlocal principal + ids = resource_ids(settings, core, identifier) + await guard_owned(arm, ids, {"tre_id": settings.tre_id, "tre_shared_service_id": identifier}) + vm = await arm.request("GET", ids["vm"], COMPUTE_API, missing_ok=True) + if vm is not None and vm.get("identity", {}).get("principalId"): + principal = uuid_value(vm["identity"]["principalId"]) + + async def after_remove(identifier): + await assert_removed(arm, resource_ids(settings, core, identifier), settings.subscription, principal) + assert firewall_rules(await current_firewall(), identifier) == baseline, ( + "CycleCloud firewall rules were not restored" + ) + + async with cyclecloud_lifecycle(verify, payload, before_remove, after_remove) as service: + firewall = await prerequisites(verify) + baseline = firewall_rules(firewall) + core = await preflight(arm, settings) + if any(ip_network(CLUSTER_TEST_RANGE).overlaps(ip_network(v)) for v in core["address_space"]): + raise ValueError("The CycleCloud cluster test range must be outside the core address space") + record = await service.create() + ids = resource_ids(settings, core, service.identifier) + + async def inspect(record, clusters=()): + assert record["templateVersion"] == load_catalog()[TEMPLATE]["source_version"] + assert record["properties"]["connection_uri"] == "https://" + core["zone"].rsplit("/", 1)[1] + record_deployed_resource(record, service.path) + identity = await validate_resources(arm, settings, core, ids, service.identifier, clusters) + validate_firewall(await current_firewall(), baseline, service.identifier, core["prefixes"]) + return identity + + # By default, only the core network and the load balancer can connect to the server. + principal = await inspect(record) + await wait_power(arm, ids["vm"], "running") + await service.change(action="stop") + await wait_power(arm, ids["vm"], "deallocated") + await service.change(action="start") + await wait_power(arm, ids["vm"], "running") + updated = await service.change( + properties={ + "overview": "CycleCloud lifecycle metadata upgrade", + "cluster_address_spaces": [CLUSTER_TEST_RANGE], + } + ) + assert await inspect(updated, [CLUSTER_TEST_RANGE]) == principal, ( + "CycleCloud upgrade replaced the VM identity" + ) + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.cyclecloud_validation +async def test_cyclecloud_lifecycle(verify): + await validate_lifecycle(verify) diff --git a/e2e_tests/test_ohdsi.py b/e2e_tests/test_ohdsi.py new file mode 100644 index 0000000000..4a80b911f8 --- /dev/null +++ b/e2e_tests/test_ohdsi.py @@ -0,0 +1,226 @@ +"""Validate a private OHDSI deployment without an external clinical data source.""" + +from uuid import uuid4 + +import pytest + +from e2e_tests import config +from e2e_tests.bundle_evidence import record_deployed_resource, record_reused_resource +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import get_admin_token +from e2e_tests.resources.ohdsi import ( + BASE, + FIREWALL, + TEMPLATE, + GROUP_API, + IDENTITY_API, + DNS_API, + assert_removed, + assert_roles_removed, + firewall_rules, + guard_owned, + owned, + postgres_core_dns_link, + preflight, + require_settings, + resource_ids, + validate_firewall, + validate_resources, +) +from e2e_tests.resources.ohdsi_lifecycle import ohdsi_lifecycle, require_uuid +from e2e_tests.resources.resource import get_resource +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.run_bundle import load_catalog + + +async def prerequisites(verify): + catalog = load_catalog() + token = await get_admin_token(verify) + for name, kind in ((TEMPLATE, "workspace-service"), (BASE, "workspace"), (FIREWALL, "shared-service")): + template = await get_resource(f"/api/{kind}-templates/{name}", token, verify) + if template["name"] != name or template["version"] != catalog[name]["source_version"]: + raise ValueError(f"Register the checkout version of {name} before OHDSI validation") + services = (await get_resource("/api/shared-services", token, verify))["sharedServices"] + firewalls = [s for s in services if s["templateName"] == FIREWALL] + if len(firewalls) != 1: + raise ValueError("OHDSI requires exactly one deployed firewall") + firewall = firewalls[0] + if ( + firewall["isEnabled"] is not True + or firewall["deploymentStatus"] not in ("deployed", "updated") + or firewall["templateVersion"] != catalog[FIREWALL]["source_version"] + ): + raise ValueError("Deploy and enable the checkout firewall version before OHDSI validation") + record_reused_resource(firewall) + return firewall + + +def payload(template, label, **properties): + return { + "templateName": template, + "properties": { + "display_name": f"OHDSI validation {label} {uuid4()}", + "description": "Owned OHDSI lifecycle validation", + **properties, + }, + } + + +async def validate_lifecycle(verify): + settings = require_settings() + async with arm_client() as arm: + async with ohdsi_lifecycle(verify) as resources: + firewall = await prerequisites(verify) + baseline = firewall_rules(firewall) + await preflight(arm, settings) + core_link = await postgres_core_dns_link(arm, settings, missing_ok=True) + workspace_path = None + workspace_id = None + callbacks = None + principal = None + + async def check_core_link(*, required=False): + nonlocal core_link + # A failed install might not reach DNS creation. Preserve any link + # observed before install, during validation or before removal. + core_link = await postgres_core_dns_link(arm, settings, missing_ok=not required and core_link is None) + + async def current_firewall(): + result = ( + await get_resource(f"/api/shared-services/{firewall['id']}", await get_admin_token(verify), verify) + )["sharedService"] + assert result["id"] == firewall["id"] and result["templateName"] == FIREWALL + return result + + async def workspace_record(): + return (await get_resource(f"/api{workspace_path}", await get_admin_token(verify), verify))["workspace"] + + def group_id(identifier): + return ( + f"/subscriptions/{settings.subscription}/resourceGroups/rg-{settings.tre_id}-ws-{identifier[-4:]}" + ) + + async def guard_workspace(_path, identifier): + group = await arm.request("GET", group_id(identifier), GROUP_API, missing_ok=True) + if group is not None: + owned(group, group_id(identifier), {"tre_id": settings.tre_id, "tre_workspace_id": identifier}) + + async def removed_workspace(_path, identifier): + await assert_removed(arm, [(group_id(identifier), GROUP_API)]) + + properties = {"auth_type": "Automatic", "address_space_size": "small", "enable_backup": False} + if config.TEST_WORKSPACE_APP_PLAN: + properties["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN + workspace_path, workspace_id = await resources.create( + payload(BASE, "workspace", **properties), + "/api/workspaces", + before_remove=guard_workspace, + after_remove=removed_workspace, + ) + workspace = await workspace_record() + assert workspace["templateVersion"] == load_catalog()[BASE]["source_version"] + assert workspace["properties"]["auth_type"] == "Automatic" + assert ( + workspace["properties"].get("workspace_subscription_id") or settings.subscription + ).lower() == settings.subscription + record_deployed_resource(workspace, workspace_path) + callbacks = list(workspace["properties"].get("aad_redirect_uris", [])) + await guard_workspace(workspace_path, workspace_id) + + async def guard_service(_path, identifier): + nonlocal principal + ids = resource_ids(settings, workspace_id, identifier) + await guard_workspace(workspace_path, workspace_id) + await guard_owned( + arm, + ids, + { + "tre_id": settings.tre_id, + "tre_workspace_id": workspace_id, + "tre_workspace_service_id": identifier, + }, + ) + identity = await arm.request("GET", ids["identity"], IDENTITY_API, missing_ok=True) + if identity is not None: + principal = require_uuid(identity["properties"]["principalId"]) + await check_core_link() + + async def removed_service(_path, identifier): + ids = resource_ids(settings, workspace_id, identifier) + dns_records = [ + (f"{ids['web_zone']}/A/{app.rsplit('/', 1)[1]}{suffix}", DNS_API) + for app in ids["apps"].values() + for suffix in ("", ".scm") + ] + await assert_removed(arm, ids["removal"] + dns_records) + await check_core_link() + if principal: + await assert_roles_removed(arm, ids["vault"], principal) + assert firewall_rules(await current_firewall(), identifier) == baseline, ( + "OHDSI firewall rules were not restored" + ) + assert (await workspace_record())["properties"].get("aad_redirect_uris", []) == callbacks, ( + "OHDSI callback was not removed" + ) + + service_path, service_id = await resources.create( + payload(TEMPLATE, "service", configure_data_source=False), + f"/api{workspace_path}/workspace-services", + workspace_id, + before_remove=guard_service, + after_remove=removed_service, + ) + ids = resource_ids(settings, workspace_id, service_id) + + async def inspect(record): + await check_core_link(required=True) + assert ( + record["id"] == service_id + and record["templateVersion"] == load_catalog()[TEMPLATE]["source_version"] + ) + record_deployed_resource(record, service_path) + props = record["properties"] + assert props["configure_data_source"] is False + assert props["is_exposed_externally"] is False + assert ( + props["connection_uri"] + == "https://" + ids["apps"]["atlas"].rsplit("/", 1)[1] + ".azurewebsites.net/atlas" + ) + assert ( + props["webapi_uri"] + == "https://" + ids["apps"]["webapi"].rsplit("/", 1)[1] + ".azurewebsites.net/WebAPI/" + ) + expected_callback = props["webapi_uri"] + "user/oauth/callback?client_name=OidcClient" + assert props["authentication_callback_uri"] == expected_callback + parent = await workspace_record() + entries = list(parent["properties"].get("aad_redirect_uris", [])) + expected = {"name": service_id, "value": expected_callback} + assert entries.count(expected) == 1 + entries.remove(expected) + assert entries == callbacks, "Unrelated workspace callbacks changed" + validate_firewall( + await current_firewall(), baseline, service_id, parent["properties"]["address_spaces"] + ) + return await validate_resources( + arm, + settings, + ids, + { + "tre_id": settings.tre_id, + "tre_workspace_id": workspace_id, + "tre_workspace_service_id": service_id, + }, + ) + + record = ( + await get_resource(f"/api{service_path}", await get_workspace_owner_token(workspace_id, verify), verify) + )["workspaceService"] + principal = await inspect(record) + updated = await resources.upgrade(resources.owned[-1], "OHDSI lifecycle metadata upgrade") + assert await inspect(updated) == principal, "OHDSI upgrade replaced the managed identity" + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.ohdsi_validation +async def test_private_ohdsi_lifecycle(verify): + await validate_lifecycle(verify) diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index c38fd4dc82..ef7bd04ea8 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -209,8 +209,7 @@ async def test_patch_firewall(verify): # Tested in test_create_certs_nexus_shared_service # strings.NEXUS_SHARED_SERVICE, strings.AIRLOCK_NOTIFIER_SHARED_SERVICE, - # TODO: fix cyclecloud and enable this - # strings.CYCLECLOUD_SHARED_SERVICE, + # CycleCloud has a separate opt-in lifecycle in test_cyclecloud.py. ] create_airlock_notifier_properties = { diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index 2adf50baa1..7a24f3de00 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -151,6 +151,15 @@ def test_base_workspace_collects_its_own_case_without_reusing_a_workspace(self): self.assertEqual(report["resources"], []) self.assertEqual(report["declared_prerequisites"], {}) + def test_cyclecloud_selects_only_its_server_case_with_firewall_dependency(self): + result, report = self.run_selector("--bundle", "tre-shared-service-cyclecloud", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_cyclecloud.py::test_cyclecloud_lifecycle"]) + self.assertEqual(set(report["declared_prerequisites"]), {"tre-shared-service-firewall"}) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + def test_unknown_partial_and_shell_like_identifiers_fail_before_collection(self): for identifier in ("azuresql", "tre-nonexistent", SQL + "; echo unexpected", SQL + "$(echo unexpected)"): with self.subTest(identifier=identifier): @@ -222,6 +231,42 @@ def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): self.assertEqual(report["source_bundle_version"], version) self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + def test_ohdsi_selects_only_its_private_lifecycle_without_provisioning(self): + result, report = self.run_selector("--bundle", "tre-workspace-service-ohdsi", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_ohdsi.py::test_private_ohdsi_lifecycle"]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + self.assertEqual(set(report["declared_prerequisites"]), {"tre-workspace-base", "tre-shared-service-firewall"}) + self.assertTrue(any("Synapse" in limit for limit in report["limitations"])) + + def test_observed_deployment_evidence_excludes_properties(self): + with tempfile.TemporaryDirectory() as directory: + report = bundle_evidence.BundleReport(Path(directory) / "report.json", {}) + with patch.object(bundle_evidence, "_ACTIVE_REPORT", report): + bundle_evidence.record_deployed_resource( + { + "id": "vm", + "templateName": "review", + "templateVersion": "2.0.9", + "properties": {"secret": "excluded"}, + }, + "/workspaces/ws/workspace-services/service/user-resources/vm", + ) + self.assertEqual( + report.data["resources"], + [ + { + "method": "GET", + "resourceId": "vm", + "resourcePath": "/workspaces/ws/workspace-services/service/user-resources/vm", + "bundle": "review", + "deployed_template_version": "2.0.9", + } + ], + ) + def test_marker_and_parallel_worker_inputs_are_rejected(self): for extra in ({"TEST_BUNDLE_MARKER": "workspace_services"}, {"E2E_TESTS_NUMBER_PROCESSES": "2"}): with self.subTest(environment=extra): diff --git a/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py new file mode 100644 index 0000000000..3c1bf9909a --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_bundle_contract.py @@ -0,0 +1,153 @@ +"""Check the CycleCloud bundle outputs, VM size and network rules.""" + +import base64 +import json +from pathlib import Path +import re +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/shared_services/cyclecloud" + + +def blocks(source, opener): + """Return the body of each block that starts with the opener, for example 'security_rule {'.""" + result = [] + for match in re.finditer(re.escape(opener), source): + depth = 0 + for index in range(match.end() - 1, len(source)): + depth += {"{": 1, "}": -1}.get(source[index], 0) + if depth == 0: + result.append(source[match.end() : index]) + break + return result + + +def setting(block, name): + match = re.search(rf"^\s*{name}\s*=\s*(.+)$", block, re.MULTILINE) + return match.group(1).strip() if match else None + + +class CycleCloudBundleContractTests(unittest.TestCase): + def test_vm_actions_have_a_declared_and_emitted_resource_id(self): + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + output = next(o for o in manifest["outputs"] if o["name"] == "azure_resource_id") + self.assertEqual(output["type"], "string") + for action in ("install", "upgrade", "start", "stop"): + with self.subTest(action=action): + self.assertIn(action, output["applyTo"]) + terraform = next(step["terraform"] for step in manifest[action] if "terraform" in step) + self.assertIn({"name": "azure_resource_id"}, terraform["outputs"]) + for action, command in (("start", "start"), ("stop", "deallocate")): + with self.subTest(action=action): + step = next(s["az"] for s in manifest[action] if s.get("az", {}).get("arguments") == ["vm", command]) + self.assertEqual(step["flags"]["ids"], "${ bundle.outputs.azure_resource_id }") + output_source = (BUNDLE / "terraform/outputs.tf").read_text() + self.assertRegex( + output_source, r'output "azure_resource_id"\s*\{\s*value\s*=\s*azurerm_virtual_machine\.cyclecloud\.id\s*\}' + ) + vm_source = (BUNDLE / "terraform/cyclecloud.tf").read_text() + self.assertIsNotNone(re.search(r'resource "azurerm_virtual_machine" "cyclecloud"', vm_source)) + + def test_new_servers_use_an_offered_size_and_upgrades_keep_the_size(self): + source = (BUNDLE / "terraform/cyclecloud.tf").read_text() + vm = source.split('resource "azurerm_virtual_machine" "cyclecloud" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(vm, r'\n vm_size\s*=\s*"Standard_D4s_v5"\n') + ignored = re.search(r"lifecycle\s*\{\s*ignore_changes\s*=\s*\[([^\]]*)\]", vm) + self.assertIsNotNone(ignored) + self.assertIn("vm_size", [name.strip() for name in ignored.group(1).split(",")]) + + def test_network_card_has_inbound_rules_for_the_core_network(self): + source = (BUNDLE / "terraform/nsg.tf").read_text() + (association,) = blocks( + source, 'resource "azurerm_network_interface_security_group_association" "cyclecloud" {' + ) + self.assertEqual(setting(association, "network_interface_id"), "azurerm_network_interface.cyclecloud.id") + self.assertEqual( + setting(association, "network_security_group_id"), "azurerm_network_security_group.cyclecloud.id" + ) + (nsg,) = blocks(source, 'resource "azurerm_network_security_group" "cyclecloud" {') + self.assertEqual(setting(nsg, "tags"), "local.tre_shared_service_tags") + self.assertEqual(setting(nsg, "resource_group_name"), "local.core_resource_group_name") + rules = blocks(nsg, "security_rule {") + self.assertEqual(len(rules), 3) + self.assertTrue(all(setting(rule, "direction") == '"Inbound"' for rule in rules)) + core = [ + r + for r in rules + if setting(r, "source_address_prefixes") == "data.azurerm_virtual_network.core.address_space" + ] + self.assertEqual(len(core), 1) + self.assertEqual(setting(core[0], "access"), '"Allow"') + self.assertEqual(setting(core[0], "protocol"), '"Tcp"') + self.assertEqual(setting(core[0], "destination_port_ranges"), '["22", "443"]') + balancer = [r for r in rules if setting(r, "source_address_prefix") == '"AzureLoadBalancer"'] + self.assertEqual([setting(r, "access") for r in balancer], ['"Allow"']) + deny = [r for r in rules if setting(r, "access") == '"Deny"'] + self.assertEqual(len(deny), 1) + self.assertEqual(setting(deny[0], "source_address_prefix"), '"VirtualNetwork"') + self.assertEqual(setting(deny[0], "destination_port_range"), '"*"') + self.assertEqual(setting(deny[0], "protocol"), '"*"') + priorities = [int(setting(rule, "priority")) for rule in rules] + self.assertEqual(max(priorities), 4096) + self.assertEqual(int(setting(deny[0], "priority")), 4096) + self.assertEqual(len(set(priorities)), len(priorities)) + vm = (BUNDLE / "terraform/cyclecloud.tf").read_text() + self.assertRegex(vm, r"depends_on\s*=\s*\[azurerm_network_interface_security_group_association\.cyclecloud\]") + + def test_cluster_port_is_open_only_for_the_configured_ranges(self): + source = (BUNDLE / "terraform/nsg.tf").read_text() + (nsg,) = blocks(source, 'resource "azurerm_network_security_group" "cyclecloud" {') + self.assertFalse(any("9443" in rule for rule in blocks(nsg, "security_rule {"))) + (dynamic,) = blocks(nsg, 'dynamic "security_rule" {') + self.assertEqual( + setting(dynamic, "for_each"), + "length(local.cluster_address_spaces) > 0 ? [local.cluster_address_spaces] : []", + ) + (content,) = blocks(dynamic, "content {") + self.assertEqual(setting(content, "access"), '"Allow"') + self.assertEqual(setting(content, "destination_port_range"), '"9443"') + self.assertEqual(setting(content, "source_address_prefixes"), "security_rule.value") + + def test_cluster_address_spaces_is_an_updateable_list_that_is_empty_by_default(self): + schema = json.loads((BUNDLE / "template_schema.json").read_text()) + prop = schema["properties"]["cluster_address_spaces"] + self.assertEqual((prop["type"], prop["default"], prop["updateable"]), ("array", [], True)) + self.assertEqual(prop["items"]["type"], "string") + pattern = re.compile(prop["items"]["pattern"]) + self.assertTrue(pattern.fullmatch("10.1.4.0/24")) + for value in ("10.1.4.0", "10.1.4.0/33", "256.1.4.0/24", "10.1.4/24"): + self.assertIsNone(pattern.fullmatch(value), value) + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + parameter = next(p for p in manifest["parameters"] if p["name"] == "cluster_address_spaces") + self.assertEqual(parameter["type"], "string") + self.assertEqual(json.loads(base64.b64decode(parameter["default"])), []) + for action in ("install", "upgrade", "uninstall"): + with self.subTest(action=action): + terraform = next(step["terraform"] for step in manifest[action] if "terraform" in step) + self.assertEqual( + terraform["vars"]["cluster_address_spaces"], "${ bundle.parameters.cluster_address_spaces }" + ) + self.assertIn('variable "cluster_address_spaces"', (BUNDLE / "terraform/variables.tf").read_text()) + self.assertRegex( + (BUNDLE / "terraform/locals.tf").read_text(), + r"for cidr in jsondecode\(base64decode\(var\.cluster_address_spaces\)\)", + ) + + def test_parameter_set_lists_each_porter_parameter(self): + # make bundle-check-params compares these names, and make bundle-install needs the check to pass. + manifest = yaml.safe_load((BUNDLE / "porter.yaml").read_text()) + parameter_set = json.loads((BUNDLE / "parameters.json").read_text()) + self.assertEqual( + {p["name"] for p in manifest["parameters"]} - {"arm_use_msi"}, + {p["name"] for p in parameter_set["parameters"]} - {"arm_use_msi"}, + ) + + def test_cluster_ranges_use_their_network_address(self): + # The schema accepts a range with host bits, so Terraform must not refuse it after the API stored it. + locals_tf = (BUNDLE / "terraform/locals.tf").read_text() + self.assertRegex(locals_tf, r"cluster_address_spaces\s*=\s*distinct\(\[") + self.assertIn('"${cidrhost(cidr, 0)}/${split("/", cidr)[1]}"', locals_tf) + self.assertNotIn("cidrhost", (BUNDLE / "terraform/variables.tf").read_text()) diff --git a/e2e_tests/unit_tests/test_cyclecloud_configuration.py b/e2e_tests/unit_tests/test_cyclecloud_configuration.py new file mode 100644 index 0000000000..2ea09e82a6 --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_configuration.py @@ -0,0 +1,317 @@ +"""Check CycleCloud evidence against private infrastructure and prerequisite cases.""" + +from copy import deepcopy +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +import yaml + +from e2e_tests.resources import cyclecloud as cc + +SUB = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +PRINCIPAL = "33333333-3333-4333-8333-333333333333" +ROOT = Path(__file__).resolve().parents[2] +CLUSTER = "172.31.254.0/24" + + +def nsg_rule(name, priority, access, protocol, ports, sources): + properties = { + "priority": priority, + "direction": "Inbound", + "access": access, + "protocol": protocol, + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRanges": [], + "sourceAddressPrefixes": [], + } + if len(ports) == 1: + properties["destinationPortRange"] = ports[0] + else: + properties["destinationPortRanges"] = ports + if sources in ("AzureLoadBalancer", "VirtualNetwork"): + properties["sourceAddressPrefix"] = sources + else: + properties["sourceAddressPrefixes"] = sources + return {"name": name, "properties": properties} + + +def default_rules(): + return [ + nsg_rule("AllowCoreHttpsSshInbound", 100, "Allow", "Tcp", ["22", "443"], ["10.0.0.0/22"]), + nsg_rule("AllowAzureLoadBalancerInbound", 120, "Allow", "*", ["*"], "AzureLoadBalancer"), + nsg_rule("DenyVirtualNetworkInbound", 4096, "Deny", "*", ["*"], "VirtualNetwork"), + ] + + +def cluster_rule(sources): + return nsg_rule("AllowClusterNodesInbound", 110, "Allow", "Tcp", ["9443"], sources) + + +def fixture(): + settings = cc.Settings(SUB, "tre-test") + net = settings.group + "/providers/Microsoft.Network/virtualNetworks/vnet-tre-test" + core = { + "vnet": net, + "address_space": ["10.0.0.0/22"], + "subnet": net + "/subnets/SharedSubnet", + "prefixes": ["10.0.1.0/24"], + "zone": settings.group + + "/providers/Microsoft.Network/privateDnsZones/cyclecloud-tre-test.switzerlandnorth.cloudapp.azure.com", + "blob_zone": settings.group + "/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net", + "location": "switzerlandnorth", + } + ids = cc.resource_ids(settings, core, SERVICE) + records = { + value: {"id": value, "tags": {"tre_id": settings.tre_id, "tre_shared_service_id": SERVICE}} + for value in ids.values() + } + records[ids["vm"]].update( + identity={"type": "SystemAssigned", "principalId": PRINCIPAL}, + plan={ + "name": "cyclecloud8", + "product": "azure-cyclecloud", + "publisher": "azurecyclecloud", + "promotionCode": None, + }, + properties={ + "provisioningState": "Succeeded", + "hardwareProfile": {"vmSize": "Standard_D4s_v5"}, + "storageProfile": { + "imageReference": { + "publisher": "azurecyclecloud", + "offer": "azure-cyclecloud", + "sku": "cyclecloud8", + "version": "8.8.0", + }, + "osDisk": {"managedDisk": {"id": ids["disk"]}}, + }, + "networkProfile": {"networkInterfaces": [{"id": ids["nic"]}]}, + }, + ) + records[ids["nic"]]["properties"] = { + "networkSecurityGroup": {"id": ids["nsg"]}, + "ipConfigurations": [{"properties": {"subnet": {"id": core["subnet"]}, "privateIPAddress": "10.0.1.5"}}], + } + records[ids["nsg"]]["properties"] = {"securityRules": default_rules()} + records[ids["zone"] + "/A/@"] = {"properties": {"aRecords": [{"ipv4Address": "10.0.1.5"}]}} + records[ids["zone"] + "/virtualNetworkLinks/cyclecloudlink-core"] = {"properties": {"virtualNetwork": {"id": net}}} + records[ids["storage"]].update( + sku={"name": "Standard_GRS"}, + properties={"encryption": {"requireInfrastructureEncryption": True}, "allowCrossTenantReplication": False}, + ) + records[ids["endpoint"]]["properties"] = { + "subnet": {"id": core["subnet"]}, + "privateLinkServiceConnections": [ + { + "properties": { + "privateLinkServiceId": ids["storage"], + "groupIds": ["blob"], + "privateLinkServiceConnectionState": {"status": "Approved"}, + } + } + ], + } + records[ids["endpoint"] + "/privateDnsZoneGroups/private-dns-zone-group"] = { + "properties": {"privateDnsZoneConfigs": [{"properties": {"privateDnsZoneId": core["blob_zone"]}}]} + } + records[f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments"] = { + "value": [ + { + "properties": { + "principalId": PRINCIPAL, + "scope": f"/subscriptions/{SUB}", + "roleDefinitionId": f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleDefinitions/{cc.CONTRIBUTOR}", + } + } + ] + } + return settings, core, ids, records + + +class ConfigurationTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.settings, self.core, self.ids, self.records = fixture() + + async def request(method, identifier, version, **kwargs): + self.assertEqual(method, "GET") + if kwargs.get("missing_ok"): + return self.records.get(identifier) + return self.records[identifier] + + self.arm = SimpleNamespace(request=AsyncMock(side_effect=request)) + + async def test_private_resources_and_subscription_role_match_the_bundle(self): + self.assertEqual(await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE), PRINCIPAL) + self.assertEqual(self.ids["storage"].rsplit("/", 1)[1], "stgcctretest2222") + + async def test_public_ip_wrong_subnet_dns_target_or_role_is_rejected(self): + for defect in ("public_ip", "subnet", "dns", "role", "tags", "endpoint"): + with self.subTest(defect=defect): + _, _, _, self.records = fixture() + ip = self.records[self.ids["nic"]]["properties"]["ipConfigurations"][0]["properties"] + if defect == "public_ip": + ip["publicIPAddress"] = {"id": "public"} + if defect == "subnet": + ip["subnet"]["id"] = "other" + if defect == "dns": + self.records[self.ids["zone"] + "/A/@"]["properties"]["aRecords"] = [] + if defect == "tags": + self.records[self.ids["vm"]]["tags"]["tre_shared_service_id"] = "other" + if defect == "endpoint": + self.records[self.ids["endpoint"]]["properties"]["privateLinkServiceConnections"][0]["properties"][ + "privateLinkServiceId" + ] = "other" + if defect == "role": + self.records[f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments"][ + "value" + ] = [] + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE) + + async def test_network_card_allows_only_the_core_network_by_default(self): + rules = self.records[self.ids["nsg"]]["properties"]["securityRules"] + for defect in ("association", "workspace", "cluster", "core", "deny", "priority", "outbound_only"): + with self.subTest(defect=defect): + rules[:] = default_rules() + self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] = {"id": self.ids["nsg"]} + if defect == "association": + del self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] + if defect == "workspace": + rules.append(nsg_rule("AllowWorkspace", 130, "Allow", "Tcp", ["443"], ["10.2.0.0/24"])) + if defect == "cluster": + rules.append(cluster_rule([CLUSTER])) + if defect == "core": + rules[0]["properties"]["sourceAddressPrefixes"] = ["10.0.0.0/8"] + if defect == "deny": + del rules[2] + if defect == "priority": + rules[2]["properties"]["priority"] = 105 + if defect == "outbound_only": + rules[2]["properties"]["direction"] = "Outbound" + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE) + rules[:] = default_rules() + self.records[self.ids["nic"]]["properties"]["networkSecurityGroup"] = {"id": self.ids["nsg"]} + self.assertEqual(await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE), PRINCIPAL) + + async def test_cluster_rule_allows_exactly_the_requested_ranges(self): + rules = self.records[self.ids["nsg"]]["properties"]["securityRules"] + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]) + rules.append(cluster_rule([CLUSTER])) + self.assertEqual( + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]), PRINCIPAL + ) + rules[-1] = cluster_rule([CLUSTER, "10.2.0.0/24"]) + with self.assertRaises(AssertionError): + await cc.validate_resources(self.arm, self.settings, self.core, self.ids, SERVICE, [CLUSTER]) + + async def test_cleanup_refuses_foreign_tags_before_any_write(self): + self.records[self.ids["storage"]]["tags"]["tre_shared_service_id"] = "other" + with self.assertRaisesRegex(AssertionError, "ownership"): + await cc.guard_owned(self.arm, self.ids, {"tre_id": self.settings.tre_id, "tre_shared_service_id": SERVICE}) + + async def test_cleanup_refuses_a_foreign_network_security_group(self): + self.records[self.ids["nsg"]]["tags"]["tre_shared_service_id"] = "other" + with self.assertRaisesRegex(AssertionError, "ownership"): + await cc.guard_owned(self.arm, self.ids, {"tre_id": self.settings.tre_id, "tre_shared_service_id": SERVICE}) + + async def test_disk_cleanup_uses_the_disk_api_and_role_removal_is_checked(self): + self.records = {} + role_endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" + self.records[role_endpoint] = {"value": [{"id": "role"}]} + + async def replicate_removal(seconds): + self.records[role_endpoint] = {"value": []} + + with patch.object(cc.asyncio, "sleep", AsyncMock(side_effect=replicate_removal)) as sleep: + await cc.assert_removed(self.arm, self.ids, SUB, PRINCIPAL) + sleep.assert_awaited_once() + self.arm.request.assert_any_await("GET", self.ids["disk"], cc.DISK_API, missing_ok=True) + self.arm.request.assert_any_await("GET", self.ids["nsg"], cc.NETWORK_API, missing_ok=True) + + async def test_role_pagination_is_complete_and_cannot_change_authority(self): + endpoint = f"/subscriptions/{SUB}/providers/Microsoft.Authorization/roleAssignments" + self.arm.request.side_effect = [ + { + "value": [1], + "nextLink": f"https://management.azure.com{endpoint}?api-version=2022-04-01&$skiptoken=second", + }, + {"value": [2]}, + ] + self.assertEqual(await cc.principal_roles(self.arm, SUB, PRINCIPAL), [1, 2]) + self.arm.request.side_effect = [{"value": [], "nextLink": f"https://other.example{endpoint}"}] + with self.assertRaisesRegex(ValueError, "continuation"): + await cc.principal_roles(self.arm, SUB, PRINCIPAL) + + async def test_terms_and_image_failures_happen_without_writes(self): + self.records[self.settings.group] = { + "id": self.settings.group, + "tags": {"tre_id": self.settings.tre_id}, + "location": self.core["location"], + } + self.records[self.core["vnet"]] = { + "id": self.core["vnet"], + "properties": {"addressSpace": {"addressPrefixes": ["10.0.0.0/22"]}}, + } + self.records[self.core["subnet"]] = {"id": self.core["subnet"], "properties": {"addressPrefix": "10.0.1.0/24"}} + self.records[self.settings.group + "/providers/Microsoft.Network/publicIPAddresses/pip-agw-tre-test"] = { + "properties": {"dnsSettings": {"fqdn": "tre-test.switzerlandnorth.cloudapp.azure.com"}} + } + self.records[self.core["blob_zone"]] = {"id": self.core["blob_zone"]} + del self.records[self.core["zone"]] + agreement = f"/subscriptions/{SUB}/providers/Microsoft.MarketplaceOrdering/offerTypes/virtualmachine/publishers/azurecyclecloud/offers/azure-cyclecloud/plans/cyclecloud8/agreements/current" + images = f"/subscriptions/{SUB}/providers/Microsoft.Compute/locations/switzerlandnorth/publishers/azurecyclecloud/artifacttypes/vmimage/offers/azure-cyclecloud/skus/cyclecloud8/versions" + self.records[agreement] = {"properties": {"accepted": False}} + with self.assertRaisesRegex(ValueError, "terms"): + await cc.preflight(self.arm, self.settings) + self.records[agreement]["properties"]["accepted"] = True + self.records[images] = [] + with self.assertRaisesRegex(ValueError, "image is unavailable"): + await cc.preflight(self.arm, self.settings) + self.records[images] = [{"name": "8.8.0"}] + self.assertEqual(await cc.preflight(self.arm, self.settings), self.core) + + +class FirewallTests(unittest.TestCase): + def test_only_owned_rules_are_added_and_unrelated_rules_are_preserved(self): + baseline = {"rule_collections": {"keep": {"name": "keep"}}, "network_rule_collections": {}} + schema = json.loads((ROOT / "templates/shared_services/cyclecloud/template_schema.json").read_text()) + properties = {key: list(value.values()) for key, value in deepcopy(baseline).items()} + for prop in schema["pipeline"]["install"][1]["properties"]: + value = prop["value"] + value["name"] = value["name"].replace("{{ resource.id }}", SERVICE) + value["rules"][0]["source_addresses"] = ["10.0.1.0/24"] + properties[prop["name"]].append(value) + cc.validate_firewall({"properties": properties}, baseline, SERVICE, ["10.0.1.0/24"]) + properties["rule_collections"][0]["changed"] = True + with self.assertRaisesRegex(AssertionError, "Unrelated"): + cc.validate_firewall({"properties": properties}, baseline, SERVICE, ["10.0.1.0/24"]) + + def test_other_services_can_change_their_own_rules(self): + other = "55555555-5555-4555-8555-555555555555" + first = {"rule_collections": [{"name": "keep"}, {"name": f"arc_svc_{other}_health_services"}]} + baseline = cc.firewall_rules({"properties": first}) + self.assertEqual(baseline, {"rule_collections": {"keep": {"name": "keep"}}, "network_rule_collections": {}}) + later = {"rule_collections": [{"name": "keep"}], "network_rule_collections": [{"name": f"nrc_svc_{other}"}]} + self.assertEqual(cc.firewall_rules({"properties": later}, SERVICE), baseline) + later["rule_collections"].append({"name": f"arc_svc_{SERVICE}_cyclecloud"}) + self.assertNotEqual(cc.firewall_rules({"properties": later}, SERVICE), baseline) + + def test_deadline_precedes_container_and_smoke_is_a_dependency(self): + jobs = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text())["jobs"] + job = jobs["e2e_tests_bundle"] + self.assertIn("e2e_tests_smoke", job["needs"]) + reserve = next(s for s in job["steps"] if "CYCLECLOUD_VALIDATION_DEADLINE" in s.get("run", "")) + run = next(s for s in job["steps"] if s["name"] == "Run E2E Tests") + self.assertLess(job["steps"].index(reserve), job["steps"].index(run)) + self.assertEqual(reserve["if"], "inputs.e2eBundle == 'tre-shared-service-cyclecloud'") + self.assertIn( + "CYCLECLOUD_VALIDATION_DEADLINE=${{ env.CYCLECLOUD_VALIDATION_DEADLINE }}", run["with"]["COMMAND"] + ) + self.assertGreater(job["timeout-minutes"], 270) diff --git a/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py new file mode 100644 index 0000000000..41c8292ea6 --- /dev/null +++ b/e2e_tests/unit_tests/test_cyclecloud_lifecycle.py @@ -0,0 +1,346 @@ +"""Exercise accepted ownership, pipeline waits, deadlines and cleanup failures.""" + +import asyncio +from builtins import ExceptionGroup +from copy import deepcopy +import os +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests.resources import cyclecloud_lifecycle as lc +from e2e_tests import test_cyclecloud as case + +SERVICE = "22222222-2222-4222-8222-222222222222" +OPERATION = "33333333-3333-4333-8333-333333333333" +OLD_OPERATION = "44444444-4444-4444-8444-444444444444" +PATH = f"/shared-services/{SERVICE}" +PAYLOAD = {"templateName": lc.TEMPLATE, "properties": {"display_name": "unique test", "description": "owned test"}} + + +def response(body=None, code=200): + return SimpleNamespace(status_code=code, json=lambda: body) + + +def accepted(**changes): + return response({"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH, **changes}}, 202) + + +def resource(state="deployed", **properties): + return { + "id": SERVICE, + "templateName": lc.TEMPLATE, + "deploymentStatus": state, + "_etag": "etag", + "properties": PAYLOAD["properties"] | properties, + } + + +def operations(state="deployed", identifier=OPERATION): + return response({"operations": [{"id": identifier, "status": state}]}) + + +class LifecycleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.before, self.after = AsyncMock(), AsyncMock() + self.service = lc.CycleCloudService( + True, asyncio.get_running_loop().time() + 2, deepcopy(PAYLOAD), self.before, self.after + ) + self.client = AsyncMock() + self.client.__aenter__.return_value = self.client + self.client.post.return_value = accepted() + self.enterContext(patch.object(lc, "AsyncClient", return_value=self.client)) + self.enterContext(patch.object(lc, "get_admin_token", AsyncMock(return_value="token"))) + self.evidence = self.enterContext(patch.object(lc, "record_resource")) + self.enterContext(patch.object(lc, "record_operation")) + self.enterContext(patch.object(lc, "get_full_endpoint", side_effect=lambda v: "https://tre" + v)) + self.remove = self.enterContext(patch.object(lc, "disable_and_delete_resource", AsyncMock())) + + def own(self): + self.service.path, self.service.identifier = PATH, SERVICE + + async def test_accepted_resource_is_owned_before_client_exit_cancellation(self): + self.client.__aexit__.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await self.service.create() + self.assertEqual((self.service.path, self.service.identifier), (PATH, SERVICE)) + self.evidence.assert_called_once() + self.client.__aexit__.side_effect = None + self.client.get.return_value = response(code=404) + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=(resource(), []))): + self.assertEqual(await self.service.close(), []) + self.remove.assert_awaited_once() + self.before.assert_awaited_once_with(SERVICE) + self.after.assert_awaited_once_with(SERVICE) + + async def test_evidence_failure_cannot_lose_accepted_identity(self): + self.evidence.side_effect = OSError("evidence disk full") + with self.assertRaises(OSError): + await self.service.create() + self.assertEqual(self.service.path, PATH) + + async def test_invalid_accepted_identity_is_not_owned(self): + for changes in ({"resourcePath": "/other"}, {"resourceId": "not-uuid"}): + self.client.post.return_value = accepted(**changes) + with self.assertRaises(ValueError): + await self.service.create() + self.assertIsNone(self.service.path) + self.evidence.assert_not_called() + + async def test_failed_firewall_pipeline_remains_owned(self): + with patch.object( + self.service, "wait_terminal", AsyncMock(return_value=(resource(), [{"status": "pipeline_failed"}])) + ): + with self.assertRaisesRegex(RuntimeError, "pipeline"): + await self.service.create() + self.assertEqual(self.service.path, PATH) + + async def test_terminal_wait_requires_current_operation_and_pipeline_completion(self): + self.own() + self.client.get.side_effect = [ + response({"sharedService": resource()}), + operations(identifier=OLD_OPERATION), + response({"sharedService": resource()}), + operations("updating"), + response({"sharedService": resource("updated")}), + operations("pipeline_succeeded"), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + result, _ = await self.service.wait_terminal(operation_id=OPERATION) + self.assertEqual(result["deploymentStatus"], "updated") + self.assertEqual(sleep.await_count, 2) + + async def test_wait_rejects_changed_ownership(self): + self.own() + self.client.get.return_value = response({"sharedService": resource(display_name="another test")}) + with self.assertRaisesRegex(ValueError, "ownership"): + await self.service.wait_terminal() + self.remove.assert_not_awaited() + + async def test_action_uses_etag_and_waits_for_returned_operation(self): + self.own() + with patch.object( + self.service, + "wait_terminal", + AsyncMock( + side_effect=[ + (resource(), []), + (resource("action_succeeded"), [{"id": OPERATION, "status": "action_succeeded"}]), + ] + ), + ) as wait: + await self.service.change(action="stop") + self.client.post.assert_awaited_once_with( + "https://tre/api" + PATH + "/invoke-action", + headers={"Authorization": "Bearer token", "etag": "etag"}, + params={"action": "stop"}, + ) + self.assertEqual(wait.await_args.kwargs, {"operation_id": OPERATION}) + + async def test_metadata_update_must_be_preserved(self): + self.own() + self.client.patch.return_value = accepted() + with patch.object( + self.service, + "wait_terminal", + AsyncMock(side_effect=[(resource(), []), (resource("updated"), [{"id": OPERATION, "status": "updated"}])]), + ): + with self.assertRaisesRegex(AssertionError, "not preserved"): + await self.service.change(properties={"overview": "new text"}) + self.assertEqual(self.client.patch.await_args.kwargs["json"], {"properties": {"overview": "new text"}}) + + async def test_cluster_address_spaces_update_is_sent_and_preserved(self): + self.own() + self.client.patch.return_value = accepted() + update = {"overview": "new text", "cluster_address_spaces": ["172.31.254.0/24"]} + with patch.object( + self.service, + "wait_terminal", + AsyncMock( + side_effect=[ + (resource(), []), + (resource("updated", **update), [{"id": OPERATION, "status": "updated"}]), + ] + ), + ): + result = await self.service.change(properties=update) + self.assertEqual(result["properties"]["cluster_address_spaces"], ["172.31.254.0/24"]) + self.assertEqual(self.client.patch.await_args.kwargs["json"], {"properties": update}) + + async def test_change_needs_one_action_or_a_property_update(self): + self.own() + for arguments in ({}, {"properties": {}}, {"action": "restart"}, {"action": "stop", "properties": {"a": 1}}): + with self.subTest(arguments=arguments), self.assertRaises(ValueError): + await self.service.change(**arguments) + self.client.patch.assert_not_awaited() + self.client.post.assert_not_awaited() + + async def test_cleanup_waits_then_guards_then_deletes_and_verifies(self): + self.own() + order = [] + + async def wait(**kwargs): + order.append("wait") + self.assertTrue(kwargs["allow_absent"]) + return resource("deployment_failed"), [] + + async def before(identifier): + order.append("guard") + + async def remove(*args, **kwargs): + order.append("delete") + + async def after(identifier): + order.append("verify") + + self.before.side_effect, self.remove.side_effect, self.after.side_effect = before, remove, after + self.client.get.return_value = response(code=404) + with patch.object(self.service, "wait_terminal", AsyncMock(side_effect=wait)): + self.assertEqual(await self.service.close(), []) + self.assertEqual(order, ["wait", "guard", "delete", "verify"]) + self.assertTrue(self.remove.await_args.kwargs["allow_failed_disable"]) + + async def test_cleanup_guard_failure_prevents_delete(self): + self.own() + self.before.side_effect = ValueError("foreign tag") + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=(resource(), []))): + failures = await self.service.close() + self.assertIsInstance(failures[0], ValueError) + self.remove.assert_not_awaited() + self.after.assert_not_awaited() + + async def test_cleanup_after_api_absence_still_checks_azure(self): + self.own() + with patch.object(self.service, "wait_terminal", AsyncMock(return_value=None)): + self.assertEqual(await self.service.close(), []) + self.remove.assert_not_awaited() + self.after.assert_awaited_once_with(SERVICE) + + async def test_cleanup_deadline_bounds_stalled_pipeline_without_delete(self): + self.own() + self.service.finish = asyncio.get_running_loop().time() + 0.02 + + async def wait(**kwargs): + await asyncio.sleep(10) + + with patch.object(self.service, "wait_terminal", side_effect=wait): + failures = await self.service.close() + self.assertIsInstance(failures[0], TimeoutError) + self.remove.assert_not_awaited() + + +class DeadlineTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE="")) + + async def test_absolute_deadline_counts_container_setup_and_caps_extension(self): + for deadline, seconds in (("10000", 9000), ("1000000", lc.LIFECYCLE_SECONDS)): + with ( + patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE=deadline), + patch.object(lc.time, "time", return_value=1000), + ): + work, finish = lc.lifecycle_deadlines() + self.assertAlmostEqual(finish - asyncio.get_running_loop().time(), seconds, delta=1) + self.assertEqual(finish - work, lc.CLEANUP_RESERVE_SECONDS) + + async def test_invalid_or_exhausted_deadline_prevents_provisioning(self): + for value in ("nan", "inf", "-inf", "invalid", "0"): + with self.subTest(value=value), patch.dict(os.environ, CYCLECLOUD_VALIDATION_DEADLINE=value): + with self.assertRaises((ValueError, TimeoutError)): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + self.fail("Invalid deadline reached provisioning") + + async def test_work_timeout_still_runs_cleanup(self): + now = asyncio.get_running_loop().time() + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 0.02, now + 1)), + patch.object(lc.CycleCloudService, "close", AsyncMock(return_value=[])) as close, + ): + with self.assertRaises(TimeoutError): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + await asyncio.sleep(10) + close.assert_awaited_once() + + async def test_cleanup_failure_preserves_original_or_fails_successful_case(self): + cleanup_error = RuntimeError("cleanup failed") + cleanup_error.add_note(PATH) + original = ValueError("validation failed") + with patch.object(lc.CycleCloudService, "close", AsyncMock(return_value=[cleanup_error])): + with self.assertRaises(ValueError) as raised: + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + raise original + self.assertIs(raised.exception, original) + self.assertIn(PATH, str(original.__notes__)) + self.assertIn("cleanup failed", str(original.__notes__)) + with self.assertRaises(ExceptionGroup): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + pass + + async def test_repeated_cancellation_waits_for_cleanup(self): + entered, cleaning, release = (asyncio.Event() for _ in range(3)) + + async def close(): + cleaning.set() + await release.wait() + return [] + + async def run(): + async with lc.cyclecloud_lifecycle(True, PAYLOAD, AsyncMock(), AsyncMock()): + entered.set() + await asyncio.sleep(10) + + with patch.object(lc.CycleCloudService, "close", side_effect=close): + task = asyncio.create_task(run()) + await entered.wait() + task.cancel() + await cleaning.wait() + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await task + + +class PrerequisiteTests(unittest.IsolatedAsyncioTestCase): + async def test_wrong_version_missing_actions_existing_service_or_disabled_firewall_prevents_creation(self): + catalog = case.load_catalog() + template = { + "name": lc.TEMPLATE, + "version": catalog[lc.TEMPLATE]["source_version"], + "customActions": [{"name": "start"}, {"name": "stop"}], + } + firewall_template = {"name": case.FIREWALL, "version": catalog[case.FIREWALL]["source_version"]} + firewall = { + "id": SERVICE, + "templateName": case.FIREWALL, + "templateVersion": firewall_template["version"], + "isEnabled": True, + "deploymentStatus": "deployed", + } + for defect in ("none", "version", "actions", "existing", "disabled"): + with self.subTest(defect=defect): + first, deployed = deepcopy(template), deepcopy(firewall) + services = [deployed] + if defect == "version": + first["version"] = "0.0.0" + if defect == "actions": + first["customActions"] = [] + if defect == "existing": + services.append({"templateName": lc.TEMPLATE}) + if defect == "disabled": + deployed["isEnabled"] = False + with ( + patch.object(case, "get_admin_token", AsyncMock()), + patch.object(case, "record_reused_resource"), + patch.object( + case, + "get_resource", + AsyncMock(side_effect=[first, firewall_template, {"sharedServices": services}]), + ), + ): + if defect == "none": + self.assertEqual(await case.prerequisites(True), deployed) + else: + with self.assertRaises(ValueError): + await case.prerequisites(True) diff --git a/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py new file mode 100644 index 0000000000..5d24634ac2 --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_bundle_contract.py @@ -0,0 +1,247 @@ +"""Check OHDSI bootstrap inputs, private access and pipeline cleanup contracts.""" + +import json +import os +import re +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/workspace_services/ohdsi" + + +class BootstrapTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.folder = Path(self.tmp.name) + self.env = os.environ | { + "PATH": str(self.folder) + os.pathsep + os.environ["PATH"], + "CALLS": str(self.folder / "calls.jsonl"), + "OHDSI_ADMIN_PASSWORD": "admin'password", + "OHDSI_APP_PASSWORD": "app'password", + "OHDSI_ADMIN_USERNAME": "ohdsi_admin_user", + "OHDSI_APP_USERNAME": "ohdsi_app_user", + "OHDSI_ADMIN_ROLE": "ohdsi_admin", + "OHDSI_APP_ROLE": "ohdsi_app", + "DATABASE_NAME": "atlas_webapi_db", + "SCHEMA_NAME": "webapi", + "MAIN_CONNECTION_STRING": "main", + "OHDSI_ADMIN_CONNECTION_STRING": "admin", + "ATLAS_USERS": "admin,secret", + "WEB_API_URL": "https://webapi.example/WebAPI/", + "WEB_API_VERSION": "2.12.1", + } + self.fake( + "psql", + """import json,os,sys +from pathlib import Path +args=sys.argv[1:] +body=Path(args[args.index('-f')+1]).read_text() if '-f' in args else sys.stdin.read() +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'args':args,'body':body})+'\\n') +# A real psql returns 0 after a failed statement unless ON_ERROR_STOP is on. +sys.exit(int(os.environ.get('PSQL_EXIT','0')) if 'ON_ERROR_STOP=1' in args else 0) +""", + ) + + def fake(self, name, body): + p = self.folder / name + p.write_text("#!" + sys.executable + "\n" + body) + p.chmod(0o755) + + def run_script(self, name, **env): + result = subprocess.run( + ["bash", str(BUNDLE / "scripts" / name)], + cwd=BUNDLE / "terraform", + env=self.env | env, + capture_output=True, + text=True, + timeout=10, + ) + calls = ( + [json.loads(line) for line in Path(self.env["CALLS"]).read_text().splitlines()] + if Path(self.env["CALLS"]).exists() + else [] + ) + return result, calls + + def security_commands(self): + self.fake( + "curl", + """import json,os,sys +from pathlib import Path +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'curl':args})+'\\n') +if args[0].endswith('/info'): + sequence=os.environ.get('INFO_VERSIONS','2.12.1').split(',') + counter=Path(os.environ['CALLS']+'.count') + count=int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(count+1)) + Path(args[args.index('--output')+1]).write_text(json.dumps({'version':sequence[min(count,len(sequence)-1)]})) + sys.exit(int(os.environ.get('CURL_EXIT','0'))) +if '--output' not in args:print('authentication-token') +""", + ) + self.fake( + "jq", + """import json,sys +from pathlib import Path +args=sys.argv[1:] +expected=args[args.index('--arg')+2] +sys.exit(0 if json.loads(Path(args[-1]).read_text()).get('version')==expected else 1) +""", + ) + self.fake("sleep", "import time\ntime.sleep(0.01)\n") + self.fake( + "timeout", + """import json,os,signal,subprocess,sys +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps({'timeout':args[:1]})+'\\n') +# Use a shorter test deadline while exercising the real shell loop. +process=subprocess.Popen(args[1:],start_new_session=True) +try: + sys.exit(process.wait(timeout=2)) +except subprocess.TimeoutExpired: + os.killpg(process.pid,signal.SIGKILL) + process.wait() + sys.exit(124) +""", + ) + self.fake("htpasswd", "print(':test-password-hash')\n") + + def test_security_waits_without_repeated_logins_or_token_output(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh") + self.assertEqual(result.returncode, 0, result.stderr) + requests = [c["curl"] for c in calls if "curl" in c] + self.assertEqual(len(requests), 2) + self.assertEqual(requests[0][0], "https://webapi.example/WebAPI/info") + self.assertEqual([c["timeout"] for c in calls if "timeout" in c], [["600"]]) + self.assertIn("--max-time", requests[0]) + self.assertFalse(Path(requests[0][requests[0].index("--output") + 1]).exists()) + self.assertNotIn("--retry", requests[1]) + self.assertEqual(requests[1][requests[1].index("--output") + 1], "/dev/null") + self.assertNotIn("authentication-token", result.stdout + result.stderr) + sql = [c for c in calls if "args" in c] + self.assertTrue(all("-e" not in c["args"] for c in sql)) + self.assertIn(":'password_hash'", sql[2]["body"]) + self.assertIn("NOT EXISTS", sql[3]["body"]) + self.assertIn(r"\if :role_assigned", sql[3]["body"]) + self.assertIn("RAISE EXCEPTION", sql[3]["body"]) + + def test_readiness_failure_prevents_database_changes_and_login(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", CURL_EXIT="22") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.returncode, 124) + self.assertTrue(all("args" not in c for c in calls)) + self.assertTrue(all(c["curl"][0].endswith("/info") for c in calls if "curl" in c)) + + def test_unexpected_version_prevents_database_changes_and_login(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", INFO_VERSIONS="2.11.0") + self.assertEqual(result.returncode, 124) + self.assertTrue(all("args" not in c for c in calls)) + self.assertTrue(all(c["curl"][0].endswith("/info") for c in calls if "curl" in c)) + + def test_stale_http_success_is_retried_before_security_initialisation(self): + self.security_commands() + result, calls = self.run_script("atlas_security.sh", INFO_VERSIONS="2.11.0,2.12.1") + self.assertEqual(result.returncode, 0, result.stderr) + requests = [c["curl"] for c in calls if "curl" in c] + self.assertEqual([r[0].rsplit("/", 1)[1] for r in requests], ["info", "info", "db"]) + first_sql = next(i for i, c in enumerate(calls) if "args" in c) + self.assertEqual(sum("curl" in c for c in calls[:first_sql]), 2) + self.assertNotIn("--retry", requests[-1]) + + def test_passwords_use_psql_quoted_variables_without_sql_echo(self): + result, calls = self.run_script("atlas_db_init.sh") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(calls), 2) + self.assertIn("admin_password=admin'password", calls[0]["args"]) + self.assertIn("app_password=app'password", calls[0]["args"]) + self.assertIn(":'admin_password'", calls[0]["body"]) + self.assertIn(":'app_password'", calls[0]["body"]) + for call in calls: + self.assertIn("ON_ERROR_STOP=1", call["args"]) + self.assertNotIn("-e", call["args"]) + self.assertNotIn("password", result.stdout + result.stderr) + + def test_role_creation_failure_stops_before_schema_creation(self): + result, calls = self.run_script("atlas_db_init.sh", PSQL_EXIT="7") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(calls), 1) + + def test_roles_and_schema_are_safe_to_repeat(self): + roles = (BUNDLE / "sql/atlas_create_roles_users.sql").read_text() + self.assertEqual(roles.count("WHERE NOT EXISTS"), 4) + self.assertEqual(roles.count("\\gexec"), 4) + self.assertIn("CREATE SCHEMA IF NOT EXISTS", (BUNDLE / "sql/atlas_create_schema.sql").read_text()) + + +class BundleContractTests(unittest.TestCase): + def test_database_and_apps_disable_public_access(self): + for filename in ("atlas_database.tf", "atlas_ui.tf", "ohdsi_web_api.tf"): + with self.subTest(filename=filename): + source = (BUNDLE / "terraform" / filename).read_text() + self.assertRegex(source, r"public_network_access_enabled\s*=\s*false") + + def test_uninstall_removes_only_owned_firewall_collections(self): + schema = json.loads((BUNDLE / "template_schema.json").read_text()) + uninstall = schema["pipeline"]["uninstall"] + cleanup = [s for s in uninstall if s.get("resourceTemplateName") == "tre-shared-service-firewall"] + self.assertEqual(len(cleanup), 1) + self.assertEqual(cleanup[0]["resourceAction"], "upgrade") + self.assertEqual( + {p["name"]: p["value"] for p in cleanup[0]["properties"]}, + { + "network_rule_collections": {"name": "nrc_svc_{{ resource.id }}"}, + "rule_collections": {"name": "arc_svc_{{ resource.id }}"}, + }, + ) + for prop in cleanup[0]["properties"]: + self.assertEqual(prop["arraySubstitutionAction"], "remove") + self.assertEqual(prop["arrayMatchField"], "name") + self.assertLess(uninstall.index(cleanup[0]), next(i for i, s in enumerate(uninstall) if s["stepId"] == "main")) + + def test_atlas_config_upload_uses_the_storage_account_key(self): + providers = (BUNDLE / "terraform/providers.tf").read_text() + alias = providers.split('alias = "shared_key"', 1)[1].split("\n}", 1)[0] + self.assertRegex(alias, r"storage_use_azuread\s*=\s*false") + source = (BUNDLE / "terraform/atlas_ui.tf").read_text() + upload = source.split('resource "azurerm_storage_share_file" "config_local" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(upload, r"provider\s*=\s*azurerm\.shared_key") + + def test_web_apps_join_the_subnet_after_the_postgres_subnet_writes(self): + terraform = BUNDLE / "terraform" + + def depends_on(path, address): + block = (terraform / path).read_text().split(f'resource "azurerm_linux_web_app" "{address}" {{', 1)[1] + return block.split("depends_on = [", 1)[1].split("]", 1)[0] + + self.assertIn("azurerm_postgresql_flexible_server.postgres", depends_on("atlas_ui.tf", "atlas_ui")) + self.assertIn("azurerm_linux_web_app.atlas_ui", depends_on("ohdsi_web_api.tf", "ohdsi_webapi")) + + def test_web_api_identity_reads_only_its_key_vault_references(self): + # The workspace Key Vault also has the secrets of the other workspace services. + source = (BUNDLE / "terraform/ohdsi_web_api.tf").read_text() + self.assertNotRegex(source, r"\n scope\s*=\s*data\.azurerm_key_vault\.ws\.id\n") + roles = source.split('resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" {', 1)[1] + roles = roles.split("\n}", 1)[0] + scoped = set(re.findall(r"=\s*((?:data\.)?azurerm_key_vault_secret\.\w+)\.resource_versionless_id", roles)) + referenced = set(re.findall(r"SecretName=\$\{((?:data\.)?azurerm_key_vault_secret\.\w+)\.name\}", source)) + self.assertTrue(referenced) + self.assertEqual(scoped, referenced) + + def test_postgres_subnet_declares_the_endpoint_that_the_service_adds(self): + source = (BUNDLE / "terraform/atlas_database.tf").read_text() + subnet = source.split('resource "azurerm_subnet" "postgres" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(subnet, r'service_endpoints\s*=\s*\["Microsoft\.Storage"\]') + + def test_bootstrap_version_matches_pinned_image(self): + source = (BUNDLE / "terraform/atlas_security.tf").read_text() + self.assertRegex(source, r"WEB_API_VERSION\s*=\s*local.ohdsi_api_docker_image_tag") diff --git a/e2e_tests/unit_tests/test_ohdsi_configuration.py b/e2e_tests/unit_tests/test_ohdsi_configuration.py new file mode 100644 index 0000000000..0d46508cae --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_configuration.py @@ -0,0 +1,440 @@ +"""Check OHDSI private configuration, prerequisite rejection and workflow wiring.""" + +from copy import deepcopy +from contextlib import asynccontextmanager +import json +from pathlib import Path +import re +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +import yaml + +from e2e_tests import test_ohdsi as case +from e2e_tests.resources import ohdsi as h + +ROOT = Path(__file__).resolve().parents[2] +SUB = "11111111-1111-4111-8111-111111111111" +TENANT = "22222222-2222-4222-8222-222222222222" +WS = "33333333-3333-4333-8333-333333333333" +SERVICE = "44444444-4444-4444-8444-444444444444" +PRINCIPAL = "55555555-5555-4555-8555-555555555555" +SETTINGS = h.Settings(SUB, TENANT, "testtre") +IDS = h.resource_ids(SETTINGS, WS, SERVICE) +TAGS = {"tre_id": "testtre", "tre_workspace_id": WS, "tre_workspace_service_id": SERVICE} +CORE_LINK = IDS["pg_zone"] + "/virtualNetworkLinks/core" +REFERENCES = IDS["apps"]["webapi"] + "/config/configreferences/appsettings" + + +def core_link(): + return { + "id": CORE_LINK, + "properties": { + "virtualNetwork": { + "id": f"{SETTINGS.group}/providers/Microsoft.Network/virtualNetworks/vnet-{SETTINGS.tre_id}" + }, + "registrationEnabled": False, + "provisioningState": "Succeeded", + }, + } + + +def arm_records(): + def resource(identifier, properties): + return {"id": identifier, "tags": TAGS.copy(), "properties": properties} + + records = { + IDS["postgres"]: resource( + IDS["postgres"], + { + "state": "Ready", + "version": "14", + "network": { + "publicNetworkAccess": "Disabled", + "delegatedSubnetResourceId": IDS["postgres_subnet"], + "privateDnsZoneArmResourceId": IDS["pg_zone"], + }, + }, + ), + IDS["postgres_subnet"]: resource( + IDS["postgres_subnet"], + { + "networkSecurityGroup": {"id": IDS["nsg"]}, + "defaultOutboundAccess": False, + "delegations": [{"properties": {"serviceName": "Microsoft.DBforPostgreSQL/flexibleServers"}}], + }, + ), + IDS["identity"]: resource(IDS["identity"], {"tenantId": TENANT, "principalId": PRINCIPAL}), + IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments": { + "value": [ + { + "properties": { + "principalId": PRINCIPAL, + "scope": secret, + "roleDefinitionId": "/roles/" + h.SECRETS_USER, + } + } + for secret in IDS["secrets"] + ] + }, + } + for key, identifier in IDS["apps"].items(): + records[identifier] = resource( + identifier, + { + "state": "Running", + "publicNetworkAccess": "Disabled", + "httpsOnly": True, + "virtualNetworkSubnetId": IDS["vnet"] + "/subnets/WebAppsSubnet", + "keyVaultReferenceIdentity": IDS["identity"], + }, + ) + records[identifier]["identity"] = {"type": "UserAssigned", "userAssignedIdentities": {IDS["identity"]: {}}} + records[identifier + "/config/web"] = { + "properties": { + "ftpsState": "Disabled", + "minTlsVersion": "1.3", + "linuxFxVersion": f"DOCKER|/index.docker.io/ohdsi/{key}:2.12.1", + } + } + endpoint = IDS["endpoints"][key] + records[endpoint] = resource( + endpoint, + { + "provisioningState": "Succeeded", + "subnet": {"id": IDS["vnet"] + "/subnets/ServicesSubnet"}, + "privateLinkServiceConnections": [ + { + "properties": { + "privateLinkServiceId": identifier, + "groupIds": ["sites"], + "privateLinkServiceConnectionState": {"status": "Approved"}, + } + } + ], + }, + ) + zone_group = endpoint + "/privateDnsZoneGroups/privatelink.azurewebsites.net" + records[zone_group] = resource( + zone_group, {"privateDnsZoneConfigs": [{"properties": {"privateDnsZoneId": IDS["web_zone"]}}]} + ) + records[REFERENCES] = { + "value": [ + {"properties": {"secretName": secret.rsplit("/", 1)[1], "status": "Resolved"}} for secret in IDS["secrets"] + ] + } + return records + + +class ConfigurationTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.records = arm_records() + self.arm = AsyncMock() + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if kwargs.get("missing_ok"): + return self.records.get(identifier) + return self.records[identifier] + + self.arm.request.side_effect = request + + async def test_private_configuration_reads_no_secrets_or_app_settings(self): + self.assertEqual(await h.validate_resources(self.arm, SETTINGS, IDS, TAGS), PRINCIPAL) + for call in self.arm.request.await_args_list: + self.assertNotIn("/secrets/", call.args[1]) + # Azure returns the status of a Key Vault reference, not the value of the app setting. + self.assertNotIn("/appsettings", call.args[1].removesuffix("/config/configreferences/appsettings")) + self.assertIn(REFERENCES, [call.args[1] for call in self.arm.request.await_args_list]) + + async def test_key_vault_references_wait_for_the_web_app_start(self): + resolved = self.records[REFERENCES] + self.records[REFERENCES] = deepcopy(resolved) + self.records[REFERENCES]["value"][0]["properties"]["status"] = "Initialized" + with patch.object( + h.asyncio, "sleep", AsyncMock(side_effect=lambda _: self.records.update({REFERENCES: resolved})) + ): + self.assertEqual(await h.validate_resources(self.arm, SETTINGS, IDS, TAGS), PRINCIPAL) + + async def test_unresolved_or_missing_key_vault_reference_is_rejected(self): + self.records[REFERENCES]["value"][0]["properties"]["status"] = "AccessToKeyVaultDenied" + with patch.object(h.asyncio, "sleep", AsyncMock()) as sleep: + with self.assertRaisesRegex(AssertionError, "AccessToKeyVaultDenied"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.assertEqual(sleep.await_count, 9) + self.records = arm_records() + self.records[REFERENCES]["value"].pop() + with self.assertRaisesRegex(AssertionError, "Unexpected OHDSI Key Vault references"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.records[REFERENCES]["nextLink"] = "next-page" + with self.assertRaisesRegex(AssertionError, "Incomplete OHDSI Key Vault references"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_roles_are_read_for_the_identity_also_below_the_vault(self): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + call = next(c for c in self.arm.request.await_args_list if c.args[1].endswith("/roleAssignments")) + self.assertEqual(call.kwargs["query"], {"$filter": f"principalId eq '{PRINCIPAL}'"}) + self.assertTrue(all(secret.startswith(IDS["vault"] + "/secrets/") for secret in IDS["secrets"])) + + def test_secrets_match_the_secret_roles_of_the_bundle(self): + terraform = ROOT / "templates/workspace_services/ohdsi/terraform" + source = "".join(path.read_text() for path in sorted(terraform.glob("*.tf"))) + roles = source.split('resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" {', 1)[1] + roles = roles.split("\n}", 1)[0] + names = set() + for data, resource in re.findall( + r"=\s*(data\.)?azurerm_key_vault_secret\.(\w+)\.resource_versionless_id", roles + ): + header = f'{"data" if data else "resource"} "azurerm_key_vault_secret" "{resource}" {{' + block = source.split(header, 1)[1].split("\n}", 1)[0] + name = re.search(r'\n name\s*=\s*"([^"]+)"', block).group(1) + names.add(name.replace("${local.short_service_id}", SERVICE[-4:])) + self.assertEqual(len(names), 5) + self.assertEqual({secret.rsplit("/", 1)[1] for secret in IDS["secrets"]}, names) + + async def test_role_cleanup_waits_for_rbac_convergence(self): + self.arm.request.side_effect = [ + {"value": [{"properties": {"principalId": PRINCIPAL}}]}, + {"value": []}, + ] + with patch.object(h.asyncio, "sleep", AsyncMock()) as sleep: + await h.assert_roles_removed(self.arm, IDS["vault"], PRINCIPAL) + self.assertEqual(self.arm.request.await_count, 2) + self.assertEqual(self.arm.request.await_args.kwargs["query"], {"$filter": f"principalId eq '{PRINCIPAL}'"}) + sleep.assert_awaited_once_with(10) + + async def test_public_web_app_fails_configuration(self): + self.records[IDS["apps"]["atlas"]]["properties"]["publicNetworkAccess"] = "Enabled" + with self.assertRaisesRegex(AssertionError, "public access"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_public_postgres_fails_configuration(self): + self.records[IDS["postgres"]]["properties"]["network"]["publicNetworkAccess"] = "Enabled" + with self.assertRaisesRegex(AssertionError, "public access"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_wrong_tenant_or_vault_role_is_rejected(self): + self.records[IDS["identity"]]["properties"]["tenantId"] = SUB + with self.assertRaises(AssertionError): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + for scope in (SETTINGS.group, IDS["vault"]): + with self.subTest(scope=scope): + self.records = arm_records() + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"][0][ + "properties" + ]["scope"] = scope + with self.assertRaisesRegex(AssertionError, "secret role scopes"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_missing_or_extra_secret_role_is_rejected(self): + roles = self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"] + extra = deepcopy(roles[0]) + roles.pop() + with self.assertRaisesRegex(AssertionError, "secret role scopes"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + self.records = arm_records() + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["value"].append(extra) + with self.assertRaisesRegex(AssertionError, "secret role scopes"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_paginated_roles_are_not_treated_as_complete(self): + self.records[IDS["vault"] + "/providers/Microsoft.Authorization/roleAssignments"]["nextLink"] = "next-page" + with self.assertRaisesRegex(AssertionError, "Incomplete"): + await h.validate_resources(self.arm, SETTINGS, IDS, TAGS) + + async def test_wrong_owner_blocks_cleanup_guard(self): + self.records[IDS["postgres"]]["tags"]["tre_workspace_service_id"] = WS + with self.assertRaisesRegex(AssertionError, "ownership"): + await h.guard_owned(self.arm, IDS, TAGS) + + async def test_removal_does_not_treat_api_failure_as_absence(self): + self.arm.request.side_effect = RuntimeError("HTTP 403") + with self.assertRaisesRegex(RuntimeError, "HTTP 403"): + await h.assert_removed(self.arm, [(IDS["postgres"], h.POSTGRES_API)]) + + def test_ids_remain_in_owned_workspace(self): + self.assertTrue(IDS["apps"]["atlas"].endswith("/app-ohdsi-atlas-testtre-ws-3333-svc-4444")) + self.assertTrue(IDS["postgres_subnet"].endswith("/PostgreSQLSubnet4444")) + self.assertTrue(all(identifier.startswith(IDS["group"] + "/") for identifier, _ in IDS["removal"])) + for value in ("", "../other", "00000000-0000-0000-0000-000000000000"): + with self.assertRaises(ValueError): + h.resource_ids(SETTINGS, value, SERVICE) + + def test_firewall_restores_unrelated_collections(self): + schema = json.loads((ROOT / "templates/workspace_services/ohdsi/template_schema.json").read_text()) + props = schema["pipeline"]["install"][-1]["properties"] + baseline = {"network_rule_collections": [{"name": "unrelated", "rules": []}], "rule_collections": []} + actual = {"properties": deepcopy(baseline)} + for prop in props: + collection = deepcopy(prop["value"]) + collection["name"] = collection["name"].replace("{{ resource.id }}", SERVICE) + collection["rules"][0]["source_addresses"] = ["10.0.1.0/24"] + actual["properties"][prop["name"]].append(collection) + h.validate_firewall(actual, baseline, SERVICE, ["10.0.1.0/24"]) + actual["properties"]["network_rule_collections"][0]["rules"] = ["changed"] + with self.assertRaisesRegex(AssertionError, "Unrelated"): + h.validate_firewall(actual, baseline, SERVICE, ["10.0.1.0/24"]) + + def test_firewall_ignores_the_collections_of_other_services(self): + other = "55555555-5555-4555-8555-555555555555" + first = { + "properties": { + "network_rule_collections": [{"name": "core"}, {"name": f"nrc_svc_{other}_health_services"}], + "rule_collections": [], + } + } + baseline = h.firewall_rules(first) + self.assertEqual(baseline, {"network_rule_collections": [{"name": "core"}], "rule_collections": []}) + later = deepcopy(first) + later["properties"]["network_rule_collections"].pop() + later["properties"]["rule_collections"].append({"name": f"arc_svc_{other}_azureml_client"}) + self.assertEqual(h.firewall_rules(later, SERVICE), baseline) + later["properties"]["rule_collections"].append({"name": f"arc_svc_{SERVICE}"}) + self.assertNotEqual(h.firewall_rules(later, SERVICE), baseline) + + def test_deadline_starts_before_container_setup_and_is_forwarded(self): + workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) + jobs = [ + j + for j in workflow["jobs"].values() + if any(s.get("name") == "Reserve OHDSI cleanup and reporting time" for s in j.get("steps", [])) + ] + self.assertEqual(len(jobs), 1) + steps = jobs[0]["steps"] + deadline = next(i for i, s in enumerate(steps) if s.get("name") == "Reserve OHDSI cleanup and reporting time") + run = next(i for i, s in enumerate(steps) if s.get("name") == "Run E2E Tests") + self.assertLess(deadline, run) + self.assertIn("270 * 60", steps[deadline]["run"]) + self.assertIn("inputs.e2eBundle == 'tre-workspace-service-ohdsi'", steps[deadline]["if"]) + self.assertIn("OHDSI_VALIDATION_DEADLINE=${{ env.OHDSI_VALIDATION_DEADLINE }}", steps[run]["with"]["COMMAND"]) + self.assertEqual(jobs[0]["timeout-minutes"], 300) + self.assertIn("e2e_tests_smoke", jobs[0]["needs"]) + + +class PrerequisiteTests(unittest.IsolatedAsyncioTestCase): + async def test_missing_firewall_is_rejected(self): + async def get(endpoint, *args): + if endpoint == "/api/shared-services": + return {"sharedServices": []} + name = endpoint.rsplit("/", 1)[1] + return {"name": name, "version": case.load_catalog()[name]["source_version"]} + + with ( + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "get_resource", side_effect=get), + ): + with self.assertRaisesRegex(ValueError, "exactly one"): + await case.prerequisites(True) + + async def test_stale_template_is_rejected(self): + with ( + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "get_resource", AsyncMock(return_value={"name": h.TEMPLATE, "version": "0.0.0"})), + ): + with self.assertRaisesRegex(ValueError, "checkout version"): + await case.prerequisites(True) + + +class SharedDnsTests(unittest.IsolatedAsyncioTestCase): + async def test_core_link_requires_expected_target_and_settings(self): + arm = AsyncMock() + arm.request.return_value = core_link() + self.assertEqual(await h.postgres_core_dns_link(arm, SETTINGS), core_link()) + for key, value in ( + ("virtualNetwork", {"id": IDS["vnet"]}), + ("registrationEnabled", True), + ("provisioningState", "Failed"), + ): + with self.subTest(key=key): + link = core_link() + link["properties"][key] = value + arm.request.return_value = link + with self.assertRaises(AssertionError): + await h.postgres_core_dns_link(arm, SETTINGS) + + async def test_core_link_does_not_treat_read_failure_as_absence(self): + arm = AsyncMock() + arm.request.side_effect = RuntimeError("HTTP 403") + with self.assertRaisesRegex(RuntimeError, "HTTP 403"): + await h.postgres_core_dns_link(arm, SETTINGS, missing_ok=True) + + async def exercise_cleanup(self, *, existing=True, create_link=False, damage=None): + records = {CORE_LINK: core_link()} if existing else {} + arm = AsyncMock() + arm.__aenter__.return_value = arm + workspace_path = f"/workspaces/{WS}" + workspace = { + "templateVersion": case.load_catalog()[h.BASE]["source_version"], + "properties": {"auth_type": "Automatic", "aad_redirect_uris": []}, + } + firewall = {"id": "firewall", "templateName": h.FIREWALL, "properties": {}} + events = [] + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if identifier == CORE_LINK: + events.append("read link") + value = records.get(identifier) + if value is None and not kwargs.get("missing_ok"): + raise AssertionError("Shared PostgreSQL core DNS link is missing") + return value + if identifier == IDS["group"]: + return {"id": identifier, "tags": {"tre_id": SETTINGS.tre_id, "tre_workspace_id": WS}} + self.assertEqual(identifier, IDS["identity"]) + return None + + async def create(payload, endpoint, *args, **kwargs): + if endpoint == "/api/workspaces": + self.assertEqual(events, ["read link"]) + return workspace_path, WS + # Exercise cleanup after a partial install, before normal inspection. + if create_link: + records[CORE_LINK] = core_link() + await kwargs["before_remove"]("service", SERVICE) + events.append("remove service") + if damage == "delete": + records.pop(CORE_LINK) + elif damage == "retarget": + records[CORE_LINK]["properties"]["virtualNetwork"]["id"] = IDS["vnet"] + await kwargs["after_remove"]("service", SERVICE) + raise RuntimeError("simulated partial install") + + @asynccontextmanager + async def lifecycle(_verify): + yield SimpleNamespace(create=create) + + arm.request.side_effect = request + with ( + patch.object(case, "arm_client", return_value=arm), + patch.object(case, "ohdsi_lifecycle", lifecycle), + patch.object(case, "require_settings", return_value=SETTINGS), + patch.object(case, "prerequisites", AsyncMock(return_value=firewall)), + patch.object(case, "preflight", AsyncMock()), + patch.object(case, "guard_owned", AsyncMock()), + patch.object(case, "assert_removed", AsyncMock()), + patch.object(case, "get_admin_token", AsyncMock(return_value="token")), + patch.object(case, "record_deployed_resource"), + patch.object( + case, "get_resource", AsyncMock(return_value={"workspace": workspace, "sharedService": firewall}) + ), + ): + await case.validate_lifecycle(True) + + async def test_existing_link_survives_cleanup(self): + with self.assertRaisesRegex(RuntimeError, "simulated partial install"): + await self.exercise_cleanup() + + async def test_cleanup_detects_deleted_existing_or_new_link(self): + for existing in (True, False): + with self.subTest(existing=existing): + with self.assertRaisesRegex(AssertionError, "core DNS link is missing"): + await self.exercise_cleanup(existing=existing, create_link=True, damage="delete") + + async def test_cleanup_detects_retargeted_core_link(self): + with self.assertRaisesRegex(AssertionError, "ARM resource ID"): + await self.exercise_cleanup(damage="retarget") + + async def test_failed_install_before_link_creation_allows_absence(self): + with self.assertRaisesRegex(RuntimeError, "simulated partial install"): + await self.exercise_cleanup(existing=False) diff --git a/e2e_tests/unit_tests/test_ohdsi_lifecycle.py b/e2e_tests/unit_tests/test_ohdsi_lifecycle.py new file mode 100644 index 0000000000..a40ebad9b4 --- /dev/null +++ b/e2e_tests/unit_tests/test_ohdsi_lifecycle.py @@ -0,0 +1,234 @@ +"""Exercise OHDSI ownership, pipeline completion and bounded cleanup failures.""" + +import asyncio +from builtins import ExceptionGroup +import os +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from e2e_tests.resources import ohdsi_lifecycle as lc + +WORKSPACE = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +OPERATION = "33333333-3333-4333-8333-333333333333" +OLD = "44444444-4444-4444-8444-444444444444" +ENDPOINT = f"/api/workspaces/{WORKSPACE}/workspace-services" +PATH = ENDPOINT.removeprefix("/api") + "/" + SERVICE +PAYLOAD = { + "templateName": "tre-workspace-service-ohdsi", + "properties": {"display_name": "unique name", "description": "owned resource", "configure_data_source": False}, +} + + +def response(body=None, code=200): + return SimpleNamespace(status_code=code, json=lambda: body) + + +def owner(): + return lc.OwnedResource( + PATH, SERVICE, "workspaceService", PAYLOAD["templateName"], "unique name", "owned resource", WORKSPACE + ) + + +def record(state="deployed", **properties): + return { + "id": SERVICE, + "templateName": PAYLOAD["templateName"], + "properties": PAYLOAD["properties"] | properties, + "deploymentStatus": state, + "_etag": "tag", + } + + +def operations(state="deployed", identifier=OPERATION): + return response({"operations": [{"id": identifier, "status": state}]}) + + +class LifecycleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + now = asyncio.get_running_loop().time() + self.resources = lc.OHDSIResources(True, now + 5, now + 10) + self.client = AsyncMock() + self.client.__aenter__.return_value = self.client + self.client.post.return_value = response( + {"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH}}, 202 + ) + self.enterContext(patch.object(lc, "AsyncClient", return_value=self.client)) + self.token = self.enterContext(patch.object(lc, "get_workspace_owner_token", AsyncMock(return_value="owner"))) + self.admin = self.enterContext(patch.object(lc, "get_admin_token", AsyncMock(return_value="admin"))) + self.evidence = self.enterContext(patch.object(lc, "record_resource")) + self.enterContext(patch.object(lc, "record_operation")) + self.enterContext(patch.object(lc, "get_full_endpoint", side_effect=lambda p: "https://tre" + p)) + self.remove = self.enterContext(patch.object(lc, "disable_and_delete_resource", AsyncMock())) + + async def test_accepted_resource_is_owned_before_client_exit_cancellation(self): + self.client.__aexit__.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].path, PATH) + self.evidence.assert_called_once() + self.token.assert_awaited_once_with(WORKSPACE, True) + self.admin.assert_not_awaited() + + async def test_evidence_error_does_not_lose_accepted_identity(self): + self.evidence.side_effect = OSError("disk full") + with self.assertRaises(OSError): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].identifier, SERVICE) + + async def test_external_source_or_wrong_parent_is_rejected_before_post(self): + for properties in ({"configure_data_source": True}, {"data_source_config": {}}, {"data_source_daimons": {}}): + with self.assertRaises(ValueError): + await self.resources.create( + PAYLOAD | {"properties": PAYLOAD["properties"] | properties}, ENDPOINT, WORKSPACE + ) + with self.assertRaises(ValueError): + await self.resources.create(PAYLOAD, "/api/shared-services", WORKSPACE) + self.client.post.assert_not_awaited() + + async def test_failed_pipeline_remains_owned(self): + with patch.object( + self.resources, "wait_terminal", AsyncMock(return_value=(record(), [{"status": "pipeline_failed"}])) + ): + with self.assertRaisesRegex(RuntimeError, "pipeline"): + await self.resources.create(PAYLOAD, ENDPOINT, WORKSPACE) + self.assertEqual(self.resources.owned[0].path, PATH) + + async def test_wait_requires_current_operation_and_terminal_pipeline(self): + self.client.get.side_effect = [ + response({"workspaceService": record()}), + operations(identifier=OLD), + response({"workspaceService": record()}), + operations("pipeline_running"), + response({"workspaceService": record()}), + operations("pipeline_succeeded"), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + await self.resources.wait_terminal(owner(), operation_id=OPERATION) + self.assertEqual(sleep.await_count, 2) + + async def test_new_resource_without_a_status_keeps_polling(self): + self.client.get.side_effect = [ + response({"workspaceService": record(None)}), + operations("awaiting_deployment"), + response({"workspaceService": record()}), + operations(), + ] + with patch.object(lc.asyncio, "sleep", AsyncMock()) as sleep: + deployed, _ = await self.resources.wait_terminal(owner()) + self.assertEqual(deployed["deploymentStatus"], "deployed") + sleep.assert_awaited_once_with(30) + + async def test_ownership_change_prevents_deletion(self): + self.resources.owned = [owner()] + self.client.get.return_value = response({"workspaceService": record(description="different owner")}) + errors = await self.resources.close() + self.assertEqual(len(errors), 1) + self.remove.assert_not_awaited() + + async def test_reverse_cleanup_reserves_parent_after_child(self): + workspace = lc.OwnedResource( + "/workspaces/" + WORKSPACE, WORKSPACE, "workspace", "tre-workspace-base", "test", "owned", None + ) + self.resources.owned = [workspace, owner()] + with patch.object(self.resources, "remove", AsyncMock()) as remove: + self.assertEqual(await self.resources.close(), []) + self.assertEqual([c.args[0].identifier for c in remove.await_args_list], [SERVICE, WORKSPACE]) + + async def test_child_failure_retains_parent_and_reports_both(self): + workspace = lc.OwnedResource( + "/workspaces/" + WORKSPACE, WORKSPACE, "workspace", "tre-workspace-base", "test", "owned", None + ) + self.resources.owned = [workspace, owner()] + with patch.object(self.resources, "remove", AsyncMock(side_effect=RuntimeError("delete failed"))) as remove: + errors = await self.resources.close() + self.assertEqual(remove.await_count, 1) + self.assertEqual(len(errors), 2) + self.assertIn("retained", str(errors[1])) + + async def test_absent_resource_still_checks_azure_and_pipeline_cleanup(self): + resource = owner() + resource.after_remove = AsyncMock() + with patch.object(self.resources, "wait_terminal", AsyncMock(return_value=None)): + await self.resources.remove(resource) + self.remove.assert_not_awaited() + resource.after_remove.assert_awaited_once_with(PATH, SERVICE) + + async def test_same_version_upgrade_waits_for_returned_operation(self): + self.client.patch.return_value = response( + {"operation": {"id": OPERATION, "resourceId": SERVICE, "resourcePath": PATH}}, 202 + ) + wait = AsyncMock( + side_effect=[ + (record(), []), + (record("updated", overview="new"), [{"id": OPERATION, "status": "pipeline_succeeded"}]), + ] + ) + with patch.object(self.resources, "wait_terminal", wait): + await self.resources.upgrade(owner(), "new") + self.assertEqual(wait.await_args.kwargs, {"operation_id": OPERATION}) + self.assertEqual(self.client.patch.call_args.kwargs["headers"]["etag"], "tag") + self.assertEqual(self.client.patch.call_args.kwargs["json"], {"properties": {"overview": "new"}}) + + async def test_deadline_rejects_nonfinite_or_exhausted_budget(self): + for value in ("nan", "inf", "0"): + with patch.dict(os.environ, OHDSI_VALIDATION_DEADLINE=value): + with self.assertRaises((ValueError, TimeoutError)): + lc.lifecycle_deadlines() + + async def test_cleanup_failure_fails_an_otherwise_successful_case(self): + now = asyncio.get_running_loop().time() + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 2, now + 3)), + patch.object(lc.OHDSIResources, "close", AsyncMock(return_value=[RuntimeError("cleanup failed")])), + ): + with self.assertRaises(ExceptionGroup): + async with lc.ohdsi_lifecycle(True): + pass + + async def test_cleanup_failure_note_keeps_the_cleanup_error_message(self): + now = asyncio.get_running_loop().time() + cleanup_error = RuntimeError("delete returned HTTP 409") + cleanup_error.add_note("OHDSI validation cleanup failed for /workspaces/w/workspace-services/s") + original = ValueError("validation failed") + with ( + patch.object(lc, "lifecycle_deadlines", return_value=(now + 2, now + 3)), + patch.object(lc.OHDSIResources, "close", AsyncMock(return_value=[cleanup_error])), + ): + with self.assertRaises(ValueError) as raised: + async with lc.ohdsi_lifecycle(True): + raise original + self.assertIs(raised.exception, original) + notes = " ".join(original.__notes__) + self.assertIn("delete returned HTTP 409", notes) + self.assertIn("/workspaces/w/workspace-services/s", notes) + + async def test_cancellation_waits_for_cleanup(self): + started = asyncio.Event() + release = asyncio.Event() + done = Mock() + + async def close(): + started.set() + await release.wait() + done() + return [] + + async def run(): + async with lc.ohdsi_lifecycle(True): + pass + + with ( + patch.object(lc.OHDSIResources, "close", side_effect=close), + patch.dict(os.environ, OHDSI_VALIDATION_DEADLINE=""), + ): + task = asyncio.create_task(run()) + await started.wait() + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await task + done.assert_called_once() diff --git a/templates/shared_services/cyclecloud/parameters.json b/templates/shared_services/cyclecloud/parameters.json index c39909257d..8e0d126ecd 100755 --- a/templates/shared_services/cyclecloud/parameters.json +++ b/templates/shared_services/cyclecloud/parameters.json @@ -57,6 +57,12 @@ "source": { "env": "KEY_STORE_ID" } + }, + { + "name": "cluster_address_spaces", + "source": { + "env": "CLUSTER_ADDRESS_SPACES" + } } ] } diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index 1cd58af1d4..272abd1d15 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-cyclecloud -version: 0.7.9 +version: 1.0.0 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl @@ -53,8 +53,19 @@ parameters: - name: key_store_id type: string default: "" + - name: cluster_address_spaces + type: string + description: "Address spaces of cluster nodes that connect to the CycleCloud server on port 9443" + default: "W10=" # b64 for [] outputs: + - name: azure_resource_id + type: string + applyTo: + - install + - upgrade + - start + - stop - name: connection_uri type: string applyTo: @@ -82,6 +93,7 @@ install: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -90,6 +102,7 @@ install: container_name: ${ bundle.parameters.tfstate_container_name } key: ${ bundle.parameters.id } outputs: + - name: azure_resource_id - name: connection_uri - name: shared_subnet_address_prefixes @@ -102,6 +115,7 @@ upgrade: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -110,6 +124,7 @@ upgrade: container_name: ${ bundle.parameters.tfstate_container_name } key: ${ bundle.parameters.id } outputs: + - name: azure_resource_id - name: connection_uri - name: shared_subnet_address_prefixes @@ -122,6 +137,7 @@ uninstall: arm_environment: ${ bundle.parameters.arm_environment } enable_cmk_encryption: ${ bundle.parameters.enable_cmk_encryption } key_store_id: ${ bundle.parameters.key_store_id } + cluster_address_spaces: ${ bundle.parameters.cluster_address_spaces } backendConfig: use_azuread_auth: "true" use_oidc: "true" diff --git a/templates/shared_services/cyclecloud/template_schema.json b/templates/shared_services/cyclecloud/template_schema.json index 425fe4bd97..d6048e27d9 100644 --- a/templates/shared_services/cyclecloud/template_schema.json +++ b/templates/shared_services/cyclecloud/template_schema.json @@ -26,6 +26,18 @@ "description": "Long form description of the shared service, in markdown syntax", "default": "Azure CycleCloud is an enterprise-friendly tool for orchestrating and managing High Performance Computing (HPC) environments on Azure.", "updateable": true + }, + "cluster_address_spaces": { + "type": "array", + "title": "Cluster address spaces", + "description": "Address spaces in CIDR format, for example 10.1.4.0/24. Cluster nodes in these address spaces can connect to the CycleCloud server on TCP port 9443. Add the address space of each workspace that has a CycleCloud cluster.", + "default": [], + "updateable": true, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])/(3[0-2]|[12]?[0-9])$" + } } }, "pipeline": { diff --git a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf index 7d58b39a60..a3e156b3ef 100644 --- a/templates/shared_services/cyclecloud/terraform/cyclecloud.tf +++ b/templates/shared_services/cyclecloud/terraform/cyclecloud.tf @@ -29,7 +29,7 @@ resource "azurerm_virtual_machine" "cyclecloud" { location = data.azurerm_resource_group.rg.location resource_group_name = data.azurerm_resource_group.rg.name network_interface_ids = [azurerm_network_interface.cyclecloud.id] - vm_size = "Standard_DS3_v2" + vm_size = "Standard_D4s_v5" delete_os_disk_on_termination = true delete_data_disks_on_termination = true @@ -70,7 +70,11 @@ resource "azurerm_virtual_machine" "cyclecloud" { tags = local.tre_shared_service_tags - lifecycle { ignore_changes = [tags] } + # Start the server only after its network card has the inbound rules. + depends_on = [azurerm_network_interface_security_group_association.cyclecloud] + + # Azure does not offer Standard_DS3_v2 to all subscriptions in all regions. Keep the size of existing servers. + lifecycle { ignore_changes = [tags, vm_size] } } diff --git a/templates/shared_services/cyclecloud/terraform/locals.tf b/templates/shared_services/cyclecloud/terraform/locals.tf index 9d75e9f53b..8e93664bca 100644 --- a/templates/shared_services/cyclecloud/terraform/locals.tf +++ b/templates/shared_services/cyclecloud/terraform/locals.tf @@ -2,8 +2,13 @@ locals { core_resource_group_name = "rg-${var.tre_id}" core_vnet = "vnet-${var.tre_id}" short_service_id = substr(var.tre_resource_id, -4, -1) - vm_name = "cyclecloud-${local.short_service_id}" - storage_name = lower(replace("stgcc${var.tre_id}${local.short_service_id}", "-", "")) + # A range with host bits, for example 10.1.4.5/24, becomes its network address, 10.1.4.0/24. + cluster_address_spaces = distinct([ + for cidr in jsondecode(base64decode(var.cluster_address_spaces)) : + "${cidrhost(cidr, 0)}/${split("/", cidr)[1]}" + ]) + vm_name = "cyclecloud-${local.short_service_id}" + storage_name = lower(replace("stgcc${var.tre_id}${local.short_service_id}", "-", "")) tre_shared_service_tags = { tre_id = var.tre_id tre_shared_service_id = var.tre_resource_id diff --git a/templates/shared_services/cyclecloud/terraform/nsg.tf b/templates/shared_services/cyclecloud/terraform/nsg.tf new file mode 100644 index 0000000000..cd6dd076c0 --- /dev/null +++ b/templates/shared_services/cyclecloud/terraform/nsg.tf @@ -0,0 +1,68 @@ +# Inbound rules for the CycleCloud server. The outbound rules stay at the Azure defaults. +resource "azurerm_network_security_group" "cyclecloud" { + name = "nsg-cyclecloud-${var.tre_id}-${local.short_service_id}" + location = data.azurerm_resource_group.rg.location + resource_group_name = local.core_resource_group_name + tags = local.tre_shared_service_tags + + # The admin jumpbox and Azure Bastion are in the core virtual network. + security_rule { + name = "AllowCoreHttpsSshInbound" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_ranges = ["22", "443"] + source_address_prefixes = data.azurerm_virtual_network.core.address_space + destination_address_prefix = "*" + } + + # Cluster nodes without a return proxy connect to the CycleCloud server on port 9443. + dynamic "security_rule" { + for_each = length(local.cluster_address_spaces) > 0 ? [local.cluster_address_spaces] : [] + content { + name = "AllowClusterNodesInbound" + priority = 110 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9443" + source_address_prefixes = security_rule.value + destination_address_prefix = "*" + } + } + + security_rule { + name = "AllowAzureLoadBalancerInbound" + priority = 120 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "*" + } + + # Peered workspace networks are part of the VirtualNetwork service tag. + security_rule { + name = "DenyVirtualNetworkInbound" + priority = 4096 + direction = "Inbound" + access = "Deny" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "*" + } + + lifecycle { ignore_changes = [tags] } +} + +resource "azurerm_network_interface_security_group_association" "cyclecloud" { + network_interface_id = azurerm_network_interface.cyclecloud.id + network_security_group_id = azurerm_network_security_group.cyclecloud.id +} diff --git a/templates/shared_services/cyclecloud/terraform/outputs.tf b/templates/shared_services/cyclecloud/terraform/outputs.tf index d6b7b5c87d..6220f4370c 100644 --- a/templates/shared_services/cyclecloud/terraform/outputs.tf +++ b/templates/shared_services/cyclecloud/terraform/outputs.tf @@ -1,3 +1,7 @@ +output "azure_resource_id" { + value = azurerm_virtual_machine.cyclecloud.id +} + output "connection_uri" { value = "https://${azurerm_private_dns_zone.cyclecloud.name}" } diff --git a/templates/shared_services/cyclecloud/terraform/variables.tf b/templates/shared_services/cyclecloud/terraform/variables.tf index 48a64a1945..62d44c519b 100644 --- a/templates/shared_services/cyclecloud/terraform/variables.tf +++ b/templates/shared_services/cyclecloud/terraform/variables.tf @@ -14,3 +14,15 @@ variable "enable_cmk_encryption" { variable "key_store_id" { type = string } +variable "cluster_address_spaces" { + type = string + description = "Address spaces of cluster nodes that connect to the CycleCloud server on port 9443 (base 64 JSON array)" + + validation { + condition = try(alltrue([ + for cidr in jsondecode(base64decode(var.cluster_address_spaces)) : + can(cidrnetmask(cidr)) + ]), false) + error_message = "cluster_address_spaces must be a list of IPv4 address ranges in CIDR format, for example 10.1.4.0/24." + } +} diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index dba38adcd1..a18290240e 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.13 +version: 0.3.14 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh b/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh index b50f073fb0..723f387164 100755 --- a/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh +++ b/templates/workspace_services/ohdsi/scripts/atlas_db_init.sh @@ -3,19 +3,18 @@ set -o errexit set -o pipefail set -o nounset -admin_user_password="${OHDSI_ADMIN_PASSWORD}${OHDSI_ADMIN_USERNAME}" -app_user_password="${OHDSI_APP_PASSWORD}${OHDSI_APP_USERNAME}" -admin_md5=("md5$(echo -n "$admin_user_password" | md5sum | awk '{ print $1 }')") -export admin_md5 -app_md5=("md5$(echo -n "$app_user_password" | md5sum | awk '{ print $1 }')'") -export app_md5 +export PGCONNECT_TIMEOUT=30 +export PGOPTIONS='-c statement_timeout=600000 -c lock_timeout=30000' -printf 'Creating roles and users' -envsubst < ../sql/atlas_create_roles_users.sql | psql -v ON_ERROR_STOP=0 -e "$MAIN_CONNECTION_STRING" -printf 'Creating roles and users: done.' +printf 'Creating roles and users\n' +psql -X -v ON_ERROR_STOP=1 "$MAIN_CONNECTION_STRING" \ + -v "admin_role=$OHDSI_ADMIN_ROLE" -v "app_role=$OHDSI_APP_ROLE" \ + -v "admin_username=$OHDSI_ADMIN_USERNAME" -v "app_username=$OHDSI_APP_USERNAME" \ + -v "admin_password=$OHDSI_ADMIN_PASSWORD" -v "app_password=$OHDSI_APP_PASSWORD" \ + -v "database_name=$DATABASE_NAME" -f ../sql/atlas_create_roles_users.sql -printf 'Creating schema' -envsubst < ../sql/atlas_create_schema.sql | psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -printf 'Creating schema: done.' - -printf 'Done' +printf 'Creating schema\n' +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "schema_name=$SCHEMA_NAME" -v "admin_role=$OHDSI_ADMIN_ROLE" \ + -v "app_role=$OHDSI_APP_ROLE" -f ../sql/atlas_create_schema.sql +printf 'Database initialisation complete\n' diff --git a/templates/workspace_services/ohdsi/scripts/atlas_security.sh b/templates/workspace_services/ohdsi/scripts/atlas_security.sh index 0d15b4bc62..ebcf8c81f9 100755 --- a/templates/workspace_services/ohdsi/scripts/atlas_security.sh +++ b/templates/workspace_services/ohdsi/scripts/atlas_security.sh @@ -3,27 +3,58 @@ set -o errexit set -o pipefail set -o nounset -psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -f "../sql/atlas_create_security.sql" +export PGCONNECT_TIMEOUT=30 +export PGOPTIONS='-c statement_timeout=600000 -c lock_timeout=30000' -psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -f "../sql/atlas_default_roles.sql" +# Retry only the unauthenticated readiness request. Failed logins can lock accounts. +info_file=$(mktemp) +trap 'rm -f "$info_file"' EXIT +# Expand readiness variables inside the bounded child shell. +# shellcheck disable=SC2016 +WEB_API_INFO_FILE="$info_file" timeout 600 bash -eu -o pipefail -c ' + until curl "${WEB_API_URL%/}/info" --fail --silent --show-error \ + --connect-timeout 5 --max-time 15 --output "$WEB_API_INFO_FILE" && + jq -e --arg version "$WEB_API_VERSION" ".version == \$version" "$WEB_API_INFO_FILE" > /dev/null; do + sleep 10 + done +' -count=1 -for i in ${ATLAS_USERS//,/ } -do - if [ "$(("$count" % 2))" -eq "1" ]; then - username=$i - else - # shellcheck disable=SC2016 - atlaspw=$(htpasswd -bnBC 4 "" "$i" | tr -d ':\n' | sed 's/$2y/$2a/') - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi_security.security (email,password) values ('$username', E'$atlaspw');" - # this step adds some required rows/ids in the db - curl "$WEB_API_URL/user/login/db" --data-urlencode "login=$username" --data-urlencode "password=$i" --fail +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" -f ../sql/atlas_create_security.sql +psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" -f ../sql/atlas_default_roles.sql - if [ "$count" = "2" ]; then - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi.sec_user_role (user_id, role_id) values ((select id from webapi.sec_user where login='$username'),2);" #admin role - else - psql -v ON_ERROR_STOP=1 -e "$OHDSI_ADMIN_CONNECTION_STRING" -c "insert into webapi.sec_user_role (user_id, role_id) values ((select id from webapi.sec_user where login='$username'),10);" #atlas user role - fi - fi - ((count++)) +IFS=',' read -r -a users <<< "$ATLAS_USERS" +if (( ${#users[@]} == 0 || ${#users[@]} % 2 != 0 )); then + printf 'ATLAS_USERS must contain username and password pairs\n' >&2 + exit 1 +fi +for ((index=0; index<${#users[@]}; index+=2)); do + username=${users[index]} + password=${users[index+1]} + # shellcheck disable=SC2016 + atlaspw=$(htpasswd -bnBC 4 "" "$password" | tr -d ':\n' | sed 's/$2y/$2a/') + psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "username=$username" -v "password_hash=$atlaspw" <<'SQL' +INSERT INTO webapi_security.security (email, password) VALUES (:'username', :'password_hash'); +SQL + # The response can contain an authentication token. Do not write it to the log. + curl "${WEB_API_URL%/}/user/login/db" --fail --silent --show-error \ + --connect-timeout 5 --max-time 30 --output /dev/null \ + --data-urlencode "login=$username" --data-urlencode "password=$password" + + role_id=10 + if (( index == 0 )); then role_id=2; fi + psql -X -v ON_ERROR_STOP=1 "$OHDSI_ADMIN_CONNECTION_STRING" \ + -v "username=$username" -v "role_id=$role_id" <<'SQL' +INSERT INTO webapi.sec_user_role (user_id, role_id) +SELECT id, :'role_id'::integer FROM webapi.sec_user u WHERE login = :'username' +AND NOT EXISTS (SELECT FROM webapi.sec_user_role r WHERE r.user_id = u.id AND r.role_id = :'role_id'::integer); +SELECT EXISTS ( + SELECT FROM webapi.sec_user u JOIN webapi.sec_user_role r ON r.user_id = u.id + WHERE u.login = :'username' AND r.role_id = :'role_id'::integer +) AS role_assigned \gset +\if :role_assigned +\else + DO $$ BEGIN RAISE EXCEPTION 'Application user role assignment failed'; END $$; +\endif +SQL done diff --git a/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql b/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql index f77ffa515f..e817e645a2 100644 --- a/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql +++ b/templates/workspace_services/ohdsi/sql/atlas_create_roles_users.sql @@ -1,23 +1,23 @@ -CREATE ROLE ${OHDSI_ADMIN_ROLE} CREATEDB REPLICATION VALID UNTIL 'infinity'; - -COMMENT ON ROLE ${OHDSI_ADMIN_ROLE} IS 'Administration group for OHDSI applications'; - -CREATE ROLE ${OHDSI_APP_ROLE} VALID UNTIL 'infinity'; - -COMMENT ON ROLE ${OHDSI_APP_ROLE} IS 'Application groupfor OHDSI applications'; - -CREATE ROLE ${OHDSI_ADMIN_USERNAME} LOGIN ENCRYPTED PASSWORD ${admin_md5} VALID UNTIL 'infinity'; - -GRANT ${OHDSI_ADMIN_ROLE} TO ${OHDSI_ADMIN_USERNAME}; - -COMMENT ON ROLE ${OHDSI_ADMIN_USERNAME} IS 'Admin user account for OHDSI applications'; - -CREATE ROLE ${OHDSI_APP_USERNAME} LOGIN ENCRYPTED PASSWORD ${app_md5} VALID UNTIL 'infinity'; - -GRANT ${OHDSI_APP_ROLE} TO ${OHDSI_APP_USERNAME}; - -COMMENT ON ROLE ${OHDSI_APP_USERNAME} IS 'Application user account for OHDSI applications'; - -GRANT ALL ON DATABASE ${DATABASE_NAME} TO GROUP ${OHDSI_ADMIN_ROLE}; - -GRANT CONNECT, TEMPORARY ON DATABASE ${DATABASE_NAME} TO GROUP ${OHDSI_APP_ROLE}; +-- Quote identifiers and values with psql. A retry can reuse roles already created. +SELECT format('CREATE ROLE %I CREATEDB REPLICATION', :'admin_role') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'admin_role') \gexec +COMMENT ON ROLE :"admin_role" IS 'Administration group for OHDSI applications'; + +SELECT format('CREATE ROLE %I', :'app_role') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_role') \gexec +COMMENT ON ROLE :"app_role" IS 'Application group for OHDSI applications'; + +SELECT format('CREATE ROLE %I LOGIN', :'admin_username') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'admin_username') \gexec +ALTER ROLE :"admin_username" PASSWORD :'admin_password' VALID UNTIL 'infinity'; +GRANT :"admin_role" TO :"admin_username"; +COMMENT ON ROLE :"admin_username" IS 'Admin user account for OHDSI applications'; + +SELECT format('CREATE ROLE %I LOGIN', :'app_username') +WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_username') \gexec +ALTER ROLE :"app_username" PASSWORD :'app_password' VALID UNTIL 'infinity'; +GRANT :"app_role" TO :"app_username"; +COMMENT ON ROLE :"app_username" IS 'Application user account for OHDSI applications'; + +GRANT ALL ON DATABASE :"database_name" TO GROUP :"admin_role"; +GRANT CONNECT, TEMPORARY ON DATABASE :"database_name" TO GROUP :"app_role"; diff --git a/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql b/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql index f6bd62d236..42fb049f60 100644 --- a/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql +++ b/templates/workspace_services/ohdsi/sql/atlas_create_schema.sql @@ -1,24 +1,24 @@ -CREATE SCHEMA ${SCHEMA_NAME} -AUTHORIZATION ${OHDSI_ADMIN_ROLE}; +CREATE SCHEMA IF NOT EXISTS :"schema_name" +AUTHORIZATION :"admin_role"; -COMMENT ON SCHEMA ${SCHEMA_NAME} IS 'Schema containing tables to support WebAPI functionality'; +COMMENT ON SCHEMA :"schema_name" IS 'Schema containing tables to support WebAPI functionality'; -GRANT USAGE ON SCHEMA ${SCHEMA_NAME} TO PUBLIC; +GRANT USAGE ON SCHEMA :"schema_name" TO PUBLIC; -GRANT ALL ON SCHEMA ${SCHEMA_NAME} TO GROUP ${OHDSI_ADMIN_ROLE}; +GRANT ALL ON SCHEMA :"schema_name" TO GROUP :"admin_role"; -GRANT USAGE ON SCHEMA ${SCHEMA_NAME} TO GROUP ${OHDSI_APP_ROLE}; +GRANT USAGE ON SCHEMA :"schema_name" TO GROUP :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT INSERT, SELECT, UPDATE, DELETE, REFERENCES, - TRIGGER ON TABLES TO ${OHDSI_APP_ROLE}; + TRIGGER ON TABLES TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT -SELECT, USAGE ON SEQUENCES TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT +SELECT, USAGE ON SEQUENCES TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT EXECUTE ON FUNCTIONS TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT EXECUTE ON FUNCTIONS TO :"app_role"; -ALTER DEFAULT PRIVILEGES IN SCHEMA ${SCHEMA_NAME} GRANT USAGE ON TYPES TO ${OHDSI_APP_ROLE}; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT USAGE ON TYPES TO :"app_role"; diff --git a/templates/workspace_services/ohdsi/template_schema.json b/templates/workspace_services/ohdsi/template_schema.json index 0794edb119..913c5dc053 100644 --- a/templates/workspace_services/ohdsi/template_schema.json +++ b/templates/workspace_services/ohdsi/template_schema.json @@ -368,6 +368,33 @@ } ] }, + { + "stepId": "83a8b148-d455-416c-9779-320fb7ae279c", + "stepTitle": "Remove OHDSI firewall rules", + "resourceTemplateName": "tre-shared-service-firewall", + "resourceType": "shared-service", + "resourceAction": "upgrade", + "properties": [ + { + "name": "network_rule_collections", + "type": "array", + "arraySubstitutionAction": "remove", + "arrayMatchField": "name", + "value": { + "name": "nrc_svc_{{ resource.id }}" + } + }, + { + "name": "rule_collections", + "type": "array", + "arraySubstitutionAction": "remove", + "arrayMatchField": "name", + "value": { + "name": "arc_svc_{{ resource.id }}" + } + } + ] + }, { "stepId": "main" } diff --git a/templates/workspace_services/ohdsi/terraform/atlas_database.tf b/templates/workspace_services/ohdsi/terraform/atlas_database.tf index 7d26918705..ee677c64c9 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_database.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_database.tf @@ -105,6 +105,8 @@ resource "azurerm_subnet" "postgres" { resource_group_name = data.azurerm_resource_group.ws.name address_prefixes = [var.address_space] default_outbound_access_enabled = false + # The PostgreSQL service adds this endpoint during the server create, so declare it to stop upgrades removing it. + service_endpoints = ["Microsoft.Storage"] delegation { name = "psql-delegation" @@ -147,18 +149,19 @@ resource "terraform_data" "postgres_subnet_wait" { } resource "azurerm_postgresql_flexible_server" "postgres" { - name = "psql-server-${local.service_suffix}" - resource_group_name = data.azurerm_resource_group.ws.name - location = data.azurerm_resource_group.ws.location - delegated_subnet_id = azurerm_subnet.postgres.id - private_dns_zone_id = data.azurerm_private_dns_zone.postgres.id - sku_name = var.postgres_sku - version = local.postgres_version - administrator_login = local.postgres_admin_username - administrator_password = azurerm_key_vault_secret.postgres_admin_password.value - storage_mb = var.postgres_storage_size_in_mb - zone = "1" - tags = local.tre_workspace_service_tags + public_network_access_enabled = false + name = "psql-server-${local.service_suffix}" + resource_group_name = data.azurerm_resource_group.ws.name + location = data.azurerm_resource_group.ws.location + delegated_subnet_id = azurerm_subnet.postgres.id + private_dns_zone_id = data.azurerm_private_dns_zone.postgres.id + sku_name = var.postgres_sku + version = local.postgres_version + administrator_login = local.postgres_admin_username + administrator_password = azurerm_key_vault_secret.postgres_admin_password.value + storage_mb = var.postgres_storage_size_in_mb + zone = "1" + tags = local.tre_workspace_service_tags timeouts { # If this doesn't complete in a realistic time, no point in waiting the full/default 60m diff --git a/templates/workspace_services/ohdsi/terraform/atlas_security.tf b/templates/workspace_services/ohdsi/terraform/atlas_security.tf index 0dd1fc3083..901bef75da 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_security.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_security.tf @@ -23,6 +23,7 @@ resource "terraform_data" "deployment_atlas_security" { ATLAS_SECURITY_ADMIN_PASSWORD = azurerm_key_vault_secret.atlas_security_admin_password.value ATLAS_USERS = "admin,${azurerm_key_vault_secret.atlas_security_admin_password.value}" WEB_API_URL = local.ohdsi_webapi_url + WEB_API_VERSION = local.ohdsi_api_docker_image_tag } command = "../scripts/atlas_security.sh" diff --git a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf index 24879a80da..8b8649e543 100644 --- a/templates/workspace_services/ohdsi/terraform/atlas_ui.tf +++ b/templates/workspace_services/ohdsi/terraform/atlas_ui.tf @@ -10,6 +10,7 @@ resource "local_file" "config_local" { } resource "azurerm_storage_share_file" "config_local" { + provider = azurerm.shared_key name = "config-local.js" storage_share_id = azurerm_storage_share.atlas_ui.id source = local.config_local_file_path @@ -20,10 +21,11 @@ resource "azurerm_storage_share_file" "config_local" { } resource "azurerm_linux_web_app" "atlas_ui" { - name = local.atlas_ui_name - location = data.azurerm_resource_group.ws.location - resource_group_name = data.azurerm_resource_group.ws.name - virtual_network_subnet_id = data.azurerm_subnet.web_app.id + public_network_access_enabled = false + name = local.atlas_ui_name + location = data.azurerm_resource_group.ws.location + resource_group_name = data.azurerm_resource_group.ws.name + virtual_network_subnet_id = data.azurerm_subnet.web_app.id service_plan_id = data.azurerm_service_plan.workspace.id https_only = true @@ -68,6 +70,9 @@ resource "azurerm_linux_web_app" "atlas_ui" { depends_on = [ azurerm_storage_share_file.config_local, + # Azure rejects the web app subnet link while another subnet write runs in the same VNet. + # The PostgreSQL server create makes the last write to the PostgreSQL subnet. + azurerm_postgresql_flexible_server.postgres, ] tags = local.tre_workspace_service_tags diff --git a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf index 2606dbc5cb..a1d732bd9f 100644 --- a/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf +++ b/templates/workspace_services/ohdsi/terraform/ohdsi_web_api.tf @@ -16,17 +16,27 @@ resource "azurerm_user_assigned_identity" "ohdsi_webapi_id" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_role_assignment" "keyvault_ohdsi_ws_role" { - scope = data.azurerm_key_vault.ws.id +# The web app reads only the secrets of its Key Vault references, so its identity gets a role on each of them. +resource "azurerm_role_assignment" "keyvault_ohdsi_secret_role" { + for_each = { + jdbc_connection_string = azurerm_key_vault_secret.jdbc_connection_string_webapi_admin.resource_versionless_id + webapi_admin_password = azurerm_key_vault_secret.postgres_webapi_admin_password.resource_versionless_id + webapi_app_password = azurerm_key_vault_secret.postgres_webapi_app_password.resource_versionless_id + workspace_client_id = data.azurerm_key_vault_secret.workspace_client_id.resource_versionless_id + workspace_client_secret = data.azurerm_key_vault_secret.workspace_client_secret.resource_versionless_id + } + + scope = each.value role_definition_name = "Key Vault Secrets User" principal_id = azurerm_user_assigned_identity.ohdsi_webapi_id.principal_id } resource "azurerm_linux_web_app" "ohdsi_webapi" { - name = local.ohdsi_webapi_name - location = data.azurerm_resource_group.ws.location - resource_group_name = data.azurerm_resource_group.ws.name - virtual_network_subnet_id = data.azurerm_subnet.web_app.id + public_network_access_enabled = false + name = local.ohdsi_webapi_name + location = data.azurerm_resource_group.ws.location + resource_group_name = data.azurerm_resource_group.ws.name + virtual_network_subnet_id = data.azurerm_subnet.web_app.id service_plan_id = data.azurerm_service_plan.workspace.id https_only = true @@ -110,7 +120,10 @@ resource "azurerm_linux_web_app" "ohdsi_webapi" { tags = local.tre_workspace_service_tags depends_on = [ - terraform_data.deployment_ohdsi_webapi_init + terraform_data.deployment_ohdsi_webapi_init, + azurerm_role_assignment.keyvault_ohdsi_secret_role, + # Link the two web apps to the web app subnet one after the other. + azurerm_linux_web_app.atlas_ui, ] lifecycle { ignore_changes = [tags] } diff --git a/templates/workspace_services/ohdsi/terraform/providers.tf b/templates/workspace_services/ohdsi/terraform/providers.tf index e2f5f98498..11b59e0570 100644 --- a/templates/workspace_services/ohdsi/terraform/providers.tf +++ b/templates/workspace_services/ohdsi/terraform/providers.tf @@ -35,6 +35,14 @@ provider "azurerm" { storage_use_azuread = true } +# Uploads the Atlas UI config with the storage account key. The share and its web app mount already use the key, +# and Azure AD access to file data needs a role that the resource processor does not have. +provider "azurerm" { + alias = "shared_key" + features {} + storage_use_azuread = false +} + module "terraform_azurerm_environment_configuration" { source = "git::https://github.com/microsoft/terraform-azurerm-environment-configuration.git?ref=0.7.0" arm_environment = var.arm_environment