diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 6849c357d8..ffc65f879a 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -718,6 +718,8 @@ jobs: BUNDLE_DIR: "./templates/workspace_services/ohdsi" - BUNDLE_TYPE: "workspace_service" BUNDLE_DIR: "./templates/workspace_services/azuresql" + - BUNDLE_TYPE: "workspace_service" + BUNDLE_DIR: "./templates/workspace_services/openai" environment: ${{ inputs.environmentName }} steps: @@ -1071,6 +1073,10 @@ jobs: if: inputs.e2eBundle == 'tre-workspace-service-azuresql' run: echo "SQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Reserve MySQL cleanup and reporting time + if: inputs.e2eBundle == 'tre-workspace-service-mysql' + run: echo "MYSQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -1098,6 +1104,7 @@ jobs: COMMAND: >- TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=${{ inputs.runCertificateTestsOnWeekends }} SQL_VALIDATION_DEADLINE=${{ env.SQL_VALIDATION_DEADLINE }} + MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index f49a503145..5e263cf715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: +* Register the OpenAI bundle for branch validation, add GPT-5.1 as the default model and check regional model and quota prerequisites before its private lifecycle test. (`tre-workspace-service-openai` 1.1.0) ([#5150](https://github.com/microsoft/AzureTRE/issues/5150)) +* Add a private MySQL lifecycle check. A Nexus-backed Linux VM writes a row through private DNS, the SKU changes from Burstable to General Purpose, and the VM reads the row again. ([#5148](https://github.com/microsoft/AzureTRE/issues/5148)) +* Run the `stop`, `start` and `reset_password` actions in the Guacamole Linux VM check, and make sure of the VM power state and the new Key Vault secret version through Azure. ([#5145](https://github.com/microsoft/AzureTRE/issues/5145)) * Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock, Linux VMs and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) @@ -35,6 +38,11 @@ BUG FIXES: (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) +* Try the storage registration up to five times before Terraform registers the workspace storage account with Azure Backup. Azure Backup usually fails the first registration in a new vault with an internal error, so most backup-enabled workspaces failed to deploy. Terraform registers after an attempt succeeds, or after five failed attempts. + (`tre-workspace-base` 2.12.4, `tre-workspace-unrestricted` 0.15.3, `tre-workspace-airlock-import-review` 0.18.3) ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) +* Clean up backup-test workspaces after failed provisioning. ([#5135](https://github.com/microsoft/AzureTRE/issues/5135)) +* Create the OpenAI private endpoint after the model deployment, so that Azure does not reject it while the deployment updates the account. (`tre-workspace-service-openai` 1.1.0) ([#5249](https://github.com/microsoft/AzureTRE/issues/5249)) +* Disable public network access on the MySQL flexible server, so that clients can connect only through the private endpoint. Move the MySQL bundle to azurerm 4.27.0, which has this setting. An upgrade changes the setting in place. (`tre-workspace-service-mysql` 1.0.16) ([#5148](https://github.com/microsoft/AzureTRE/issues/5148)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index c62d54490e..e4f727a0c6 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -82,6 +82,25 @@ That case requires an owned workspace so cascade cleanup removes its new service Some shared-service cases replace existing services, so use an isolated validation environment. Check the selected entry's prerequisites and limitations before running it. +### OpenAI selection + +Select `tre-workspace-service-openai` to run `test_openai.py::test_private_openai_lifecycle`. +The branch workflow registers the OpenAI bundle. The test also replaces its generic case in the `workspace_services` group. +The test requests `gpt-5.1 | 2025-11-13`, private access and one regional `Standard` capacity unit. +It does not switch models or deployment types when Azure cannot satisfy those inputs. + +The prerequisite check uses the workspace resource group's actual region and the current subscription model and usage APIs. +It requires a generally available chat model, a current Standard tier and at least one free capacity unit. +It rejects retired model/tier dates, fine-tuning quotas and incomplete or blocked quota data. +The CI identity needs permission to read model and quota data at subscription scope. +The check does not grant roles or reserve capacity. Azure deployment can still fail after it passes. + +Prerequisite failures fail the selected test before service creation. They do not count as skipped or passing coverage. +Service creation has a 45-minute deadline. Verification has a separate five-minute deadline. +Failed creation and teardown use the existing helper's separate cleanup deadline. +A configured workspace is retained. A fixture-created workspace uses the fixture's cleanup. +Private inference, access control, upgrades and complete physical cleanup remain separate checks. + ### Certificate and Nexus selection Select `tre-shared-service-certs` to install, uninstall and reinstall the certificate shared service with the same hostname and certificate name. @@ -124,6 +143,14 @@ Select `e2eBundle=tre-shared-service-certs` in `deploy_tre_branch.yml` and leave PR slash commands use workflow definitions from `main`. `/test` runs smoke tests and `/test-shared-services` excludes this certificate-only case. A test-only PR without the new certificate bundle cannot validate reuse. Run Nexus and Airlock only after this case and its cleanup pass. +### Linux VM selection + +Select `tre-service-guacamole-linuxvm` to run `test_guacamole_linuxvm.py::test_create_guacamole_linux_vm`. +The case creates a Linux VM and waits for its Nexus-backed bootstrap. Thus the run needs a suitable Nexus instance or `acceptNexusEula=true`. +The case then runs the `stop`, `start` and `reset_password` actions. +After `stop` and `start`, the case reads the VM power state from Azure. After `reset_password`, it makes sure that the Key Vault secret has a new version. +The case does not sign in to the VM, so it does not prove interactive access or the new password. + ### Base workspace lifecycle Select `tre-workspace-base` to run `test_workspace_base.py::test_base_workspace_lifecycle` independently. @@ -149,6 +176,16 @@ Failed-create recovery and temporary probe cleanup also count towards the lifecy The final 30 minutes are reserved for results and job overhead. Cleanup failures still fail the test. For a local run, the 270-minute lifecycle budget starts when the SQL case begins. +### MySQL selection + +Select `tre-workspace-service-mysql` to run `test_mysql.py::test_private_mysql_query_survives_sku_change`. +The case creates its own workspace, Guacamole service, Linux VM and MySQL service. +The Linux VM installs the MySQL client through Nexus. Thus the run needs a suitable Nexus instance or `acceptNexusEula=true`. +The case writes a row through private DNS with the Burstable SKU. It then changes the SKU to General Purpose and reads the row again. +The VM reads the administrator password from the workspace Key Vault through a temporary role assignment on that secret only. +The MySQL job uses the SQL time limits above with its own `MYSQL_VALIDATION_DEADLINE`. If the case creates Nexus, it removes Nexus after the VM. +These checks do not prove previous-version upgrades or researcher access. + ### Run the branch workflow before merge 1. Push the reviewed changes to a branch in `microsoft/AzureTRE`. diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index d6513b3ae3..9722956fef 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -22,7 +22,7 @@ The table describes the selected assertions. A passing command does not prove ev | `/test-airlock` | Import requests, access checks, the import review VM flow and export request storage routing | No export review VM or migration proof. | | `/test-backups` | Base workspace deployment with backups enabled and disabled | No backup restore or retention proof. | -The `workspace_services`, `linux_vm`, `airlock_validation` and `sql_validation` markers have no dedicated slash command. +The `workspace_services`, `linux_vm`, `airlock_validation`, `sql_validation` and `mysql_validation` markers have no dedicated slash command. Use the branch workflow or the local test runner for these selections. The `airlock_validation` marker selects two Airlock cases. It does not select the full `airlock` suite. See [bundle selection](end-to-end-tests.md#validate-one-bundle) for exact cases and declared coverage gaps. diff --git a/docs/tre-templates/workspace-services/mysql.md b/docs/tre-templates/workspace-services/mysql.md index c7174cf292..0c979d9a21 100644 --- a/docs/tre-templates/workspace-services/mysql.md +++ b/docs/tre-templates/workspace-services/mysql.md @@ -10,6 +10,10 @@ See: [MySQL Azure](https://learn.microsoft.com/en-GB/azure/mysql/) `make workspace_service_bundle BUNDLE=mysql` +## Network access + +The MySQL server has public network access disabled. Clients connect through its private endpoint, for example from a VM in the workspace. + ## Authenticating to MySQL 1. Navigate to the MySQL workspace service using the `Mysql fqdn` from the details tab. diff --git a/docs/tre-templates/workspace-services/openai.md b/docs/tre-templates/workspace-services/openai.md index 5ad0b57395..1ea4bf8077 100644 --- a/docs/tre-templates/workspace-services/openai.md +++ b/docs/tre-templates/workspace-services/openai.md @@ -17,6 +17,28 @@ See: [Azure OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/ ## Properties - `is_exposed_externally` - If `True`, the OpenAI workspace is accessible from outside of the workspace virtual network. -- `openai_model` - The model to use for the OpenAI deployment ` | `. The default is `gpt-35-turbo | 0301`. +- `openai_model` - The model to use for the OpenAI deployment ` | `. The default is `gpt-5.1 | 2025-11-13`. - Important note: Models are subject to different quota and region availability and the deployment may fail if you don't have the correct quota. Please review this link on current limits and how to request increases: [Open AI Quotas](https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits) + + +## Model prerequisites + +The bundle uses the regional `Standard` deployment type with one capacity unit. +Check availability and quota for the exact model version in the workspace region before deployment. +The older model values remain in the schema for existing resource records. Their presence does not mean Azure can deploy them. +Changing the template default does not change the model value stored on an existing service. +An explicit model change can replace the deployment and needs separate upgrade validation. + +The OpenAI bundle currently requires the workspace and core in the same subscription. +Private network access remains the default. The bundle does not add researcher RBAC roles or change key authentication. + +See [model availability](https://learn.microsoft.com/en-us/rest/api/aiservices/accountmanagement/models/list?view=rest-aiservices-accountmanagement-2024-10-01) +and [subscription quota](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/quota). + +## Independent validation + +Select `tre-workspace-service-openai` through the [bundle validation procedure](../../tre-developers/end-to-end-tests.md#validate-one-bundle). +The test checks current model and quota prerequisites, deploys a private service, checks its model and network settings, and uninstalls it. +It confirms removal of the API record and active Azure account. +Private inference, researcher access, model upgrades, secret removal and soft-deleted account purge require separate evidence. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index d443823a66..927319df11 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -325,10 +325,13 @@ "coverage": [ "install", "bootstrap-completion", + "stop action with deallocated power state", + "start action with running power state", + "reset_password action with a new Key Vault secret version", "uninstall" ], "limitations": [ - "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites. VM availability and interactive access need separate validation." + "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites. Interactive access, sign-in with the new password and other VM sizes need separate validation." ], "unavailable_reason": "" }, @@ -376,18 +379,27 @@ "manifest": "templates/workspace_services/mysql/porter.yaml", "family": "workspace_service", "prerequisites": [ - "tre-workspace-base" + "tre-workspace-base", + "tre-service-guacamole", + "tre-service-guacamole-linuxvm", + "tre-shared-service-certs", + "tre-shared-service-sonatype-nexus" ], "tests": [ + "test_mysql.py::test_private_mysql_query_survives_sku_change" + ], + "related_tests": [ "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-mysql]" ], - "related_tests": [], "coverage": [ "install", + "private MySQL write/read", + "Burstable-to-General Purpose SKU change with retained data", "uninstall" ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "AzureCloud only. Requires a suitable Nexus instance or explicit Nexus EULA consent, permission to run commands on a test-created workspace VM and a temporary secret-scoped role assignment.", + "The SKU change uses the current bundle version. Previous-version migration remains separate (#5128). The MySQL bundle has no custom actions." ], "unavailable_reason": "" }, @@ -411,15 +423,25 @@ "prerequisites": [ "tre-workspace-base" ], - "tests": [], - "related_tests": [ - "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-openai]" + "tests": [ + "test_openai.py::test_private_openai_lifecycle" + ], + "related_tests": [], + "coverage": [ + "model-prerequisites", + "install", + "deployment-status", + "private-network-setting", + "uninstall", + "api-removal", + "active-account-removal" ], - "coverage": [], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Requires AzureCloud, a registered 1.1.0 or later template, regional Standard model availability and subscription quota read access. Preflight does not reserve Azure capacity.", + "Uses the workspace subscription and actual resource-group region. A workspace in a different subscription from ARM_SUBSCRIPTION_ID is unsupported by this bundle.", + "Does not prove private inference, researcher access, model upgrades, private endpoint connectivity, secret removal or soft-deleted account purge." ], - "unavailable_reason": "The branch deployment does not register this bundle (#5150)." + "unavailable_reason": "" }, "tre-workspace-airlock-import-review": { "manifest": "templates/workspaces/airlock-import-review/porter.yaml", diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index b810211fe8..6613a900a9 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -15,7 +15,9 @@ markers = airlock_validation: draft-container deletion and review-VM flow, including Nexus prerequisites and cleanup workspace_services workspace_validation: opt-in base-workspace creation, scoped API access and deletion + openai_validation: opt-in private OpenAI deployment with model and quota preflight sql_validation: Private SQL query and configuration upgrade lifecycle + mysql_validation: opt-in private MySQL query and SKU change lifecycle from a Nexus-backed Linux VM asyncio_mode = auto asyncio_default_fixture_loop_scope = session diff --git a/e2e_tests/resources/mysql_probe.py b/e2e_tests/resources/mysql_probe.py new file mode 100644 index 0000000000..64247da849 --- /dev/null +++ b/e2e_tests/resources/mysql_probe.py @@ -0,0 +1,87 @@ +"""Run a private MySQL probe from a test-created workspace Linux VM.""" + +import asyncio +import json +from pathlib import Path +import re +from uuid import uuid4 + +from e2e_tests.resources.sql_probe import COMPUTE_API, delete_after + +SCRIPT = Path(__file__).with_name("mysql_probe.sh") +RESULT = "MYSQL_PROBE_RESULT=" +FAILED = "MYSQL_PROBE_FAILED=" +PATTERNS = { + "SERVER": r"[a-z0-9-]+\.mysql\.database\.azure\.com", + "VAULT_NAME": r"[a-z0-9-]+", + "SECRET_NAME": r"[a-z0-9-]+", + "DATABASE": r"[a-z0-9_]+", + "PROBE_ID": r"[a-f0-9]{32}", + "PHASE": r"write|read", +} + + +def probe_script(**values): + """Set the validated values in the script text. They contain no secrets.""" + if set(values) != set(PATTERNS): + raise ValueError("Unexpected MySQL probe values") + for name, value in values.items(): + if not re.fullmatch(PATTERNS[name], value): + raise ValueError(f"Invalid MySQL probe value for {name}") + header = "".join(f"{name}='{value}'\n" for name, value in values.items()) + shebang, body = SCRIPT.read_text().split("\n", 1) + return f"{shebang}\n{header}{body}" + + +def probe_result(resource, phase, probe_id): + properties = resource.get("properties", {}) + view = properties.get("instanceView", {}) + state = view.get("executionState", "") + lines = view.get("output", "").splitlines() + if properties.get("provisioningState") in ("Failed", "Canceled") or state in ("Failed", "Canceled", "TimedOut"): + stages = [line[len(FAILED) :] for line in lines if line.startswith(FAILED)] + raise AssertionError( + f"MySQL probe failed: state={state}, exitCode={view.get('exitCode')}, stage={','.join(stages) or 'unknown'}" + ) + if state != "Succeeded": + return False + if view.get("exitCode") != 0: + raise AssertionError("MySQL probe returned a non-zero exit code") + results = [line[len(RESULT) :] for line in lines if line.startswith(RESULT)] + expected = {"phase": phase, "probe": probe_id, "row_count": 1, "private_dns": True} + if len(results) != 1 or json.loads(results[0]) != expected: + raise AssertionError("MySQL probe did not confirm the expected data and private DNS") + return True + + +async def run_probe(arm, vm_id, location, *, server, vault_name, secret_name, database, probe_id, phase): + script = probe_script( + SERVER=server, + VAULT_NAME=vault_name, + SECRET_NAME=secret_name, + DATABASE=database, + PROBE_ID=probe_id, + PHASE=phase, + ) + command = f"{vm_id}/runCommands/mysql-probe-{uuid4().hex}" + # Removing our command also terminates it if its deadline expired. + async with delete_after(arm, command, COMPUTE_API): + async with asyncio.timeout(20 * 60): + await arm.request( + "PUT", + command, + COMPUTE_API, + body={ + "location": location, + "properties": { + "source": {"script": script}, + "timeoutInSeconds": 900, + "treatFailureAsDeploymentFailure": True, + }, + }, + ) + while True: + result = await arm.request("GET", command, COMPUTE_API, expand=True) + if probe_result(result, phase, probe_id): + return + await asyncio.sleep(10) diff --git a/e2e_tests/resources/mysql_probe.sh b/e2e_tests/resources/mysql_probe.sh new file mode 100755 index 0000000000..c02a5ff581 --- /dev/null +++ b/e2e_tests/resources/mysql_probe.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# The test validates SERVER, VAULT_NAME, SECRET_NAME, DATABASE, PROBE_ID and PHASE and sets them above this line. +set -euo pipefail +stage=input +fail() { + # Do not print command output, tokens or the password. + echo "MYSQL_PROBE_FAILED=$stage" + exit 1 +} +[[ "$SERVER" =~ ^[a-z0-9-]+\.mysql\.database\.azure\.com$ ]] || fail +[[ "$VAULT_NAME" =~ ^[a-z0-9-]+$ ]] || fail +[[ "$SECRET_NAME" =~ ^[a-z0-9-]+$ ]] || fail +[[ "$DATABASE" =~ ^[a-z0-9_]+$ ]] || fail +[[ "$PROBE_ID" =~ ^[a-f0-9]{32}$ ]] || fail +[[ "$PHASE" =~ ^(write|read)$ ]] || fail + +stage=private-dns +addresses=$(getent ahostsv4 "$SERVER" | awk '{print $1}' | sort -u) || fail +[ -n "$addresses" ] || fail +for address in $addresses; do + case "$address" in + 10.* | 192.168.* | 172.1[6-9].* | 172.2[0-9].* | 172.3[01].*) ;; + *) fail ;; + esac +done + +stage=mysql-client +# The VM bootstrap points apt at Nexus. Wait for it before the install. +cloud-init status --wait >/dev/null 2>&1 || true +attempt=0 +until command -v mysql >/dev/null; do + attempt=$((attempt + 1)) + [ "$attempt" -le 3 ] || fail + export DEBIAN_FRONTEND=noninteractive + if ! { apt-get -o DPkg::Lock::Timeout=300 update -qq && apt-get -o DPkg::Lock::Timeout=300 install -y -qq --no-install-recommends mysql-client-core-8.0; } >/dev/null 2>&1; then + sleep 30 + fi +done + +stage=secret-access +deadline=$(($(date +%s) + 300)) +password="" +until [ -n "$password" ]; do + token=$(curl -sf --max-time 15 -H Metadata:true "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net" | + python3 -c 'import json, sys; print(json.load(sys.stdin)["access_token"])' 2>/dev/null) || token="" + if [ -n "$token" ]; then + password=$(curl -sf --max-time 15 -H "Authorization: Bearer $token" "https://$VAULT_NAME.vault.azure.net/secrets/$SECRET_NAME?api-version=7.4" | + python3 -c 'import json, sys; print(json.load(sys.stdin)["value"])' 2>/dev/null) || password="" + fi + if [ -z "$password" ]; then + [ "$(date +%s)" -lt "$deadline" ] || fail + sleep 10 + fi +done +token="" + +query() { + MYSQL_PWD="$password" mysql --host="$SERVER" --user=mysqladmin --database="$DATABASE" \ + --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/certs/ca-certificates.crt \ + --connect-timeout=30 --batch --skip-column-names --execute="$1" 2>/dev/null +} + +stage=mysql-connect +query "SELECT 1" >/dev/null || fail + +stage=mysql-data +if [ "$PHASE" = write ]; then + query "CREATE TABLE tre_e2e_probe (probe CHAR(32) NOT NULL PRIMARY KEY); INSERT INTO tre_e2e_probe (probe) VALUES ('$PROBE_ID');" >/dev/null || fail +fi +count=$(query "SELECT COUNT(*) FROM tre_e2e_probe WHERE probe = '$PROBE_ID'") || fail +password="" +[ "$count" = 1 ] || fail +echo "MYSQL_PROBE_RESULT={\"phase\": \"$PHASE\", \"probe\": \"$PROBE_ID\", \"row_count\": $count, \"private_dns\": true}" diff --git a/e2e_tests/resources/openai.py b/e2e_tests/resources/openai.py new file mode 100644 index 0000000000..3465f142c8 --- /dev/null +++ b/e2e_tests/resources/openai.py @@ -0,0 +1,114 @@ +"""Check the prerequisites for the private OpenAI lifecycle case.""" + +from datetime import datetime, timezone +import math +from urllib.parse import parse_qsl, urlsplit + +COGNITIVE_API = "2024-10-01" +MODEL = "gpt-5.1 | 2025-11-13" +CAPACITY = 1 + + +async def list_values(arm, resource_id): + """Read all ARM pages, keeping credentials on the original collection.""" + values, seen, query = [], set(), {} + for _ in range(100): + page = await arm.request("GET", resource_id, COGNITIVE_API, query=query) + if not isinstance(page.get("value"), list): + raise ValueError("Azure returned an invalid model or quota list") + values.extend(page["value"]) + link = page.get("nextLink") + if not link: + return values + parsed = urlsplit(link) + pairs = parse_qsl(parsed.query, keep_blank_values=True) + query = dict(pairs) + if ( + parsed.scheme != "https" + or parsed.netloc != "management.azure.com" + or parsed.path != resource_id + or parsed.fragment + or len(query) != len(pairs) + or query.get("api-version") != COGNITIVE_API + or link in seen + ): + raise ValueError("Azure returned an invalid model or quota continuation link") + seen.add(link) + query.pop("api-version") + raise ValueError("Azure model or quota list exceeded the page limit") + + +def unexpired(value, now): + if value is None: + return True + try: + deadline = datetime.fromisoformat(value.replace("Z", "+00:00")) + return deadline.tzinfo is not None and deadline > now + except (ValueError, TypeError, AttributeError): + return False + + +def supported_sku(models, now): + """Match the exact approved model and regional Standard deployment type.""" + name, version = (part.strip() for part in MODEL.split("|")) + for entry in models: + model = entry.get("model", {}) + if ( + entry.get("kind") != "OpenAI" + or entry.get("skuName") != "S0" + or model.get("format") != "OpenAI" + or model.get("name") != name + or model.get("version") != version + or model.get("lifecycleStatus") != "GenerallyAvailable" + or model.get("capabilities", {}).get("chatCompletion") != "true" + or not unexpired(model.get("deprecation", {}).get("inference"), now) + ): + continue + for sku in model.get("skus", []): + usage = sku.get("usageName", "") + if ( + sku.get("name") == "Standard" + and usage + and "finetune" not in usage.lower() + and unexpired(sku.get("deprecationDate"), now) + ): + return sku + raise ValueError(f"OpenAI prerequisite failed: {MODEL} has no current regional Standard chat deployment") + + +def check_capacity(sku, usages): + capacity = sku.get("capacity", {}) + minimum, maximum, step = capacity.get("minimum", 1), capacity.get("maximum", CAPACITY), capacity.get("step", 1) + if ( + any(type(value) is not int for value in (minimum, maximum, step)) + or step < 1 + or not minimum <= CAPACITY <= maximum + or (CAPACITY - minimum) % step + or ("allowedValues" in capacity and CAPACITY not in capacity["allowedValues"]) + ): + raise ValueError("OpenAI prerequisite failed: Standard does not support one capacity unit") + for usage in usages: + if usage.get("name", {}).get("value") != sku["usageName"]: + continue + current, limit = usage.get("currentValue"), usage.get("limit") + if ( + type(current) not in (int, float) + or type(limit) not in (int, float) + or not math.isfinite(current) + or not math.isfinite(limit) + or current < 0 + or limit < 0 + or usage.get("unit") != "Count" + or usage.get("status") in ("Blocked", "InOverage", "Unknown") + ): + raise ValueError("OpenAI prerequisite failed: quota values are invalid or blocked") + if limit - current < CAPACITY: + raise ValueError("OpenAI prerequisite failed: Standard model quota has no free capacity unit") + return + raise ValueError("OpenAI prerequisite failed: the Standard model quota was not returned") + + +async def check_prerequisites(arm, subscription_id, location): + collection = f"/subscriptions/{subscription_id}/providers/Microsoft.CognitiveServices/locations/{location}" + sku = supported_sku(await list_values(arm, collection + "/models"), datetime.now(timezone.utc)) + check_capacity(sku, await list_values(arm, collection + "/usages")) diff --git a/e2e_tests/resources/sql_lifecycle.py b/e2e_tests/resources/sql_lifecycle.py index b574849fca..8b19ae9986 100644 --- a/e2e_tests/resources/sql_lifecycle.py +++ b/e2e_tests/resources/sql_lifecycle.py @@ -37,10 +37,10 @@ } -def lifecycle_deadlines(): +def lifecycle_deadlines(variable="SQL_VALIDATION_DEADLINE"): """Use the job's absolute deadline, including time spent starting the container.""" loop = asyncio.get_running_loop() - configured = os.environ.get("SQL_VALIDATION_DEADLINE", "") + configured = os.environ.get(variable, "") deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS if not math.isfinite(deadline): raise ValueError("SQL validation deadline must be finite") @@ -60,6 +60,11 @@ def __init__(self, verify, finish): def own(self, name, callback, *args, **kwargs): self.finalisers.append((name, partial(callback, *args, **kwargs))) + async def enter(self, name, context): + """Enter a prerequisite context now and exit it in order with the owned resources.""" + await context.__aenter__() + self.own(name, context.__aexit__, None, None, None) + async def setup(self): # Keep these resources inside the SQL budget rather than session fixtures. workspace_path, workspace_id = await create_or_get_test_workspace( @@ -113,8 +118,8 @@ async def close(self): @asynccontextmanager -async def sql_lifecycle(verify): - work_finish, finish = lifecycle_deadlines() +async def sql_lifecycle(verify, deadline_variable="SQL_VALIDATION_DEADLINE"): + work_finish, finish = lifecycle_deadlines(deadline_variable) resources = SQLResources(verify, finish) original = None # Accepted-create recovery inherits the same absolute limit. It cannot add diff --git a/e2e_tests/resources/sql_probe.py b/e2e_tests/resources/sql_probe.py index e72333139f..81b3e527f5 100644 --- a/e2e_tests/resources/sql_probe.py +++ b/e2e_tests/resources/sql_probe.py @@ -30,11 +30,11 @@ def __init__(self, client, credential): self.client = client self.credential = credential - async def request(self, method, resource_id, api_version, *, body=None, missing_ok=False, expand=False): + async def request(self, method, resource_id, api_version, *, body=None, missing_ok=False, expand=False, query=None): if not resource_id.startswith("/subscriptions/") or any(c in resource_id for c in "?#"): raise ValueError("Expected an ARM resource ID") token = await asyncio.to_thread(self.credential.get_token, f"{ARM}/.default") - params = {"api-version": api_version} + params = {**(query or {}), "api-version": api_version} if expand: params["$expand"] = "instanceView" response = await self.client.request( diff --git a/e2e_tests/resources/vm_actions.py b/e2e_tests/resources/vm_actions.py new file mode 100644 index 0000000000..43ae2ec99d --- /dev/null +++ b/e2e_tests/resources/vm_actions.py @@ -0,0 +1,55 @@ +"""Invoke user resource actions and read the VM state that they change.""" + +from httpx import AsyncClient +from starlette import status + +from e2e_tests.bundle_evidence import record_resource +from e2e_tests.helpers import assert_status, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.deployment import check_deployment +from e2e_tests.resources.resource import TIMEOUT, wait_for +from e2e_tests.resources.sql_probe import COMPUTE_API + +KEY_VAULT_API = "2023-07-01" +ACTION_TERMINAL = (strings.RESOURCE_ACTION_STATUS_SUCCEEDED, strings.RESOURCE_ACTION_STATUS_FAILED) + + +async def action_done(client, operation_endpoint, access_token): + action_status, message, steps = await check_deployment(client, operation_endpoint, access_token) + return action_status in ACTION_TERMINAL, action_status, message, steps + + +async def invoke_action(resource_path, action, access_token, verify): + """Run one custom action and fail unless it succeeds.""" + async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: + response = await client.post( + get_full_endpoint(f"/api{resource_path}/invoke-action"), + headers=get_auth_header(access_token), + params={"action": action}, + ) + assert_status(response, [status.HTTP_202_ACCEPTED], f"The {action} action could not be sent") + operation = response.json()["operation"] + if operation["resourcePath"] != resource_path: + raise ValueError("The action returned an operation for a different resource") + record_resource(None, operation, "POST") + await wait_for( + action_done, + client, + response.headers["Location"], + access_token, + [strings.RESOURCE_ACTION_STATUS_FAILED], + ) + + +async def power_state(arm, vm_id): + view = await arm.request("GET", f"{vm_id}/instanceView", COMPUTE_API) + states = [s["code"] for s in view.get("statuses", []) if s.get("code", "").startswith("PowerState/")] + if len(states) != 1: + raise AssertionError("The VM reported no single power state") + return states[0] + + +async def secret_version(arm, vault_id, secret_name): + """Read the current secret version through ARM. The response contains no secret value.""" + secret = await arm.request("GET", f"{vault_id}/secrets/{secret_name}", KEY_VAULT_API) + return secret["properties"]["secretUriWithVersion"].rsplit("/", 1)[1] diff --git a/e2e_tests/test_backups.py b/e2e_tests/test_backups.py index 77504ba5da..986511110d 100644 --- a/e2e_tests/test_backups.py +++ b/e2e_tests/test_backups.py @@ -1,14 +1,11 @@ import uuid import pytest -from httpx import AsyncClient from starlette import status -from e2e_tests.conftest import clean_up_test_workspace -from e2e_tests.helpers import assert_status, get_admin_token, get_auth_header, get_template +from e2e_tests.helpers import assert_status, get_admin_token, get_template from e2e_tests.resources import strings -from e2e_tests.resources.resource import post_resource -from e2e_tests.resources.workspace import get_workspace +from e2e_tests.resources.resource import get_resource, temporary_resource pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -30,33 +27,22 @@ async def test_create_base_workspace_with_backup_setting(enable_backup, expected template_properties = response.json().get("properties", {}) assert "enable_backup" in template_properties - workspace_path = "" - workspace_id = "" - - try: - properties = { - "display_name": f"E2E Backup Workspace {uuid.uuid4().hex[:8]}", - "description": "Base workspace for backup E2E tests", - "auth_type": "Automatic", - "address_space_size": "small", - "enable_backup": enable_backup, - } - if enable_backup: - properties["delete_backups_on_uninstall"] = True - payload = { - "templateName": strings.BASE_WORKSPACE, - "properties": properties, - } - - workspace_path, workspace_id = await post_resource( - payload, - strings.API_WORKSPACES, - access_token=admin_token, - verify=verify, - ) - - async with AsyncClient(verify=verify) as client: - workspace = await get_workspace(client, workspace_id, get_auth_header(admin_token)) + properties = { + "display_name": f"E2E Backup Workspace {uuid.uuid4().hex[:8]}", + "description": "Base workspace for backup E2E tests", + "auth_type": "Automatic", + "address_space_size": "small", + "enable_backup": enable_backup, + } + if enable_backup: + properties["delete_backups_on_uninstall"] = True + payload = { + "templateName": strings.BASE_WORKSPACE, + "properties": properties, + } + + async with temporary_resource(payload, strings.API_WORKSPACES, admin_token, verify) as workspace_path: + workspace = (await get_resource(f"/api{workspace_path}", admin_token, verify))["workspace"] assert workspace["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED @@ -73,7 +59,3 @@ async def test_create_base_workspace_with_backup_setting(enable_backup, expected assert value, f"Expected non-empty workspace property '{prop}' when backup is enabled" else: assert not value, f"Expected empty workspace property '{prop}' when backup is disabled" - - finally: - if workspace_path: - await clean_up_test_workspace(pre_created_workspace_id="", workspace_path=workspace_path, verify=verify) diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index 1cc4ada88f..99141ddfb5 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -1,21 +1,27 @@ -"""Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap.""" +"""Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap and runs its actions.""" + +import logging +import os import pytest +from e2e_tests import config from e2e_tests.conftest import get_workspace_owner_token from e2e_tests.resources import strings -from e2e_tests.resources.resource import temporary_resource +from e2e_tests.resources.resource import get_resource, temporary_resource from e2e_tests.resources.nexus import nexus_prerequisites +from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.resources.vm_actions import invoke_action, power_state, secret_version pytestmark = pytest.mark.asyncio(loop_scope="session") +LOGGER = logging.getLogger(__name__) @pytest.mark.linux_vm @pytest.mark.timeout(150 * 60) async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_service, verify): _, workspace_id, workspace_service_path, _ = setup_test_workspace_and_guacamole_service - workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) payload = { "templateName": strings.GUACAMOLE_LINUX_USER_RESOURCE, "properties": { @@ -29,7 +35,33 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv } async with nexus_prerequisites(verify): + workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) async with temporary_resource( payload, f"/api{workspace_service_path}/{strings.API_USER_RESOURCES}", workspace_owner_token, verify - ): - pass + ) as vm_path: + await check_vm_actions(vm_path, workspace_id, verify) + + +async def check_vm_actions(vm_path, workspace_id, verify): + """Run the VM actions and read their effect from Azure.""" + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + # The Resource Manager client of these checks supports AzureCloud only. Run the actions without the reads. + LOGGER.warning("Not reading the VM power state and secret version outside AzureCloud") + for action in ("stop", "start", "reset_password"): + await invoke_action(vm_path, action, await get_workspace_owner_token(workspace_id, verify), verify) + return + token = await get_workspace_owner_token(workspace_id, verify) + vm_id = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"]["azure_resource_id"] + group = vm_id.split("/providers/", 1)[0] + vault = f"kv-{f'{config.TRE_ID}-ws-{workspace_id[-4:]}'[-20:]}".lower() + vault_id = f"{group}/providers/Microsoft.KeyVault/vaults/{vault}" + secret_name = f"{vm_id.rsplit('/', 1)[1]}-admin-credentials" + async with arm_client() as arm: + assert await power_state(arm, vm_id) == "PowerState/running" + await invoke_action(vm_path, "stop", await get_workspace_owner_token(workspace_id, verify), verify) + assert await power_state(arm, vm_id) == "PowerState/deallocated" + await invoke_action(vm_path, "start", await get_workspace_owner_token(workspace_id, verify), verify) + assert await power_state(arm, vm_id) == "PowerState/running" + version = await secret_version(arm, vault_id, secret_name) + await invoke_action(vm_path, "reset_password", await get_workspace_owner_token(workspace_id, verify), verify) + assert await secret_version(arm, vault_id, secret_name) != version, "reset_password kept the secret" diff --git a/e2e_tests/test_mysql.py b/e2e_tests/test_mysql.py new file mode 100644 index 0000000000..901fd5b1cc --- /dev/null +++ b/e2e_tests/test_mysql.py @@ -0,0 +1,105 @@ +"""Exercise MySQL data access and a supported SKU change privately.""" + +import asyncio +import logging +import os +from uuid import uuid4 + +import pytest + +from e2e_tests import config +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.resources import strings +from e2e_tests.resources.mysql_probe import run_probe +from e2e_tests.resources.nexus import nexus_prerequisites +from e2e_tests.resources.resource import get_resource, post_resource +from e2e_tests.resources.sql_lifecycle import sql_lifecycle +from e2e_tests.resources.sql_probe import arm_client, COMPUTE_API, secret_access, validate_vm + +LOGGER = logging.getLogger(__name__) +MYSQL_API = "2023-12-30" +DATABASE = "tredb" +MYSQL_PAYLOAD = { + "templateName": strings.MYSQL_SERVICE, + "properties": { + "display_name": "MySQL lifecycle test", + "description": "Private MySQL query and SKU change validation", + "sql_sku": "B | 4GB 2vCores", + "db_name": DATABASE, + }, +} +VM_PAYLOAD = { + "templateName": strings.GUACAMOLE_LINUX_USER_RESOURCE, + "properties": { + "display_name": "MySQL test client", + "description": "Temporary private MySQL validation client", + "os_image": "Ubuntu 22.04 LTS", + "vm_size": "2 CPU | 8GB RAM", + # CI uses a service principal, so the bundle cannot derive a username from its owner. + "admin_username": "researcher", + }, +} +SKUS = {"B | 4GB 2vCores": "Standard_B2s", "GP | 8GB 2vCores": "Standard_D2ds_v4"} + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.mysql_validation +@pytest.mark.timeout(285 * 60) +async def test_private_mysql_query_survives_sku_change(verify): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("The MySQL probe currently supports AzureCloud only") + async with sql_lifecycle(verify, deadline_variable="MYSQL_VALIDATION_DEADLINE") as resources: + workspace_path, workspace_id, guacamole_path, guacamole_id = await resources.setup() + # The Linux VM installs its packages through Nexus. + async with asyncio.timeout(60 * 60): + await resources.enter("Nexus prerequisites", nexus_prerequisites(verify)) + token = await get_workspace_owner_token(workspace_id, verify) + mysql_path, mysql_id = await resources.create(MYSQL_PAYLOAD, f"/api{workspace_path}/workspace-services", token) + vm_path, vm_id = await resources.create(VM_PAYLOAD, f"/api{guacamole_path}/user-resources", token) + mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] + vm_properties = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"] + subscription_id = vm_properties.get("workspace_subscription_id") or os.environ["ARM_SUBSCRIPTION_ID"] + server = f"mysql-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{mysql_id[-4:]}" + assert mysql["properties"]["mysql_fqdn"] == f"{server}.mysql.database.azure.com" + vault = f"kv-{f'{config.TRE_ID}-ws-{workspace_id[-4:]}'[-20:]}".lower() + azure_vm_id = vm_properties["azure_resource_id"] + async with arm_client() as arm: + vm = await arm.request("GET", azure_vm_id, COMPUTE_API) + group = validate_vm(vm, azure_vm_id, subscription_id, config.TRE_ID, workspace_id, guacamole_id, vm_id) + server_id = f"{group}/providers/Microsoft.DBforMySQL/flexibleServers/{server}" + server_record = await arm.request("GET", server_id, MYSQL_API) + assert server_record["sku"]["name"] == SKUS["B | 4GB 2vCores"] + assert server_record["properties"]["network"]["publicNetworkAccess"] == "Disabled" + secure = await arm.request("GET", f"{server_id}/configurations/require_secure_transport", MYSQL_API) + assert secure["properties"]["value"].upper() == "ON" + vault_id = f"{group}/providers/Microsoft.KeyVault/vaults/{vault}" + secret_name = f"{server}-administrator-password" + probe_id = uuid4().hex + async with secret_access(arm, vault_id, secret_name, vm["identity"]["principalId"], subscription_id): + probe = dict( + server=f"{server}.mysql.database.azure.com", + vault_name=vault, + secret_name=secret_name, + database=DATABASE, + probe_id=probe_id, + ) + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="write") + LOGGER.info("MySQL query passed through private DNS on the Burstable SKU") + mysql = (await get_resource(f"/api{mysql_path}", token, verify))["workspaceService"] + async with asyncio.timeout(60 * 60): + await post_resource( + {"properties": {"sql_sku": "GP | 8GB 2vCores"}}, + f"/api{mysql_path}", + token, + verify, + method="PATCH", + etag=mysql["_etag"], + ) + server_record = await arm.request("GET", server_id, MYSQL_API) + assert server_record["sku"]["name"] == SKUS["GP | 8GB 2vCores"] + # The SKU change must keep the network and TLS settings. + assert server_record["properties"]["network"]["publicNetworkAccess"] == "Disabled" + secure = await arm.request("GET", f"{server_id}/configurations/require_secure_transport", MYSQL_API) + assert secure["properties"]["value"].upper() == "ON" + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="read") + LOGGER.info("MySQL data survived the change to the General Purpose SKU") diff --git a/e2e_tests/test_openai.py b/e2e_tests/test_openai.py new file mode 100644 index 0000000000..e046e53159 --- /dev/null +++ b/e2e_tests/test_openai.py @@ -0,0 +1,93 @@ +"""Validate private OpenAI deployment after checking model and quota prerequisites.""" + +import asyncio +import os +import re + +import pytest +from httpx import AsyncClient + +from e2e_tests import config +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.helpers import TIMEOUT, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.openai import CAPACITY, COGNITIVE_API, MODEL, check_prerequisites +from e2e_tests.resources.resource import get_resource, temporary_resource +from e2e_tests.resources.sql_probe import arm_client + +CREATE_TIMEOUT_SECONDS = 45 * 60 +VERIFY_TIMEOUT_SECONDS = 5 * 60 + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.workspace_services +@pytest.mark.openai_validation +async def test_private_openai_lifecycle(verify, setup_test_workspace): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("The OpenAI prerequisite check currently supports AzureCloud only") + workspace_path, workspace_id = setup_test_workspace + token = await get_workspace_owner_token(workspace_id, verify) + async with arm_client() as arm: + async with asyncio.timeout(5 * 60): + workspace = (await get_resource(f"/api{workspace_path}", token, verify))["workspace"] + subscription = os.environ["ARM_SUBSCRIPTION_ID"] + if ( + workspace["properties"].get("workspace_subscription_id") or subscription + ).lower() != subscription.lower(): + raise ValueError("This OpenAI bundle requires the workspace and core in the same subscription") + if not re.fullmatch(r"[a-fA-F0-9-]{36}", subscription) or not re.fullmatch(r"[a-z0-9-]+", config.TRE_ID): + raise ValueError("Invalid subscription or TRE identifier for the OpenAI prerequisite check") + group = f"/subscriptions/{subscription}/resourceGroups/rg-{config.TRE_ID}-ws-{workspace_id[-4:]}" + region = (await arm.request("GET", group, "2022-09-01"))["location"] + if not re.fullmatch(r"[a-z0-9]+", region): + raise ValueError("Invalid workspace region for the OpenAI prerequisite check") + await check_prerequisites(arm, subscription, region) + payload = { + "templateName": strings.OPENAI_SERVICE, + "properties": { + "display_name": "Private OpenAI lifecycle test", + "description": "OpenAI deployment and cleanup validation", + "openai_model": MODEL, + "is_exposed_externally": False, + }, + } + # Cancel the creation deadline after deployment. Verification and cleanup + # each have their own budget, even if creation uses its full allowance. + async with asyncio.timeout(CREATE_TIMEOUT_SECONDS) as creation_deadline: + async with temporary_resource( + payload, f"/api{workspace_path}/workspace-services", token, verify + ) as service_path: + creation_deadline.reschedule(None) + async with asyncio.timeout(VERIFY_TIMEOUT_SECONDS): + service = (await get_resource(f"/api{service_path}", token, verify))["workspaceService"] + assert service["templateName"] == strings.OPENAI_SERVICE + assert service["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED + properties = service["properties"] + assert properties["openai_model"] == MODEL + assert properties["is_exposed_externally"] is False + account_name = f"openai-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{service['id'][-4:]}" + account_id = f"{group}/providers/Microsoft.CognitiveServices/accounts/{account_name}" + account = await arm.request("GET", account_id, COGNITIVE_API) + assert account["properties"]["provisioningState"] == "Succeeded" + assert account["properties"]["publicNetworkAccess"] == "Disabled" + assert properties["openai_fqdn"] == account["properties"]["endpoint"] + name, version = (part.strip() for part in MODEL.split("|")) + deployment_name = ( + f"openai-{name}-{version}-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{service['id'][-4:]}" + ) + assert properties["openai_deployment_id"] == deployment_name + deployment = await arm.request("GET", f"{account_id}/deployments/{deployment_name}", COGNITIVE_API) + assert deployment["properties"]["provisioningState"] == "Succeeded" + model = deployment["properties"]["model"] + assert (model["format"], model["name"], model["version"]) == ("OpenAI", name, version) + assert deployment["sku"]["name"] == "Standard" + assert deployment["sku"]["capacity"] == CAPACITY + async with asyncio.timeout(5 * 60): + async with AsyncClient(verify=verify) as client: + response = await client.get( + get_full_endpoint(f"/api{service_path}"), headers=get_auth_header(token), timeout=TIMEOUT + ) + assert response.status_code == 404, f"OpenAI service remains visible: HTTP {response.status_code}" + assert await arm.request("GET", account_id, COGNITIVE_API, missing_ok=True) is None, ( + "OpenAI account remains active after uninstall" + ) diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index fbb4704278..3ce22c5614 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -13,7 +13,6 @@ strings.MYSQL_SERVICE, strings.HEALTH_SERVICE, strings.AZURESQL_SERVICE, - strings.OPENAI_SERVICE, ] diff --git a/e2e_tests/unit_tests/test_backup_registration.py b/e2e_tests/unit_tests/test_backup_registration.py new file mode 100644 index 0000000000..d3d2e13779 --- /dev/null +++ b/e2e_tests/unit_tests/test_backup_registration.py @@ -0,0 +1,299 @@ +"""Run the backup registration warm-up from the base workspace Terraform against a fake Azure CLI.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +STORAGE = ROOT / "templates/workspaces/base/terraform/storage.tf" +CONTAINER = "/vault/backupFabrics/Azure/protectionContainers/container?api-version=2023-04-01" +OPERATION = "https://management.azure.com/vault/operations/1" + +FAKE_AZ = """import json,os,sys +args=sys.argv[1:] +with open(os.environ['CALLS'],'a') as f:f.write(json.dumps(args)+'\\n') +state=json.load(open(os.environ['STATE'])) +def save():json.dump(state,open(os.environ['STATE'],'w')) +if args[0]!='rest':sys.exit(0) +method=args[args.index('--method')+1];url=args[args.index('--url')+1] +if url==os.environ['OPERATION']: + outcome=state['outcomes'].pop(0) if len(state['outcomes'])>1 else state['outcomes'][0] + if outcome=='ERROR':save();sys.exit(1) + print(outcome) + state['registered']=outcome=='Succeeded';save();sys.exit(0) +if method=='get': + if state['get_errors']: + state['get_errors']-=1;save();print('ERROR: (TooManyRequests) The request was throttled.',file=sys.stderr);sys.exit(1) + if state['registered'] and '--query' in args: + statuses=state['registration_statuses'];status=statuses.pop(0) if len(statuses)>1 else statuses[0];save();print(status);sys.exit(0) + if state['registered']:sys.exit(0) + print('ERROR: Not Found({"error":{"code":"BMSUserErrorContainerObjectNotFound"}})',file=sys.stderr);sys.exit(1) +if method=='put': + if state['put_lost']: + if state['put_registers']:state['registered']=True + save();print('ERROR: (ConnectionResetError) Connection aborted.',file=sys.stderr);sys.exit(1) + if state['put_failures']: + state['put_failures']-=1;save();print('INFO: cli.azure.cli.core.util: Response status: 500',file=sys.stderr);print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) + if state['put_registers']:state['registered']=True;save() + if state['header']:print("INFO: cli.azure.cli.core.util: '"+state['header_name']+"': '"+os.environ['OPERATION']+"'",file=sys.stderr) + sys.exit(0) +if method=='delete': + if state['delete_failures']: + state['delete_failures']-=1;save();print('ERROR: (CloudInternalError) Microsoft Azure Backup encountered an internal error.',file=sys.stderr);sys.exit(1) + state['registered']=False;save();sys.exit(0) +""" + + +def warm_up_command(): + source = STORAGE.read_text() + block = source.split('resource "terraform_data" "backup_registration_warm_up" {', 1)[1] + command = block.split("< 1 else state[0] + if isinstance(state, Exception): + raise state + return Response( + response_status, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourcePath": self.resource_path, + "resourceId": "backup-workspace", + "status": state, + "message": "Mock operation outcome", + "steps": [], + } + }, + ) + + def create_client(**_): + return AsyncClient(transport=MockTransport(handle)) + + self.enterContext(patch.object(helpers, "AsyncClient", side_effect=create_client)) + self.enterContext(patch.object(resource, "AsyncClient", side_effect=create_client)) + + def assert_deleted(self): + self.assertEqual([r.method for r in self.requests if r.method != "GET"], ["POST", "PATCH", "DELETE"]) + self.assertEqual(self.requests[-1].url.path, "/api/operations/delete") + self.assertEqual(self.requests[-2].url.path, f"/api{self.resource_path}") + + async def test_success_cleans_up_both_backup_settings(self): + for enabled in [True, False]: + with self.subTest(enabled=enabled): + self.requests = [] + self.mock_api(enabled=enabled) + await backups.test_create_base_workspace_with_backup_setting(enabled, enabled, True) + self.assert_deleted() + + async def test_failed_deployment_cleans_up_even_if_disable_fails(self): + self.mock_api( + install=strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, disable=strings.RESOURCE_STATUS_UPDATING_FAILED + ) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + self.assertEqual( + [r.method for r in self.requests], ["POST", "GET", "GET", "PATCH", "GET", "GET", "DELETE", "GET"] + ) + + async def test_output_assertion_failure_still_cleans_up(self): + self.mock_api(missing_output=True) + with self.assertRaisesRegex(AssertionError, "fileshare_backup_policy_id"): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + + async def test_cleanup_failure_preserves_original_polling_error(self): + failure = RuntimeError("backup registration polling failed") + self.mock_api( + install=[failure, strings.RESOURCE_STATUS_DEPLOYMENT_FAILED], delete=strings.RESOURCE_STATUS_DELETING_FAILED + ) + with self.assertRaises(RuntimeError) as caught: + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertIs(caught.exception, failure) + self.assertIn( + "Cleanup of /workspaces/backup-workspace after /api/operations/install also failed", + caught.exception.__notes__[0], + ) + self.assert_deleted() + + async def test_unreadable_install_operation_preserves_resource_and_original_error(self): + failure = RuntimeError("backup registration polling failed") + self.mock_api(install=failure) + with self.assertRaises(RuntimeError) as caught: + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertIs(caught.exception, failure) + self.assertIn("also failed", caught.exception.__notes__[0]) + self.assertEqual([r.method for r in self.requests], ["POST", "GET", "GET"]) + + async def test_cleanup_failure_fails_a_successful_deployment(self): + self.mock_api(delete=strings.RESOURCE_STATUS_DELETING_FAILED) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assert_deleted() + + async def test_rejected_creation_does_not_attempt_cleanup(self): + self.mock_api(post_status=403) + with self.assertRaises(AssertionError): + await backups.test_create_base_workspace_with_backup_setting(True, True, True) + self.assertEqual([r.method for r in self.requests], ["POST"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index fdca790ea5..2adf50baa1 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -3,6 +3,7 @@ import json import os from pathlib import Path +import re import subprocess import sys import tempfile @@ -210,6 +211,17 @@ def test_unavailable_bundle_fails_with_its_coverage_gap(self): self.assertIn("#5127", report["error"]) self.assertEqual(report["collected_tests"], []) + def test_openai_selects_the_guarded_case_and_reports_remaining_coverage(self): + result, report = self.run_selector("--bundle", "tre-workspace-service-openai", "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_openai.py::test_private_openai_lifecycle"]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + manifest = (ROOT / "templates/workspace_services/openai/porter.yaml").read_text() + version = re.search(r"^version: (\S+)$", manifest, re.MULTILINE).group(1) + self.assertEqual(report["source_bundle_version"], version) + self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + def test_marker_and_parallel_worker_inputs_are_rejected(self): for extra in ({"TEST_BUNDLE_MARKER": "workspace_services"}, {"E2E_TESTS_NUMBER_PROCESSES": "2"}): with self.subTest(environment=extra): diff --git a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py index e615f7a047..7f99369cd7 100644 --- a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py +++ b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py @@ -1,9 +1,10 @@ """Exercise Linux VM cleanup through the real polling and deletion helpers.""" import json +import os import unittest from pathlib import Path -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch from httpx import AsyncClient, MockTransport, Response from jsonschema import validate @@ -18,6 +19,7 @@ def setUp(self): self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) self.enterContext(patch.object(linuxvm, "get_workspace_owner_token", new=AsyncMock(return_value="token"))) self.enterContext(patch.object(nexus, "get_admin_token", new=AsyncMock(return_value="admin-token"))) + self.actions = self.enterContext(patch.object(linuxvm, "check_vm_actions", new=AsyncMock())) self.fixture = ("/workspaces/ws", "ws", "/workspaces/ws/workspace-services/service", "service") self.resource_path = self.fixture[2] + "/user-resources/linux" self.requests = [] @@ -112,6 +114,14 @@ async def test_success_uses_explicit_username_and_cleans_up_vm(self): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) payload = json.loads(self.requests[0].content) self.assertEqual(payload["properties"]["admin_username"], "researcher") + self.actions.assert_awaited_once_with(self.resource_path, "ws", True) + self.assert_deleted() + + async def test_failed_action_still_cleans_up_vm(self): + self.mock_api(strings.RESOURCE_STATUS_DEPLOYED) + self.actions.side_effect = AssertionError("stop failed") + with self.assertRaisesRegex(AssertionError, "stop failed"): + await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assert_deleted() async def test_failed_bootstrap_cleans_up_after_failed_disable(self): @@ -121,6 +131,7 @@ async def test_failed_bootstrap_cleans_up_after_failed_disable(self): with self.assertRaises(AssertionError): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assert_deleted(failed_disable=True, failed_create=True) + self.actions.assert_not_awaited() async def test_failed_disable_does_not_delete_an_enabled_resource(self): self.mock_api( @@ -177,5 +188,20 @@ async def test_successful_bootstrap_still_requires_successful_disable(self): self.assertEqual([request.method for request in self.requests], ["POST", "GET", "PATCH", "GET"]) +class LinuxVmCloudTests(unittest.IsolatedAsyncioTestCase): + async def test_other_clouds_run_the_actions_without_azure_reads(self): + invoke = AsyncMock() + arm = Mock(side_effect=AssertionError("The public-cloud client must not run outside AzureCloud")) + with ( + patch.dict(os.environ, {"AZURE_ENVIRONMENT": "AzureUSGovernment"}), + patch.object(linuxvm, "invoke_action", invoke), + patch.object(linuxvm, "arm_client", arm), + patch.object(linuxvm, "get_workspace_owner_token", new=AsyncMock(return_value="token")), + ): + await linuxvm.check_vm_actions("/workspaces/ws/vm", "ws", True) + self.assertEqual([call.args[1] for call in invoke.await_args_list], ["stop", "start", "reset_password"]) + arm.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/e2e_tests/unit_tests/test_mysql_bundle_contract.py b/e2e_tests/unit_tests/test_mysql_bundle_contract.py new file mode 100644 index 0000000000..73b88897ec --- /dev/null +++ b/e2e_tests/unit_tests/test_mysql_bundle_contract.py @@ -0,0 +1,24 @@ +"""Check that the MySQL server accepts private endpoint connections only.""" + +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +BUNDLE = ROOT / "templates/workspace_services/mysql/terraform" + + +class MySQLBundleContractTests(unittest.TestCase): + def test_server_disables_public_network_access(self): + source = (BUNDLE / "mysql.tf").read_text() + server = source.split('resource "azurerm_mysql_flexible_server" "mysql" {', 1)[1].split("\n}", 1)[0] + self.assertRegex(server, r'\n public_network_access\s*=\s*"Disabled"\n') + + def test_provider_supports_the_public_network_access_argument(self): + # azurerm added public_network_access for MySQL flexible servers in the 4.x releases. + main = (BUNDLE / "main.tf").read_text() + version = re.search(r'source\s*=\s*"hashicorp/azurerm"\s*\n\s*version\s*=\s*"=?\s*(\d+)\.', main) + self.assertIsNotNone(version) + self.assertGreaterEqual(int(version.group(1)), 4) + lock = (BUNDLE / ".terraform.lock.hcl").read_text() + self.assertRegex(lock, r'provider "registry\.terraform\.io/hashicorp/azurerm" \{\n version = "4\.') diff --git a/e2e_tests/unit_tests/test_mysql_probe.py b/e2e_tests/unit_tests/test_mysql_probe.py new file mode 100644 index 0000000000..b555ef6d99 --- /dev/null +++ b/e2e_tests/unit_tests/test_mysql_probe.py @@ -0,0 +1,180 @@ +"""Check the MySQL probe's evidence, input handling and cleanup boundaries.""" + +import json +import os +from pathlib import Path +import re +import subprocess +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from jsonschema import validate +import yaml + +from api_app.services.schema_service import enrich_user_resource_template, enrich_workspace_service_template +from e2e_tests import test_mysql as mysql +from e2e_tests.resources import mysql_probe as probe +from e2e_tests.resources import sql_lifecycle as lifecycle + + +ROOT = Path(__file__).resolve().parents[2] +VM_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.Compute/virtualMachines/linuxvm1234" +PROBE_ID = "0123456789abcdef0123456789abcdef" +VALUES = { + "SERVER": "mysql-tretest-ws-abcd-svc-1234.mysql.database.azure.com", + "VAULT_NAME": "kv-tretest-ws-abcd", + "SECRET_NAME": "mysql-tretest-ws-abcd-svc-1234-administrator-password", + "DATABASE": "tredb", + "PROBE_ID": PROBE_ID, + "PHASE": "read", +} + + +def result(output=None, state="Succeeded", code=0, **overrides): + evidence = {"phase": "read", "probe": PROBE_ID, "row_count": 1, "private_dns": True, **overrides} + return { + "properties": { + "provisioningState": "Succeeded", + "instanceView": { + "executionState": state, + "exitCode": code, + "output": output if output is not None else "MYSQL_PROBE_RESULT=" + json.dumps(evidence), + }, + } + } + + +class ProbeScriptTests(unittest.TestCase): + def test_script_sets_only_validated_values(self): + script = probe.probe_script(**VALUES) + self.assertTrue(script.startswith("#!/usr/bin/env bash\n")) + self.assertEqual(script.count("#!"), 1) + for name, value in VALUES.items(): + self.assertIn(f"{name}='{value}'\n", script) + self.assertNotIn("password=", script.split("set -euo pipefail")[0]) + + def test_unexpected_or_unsafe_values_are_rejected(self): + for name, value in ( + ("SERVER", "evil.example.com"), + ("SERVER", "a'; reboot; '.mysql.database.azure.com"), + ("DATABASE", "tredb; DROP DATABASE mysql"), + ("PROBE_ID", "abc"), + ("PHASE", "delete"), + ): + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + probe.probe_script(**{**VALUES, name: value}) + with self.assertRaises(ValueError): + probe.probe_script(**{k: v for k, v in VALUES.items() if k != "PHASE"}) + + def test_script_is_valid_bash(self): + completed = subprocess.run( + ["bash", "-n"], input=probe.probe_script(**VALUES), text=True, capture_output=True, check=False + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_script_rejects_values_that_skip_python_validation(self): + script = probe.probe_script(**VALUES).replace("PHASE='read'", "PHASE='drop'") + completed = subprocess.run(["bash", "-c", script], text=True, capture_output=True, check=False) + self.assertEqual(completed.returncode, 1) + self.assertEqual(completed.stdout.strip(), "MYSQL_PROBE_FAILED=input") + + +class ProbeEvidenceTests(unittest.TestCase): + def test_success_requires_expected_data_and_private_dns(self): + self.assertTrue(probe.probe_result(result(), "read", PROBE_ID)) + for change in ({"phase": "write"}, {"probe": "other"}, {"row_count": 0}, {"private_dns": False}): + with self.subTest(change=change), self.assertRaises(AssertionError): + probe.probe_result(result(**change), "read", PROBE_ID) + + def test_running_command_is_not_a_pass(self): + resource = result() + resource["properties"]["instanceView"] = {} + self.assertFalse(probe.probe_result(resource, "read", PROBE_ID)) + + def test_failure_reports_only_the_failed_stage(self): + failed = result(output="MYSQL_PROBE_FAILED=mysql-connect\n", state="Failed", code=1) + with self.assertRaisesRegex(AssertionError, "stage=mysql-connect"): + probe.probe_result(failed, "read", PROBE_ID) + with self.assertRaisesRegex(AssertionError, "non-zero"): + probe.probe_result(result(code=1), "read", PROBE_ID) + + def test_payloads_match_enriched_api_schemas(self): + cases = ( + (mysql.MYSQL_PAYLOAD, "mysql", enrich_workspace_service_template), + (mysql.VM_PAYLOAD, "guacamole/user_resources/guacamole-azure-linuxvm", enrich_user_resource_template), + ) + for payload, folder, enrich in cases: + with self.subTest(bundle=payload["templateName"]): + schema = json.loads( + (ROOT / "templates/workspace_services" / folder / "template_schema.json").read_text() + ) + validate(payload["properties"], enrich(SimpleNamespace(model_dump=lambda **_: schema))) + + def test_expected_skus_match_the_bundle(self): + # Terraform uses a tier prefix, for example B_Standard_B2s. ARM returns the SKU name without it. + locals_tf = (ROOT / "templates/workspace_services/mysql/terraform/locals.tf").read_text() + for option, sku in mysql.SKUS.items(): + with self.subTest(option=option): + self.assertRegex(locals_tf, rf'"{re.escape(option)}"\s*=\s*{{ value = "[A-Z]+_{sku}" }}') + + def test_deadline_precedes_container(self): + jobs = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text())["jobs"] + job = jobs["e2e_tests_bundle"] + reserve = next(s for s in job["steps"] if "MYSQL_VALIDATION_DEADLINE" in s.get("run", "")) + run = next(s for s in job["steps"] if s["name"] == "Run E2E Tests") + self.assertLess(job["steps"].index(reserve), job["steps"].index(run)) + self.assertEqual(reserve["if"], "inputs.e2eBundle == 'tre-workspace-service-mysql'") + self.assertIn("MYSQL_VALIDATION_DEADLINE=${{ env.MYSQL_VALIDATION_DEADLINE }}", run["with"]["COMMAND"]) + + +class ProbeRecoveryTests(unittest.IsolatedAsyncioTestCase): + async def test_failed_command_is_removed_and_never_counts_as_pass(self): + failed = result(output="MYSQL_PROBE_FAILED=secret-access\n", state="Failed", code=1) + arm = SimpleNamespace(request=AsyncMock(side_effect=[{}, failed]), delete=AsyncMock()) + with self.assertRaisesRegex(AssertionError, "secret-access"): + await probe.run_probe( + arm, + VM_ID, + "switzerlandnorth", + server=VALUES["SERVER"], + vault_name=VALUES["VAULT_NAME"], + secret_name=VALUES["SECRET_NAME"], + database="tredb", + probe_id=PROBE_ID, + phase="read", + ) + command = arm.request.call_args_list[0].args[1] + self.assertTrue(command.startswith(VM_ID + "/runCommands/mysql-probe-")) + arm.delete.assert_awaited_once_with(command, probe.COMPUTE_API) + body = arm.request.call_args_list[0].kwargs["body"] + self.assertNotIn("parameters", body["properties"]) + self.assertNotIn("protectedParameters", body["properties"]) + + async def test_prerequisite_context_exits_after_later_resources(self): + events = [] + + class Context: + async def __aenter__(self): + events.append("enter") + + async def __aexit__(self, *args): + events.append(("exit", args)) + + async def delete(*_, **__): + events.append("delete") + + with ( + patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE=""), + patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/mysql", "id"))), + patch.object(lifecycle, "get_resource", AsyncMock(return_value={"operations": []})), + patch.object(lifecycle, "disable_and_delete_resource", side_effect=delete), + ): + async with lifecycle.sql_lifecycle(True, deadline_variable="MYSQL_VALIDATION_DEADLINE") as owned: + await owned.enter("Nexus", Context()) + await owned.create({}, "/services", "token") + self.assertEqual(events, ["enter", "delete", ("exit", (None, None, None))]) + + async def test_mysql_deadline_variable_is_read(self): + with patch.dict(os.environ, MYSQL_VALIDATION_DEADLINE="nan"), self.assertRaisesRegex(ValueError, "finite"): + lifecycle.lifecycle_deadlines("MYSQL_VALIDATION_DEADLINE") diff --git a/e2e_tests/unit_tests/test_openai_validation.py b/e2e_tests/unit_tests/test_openai_validation.py new file mode 100644 index 0000000000..0f53f96472 --- /dev/null +++ b/e2e_tests/unit_tests/test_openai_validation.py @@ -0,0 +1,356 @@ +"""Check OpenAI prerequisite failures, evidence and owned-resource cleanup offline.""" + +import asyncio +from contextlib import asynccontextmanager +from copy import deepcopy +from datetime import datetime, timezone +import json +import os +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from httpx import AsyncClient, MockTransport, Response +from jsonschema import validate +import yaml + +from e2e_tests import test_openai as lifecycle +from e2e_tests.resources import openai, resource +from e2e_tests.resources.sql_probe import ArmClient + +NOW = datetime(2026, 10, 10, tzinfo=timezone.utc) +ROOT = Path(__file__).resolve().parents[2] +SUBSCRIPTION = "00000000-0000-4000-8000-000000000000" +WORKSPACE = "11111111-1111-4111-8111-111111111111" +SERVICE = "22222222-2222-4222-8222-222222222222" +PATH = f"/workspaces/{WORKSPACE}/workspace-services/{SERVICE}" +GROUP = f"/subscriptions/{SUBSCRIPTION}/resourceGroups/rg-tretest-ws-1111" +ACCOUNT = GROUP + "/providers/Microsoft.CognitiveServices/accounts/openai-tretest-ws-1111-svc-2222" +DEPLOYMENT = "openai-gpt-5.1-2025-11-13-tretest-ws-1111-svc-2222" + + +def model_entry(): + return { + "kind": "OpenAI", + "skuName": "S0", + "model": { + "format": "OpenAI", + "name": "gpt-5.1", + "version": "2025-11-13", + "lifecycleStatus": "GenerallyAvailable", + "capabilities": {"chatCompletion": "true"}, + "skus": [{"name": "Standard", "usageName": "OpenAI.Standard.gpt-5.1"}], + }, + } + + +def usage_entry(): + return {"name": {"value": "OpenAI.Standard.gpt-5.1"}, "currentValue": 9, "limit": 10, "unit": "Count"} + + +class ModelPrerequisiteTests(unittest.TestCase): + def test_current_exact_model_and_last_capacity_unit_are_accepted(self): + sku = openai.supported_sku([model_entry()], NOW) + openai.check_capacity(sku, [usage_entry()]) + + def test_wrong_version_preview_and_non_chat_models_are_rejected(self): + for field, value in ( + ("version", "wrong"), + ("lifecycleStatus", "Preview"), + ("capabilities", {}), + ("format", "other"), + ): + entry = model_entry() + entry["model"][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "no current regional"): + openai.supported_sku([entry], NOW) + + def test_global_fine_tuning_and_expired_standard_tiers_are_rejected(self): + cases = ( + ("name", "GlobalStandard"), + ("usageName", "OpenAI.Standard.gpt-5.1-finetune"), + ("usageName", ""), + ("deprecationDate", "2026-10-10T00:00:00Z"), + ("deprecationDate", "invalid"), + ("deprecationDate", "2027-01-01"), + ) + for key, value in cases: + entry = model_entry() + entry["model"]["skus"][0][key] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + openai.supported_sku([entry], NOW) + entry = model_entry() + entry["model"]["deprecation"] = {"inference": "2020-01-01T00:00:00Z"} + with self.assertRaises(ValueError): + openai.supported_sku([entry], NOW) + + def test_missing_exhausted_and_invalid_quota_never_pass(self): + sku = openai.supported_sku([model_entry()], NOW) + cases = [[], [dict(usage_entry(), name={"value": "GlobalStandard"})]] + for key, value in ( + ("currentValue", 10), + ("currentValue", -1), + ("limit", None), + ("limit", float("nan")), + ("limit", True), + ("status", "Blocked"), + ("status", "Unknown"), + ): + cases.append([dict(usage_entry(), **{key: value})]) + for usages in cases: + with self.subTest(usages=usages), self.assertRaises(ValueError): + openai.check_capacity(sku, usages) + + def test_model_capacity_minimum_and_step_can_prevent_deployment(self): + for capacity in ({"minimum": 10}, {"allowedValues": [2, 4]}, {"step": 0}, {"minimum": 0, "step": 2}): + sku = dict(openai.supported_sku([model_entry()], NOW), capacity=capacity) + with self.subTest(capacity=capacity), self.assertRaisesRegex(ValueError, "capacity unit"): + openai.check_capacity(sku, [usage_entry()]) + + def test_bundle_schema_and_ci_registration_agree_with_selection(self): + folder = ROOT / "templates/workspace_services/openai" + schema = json.loads((folder / "template_schema.json").read_text()) + validate({"openai_model": openai.MODEL, "is_exposed_externally": False}, schema) + self.assertEqual(schema["properties"]["openai_model"]["default"], openai.MODEL) + manifest = yaml.safe_load((folder / "porter.yaml").read_text()) + # 1.1.0 is the first version with this default model. Later versions keep it. + self.assertGreaterEqual(tuple(int(part) for part in manifest["version"].split(".")), (1, 1, 0)) + workflow = yaml.safe_load((ROOT / ".github/workflows/deploy_tre_reusable.yml").read_text()) + registrations = workflow["jobs"]["register_bundles"]["strategy"]["matrix"]["include"] + self.assertIn( + {"BUNDLE_TYPE": "workspace_service", "BUNDLE_DIR": "./templates/workspace_services/openai"}, registrations + ) + + def test_private_endpoint_waits_for_the_model_deployment(self): + source = (ROOT / "templates/workspace_services/openai/terraform/openai.tf").read_text() + endpoint = source.split('resource "azurerm_private_endpoint" "openai_private_endpoint" {', 1)[1] + endpoint = endpoint.split("\nresource ", 1)[0] + self.assertRegex(endpoint, r"depends_on\s*=\s*\[\s*azurerm_cognitive_deployment\.openai\s*\]") + + +class PaginationTests(unittest.IsolatedAsyncioTestCase): + async def test_pagination_retains_api_version_and_renews_credentials(self): + collection = f"/subscriptions/{SUBSCRIPTION}/providers/Microsoft.CognitiveServices/locations/westeurope/models" + requests = [] + + def handle(request): + requests.append(request) + self.assertEqual(request.url.params["api-version"], openai.COGNITIVE_API) + self.assertEqual(request.headers["Authorization"], "Bearer token") + if len(requests) == 1: + return Response( + 200, + json={ + "value": [1], + "nextLink": f"https://management.azure.com{collection}?api-version={openai.COGNITIVE_API}&$skiptoken=a%2Bb", + }, + ) + self.assertEqual(request.url.params["$skiptoken"], "a+b") + return Response(200, json={"value": [2]}) + + credential = Mock() + credential.get_token.return_value = SimpleNamespace(token="token") + async with AsyncClient(transport=MockTransport(handle)) as client: + self.assertEqual(await openai.list_values(ArmClient(client, credential), collection), [1, 2]) + self.assertEqual(credential.get_token.call_count, 2) + + async def test_cross_host_path_and_version_links_fail_before_next_request(self): + collection = f"/subscriptions/{SUBSCRIPTION}/models" + for link in ( + "https://example.test" + collection + "?api-version=" + openai.COGNITIVE_API, + "https://management.azure.com/another?api-version=" + openai.COGNITIVE_API, + "https://management.azure.com" + collection + "?api-version=wrong", + ): + arm = SimpleNamespace(request=AsyncMock(return_value={"value": [], "nextLink": link})) + with self.subTest(link=link), self.assertRaises(ValueError): + await openai.list_values(arm, collection) + self.assertEqual(arm.request.await_count, 1) + + async def test_repeated_continuation_is_rejected(self): + collection = f"/subscriptions/{SUBSCRIPTION}/models" + arm = SimpleNamespace( + request=AsyncMock( + return_value={ + "value": [], + "nextLink": f"https://management.azure.com{collection}?api-version={openai.COGNITIVE_API}&$skiptoken=same", + } + ) + ) + with self.assertRaises(ValueError): + await openai.list_values(arm, collection) + self.assertEqual(arm.request.await_count, 2) + + +class OpenAILifecycleTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext( + patch.dict(os.environ, {"ARM_SUBSCRIPTION_ID": SUBSCRIPTION, "AZURE_ENVIRONMENT": "AzureCloud"}) + ) + self.enterContext(patch.object(lifecycle.config, "TRE_ID", "tretest")) + self.enterContext(patch.object(lifecycle, "get_workspace_owner_token", AsyncMock(return_value="token"))) + self.preflight = self.enterContext(patch.object(lifecycle, "check_prerequisites", AsyncMock())) + self.post = self.enterContext(patch.object(resource, "post_resource", AsyncMock(return_value=(PATH, SERVICE)))) + self.cleanup = self.enterContext(patch.object(resource, "disable_and_delete_resource", AsyncMock())) + self.lookup_status = 404 + self.workspace = {"workspace": {"properties": {}}} + self.service = { + "workspaceService": { + "id": SERVICE, + "templateName": "tre-workspace-service-openai", + "deploymentStatus": "deployed", + "properties": { + "openai_model": openai.MODEL, + "is_exposed_externally": False, + "openai_fqdn": "https://openai.example.test/", + "openai_deployment_id": DEPLOYMENT, + }, + } + } + self.get = self.enterContext( + patch.object( + lifecycle, + "get_resource", + AsyncMock( + side_effect=lambda endpoint, *_: deepcopy( + self.service if "workspace-services" in endpoint else self.workspace + ) + ), + ) + ) + self.account = { + "properties": { + "provisioningState": "Succeeded", + "publicNetworkAccess": "Disabled", + "endpoint": "https://openai.example.test/", + } + } + self.deployment = { + "properties": { + "provisioningState": "Succeeded", + "model": {"format": "OpenAI", "name": "gpt-5.1", "version": "2025-11-13"}, + }, + "sku": {"name": "Standard", "capacity": 1}, + } + self.retained = False + self.group = GROUP + self.account_id = ACCOUNT + self.deployment_name = DEPLOYMENT + + async def request(method, identifier, api, **kwargs): + self.assertEqual(method, "GET") + if identifier == self.group: + return {"location": "westeurope"} + if identifier == self.account_id: + return self.account if not kwargs.get("missing_ok") or self.retained else None + if identifier == self.account_id + "/deployments/" + self.deployment_name: + return self.deployment + raise AssertionError("Unexpected ARM resource") + + self.arm = SimpleNamespace(request=AsyncMock(side_effect=request)) + + @asynccontextmanager + async def arm(): + yield self.arm + + self.enterContext(patch.object(lifecycle, "arm_client", arm)) + + def lookup(request): + self.assertEqual(request.url.path, "/api" + PATH) + return Response(self.lookup_status) + + self.enterContext( + patch.object(lifecycle, "AsyncClient", lambda **_: AsyncClient(transport=MockTransport(lookup))) + ) + self.enterContext( + patch.object(lifecycle, "get_full_endpoint", lambda endpoint: "https://tre.example.test" + endpoint) + ) + + async def run_case(self): + await lifecycle.test_private_openai_lifecycle(True, (f"/workspaces/{WORKSPACE}", WORKSPACE)) + + async def test_success_checks_actual_region_and_removes_only_the_created_service(self): + await self.run_case() + self.preflight.assert_awaited_once_with(self.arm, SUBSCRIPTION, "westeurope") + self.assertTrue(self.post.await_args.kwargs["cleanup_failed_create"]) + payload = self.post.await_args.args[0] + self.assertEqual(payload["properties"]["openai_model"], openai.MODEL) + self.assertIs(payload["properties"]["is_exposed_externally"], False) + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True) + + async def test_hyphenated_tre_id_reaches_validation_and_cleanup(self): + self.group = GROUP.replace("tretest", "tre-test") + self.account_id = ACCOUNT.replace("tretest", "tre-test") + self.deployment_name = DEPLOYMENT.replace("tretest", "tre-test") + self.service["workspaceService"]["properties"]["openai_deployment_id"] = self.deployment_name + with patch.object(lifecycle.config, "TRE_ID", "tre-test"): + await self.run_case() + self.preflight.assert_awaited_once_with(self.arm, SUBSCRIPTION, "westeurope") + self.post.assert_awaited_once() + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True) + + async def test_path_and_query_characters_are_rejected_before_arm_access(self): + for tre_id in ("tre/test", "tre?test", "tre#test", "tre%2ftest"): + with self.subTest(tre_id=tre_id), patch.object(lifecycle.config, "TRE_ID", tre_id): + with self.assertRaisesRegex(ValueError, "Invalid subscription or TRE identifier"): + await self.run_case() + self.arm.request.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_model_or_quota_failure_prevents_service_creation(self): + self.preflight.side_effect = ValueError("quota unavailable") + with self.assertRaisesRegex(ValueError, "quota unavailable"): + await self.run_case() + self.post.assert_not_awaited() + self.cleanup.assert_not_awaited() + + async def test_other_subscription_fails_before_arm_access_or_service_creation(self): + self.workspace["workspace"]["properties"]["workspace_subscription_id"] = "another" + with self.assertRaisesRegex(ValueError, "same subscription"): + await self.run_case() + self.arm.request.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_public_access_failure_still_cleans_up(self): + self.account["properties"]["publicNetworkAccess"] = "Enabled" + with self.assertRaises(AssertionError): + await self.run_case() + self.cleanup.assert_awaited_once_with("/api" + PATH, "token", True, allow_failed_disable=True) + + async def test_wrong_model_deployment_still_cleans_up(self): + self.deployment["properties"]["model"]["version"] = "other" + with self.assertRaises(AssertionError): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_normal_cleanup_can_outlast_the_creation_deadline(self): + async def cleanup(*args, **kwargs): + await asyncio.sleep(0.1) + + self.cleanup.side_effect = cleanup + with patch.object(lifecycle, "CREATE_TIMEOUT_SECONDS", 0.05): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_verification_timeout_still_cleans_up(self): + async def block_account(method, identifier, api, **kwargs): + if identifier == GROUP: + return {"location": "westeurope"} + await asyncio.Event().wait() + raise AssertionError("The blocking ARM request unexpectedly resumed") + + self.arm.request.side_effect = block_account + with patch.object(lifecycle, "VERIFY_TIMEOUT_SECONDS", 0.01), self.assertRaises(TimeoutError): + await self.run_case() + self.cleanup.assert_awaited_once() + + async def test_cleanup_failure_is_not_success(self): + self.cleanup.side_effect = RuntimeError("delete failed") + with self.assertRaisesRegex(RuntimeError, "delete failed"): + await self.run_case() + + async def test_retained_api_record_or_account_is_not_success(self): + for api_status, retained in ((200, False), (404, True)): + self.lookup_status, self.retained = api_status, retained + with self.subTest(api=api_status, retained=retained), self.assertRaises(AssertionError): + await self.run_case() diff --git a/e2e_tests/unit_tests/test_vm_actions.py b/e2e_tests/unit_tests/test_vm_actions.py new file mode 100644 index 0000000000..22d413555f --- /dev/null +++ b/e2e_tests/unit_tests/test_vm_actions.py @@ -0,0 +1,74 @@ +"""Check the user resource action helper and the VM state readers.""" + +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import Response + +from e2e_tests.resources import strings +from e2e_tests.resources import vm_actions as actions + +VM_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.Compute/virtualMachines/linuxvm1234" +VAULT_ID = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd/providers/Microsoft.KeyVault/vaults/kv-tretest-ws-abcd" +PATH = "/workspaces/ws/workspace-services/svc/user-resources/vm" + + +class ActionTests(unittest.IsolatedAsyncioTestCase): + async def test_action_success_and_failure_are_terminal(self): + for state, done in ( + (strings.RESOURCE_ACTION_STATUS_SUCCEEDED, True), + (strings.RESOURCE_ACTION_STATUS_FAILED, True), + (strings.RESOURCE_ACTION_STATUS_INVOKING, False), + (strings.RESOURCE_STATUS_DEPLOYED, False), + ): + with ( + self.subTest(state=state), + patch.object(actions, "check_deployment", AsyncMock(return_value=(state, "", []))), + ): + self.assertEqual((await actions.action_done(None, "/op", "token"))[:2], (done, state)) + + async def test_invoke_waits_for_the_action_and_rejects_another_resource(self): + def response(path): + return Response( + 202, + json={"operation": {"id": "op", "resourcePath": path, "resourceId": "vm"}}, + headers={"Location": "/api/op"}, + ) + + for path, error in ((PATH, None), ("/workspaces/ws/other", ValueError)): + post = AsyncMock(return_value=response(path)) + client = SimpleNamespace(post=post) + client_context = AsyncMock() + client_context.__aenter__.return_value = client + with ( + self.subTest(path=path), + patch.object(actions, "AsyncClient", return_value=client_context), + patch.object(actions, "record_resource"), + patch.object(actions, "wait_for", AsyncMock()) as wait, + ): + if error: + with self.assertRaises(error): + await actions.invoke_action(PATH, "stop", "token", True) + wait.assert_not_awaited() + else: + await actions.invoke_action(PATH, "stop", "token", True) + self.assertEqual(post.await_args.kwargs["params"], {"action": "stop"}) + self.assertTrue(post.await_args.args[0].endswith(f"/api{PATH}/invoke-action")) + self.assertEqual(wait.await_args.args[2], "/api/op") + self.assertEqual(wait.await_args.args[4], [strings.RESOURCE_ACTION_STATUS_FAILED]) + + async def test_power_state_needs_exactly_one_value(self): + view = {"statuses": [{"code": "ProvisioningState/succeeded"}, {"code": "PowerState/deallocated"}]} + arm = SimpleNamespace(request=AsyncMock(return_value=view)) + self.assertEqual(await actions.power_state(arm, VM_ID), "PowerState/deallocated") + self.assertEqual(arm.request.await_args.args[1], VM_ID + "/instanceView") + arm.request = AsyncMock(return_value={"statuses": [{"code": "ProvisioningState/succeeded"}]}) + with self.assertRaises(AssertionError): + await actions.power_state(arm, VM_ID) + + async def test_secret_version_comes_from_arm_metadata(self): + secret = {"properties": {"secretUriWithVersion": "https://kv.vault.azure.net/secrets/name/abc123"}} + arm = SimpleNamespace(request=AsyncMock(return_value=secret)) + self.assertEqual(await actions.secret_version(arm, VAULT_ID, "linuxvm1234-admin-credentials"), "abc123") + self.assertEqual(arm.request.await_args.args[1], VAULT_ID + "/secrets/linuxvm1234-admin-credentials") diff --git a/templates/workspace_services/mysql/porter.yaml b/templates/workspace_services/mysql/porter.yaml index cc39df9638..fab5984d88 100644 --- a/templates/workspace_services/mysql/porter.yaml +++ b/templates/workspace_services/mysql/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-mysql -version: 1.0.15 +version: 1.0.16 description: "A MySQL workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/mysql/terraform/.terraform.lock.hcl b/templates/workspace_services/mysql/terraform/.terraform.lock.hcl index 904ea2b772..86a0086e86 100644 --- a/templates/workspace_services/mysql/terraform/.terraform.lock.hcl +++ b/templates/workspace_services/mysql/terraform/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.117.0" - constraints = "3.117.0" + version = "4.27.0" + constraints = "4.27.0" hashes = [ - "h1:Ynfg+Iy7x6K8M6W1AhqXCe3wkoiqIQhROlca7C3KC3w=", - "zh:2e25f47492366821a786762369f0e0921cc9452d64bfd5075f6fdfcf1a9c6d70", - "zh:41eb34f2f7469bf3eb1019dfb0e7fc28256f809824016f4f8b9d691bf473b2ac", - "zh:48bb9c87b3d928da1abc1d3db75453c9725de4674c612daf3800160cc7145d30", - "zh:5d6b0de0bbd78943fcc65c53944ef4496329e247f434c6eab86ed051c5cea67b", - "zh:78c9f6fdb1206a89cf0e6706b4f46178169a93b6c964a4cad8a321058ccbd9b4", - "zh:793b702c352589d4360b580d4a1cf654a7439d2ad6bdb7bfea91de07bc4b0fac", - "zh:7ed687ff0a5509463a592f97431863574fe5cc80a34e395be06766215b8c6285", - "zh:955ba18789bd15592824eb426a8d0f38595bd09fffc6939c1c58933489c1a71e", - "zh:bf5949a55be0714cd9c8815d472eae4baa48ba06d0f6bf2b96775869acda8a54", - "zh:da5d31f635abd2c645ffc76d6176d73f646128e73720cc368247cc424975c127", - "zh:eed5a66d59883c9c56729b0a964a2b60d758ea7489ef3e920a6fbd48518ce5f5", + "h1:2fs47aLDaEm93ANXXVRdTjlbUBmFBZRsFjyshKoPE3o=", + "zh:0c69edea1995bd3bd9e61980757169c35bf22281b660b5c755b6cb13d08d29d2", + "zh:25b86bf7b9678371d8573983954c571696f3e64a3967133be3b835da36307106", + "zh:49921cff4f26a49bafada60cd07dabb52c5eb35231059ed928a4f4722e269c82", + "zh:4b986166531f9fd1289f01d8220519443e74888a21da512c1b841b006dad6215", + "zh:53fb65b2ca4df637f03e4748a100a7d7fc77249e307c03e294d6259cec0310f6", + "zh:5c0d021a387ca4e2a5a01da009746a08c45f08e971c10d9bda54539d7264d671", + "zh:600043f2b20dc5a45275e43f175c19fe8b6e8e9557a0c884aef018f1f63de90e", + "zh:a0284f6f38912f67bb4cb7829fda3fa75be81fea6a9b21119965c2a839430092", + "zh:a7ac0576e2069ef77557042c6b5157ded364fbd355b2f9bf7f5441622424086e", + "zh:c5db0bcafe986868e28cc6225b68b2d1cf4bf631939d260ca845f17a9aa1677d", + "zh:ce620c0eb71b1fdd925828b30cf232a869abccf1c459180f2f991c4166315251", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } diff --git a/templates/workspace_services/mysql/terraform/main.tf b/templates/workspace_services/mysql/terraform/main.tf index d5761a8686..bc80c2d3ee 100644 --- a/templates/workspace_services/mysql/terraform/main.tf +++ b/templates/workspace_services/mysql/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.117.0" + version = "= 4.27.0" } random = { source = "hashicorp/random" diff --git a/templates/workspace_services/mysql/terraform/mysql.tf b/templates/workspace_services/mysql/terraform/mysql.tf index 0b878e6541..091c0e1e8e 100644 --- a/templates/workspace_services/mysql/terraform/mysql.tf +++ b/templates/workspace_services/mysql/terraform/mysql.tf @@ -16,7 +16,9 @@ resource "azurerm_mysql_flexible_server" "mysql" { version = "8.0.21" backup_retention_days = 7 geo_redundant_backup_enabled = false - tags = local.workspace_service_tags + # Clients connect through the private endpoint only. + public_network_access = "Disabled" + tags = local.workspace_service_tags storage { size_gb = local.max_storage_gb diff --git a/templates/workspace_services/openai/porter.yaml b/templates/workspace_services/openai/porter.yaml index 0fc2de4256..2a598f58fe 100644 --- a/templates/workspace_services/openai/porter.yaml +++ b/templates/workspace_services/openai/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-openai -version: 1.0.11 +version: 1.1.0 description: "An OpenAI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/openai/template_schema.json b/templates/workspace_services/openai/template_schema.json index bc0b640ec3..be77422c97 100644 --- a/templates/workspace_services/openai/template_schema.json +++ b/templates/workspace_services/openai/template_schema.json @@ -37,7 +37,7 @@ "openai_model": { "type": "string", "title": "OpenAI Model", - "description": "Which OpenAI Model should be used? (be mindful of subscription limits)", + "description": "Select a model available with the Standard deployment type in the workspace region. Check subscription quota before deployment.", "enum": [ "gpt-35-turbo | 0301", "gpt-35-turbo | 1106", @@ -45,9 +45,10 @@ "gpt-35-turbo | 0125", "gpt-4 | 0613", "gpt-4 | turbo-2024-04-09", - "gpt-4o | 2024-05-13" + "gpt-4o | 2024-05-13", + "gpt-5.1 | 2025-11-13" ], - "default": "gpt-35-turbo | 0301", + "default": "gpt-5.1 | 2025-11-13", "updateable": true } } diff --git a/templates/workspace_services/openai/terraform/openai.tf b/templates/workspace_services/openai/terraform/openai.tf index 44a0061c9c..3926deb93c 100644 --- a/templates/workspace_services/openai/terraform/openai.tf +++ b/templates/workspace_services/openai/terraform/openai.tf @@ -24,7 +24,8 @@ resource "azurerm_cognitive_deployment" "openai" { } scale { - type = "Standard" + type = "Standard" + capacity = 1 } } @@ -47,6 +48,9 @@ resource "azurerm_private_endpoint" "openai_private_endpoint" { private_dns_zone_ids = [data.azurerm_private_dns_zone.openai.id] } + # Azure rejects the endpoint with AccountProvisioningStateInvalid while a model deployment updates the account. + depends_on = [azurerm_cognitive_deployment.openai] + lifecycle { ignore_changes = [tags] } } diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index a91f3c85c2..aa37846ef0 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.18.2 +version: 0.18.3 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 276ea55b97..b973e35c0b 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 2.12.3 +version: 2.12.4 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/terraform/storage.tf b/templates/workspaces/base/terraform/storage.tf index 93315263c0..53c45fd79e 100644 --- a/templates/workspaces/base/terraform/storage.tf +++ b/templates/workspaces/base/terraform/storage.tf @@ -166,11 +166,164 @@ resource "azurerm_private_endpoint" "stgdfspe" { } } +# Azure Backup can fail the first storage registrations in a new vault with CloudInternalError. +# This step tries up to five times and removes the first registration that succeeds. Then the +# registration below is not the first one, and Terraform creates and owns the registration. +resource "terraform_data" "backup_registration_warm_up" { + count = var.enable_backup ? 1 : 0 + + input = { + container_url = "${module.backup[0].vault_id}/backupFabrics/Azure/protectionContainers/StorageContainer;storage;${azurerm_resource_group.ws.name};${azurerm_storage_account.stg.name}?api-version=2023-04-01" + storage_account_id = azurerm_storage_account.stg.id + subscription_id = data.azurerm_client_config.current.subscription_id + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = <&1); then + echo registered + return 0 + fi + if grep -qiE 'NotFound|Not Found|was not found|does not exist' <<< "$output"; then + echo missing + return 0 + fi + if [ "$read" -lt 3 ]; then + sleep 10 + fi + done + echo "Could not read the backup registration of the storage account: $output" >&2 + return 1 +} + +# Print the registration status of the container. Read it for up to 10 minutes, until it is Registered. +# The container can exist before its registration ends, and the step must not unregister it then. +registration_status() { + local registration=Unknown + for _ in $(seq 1 60); do + registration=$(az rest --method get --url "$url" --query properties.registrationStatus --output tsv 2>/dev/null || echo Unknown) + if [ "$registration" = Registered ]; then + break + fi + sleep 10 + done + echo "$registration" +} + +state=$(container_state) +if [ "$state" = registered ]; then + # Terraform owns a registration that exists, for example when a workspace upgrades to this version. + # The step cannot tell it from a registration of an earlier warm-up, so it does not remove it. + echo "The storage account is already registered with the vault" + exit 0 +fi +body='{"properties":{"backupManagementType":"AzureStorage","containerType":"StorageContainer","sourceResourceId":"${self.input.storage_account_id}"}}' +for try in 1 2 3 4 5; do + if [ "$try" -gt 1 ]; then + sleep 60 + fi + # Keep the result of the PUT. A rejected PUT starts no registration, but an accepted PUT can. + # An error can also hide an accepted PUT, for example when the response is lost. + if put_output=$(az rest --method put --url "$url" --body "$body" --debug 2>&1); then + put_accepted=true + else + put_accepted=false + fi + operation=$(sed -n "s/.*'Azure-AsyncOperation': '\([^']*\)'.*/\1/Ip" <<< "$put_output" | head -n 1 || true) + status=Unknown + if [ -n "$operation" ]; then + # A failed status read gives Unknown, and the step reads again. Another PUT must not overlap a running attempt. + for attempt in $(seq 1 60); do + status=$(az rest --method get --url "$operation" --query status --output tsv 2>/dev/null || echo Unknown) + case "$status" in + Succeeded | Failed | Canceled | Cancelled) break ;; + esac + sleep 10 + done + case "$status" in + Succeeded | Failed | Canceled | Cancelled) ;; + *) + echo "Registration attempt $try did not end within 10 minutes. The last status was $status." >&2 + exit 1 + ;; + esac + elif [ "$put_accepted" = true ]; then + # Azure accepted the PUT, but the step has no operation to follow. Continue only when the container + # shows the status Registered. + registration=$(registration_status) + if [ "$registration" != Registered ]; then + echo "Registration attempt $try has no operation to follow, and the container did not show the status Registered within 10 minutes. The last status was $registration." >&2 + exit 1 + fi + status=Succeeded + else + # Only an error answer from Azure proves a rejection. Without an answer, Azure can still run the + # registration, so the step stops, and no other registration overlaps it. + if ! grep -qE 'Response status: [45][0-9]{2}' <<< "$put_output"; then + echo "Registration attempt $try got no answer from Azure, so its result is unknown. The step stops, so that no other registration overlaps it." >&2 + exit 1 + fi + status=Rejected + sleep 120 + fi + echo "Registration attempt $try ended with status $status" + state=$(container_state) + if [ "$state" = registered ]; then + # Unregister only a registration that ended. After any attempt, for example one with a lost + # response, the registration can still run. + registration=$(registration_status) + if [ "$registration" != Registered ]; then + echo "After registration attempt $try, the container did not show the status Registered within 10 minutes. The last status was $registration. The step does not unregister a registration that has not ended." >&2 + exit 1 + fi + # Remove the registration, so that Terraform creates and owns the registration. + # Azure can answer the unregister with a short CloudInternalError, so send it again each minute. + for attempt in $(seq 1 60); do + if [ $((attempt % 6)) -eq 1 ]; then + az rest --method delete --url "$url" --output none 2>/dev/null || true + fi + sleep 10 + state=$(container_state) + if [ "$state" = missing ]; then + exit 0 + fi + done + echo "The storage account is still registered with the vault after 10 minutes." >&2 + echo "Terraform cannot create a registration that exists. Unregister the storage account in the vault, then update the workspace again." >&2 + exit 1 + fi +done +EOT + } +} + resource "azurerm_backup_container_storage_account" "storage_account" { count = var.enable_backup ? 1 : 0 resource_group_name = azurerm_resource_group.ws.name recovery_vault_name = module.backup[0].vault_name storage_account_id = azurerm_storage_account.stg.id + + depends_on = [ + terraform_data.backup_registration_warm_up + ] } resource "terraform_data" "wait_for_backup_cleanup" { diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index 5f37520600..805be3a171 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.15.2 +version: 0.15.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre