diff --git a/.github/workflows/e2e_helper_tests.yml b/.github/workflows/e2e_helper_tests.yml index ec664e5446..2c4f0a5277 100644 --- a/.github/workflows/e2e_helper_tests.yml +++ b/.github/workflows/e2e_helper_tests.yml @@ -9,6 +9,9 @@ on: # yamllint disable-line rule:truthy paths: - 'e2e_tests/**' - 'templates/**/porter.yaml' + # test_azure_monitor_contract.py reads these Terraform files. + - 'core/terraform/azure-monitor/**' + - 'templates/workspaces/base/terraform/azure-monitor/**' - '.github/workflows/e2e_helper_tests.yml' permissions: diff --git a/CHANGELOG.md b/CHANGELOG.md index 52820a709f..f49a503145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,11 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Update Nexus from 3.94.0 to 3.94.3. The Nexus 3.94.3 image stops trusting the eMudhra certificate chains that Sonatype reports as compromised. (`tre-shared-service-sonatype-nexus` 3.11.3) ([#5276](https://github.com/microsoft/AzureTRE/issues/5276)) +* Change the Azure Monitor private link scope of the core and of each workspace only after its private endpoint exists. In workspaces, also wait 5 minutes after the scope changes before Terraform deletes the endpoint. + A scope change makes Azure update the endpoint, and during the update Azure rejects DNS zone group changes with 409 AnotherOperationInProgress, so some workspaces failed to deploy or delete. + (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) +* Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. + Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/core/terraform/azure-monitor/azure-monitor.tf b/core/terraform/azure-monitor/azure-monitor.tf index 81f5c3c18d..edf502cfa2 100644 --- a/core/terraform/azure-monitor/azure-monitor.tf +++ b/core/terraform/azure-monitor/azure-monitor.tf @@ -84,6 +84,11 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { resource_group_name = var.resource_group_name scope_name = azurerm_monitor_private_link_scope.ampls_core.name linked_resource_id = azurerm_log_analytics_workspace.core.id + + # A link makes Azure update the private endpoint. During the update, Azure rejects a create of the DNS zone group. + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } @@ -104,10 +109,16 @@ resource "azurerm_application_insights" "core" { } resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { - name = "ampls-app-insights-service" + # The name changed from ampls-app-insights-service to replace the link in existing deployments. + name = "ampls-app-insights" resource_group_name = var.resource_group_name scope_name = azurerm_monitor_private_link_scope.ampls_core.name linked_resource_id = azurerm_application_insights.core.id + + # App Insights rejects ingestion through a private endpoint created after this link, with HTTP 403. + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { @@ -118,9 +129,6 @@ resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } - depends_on = [ - azurerm_monitor_private_link_scoped_service.ampls_app_insights, - ] private_service_connection { private_connection_resource_id = azurerm_monitor_private_link_scope.ampls_core.id diff --git a/core/version.txt b/core/version.txt index 9378232641..c1c747e2aa 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.15" +__version__ = "0.18.16" diff --git a/e2e_tests/unit_tests/README.md b/e2e_tests/unit_tests/README.md index 9d434e576c..2970d57445 100644 --- a/e2e_tests/unit_tests/README.md +++ b/e2e_tests/unit_tests/README.md @@ -11,6 +11,7 @@ These unit tests check the code used by the E2E test suite. They cover authentication, operation polling, cleanup, timeouts, test selection and result reporting. They run without Azure credentials or deployed resources. +`test_azure_monitor_contract.py` reads the Azure Monitor Terraform of the core and the base workspace. It makes sure of the order of the private link scope changes and the private endpoint. Cases named after SQL, Nexus or certificates check the E2E setup, validation and cleanup for those cases. The tests stay beside the E2E helpers they exercise. diff --git a/e2e_tests/unit_tests/test_azure_monitor_contract.py b/e2e_tests/unit_tests/test_azure_monitor_contract.py new file mode 100644 index 0000000000..b60551b8a0 --- /dev/null +++ b/e2e_tests/unit_tests/test_azure_monitor_contract.py @@ -0,0 +1,67 @@ +"""Check the order of Azure Monitor private link scope changes and the private endpoint.""" + +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +CORE = "core/terraform/azure-monitor/azure-monitor.tf" +WORKSPACE = "templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf" +FILES = (CORE, WORKSPACE) +LINKS = ( + "azurerm_monitor_private_link_scoped_service.ampls_app_insights", + "azurerm_monitor_private_link_scoped_service.ampls_log_anaytics", +) +ENDPOINT = "azurerm_private_endpoint.azure_monitor_private_endpoint" +WAIT = "terraform_data.ampls_endpoint_update_wait" + + +def read(path): + # A comment does not change the order, so a depends_on in a comment line must not count. + return re.sub(r"(?m)^\s*#.*\n", "", (ROOT / path).read_text()) + + +def header(address): + kind, label = address.split(".") + return f'resource "{kind}" "{label}" {{' + + +def block(source, address): + body = source.split(header(address), 1)[1] + # terraform fmt puts the closing brace of a top-level block in the first column. + return body[: re.search(r"(?m)^}", body).start()] + + +def waits_for(source, address, target): + found = re.search(r"depends_on\s*=\s*\[([^\]]*)\]", block(source, address)) + names = re.findall(r"\w+\.\w+", found.group(1)) if found else [] + return target in names or any(waits_for(source, name, target) for name in names if header(name) in source) + + +class PrivateLinkScopeOrderTests(unittest.TestCase): + def test_links_wait_for_the_private_endpoint(self): + # App Insights rejects ingestion through a private endpoint that Azure creates after its link. + for path in FILES: + source = read(path) + for link in LINKS: + with self.subTest(path=path, link=link): + self.assertTrue(waits_for(source, link, ENDPOINT)) + self.assertNotIn(link, block(source, ENDPOINT)) + + +class WorkspaceEndpointUpdateTests(unittest.TestCase): + def test_scope_changes_wait_for_the_endpoint_update(self): + source = read(WORKSPACE) + for link in LINKS: + with self.subTest(link=link): + self.assertTrue(waits_for(source, link, WAIT)) + self.assertTrue(waits_for(source, WAIT, ENDPOINT)) + + def test_destroy_waits_longer_than_the_measured_updates(self): + source = read(WORKSPACE) + wait = block(source, WAIT) + self.assertRegex(wait, r"when\s*=\s*destroy") + # Azure runs one endpoint update of up to 132 seconds for both unlinks, or one update of up to about + # 100 seconds for each unlink, one after the other. The longest measured total was 203 seconds. + unlinks = source.count('resource "azurerm_monitor_private_link_scoped_service"') + self.assertGreater(int(re.search(r'command\s*=\s*"sleep (\d+)"', wait).group(1)), unlinks * 100) diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 1b0380bf70..a91f3c85c2 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.18.1 +version: 0.18.2 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index d704c94bcf..276ea55b97 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 2.12.2 +version: 2.12.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf index 08822baa80..a49e8bc2ef 100644 --- a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf +++ b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf @@ -108,6 +108,27 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { resource_group_name = var.resource_group_name scope_name = azapi_resource.ampls_workspace.name linked_resource_id = azurerm_log_analytics_workspace.workspace.id + + depends_on = [ + terraform_data.ampls_endpoint_update_wait, + ] +} + +# A link or unlink in the scope makes Azure update the private endpoint. For the two unlinks, Azure runs one update +# of up to 132 seconds, or one update of up to about 100 seconds for each unlink, one after the other. +# In the measured deletes, the endpoint stayed busy for up to 3 minutes and 23 seconds. +# During the update, Azure rejects changes to the endpoint DNS zone group with 409 AnotherOperationInProgress. +# The scoped services depend on this resource, so they change only after the endpoint exists, +# and on destroy Terraform waits here before it deletes the endpoint. +resource "terraform_data" "ampls_endpoint_update_wait" { + provisioner "local-exec" { + when = destroy + command = "sleep 300" + } + + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } @@ -157,11 +178,17 @@ resource "azapi_resource" "appinsights" { } resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { - name = "ampls-app-insights-service" + # The name changed from ampls-app-insights-service to replace the link in existing deployments. + name = "ampls-app-insights" resource_group_name = var.resource_group_name scope_name = azapi_resource.ampls_workspace.name linked_resource_id = azapi_resource.appinsights.id + + # App Insights rejects ingestion through a private endpoint created after this link, with HTTP 403. + depends_on = [ + terraform_data.ampls_endpoint_update_wait, + ] } resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { @@ -191,10 +218,6 @@ resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { var.blob_core_dns_zone_id, ] } - - depends_on = [ - azurerm_monitor_private_link_scoped_service.ampls_app_insights, - ] } # We don't really need this, but if not present the RG will not be empty and won't be destroyed diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index 3813a3ec5a..5f37520600 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.15.1 +version: 0.15.2 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre