From ad938a33b5c86a258c40339d0446f0fa009e82c1 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 21 Oct 2026 23:06:55 +0100 Subject: [PATCH 1/5] Link App Insights to the private link scope after its endpoint App Insights in core and in each workspace received no telemetry. Public ingestion is disabled, and ingestion through the Azure Monitor private endpoint returned HTTP 403: "Component private link id does not match incoming link and has publicNetworkAccessForIngestion=Disabled". Terraform added App Insights to the scope before it created the private endpoint. A scratch reproduction in switzerlandnorth showed: - scoped resource, then endpoint: 15 of 15 probes returned 403 - endpoint, then scoped resource: 15 of 15 probes returned 200 - deleting and creating the scoped resource after the endpoint: 200 The scoped resource now depends on the private endpoint. The new name makes Terraform replace the link in existing deployments, which repairs them on upgrade. Contract tests check both files. Part of #5280. --- CHANGELOG.md | 2 + core/terraform/azure-monitor/azure-monitor.tf | 11 ++++-- core/version.txt | 2 +- .../unit_tests/test_azure_monitor_contract.py | 38 +++++++++++++++++++ .../airlock-import-review/porter.yaml | 2 +- templates/workspaces/base/porter.yaml | 2 +- .../terraform/azure-monitor/azure-monitor.tf | 12 +++--- templates/workspaces/unrestricted/porter.yaml | 2 +- 8 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 e2e_tests/unit_tests/test_azure_monitor_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 52820a709f..4a7c9e9e42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Update Nexus from 3.94.0 to 3.94.3. The Nexus 3.94.3 image stops trusting the eMudhra certificate chains that Sonatype reports as compromised. (`tre-shared-service-sonatype-nexus` 3.11.3) ([#5276](https://github.com/microsoft/AzureTRE/issues/5276)) +* Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. + Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) * Update `aiohttp` from 3.14.3 to 3.14.4 in `api_app`, `resource_processor`, and the CLI to address security advisories. (`API` 0.28.7, `resource_processor` 0.13.15, `CLI` 0.2.13) ([#5270](https://github.com/microsoft/AzureTRE/pull/5270)) * Rename a test variable in the E2E polling tests so that CodeQL stops reporting false clear-text logging alerts. ([#5266](https://github.com/microsoft/AzureTRE/pull/5266)) diff --git a/core/terraform/azure-monitor/azure-monitor.tf b/core/terraform/azure-monitor/azure-monitor.tf index 81f5c3c18d..85525e6893 100644 --- a/core/terraform/azure-monitor/azure-monitor.tf +++ b/core/terraform/azure-monitor/azure-monitor.tf @@ -104,10 +104,16 @@ resource "azurerm_application_insights" "core" { } resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { - name = "ampls-app-insights-service" + # The name changed from ampls-app-insights-service to replace the link in existing deployments. + name = "ampls-app-insights" resource_group_name = var.resource_group_name scope_name = azurerm_monitor_private_link_scope.ampls_core.name linked_resource_id = azurerm_application_insights.core.id + + # App Insights rejects ingestion through a private endpoint created after this link, with HTTP 403. + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { @@ -118,9 +124,6 @@ resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { tags = var.tre_core_tags lifecycle { ignore_changes = [tags] } - depends_on = [ - azurerm_monitor_private_link_scoped_service.ampls_app_insights, - ] private_service_connection { private_connection_resource_id = azurerm_monitor_private_link_scope.ampls_core.id diff --git a/core/version.txt b/core/version.txt index 9378232641..c1c747e2aa 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.15" +__version__ = "0.18.16" diff --git a/e2e_tests/unit_tests/test_azure_monitor_contract.py b/e2e_tests/unit_tests/test_azure_monitor_contract.py new file mode 100644 index 0000000000..281a577f5e --- /dev/null +++ b/e2e_tests/unit_tests/test_azure_monitor_contract.py @@ -0,0 +1,38 @@ +"""Check that App Insights joins the private link scope after its private endpoint exists.""" + +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +FILES = ( + "core/terraform/azure-monitor/azure-monitor.tf", + "templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf", +) + + +def block(source, header): + return source.split(header, 1)[1].split("\nresource ", 1)[0] + + +class AppInsightsPrivateLinkTests(unittest.TestCase): + def test_app_insights_link_waits_for_the_private_endpoint(self): + for path in FILES: + with self.subTest(path=path): + source = (ROOT / path).read_text() + link = block(source, 'resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" {') + endpoint = block(source, 'resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" {') + self.assertRegex( + link, r"depends_on\s*=\s*\[\s*azurerm_private_endpoint\.azure_monitor_private_endpoint,?\s*\]" + ) + self.assertNotIn("ampls_app_insights", endpoint) + + def test_existing_deployments_replace_the_link(self): + for path in FILES: + with self.subTest(path=path): + link = block( + (ROOT / path).read_text(), + 'resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" {', + ) + name = re.search(r'^\s*name\s*=\s*"([^"]+)"', link, re.MULTILINE).group(1) + self.assertNotEqual(name, "ampls-app-insights-service") diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 1b0380bf70..a91f3c85c2 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.18.1 +version: 0.18.2 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index d704c94bcf..276ea55b97 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 2.12.2 +version: 2.12.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf index 08822baa80..6261b2b554 100644 --- a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf +++ b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf @@ -157,11 +157,17 @@ resource "azapi_resource" "appinsights" { } resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { - name = "ampls-app-insights-service" + # The name changed from ampls-app-insights-service to replace the link in existing deployments. + name = "ampls-app-insights" resource_group_name = var.resource_group_name scope_name = azapi_resource.ampls_workspace.name linked_resource_id = azapi_resource.appinsights.id + + # App Insights rejects ingestion through a private endpoint created after this link, with HTTP 403. + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { @@ -191,10 +197,6 @@ resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" { var.blob_core_dns_zone_id, ] } - - depends_on = [ - azurerm_monitor_private_link_scoped_service.ampls_app_insights, - ] } # We don't really need this, but if not present the RG will not be empty and won't be destroyed diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index 3813a3ec5a..5f37520600 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.15.1 +version: 0.15.2 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre From 57f98064add39e1ef3ffdba7e50bb393bdfe817e Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 13:50:11 +0000 Subject: [PATCH 2/5] Wait for Azure Monitor endpoint updates around workspace scope changes A link or unlink in the workspace Azure Monitor private link scope makes Azure update the scope private endpoint. The activity log shows 13 to 16 seconds for a link. For the two unlinks on delete, Azure ran one update of up to 132 seconds, or one update of 80 to 89 seconds for each unlink, one after the other. During an update, Azure rejects a DNS zone group create or delete with 409 AnotherOperationInProgress. Terraform changed the Log Analytics link at the same time as the endpoint, so some workspace deploys and deletes failed and left the workspace resource group behind. Both scoped services now depend on a terraform_data resource that depends on the endpoint. On create, the scope changes after the endpoint and its zone group exist. On destroy, Terraform removes the links, waits 5 minutes, then deletes the endpoint. In the measured deletes, the endpoint stayed busy for 136 to 177 seconds. The workspace versions stay at the bump of the previous commit. Part of #3193. --- CHANGELOG.md | 2 + .../unit_tests/test_azure_monitor_contract.py | 54 ++++++++++++++----- .../terraform/azure-monitor/azure-monitor.tf | 23 +++++++- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7c9e9e42..741c5c4fca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Update Nexus from 3.94.0 to 3.94.3. The Nexus 3.94.3 image stops trusting the eMudhra certificate chains that Sonatype reports as compromised. (`tre-shared-service-sonatype-nexus` 3.11.3) ([#5276](https://github.com/microsoft/AzureTRE/issues/5276)) +* Change the workspace Azure Monitor private link scope only after its private endpoint exists, and wait 5 minutes after the scope changes before Terraform deletes the endpoint. A scope change makes Azure update the endpoint, and during the update Azure rejects DNS zone group changes with 409 AnotherOperationInProgress, so some workspaces failed to deploy or delete. + (`tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) diff --git a/e2e_tests/unit_tests/test_azure_monitor_contract.py b/e2e_tests/unit_tests/test_azure_monitor_contract.py index 281a577f5e..10350aa0cf 100644 --- a/e2e_tests/unit_tests/test_azure_monitor_contract.py +++ b/e2e_tests/unit_tests/test_azure_monitor_contract.py @@ -1,18 +1,26 @@ -"""Check that App Insights joins the private link scope after its private endpoint exists.""" +"""Check the order of Azure Monitor private link scope changes and the private endpoint.""" from pathlib import Path import re import unittest ROOT = Path(__file__).resolve().parents[2] -FILES = ( - "core/terraform/azure-monitor/azure-monitor.tf", - "templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf", -) +CORE = "core/terraform/azure-monitor/azure-monitor.tf" +WORKSPACE = "templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf" +FILES = (CORE, WORKSPACE) +ENDPOINT = "azurerm_private_endpoint.azure_monitor_private_endpoint" +WAIT = "terraform_data.ampls_endpoint_update_wait" -def block(source, header): - return source.split(header, 1)[1].split("\nresource ", 1)[0] +def block(source, address): + kind, label = address.split(".") + return source.split(f'resource "{kind}" "{label}" {{', 1)[1].split("\nresource ", 1)[0] + + +def waits_for(source, address, target): + found = re.search(r"depends_on\s*=\s*\[([^\]]*)\]", block(source, address)) + names = re.findall(r"[\w.]+", found.group(1)) if found else [] + return any(name == target or waits_for(source, name, target) for name in names) class AppInsightsPrivateLinkTests(unittest.TestCase): @@ -20,19 +28,37 @@ def test_app_insights_link_waits_for_the_private_endpoint(self): for path in FILES: with self.subTest(path=path): source = (ROOT / path).read_text() - link = block(source, 'resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" {') - endpoint = block(source, 'resource "azurerm_private_endpoint" "azure_monitor_private_endpoint" {') - self.assertRegex( - link, r"depends_on\s*=\s*\[\s*azurerm_private_endpoint\.azure_monitor_private_endpoint,?\s*\]" + self.assertTrue( + waits_for(source, "azurerm_monitor_private_link_scoped_service.ampls_app_insights", ENDPOINT) ) - self.assertNotIn("ampls_app_insights", endpoint) + self.assertNotIn("ampls_app_insights", block(source, ENDPOINT)) def test_existing_deployments_replace_the_link(self): for path in FILES: with self.subTest(path=path): link = block( - (ROOT / path).read_text(), - 'resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" {', + (ROOT / path).read_text(), "azurerm_monitor_private_link_scoped_service.ampls_app_insights" ) name = re.search(r'^\s*name\s*=\s*"([^"]+)"', link, re.MULTILINE).group(1) self.assertNotEqual(name, "ampls-app-insights-service") + + +class WorkspaceEndpointUpdateTests(unittest.TestCase): + def test_scope_changes_wait_for_the_endpoint_update(self): + source = (ROOT / WORKSPACE).read_text() + for address in ( + "azurerm_monitor_private_link_scoped_service.ampls_app_insights", + "azurerm_monitor_private_link_scoped_service.ampls_log_anaytics", + ): + with self.subTest(address=address): + self.assertTrue(waits_for(source, address, WAIT)) + self.assertTrue(waits_for(source, WAIT, ENDPOINT)) + + def test_destroy_waits_longer_than_the_measured_updates(self): + source = (ROOT / WORKSPACE).read_text() + wait = block(source, WAIT) + self.assertRegex(wait, r"when\s*=\s*destroy") + # Azure runs one endpoint update of up to 132 seconds for both unlinks, or one update of up to + # 90 seconds for each unlink, one after the other. The measured total stayed below 3 minutes. + unlinks = source.count('resource "azurerm_monitor_private_link_scoped_service"') + self.assertGreater(int(re.search(r'command\s*=\s*"sleep (\d+)"', wait).group(1)), unlinks * 90) diff --git a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf index 6261b2b554..f0f9c68b2b 100644 --- a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf +++ b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf @@ -108,6 +108,27 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { resource_group_name = var.resource_group_name scope_name = azapi_resource.ampls_workspace.name linked_resource_id = azurerm_log_analytics_workspace.workspace.id + + depends_on = [ + terraform_data.ampls_endpoint_update_wait, + ] +} + +# A link or unlink in the scope makes Azure update the private endpoint. For the two unlinks, Azure runs one update +# of up to 132 seconds, or one update of up to 90 seconds for each unlink, one after the other. +# In the measured deletes, the endpoint stayed busy for less than 3 minutes. +# During the update, Azure rejects changes to the endpoint DNS zone group with 409 AnotherOperationInProgress. +# The scoped services depend on this resource, so they change only after the endpoint exists, +# and on destroy Terraform waits here before it deletes the endpoint. +resource "terraform_data" "ampls_endpoint_update_wait" { + provisioner "local-exec" { + when = destroy + command = "sleep 300" + } + + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } @@ -166,7 +187,7 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_app_insights" { # App Insights rejects ingestion through a private endpoint created after this link, with HTTP 403. depends_on = [ - azurerm_private_endpoint.azure_monitor_private_endpoint, + terraform_data.ampls_endpoint_update_wait, ] } From c2f6906ff490d4b9c7af569c42a0e200c3caa6b7 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 14:36:25 +0000 Subject: [PATCH 3/5] Create the core Log Analytics link after the Azure Monitor endpoint The previous commit made the core App Insights link depend on the private endpoint. The core Log Analytics link had no order, so on a new core it could change the scope while Terraform created the endpoint and its DNS zone group. A scope change makes Azure update the endpoint, and during the update Azure rejects a zone group create with 409 AnotherOperationInProgress. Both core links now depend on the endpoint, as in the workspace. A depends_on change does not replace the link, so an existing core gets no change. The contract test now checks both links in both files. It ignores comment lines, so a depends_on in a comment does not count. It ends a block at the closing brace in the first column, and it follows only names that have a resource block in the same file. The test that the App Insights link has a new name is removed. The comment on the name explains the replacement. Part of #3193. --- CHANGELOG.md | 5 +- core/terraform/azure-monitor/azure-monitor.tf | 5 ++ .../unit_tests/test_azure_monitor_contract.py | 61 ++++++++++--------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 741c5c4fca..f49a503145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,9 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Update Nexus from 3.94.0 to 3.94.3. The Nexus 3.94.3 image stops trusting the eMudhra certificate chains that Sonatype reports as compromised. (`tre-shared-service-sonatype-nexus` 3.11.3) ([#5276](https://github.com/microsoft/AzureTRE/issues/5276)) -* Change the workspace Azure Monitor private link scope only after its private endpoint exists, and wait 5 minutes after the scope changes before Terraform deletes the endpoint. A scope change makes Azure update the endpoint, and during the update Azure rejects DNS zone group changes with 409 AnotherOperationInProgress, so some workspaces failed to deploy or delete. - (`tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) +* Change the Azure Monitor private link scope of the core and of each workspace only after its private endpoint exists. In workspaces, also wait 5 minutes after the scope changes before Terraform deletes the endpoint. + A scope change makes Azure update the endpoint, and during the update Azure rejects DNS zone group changes with 409 AnotherOperationInProgress, so some workspaces failed to deploy or delete. + (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#3193](https://github.com/microsoft/AzureTRE/issues/3193)) * Link App Insights to the Azure Monitor private link scope after the private endpoint exists. Before this fix, App Insights rejected all private ingestion with HTTP 403, so the core and workspace App Insights received no telemetry. Rename the scoped service, so that existing deployments replace the link on upgrade. (`core` 0.18.16, `tre-workspace-base` 2.12.3, `tre-workspace-unrestricted` 0.15.2, `tre-workspace-airlock-import-review` 0.18.2) ([#5280](https://github.com/microsoft/AzureTRE/issues/5280)) * Update `vite` from 7.3.6 to 7.3.7 and the UI Vitest packages from 4.1.11 to 4.1.12 to address security advisories in the development server and the test mocker. (`ui` 0.10.4) ([#5274](https://github.com/microsoft/AzureTRE/pull/5274)) diff --git a/core/terraform/azure-monitor/azure-monitor.tf b/core/terraform/azure-monitor/azure-monitor.tf index 85525e6893..edf502cfa2 100644 --- a/core/terraform/azure-monitor/azure-monitor.tf +++ b/core/terraform/azure-monitor/azure-monitor.tf @@ -84,6 +84,11 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { resource_group_name = var.resource_group_name scope_name = azurerm_monitor_private_link_scope.ampls_core.name linked_resource_id = azurerm_log_analytics_workspace.core.id + + # A link makes Azure update the private endpoint. During the update, Azure rejects a create of the DNS zone group. + depends_on = [ + azurerm_private_endpoint.azure_monitor_private_endpoint, + ] } diff --git a/e2e_tests/unit_tests/test_azure_monitor_contract.py b/e2e_tests/unit_tests/test_azure_monitor_contract.py index 10350aa0cf..50c91bf5b5 100644 --- a/e2e_tests/unit_tests/test_azure_monitor_contract.py +++ b/e2e_tests/unit_tests/test_azure_monitor_contract.py @@ -8,54 +8,57 @@ CORE = "core/terraform/azure-monitor/azure-monitor.tf" WORKSPACE = "templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf" FILES = (CORE, WORKSPACE) +LINKS = ( + "azurerm_monitor_private_link_scoped_service.ampls_app_insights", + "azurerm_monitor_private_link_scoped_service.ampls_log_anaytics", +) ENDPOINT = "azurerm_private_endpoint.azure_monitor_private_endpoint" WAIT = "terraform_data.ampls_endpoint_update_wait" -def block(source, address): +def read(path): + # A comment does not change the order, so a depends_on in a comment line must not count. + return re.sub(r"(?m)^\s*#.*\n", "", (ROOT / path).read_text()) + + +def header(address): kind, label = address.split(".") - return source.split(f'resource "{kind}" "{label}" {{', 1)[1].split("\nresource ", 1)[0] + return f'resource "{kind}" "{label}" {{' + + +def block(source, address): + body = source.split(header(address), 1)[1] + # terraform fmt puts the closing brace of a top-level block in the first column. + return body[: re.search(r"(?m)^}", body).start()] def waits_for(source, address, target): found = re.search(r"depends_on\s*=\s*\[([^\]]*)\]", block(source, address)) - names = re.findall(r"[\w.]+", found.group(1)) if found else [] - return any(name == target or waits_for(source, name, target) for name in names) + names = re.findall(r"\w+\.\w+", found.group(1)) if found else [] + return target in names or any(waits_for(source, name, target) for name in names if header(name) in source) -class AppInsightsPrivateLinkTests(unittest.TestCase): - def test_app_insights_link_waits_for_the_private_endpoint(self): - for path in FILES: - with self.subTest(path=path): - source = (ROOT / path).read_text() - self.assertTrue( - waits_for(source, "azurerm_monitor_private_link_scoped_service.ampls_app_insights", ENDPOINT) - ) - self.assertNotIn("ampls_app_insights", block(source, ENDPOINT)) - - def test_existing_deployments_replace_the_link(self): +class PrivateLinkScopeOrderTests(unittest.TestCase): + def test_links_wait_for_the_private_endpoint(self): + # App Insights rejects ingestion through a private endpoint that Azure creates after its link. for path in FILES: - with self.subTest(path=path): - link = block( - (ROOT / path).read_text(), "azurerm_monitor_private_link_scoped_service.ampls_app_insights" - ) - name = re.search(r'^\s*name\s*=\s*"([^"]+)"', link, re.MULTILINE).group(1) - self.assertNotEqual(name, "ampls-app-insights-service") + source = read(path) + for link in LINKS: + with self.subTest(path=path, link=link): + self.assertTrue(waits_for(source, link, ENDPOINT)) + self.assertNotIn(link, block(source, ENDPOINT)) class WorkspaceEndpointUpdateTests(unittest.TestCase): def test_scope_changes_wait_for_the_endpoint_update(self): - source = (ROOT / WORKSPACE).read_text() - for address in ( - "azurerm_monitor_private_link_scoped_service.ampls_app_insights", - "azurerm_monitor_private_link_scoped_service.ampls_log_anaytics", - ): - with self.subTest(address=address): - self.assertTrue(waits_for(source, address, WAIT)) + source = read(WORKSPACE) + for link in LINKS: + with self.subTest(link=link): + self.assertTrue(waits_for(source, link, WAIT)) self.assertTrue(waits_for(source, WAIT, ENDPOINT)) def test_destroy_waits_longer_than_the_measured_updates(self): - source = (ROOT / WORKSPACE).read_text() + source = read(WORKSPACE) wait = block(source, WAIT) self.assertRegex(wait, r"when\s*=\s*destroy") # Azure runs one endpoint update of up to 132 seconds for both unlinks, or one update of up to From 71362739d25f884714dcfc0b9c4ad609203acbb4 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 14:36:29 +0000 Subject: [PATCH 4/5] Run the E2E helper tests when the Azure Monitor Terraform changes test_azure_monitor_contract.py reads the Azure Monitor Terraform of the core and of the base workspace. The E2E helper tests ran only for changes in e2e_tests, porter.yaml files and the workflow, so a change to these Terraform files alone did not run the contract test. The path filter now includes the two azure-monitor folders. The README of the unit tests names the test. Part of #5280 and #3193. --- .github/workflows/e2e_helper_tests.yml | 3 +++ e2e_tests/unit_tests/README.md | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/e2e_helper_tests.yml b/.github/workflows/e2e_helper_tests.yml index ec664e5446..2c4f0a5277 100644 --- a/.github/workflows/e2e_helper_tests.yml +++ b/.github/workflows/e2e_helper_tests.yml @@ -9,6 +9,9 @@ on: # yamllint disable-line rule:truthy paths: - 'e2e_tests/**' - 'templates/**/porter.yaml' + # test_azure_monitor_contract.py reads these Terraform files. + - 'core/terraform/azure-monitor/**' + - 'templates/workspaces/base/terraform/azure-monitor/**' - '.github/workflows/e2e_helper_tests.yml' permissions: diff --git a/e2e_tests/unit_tests/README.md b/e2e_tests/unit_tests/README.md index 9d434e576c..2970d57445 100644 --- a/e2e_tests/unit_tests/README.md +++ b/e2e_tests/unit_tests/README.md @@ -11,6 +11,7 @@ These unit tests check the code used by the E2E test suite. They cover authentication, operation polling, cleanup, timeouts, test selection and result reporting. They run without Azure credentials or deployed resources. +`test_azure_monitor_contract.py` reads the Azure Monitor Terraform of the core and the base workspace. It makes sure of the order of the private link scope changes and the private endpoint. Cases named after SQL, Nexus or certificates check the E2E setup, validation and cleanup for those cases. The tests stay beside the E2E helpers they exercise. From 0c9a1927eab4e3a4f75b6da93558ea056e1e7fe4 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 28 Oct 2026 18:47:06 +0000 Subject: [PATCH 5/5] Match the endpoint update comments to the latest measurements A later run measured endpoint updates of 91 and 92 seconds for each unlink, one after the other, and the endpoint stayed busy for up to 203 seconds after the last unlink. The comments now give about 100 seconds for each unlink and up to 3 minutes and 23 seconds in total. The contract test now requires a wait longer than 100 seconds for each unlink. The 5-minute wait does not change. Part of #3193. --- e2e_tests/unit_tests/test_azure_monitor_contract.py | 6 +++--- .../base/terraform/azure-monitor/azure-monitor.tf | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e_tests/unit_tests/test_azure_monitor_contract.py b/e2e_tests/unit_tests/test_azure_monitor_contract.py index 50c91bf5b5..b60551b8a0 100644 --- a/e2e_tests/unit_tests/test_azure_monitor_contract.py +++ b/e2e_tests/unit_tests/test_azure_monitor_contract.py @@ -61,7 +61,7 @@ def test_destroy_waits_longer_than_the_measured_updates(self): source = read(WORKSPACE) wait = block(source, WAIT) self.assertRegex(wait, r"when\s*=\s*destroy") - # Azure runs one endpoint update of up to 132 seconds for both unlinks, or one update of up to - # 90 seconds for each unlink, one after the other. The measured total stayed below 3 minutes. + # Azure runs one endpoint update of up to 132 seconds for both unlinks, or one update of up to about + # 100 seconds for each unlink, one after the other. The longest measured total was 203 seconds. unlinks = source.count('resource "azurerm_monitor_private_link_scoped_service"') - self.assertGreater(int(re.search(r'command\s*=\s*"sleep (\d+)"', wait).group(1)), unlinks * 90) + self.assertGreater(int(re.search(r'command\s*=\s*"sleep (\d+)"', wait).group(1)), unlinks * 100) diff --git a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf index f0f9c68b2b..a49e8bc2ef 100644 --- a/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf +++ b/templates/workspaces/base/terraform/azure-monitor/azure-monitor.tf @@ -115,8 +115,8 @@ resource "azurerm_monitor_private_link_scoped_service" "ampls_log_anaytics" { } # A link or unlink in the scope makes Azure update the private endpoint. For the two unlinks, Azure runs one update -# of up to 132 seconds, or one update of up to 90 seconds for each unlink, one after the other. -# In the measured deletes, the endpoint stayed busy for less than 3 minutes. +# of up to 132 seconds, or one update of up to about 100 seconds for each unlink, one after the other. +# In the measured deletes, the endpoint stayed busy for up to 3 minutes and 23 seconds. # During the update, Azure rejects changes to the endpoint DNS zone group with 409 AnotherOperationInProgress. # The scoped services depend on this resource, so they change only after the endpoint exists, # and on destroy Terraform waits here before it deletes the endpoint.