Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
112 commits
Select commit Hold shift + click to select a range
c83471e
Implement address space removal
JC-wk Nov 4, 2025
029f506
update template versions
JC-wk Nov 4, 2025
1abd135
Merge branch 'main' into deallocate-ip-addresses
JC-wk Nov 10, 2025
c9f9307
Move address_space deletion to service_bus
JC-wk Nov 12, 2025
f38ef4a
Merge branch 'main' into deallocate-ip-addresses
JC-wk Nov 13, 2025
8f722aa
Merge branch 'main' into deallocate-ip-addresses
Dec 15, 2025
539c4ab
Merge branch 'main' into deallocate-ip-addresses
JC-wk Dec 24, 2025
a1ca86a
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jan 9, 2026
6ebeb08
bump api version
Jan 9, 2026
3583506
Merge branch 'main' into deallocate-ip-addresses
JC-wk Feb 6, 2026
b32d4ef
changelog
Feb 6, 2026
48908ea
Merge branch 'main' into deallocate-ip-addresses
JC-wk May 22, 2026
fc9ba33
Copilot feedback
Jun 9, 2026
69f0b3e
remove spaces
Jun 9, 2026
2924a12
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jun 9, 2026
84c6915
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jun 24, 2026
88ba948
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jun 30, 2026
b352c12
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jul 3, 2026
e962d53
Bump version to 0.25.27
JC-wk Jul 3, 2026
32c56a1
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jul 3, 2026
ece05e3
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jul 21, 2026
f1a6711
Merge branch 'main' into deallocate-ip-addresses
JC-wk Jul 31, 2026
702f078
### Summary of Changes
Jul 31, 2026
232c088
copilot feedback
Jul 31, 2026
a622859
• Refactored AutoLockRenewer in deployment_status_updater.py to use…
Jul 31, 2026
1d65c72
1. **deployment_status_updater.py**:
Jul 31, 2026
bf72041
Add SERVICE_BUS_MAX_WAIT_TIME configuration and improve error handlin…
Jul 31, 2026
04be27d
Fix CosmosAccessConditionFailedError instantiation in tests for airlo…
Jul 31, 2026
ad7bdb6
Fix type casting for span attributes in DeploymentStatusUpdater and u…
Jul 31, 2026
9a08ecd
Merge branch 'main' into deallocate-ip-addresses
JC-wk Aug 4, 2026
f6b7e09
Merge branch 'main' into deallocate-ip-addresses
JC-wk Aug 17, 2026
1ce7cb0
Handle workspace address cleanup failures
Aug 21, 2026
8ac5d24
Defer workspace address cleanup until upgrade
Aug 21, 2026
2711e28
Select the matching resource when freeing workspace address space
Aug 21, 2026
5d668e8
Mock workspace repository in deployment status tests
Aug 21, 2026
60e52b6
Require successful root step for workspace address cleanup
Aug 21, 2026
9e78958
standardize the deallocation of IP addresses for workspace services u…
Aug 21, 2026
a360697
Merge branch 'main' into deallocate-ip-addresses
JC-wk Sep 3, 2026
9fb1eb3
CHANGELOG
JC-wk Sep 3, 2026
6e71bb1
Fix address space cleanup handling for cascade uninstalls
Sep 3, 2026
b2d187f
Fix workspace address space removal payload and failure status handling
Sep 3, 2026
bdca9b2
Serialize workspace address allocation with active operations and res…
Sep 3, 2026
1732ec2
Serialize workspace mutations against active cleanup and ensure clean…
Sep 3, 2026
3586aac
Lock workspace from start via resourcePath, keep active status on ret…
Sep 3, 2026
0fc006d
Append fallback step before persisting status and treat address confl…
Sep 3, 2026
69dded3
Validate ownership before enqueue, make enqueue retryable, and handle…
Sep 3, 2026
624de53
revert service bus changes in favour of PR 4930
Sep 3, 2026
95e9305
Adjust delivery count handling for final cleanup in DeploymentStatusU…
Sep 3, 2026
4b2b20f
Guard completed address space cleanup against redelivered status mess…
Sep 4, 2026
9e8207a
Resolve cleanup redelivery, terminal failure, and concurrent workspac…
Sep 4, 2026
4ac97bf
Enhance deployment message handling with deterministic message IDs an…
JC-wk Sep 4, 2026
7d830ec
Refactor workspace service operations and update delivery count handl…
JC-wk Sep 4, 2026
9b2751f
Address review feedback on deduplication, delivery counting, and work…
JC-wk Sep 4, 2026
f94c4ea
Make cleanup dispatch idempotent in application and revert queue recr…
JC-wk Sep 4, 2026
0d551dc
Enable duplicate detection on workspacequeue and bump core version to…
JC-wk Sep 4, 2026
5722c2a
Handle cleanup dispatch idempotency in application and revert Service…
JC-wk Sep 4, 2026
13ae073
Inject empty active-services mock for workspace_services_repo in depl…
JC-wk Sep 4, 2026
eaf6150
Address Copilot review on terminal-state guards, resource lookups, an…
JC-wk Sep 4, 2026
45bf936
Enhance address space cleanup validation and improve deployment statu…
JC-wk Sep 4, 2026
3d6371b
Implement ETag handling in Operation updates and enhance step merging…
JC-wk Sep 4, 2026
148bd61
Address Copilot feedback on etag exclusion, template validation, and …
JC-wk Sep 4, 2026
5770364
Correct delivery_count indexing in updater and update pipeline schema…
JC-wk Sep 4, 2026
ab99415
Roll back operations on dispatch failure and correct repository mocki…
JC-wk Sep 4, 2026
5c1892f
Persist failure status on dispatch failure and restore 0-based final …
JC-wk Sep 4, 2026
18d5faf
Fix terminal status mapping and missing resource failure persistence …
JC-wk Sep 4, 2026
c1ae6c5
Reduce comment verbosity and remove obvious comments across cleanup i…
JC-wk Sep 4, 2026
24740e4
Resolve stale-status concurrency on ETag conflict and derive step fai…
JC-wk Sep 4, 2026
d940017
Refactor failure status handling and improve UUID validation in resou…
JC-wk Sep 4, 2026
92a52d3
Refactor resource path handling to include cascading operations and a…
JC-wk Sep 4, 2026
098f3c5
Enforce workspace lock on descendant mutating routes and preserve del…
JC-wk Sep 4, 2026
ae0608c
Acquire atomic workspace lease on operation admission and roll back a…
JC-wk Sep 4, 2026
57facf6
Acquire workspace lease before resource mutations, keep lease ownersh…
JC-wk Sep 4, 2026
f32fe94
Reclaim orphaned workspace leases after bounded expiry and retry on c…
JC-wk Sep 4, 2026
c3cd999
Reconcile stale operations and use conditional lease release
JC-wk Sep 4, 2026
340ed43
Merge branch 'main' into deallocate-ip-addresses
JC-wk Sep 4, 2026
f91b6cd
bump version
JC-wk Sep 4, 2026
7265c8b
Address Copilot review on lease expiry, acquisition retry, and fail-c…
JC-wk Sep 4, 2026
64d1671
Exclude terminal statuses in active ops query and require non-empty s…
JC-wk Sep 4, 2026
ce885d0
Extend lease expiry margin, keep lease during caller rollback, and se…
JC-wk Sep 4, 2026
4efc650
Release lease on internal dispatch failures and serialize airlock ope…
JC-wk Sep 4, 2026
7877eeb
Address Copilot review: conditional rollback, ETag delete check, and …
JC-wk Sep 4, 2026
6248f86
Address Copilot review feedback on background tasks, lease retention,…
JC-wk Sep 4, 2026
9acf41a
Retain lease on ambiguous dispatch, await airlock cleanup, and retry …
JC-wk Sep 4, 2026
54440d6
Propagate rollback errors, preserve lease_retained on HTTPException, …
JC-wk Sep 4, 2026
a08268f
Address Copilot feedback: try-finally lease release, reconciled opera…
JC-wk Sep 4, 2026
15f6787
Schedule review user resource deletion via BackgroundTasks to avoid H…
JC-wk Sep 4, 2026
ecc79b5
Use step resourceAction for stale step status and await uninstall in …
JC-wk Sep 4, 2026
ef5a1b6
Enhance error handling for CosmosAccessConditionFailedError
JC-wk Sep 7, 2026
f17e48d
Await airlock resource cleanup during review and cancellation
Sep 7, 2026
3530553
Improve lease retention, stale operation reconciliation, and review r…
Sep 7, 2026
e205c2e
Improve workspace lease handling and stale operation reconciliation
Sep 8, 2026
7ebff4e
Improve Airlock VM replacement and workspace lease cleanup
Sep 8, 2026
6e1af03
Merge branch 'main' into deallocate-ip-addresses
JC-wk Sep 8, 2026
2865635
Bump API version to 0.26.10
Sep 8, 2026
2c96e1d
Merge branch 'deallocate-ip-addresses' of https://github.com/JC-wk/Az…
Sep 8, 2026
705d1bc
Improve Airlock resource cleanup and redeployment workflows
Sep 9, 2026
8596924
Improve Airlock workflow cleanup and lease handling
Sep 9, 2026
c983ac7
Improve Airlock redeployment recovery and workspace lease cleanup
Sep 9, 2026
f94474d
Merge branch 'main' into deallocate-ip-addresses
JC-wk Sep 9, 2026
3e17a00
fix tests
Sep 9, 2026
4273197
Merge branch 'deallocate-ip-addresses' of https://github.com/JC-wk/Az…
Sep 9, 2026
3ed812e
Fix Airlock deletion sequencing and operation reconciliation
Sep 9, 2026
3022bda
Include operation ID in redeploy workflow ID
JC-wk Sep 10, 2026
ad6cefb
Serialize address space allocation with allocator leases
Sep 10, 2026
f0d0040
Serialize resource deletion and Airlock redeployment operations
Sep 10, 2026
25a9c7b
Preserve lease state and Airlock reviewer identity
Sep 10, 2026
609f49c
Fix Airlock redeployment workspace and retry handling
Sep 10, 2026
06acaf3
Improve Airlock workflow retries and lease handling
Sep 10, 2026
8e94364
Fix await usage in VM deployment function
JC-wk Sep 10, 2026
8128e69
Reconcile expired workspace operations
Sep 10, 2026
6e33abb
Enhance Airlock workflow processing and improve operation reconciliat…
Sep 10, 2026
672526b
Improve address space lease and Airlock redeploy handling
Sep 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ ENHANCEMENTS:

BUG FIXES:
* Configure the default pytest-asyncio fixture loop scope to `function` to remove the deprecation warning. ([#5055](https://github.com/microsoft/AzureTRE/pull/5055))
* A workspace service's allocated `address_space` will now be removed from its parent workspace's `address_spaces` list when the service is deleted, preventing IP address range exhaustion, and workspace-level operation leasing ensures safe serialization of concurrent workspace mutations ([#4727](https://github.com/microsoft/AzureTRE/issues/4727))
* Fix pytest warnings ([#5066](https://github.com/microsoft/AzureTRE/issues/5066))
* Fix inconsistent ServiceBusClient lifecycle management in deployment_status_updater.py, airlock_request_status_update.py, and runner.py to prevent connection socket and AMQP channel leaks ([#4930](https://github.com/microsoft/AzureTRE/pull/4930))

Expand Down
1 change: 1 addition & 0 deletions api_app/.env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ RESOURCE_GROUP_NAME=__CHANGE_ME__
SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE=__CHANGE_ME__
SERVICE_BUS_RESOURCE_REQUEST_QUEUE=workspacequeue
SERVICE_BUS_DEPLOYMENT_STATUS_UPDATE_QUEUE=deploymentstatus
SERVICE_BUS_AIRLOCK_WORKFLOW_QUEUE=airlock-workflow
SERVICE_BUS_STEP_RESULT_QUEUE=airlock-step-result

# Event grid configuration
Expand Down
2 changes: 1 addition & 1 deletion api_app/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.26.9"
__version__ = "0.26.10"
71 changes: 53 additions & 18 deletions api_app/api/routes/resource_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ async def save_and_deploy_resource(
resource_history_repo: ResourceHistoryRepository,
user: User,
resource_template: ResourceTemplate,
operation_id: Optional[str] = None,
) -> Operation:
try:
resource.user = user.model_dump()
Expand All @@ -82,6 +83,9 @@ async def save_and_deploy_resource(
)

try:
kwargs = {}
if operation_id is not None:
kwargs["operation_id"] = operation_id
operation = await send_resource_request_message(
resource=resource,
operations_repo=operations_repo,
Expand All @@ -90,15 +94,24 @@ async def save_and_deploy_resource(
resource_template_repo=resource_template_repo,
resource_history_repo=resource_history_repo,
action=RequestAction.Install,
**kwargs,
)
return operation
except Exception:
await resource_repo.delete_item(resource.id)
except HTTPException as ex:
if not getattr(ex, "lease_retained", False):
await resource_repo.delete_item(resource.id)
raise
except Exception as ex:
lease_retained = getattr(ex, "lease_retained", False)
if not lease_retained:
await resource_repo.delete_item(resource.id)
logger.exception("Failed send resource request message")
raise HTTPException(
http_ex = HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE,
)
setattr(http_ex, "lease_retained", lease_retained)
raise http_ex from ex


def mask_sensitive_properties(
Expand Down Expand Up @@ -169,9 +182,13 @@ async def send_uninstall_message(
resource_template_repo: ResourceTemplateRepository,
resource_history_repo: ResourceHistoryRepository,
user: User,
is_cascade: str = False
is_cascade: str = False,
operation_id: Optional[str] = None,
) -> Operation:
try:
operation_kwargs = {}
if operation_id is not None:
operation_kwargs["operation_id"] = operation_id
operation = await send_resource_request_message(
resource=resource,
operations_repo=operations_repo,
Expand All @@ -180,15 +197,20 @@ async def send_uninstall_message(
resource_template_repo=resource_template_repo,
resource_history_repo=resource_history_repo,
action=RequestAction.UnInstall,
is_cascade=is_cascade
is_cascade=is_cascade,
**operation_kwargs,
)
return operation
except Exception:
except HTTPException:
raise
except Exception as ex:
logger.exception(f"Failed to send {resource_type} resource delete message")
raise HTTPException(
http_ex = HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=strings.SERVICE_BUS_GENERAL_ERROR_MESSAGE,
)
http_ex.lease_retained = getattr(ex, "lease_retained", False)
raise http_ex from ex


async def send_custom_action_message(
Expand Down Expand Up @@ -234,6 +256,8 @@ async def send_custom_action_message(
action=custom_action,
)
return operation
except HTTPException:
raise
except Exception:
logger.exception(f"Failed to send {resource_type} resource custom action message")
raise HTTPException(
Expand Down Expand Up @@ -296,17 +320,28 @@ async def update_user_resource(
operations_repo: OperationRepository,
resource_history_repo: ResourceHistoryRepository) -> Operation:

patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update)
operation = await send_resource_request_message(
resource=patched_user_resource,
operations_repo=operations_repo,
resource_repo=user_resource_repo,
user=user,
resource_template_repo=resource_template_repo,
resource_history_repo=resource_history_repo,
action=RequestAction.Upgrade)

return operation
target_workspace_id = getattr(workspace_service, "workspaceId", None) or getattr(user_resource, "workspaceId", None)
operation_id = operations_repo.create_operation_id()
if target_workspace_id and hasattr(operations_repo, "acquire_workspace_lease"):
await operations_repo.acquire_workspace_lease(target_workspace_id, operation_id)
Comment thread
JC-wk marked this conversation as resolved.
Resolved

try:
patched_user_resource, _ = await user_resource_repo.patch_user_resource(user_resource, resource_patch, etag, resource_template_repo, resource_history_repo, workspace_service.templateName, user, force_version_update)
operation = await send_resource_request_message(
resource=patched_user_resource,
operations_repo=operations_repo,
resource_repo=user_resource_repo,
user=user,
resource_template_repo=resource_template_repo,
resource_history_repo=resource_history_repo,
action=RequestAction.Upgrade,
operation_id=operation_id)

return operation
except Exception as ex:
if not getattr(ex, "lease_retained", False) and target_workspace_id and hasattr(operations_repo, "release_workspace_lease"):
await operations_repo.release_workspace_lease(target_workspace_id, operation_id)
raise


async def enrich_resource_with_available_upgrades(resource: Resource, resource_template_repo: ResourceTemplateRepository):
Expand Down
Loading
Loading