Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions acceptance/bin/print_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,43 @@ def del_path(body, field):
body.pop(leaf, None)


def fold_pending_operation_writes(requests):
"""Drop a deployment-history PATCH that marks an operation PENDING when a later PATCH to
the same operation follows, and drop sequence_id from every operation PATCH.

A recreate writes an operation twice: the drop marks it PENDING, the save (or failure)
finishes it. The CLI uploads writes in the background and merges a write still waiting
to be sent into the next one for the same resource, so depending on timing the service
gets one PATCH at sequence_id 0 or two at 0 and 1. Folding makes both read the same.

>>> pending = {"method": "PATCH", "path": "/v/1/operations/a", "body": {"sequence_id": 0, "status": "OPERATION_STATUS_PENDING"}}
>>> done = {"method": "PATCH", "path": "/v/1/operations/a", "body": {"sequence_id": 1, "status": "OPERATION_STATUS_SUCCEEDED"}}
>>> merged = {"method": "PATCH", "path": "/v/1/operations/a", "body": {"sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
>>> fold_pending_operation_writes([pending, done]) == fold_pending_operation_writes([merged])
True
>>> fold_pending_operation_writes([pending, done])
[{'method': 'PATCH', 'path': '/v/1/operations/a', 'body': {'status': 'OPERATION_STATUS_SUCCEEDED'}}]
>>> fold_pending_operation_writes([pending])
[{'method': 'PATCH', 'path': '/v/1/operations/a', 'body': {'status': 'OPERATION_STATUS_PENDING'}}]
"""

def is_operation_patch(req):
return req.get("method") == "PATCH" and "/operations/" in req.get("path", "")

result = []
for i, req in enumerate(requests):
if not is_operation_patch(req):
result.append(req)
continue
body = req.get("body") or {}
if body.get("status") == "OPERATION_STATUS_PENDING" and any(
is_operation_patch(r) and r["path"] == req["path"] for r in requests[i + 1 :]
):
continue
result.append(dict(req, body={k: v for k, v in body.items() if k != "sequence_id"}))
return result


def main():
parser = argparse.ArgumentParser()
parser.add_argument("path_filters", nargs="*", help="Path substring filters")
Expand Down Expand Up @@ -274,6 +311,13 @@ def main():
"bodies, so a test asserts the same requests whether or not deployment history "
"recording is on. Shorthand for the --del-body fields it implies.",
)
parser.add_argument(
"--fold-pending",
action="store_true",
help="Fold the PENDING write of a recreated resource into the write that finishes it, and "
"drop operation sequence_ids, so a test reads the same whether or not the CLI merged them. "
"See fold_pending_operation_writes.",
)
parser.add_argument("--fname", default=os.environ["OUT_REQUESTS"])
args = parser.parse_args()

Expand Down Expand Up @@ -302,6 +346,8 @@ def main():
filtered_requests = filter_requests(
requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms
)
if args.fold_pending:
filtered_requests = fold_pending_operation_writes(filtered_requests)

for req in filtered_requests:
body = req.get("body")
Expand Down
15 changes: 1 addition & 14 deletions acceptance/bundle/dms/failed-recreate/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ API message: Fault injected by test.

Files: 2 uploaded, 0 deleted

>>> print_requests.py --dms //api/2.0/bundle
>>> print_requests.py --dms --fold-pending //api/2.0/bundle
{
"method": "POST",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
Expand All @@ -150,18 +150,6 @@ Files: 2 uploaded, 0 deleted
"version_type": "VERSION_TYPE_DEPLOY"
}
}
{
"method": "PATCH",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo",
"q": {
"update_mask": "state,error_message,status"
},
"body": {
"error_message": "",
"sequence_id": 0,
"status": "OPERATION_STATUS_PENDING"
}
}
{
"method": "PATCH",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo",
Expand All @@ -170,7 +158,6 @@ Files: 2 uploaded, 0 deleted
},
"body": {
"error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)",
"sequence_id": 1,
"status": "OPERATION_STATUS_FAILED"
}
}
Expand Down
4 changes: 3 additions & 1 deletion acceptance/bundle/dms/failed-recreate/script
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_
trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE
trace musterr $CLI bundle deploy --auto-approve
# Not sorted: the point of this test is the order the operations are recorded in.
trace print_requests.py --dms //api/2.0/bundle
# --fold-pending: the CLI merges the delete's PENDING write into the failure if it has not been
# sent yet, so the failure is the one write that always reaches the service.
trace print_requests.py --dms --fold-pending //api/2.0/bundle

title "The resource is not listed: state is what projects a resource, and the failed recreate left none"
deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
Expand Down
15 changes: 1 addition & 14 deletions acceptance/bundle/dms/successful-recreate/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ Recreated schemas.foo
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 1 deleted, 0 unchanged

>>> print_requests.py --dms //api/2.0/bundle
>>> print_requests.py --dms --fold-pending //api/2.0/bundle
{
"method": "POST",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
Expand All @@ -143,18 +143,6 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
"version_type": "VERSION_TYPE_DEPLOY"
}
}
{
"method": "PATCH",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo",
"q": {
"update_mask": "state,error_message,status"
},
"body": {
"error_message": "",
"sequence_id": 0,
"status": "OPERATION_STATUS_PENDING"
}
}
{
"method": "PATCH",
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo",
Expand All @@ -164,7 +152,6 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
"body": {
"error_message": "",
"resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]",
"sequence_id": 1,
"state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][2]\"}}",
"status": "OPERATION_STATUS_SUCCEEDED"
}
Expand Down
5 changes: 3 additions & 2 deletions acceptance/bundle/dms/successful-recreate/script
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ trace print_state.py --no-dms

title "Recreate writes state twice - the entry is dropped, then the new resource is saved"
# One operation per resource per version; both writes land on the same one.
# The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED.
# The drop marks it PENDING, the save patches it to SUCCEEDED. The CLI merges the drop into the
# save if the drop has not been sent yet, so --fold-pending prints the same either way.
# Unity Catalog has no catalog called "other", so make one to move the schema into.
other_catalog="dms_other_${UNIQUE_NAME}"
$CLI catalogs create "$other_catalog" &> LOG.catalog
trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT
trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}"
trace $CLI bundle deploy --auto-approve
trace print_requests.py --dms //api/2.0/bundle
trace print_requests.py --dms --fold-pending //api/2.0/bundle

title "The recreate is done: nothing left to plan, and the serial moved to 2"
dms_plan
Expand Down
Loading