diff --git a/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl b/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl new file mode 100644 index 00000000000..841c2f31fdf --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl @@ -0,0 +1,6 @@ +bundle: + name: dms-create-response-lost-$UNIQUE_NAME +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/create-response-lost/out.test.toml b/acceptance/bundle/dms/create-response-lost/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/create-response-lost/output.txt b/acceptance/bundle/dms/create-response-lost/output.txt new file mode 100644 index 00000000000..17236a569c0 --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/output.txt @@ -0,0 +1,68 @@ + +=== The first deploy's create is applied, but its response is lost + +>>> fault.py --after-handler POST /api/2.0/bundle/deployments 500 0 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== The deploy recovered the deployment: the job carries its id + +>>> [CLI] jobs get [JOB_ID] +{ + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" +} + +=== No drift after the deploy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A create that finds an earlier deployment with versions does not adopt it + +>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 3 + +>>> [CLI] bundle deploy +Warn: Could not read deployment history: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists +Updated jobs.foo +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Destroy, so the next deploy creates a new deployment + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== A create retried into ALREADY_EXISTS is recovered + +>>> fault.py --after-handler POST /api/2.0/bundle/deployments 503 0 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [JOB_ID_2] +{ + "deployment_id": "[DEPLOYMENT_ID_2]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/dms/create-response-lost/script b/acceptance/bundle/dms/create-response-lost/script new file mode 100644 index 00000000000..b0b1d7df064 --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/script @@ -0,0 +1,42 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_node="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-create-response-lost-${UNIQUE_NAME}/default/state/resources.deployment.json" + +show_job_deployment() { + job_id=$(jq -r '.state["resources.jobs.foo"].__id__' .databricks/bundle/default/resources.json) + add_repl "$job_id" JOB_ID + trace $CLI jobs get "$job_id" | jq '.settings.deployment' +} + +# The service creates the deployment, but the client never sees the answer (in CI: an HTTP/2 +# stream reset while the body is read). +title "The first deploy's create is applied, but its response is lost" +trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 500 0 1 +trace $CLI bundle deploy + +title "The deploy recovered the deployment: the job carries its id" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id) +add_repl "$deployment_id" DEPLOYMENT_ID +show_job_deployment + +title "No drift after the deploy" +trace $CLI bundle plan + +# The startup lookup fails, so the deploy tries to create the deployment again. The service +# rejects it as ALREADY_EXISTS, and the deployment it finds has versions: it existed before this +# run, so the deploy does not take it over. +title "A create that finds an earlier deployment with versions does not adopt it" +trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 3 +trace $CLI bundle deploy + +title "Destroy, so the next deploy creates a new deployment" +trace $CLI bundle destroy --auto-approve + +# The service answers 503 after the create was applied; the SDK retries it, and the retry is +# rejected as ALREADY_EXISTS. +title "A create retried into ALREADY_EXISTS is recovered" +trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 503 0 1 +trace $CLI bundle deploy +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id) +add_repl "$deployment_id" DEPLOYMENT_ID_2 +show_job_deployment +trace $CLI bundle plan diff --git a/acceptance/bundle/dms/create-response-lost/test.toml b/acceptance/bundle/dms/create-response-lost/test.toml new file mode 100644 index 00000000000..0f61e6a606c --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/test.toml @@ -0,0 +1,6 @@ +RecordRequests = false +# Local only: the lost responses are injected by fault.py, which a cloud run does not have. +Cloud = false + +# The startup lookup is retried after the injected 500s; do not wait between attempts. +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "1" diff --git a/acceptance/bundle/dms_fault/get_deployment/output.txt b/acceptance/bundle/dms_fault/get_deployment/output.txt index 0dda68edc1e..a2c05be253f 100644 --- a/acceptance/bundle/dms_fault/get_deployment/output.txt +++ b/acceptance/bundle/dms_fault/get_deployment/output.txt @@ -16,7 +16,9 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated jobs.foo Updated pipelines.bar Files: 3 uploaded, 0 deleted @@ -91,20 +93,20 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 - Updated jobs.foo + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated pipelines.bar Files: 6 uploaded, 0 deleted - Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === deployment_history: false >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... Updated jobs.foo - Updated pipelines.bar Files: 2 uploaded, 0 deleted - Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === deployment_history: true @@ -113,7 +115,9 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated jobs.foo Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -132,6 +136,9 @@ All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not record deployment history for this destroy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Destroy: 2 deleted === deployment_history: false diff --git a/acceptance/bundle/dms_fault/run.py b/acceptance/bundle/dms_fault/run.py index 11e22795ccc..b996d10d2cc 100644 --- a/acceptance/bundle/dms_fault/run.py +++ b/acceptance/bundle/dms_fault/run.py @@ -29,7 +29,7 @@ def main(): "create_deployment": "POST /api/2.0/bundle/deployments", "create_version": "POST /api/2.0/bundle/deployments/{id}/versions", "delete_deployment": "DELETE /api/2.0/bundle/deployments/{id}", - # Keep failing lookups even if the CLI registers a replacement deployment. + # Any id: the lookup after the rejected re-create fails too, so nothing is adopted. "get_deployment": "GET /api/2.0/bundle/deployments/*", "get_operation": "GET /api/2.0/bundle/deployments/{id}/versions/*/operations/*", "get_resource": "GET /api/2.0/bundle/deployments/{id}/resources/*", @@ -125,6 +125,7 @@ def activate(cwd, root): run(cwd, "workspace", "get-status", root + "/state/resources.deployment.json", "-o", "json").stdout ) deployment = str(node["object_id"]) + add_repl(deployment, "DEPLOYMENT_ID") pattern = PATTERNS[API].format(id=deployment) fault(pattern, True) print(f"\n=== {cwd.name}: {API} fault enabled", flush=True) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 38210913ce8..bd5d846420d 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -85,12 +85,16 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu dep.InitialParentPath = b.Config.Workspace.StatePath created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) - } - // The id is the workspace node the server makes under the parent path. - deploymentID, err = dms.DeploymentIDFromName(created.Name) - if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + deploymentID = recoverCreatedDeployment(ctx, b, err) + if deploymentID == "" { + return fmt.Errorf("failed to create deployment: %w", err) + } + } else { + // The id is the workspace node the server makes under the parent path. + deploymentID, err = dms.DeploymentIDFromName(created.Name) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } } db.StateDB.DeploymentID = deploymentID } else if mask := metadata.StaleFields(current); mask != "" { @@ -113,6 +117,26 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu return nil } +// recoverCreatedDeployment returns the id of the deployment a failed CreateDeployment may still +// have made, or "" if there is none. The server can apply the create and then lose the response +// (an HTTP/2 stream reset while the body is read, or a 504 the SDK retries into ALREADY_EXISTS). +// Without the id the deploy would write its resources without it, and the next plan, which finds +// the deployment, would report them as changed. A create that was rejected leaves no node, so the +// lookup runs after any error. Only a deployment with no versions is taken: one with versions +// existed before this run, and its version chain is not ours to continue. +func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr error) string { + id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + log.Debugf(ctx, "Could not look up the deployment after a failed create: %s", err) + return "" + } + if id == "" || deployment.LastVersionId != "" { + return "" + } + log.Debugf(ctx, "Deployment create failed (%s), but the deployment %s exists; using it", createErr, id) + return id +} + // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the // earlier attempt may have deleted it before its response was lost. func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploymentID string) error { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 68798a4bb01..a39f8096ff3 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "path" "path/filepath" "slices" "strconv" @@ -32,14 +31,10 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" "golang.org/x/mod/semver" @@ -411,7 +406,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState && stateMode == dstate.WritingDeploymentHistoryOnly { var err error - dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err = dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -637,40 +632,6 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { return nil } -// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json -// -// TODO: a deployment is only usable when both the node and the service's record exist, and a -// half-created one - node present, record missing - blocks the bundle here even though -// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) -// that returns an empty id and version unless both halves are there, leaving the caller to call -// CreateDeployment to create or finalize it. -// -// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a -// workspace lookup to turn the node into an id, then a get by that id. -func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { - nodePath := path.Join(statePath, dms.DeploymentNodeName) - - obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { - return w.Workspace.GetStatusByPath(ctx, nodePath) - }) - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - return "", nil, nil - } - if err != nil { - return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) - } - deploymentID := strconv.FormatInt(obj.ObjectId, 10) - deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { - return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: dms.DeploymentName(deploymentID), - }) - }) - if err != nil { - return "", nil, err - } - return deploymentID, deployment, nil -} - // parseLastVersionID parses the deployment's last recorded version, which the service reports // as a string. It returns 0 when the deployment does not exist yet or has no recorded version. func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { @@ -696,7 +657,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return nil } - dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return err } @@ -716,7 +677,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st // prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { - id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return nil, err } diff --git a/libs/dms/lookup.go b/libs/dms/lookup.go new file mode 100644 index 00000000000..bd09f8da4ed --- /dev/null +++ b/libs/dms/lookup.go @@ -0,0 +1,49 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// FetchDeployment returns the deployment recorded under statePath and its id, the object id of +// the workspace node the service creates there. It returns an empty id and a nil deployment when +// there is no node yet. +// +// TODO: a deployment is only usable when both the node and the service's record exist, and a +// half-created one - node present, record missing - blocks the bundle at startup even though +// CreateDeployment already recovers from it. Return an empty id and version unless both halves +// are there, leaving the caller to call CreateDeployment to create or finalize it. +// +// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a +// workspace lookup to turn the node into an id, then a get by that id. +func FetchDeployment(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { + return w.Workspace.GetStatusByPath(ctx, nodePath) + }) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { + return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index 9b524b4a016..357c8d3a573 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -124,15 +124,28 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { if resp, ok := s.requireParentDirectory(nodePath); !ok { return resp } - objectID := nextTreenodeID() - s.files[nodePath] = FileEntry{ - Info: workspace.ObjectInfo{ - // The service registers its own node type, which get-status reports back; - // a plain FILE is what the fake used to claim and the real API never returns. - ObjectType: "BUNDLE_DEPLOYMENT", - Path: nodePath, - ObjectId: objectID, - }, + // Like the service, a second create in the same folder is rejected once the record exists, + // and a node left without a record (a create that failed half-way) is finalized in place. + var objectID int64 + if existing, ok := s.files[nodePath]; ok { + objectID = existing.Info.ObjectId + if _, ok := s.DmsDeployments[strconv.FormatInt(objectID, 10)]; ok { + return Response{ + StatusCode: 409, + Body: map[string]string{"error_code": "ALREADY_EXISTS", "message": "deployment " + strconv.FormatInt(objectID, 10) + " already exists"}, + } + } + } else { + objectID = nextTreenodeID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + // The service registers its own node type, which get-status reports back; + // a plain FILE is what the fake used to claim and the real API never returns. + ObjectType: "BUNDLE_DEPLOYMENT", + Path: nodePath, + ObjectId: objectID, + }, + } } // The record carries no version yet; last_version_id stays empty until