From eedf1296de3f24f72c0ed4a73faef939728a4183 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 12:33:23 +0000 Subject: [PATCH 1/3] Recover the deployment when its create response is lost When CreateDeployment fails after the server applied it (an HTTP/2 stream reset while the body is read), the deploy wrote its jobs without deployment_id, and the next plan, which finds the deployment at the state path, reported them as changed. On any create error other than a 4xx, look the deployment up at the state path and use it if it exists. Also print the invariant tests' LOG.*.err files on failure: the "*.err" ignore pattern hid them, so a failing `bundle plan` left only "Exit code: 1". aws: TestAccept/bundle/invariant/no_drift/DMS=true Co-authored-by: Isaac --- .../create-response-lost/databricks.yml.tmpl | 6 +++ .../dms/create-response-lost/out.test.toml | 2 + .../dms/create-response-lost/output.txt | 25 +++++++++++ .../bundle/dms/create-response-lost/script | 17 ++++++++ .../bundle/dms/create-response-lost/test.toml | 3 ++ acceptance/bundle/invariant/test.toml | 1 - bundle/phases/dms.go | 43 ++++++++++++++++--- cmd/bundle/utils/process.go | 17 ++------ libs/dms/lookup.go | 30 +++++++++++++ 9 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/create-response-lost/out.test.toml create mode 100644 acceptance/bundle/dms/create-response-lost/output.txt create mode 100644 acceptance/bundle/dms/create-response-lost/script create mode 100644 acceptance/bundle/dms/create-response-lost/test.toml create mode 100644 libs/dms/lookup.go diff --git a/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl b/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl new file mode 100644 index 00000000000..841c2f31fdf --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/databricks.yml.tmpl @@ -0,0 +1,6 @@ +bundle: + name: dms-create-response-lost-$UNIQUE_NAME +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/create-response-lost/out.test.toml b/acceptance/bundle/dms/create-response-lost/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/create-response-lost/output.txt b/acceptance/bundle/dms/create-response-lost/output.txt new file mode 100644 index 00000000000..d031f27ece4 --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/output.txt @@ -0,0 +1,25 @@ + +=== The first deploy's create is applied, but its response is lost + +>>> fault.py --after-handler POST /api/2.0/bundle/deployments 500 0 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== The deploy recovered the deployment: the job carries its id + +>>> [CLI] jobs get [JOB_ID] +{ + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" +} + +=== No drift after the deploy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/dms/create-response-lost/script b/acceptance/bundle/dms/create-response-lost/script new file mode 100644 index 00000000000..aadabfd0921 --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# The service creates the deployment, but the client never sees the answer (in CI: an HTTP/2 +# stream reset while the body is read). A retried create would be rejected as ALREADY_EXISTS. +title "The first deploy's create is applied, but its response is lost" +trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 500 0 1 +trace $CLI bundle deploy + +title "The deploy recovered the deployment: the job carries its id" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-create-response-lost-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID +job_id=$(jq -r '.state["resources.jobs.foo"].__id__' .databricks/bundle/default/resources.json) +add_repl "$job_id" JOB_ID +trace $CLI jobs get "$job_id" | jq '.settings.deployment' + +title "No drift after the deploy" +trace $CLI bundle plan diff --git a/acceptance/bundle/dms/create-response-lost/test.toml b/acceptance/bundle/dms/create-response-lost/test.toml new file mode 100644 index 00000000000..f9329de096d --- /dev/null +++ b/acceptance/bundle/dms/create-response-lost/test.toml @@ -0,0 +1,3 @@ +RecordRequests = false +# Local only: the lost response is injected by fault.py, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 1c1abd19348..ba2889d9914 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -8,7 +8,6 @@ Ignore = [ "plan.json", "*.py", "*.json", - "*.err", "app", ] diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 38210913ce8..4655de8f705 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,7 +2,9 @@ package phases import ( "context" + "errors" "fmt" + "net/http" "net/url" "path" "strconv" @@ -85,12 +87,16 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu dep.InitialParentPath = b.Config.Workspace.StatePath created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) - } - // The id is the workspace node the server makes under the parent path. - deploymentID, err = dms.DeploymentIDFromName(created.Name) - if err != nil { - return fmt.Errorf("failed to create deployment: %w", err) + deploymentID = recoverCreatedDeployment(ctx, b, err) + if deploymentID == "" { + return fmt.Errorf("failed to create deployment: %w", err) + } + } else { + // The id is the workspace node the server makes under the parent path. + deploymentID, err = dms.DeploymentIDFromName(created.Name) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } } db.StateDB.DeploymentID = deploymentID } else if mask := metadata.StaleFields(current); mask != "" { @@ -113,6 +119,31 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu return nil } +// recoverCreatedDeployment returns the id of the deployment a failed CreateDeployment may still +// have made, or "" if there is none. The server can apply the create and then lose the response +// (an HTTP/2 stream reset while the body is read), and a repeated create is rejected as +// ALREADY_EXISTS. Without the id the deploy would write its resources without it, and the next +// plan, which finds the deployment, would report them as changed. A client error (4xx) means the +// create was not applied, so there is nothing to look up. +func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr error) string { + if apiErr, ok := errors.AsType[*apierr.APIError](createErr); ok && isClientError(apiErr.StatusCode) { + return "" + } + id, err := dms.LookupDeploymentID(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + log.Debugf(ctx, "Could not look up the deployment after a failed create: %s", err) + return "" + } + if id != "" { + log.Debugf(ctx, "Deployment create failed (%s), but the deployment %s exists; using it", createErr, id) + } + return id +} + +func isClientError(statusCode int) bool { + return statusCode >= 400 && statusCode < 500 && statusCode != http.StatusRequestTimeout && statusCode != http.StatusTooManyRequests +} + // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the // earlier attempt may have deleted it before its response was lost. func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploymentID string) error { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 68798a4bb01..d1abf94b41d 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "path" "path/filepath" "slices" "strconv" @@ -37,9 +36,7 @@ import ( "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" "golang.org/x/mod/semver" @@ -648,18 +645,10 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { // TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a // workspace lookup to turn the node into an id, then a get by that id. func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { - nodePath := path.Join(statePath, dms.DeploymentNodeName) - - obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { - return w.Workspace.GetStatusByPath(ctx, nodePath) - }) - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - return "", nil, nil - } - if err != nil { - return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + deploymentID, err := dms.LookupDeploymentID(ctx, w, statePath) + if err != nil || deploymentID == "" { + return "", nil, err } - deploymentID := strconv.FormatInt(obj.ObjectId, 10) deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: dms.DeploymentName(deploymentID), diff --git a/libs/dms/lookup.go b/libs/dms/lookup.go new file mode 100644 index 00000000000..f3430f465f2 --- /dev/null +++ b/libs/dms/lookup.go @@ -0,0 +1,30 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// LookupDeploymentID returns the id of the deployment recorded under statePath: the object id of +// the workspace node the service creates there. It returns "" when there is no deployment yet. +func LookupDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { + return w.Workspace.GetStatusByPath(ctx, nodePath) + }) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + return strconv.FormatInt(obj.ObjectId, 10), nil +} From fdf2714518aff785c73965e23f7b289a5ee90237 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 13:54:53 +0000 Subject: [PATCH 2/3] Recover on ALREADY_EXISTS too, and only adopt a deployment with no versions A create the SDK retries after a 504 or a timeout can come back as 409 once the first attempt was applied. A deployment that already has versions existed before this run, so it is not taken over. aws: TestAccept/bundle/invariant/no_drift/DMS=true Co-authored-by: Isaac --- bundle/phases/dms.go | 28 ++++++++++++++++++---------- bundle/phases/dms_test.go | 19 +++++++++++++++++++ cmd/bundle/utils/process.go | 34 +++------------------------------- libs/dms/lookup.go | 31 +++++++++++++++++++++++++------ 4 files changed, 65 insertions(+), 47 deletions(-) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 4655de8f705..b37d222d8bf 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -121,27 +121,35 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu // recoverCreatedDeployment returns the id of the deployment a failed CreateDeployment may still // have made, or "" if there is none. The server can apply the create and then lose the response -// (an HTTP/2 stream reset while the body is read), and a repeated create is rejected as -// ALREADY_EXISTS. Without the id the deploy would write its resources without it, and the next -// plan, which finds the deployment, would report them as changed. A client error (4xx) means the -// create was not applied, so there is nothing to look up. +// (an HTTP/2 stream reset while the body is read, or a 504 the SDK retries into ALREADY_EXISTS). +// Without the id the deploy would write its resources without it, and the next plan, which finds +// the deployment, would report them as changed. Only a deployment with no versions is taken: one +// with versions existed before this run, and its version chain is not ours to continue. func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr error) string { - if apiErr, ok := errors.AsType[*apierr.APIError](createErr); ok && isClientError(apiErr.StatusCode) { + if apiErr, ok := errors.AsType[*apierr.APIError](createErr); ok && !mayHaveCreated(apiErr) { return "" } - id, err := dms.LookupDeploymentID(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { log.Debugf(ctx, "Could not look up the deployment after a failed create: %s", err) return "" } - if id != "" { - log.Debugf(ctx, "Deployment create failed (%s), but the deployment %s exists; using it", createErr, id) + if id == "" || deployment.LastVersionId != "" { + return "" } + log.Debugf(ctx, "Deployment create failed (%s), but the deployment %s exists; using it", createErr, id) return id } -func isClientError(statusCode int) bool { - return statusCode >= 400 && statusCode < 500 && statusCode != http.StatusRequestTimeout && statusCode != http.StatusTooManyRequests +// mayHaveCreated reports whether a create that failed with err may still have been applied: a +// server error, a timeout, or a conflict with an earlier attempt. Other client errors mean the +// request was rejected. +func mayHaveCreated(err *apierr.APIError) bool { + switch err.StatusCode { + case http.StatusRequestTimeout, http.StatusConflict, http.StatusTooManyRequests: + return true + } + return err.StatusCode >= 500 || err.StatusCode < 400 } // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go index 6b045f863ae..512b5df7f72 100644 --- a/bundle/phases/dms_test.go +++ b/bundle/phases/dms_test.go @@ -1,9 +1,11 @@ package phases import ( + "net/http" "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -78,3 +80,20 @@ func TestActionToSDK(t *testing.T) { _, err = actionToSDK(deployplan.Undefined) assert.Error(t, err) } + +func TestMayHaveCreated(t *testing.T) { + // A lost response can surface as a server error, a timeout, or, after the SDK retried the + // create, a conflict with the attempt that was applied. Other client errors were rejected. + for status, want := range map[int]bool{ + http.StatusInternalServerError: true, + http.StatusGatewayTimeout: true, + http.StatusRequestTimeout: true, + http.StatusTooManyRequests: true, + http.StatusConflict: true, + http.StatusBadRequest: false, + http.StatusForbidden: false, + http.StatusNotFound: false, + } { + assert.Equal(t, want, mayHaveCreated(&apierr.APIError{StatusCode: status}), "status %d", status) + } +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index d1abf94b41d..a39f8096ff3 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -31,11 +31,9 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" @@ -408,7 +406,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState && stateMode == dstate.WritingDeploymentHistoryOnly { var err error - dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err = dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -634,32 +632,6 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { return nil } -// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json -// -// TODO: a deployment is only usable when both the node and the service's record exist, and a -// half-created one - node present, record missing - blocks the bundle here even though -// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) -// that returns an empty id and version unless both halves are there, leaving the caller to call -// CreateDeployment to create or finalize it. -// -// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a -// workspace lookup to turn the node into an id, then a get by that id. -func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { - deploymentID, err := dms.LookupDeploymentID(ctx, w, statePath) - if err != nil || deploymentID == "" { - return "", nil, err - } - deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { - return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: dms.DeploymentName(deploymentID), - }) - }) - if err != nil { - return "", nil, err - } - return deploymentID, deployment, nil -} - // parseLastVersionID parses the deployment's last recorded version, which the service reports // as a string. It returns 0 when the deployment does not exist yet or has no recorded version. func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { @@ -685,7 +657,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return nil } - dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return err } @@ -705,7 +677,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st // prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { - id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return nil, err } diff --git a/libs/dms/lookup.go b/libs/dms/lookup.go index f3430f465f2..bd09f8da4ed 100644 --- a/libs/dms/lookup.go +++ b/libs/dms/lookup.go @@ -10,21 +10,40 @@ import ( "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/databricks/databricks-sdk-go/service/workspace" ) -// LookupDeploymentID returns the id of the deployment recorded under statePath: the object id of -// the workspace node the service creates there. It returns "" when there is no deployment yet. -func LookupDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { +// FetchDeployment returns the deployment recorded under statePath and its id, the object id of +// the workspace node the service creates there. It returns an empty id and a nil deployment when +// there is no node yet. +// +// TODO: a deployment is only usable when both the node and the service's record exist, and a +// half-created one - node present, record missing - blocks the bundle at startup even though +// CreateDeployment already recovers from it. Return an empty id and version unless both halves +// are there, leaving the caller to call CreateDeployment to create or finalize it. +// +// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a +// workspace lookup to turn the node into an id, then a get by that id. +func FetchDeployment(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { nodePath := path.Join(statePath, DeploymentNodeName) obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) { return w.Workspace.GetStatusByPath(ctx, nodePath) }) if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - return "", nil + return "", nil, nil } if err != nil { - return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) } - return strconv.FormatInt(obj.ObjectId, 10), nil + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) { + return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil } From b376767636237cd4e99c0c3b1e8005dfd6d6116a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 16:41:59 +0000 Subject: [PATCH 3/3] Look up the deployment after any create error; fake rejects a repeated create Per review: a rejected create leaves no node, so the lookup can run after every error, and the status classification goes. The testserver's CreateDeployment now matches the service: a second create in the same folder fails with ALREADY_EXISTS (it used to overwrite the node), and a node with no record is finalized in place. create-response-lost covers the recovered 409 after an SDK retry and the refused deployment with versions. The LOG.*.err change moves to its own PR. aws: TestAccept/bundle/invariant/no_drift/DMS=true Co-authored-by: Isaac --- .../dms/create-response-lost/output.txt | 43 +++++++++++++++++++ .../bundle/dms/create-response-lost/script | 35 ++++++++++++--- .../bundle/dms/create-response-lost/test.toml | 5 ++- .../dms_fault/get_deployment/output.txt | 21 ++++++--- acceptance/bundle/dms_fault/run.py | 3 +- acceptance/bundle/invariant/test.toml | 1 + bundle/phases/dms.go | 21 ++------- bundle/phases/dms_test.go | 19 -------- libs/testserver/bundledeployments.go | 31 +++++++++---- 9 files changed, 119 insertions(+), 60 deletions(-) diff --git a/acceptance/bundle/dms/create-response-lost/output.txt b/acceptance/bundle/dms/create-response-lost/output.txt index d031f27ece4..17236a569c0 100644 --- a/acceptance/bundle/dms/create-response-lost/output.txt +++ b/acceptance/bundle/dms/create-response-lost/output.txt @@ -23,3 +23,46 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A create that finds an earlier deployment with versions does not adopt it + +>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 3 + +>>> [CLI] bundle deploy +Warn: Could not read deployment history: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists +Updated jobs.foo +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Destroy, so the next deploy creates a new deployment + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== A create retried into ALREADY_EXISTS is recovered + +>>> fault.py --after-handler POST /api/2.0/bundle/deployments 503 0 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [JOB_ID_2] +{ + "deployment_id": "[DEPLOYMENT_ID_2]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/dms/create-response-lost/script b/acceptance/bundle/dms/create-response-lost/script index aadabfd0921..b0b1d7df064 100644 --- a/acceptance/bundle/dms/create-response-lost/script +++ b/acceptance/bundle/dms/create-response-lost/script @@ -1,17 +1,42 @@ envsubst < databricks.yml.tmpl > databricks.yml +state_node="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-create-response-lost-${UNIQUE_NAME}/default/state/resources.deployment.json" + +show_job_deployment() { + job_id=$(jq -r '.state["resources.jobs.foo"].__id__' .databricks/bundle/default/resources.json) + add_repl "$job_id" JOB_ID + trace $CLI jobs get "$job_id" | jq '.settings.deployment' +} # The service creates the deployment, but the client never sees the answer (in CI: an HTTP/2 -# stream reset while the body is read). A retried create would be rejected as ALREADY_EXISTS. +# stream reset while the body is read). title "The first deploy's create is applied, but its response is lost" trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 500 0 1 trace $CLI bundle deploy title "The deploy recovered the deployment: the job carries its id" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-create-response-lost-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id) add_repl "$deployment_id" DEPLOYMENT_ID -job_id=$(jq -r '.state["resources.jobs.foo"].__id__' .databricks/bundle/default/resources.json) -add_repl "$job_id" JOB_ID -trace $CLI jobs get "$job_id" | jq '.settings.deployment' +show_job_deployment title "No drift after the deploy" trace $CLI bundle plan + +# The startup lookup fails, so the deploy tries to create the deployment again. The service +# rejects it as ALREADY_EXISTS, and the deployment it finds has versions: it existed before this +# run, so the deploy does not take it over. +title "A create that finds an earlier deployment with versions does not adopt it" +trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 3 +trace $CLI bundle deploy + +title "Destroy, so the next deploy creates a new deployment" +trace $CLI bundle destroy --auto-approve + +# The service answers 503 after the create was applied; the SDK retries it, and the retry is +# rejected as ALREADY_EXISTS. +title "A create retried into ALREADY_EXISTS is recovered" +trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 503 0 1 +trace $CLI bundle deploy +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id) +add_repl "$deployment_id" DEPLOYMENT_ID_2 +show_job_deployment +trace $CLI bundle plan diff --git a/acceptance/bundle/dms/create-response-lost/test.toml b/acceptance/bundle/dms/create-response-lost/test.toml index f9329de096d..0f61e6a606c 100644 --- a/acceptance/bundle/dms/create-response-lost/test.toml +++ b/acceptance/bundle/dms/create-response-lost/test.toml @@ -1,3 +1,6 @@ RecordRequests = false -# Local only: the lost response is injected by fault.py, which a cloud run does not have. +# Local only: the lost responses are injected by fault.py, which a cloud run does not have. Cloud = false + +# The startup lookup is retried after the injected 500s; do not wait between attempts. +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "1" diff --git a/acceptance/bundle/dms_fault/get_deployment/output.txt b/acceptance/bundle/dms_fault/get_deployment/output.txt index 0dda68edc1e..a2c05be253f 100644 --- a/acceptance/bundle/dms_fault/get_deployment/output.txt +++ b/acceptance/bundle/dms_fault/get_deployment/output.txt @@ -16,7 +16,9 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated jobs.foo Updated pipelines.bar Files: 3 uploaded, 0 deleted @@ -91,20 +93,20 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 - Updated jobs.foo + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated pipelines.bar Files: 6 uploaded, 0 deleted - Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === deployment_history: false >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... Updated jobs.foo - Updated pipelines.bar Files: 2 uploaded, 0 deleted - Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === deployment_history: true @@ -113,7 +115,9 @@ Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] Warn: Could not read deployment history: Fault injected by test. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Updated jobs.foo Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -132,6 +136,9 @@ All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Could not record deployment history for this destroy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists Destroy: 2 deleted === deployment_history: false diff --git a/acceptance/bundle/dms_fault/run.py b/acceptance/bundle/dms_fault/run.py index 11e22795ccc..b996d10d2cc 100644 --- a/acceptance/bundle/dms_fault/run.py +++ b/acceptance/bundle/dms_fault/run.py @@ -29,7 +29,7 @@ def main(): "create_deployment": "POST /api/2.0/bundle/deployments", "create_version": "POST /api/2.0/bundle/deployments/{id}/versions", "delete_deployment": "DELETE /api/2.0/bundle/deployments/{id}", - # Keep failing lookups even if the CLI registers a replacement deployment. + # Any id: the lookup after the rejected re-create fails too, so nothing is adopted. "get_deployment": "GET /api/2.0/bundle/deployments/*", "get_operation": "GET /api/2.0/bundle/deployments/{id}/versions/*/operations/*", "get_resource": "GET /api/2.0/bundle/deployments/{id}/resources/*", @@ -125,6 +125,7 @@ def activate(cwd, root): run(cwd, "workspace", "get-status", root + "/state/resources.deployment.json", "-o", "json").stdout ) deployment = str(node["object_id"]) + add_repl(deployment, "DEPLOYMENT_ID") pattern = PATTERNS[API].format(id=deployment) fault(pattern, True) print(f"\n=== {cwd.name}: {API} fault enabled", flush=True) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ba2889d9914..1c1abd19348 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -8,6 +8,7 @@ Ignore = [ "plan.json", "*.py", "*.json", + "*.err", "app", ] diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index b37d222d8bf..bd5d846420d 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,9 +2,7 @@ package phases import ( "context" - "errors" "fmt" - "net/http" "net/url" "path" "strconv" @@ -123,12 +121,10 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu // have made, or "" if there is none. The server can apply the create and then lose the response // (an HTTP/2 stream reset while the body is read, or a 504 the SDK retries into ALREADY_EXISTS). // Without the id the deploy would write its resources without it, and the next plan, which finds -// the deployment, would report them as changed. Only a deployment with no versions is taken: one -// with versions existed before this run, and its version chain is not ours to continue. +// the deployment, would report them as changed. A create that was rejected leaves no node, so the +// lookup runs after any error. Only a deployment with no versions is taken: one with versions +// existed before this run, and its version chain is not ours to continue. func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr error) string { - if apiErr, ok := errors.AsType[*apierr.APIError](createErr); ok && !mayHaveCreated(apiErr) { - return "" - } id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { log.Debugf(ctx, "Could not look up the deployment after a failed create: %s", err) @@ -141,17 +137,6 @@ func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr e return id } -// mayHaveCreated reports whether a create that failed with err may still have been applied: a -// server error, a timeout, or a conflict with an earlier attempt. Other client errors mean the -// request was rejected. -func mayHaveCreated(err *apierr.APIError) bool { - switch err.StatusCode { - case http.StatusRequestTimeout, http.StatusConflict, http.StatusTooManyRequests: - return true - } - return err.StatusCode >= 500 || err.StatusCode < 400 -} - // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the // earlier attempt may have deleted it before its response was lost. func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploymentID string) error { diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go index 512b5df7f72..6b045f863ae 100644 --- a/bundle/phases/dms_test.go +++ b/bundle/phases/dms_test.go @@ -1,11 +1,9 @@ package phases import ( - "net/http" "testing" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -80,20 +78,3 @@ func TestActionToSDK(t *testing.T) { _, err = actionToSDK(deployplan.Undefined) assert.Error(t, err) } - -func TestMayHaveCreated(t *testing.T) { - // A lost response can surface as a server error, a timeout, or, after the SDK retried the - // create, a conflict with the attempt that was applied. Other client errors were rejected. - for status, want := range map[int]bool{ - http.StatusInternalServerError: true, - http.StatusGatewayTimeout: true, - http.StatusRequestTimeout: true, - http.StatusTooManyRequests: true, - http.StatusConflict: true, - http.StatusBadRequest: false, - http.StatusForbidden: false, - http.StatusNotFound: false, - } { - assert.Equal(t, want, mayHaveCreated(&apierr.APIError{StatusCode: status}), "status %d", status) - } -} diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index 9b524b4a016..357c8d3a573 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -124,15 +124,28 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { if resp, ok := s.requireParentDirectory(nodePath); !ok { return resp } - objectID := nextTreenodeID() - s.files[nodePath] = FileEntry{ - Info: workspace.ObjectInfo{ - // The service registers its own node type, which get-status reports back; - // a plain FILE is what the fake used to claim and the real API never returns. - ObjectType: "BUNDLE_DEPLOYMENT", - Path: nodePath, - ObjectId: objectID, - }, + // Like the service, a second create in the same folder is rejected once the record exists, + // and a node left without a record (a create that failed half-way) is finalized in place. + var objectID int64 + if existing, ok := s.files[nodePath]; ok { + objectID = existing.Info.ObjectId + if _, ok := s.DmsDeployments[strconv.FormatInt(objectID, 10)]; ok { + return Response{ + StatusCode: 409, + Body: map[string]string{"error_code": "ALREADY_EXISTS", "message": "deployment " + strconv.FormatInt(objectID, 10) + " already exists"}, + } + } + } else { + objectID = nextTreenodeID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + // The service registers its own node type, which get-status reports back; + // a plain FILE is what the fake used to claim and the real API never returns. + ObjectType: "BUNDLE_DEPLOYMENT", + Path: nodePath, + ObjectId: objectID, + }, + } } // The record carries no version yet; last_version_id stays empty until