Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
bundle:
name: dms-create-response-lost-$UNIQUE_NAME
resources:
jobs:
foo:
name: foo
2 changes: 2 additions & 0 deletions acceptance/bundle/dms/create-response-lost/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["true"]
68 changes: 68 additions & 0 deletions acceptance/bundle/dms/create-response-lost/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@

=== The first deploy's create is applied, but its response is lost

>>> fault.py --after-handler POST /api/2.0/bundle/deployments 500 0 1

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files...
Created jobs.foo
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== The deploy recovered the deployment: the job carries its id

>>> [CLI] jobs get [JOB_ID]
{
"deployment_id": "[DEPLOYMENT_ID]",
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json",
"version_id": "1"
}

=== No drift after the deploy

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

=== A create that finds an earlier deployment with versions does not adopt it

>>> fault.py GET /api/2.0/bundle/deployments/* 500 0 3

>>> [CLI] bundle deploy
Warn: Could not read deployment history: Fault injected by test.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files...
Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists
Updated jobs.foo
Files: 0 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 0 unchanged

=== Destroy, so the next deploy creates a new deployment

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.foo

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default

Destroy: 1 deleted

=== A create retried into ALREADY_EXISTS is recovered

>>> fault.py --after-handler POST /api/2.0/bundle/deployments 503 0 1

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/files...
Created jobs.foo
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> [CLI] jobs get [JOB_ID_2]
{
"deployment_id": "[DEPLOYMENT_ID_2]",
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-create-response-lost-[UNIQUE_NAME]/default/state/metadata.json",
"version_id": "1"
}

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
42 changes: 42 additions & 0 deletions acceptance/bundle/dms/create-response-lost/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
envsubst < databricks.yml.tmpl > databricks.yml
state_node="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-create-response-lost-${UNIQUE_NAME}/default/state/resources.deployment.json"

show_job_deployment() {
job_id=$(jq -r '.state["resources.jobs.foo"].__id__' .databricks/bundle/default/resources.json)
add_repl "$job_id" JOB_ID
trace $CLI jobs get "$job_id" | jq '.settings.deployment'
}

# The service creates the deployment, but the client never sees the answer (in CI: an HTTP/2
# stream reset while the body is read).
title "The first deploy's create is applied, but its response is lost"
trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 500 0 1
trace $CLI bundle deploy

title "The deploy recovered the deployment: the job carries its id"
deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id)
add_repl "$deployment_id" DEPLOYMENT_ID
show_job_deployment

title "No drift after the deploy"
trace $CLI bundle plan

# The startup lookup fails, so the deploy tries to create the deployment again. The service
# rejects it as ALREADY_EXISTS, and the deployment it finds has versions: it existed before this
# run, so the deploy does not take it over.
title "A create that finds an earlier deployment with versions does not adopt it"
trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 3
trace $CLI bundle deploy

title "Destroy, so the next deploy creates a new deployment"
trace $CLI bundle destroy --auto-approve

# The service answers 503 after the create was applied; the SDK retries it, and the retry is
# rejected as ALREADY_EXISTS.
title "A create retried into ALREADY_EXISTS is recovered"
trace fault.py --after-handler "POST /api/2.0/bundle/deployments" 503 0 1
trace $CLI bundle deploy
deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_node" | jq -r .object_id)
add_repl "$deployment_id" DEPLOYMENT_ID_2
show_job_deployment
trace $CLI bundle plan
6 changes: 6 additions & 0 deletions acceptance/bundle/dms/create-response-lost/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
RecordRequests = false
# Local only: the lost responses are injected by fault.py, which a cloud run does not have.
Cloud = false

# The startup lookup is retried after the injected 500s; do not wait between attempts.
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "1"
21 changes: 14 additions & 7 deletions acceptance/bundle/dms_fault/get_deployment/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not read deployment history: Fault injected by test.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files...
Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists
Updated jobs.foo
Updated pipelines.bar
Files: 3 uploaded, 0 deleted
Expand Down Expand Up @@ -91,20 +93,20 @@
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not read deployment history: Fault injected by test.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files...
Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9
Updated jobs.foo
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists
Updated pipelines.bar
Files: 6 uploaded, 0 deleted
Resources: 0 created, 2 changed, 0 deleted, 0 unchanged
Resources: 0 created, 1 changed, 0 deleted, 1 unchanged

=== deployment_history: false

>>> bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files...
Updated jobs.foo
Updated pipelines.bar
Files: 2 uploaded, 0 deleted
Resources: 0 created, 2 changed, 0 deleted, 0 unchanged
Resources: 0 created, 1 changed, 0 deleted, 1 unchanged

=== deployment_history: true

Expand All @@ -113,7 +115,9 @@
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not read deployment history: Fault injected by test.
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]/files...
Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not enable deployment history for this deploy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists
Updated jobs.foo
Files: 2 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
Expand All @@ -132,6 +136,9 @@

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_deployment-commands-[UNIQUE_NAME]

Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID]
Warn: Could not record deployment history for this destroy: failed to create deployment: deployment [DEPLOYMENT_ID] already exists
Destroy: 2 deleted

=== deployment_history: false
Expand Down
3 changes: 2 additions & 1 deletion acceptance/bundle/dms_fault/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def main():
"create_deployment": "POST /api/2.0/bundle/deployments",
"create_version": "POST /api/2.0/bundle/deployments/{id}/versions",
"delete_deployment": "DELETE /api/2.0/bundle/deployments/{id}",
# Keep failing lookups even if the CLI registers a replacement deployment.
# Any id: the lookup after the rejected re-create fails too, so nothing is adopted.
"get_deployment": "GET /api/2.0/bundle/deployments/*",
"get_operation": "GET /api/2.0/bundle/deployments/{id}/versions/*/operations/*",
"get_resource": "GET /api/2.0/bundle/deployments/{id}/resources/*",
Expand Down Expand Up @@ -125,6 +125,7 @@ def activate(cwd, root):
run(cwd, "workspace", "get-status", root + "/state/resources.deployment.json", "-o", "json").stdout
)
deployment = str(node["object_id"])
add_repl(deployment, "DEPLOYMENT_ID")
pattern = PATTERNS[API].format(id=deployment)
fault(pattern, True)
print(f"\n=== {cwd.name}: {API} fault enabled", flush=True)
Expand Down
36 changes: 30 additions & 6 deletions bundle/phases/dms.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,16 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu
dep.InitialParentPath = b.Config.Workspace.StatePath
created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep})
if err != nil {
return fmt.Errorf("failed to create deployment: %w", err)
}
// The id is the workspace node the server makes under the parent path.
deploymentID, err = dms.DeploymentIDFromName(created.Name)
if err != nil {
return fmt.Errorf("failed to create deployment: %w", err)
deploymentID = recoverCreatedDeployment(ctx, b, err)
if deploymentID == "" {
return fmt.Errorf("failed to create deployment: %w", err)
}
} else {
// The id is the workspace node the server makes under the parent path.
deploymentID, err = dms.DeploymentIDFromName(created.Name)
if err != nil {
return fmt.Errorf("failed to create deployment: %w", err)
}
}
db.StateDB.DeploymentID = deploymentID
} else if mask := metadata.StaleFields(current); mask != "" {
Expand All @@ -113,6 +117,26 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu
return nil
}

// recoverCreatedDeployment returns the id of the deployment a failed CreateDeployment may still
// have made, or "" if there is none. The server can apply the create and then lose the response
// (an HTTP/2 stream reset while the body is read, or a 504 the SDK retries into ALREADY_EXISTS).
// Without the id the deploy would write its resources without it, and the next plan, which finds
// the deployment, would report them as changed. A create that was rejected leaves no node, so the
// lookup runs after any error. Only a deployment with no versions is taken: one with versions
// existed before this run, and its version chain is not ours to continue.
func recoverCreatedDeployment(ctx context.Context, b *bundle.Bundle, createErr error) string {
id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
if err != nil {
log.Debugf(ctx, "Could not look up the deployment after a failed create: %s", err)
return ""
}
if id == "" || deployment.LastVersionId != "" {
Comment thread
denik marked this conversation as resolved.
Resolved
return ""
}
log.Debugf(ctx, "Deployment create failed (%s), but the deployment %s exists; using it", createErr, id)
return id
}

// deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the
// earlier attempt may have deleted it before its response was lost.
func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploymentID string) error {
Expand Down
45 changes: 3 additions & 42 deletions cmd/bundle/utils/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"path"
"path/filepath"
"slices"
"strconv"
Expand Down Expand Up @@ -32,14 +31,10 @@ import (
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/logdiag"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/cli/libs/sync"
"github.com/databricks/cli/libs/telemetry"
"github.com/databricks/cli/libs/telemetry/protos"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/bundledeployments"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/databricks/databricks-sdk-go/useragent"
"github.com/spf13/cobra"
"golang.org/x/mod/semver"
Expand Down Expand Up @@ -411,7 +406,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle

if needDirectState && stateMode == dstate.WritingDeploymentHistoryOnly {
var err error
dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
dmsDeploymentID, dmsDeployment, err = dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
if err != nil {
logdiag.LogError(ctx, err)
return b, stateDesc, root.ErrAlreadyPrinted
Expand Down Expand Up @@ -637,40 +632,6 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error {
return nil
}

// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json
//
// TODO: a deployment is only usable when both the node and the service's record exist, and a
// half-created one - node present, record missing - blocks the bundle here even though
// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath)
// that returns an empty id and version unless both halves are there, leaving the caller to call
// CreateDeployment to create or finalize it.
//
// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a
// workspace lookup to turn the node into an id, then a get by that id.
func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) {
nodePath := path.Join(statePath, dms.DeploymentNodeName)

obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) {
return w.Workspace.GetStatusByPath(ctx, nodePath)
})
if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) {
return "", nil, nil
}
if err != nil {
return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err)
}
deploymentID := strconv.FormatInt(obj.ObjectId, 10)
deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) {
return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{
Name: dms.DeploymentName(deploymentID),
})
})
if err != nil {
return "", nil, err
}
return deploymentID, deployment, nil
}

// parseLastVersionID parses the deployment's last recorded version, which the service reports
// as a string. It returns 0 when the deployment does not exist yet or has no recorded version.
func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) {
Expand All @@ -696,7 +657,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st
return nil
}

dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
dmsDeploymentID, dmsDeployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
if err != nil {
return err
}
Expand All @@ -716,7 +677,7 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st

// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS.
func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) {
id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
id, deployment, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
if err != nil {
return nil, err
}
Expand Down
49 changes: 49 additions & 0 deletions libs/dms/lookup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package dms

import (
"context"
"errors"
"fmt"
"path"
"strconv"

"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/bundledeployments"
"github.com/databricks/databricks-sdk-go/service/workspace"
)

// FetchDeployment returns the deployment recorded under statePath and its id, the object id of
// the workspace node the service creates there. It returns an empty id and a nil deployment when
// there is no node yet.
//
// TODO: a deployment is only usable when both the node and the service's record exist, and a
// half-created one - node present, record missing - blocks the bundle at startup even though
// CreateDeployment already recovers from it. Return an empty id and version unless both halves
// are there, leaving the caller to call CreateDeployment to create or finalize it.
//
// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a
// workspace lookup to turn the node into an id, then a get by that id.
func FetchDeployment(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) {
nodePath := path.Join(statePath, DeploymentNodeName)
obj, err := retry.With(ctx, retry.IsTransientRead, func() (*workspace.ObjectInfo, error) {
return w.Workspace.GetStatusByPath(ctx, nodePath)
})
if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) {
return "", nil, nil
}
if err != nil {
return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err)
}
deploymentID := strconv.FormatInt(obj.ObjectId, 10)
deployment, err := retry.With(ctx, retry.IsTransientRead, func() (*bundledeployments.Deployment, error) {
return w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{
Name: DeploymentName(deploymentID),
})
})
if err != nil {
return "", nil, err
}
return deploymentID, deployment, nil
}
Loading
Loading