Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7757.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the feature, model serving endpoint or dashboard that a create request created despite timing out, instead of failing the deploy, in the direct deployment engine. ([#7757](https://github.com/databricks/cli/pull/7757))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ }
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
bundle:
name: test-bundle

resources:
dashboards:
dashboard1:
display_name: test-dashboard
warehouse_id: warehouse-1234
file_path: dashboard.lvdash.json
parent_path: /Users/${workspace.current_user.userName}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
33 changes: 33 additions & 0 deletions acceptance/bundle/resources/dashboards/create_504/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

=== The create request times out but the dashboard was created: deploy uses the dashboard

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards
Created dashboards.dashboard1
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //lakeview //workspace/get-status
POST /api/2.0/lakeview/dashboards
GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json
GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID]
POST /api/2.0/lakeview/dashboards
GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json
GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID]
POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published

=== The create request times out and the dashboard was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards
Created dashboards.dashboard1
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //lakeview //workspace/get-status
POST /api/2.0/lakeview/dashboards
GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json
POST /api/2.0/lakeview/dashboards
POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published
20 changes: 20 additions & 0 deletions acceptance/bundle/resources/dashboards/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# dashboards/test.toml sets MSYS_NO_PATHCONV, which keeps Python from finding these helpers on
# Windows; see the same workaround in acceptance/script.prepare.
fault.py() { env -u MSYS_NO_PATHCONV "$(type -P fault.py)" "$@"; }
print_requests.py() { env -u MSYS_NO_PATHCONV "$(type -P print_requests.py)" "$@"; }

title "The create request times out but the dashboard was created: deploy uses the dashboard"
# The first read of a new dashboard returns 404 (INJECT_STALE_ON_DIRECT), so the deploy repeats
# the create, gets ALREADY_EXISTS, and finds the dashboard on the next look.
fault.py --after-handler "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out and the dashboard was not created: deploy repeats the create"
fault.py "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
6 changes: 6 additions & 0 deletions acceptance/bundle/resources/dashboards/create_504/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
23 changes: 23 additions & 0 deletions acceptance/bundle/resources/features/create_504/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
bundle:
name: test-bundle

resources:
features:
feature1:
# UC lowercases the name; the lookup after a timed-out create must too.
full_name: main.myschema.MyFeature
source:
delta_table_source:
full_name: main.myschema.mysource
function:
aggregation_function:
count_function:
input: amount
time_window:
sliding:
window_duration: 604800s
slide_duration: 86400s
entities:
- name: account_id
timeseries_column:
name: event_ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
46 changes: 46 additions & 0 deletions acceptance/bundle/resources/features/create_504/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@

=== The create request times out but the feature was created: deploy uses the feature

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created features.feature1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //feature-engineering
POST /api/2.0/feature-engineering/features
GET /api/2.0/feature-engineering/features/main.myschema.myfeature

=== The create request times out and the feature was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.features.feature1: retrying after 504 Gateway Timeout from POST /api/2.0/feature-engineering/features
Created features.feature1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //feature-engineering
POST /api/2.0/feature-engineering/features
GET /api/2.0/feature-engineering/features/main.myschema.myfeature
POST /api/2.0/feature-engineering/features

=== The feature cannot be read after the create request times out: deploy fails with the create error

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.features.feature1: checking whether feature main.myschema.myfeature was created: Fault injected by test.
Error: cannot create resources.features.feature1: Fault injected by test. (504 DEADLINE_EXCEEDED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/features
HTTP Status: 504 Gateway Timeout
API error_code: DEADLINE_EXCEEDED
API message: Fault injected by test.

Files: 1 uploaded, 0 deleted

Exit code: 1

>>> print_requests.py --get //feature-engineering
POST /api/2.0/feature-engineering/features
GET /api/2.0/feature-engineering/features/main.myschema.myfeature
20 changes: 20 additions & 0 deletions acceptance/bundle/resources/features/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
title "The create request times out but the feature was created: deploy uses the feature"
fault.py --after-handler "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out and the feature was not created: deploy repeats the create"
fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
Comment thread
denik marked this conversation as resolved.
Resolved

title "The feature cannot be read after the create request times out: deploy fails with the create error"
fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.0/feature-engineering/features/*" 403 0 1 PERMISSION_DENIED
errcode trace $CLI bundle deploy
trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
6 changes: 6 additions & 0 deletions acceptance/bundle/resources/features/create_504/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bundle:
name: test-bundle

resources:
model_serving_endpoints:
my_endpoint:
name: test-endpoint
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@

=== The create request times out but the endpoint was created: deploy uses the endpoint

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created model_serving_endpoints.my_endpoint
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //serving-endpoints
POST /api/2.0/serving-endpoints
GET /api/2.0/serving-endpoints/test-endpoint
GET /api/2.0/serving-endpoints/test-endpoint

=== The create request times out and the endpoint was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_serving_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/serving-endpoints
Created model_serving_endpoints.my_endpoint
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //serving-endpoints
POST /api/2.0/serving-endpoints
GET /api/2.0/serving-endpoints/test-endpoint
POST /api/2.0/serving-endpoints
GET /api/2.0/serving-endpoints/test-endpoint
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
title "The create request times out but the endpoint was created: deploy uses the endpoint"
fault.py --after-handler "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out and the endpoint was not created: deploy repeats the create"
fault.py "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
112 changes: 112 additions & 0 deletions bundle/direct/dresources/create_recovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package dresources

import (
"context"
"errors"
"strings"
"time"

"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/apierr"
)

// createTimeSkew is how much earlier than our first create request a found resource's create time
// may be, to allow for clock skew, and still count as created by that request.
const createTimeSkew = time.Minute

// createRecovery describes a create request that failed, for a resource whose name the bundle
// chooses and the API keeps unique. T is the SDK type the create and the lookup return.
type createRecovery[T any] struct {
// name identifies the resource in messages, e.g. the endpoint name or the dashboard path.
name string
// start is when the first create request was sent.
start time.Time
// get looks the resource up by name. It must fail with a 404 when the resource is missing.
get func(ctx context.Context) (*T, error)
// createTime returns when the found resource was created, or false if the response omits it.
createTime func(found *T) (time.Time, bool)
// create repeats the create request.
create func(ctx context.Context) (*T, error)
}

// recoverCreate handles a create request that failed with createErr. A slow create is answered
// with 504 DEADLINE_EXCEEDED, which does not say whether the resource was created, so repeating
// the request blindly can fail with "already exists" for a resource that the next deploy would
// then also fail to create. Instead, look the resource up: use it if our request created it, and
// repeat the request while it is missing. A repeat that fails because the resource exists means an
// earlier request has landed in the meantime, so the next look finds it.
//
// Errors that are not transient, and a done context, are returned as is.
func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) {
if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) {
return nil, createErr
}

lastErr := createErr
for attempt := 1; ; attempt++ {
found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) {
return rc.get(ctx)
})
if ctx.Err() != nil {
return nil, ctx.Err()
}
if err == nil {
// Without the time check, a resource that existed before the deploy would be taken over:
// our request would have failed with "already exists" only if it reached that check.
created, ok := rc.createTime(found)
if !ok {
return nil, &recoveryError{msg: rc.name + " exists but has no create time, so it is not known to be created by this deployment", err: createErr}
}
if created.Before(rc.start.Add(-createTimeSkew)) {
return nil, &recoveryError{msg: rc.name + " exists but was not created by this deployment", err: createErr}
}
// Info, not Warn: the deploy got the resource it asked for, so there is nothing to act on.
log.Infof(ctx, "using %s, created despite the failed create request: %s", rc.name, strings.TrimSpace(diag.FormatAPIErrorSummary(createErr)))
return found, nil
}
if !apierr.IsMissing(err) {
log.Warnf(ctx, "checking whether %s was created: %s", rc.name, err)
return nil, createErr
}
if attempt > retry.MaxRetries {
return nil, lastErr
}
if err := retry.Pause(ctx, lastErr); err != nil {
return nil, err
}
created, err := rc.create(ctx)
if err == nil {
return created, nil
}
if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) {
return nil, err
}
lastErr = err
}
}

// recoveryError says why a resource found after a failed create request is not used, and wraps
// that request's error so the bundle error output still shows its status code and details. The
// 504s these requests get carry an empty message, so the message is left out when it is empty.
type recoveryError struct {
msg string
err error
}

func (e *recoveryError) Error() string {
if m := e.err.Error(); m != "" {
return e.msg + ": " + m
}
return e.msg
}

func (e *recoveryError) Unwrap() error { return e.err }

// isAlreadyExists reports whether a create request failed because the resource exists. Features,
// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with
// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11).
func isAlreadyExists(err error) bool {
return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists)
}
Loading
Loading