diff --git a/.nextchanges/bundles/7757.md b/.nextchanges/bundles/7757.md new file mode 100644 index 00000000000..144244eafe9 --- /dev/null +++ b/.nextchanges/bundles/7757.md @@ -0,0 +1 @@ +* Use the feature, model serving endpoint or dashboard that a create request created despite timing out, instead of failing the deploy, in the direct deployment engine. ([#7757](https://github.com/databricks/cli/pull/7757)) diff --git a/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json b/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json new file mode 100644 index 00000000000..ffcd4415b08 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json @@ -0,0 +1 @@ +{ } diff --git a/acceptance/bundle/resources/dashboards/create_504/databricks.yml b/acceptance/bundle/resources/dashboards/create_504/databricks.yml new file mode 100644 index 00000000000..8c1157635d3 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + dashboards: + dashboard1: + display_name: test-dashboard + warehouse_id: warehouse-1234 + file_path: dashboard.lvdash.json + parent_path: /Users/${workspace.current_user.userName} diff --git a/acceptance/bundle/resources/dashboards/create_504/out.test.toml b/acceptance/bundle/resources/dashboards/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/dashboards/create_504/output.txt b/acceptance/bundle/resources/dashboards/create_504/output.txt new file mode 100644 index 00000000000..4dca4601cca --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/output.txt @@ -0,0 +1,33 @@ + +=== The create request times out but the dashboard was created: deploy uses the dashboard + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards +Created dashboards.dashboard1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //lakeview //workspace/get-status +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID] +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID] +POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published + +=== The create request times out and the dashboard was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards +Created dashboards.dashboard1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //lakeview //workspace/get-status +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +POST /api/2.0/lakeview/dashboards +POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published diff --git a/acceptance/bundle/resources/dashboards/create_504/script b/acceptance/bundle/resources/dashboards/create_504/script new file mode 100644 index 00000000000..19cfefa4184 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/script @@ -0,0 +1,20 @@ +# dashboards/test.toml sets MSYS_NO_PATHCONV, which keeps Python from finding these helpers on +# Windows; see the same workaround in acceptance/script.prepare. +fault.py() { env -u MSYS_NO_PATHCONV "$(type -P fault.py)" "$@"; } +print_requests.py() { env -u MSYS_NO_PATHCONV "$(type -P print_requests.py)" "$@"; } + +title "The create request times out but the dashboard was created: deploy uses the dashboard" +# The first read of a new dashboard returns 404 (INJECT_STALE_ON_DIRECT), so the deploy repeats +# the create, gets ALREADY_EXISTS, and finds the dashboard on the next look. +fault.py --after-handler "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the dashboard was not created: deploy repeats the create" +fault.py "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/dashboards/create_504/test.toml b/acceptance/bundle/resources/dashboards/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/features/create_504/databricks.yml b/acceptance/bundle/resources/features/create_504/databricks.yml new file mode 100644 index 00000000000..4c2fb81cb8b --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bundle + +resources: + features: + feature1: + # UC lowercases the name; the lookup after a timed-out create must too. + full_name: main.myschema.MyFeature + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts diff --git a/acceptance/bundle/resources/features/create_504/out.test.toml b/acceptance/bundle/resources/features/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/features/create_504/output.txt b/acceptance/bundle/resources/features/create_504/output.txt new file mode 100644 index 00000000000..38a73d5fe72 --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/output.txt @@ -0,0 +1,46 @@ + +=== The create request times out but the feature was created: deploy uses the feature + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature + +=== The create request times out and the feature was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.features.feature1: retrying after 504 Gateway Timeout from POST /api/2.0/feature-engineering/features +Created features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature +POST /api/2.0/feature-engineering/features + +=== The feature cannot be read after the create request times out: deploy fails with the create error + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.features.feature1: checking whether feature main.myschema.myfeature was created: Fault injected by test. +Error: cannot create resources.features.feature1: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/features +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature diff --git a/acceptance/bundle/resources/features/create_504/script b/acceptance/bundle/resources/features/create_504/script new file mode 100644 index 00000000000..ba647ee37cb --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/script @@ -0,0 +1,20 @@ +title "The create request times out but the feature was created: deploy uses the feature" +fault.py --after-handler "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the feature was not created: deploy repeats the create" +fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The feature cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/feature-engineering/features/*" 403 0 1 PERMISSION_DENIED +errcode trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/acceptance/bundle/resources/features/create_504/test.toml b/acceptance/bundle/resources/features/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml new file mode 100644 index 00000000000..9813e7c7d66 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + my_endpoint: + name: test-endpoint diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt b/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt new file mode 100644 index 00000000000..390a38f4ba9 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt @@ -0,0 +1,28 @@ + +=== The create request times out but the endpoint was created: deploy uses the endpoint + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_serving_endpoints.my_endpoint +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //serving-endpoints +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint +GET /api/2.0/serving-endpoints/test-endpoint + +=== The create request times out and the endpoint was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_serving_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/serving-endpoints +Created model_serving_endpoints.my_endpoint +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //serving-endpoints +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/script b/acceptance/bundle/resources/model_serving_endpoints/create_504/script new file mode 100644 index 00000000000..4b5028c747a --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/script @@ -0,0 +1,13 @@ +title "The create request times out but the endpoint was created: deploy uses the endpoint" +fault.py --after-handler "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the endpoint was not created: deploy repeats the create" +fault.py "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml b/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go new file mode 100644 index 00000000000..f27a078364e --- /dev/null +++ b/bundle/direct/dresources/create_recovery.go @@ -0,0 +1,112 @@ +package dresources + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// createTimeSkew is how much earlier than our first create request a found resource's create time +// may be, to allow for clock skew, and still count as created by that request. +const createTimeSkew = time.Minute + +// createRecovery describes a create request that failed, for a resource whose name the bundle +// chooses and the API keeps unique. T is the SDK type the create and the lookup return. +type createRecovery[T any] struct { + // name identifies the resource in messages, e.g. the endpoint name or the dashboard path. + name string + // start is when the first create request was sent. + start time.Time + // get looks the resource up by name. It must fail with a 404 when the resource is missing. + get func(ctx context.Context) (*T, error) + // createTime returns when the found resource was created, or false if the response omits it. + createTime func(found *T) (time.Time, bool) + // create repeats the create request. + create func(ctx context.Context) (*T, error) +} + +// recoverCreate handles a create request that failed with createErr. A slow create is answered +// with 504 DEADLINE_EXCEEDED, which does not say whether the resource was created, so repeating +// the request blindly can fail with "already exists" for a resource that the next deploy would +// then also fail to create. Instead, look the resource up: use it if our request created it, and +// repeat the request while it is missing. A repeat that fails because the resource exists means an +// earlier request has landed in the meantime, so the next look finds it. +// +// Errors that are not transient, and a done context, are returned as is. +func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { + return nil, createErr + } + + lastErr := createErr + for attempt := 1; ; attempt++ { + found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) { + return rc.get(ctx) + }) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err == nil { + // Without the time check, a resource that existed before the deploy would be taken over: + // our request would have failed with "already exists" only if it reached that check. + created, ok := rc.createTime(found) + if !ok { + return nil, &recoveryError{msg: rc.name + " exists but has no create time, so it is not known to be created by this deployment", err: createErr} + } + if created.Before(rc.start.Add(-createTimeSkew)) { + return nil, &recoveryError{msg: rc.name + " exists but was not created by this deployment", err: createErr} + } + // Info, not Warn: the deploy got the resource it asked for, so there is nothing to act on. + log.Infof(ctx, "using %s, created despite the failed create request: %s", rc.name, strings.TrimSpace(diag.FormatAPIErrorSummary(createErr))) + return found, nil + } + if !apierr.IsMissing(err) { + log.Warnf(ctx, "checking whether %s was created: %s", rc.name, err) + return nil, createErr + } + if attempt > retry.MaxRetries { + return nil, lastErr + } + if err := retry.Pause(ctx, lastErr); err != nil { + return nil, err + } + created, err := rc.create(ctx) + if err == nil { + return created, nil + } + if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) { + return nil, err + } + lastErr = err + } +} + +// recoveryError says why a resource found after a failed create request is not used, and wraps +// that request's error so the bundle error output still shows its status code and details. The +// 504s these requests get carry an empty message, so the message is left out when it is empty. +type recoveryError struct { + msg string + err error +} + +func (e *recoveryError) Error() string { + if m := e.err.Error(); m != "" { + return e.msg + ": " + m + } + return e.msg +} + +func (e *recoveryError) Unwrap() error { return e.err } + +// isAlreadyExists reports whether a create request failed because the resource exists. Features, +// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with +// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). +func isAlreadyExists(err error) bool { + return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) +} diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go new file mode 100644 index 00000000000..2ce23bcf248 --- /dev/null +++ b/bundle/direct/dresources/create_recovery_test.go @@ -0,0 +1,193 @@ +package dresources + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testFeatureName = "main.default.feat" + +// featureCreateScript plays the given responses, in order, to the feature create and get +// requests. A response is an ml.Feature or a testserver.Response for an error. +type featureCreateScript struct { + t *testing.T + creates []any + gets []any + // onGet runs when a get request arrives, before its response is sent. + onGet func() + + createCount int + getCount int +} + +func (s *featureCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/feature-engineering/features", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + return s.creates[s.createCount-1] + }) + server.Handle("GET", "/api/2.0/feature-engineering/features/{full_name}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + if s.onGet != nil { + s.onGet() + } + return s.gets[s.getCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +func featureError(status int, code, message string) testserver.Response { + return testserver.Response{StatusCode: status, Body: map[string]string{"error_code": code, "message": message}} +} + +var ( + // Real 504s from the feature-engineering gateway carry an empty message, which the SDK does + // not retry. + featureTimeout = featureError(504, "DEADLINE_EXCEEDED", "") + featureNotFound = featureError(404, "NOT_FOUND", "Feature 'main.default.feat' not found.") + featureAlreadyExists = featureError(409, "ALREADY_EXISTS", "Feature 'main.default.feat' already exists.") + featureDenied = featureError(403, "PERMISSION_DENIED", "denied") + featureInvalid = featureError(400, "INVALID_PARAMETER_VALUE", "invalid") +) + +func requireStatus(t *testing.T, err error, status int) { + t.Helper() + apiErr, ok := errors.AsType[*apierr.APIError](err) + require.True(t, ok, "not an API error: %v", err) + assert.Equal(t, status, apiErr.StatusCode) +} + +func testFeature(created time.Time) ml.Feature { + return ml.Feature{FullName: testFeatureName, Description: "d", CreatedAt: sdktime.New(created)} +} + +// createTestFeature runs DoCreate with fast retries. +func createTestFeature(t *testing.T, s *featureCreateScript, ctx context.Context) (string, *ml.Feature, error) { + t.Helper() + ctx = env.Set(ctx, retry.IntervalMsVariable, "1") + r := (&ResourceFeature{}).New(s.client()) + return r.DoCreate(ctx, &ml.Feature{FullName: testFeatureName, Description: "d"}) +} + +func TestCreateRecoveryUsesResourceCreatedDespiteTimeout(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{testFeature(time.Now())}} + id, remote, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, testFeatureName, remote.FullName) + assert.Equal(t, 1, s.createCount) +} + +func TestCreateRecoveryRejectsResourceThatPredatesRequest(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{testFeature(time.Now().Add(-time.Hour))}} + _, _, err := createTestFeature(t, s, t.Context()) + // The 504 has an empty message, so nothing follows the explanation; the bundle error output + // appends the status code and details from the wrapped error. + require.EqualError(t, err, "feature main.default.feat exists but was not created by this deployment") + requireStatus(t, err, 504) +} + +func TestCreateRecoveryKeepsCreateErrorMessage(t *testing.T) { + timeout := featureError(504, "DEADLINE_EXCEEDED", "The request timed out.") + s := &featureCreateScript{t: t, creates: []any{timeout}, gets: []any{testFeature(time.Now().Add(-time.Hour))}} + _, _, err := createTestFeature(t, s, t.Context()) + require.EqualError(t, err, "feature main.default.feat exists but was not created by this deployment: The request timed out.") + requireStatus(t, err, 504) +} + +func TestCreateRecoveryRejectsResourceWithoutCreateTime(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{ml.Feature{FullName: testFeatureName}}} + _, _, err := createTestFeature(t, s, t.Context()) + require.EqualError(t, err, "feature main.default.feat exists but has no create time, so it is not known to be created by this deployment") + requireStatus(t, err, 504) +} + +func TestCreateRecoveryRepeatsCreateWhileMissing(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, testFeature(time.Now())}, + gets: []any{featureNotFound}, + } + id, _, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, 2, s.createCount) +} + +func TestCreateRecoveryLooksAgainAfterAlreadyExists(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, featureAlreadyExists}, + gets: []any{featureNotFound, testFeature(time.Now())}, + } + id, _, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestCreateRecoveryGivesUpAfterMaxRetries(t *testing.T) { + creates := []any{featureTimeout} + var gets []any + for range retry.MaxRetries { + creates = append(creates, featureTimeout) + gets = append(gets, featureNotFound) + } + gets = append(gets, featureNotFound) + s := &featureCreateScript{t: t, creates: creates, gets: gets} + _, _, err := createTestFeature(t, s, t.Context()) + requireStatus(t, err, 504) + assert.Equal(t, 1+retry.MaxRetries, s.createCount) + assert.Equal(t, 1+retry.MaxRetries, s.getCount) +} + +func TestCreateRecoveryReturnsCreateErrorWhenReadFails(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{featureDenied}} + _, _, err := createTestFeature(t, s, t.Context()) + requireStatus(t, err, 504) + assert.Equal(t, 1, s.createCount) +} + +func TestCreateRecoveryReturnsNonTransientRepeatError(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, featureInvalid}, + gets: []any{featureNotFound}, + } + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "invalid") +} + +func TestCreateRecoveryIgnoresNonTransientError(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureAlreadyExists}} + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "already exists") + assert.Equal(t, 0, s.getCount) +} + +func TestCreateRecoveryStopsOnCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{featureNotFound}, onGet: cancel} + _, _, err := createTestFeature(t, s, ctx) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, s.createCount) +} diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index c47fed477fc..602cb0931dc 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" "golang.org/x/sync/errgroup" ) @@ -258,13 +259,17 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState return "", nil, err } - createResp, err := r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ - Dashboard: dashboard, - DatasetCatalog: config.DatasetCatalog, - DatasetSchema: config.DatasetSchema, + create := func(ctx context.Context) (*dashboards.Dashboard, error) { + return r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ + Dashboard: dashboard, + DatasetCatalog: config.DatasetCatalog, + DatasetSchema: config.DatasetSchema, - ForceSendFields: nil, - }) + ForceSendFields: nil, + }) + } + start := time.Now() + createResp, err := create(ctx) // The API returns 404 if the parent directory doesn't exist. // If the parent directory doesn't exist, create it and try again. @@ -273,13 +278,10 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState if err != nil { return "", nil, fmt.Errorf("failed to create parent directory: %w", err) } - createResp, err = r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ - Dashboard: dashboard, - DatasetCatalog: config.DatasetCatalog, - DatasetSchema: config.DatasetSchema, - - ForceSendFields: nil, - }) + createResp, err = create(ctx) + } + if err != nil && config.ParentPath != "" { + createResp, err = r.recoverFailedCreate(ctx, config, start, create, err) } if err != nil { return "", nil, err @@ -309,6 +311,33 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState return createResp.DashboardId, responseToState(createResp, publishResp, dashboard.SerializedDashboard, config.Published), nil } +// recoverFailedCreate handles a create request that failed; see recoverCreate. A dashboard is +// stored as the workspace file /.lvdash.json, so the API rejects a +// second dashboard with the same display name in the same folder, and the file names the +// dashboard to look up. +func (r *ResourceDashboard) recoverFailedCreate(ctx context.Context, config *DashboardState, start time.Time, create func(context.Context) (*dashboards.Dashboard, error), createErr error) (*dashboards.Dashboard, error) { + filePath := path.Join(ensureWorkspacePrefix(config.ParentPath), config.DisplayName+".lvdash.json") + return recoverCreate(ctx, createRecovery[dashboards.Dashboard]{ + name: "dashboard " + filePath, + start: start, + get: func(ctx context.Context) (*dashboards.Dashboard, error) { + info, err := r.client.Workspace.GetStatusByPath(ctx, filePath) + if err != nil { + return nil, err + } + if info.ObjectType != workspace.ObjectTypeDashboard || info.ResourceId == "" { + return nil, fmt.Errorf("%s is a %s, not a dashboard", filePath, info.ObjectType) + } + return r.client.Lakeview.Get(ctx, dashboards.GetDashboardRequest{DashboardId: info.ResourceId}) + }, + createTime: func(found *dashboards.Dashboard) (time.Time, bool) { + created, err := time.Parse(time.RFC3339, found.CreateTime) + return created, err == nil + }, + create: create, + }, createErr) +} + func (r *ResourceDashboard) DoUpdate(ctx context.Context, id string, config *DashboardState, _ *PlanEntry) (*DashboardState, error) { dashboard, err := prepareDashboardRequest(config) if err != nil { diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8d95e3d0eac..8ec576a1f87 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -3,6 +3,8 @@ package dresources import ( "context" "slices" + "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -28,9 +30,32 @@ func (r *ResourceFeature) DoRead(ctx context.Context, id string) (*ml.Feature, e } func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (string, *ml.Feature, error) { - response, err := r.client.FeatureEngineering.CreateFeature(ctx, ml.CreateFeatureRequest{Feature: *config}) + create := func(ctx context.Context) (*ml.Feature, error) { + return r.client.FeatureEngineering.CreateFeature(ctx, ml.CreateFeatureRequest{Feature: *config}) + } + start := time.Now() + response, err := create(ctx) if err != nil { - return "", nil, err + // The feature is keyed by full_name, which the bundle chooses and the API keeps unique. UC + // stores the name lowercased, which is also the id a successful create returns. + fullName := strings.ToLower(config.FullName) + response, err = recoverCreate(ctx, createRecovery[ml.Feature]{ + name: "feature " + fullName, + start: start, + get: func(ctx context.Context) (*ml.Feature, error) { + return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: fullName}) + }, + createTime: func(found *ml.Feature) (time.Time, bool) { + if found.CreatedAt == nil { + return time.Time{}, false + } + return found.CreatedAt.AsTime(), true + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } return response.FullName, response, nil } diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index f9c0a2314ac..26f181ab205 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -169,12 +169,35 @@ func (r *ResourceModelServingEndpoint) DoRead(ctx context.Context, id string) (* } func (r *ResourceModelServingEndpoint) DoCreate(ctx context.Context, config *serving.CreateServingEndpoint) (string, *ModelServingEndpointRemote, error) { - waiter, err := r.client.ServingEndpoints.Create(ctx, *config) + create := func(ctx context.Context) (*serving.ServingEndpointDetailed, error) { + waiter, err := r.client.ServingEndpoints.Create(ctx, *config) + if err != nil { + return nil, err + } + return waiter.Response, nil + } + start := time.Now() + endpoint, err := create(ctx) if err != nil { - return "", nil, err + // Endpoint names are chosen by the bundle and unique in the workspace. WaitAfterCreate waits + // for a recovered endpoint as it does for one the first request created. + endpoint, err = recoverCreate(ctx, createRecovery[serving.ServingEndpointDetailed]{ + name: "serving endpoint " + config.Name, + start: start, + get: func(ctx context.Context) (*serving.ServingEndpointDetailed, error) { + return r.client.ServingEndpoints.Get(ctx, serving.GetServingEndpointRequest{Name: config.Name}) + }, + createTime: func(found *serving.ServingEndpointDetailed) (time.Time, bool) { + return time.UnixMilli(found.CreationTimestamp), found.CreationTimestamp != 0 + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } - return waiter.Response.Name, nil, nil + return endpoint.Name, nil, nil } // waitForEndpointReady waits for the serving endpoint to be ready (not updating) diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 46a229dbf6d..4e39815fceb 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -2,7 +2,6 @@ package dresources import ( "context" - "errors" "time" "github.com/databricks/cli/bundle/config/resources" @@ -11,7 +10,6 @@ import ( "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) @@ -97,7 +95,7 @@ func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vec if err == nil { return waiter.Response, nil } - if afterTransient && isEndpointAlreadyExists(err) { + if afterTransient && isAlreadyExists(err) { info, getErr := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, config.Name) if getErr != nil { return nil, getErr @@ -117,19 +115,10 @@ func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vec return config.Name, newVectorSearchEndpointRemote(info), nil } -// isEndpointAlreadyExists matches both codes: the cloud returns 409 ALREADY_EXISTS (observed on -// AWS, 2026-11), while RESOURCE_ALREADY_EXISTS is the generic code for the same conflict. -func isEndpointAlreadyExists(err error) bool { - return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) -} - -// endpointClockSkew allows for the server's clock running behind the client's. -const endpointClockSkew = time.Minute - // createdSince reports whether the endpoint was created no earlier than start, the time of the // first create attempt, so that it can only have been created by that attempt. func createdSince(info *vectorsearch.EndpointInfo, start time.Time) bool { - return info.CreationTimestamp >= start.Add(-endpointClockSkew).UnixMilli() + return info.CreationTimestamp >= start.Add(-createTimeSkew).UnixMilli() } func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) { diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go index 098f790a229..809849609c4 100644 --- a/libs/testserver/dashboards.go +++ b/libs/testserver/dashboards.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "net/http" "path" "strconv" "strings" @@ -141,6 +142,17 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { dashboard.Path = dashboard.ParentPath + "/" + dashboard.DisplayName + ".lvdash.json" } + workspacePath := path.Join("/Workspace", dashboard.Path) + if _, exists := s.files[workspacePath]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("A node with name '%s' already exists under the specified parent", dashboard.DisplayName), + }, + } + } + dashboard.CreateTime = nextTimestamp() dashboard.UpdateTime = dashboard.CreateTime @@ -163,7 +175,6 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { InputSerializedDashboard: inputSerializedDashboard, }) - workspacePath := path.Join("/Workspace", dashboard.Path) s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "DASHBOARD", @@ -220,6 +231,7 @@ func (s *FakeWorkspace) DashboardUpdate(req Request) Response { updated.DisplayName = updateReq.DisplayName dir := path.Dir(updated.Path) base := updateReq.DisplayName + ".lvdash.json" + s.moveDashboardFile(updated.Path, path.Join(dir, base), dashboardId) updated.Path = path.Join(dir, base) } if updateReq.SerializedDashboard != "" { @@ -298,7 +310,8 @@ func (s *FakeWorkspace) DashboardTrash(req Request) Response { } } - // The dashboard is marked as trashed and moved to the trash. + // The dashboard is marked as trashed and moved to the trash, which frees its path. + s.moveDashboardFile(dashboard.Path, "", dashboardId) s.Dashboards.Put(dashboardId, &fakeDashboard{ Etag: dashboard.Etag, DashboardId: dashboardId, @@ -350,3 +363,18 @@ func (s *FakeWorkspace) DashboardGetPublished(req Request) Response { Body: publishedDashboard, } } + +// moveDashboardFile moves the workspace file of dashboard dashboardId from oldPath to newPath, or +// removes it if newPath is empty. Paths are as in Dashboard.Path, without the /Workspace prefix. +func (s *FakeWorkspace) moveDashboardFile(oldPath, newPath, dashboardId string) { + oldWorkspacePath := path.Join("/Workspace", oldPath) + entry, ok := s.files[oldWorkspacePath] + if !ok || entry.Info.ResourceId != dashboardId { + return + } + delete(s.files, oldWorkspacePath) + if newPath != "" { + entry.Info.Path = path.Join("/Workspace", newPath) + s.files[entry.Info.Path] = entry + } +} diff --git a/libs/testserver/feature.go b/libs/testserver/feature.go index 6ebdacc74f6..d1c63a9b5da 100644 --- a/libs/testserver/feature.go +++ b/libs/testserver/feature.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -33,8 +35,18 @@ func (s *FakeWorkspace) FeaturesCreate(req Request) Response { Body: fmt.Sprintf("full_name must be a three-part name, got %q", feature.FullName), } } + if _, exists := s.Features[feature.FullName]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("Feature '%s' already exists.", feature.FullName), + }, + } + } feature.CatalogName, feature.SchemaName, feature.Name = parts[0], parts[1], parts[2] feature.CreatedBy = s.CurrentUser().UserName + feature.CreatedAt = sdktime.New(time.Unix(0, nowNano()).UTC()) s.Features[feature.FullName] = feature return Response{ diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 48efd7232ce..43e71a6475f 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -966,7 +966,8 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.FeaturesCreate(req) }) server.Handle("GET", "/api/2.0/feature-engineering/features/{full_name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.Features, req.Vars["full_name"]) + // Features are stored lowercased, and the cloud GET matches the name case-insensitively. + return MapGet(req.Workspace, req.Workspace.Features, strings.ToLower(req.Vars["full_name"])) }) server.Handle("PATCH", "/api/2.0/feature-engineering/features/{full_name}", func(req Request) any { return req.Workspace.FeaturesUpdate(req, req.Vars["full_name"])