From e170398b858d8c5d1b3e5c886f855b0e27ef8c48 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 08:55:34 +0000 Subject: [PATCH 1/4] direct: use features, serving endpoints and dashboards that a timed-out create created A create request answered with 504 DEADLINE_EXCEEDED may still land. Look the resource up by the name the bundle chose: use it if it was created after our first request, else repeat the create (up to retry.MaxRetries). cloud: TestAccept/bundle/resources/features/lifecycle TestAccept/bundle/resources/dashboards/nested-folders TestAccept/bundle/resources/model_serving_endpoints/basic Co-authored-by: Isaac --- .../create_504/dashboard.lvdash.json | 1 + .../dashboards/create_504/databricks.yml | 10 + .../dashboards/create_504/out.test.toml | 3 + .../dashboards/create_504/output.txt | 33 ++++ .../resources/dashboards/create_504/script | 15 ++ .../resources/dashboards/create_504/test.toml | 6 + .../features/create_504/databricks.yml | 22 +++ .../features/create_504/out.test.toml | 3 + .../resources/features/create_504/output.txt | 26 +++ .../resources/features/create_504/script | 13 ++ .../resources/features/create_504/test.toml | 6 + .../create_504/databricks.yml | 7 + .../create_504/out.test.toml | 3 + .../create_504/output.txt | 28 +++ .../model_serving_endpoints/create_504/script | 13 ++ .../create_504/test.toml | 6 + bundle/direct/dresources/create_recovery.go | 90 +++++++++ .../direct/dresources/create_recovery_test.go | 182 ++++++++++++++++++ bundle/direct/dresources/dashboard.go | 55 ++++-- bundle/direct/dresources/feature.go | 26 ++- .../dresources/model_serving_endpoint.go | 29 ++- libs/testserver/dashboards.go | 32 ++- libs/testserver/feature.go | 12 ++ 23 files changed, 601 insertions(+), 20 deletions(-) create mode 100644 acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json create mode 100644 acceptance/bundle/resources/dashboards/create_504/databricks.yml create mode 100644 acceptance/bundle/resources/dashboards/create_504/out.test.toml create mode 100644 acceptance/bundle/resources/dashboards/create_504/output.txt create mode 100644 acceptance/bundle/resources/dashboards/create_504/script create mode 100644 acceptance/bundle/resources/dashboards/create_504/test.toml create mode 100644 acceptance/bundle/resources/features/create_504/databricks.yml create mode 100644 acceptance/bundle/resources/features/create_504/out.test.toml create mode 100644 acceptance/bundle/resources/features/create_504/output.txt create mode 100644 acceptance/bundle/resources/features/create_504/script create mode 100644 acceptance/bundle/resources/features/create_504/test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml create mode 100644 acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt create mode 100644 acceptance/bundle/resources/model_serving_endpoints/create_504/script create mode 100644 acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml create mode 100644 bundle/direct/dresources/create_recovery.go create mode 100644 bundle/direct/dresources/create_recovery_test.go diff --git a/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json b/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json new file mode 100644 index 00000000000..ffcd4415b08 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/dashboard.lvdash.json @@ -0,0 +1 @@ +{ } diff --git a/acceptance/bundle/resources/dashboards/create_504/databricks.yml b/acceptance/bundle/resources/dashboards/create_504/databricks.yml new file mode 100644 index 00000000000..8c1157635d3 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + dashboards: + dashboard1: + display_name: test-dashboard + warehouse_id: warehouse-1234 + file_path: dashboard.lvdash.json + parent_path: /Users/${workspace.current_user.userName} diff --git a/acceptance/bundle/resources/dashboards/create_504/out.test.toml b/acceptance/bundle/resources/dashboards/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/dashboards/create_504/output.txt b/acceptance/bundle/resources/dashboards/create_504/output.txt new file mode 100644 index 00000000000..4dca4601cca --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/output.txt @@ -0,0 +1,33 @@ + +=== The create request times out but the dashboard was created: deploy uses the dashboard + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards +Created dashboards.dashboard1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //lakeview //workspace/get-status +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID] +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +GET /api/2.0/lakeview/dashboards/[DASHBOARD_ID] +POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published + +=== The create request times out and the dashboard was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.dashboards.dashboard1: retrying after 504 Gateway Timeout from POST /api/2.0/lakeview/dashboards +Created dashboards.dashboard1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //lakeview //workspace/get-status +POST /api/2.0/lakeview/dashboards +GET /api/2.0/workspace/get-status /Workspace/Users/[USERNAME]/test-dashboard.lvdash.json +POST /api/2.0/lakeview/dashboards +POST /api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published diff --git a/acceptance/bundle/resources/dashboards/create_504/script b/acceptance/bundle/resources/dashboards/create_504/script new file mode 100644 index 00000000000..355e8595cf4 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/script @@ -0,0 +1,15 @@ +title "The create request times out but the dashboard was created: deploy uses the dashboard" +# The first read of a new dashboard returns 404 (INJECT_STALE_ON_DIRECT), so the deploy repeats +# the create, gets ALREADY_EXISTS, and finds the dashboard on the next look. +fault.py --after-handler "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the dashboard was not created: deploy repeats the create" +fault.py "POST /api/2.0/lakeview/dashboards" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //lakeview //workspace/get-status | jq -r 'select(.path != "/api/2.0/workspace/get-status" or (.q.path | endswith(".lvdash.json"))) | .method + " " + .path + (if .q.path then " " + .q.path else "" end)' | sed -E 's/[0-9a-f]{32}/[DASHBOARD_ID]/' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/dashboards/create_504/test.toml b/acceptance/bundle/resources/dashboards/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/features/create_504/databricks.yml b/acceptance/bundle/resources/features/create_504/databricks.yml new file mode 100644 index 00000000000..61f14c0564b --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: test-bundle + +resources: + features: + feature1: + full_name: main.myschema.myfeature + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts diff --git a/acceptance/bundle/resources/features/create_504/out.test.toml b/acceptance/bundle/resources/features/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/features/create_504/output.txt b/acceptance/bundle/resources/features/create_504/output.txt new file mode 100644 index 00000000000..4dfe9fb1b5e --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/output.txt @@ -0,0 +1,26 @@ + +=== The create request times out but the feature was created: deploy uses the feature + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature + +=== The create request times out and the feature was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.features.feature1: retrying after 504 Gateway Timeout from POST /api/2.0/feature-engineering/features +Created features.feature1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature +POST /api/2.0/feature-engineering/features diff --git a/acceptance/bundle/resources/features/create_504/script b/acceptance/bundle/resources/features/create_504/script new file mode 100644 index 00000000000..43ee407fb14 --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/script @@ -0,0 +1,13 @@ +title "The create request times out but the feature was created: deploy uses the feature" +fault.py --after-handler "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the feature was not created: deploy repeats the create" +fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/features/create_504/test.toml b/acceptance/bundle/resources/features/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/features/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml b/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml new file mode 100644 index 00000000000..9813e7c7d66 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + model_serving_endpoints: + my_endpoint: + name: test-endpoint diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt b/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt new file mode 100644 index 00000000000..390a38f4ba9 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/output.txt @@ -0,0 +1,28 @@ + +=== The create request times out but the endpoint was created: deploy uses the endpoint + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_serving_endpoints.my_endpoint +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //serving-endpoints +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint +GET /api/2.0/serving-endpoints/test-endpoint + +=== The create request times out and the endpoint was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_serving_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/serving-endpoints +Created model_serving_endpoints.my_endpoint +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //serving-endpoints +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint +POST /api/2.0/serving-endpoints +GET /api/2.0/serving-endpoints/test-endpoint diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/script b/acceptance/bundle/resources/model_serving_endpoints/create_504/script new file mode 100644 index 00000000000..4b5028c747a --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/script @@ -0,0 +1,13 @@ +title "The create request times out but the endpoint was created: deploy uses the endpoint" +fault.py --after-handler "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the endpoint was not created: deploy repeats the create" +fault.py "POST /api/2.0/serving-endpoints" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //serving-endpoints | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml b/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go new file mode 100644 index 00000000000..17b60683aa5 --- /dev/null +++ b/bundle/direct/dresources/create_recovery.go @@ -0,0 +1,90 @@ +package dresources + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// createTimeSkew is how much earlier than our first create request a found resource's create time +// may be, to allow for clock skew, and still count as created by that request. +const createTimeSkew = time.Minute + +// createRecovery describes a create request that failed, for a resource whose name the bundle +// chooses and the API keeps unique. T is the SDK type the create and the lookup return. +type createRecovery[T any] struct { + // name identifies the resource in messages, e.g. the endpoint name or the dashboard path. + name string + // start is when the first create request was sent. + start time.Time + // get looks the resource up by name. It must fail with a 404 when the resource is missing. + get func(ctx context.Context) (*T, error) + // createTime returns when the found resource was created, or false if the response omits it. + createTime func(found *T) (time.Time, bool) + // create repeats the create request. + create func(ctx context.Context) (*T, error) +} + +// recoverCreate handles a create request that failed with createErr. A slow create is answered +// with 504 DEADLINE_EXCEEDED, which does not say whether the resource was created, so repeating +// the request blindly can fail with "already exists" for a resource that the next deploy would +// then also fail to create. Instead, look the resource up: use it if our request created it, and +// repeat the request while it is missing. A repeat that fails because the resource exists means an +// earlier request has landed in the meantime, so the next look finds it. +// +// Errors that are not transient, and a done context, are returned as is. +func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { + return nil, createErr + } + + lastErr := createErr + for attempt := 1; ; attempt++ { + found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) { + return rc.get(ctx) + }) + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err == nil { + // Without the time check, a resource that existed before the deploy would be taken over: + // our request would have failed with "already exists" only if it reached that check. + created, ok := rc.createTime(found) + if !ok || created.Before(rc.start.Add(-createTimeSkew)) { + return nil, fmt.Errorf("%w; %s exists but was not created by this deployment", createErr, rc.name) + } + // Info, not Warn: the deploy got the resource it asked for, so there is nothing to act on. + log.Infof(ctx, "using %s, created despite the failed create request: %s", rc.name, createErr) + return found, nil + } + if !apierr.IsMissing(err) { + log.Warnf(ctx, "checking whether %s was created: %s", rc.name, err) + return nil, createErr + } + if attempt > retry.MaxRetries { + return nil, lastErr + } + if err := retry.Pause(ctx, lastErr); err != nil { + return nil, err + } + created, err := rc.create(ctx) + if err == nil { + return created, nil + } + if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) { + return nil, err + } + lastErr = err + } +} + +// isAlreadyExists reports whether a create request failed because the resource exists. Features +// and dashboards answer with 409 ALREADY_EXISTS, serving endpoints with 400 RESOURCE_ALREADY_EXISTS. +func isAlreadyExists(err error) bool { + return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) +} diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go new file mode 100644 index 00000000000..056f9ed83e7 --- /dev/null +++ b/bundle/direct/dresources/create_recovery_test.go @@ -0,0 +1,182 @@ +package dresources + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testFeatureName = "main.default.feat" + +// featureCreateScript plays the given responses, in order, to the feature create and get +// requests. A response is an ml.Feature or a testserver.Response for an error. +type featureCreateScript struct { + t *testing.T + creates []any + gets []any + // onGet runs when a get request arrives, before its response is sent. + onGet func() + + createCount int + getCount int +} + +func (s *featureCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/feature-engineering/features", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + return s.creates[s.createCount-1] + }) + server.Handle("GET", "/api/2.0/feature-engineering/features/{full_name}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + if s.onGet != nil { + s.onGet() + } + return s.gets[s.getCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +func featureError(status int, code, message string) testserver.Response { + return testserver.Response{StatusCode: status, Body: map[string]string{"error_code": code, "message": message}} +} + +var ( + // Real 504s from the feature-engineering gateway carry an empty message, which the SDK does + // not retry. + featureTimeout = featureError(504, "DEADLINE_EXCEEDED", "") + featureNotFound = featureError(404, "NOT_FOUND", "Feature 'main.default.feat' not found.") + featureAlreadyExists = featureError(409, "ALREADY_EXISTS", "Feature 'main.default.feat' already exists.") + featureDenied = featureError(403, "PERMISSION_DENIED", "denied") + featureInvalid = featureError(400, "INVALID_PARAMETER_VALUE", "invalid") +) + +func requireStatus(t *testing.T, err error, status int) { + t.Helper() + apiErr, ok := errors.AsType[*apierr.APIError](err) + require.True(t, ok, "not an API error: %v", err) + assert.Equal(t, status, apiErr.StatusCode) +} + +func testFeature(created time.Time) ml.Feature { + return ml.Feature{FullName: testFeatureName, Description: "d", CreatedAt: sdktime.New(created)} +} + +// createTestFeature runs DoCreate with fast retries. +func createTestFeature(t *testing.T, s *featureCreateScript, ctx context.Context) (string, *ml.Feature, error) { + t.Helper() + ctx = env.Set(ctx, retry.IntervalMsVariable, "1") + r := (&ResourceFeature{}).New(s.client()) + return r.DoCreate(ctx, &ml.Feature{FullName: testFeatureName, Description: "d"}) +} + +func TestCreateRecoveryUsesResourceCreatedDespiteTimeout(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{testFeature(time.Now())}} + id, remote, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, testFeatureName, remote.FullName) + assert.Equal(t, 1, s.createCount) +} + +func TestCreateRecoveryRejectsResourceThatPredatesRequest(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{testFeature(time.Now().Add(-time.Hour))}} + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "feature main.default.feat exists but was not created by this deployment") + requireStatus(t, err, 504) +} + +func TestCreateRecoveryRejectsResourceWithoutCreateTime(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{ml.Feature{FullName: testFeatureName}}} + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "exists but was not created by this deployment") +} + +func TestCreateRecoveryRepeatsCreateWhileMissing(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, testFeature(time.Now())}, + gets: []any{featureNotFound}, + } + id, _, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, 2, s.createCount) +} + +func TestCreateRecoveryLooksAgainAfterAlreadyExists(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, featureAlreadyExists}, + gets: []any{featureNotFound, testFeature(time.Now())}, + } + id, _, err := createTestFeature(t, s, t.Context()) + require.NoError(t, err) + assert.Equal(t, testFeatureName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestCreateRecoveryGivesUpAfterMaxRetries(t *testing.T) { + creates := []any{featureTimeout} + var gets []any + for range retry.MaxRetries { + creates = append(creates, featureTimeout) + gets = append(gets, featureNotFound) + } + gets = append(gets, featureNotFound) + s := &featureCreateScript{t: t, creates: creates, gets: gets} + _, _, err := createTestFeature(t, s, t.Context()) + requireStatus(t, err, 504) + assert.Equal(t, 1+retry.MaxRetries, s.createCount) + assert.Equal(t, 1+retry.MaxRetries, s.getCount) +} + +func TestCreateRecoveryReturnsCreateErrorWhenReadFails(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{featureDenied}} + _, _, err := createTestFeature(t, s, t.Context()) + requireStatus(t, err, 504) + assert.Equal(t, 1, s.createCount) +} + +func TestCreateRecoveryReturnsNonTransientRepeatError(t *testing.T) { + s := &featureCreateScript{ + t: t, + creates: []any{featureTimeout, featureInvalid}, + gets: []any{featureNotFound}, + } + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "invalid") +} + +func TestCreateRecoveryIgnoresNonTransientError(t *testing.T) { + s := &featureCreateScript{t: t, creates: []any{featureAlreadyExists}} + _, _, err := createTestFeature(t, s, t.Context()) + require.ErrorContains(t, err, "already exists") + assert.Equal(t, 0, s.getCount) +} + +func TestCreateRecoveryStopsOnCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{featureNotFound}, onGet: cancel} + _, _, err := createTestFeature(t, s, ctx) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, s.createCount) +} diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index c47fed477fc..602cb0931dc 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" "golang.org/x/sync/errgroup" ) @@ -258,13 +259,17 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState return "", nil, err } - createResp, err := r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ - Dashboard: dashboard, - DatasetCatalog: config.DatasetCatalog, - DatasetSchema: config.DatasetSchema, + create := func(ctx context.Context) (*dashboards.Dashboard, error) { + return r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ + Dashboard: dashboard, + DatasetCatalog: config.DatasetCatalog, + DatasetSchema: config.DatasetSchema, - ForceSendFields: nil, - }) + ForceSendFields: nil, + }) + } + start := time.Now() + createResp, err := create(ctx) // The API returns 404 if the parent directory doesn't exist. // If the parent directory doesn't exist, create it and try again. @@ -273,13 +278,10 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState if err != nil { return "", nil, fmt.Errorf("failed to create parent directory: %w", err) } - createResp, err = r.client.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ - Dashboard: dashboard, - DatasetCatalog: config.DatasetCatalog, - DatasetSchema: config.DatasetSchema, - - ForceSendFields: nil, - }) + createResp, err = create(ctx) + } + if err != nil && config.ParentPath != "" { + createResp, err = r.recoverFailedCreate(ctx, config, start, create, err) } if err != nil { return "", nil, err @@ -309,6 +311,33 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState return createResp.DashboardId, responseToState(createResp, publishResp, dashboard.SerializedDashboard, config.Published), nil } +// recoverFailedCreate handles a create request that failed; see recoverCreate. A dashboard is +// stored as the workspace file /.lvdash.json, so the API rejects a +// second dashboard with the same display name in the same folder, and the file names the +// dashboard to look up. +func (r *ResourceDashboard) recoverFailedCreate(ctx context.Context, config *DashboardState, start time.Time, create func(context.Context) (*dashboards.Dashboard, error), createErr error) (*dashboards.Dashboard, error) { + filePath := path.Join(ensureWorkspacePrefix(config.ParentPath), config.DisplayName+".lvdash.json") + return recoverCreate(ctx, createRecovery[dashboards.Dashboard]{ + name: "dashboard " + filePath, + start: start, + get: func(ctx context.Context) (*dashboards.Dashboard, error) { + info, err := r.client.Workspace.GetStatusByPath(ctx, filePath) + if err != nil { + return nil, err + } + if info.ObjectType != workspace.ObjectTypeDashboard || info.ResourceId == "" { + return nil, fmt.Errorf("%s is a %s, not a dashboard", filePath, info.ObjectType) + } + return r.client.Lakeview.Get(ctx, dashboards.GetDashboardRequest{DashboardId: info.ResourceId}) + }, + createTime: func(found *dashboards.Dashboard) (time.Time, bool) { + created, err := time.Parse(time.RFC3339, found.CreateTime) + return created, err == nil + }, + create: create, + }, createErr) +} + func (r *ResourceDashboard) DoUpdate(ctx context.Context, id string, config *DashboardState, _ *PlanEntry) (*DashboardState, error) { dashboard, err := prepareDashboardRequest(config) if err != nil { diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8d95e3d0eac..130ac57db4e 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -3,6 +3,7 @@ package dresources import ( "context" "slices" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -28,9 +29,30 @@ func (r *ResourceFeature) DoRead(ctx context.Context, id string) (*ml.Feature, e } func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (string, *ml.Feature, error) { - response, err := r.client.FeatureEngineering.CreateFeature(ctx, ml.CreateFeatureRequest{Feature: *config}) + create := func(ctx context.Context) (*ml.Feature, error) { + return r.client.FeatureEngineering.CreateFeature(ctx, ml.CreateFeatureRequest{Feature: *config}) + } + start := time.Now() + response, err := create(ctx) if err != nil { - return "", nil, err + // The feature is keyed by full_name, which the bundle chooses and the API keeps unique. + response, err = recoverCreate(ctx, createRecovery[ml.Feature]{ + name: "feature " + config.FullName, + start: start, + get: func(ctx context.Context) (*ml.Feature, error) { + return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: config.FullName}) + }, + createTime: func(found *ml.Feature) (time.Time, bool) { + if found.CreatedAt == nil { + return time.Time{}, false + } + return found.CreatedAt.AsTime(), true + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } return response.FullName, response, nil } diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index f9c0a2314ac..26f181ab205 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -169,12 +169,35 @@ func (r *ResourceModelServingEndpoint) DoRead(ctx context.Context, id string) (* } func (r *ResourceModelServingEndpoint) DoCreate(ctx context.Context, config *serving.CreateServingEndpoint) (string, *ModelServingEndpointRemote, error) { - waiter, err := r.client.ServingEndpoints.Create(ctx, *config) + create := func(ctx context.Context) (*serving.ServingEndpointDetailed, error) { + waiter, err := r.client.ServingEndpoints.Create(ctx, *config) + if err != nil { + return nil, err + } + return waiter.Response, nil + } + start := time.Now() + endpoint, err := create(ctx) if err != nil { - return "", nil, err + // Endpoint names are chosen by the bundle and unique in the workspace. WaitAfterCreate waits + // for a recovered endpoint as it does for one the first request created. + endpoint, err = recoverCreate(ctx, createRecovery[serving.ServingEndpointDetailed]{ + name: "serving endpoint " + config.Name, + start: start, + get: func(ctx context.Context) (*serving.ServingEndpointDetailed, error) { + return r.client.ServingEndpoints.Get(ctx, serving.GetServingEndpointRequest{Name: config.Name}) + }, + createTime: func(found *serving.ServingEndpointDetailed) (time.Time, bool) { + return time.UnixMilli(found.CreationTimestamp), found.CreationTimestamp != 0 + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } - return waiter.Response.Name, nil, nil + return endpoint.Name, nil, nil } // waitForEndpointReady waits for the serving endpoint to be ready (not updating) diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go index 098f790a229..809849609c4 100644 --- a/libs/testserver/dashboards.go +++ b/libs/testserver/dashboards.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "net/http" "path" "strconv" "strings" @@ -141,6 +142,17 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { dashboard.Path = dashboard.ParentPath + "/" + dashboard.DisplayName + ".lvdash.json" } + workspacePath := path.Join("/Workspace", dashboard.Path) + if _, exists := s.files[workspacePath]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("A node with name '%s' already exists under the specified parent", dashboard.DisplayName), + }, + } + } + dashboard.CreateTime = nextTimestamp() dashboard.UpdateTime = dashboard.CreateTime @@ -163,7 +175,6 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { InputSerializedDashboard: inputSerializedDashboard, }) - workspacePath := path.Join("/Workspace", dashboard.Path) s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "DASHBOARD", @@ -220,6 +231,7 @@ func (s *FakeWorkspace) DashboardUpdate(req Request) Response { updated.DisplayName = updateReq.DisplayName dir := path.Dir(updated.Path) base := updateReq.DisplayName + ".lvdash.json" + s.moveDashboardFile(updated.Path, path.Join(dir, base), dashboardId) updated.Path = path.Join(dir, base) } if updateReq.SerializedDashboard != "" { @@ -298,7 +310,8 @@ func (s *FakeWorkspace) DashboardTrash(req Request) Response { } } - // The dashboard is marked as trashed and moved to the trash. + // The dashboard is marked as trashed and moved to the trash, which frees its path. + s.moveDashboardFile(dashboard.Path, "", dashboardId) s.Dashboards.Put(dashboardId, &fakeDashboard{ Etag: dashboard.Etag, DashboardId: dashboardId, @@ -350,3 +363,18 @@ func (s *FakeWorkspace) DashboardGetPublished(req Request) Response { Body: publishedDashboard, } } + +// moveDashboardFile moves the workspace file of dashboard dashboardId from oldPath to newPath, or +// removes it if newPath is empty. Paths are as in Dashboard.Path, without the /Workspace prefix. +func (s *FakeWorkspace) moveDashboardFile(oldPath, newPath, dashboardId string) { + oldWorkspacePath := path.Join("/Workspace", oldPath) + entry, ok := s.files[oldWorkspacePath] + if !ok || entry.Info.ResourceId != dashboardId { + return + } + delete(s.files, oldWorkspacePath) + if newPath != "" { + entry.Info.Path = path.Join("/Workspace", newPath) + s.files[entry.Info.Path] = entry + } +} diff --git a/libs/testserver/feature.go b/libs/testserver/feature.go index 6ebdacc74f6..d1c63a9b5da 100644 --- a/libs/testserver/feature.go +++ b/libs/testserver/feature.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -33,8 +35,18 @@ func (s *FakeWorkspace) FeaturesCreate(req Request) Response { Body: fmt.Sprintf("full_name must be a three-part name, got %q", feature.FullName), } } + if _, exists := s.Features[feature.FullName]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("Feature '%s' already exists.", feature.FullName), + }, + } + } feature.CatalogName, feature.SchemaName, feature.Name = parts[0], parts[1], parts[2] feature.CreatedBy = s.CurrentUser().UserName + feature.CreatedAt = sdktime.New(time.Unix(0, nowNano()).UTC()) s.Features[feature.FullName] = feature return Response{ From 59dc3e1b63c97b293710b56899acca406b956054 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 08:56:35 +0000 Subject: [PATCH 2/4] Add changelog entry cloud: TestAccept/bundle/resources/features/lifecycle TestAccept/bundle/resources/dashboards/nested-folders TestAccept/bundle/resources/model_serving_endpoints/basic Co-authored-by: Isaac --- .nextchanges/bundles/7757.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7757.md diff --git a/.nextchanges/bundles/7757.md b/.nextchanges/bundles/7757.md new file mode 100644 index 00000000000..144244eafe9 --- /dev/null +++ b/.nextchanges/bundles/7757.md @@ -0,0 +1 @@ +* Use the feature, model serving endpoint or dashboard that a create request created despite timing out, instead of failing the deploy, in the direct deployment engine. ([#7757](https://github.com/databricks/cli/pull/7757)) From 07edd1576575e4ec5a02f87e46759e538a716574 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 09:21:06 +0000 Subject: [PATCH 3/4] Fix dashboards/create_504 on Windows, clarify missing create time dashboards/test.toml sets MSYS_NO_PATHCONV, which keeps Python from finding fault.py on Windows; unset it for the helpers, as script.prepare does. Say so when a found resource has no create time instead of claiming it predates the deploy. Reuse isAlreadyExists and createTimeSkew in the vector search endpoint create. cloud: TestAccept/bundle/resources/features/lifecycle TestAccept/bundle/resources/dashboards/nested-folders TestAccept/bundle/resources/model_serving_endpoints/basic Co-authored-by: Isaac --- .../bundle/resources/dashboards/create_504/script | 5 +++++ bundle/direct/dresources/create_recovery.go | 10 +++++++--- bundle/direct/dresources/create_recovery_test.go | 2 +- .../direct/dresources/vector_search_endpoint.go | 15 ++------------- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/create_504/script b/acceptance/bundle/resources/dashboards/create_504/script index 355e8595cf4..19cfefa4184 100644 --- a/acceptance/bundle/resources/dashboards/create_504/script +++ b/acceptance/bundle/resources/dashboards/create_504/script @@ -1,3 +1,8 @@ +# dashboards/test.toml sets MSYS_NO_PATHCONV, which keeps Python from finding these helpers on +# Windows; see the same workaround in acceptance/script.prepare. +fault.py() { env -u MSYS_NO_PATHCONV "$(type -P fault.py)" "$@"; } +print_requests.py() { env -u MSYS_NO_PATHCONV "$(type -P print_requests.py)" "$@"; } + title "The create request times out but the dashboard was created: deploy uses the dashboard" # The first read of a new dashboard returns 404 (INJECT_STALE_ON_DIRECT), so the deploy repeats # the create, gets ALREADY_EXISTS, and finds the dashboard on the next look. diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index 17b60683aa5..ecdfb973913 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -55,7 +55,10 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e // Without the time check, a resource that existed before the deploy would be taken over: // our request would have failed with "already exists" only if it reached that check. created, ok := rc.createTime(found) - if !ok || created.Before(rc.start.Add(-createTimeSkew)) { + if !ok { + return nil, fmt.Errorf("%w; %s exists but has no create time, so it is not known to be created by this deployment", createErr, rc.name) + } + if created.Before(rc.start.Add(-createTimeSkew)) { return nil, fmt.Errorf("%w; %s exists but was not created by this deployment", createErr, rc.name) } // Info, not Warn: the deploy got the resource it asked for, so there is nothing to act on. @@ -83,8 +86,9 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e } } -// isAlreadyExists reports whether a create request failed because the resource exists. Features -// and dashboards answer with 409 ALREADY_EXISTS, serving endpoints with 400 RESOURCE_ALREADY_EXISTS. +// isAlreadyExists reports whether a create request failed because the resource exists. Features, +// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with +// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). func isAlreadyExists(err error) bool { return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) } diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go index 056f9ed83e7..7b6d8f02494 100644 --- a/bundle/direct/dresources/create_recovery_test.go +++ b/bundle/direct/dresources/create_recovery_test.go @@ -106,7 +106,7 @@ func TestCreateRecoveryRejectsResourceThatPredatesRequest(t *testing.T) { func TestCreateRecoveryRejectsResourceWithoutCreateTime(t *testing.T) { s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{ml.Feature{FullName: testFeatureName}}} _, _, err := createTestFeature(t, s, t.Context()) - require.ErrorContains(t, err, "exists but was not created by this deployment") + require.ErrorContains(t, err, "exists but has no create time") } func TestCreateRecoveryRepeatsCreateWhileMissing(t *testing.T) { diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 46a229dbf6d..4e39815fceb 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -2,7 +2,6 @@ package dresources import ( "context" - "errors" "time" "github.com/databricks/cli/bundle/config/resources" @@ -11,7 +10,6 @@ import ( "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) @@ -97,7 +95,7 @@ func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vec if err == nil { return waiter.Response, nil } - if afterTransient && isEndpointAlreadyExists(err) { + if afterTransient && isAlreadyExists(err) { info, getErr := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, config.Name) if getErr != nil { return nil, getErr @@ -117,19 +115,10 @@ func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vec return config.Name, newVectorSearchEndpointRemote(info), nil } -// isEndpointAlreadyExists matches both codes: the cloud returns 409 ALREADY_EXISTS (observed on -// AWS, 2026-11), while RESOURCE_ALREADY_EXISTS is the generic code for the same conflict. -func isEndpointAlreadyExists(err error) bool { - return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) -} - -// endpointClockSkew allows for the server's clock running behind the client's. -const endpointClockSkew = time.Minute - // createdSince reports whether the endpoint was created no earlier than start, the time of the // first create attempt, so that it can only have been created by that attempt. func createdSince(info *vectorsearch.EndpointInfo, start time.Time) bool { - return info.CreationTimestamp >= start.Add(-endpointClockSkew).UnixMilli() + return info.CreationTimestamp >= start.Add(-createTimeSkew).UnixMilli() } func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) { From 9b9ec98789509febcc67abaf5349b82ceb65a5ce Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 11:30:56 +0000 Subject: [PATCH 4/4] Address review: error text, feature name case, failed-lookup golden - When a found resource is not used, the error no longer starts with "; " for the empty-message 504s: the explanation comes first and the create error's message, if any, after it. The 504 stays wrapped, so the output still ends with its status code and details. - Look the feature up by its lowercased full_name, as UC stores it and as a successful create returns it. The fake GET now matches the name case-insensitively, as the cloud does (checked on AWS). - features/create_504 uses a mixed-case name and adds the scenario where the lookup fails. cloud: TestAccept/bundle/resources/features/lifecycle TestAccept/bundle/resources/features/uppercase-name Co-authored-by: Isaac --- .../features/create_504/databricks.yml | 3 ++- .../resources/features/create_504/output.txt | 20 ++++++++++++++ .../resources/features/create_504/script | 7 +++++ bundle/direct/dresources/create_recovery.go | 26 ++++++++++++++++--- .../direct/dresources/create_recovery_test.go | 15 +++++++++-- bundle/direct/dresources/feature.go | 9 ++++--- libs/testserver/handlers.go | 3 ++- 7 files changed, 72 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/resources/features/create_504/databricks.yml b/acceptance/bundle/resources/features/create_504/databricks.yml index 61f14c0564b..4c2fb81cb8b 100644 --- a/acceptance/bundle/resources/features/create_504/databricks.yml +++ b/acceptance/bundle/resources/features/create_504/databricks.yml @@ -4,7 +4,8 @@ bundle: resources: features: feature1: - full_name: main.myschema.myfeature + # UC lowercases the name; the lookup after a timed-out create must too. + full_name: main.myschema.MyFeature source: delta_table_source: full_name: main.myschema.mysource diff --git a/acceptance/bundle/resources/features/create_504/output.txt b/acceptance/bundle/resources/features/create_504/output.txt index 4dfe9fb1b5e..38a73d5fe72 100644 --- a/acceptance/bundle/resources/features/create_504/output.txt +++ b/acceptance/bundle/resources/features/create_504/output.txt @@ -24,3 +24,23 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged POST /api/2.0/feature-engineering/features GET /api/2.0/feature-engineering/features/main.myschema.myfeature POST /api/2.0/feature-engineering/features + +=== The feature cannot be read after the create request times out: deploy fails with the create error + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.features.feature1: checking whether feature main.myschema.myfeature was created: Fault injected by test. +Error: cannot create resources.features.feature1: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/features +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py --get //feature-engineering +POST /api/2.0/feature-engineering/features +GET /api/2.0/feature-engineering/features/main.myschema.myfeature diff --git a/acceptance/bundle/resources/features/create_504/script b/acceptance/bundle/resources/features/create_504/script index 43ee407fb14..ba647ee37cb 100644 --- a/acceptance/bundle/resources/features/create_504/script +++ b/acceptance/bundle/resources/features/create_504/script @@ -11,3 +11,10 @@ trace $CLI bundle deploy trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' $CLI bundle destroy --auto-approve > /dev/null 2>&1 rm -f "$OUT_REQUESTS" + +title "The feature cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/feature-engineering/features" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/feature-engineering/features/*" 403 0 1 PERMISSION_DENIED +errcode trace $CLI bundle deploy +trace print_requests.py --get //feature-engineering | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index ecdfb973913..f27a078364e 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -3,9 +3,10 @@ package dresources import ( "context" "errors" - "fmt" + "strings" "time" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/apierr" @@ -56,13 +57,13 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e // our request would have failed with "already exists" only if it reached that check. created, ok := rc.createTime(found) if !ok { - return nil, fmt.Errorf("%w; %s exists but has no create time, so it is not known to be created by this deployment", createErr, rc.name) + return nil, &recoveryError{msg: rc.name + " exists but has no create time, so it is not known to be created by this deployment", err: createErr} } if created.Before(rc.start.Add(-createTimeSkew)) { - return nil, fmt.Errorf("%w; %s exists but was not created by this deployment", createErr, rc.name) + return nil, &recoveryError{msg: rc.name + " exists but was not created by this deployment", err: createErr} } // Info, not Warn: the deploy got the resource it asked for, so there is nothing to act on. - log.Infof(ctx, "using %s, created despite the failed create request: %s", rc.name, createErr) + log.Infof(ctx, "using %s, created despite the failed create request: %s", rc.name, strings.TrimSpace(diag.FormatAPIErrorSummary(createErr))) return found, nil } if !apierr.IsMissing(err) { @@ -86,6 +87,23 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e } } +// recoveryError says why a resource found after a failed create request is not used, and wraps +// that request's error so the bundle error output still shows its status code and details. The +// 504s these requests get carry an empty message, so the message is left out when it is empty. +type recoveryError struct { + msg string + err error +} + +func (e *recoveryError) Error() string { + if m := e.err.Error(); m != "" { + return e.msg + ": " + m + } + return e.msg +} + +func (e *recoveryError) Unwrap() error { return e.err } + // isAlreadyExists reports whether a create request failed because the resource exists. Features, // dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with // 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go index 7b6d8f02494..2ce23bcf248 100644 --- a/bundle/direct/dresources/create_recovery_test.go +++ b/bundle/direct/dresources/create_recovery_test.go @@ -99,14 +99,25 @@ func TestCreateRecoveryUsesResourceCreatedDespiteTimeout(t *testing.T) { func TestCreateRecoveryRejectsResourceThatPredatesRequest(t *testing.T) { s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{testFeature(time.Now().Add(-time.Hour))}} _, _, err := createTestFeature(t, s, t.Context()) - require.ErrorContains(t, err, "feature main.default.feat exists but was not created by this deployment") + // The 504 has an empty message, so nothing follows the explanation; the bundle error output + // appends the status code and details from the wrapped error. + require.EqualError(t, err, "feature main.default.feat exists but was not created by this deployment") + requireStatus(t, err, 504) +} + +func TestCreateRecoveryKeepsCreateErrorMessage(t *testing.T) { + timeout := featureError(504, "DEADLINE_EXCEEDED", "The request timed out.") + s := &featureCreateScript{t: t, creates: []any{timeout}, gets: []any{testFeature(time.Now().Add(-time.Hour))}} + _, _, err := createTestFeature(t, s, t.Context()) + require.EqualError(t, err, "feature main.default.feat exists but was not created by this deployment: The request timed out.") requireStatus(t, err, 504) } func TestCreateRecoveryRejectsResourceWithoutCreateTime(t *testing.T) { s := &featureCreateScript{t: t, creates: []any{featureTimeout}, gets: []any{ml.Feature{FullName: testFeatureName}}} _, _, err := createTestFeature(t, s, t.Context()) - require.ErrorContains(t, err, "exists but has no create time") + require.EqualError(t, err, "feature main.default.feat exists but has no create time, so it is not known to be created by this deployment") + requireStatus(t, err, 504) } func TestCreateRecoveryRepeatsCreateWhileMissing(t *testing.T) { diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 130ac57db4e..8ec576a1f87 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -3,6 +3,7 @@ package dresources import ( "context" "slices" + "strings" "time" "github.com/databricks/cli/bundle/config/resources" @@ -35,12 +36,14 @@ func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (str start := time.Now() response, err := create(ctx) if err != nil { - // The feature is keyed by full_name, which the bundle chooses and the API keeps unique. + // The feature is keyed by full_name, which the bundle chooses and the API keeps unique. UC + // stores the name lowercased, which is also the id a successful create returns. + fullName := strings.ToLower(config.FullName) response, err = recoverCreate(ctx, createRecovery[ml.Feature]{ - name: "feature " + config.FullName, + name: "feature " + fullName, start: start, get: func(ctx context.Context) (*ml.Feature, error) { - return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: config.FullName}) + return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: fullName}) }, createTime: func(found *ml.Feature) (time.Time, bool) { if found.CreatedAt == nil { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 48efd7232ce..43e71a6475f 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -966,7 +966,8 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.FeaturesCreate(req) }) server.Handle("GET", "/api/2.0/feature-engineering/features/{full_name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.Features, req.Vars["full_name"]) + // Features are stored lowercased, and the cloud GET matches the name case-insensitively. + return MapGet(req.Workspace, req.Workspace.Features, strings.ToLower(req.Vars["full_name"])) }) server.Handle("PATCH", "/api/2.0/feature-engineering/features/{full_name}", func(req Request) any { return req.Workspace.FeaturesUpdate(req, req.Vars["full_name"])