diff --git a/.nextchanges/bundles/7782.md b/.nextchanges/bundles/7782.md new file mode 100644 index 00000000000..b89b01873d3 --- /dev/null +++ b/.nextchanges/bundles/7782.md @@ -0,0 +1 @@ +* Use the MCP service, model service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. ([#7782](https://github.com/databricks/cli/pull/7782)) diff --git a/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml b/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml new file mode 100644 index 00000000000..9b842c092b9 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + mcp_services: + mcp1: + parent: schemas/main.myschema + mcp_service_id: myservice + comment: COMMENT1 diff --git a/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml b/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/mcp_services/create_failure/output.txt b/acceptance/bundle/resources/mcp_services/create_failure/output.txt new file mode 100644 index 00000000000..29873bf8303 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/output.txt @@ -0,0 +1,41 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.mcp_services.mcp1: retrying after 499 from POST /api/2.1/unity-catalog/mcp-services +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice +POST /api/2.1/unity-catalog/mcp-services + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.mcp_services.mcp1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/mcp-services +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/mcp_services/create_failure/script b/acceptance/bundle/resources/mcp_services/create_failure/script new file mode 100644 index 00000000000..bf89a0e2986 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/script @@ -0,0 +1,21 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/mcp-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/mcp-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/mcp_services/create_failure/test.toml b/acceptance/bundle/resources/mcp_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml b/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml new file mode 100644 index 00000000000..aeac81263d9 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + model_provider_services: + mps1: + parent: schemas/main.myschema + model_provider_service_id: myservice + comment: COMMENT1 + config: + provider_type: EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM + targets: + - model: gpt-4o + native_api_types: + - openai/v1/chat/completions + custom: + direct: + base_url: https://api.example.com/v1 + api_key: + plaintext: dummy-api-key diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml b/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/output.txt b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt new file mode 100644 index 00000000000..3298c16e583 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt @@ -0,0 +1,55 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 499 from POST /api/2.1/unity-catalog/model-provider-services +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-provider-services + +=== The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 499 from GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-provider-services +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/script b/acceptance/bundle/resources/model_provider_services/create_failure/script new file mode 100644 index 00000000000..06ac61b3c8a --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/script @@ -0,0 +1,29 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED +fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/test.toml b/acceptance/bundle/resources/model_provider_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/model_services/create_failure/databricks.yml b/acceptance/bundle/resources/model_services/create_failure/databricks.yml new file mode 100644 index 00000000000..3f82564f271 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + model_services: + ms1: + parent: schemas/main.myschema + model_service_id: myservice + comment: COMMENT1 diff --git a/acceptance/bundle/resources/model_services/create_failure/out.test.toml b/acceptance/bundle/resources/model_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/model_services/create_failure/output.txt b/acceptance/bundle/resources/model_services/create_failure/output.txt new file mode 100644 index 00000000000..6b28bf3550a --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/output.txt @@ -0,0 +1,41 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_services.ms1: retrying after 499 from POST /api/2.1/unity-catalog/model-services +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-services + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_services.ms1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-services +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/model_services/create_failure/script b/acceptance/bundle/resources/model_services/create_failure/script new file mode 100644 index 00000000000..a0c9e579719 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/script @@ -0,0 +1,21 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/model-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/model-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_services/create_failure/test.toml b/acceptance/bundle/resources/model_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index f27a078364e..e854dc51fbf 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/apierr" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" ) // createTimeSkew is how much earlier than our first create request a found resource's create time @@ -38,15 +39,15 @@ type createRecovery[T any] struct { // repeat the request while it is missing. A repeat that fails because the resource exists means an // earlier request has landed in the meantime, so the next look finds it. // -// Errors that are not transient, and a done context, are returned as is. +// Errors that leave no doubt about the outcome, and a done context, are returned as is. func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) { - if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { + if ctx.Err() != nil || !isUncertainCreateError(ctx, createErr) { return nil, createErr } lastErr := createErr for attempt := 1; ; attempt++ { - found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) { + found, err := retry.With(ctx, isUncertainRead, func() (*T, error) { return rc.get(ctx) }) if ctx.Err() != nil { @@ -80,13 +81,51 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e if err == nil { return created, nil } - if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) { + if !isUncertainCreateError(ctx, err) && !isAlreadyExists(err) { return nil, err } lastErr = err } } +// isUncertainCreateError reports whether a create request failed in a way that leaves it unknown +// whether the resource was created. Besides the transient errors retry.IsTransient covers, these are: +// - 499 CANCELLED: the backend cancelled its own handling of the request (observed on model +// provider service creates, 2026-11), possibly after the resource was written; +// - an HTTP/2 stream reset, e.g. "response body: stream error: stream ID N; INTERNAL_ERROR" +// (observed on AI Gateway creates, 2026-11). It may come before or after the request was +// handled. +// +// Recovery looks the resource up before repeating the request, so treating these as uncertain +// cannot create a duplicate. +func isUncertainCreateError(ctx context.Context, err error) bool { + if retry.IsTransient(ctx, err) { + return true + } + if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { + // Match the status, not apierr.ErrCancelled: a mapped error_code takes precedence over it. + return apiErr.StatusCode == statusCancelled + } + // net/http bundles its http2 types unexported, so match the message of http2StreamError. + return strings.Contains(err.Error(), "stream error: stream ID ") +} + +// statusCancelled is the non-standard status the API uses for CANCELLED. +const statusCancelled = 499 + +// isUncertainRead reports whether the lookup after a failed create is worth repeating. +func isUncertainRead(ctx context.Context, err error) bool { + return retry.IsTransientRead(ctx, err) || isUncertainCreateError(ctx, err) +} + +// sdkTime converts a create time from an SDK response for createRecovery.createTime. +func sdkTime(t *sdktime.Time) (time.Time, bool) { + if t == nil { + return time.Time{}, false + } + return t.AsTime(), true +} + // recoveryError says why a resource found after a failed create request is not used, and wraps // that request's error so the bundle error output still shows its status code and details. The // 504s these requests get carry an empty message, so the message is left out when it is empty. @@ -105,7 +144,8 @@ func (e *recoveryError) Error() string { func (e *recoveryError) Unwrap() error { return e.err } // isAlreadyExists reports whether a create request failed because the resource exists. Features, -// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with +// dashboards, vector search endpoints and AI Gateway services (checked for model provider services) +// answer with 409 ALREADY_EXISTS, serving endpoints with // 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). func isAlreadyExists(err error) bool { return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go index 2ce23bcf248..31c5059d69b 100644 --- a/bundle/direct/dresources/create_recovery_test.go +++ b/bundle/direct/dresources/create_recovery_test.go @@ -3,6 +3,7 @@ package dresources import ( "context" "errors" + "fmt" "testing" "time" @@ -191,3 +192,25 @@ func TestCreateRecoveryStopsOnCancelledContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) assert.Equal(t, 1, s.createCount) } + +func TestIsUncertainCreateError(t *testing.T) { + ctx := t.Context() + for _, tc := range []struct { + name string + err error + want bool + }{ + {"504", &apierr.APIError{StatusCode: 504, ErrorCode: "DEADLINE_EXCEEDED", Message: "Storage timeout"}, true}, + {"499", &apierr.APIError{StatusCode: 499, ErrorCode: "CANCELLED"}, true}, + {"stream reset", fmt.Errorf("response body: %w", errors.New("stream error: stream ID 5; INTERNAL_ERROR; received from peer")), true}, + {"499 with mapped code", &apierr.APIError{StatusCode: 499, ErrorCode: "UNKNOWN"}, true}, + {"API error quoting a stream error", &apierr.APIError{StatusCode: 400, ErrorCode: "INVALID_PARAMETER_VALUE", Message: "upstream: stream error: stream ID 1; PROTOCOL_ERROR"}, false}, + {"400", &apierr.APIError{StatusCode: 400, ErrorCode: "INVALID_PARAMETER_VALUE"}, false}, + {"409", &apierr.APIError{StatusCode: 409, ErrorCode: "ALREADY_EXISTS"}, false}, + {"other", errors.New("boom"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isUncertainCreateError(ctx, tc.err)) + }) + } +} diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8ec576a1f87..c5c8265bff4 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -45,13 +45,8 @@ func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (str get: func(ctx context.Context) (*ml.Feature, error) { return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: fullName}) }, - createTime: func(found *ml.Feature) (time.Time, bool) { - if found.CreatedAt == nil { - return time.Time{}, false - } - return found.CreatedAt.AsTime(), true - }, - create: create, + createTime: func(found *ml.Feature) (time.Time, bool) { return sdkTime(found.CreatedAt) }, + create: create, }, err) if err != nil { return "", nil, err diff --git a/bundle/direct/dresources/mcp_service.go b/bundle/direct/dresources/mcp_service.go index 0882190dc5d..b44309375e3 100644 --- a/bundle/direct/dresources/mcp_service.go +++ b/bundle/direct/dresources/mcp_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -90,13 +91,30 @@ func (r *ResourceMcpService) DoRead(ctx context.Context, id string) (*resources. } func (r *ResourceMcpService) DoCreate(ctx context.Context, config *resources.McpServiceConfig) (string, *resources.McpServiceConfig, error) { - resp, err := r.client.AiGateway.CreateMcpService(ctx, catalog.CreateMcpServiceRequest{ - Parent: config.Parent, - McpServiceId: config.McpServiceId, - McpService: mcpServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.McpService, error) { + return r.client.AiGateway.CreateMcpService(ctx, catalog.CreateMcpServiceRequest{ + Parent: config.Parent, + McpServiceId: config.McpServiceId, + McpService: mcpServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and mcp_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.McpServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.McpService]{ + name: "mcp service " + id, + start: start, + get: func(ctx context.Context) (*catalog.McpService, error) { + return r.client.AiGateway.GetMcpService(ctx, catalog.GetMcpServiceRequest{Name: mcpServiceNamePrefix + id}) + }, + createTime: func(found *catalog.McpService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToMcpServiceConfig(resp) if err != nil { diff --git a/bundle/direct/dresources/model_provider_service.go b/bundle/direct/dresources/model_provider_service.go index ebf02e9c3f6..c71a1c30818 100644 --- a/bundle/direct/dresources/model_provider_service.go +++ b/bundle/direct/dresources/model_provider_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -90,13 +91,30 @@ func (r *ResourceModelProviderService) DoRead(ctx context.Context, id string) (* } func (r *ResourceModelProviderService) DoCreate(ctx context.Context, config *resources.ModelProviderServiceConfig) (string, *resources.ModelProviderServiceConfig, error) { - resp, err := r.client.AiGateway.CreateModelProviderService(ctx, catalog.CreateModelProviderServiceRequest{ - Parent: config.Parent, - ModelProviderServiceId: config.ModelProviderServiceId, - ModelProviderService: modelProviderServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.ModelProviderService, error) { + return r.client.AiGateway.CreateModelProviderService(ctx, catalog.CreateModelProviderServiceRequest{ + Parent: config.Parent, + ModelProviderServiceId: config.ModelProviderServiceId, + ModelProviderService: modelProviderServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and model_provider_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.ModelProviderServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.ModelProviderService]{ + name: "model provider service " + id, + start: start, + get: func(ctx context.Context) (*catalog.ModelProviderService, error) { + return r.client.AiGateway.GetModelProviderService(ctx, catalog.GetModelProviderServiceRequest{Name: modelProviderServiceNamePrefix + id}) + }, + createTime: func(found *catalog.ModelProviderService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToModelProviderServiceConfig(resp) if err != nil { diff --git a/bundle/direct/dresources/model_service.go b/bundle/direct/dresources/model_service.go index 7eb3860d191..7f6f8324aac 100644 --- a/bundle/direct/dresources/model_service.go +++ b/bundle/direct/dresources/model_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -96,13 +97,30 @@ func (r *ResourceModelService) DoRead(ctx context.Context, id string) (*resource } func (r *ResourceModelService) DoCreate(ctx context.Context, config *resources.ModelServiceConfig) (string, *resources.ModelServiceConfig, error) { - resp, err := r.client.AiGateway.CreateModelService(ctx, catalog.CreateModelServiceRequest{ - Parent: config.Parent, - ModelServiceId: config.ModelServiceId, - ModelService: modelServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.ModelService, error) { + return r.client.AiGateway.CreateModelService(ctx, catalog.CreateModelServiceRequest{ + Parent: config.Parent, + ModelServiceId: config.ModelServiceId, + ModelService: modelServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and model_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.ModelServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.ModelService]{ + name: "model service " + id, + start: start, + get: func(ctx context.Context) (*catalog.ModelService, error) { + return r.client.AiGateway.GetModelService(ctx, catalog.GetModelServiceRequest{Name: modelServiceNamePrefix + id}) + }, + createTime: func(found *catalog.ModelService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToModelServiceConfig(resp) if err != nil { diff --git a/libs/retry/retry.go b/libs/retry/retry.go index db2de0d4524..b575e67309a 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -136,7 +136,12 @@ func retryMessage(err error) string { req := rw.Response.Request endpoint = fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) } - msg = fmt.Sprintf("retrying after %d %s%s", apiErr.StatusCode, http.StatusText(apiErr.StatusCode), endpoint) + status := strconv.Itoa(apiErr.StatusCode) + // Non-standard codes such as 499 have no status text. + if text := http.StatusText(apiErr.StatusCode); text != "" { + status += " " + text + } + msg = fmt.Sprintf("retrying after %s%s", status, endpoint) // Include the context the caller added to the error, e.g. the path of the file being written. if err.Error() != apiErr.Error() { msg += ": " + err.Error() diff --git a/libs/testserver/mcp_services.go b/libs/testserver/mcp_services.go index 41200d06951..d94655826ed 100644 --- a/libs/testserver/mcp_services.go +++ b/libs/testserver/mcp_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -27,11 +29,22 @@ func (s *FakeWorkspace) McpServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("mcp_service_id") + if _, exists := s.McpServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("McpService '%s' already exists", key), + }, + } + } + ms.Name = "mcp-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.McpServices[key] = ms return Response{ diff --git a/libs/testserver/model_provider_services.go b/libs/testserver/model_provider_services.go index 622bf75888c..c1b77417046 100644 --- a/libs/testserver/model_provider_services.go +++ b/libs/testserver/model_provider_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -28,11 +30,22 @@ func (s *FakeWorkspace) ModelProviderServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("model_provider_service_id") + if _, exists := s.ModelProviderServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("ModelProviderService '%s' already exists", key), + }, + } + } + ms.Name = "model-provider-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.ModelProviderServices[key] = ms return Response{ diff --git a/libs/testserver/model_services.go b/libs/testserver/model_services.go index ce653543ac5..d9189b5a30e 100644 --- a/libs/testserver/model_services.go +++ b/libs/testserver/model_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -33,11 +35,22 @@ func (s *FakeWorkspace) ModelServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("model_service_id") + if _, exists := s.ModelServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("ModelService '%s' already exists", key), + }, + } + } + ms.Name = "model-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.ModelServices[key] = ms return Response{