From 6facda457c679fba7c8d8e36f658ccec58cd62aa Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 18:05:58 +0000 Subject: [PATCH 1/3] direct: use MCP and model provider services that a failed create request created Create requests for mcp_services and model_provider_services fail transiently (504 DEADLINE_EXCEEDED, 499 CANCELLED, HTTP/2 stream reset). Look the service up and use it if the failed request created it, or repeat the create while it is missing, as recoverCreate does for features, serving endpoints and dashboards. recoverCreate now also treats 499 and stream resets as uncertain. aws: TestAccept/bundle/resources/mcp_services/lifecycle TestAccept/bundle/resources/model_provider_services/lifecycle Co-authored-by: Isaac --- .../bundles/ai-gateway-create-recovery.md | 1 + .../create_failure/databricks.yml | 9 ++++ .../mcp_services/create_failure/out.test.toml | 2 + .../mcp_services/create_failure/output.txt | 26 ++++++++++++ .../mcp_services/create_failure/script | 13 ++++++ .../mcp_services/create_failure/test.toml | 4 ++ .../create_failure/databricks.yml | 20 +++++++++ .../create_failure/out.test.toml | 2 + .../create_failure/output.txt | 40 ++++++++++++++++++ .../create_failure/script | 21 ++++++++++ .../create_failure/test.toml | 4 ++ bundle/direct/dresources/create_recovery.go | 41 ++++++++++++++++--- .../direct/dresources/create_recovery_test.go | 23 +++++++++++ bundle/direct/dresources/mcp_service.go | 35 +++++++++++++--- .../dresources/model_provider_service.go | 35 +++++++++++++--- libs/retry/retry.go | 7 +++- libs/testserver/mcp_services.go | 13 ++++++ libs/testserver/model_provider_services.go | 13 ++++++ 18 files changed, 291 insertions(+), 18 deletions(-) create mode 100644 .nextchanges/bundles/ai-gateway-create-recovery.md create mode 100644 acceptance/bundle/resources/mcp_services/create_failure/databricks.yml create mode 100644 acceptance/bundle/resources/mcp_services/create_failure/out.test.toml create mode 100644 acceptance/bundle/resources/mcp_services/create_failure/output.txt create mode 100644 acceptance/bundle/resources/mcp_services/create_failure/script create mode 100644 acceptance/bundle/resources/mcp_services/create_failure/test.toml create mode 100644 acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml create mode 100644 acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml create mode 100644 acceptance/bundle/resources/model_provider_services/create_failure/output.txt create mode 100644 acceptance/bundle/resources/model_provider_services/create_failure/script create mode 100644 acceptance/bundle/resources/model_provider_services/create_failure/test.toml diff --git a/.nextchanges/bundles/ai-gateway-create-recovery.md b/.nextchanges/bundles/ai-gateway-create-recovery.md new file mode 100644 index 00000000000..48314ff4cbe --- /dev/null +++ b/.nextchanges/bundles/ai-gateway-create-recovery.md @@ -0,0 +1 @@ +* Use the MCP service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. diff --git a/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml b/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml new file mode 100644 index 00000000000..9b842c092b9 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + mcp_services: + mcp1: + parent: schemas/main.myschema + mcp_service_id: myservice + comment: COMMENT1 diff --git a/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml b/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/mcp_services/create_failure/output.txt b/acceptance/bundle/resources/mcp_services/create_failure/output.txt new file mode 100644 index 00000000000..aea5609def3 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/output.txt @@ -0,0 +1,26 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.mcp_services.mcp1: retrying after 499 from POST /api/2.1/unity-catalog/mcp-services +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice +POST /api/2.1/unity-catalog/mcp-services diff --git a/acceptance/bundle/resources/mcp_services/create_failure/script b/acceptance/bundle/resources/mcp_services/create_failure/script new file mode 100644 index 00000000000..c3a88cc0766 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/script @@ -0,0 +1,13 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/mcp-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/mcp_services/create_failure/test.toml b/acceptance/bundle/resources/mcp_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/mcp_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml b/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml new file mode 100644 index 00000000000..aeac81263d9 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + model_provider_services: + mps1: + parent: schemas/main.myschema + model_provider_service_id: myservice + comment: COMMENT1 + config: + provider_type: EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM + targets: + - model: gpt-4o + native_api_types: + - openai/v1/chat/completions + custom: + direct: + base_url: https://api.example.com/v1 + api_key: + plaintext: dummy-api-key diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml b/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/output.txt b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt new file mode 100644 index 00000000000..a849a2eadcc --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt @@ -0,0 +1,40 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 499 from POST /api/2.1/unity-catalog/model-provider-services +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-provider-services + +=== The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 499 from GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/script b/acceptance/bundle/resources/model_provider_services/create_failure/script new file mode 100644 index 00000000000..ac208183c21 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/script @@ -0,0 +1,21 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED +fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/test.toml b/acceptance/bundle/resources/model_provider_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/model_provider_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index f27a078364e..c08b1467eb2 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -38,15 +38,15 @@ type createRecovery[T any] struct { // repeat the request while it is missing. A repeat that fails because the resource exists means an // earlier request has landed in the meantime, so the next look finds it. // -// Errors that are not transient, and a done context, are returned as is. +// Errors that leave no doubt about the outcome, and a done context, are returned as is. func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) { - if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { + if ctx.Err() != nil || !isUncertainCreateError(ctx, createErr) { return nil, createErr } lastErr := createErr for attempt := 1; ; attempt++ { - found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) { + found, err := retry.With(ctx, isUncertainRead, func() (*T, error) { return rc.get(ctx) }) if ctx.Err() != nil { @@ -80,13 +80,43 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e if err == nil { return created, nil } - if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) { + if !isUncertainCreateError(ctx, err) && !isAlreadyExists(err) { return nil, err } lastErr = err } } +// isUncertainCreateError reports whether a create request failed in a way that leaves it unknown +// whether the resource was created. Besides the transient errors retry.IsTransient covers, these are: +// - 499 CANCELLED: the backend cancelled its own handling of the request (observed on model +// provider service creates, 2026-11), possibly after the resource was written; +// - an HTTP/2 stream reset, e.g. "response body: stream error: stream ID N; INTERNAL_ERROR" +// (observed on AI Gateway creates, 2026-11). It may come before or after the request was +// handled. +// +// Recovery looks the resource up before repeating the request, so treating these as uncertain +// cannot create a duplicate. +func isUncertainCreateError(ctx context.Context, err error) bool { + if retry.IsTransient(ctx, err) { + return true + } + if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { + // Match the status, not apierr.ErrCancelled: a mapped error_code takes precedence over it. + return apiErr.StatusCode == statusCancelled + } + // net/http bundles its http2 types unexported, so match the message of http2StreamError. + return strings.Contains(err.Error(), "stream error: stream ID ") +} + +// statusCancelled is the non-standard status the API uses for CANCELLED. +const statusCancelled = 499 + +// isUncertainRead reports whether the lookup after a failed create is worth repeating. +func isUncertainRead(ctx context.Context, err error) bool { + return retry.IsTransientRead(ctx, err) || isUncertainCreateError(ctx, err) +} + // recoveryError says why a resource found after a failed create request is not used, and wraps // that request's error so the bundle error output still shows its status code and details. The // 504s these requests get carry an empty message, so the message is left out when it is empty. @@ -105,7 +135,8 @@ func (e *recoveryError) Error() string { func (e *recoveryError) Unwrap() error { return e.err } // isAlreadyExists reports whether a create request failed because the resource exists. Features, -// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with +// dashboards, vector search endpoints and AI Gateway services (checked for model provider services) +// answer with 409 ALREADY_EXISTS, serving endpoints with // 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). func isAlreadyExists(err error) bool { return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) diff --git a/bundle/direct/dresources/create_recovery_test.go b/bundle/direct/dresources/create_recovery_test.go index 2ce23bcf248..31c5059d69b 100644 --- a/bundle/direct/dresources/create_recovery_test.go +++ b/bundle/direct/dresources/create_recovery_test.go @@ -3,6 +3,7 @@ package dresources import ( "context" "errors" + "fmt" "testing" "time" @@ -191,3 +192,25 @@ func TestCreateRecoveryStopsOnCancelledContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) assert.Equal(t, 1, s.createCount) } + +func TestIsUncertainCreateError(t *testing.T) { + ctx := t.Context() + for _, tc := range []struct { + name string + err error + want bool + }{ + {"504", &apierr.APIError{StatusCode: 504, ErrorCode: "DEADLINE_EXCEEDED", Message: "Storage timeout"}, true}, + {"499", &apierr.APIError{StatusCode: 499, ErrorCode: "CANCELLED"}, true}, + {"stream reset", fmt.Errorf("response body: %w", errors.New("stream error: stream ID 5; INTERNAL_ERROR; received from peer")), true}, + {"499 with mapped code", &apierr.APIError{StatusCode: 499, ErrorCode: "UNKNOWN"}, true}, + {"API error quoting a stream error", &apierr.APIError{StatusCode: 400, ErrorCode: "INVALID_PARAMETER_VALUE", Message: "upstream: stream error: stream ID 1; PROTOCOL_ERROR"}, false}, + {"400", &apierr.APIError{StatusCode: 400, ErrorCode: "INVALID_PARAMETER_VALUE"}, false}, + {"409", &apierr.APIError{StatusCode: 409, ErrorCode: "ALREADY_EXISTS"}, false}, + {"other", errors.New("boom"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isUncertainCreateError(ctx, tc.err)) + }) + } +} diff --git a/bundle/direct/dresources/mcp_service.go b/bundle/direct/dresources/mcp_service.go index 0882190dc5d..06301797e07 100644 --- a/bundle/direct/dresources/mcp_service.go +++ b/bundle/direct/dresources/mcp_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -90,13 +91,35 @@ func (r *ResourceMcpService) DoRead(ctx context.Context, id string) (*resources. } func (r *ResourceMcpService) DoCreate(ctx context.Context, config *resources.McpServiceConfig) (string, *resources.McpServiceConfig, error) { - resp, err := r.client.AiGateway.CreateMcpService(ctx, catalog.CreateMcpServiceRequest{ - Parent: config.Parent, - McpServiceId: config.McpServiceId, - McpService: mcpServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.McpService, error) { + return r.client.AiGateway.CreateMcpService(ctx, catalog.CreateMcpServiceRequest{ + Parent: config.Parent, + McpServiceId: config.McpServiceId, + McpService: mcpServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and mcp_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.McpServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.McpService]{ + name: "mcp service " + id, + start: start, + get: func(ctx context.Context) (*catalog.McpService, error) { + return r.client.AiGateway.GetMcpService(ctx, catalog.GetMcpServiceRequest{Name: mcpServiceNamePrefix + id}) + }, + createTime: func(found *catalog.McpService) (time.Time, bool) { + if found.CreateTime == nil { + return time.Time{}, false + } + return found.CreateTime.AsTime(), true + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToMcpServiceConfig(resp) if err != nil { diff --git a/bundle/direct/dresources/model_provider_service.go b/bundle/direct/dresources/model_provider_service.go index ebf02e9c3f6..0eae8b9ddbb 100644 --- a/bundle/direct/dresources/model_provider_service.go +++ b/bundle/direct/dresources/model_provider_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -90,13 +91,35 @@ func (r *ResourceModelProviderService) DoRead(ctx context.Context, id string) (* } func (r *ResourceModelProviderService) DoCreate(ctx context.Context, config *resources.ModelProviderServiceConfig) (string, *resources.ModelProviderServiceConfig, error) { - resp, err := r.client.AiGateway.CreateModelProviderService(ctx, catalog.CreateModelProviderServiceRequest{ - Parent: config.Parent, - ModelProviderServiceId: config.ModelProviderServiceId, - ModelProviderService: modelProviderServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.ModelProviderService, error) { + return r.client.AiGateway.CreateModelProviderService(ctx, catalog.CreateModelProviderServiceRequest{ + Parent: config.Parent, + ModelProviderServiceId: config.ModelProviderServiceId, + ModelProviderService: modelProviderServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and model_provider_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.ModelProviderServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.ModelProviderService]{ + name: "model provider service " + id, + start: start, + get: func(ctx context.Context) (*catalog.ModelProviderService, error) { + return r.client.AiGateway.GetModelProviderService(ctx, catalog.GetModelProviderServiceRequest{Name: modelProviderServiceNamePrefix + id}) + }, + createTime: func(found *catalog.ModelProviderService) (time.Time, bool) { + if found.CreateTime == nil { + return time.Time{}, false + } + return found.CreateTime.AsTime(), true + }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToModelProviderServiceConfig(resp) if err != nil { diff --git a/libs/retry/retry.go b/libs/retry/retry.go index db2de0d4524..b575e67309a 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -136,7 +136,12 @@ func retryMessage(err error) string { req := rw.Response.Request endpoint = fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) } - msg = fmt.Sprintf("retrying after %d %s%s", apiErr.StatusCode, http.StatusText(apiErr.StatusCode), endpoint) + status := strconv.Itoa(apiErr.StatusCode) + // Non-standard codes such as 499 have no status text. + if text := http.StatusText(apiErr.StatusCode); text != "" { + status += " " + text + } + msg = fmt.Sprintf("retrying after %s%s", status, endpoint) // Include the context the caller added to the error, e.g. the path of the file being written. if err.Error() != apiErr.Error() { msg += ": " + err.Error() diff --git a/libs/testserver/mcp_services.go b/libs/testserver/mcp_services.go index 41200d06951..d94655826ed 100644 --- a/libs/testserver/mcp_services.go +++ b/libs/testserver/mcp_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -27,11 +29,22 @@ func (s *FakeWorkspace) McpServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("mcp_service_id") + if _, exists := s.McpServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("McpService '%s' already exists", key), + }, + } + } + ms.Name = "mcp-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.McpServices[key] = ms return Response{ diff --git a/libs/testserver/model_provider_services.go b/libs/testserver/model_provider_services.go index 622bf75888c..c1b77417046 100644 --- a/libs/testserver/model_provider_services.go +++ b/libs/testserver/model_provider_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -28,11 +30,22 @@ func (s *FakeWorkspace) ModelProviderServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("model_provider_service_id") + if _, exists := s.ModelProviderServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("ModelProviderService '%s' already exists", key), + }, + } + } + ms.Name = "model-provider-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.ModelProviderServices[key] = ms return Response{ From 517bbff186d0ea6bae4615146f94374d3fbfa69a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 18:06:35 +0000 Subject: [PATCH 2/3] Add PR link to the changelog entry aws: TestAccept/bundle/resources/mcp_services/lifecycle TestAccept/bundle/resources/model_provider_services/lifecycle Co-authored-by: Isaac --- .nextchanges/bundles/{ai-gateway-create-recovery.md => 7782.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .nextchanges/bundles/{ai-gateway-create-recovery.md => 7782.md} (71%) diff --git a/.nextchanges/bundles/ai-gateway-create-recovery.md b/.nextchanges/bundles/7782.md similarity index 71% rename from .nextchanges/bundles/ai-gateway-create-recovery.md rename to .nextchanges/bundles/7782.md index 48314ff4cbe..3d25b6cb7c2 100644 --- a/.nextchanges/bundles/ai-gateway-create-recovery.md +++ b/.nextchanges/bundles/7782.md @@ -1 +1 @@ -* Use the MCP service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. +* Use the MCP service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. ([#7782](https://github.com/databricks/cli/pull/7782)) From 7a92076230c18a0a1a7d8b6716378767f1365558 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 19:21:33 +0000 Subject: [PATCH 3/3] Recover model_services creates too; cover the ALREADY_EXISTS repeat aws: TestAccept/bundle/resources/mcp_services/lifecycle TestAccept/bundle/resources/model_provider_services/lifecycle TestAccept/bundle/resources/model_services/lifecycle Co-authored-by: Isaac --- .nextchanges/bundles/7782.md | 2 +- .../mcp_services/create_failure/output.txt | 15 +++++++ .../mcp_services/create_failure/script | 8 ++++ .../create_failure/output.txt | 15 +++++++ .../create_failure/script | 8 ++++ .../create_failure/databricks.yml | 9 ++++ .../create_failure/out.test.toml | 2 + .../model_services/create_failure/output.txt | 41 +++++++++++++++++++ .../model_services/create_failure/script | 21 ++++++++++ .../model_services/create_failure/test.toml | 4 ++ bundle/direct/dresources/create_recovery.go | 9 ++++ bundle/direct/dresources/feature.go | 9 +--- bundle/direct/dresources/mcp_service.go | 9 +--- .../dresources/model_provider_service.go | 9 +--- bundle/direct/dresources/model_service.go | 30 +++++++++++--- libs/testserver/model_services.go | 13 ++++++ 16 files changed, 176 insertions(+), 28 deletions(-) create mode 100644 acceptance/bundle/resources/model_services/create_failure/databricks.yml create mode 100644 acceptance/bundle/resources/model_services/create_failure/out.test.toml create mode 100644 acceptance/bundle/resources/model_services/create_failure/output.txt create mode 100644 acceptance/bundle/resources/model_services/create_failure/script create mode 100644 acceptance/bundle/resources/model_services/create_failure/test.toml diff --git a/.nextchanges/bundles/7782.md b/.nextchanges/bundles/7782.md index 3d25b6cb7c2..b89b01873d3 100644 --- a/.nextchanges/bundles/7782.md +++ b/.nextchanges/bundles/7782.md @@ -1 +1 @@ -* Use the MCP service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. ([#7782](https://github.com/databricks/cli/pull/7782)) +* Use the MCP service, model service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. ([#7782](https://github.com/databricks/cli/pull/7782)) diff --git a/acceptance/bundle/resources/mcp_services/create_failure/output.txt b/acceptance/bundle/resources/mcp_services/create_failure/output.txt index aea5609def3..29873bf8303 100644 --- a/acceptance/bundle/resources/mcp_services/create_failure/output.txt +++ b/acceptance/bundle/resources/mcp_services/create_failure/output.txt @@ -24,3 +24,18 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged POST /api/2.1/unity-catalog/mcp-services GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice POST /api/2.1/unity-catalog/mcp-services + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.mcp_services.mcp1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/mcp-services +Created mcp_services.mcp1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //mcp-services +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice +POST /api/2.1/unity-catalog/mcp-services +GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/mcp_services/create_failure/script b/acceptance/bundle/resources/mcp_services/create_failure/script index c3a88cc0766..bf89a0e2986 100644 --- a/acceptance/bundle/resources/mcp_services/create_failure/script +++ b/acceptance/bundle/resources/mcp_services/create_failure/script @@ -11,3 +11,11 @@ trace $CLI bundle deploy trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' $CLI bundle destroy --auto-approve > /dev/null 2>&1 rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/mcp-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/output.txt b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt index a849a2eadcc..3298c16e583 100644 --- a/acceptance/bundle/resources/model_provider_services/create_failure/output.txt +++ b/acceptance/bundle/resources/model_provider_services/create_failure/output.txt @@ -38,3 +38,18 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged POST /api/2.1/unity-catalog/model-provider-services GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_provider_services.mps1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-provider-services +Created model_provider_services.mps1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-provider-services +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-provider-services +GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/model_provider_services/create_failure/script b/acceptance/bundle/resources/model_provider_services/create_failure/script index ac208183c21..06ac61b3c8a 100644 --- a/acceptance/bundle/resources/model_provider_services/create_failure/script +++ b/acceptance/bundle/resources/model_provider_services/create_failure/script @@ -19,3 +19,11 @@ trace $CLI bundle deploy trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' $CLI bundle destroy --auto-approve > /dev/null 2>&1 rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_services/create_failure/databricks.yml b/acceptance/bundle/resources/model_services/create_failure/databricks.yml new file mode 100644 index 00000000000..3f82564f271 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + model_services: + ms1: + parent: schemas/main.myschema + model_service_id: myservice + comment: COMMENT1 diff --git a/acceptance/bundle/resources/model_services/create_failure/out.test.toml b/acceptance/bundle/resources/model_services/create_failure/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/model_services/create_failure/output.txt b/acceptance/bundle/resources/model_services/create_failure/output.txt new file mode 100644 index 00000000000..6b28bf3550a --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/output.txt @@ -0,0 +1,41 @@ + +=== The create request times out but the service was created: deploy uses the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice + +=== The create request is cancelled and the service was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_services.ms1: retrying after 499 from POST /api/2.1/unity-catalog/model-services +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-services + +=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.model_services.ms1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-services +Created model_services.ms1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //model-services +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice +POST /api/2.1/unity-catalog/model-services +GET /api/2.1/unity-catalog/model-services/main.myschema.myservice diff --git a/acceptance/bundle/resources/model_services/create_failure/script b/acceptance/bundle/resources/model_services/create_failure/script new file mode 100644 index 00000000000..a0c9e579719 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/script @@ -0,0 +1,21 @@ +title "The create request times out but the service was created: deploy uses the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request is cancelled and the service was not created: deploy repeats the create" +fault.py "POST /api/2.1/unity-catalog/model-services" 499 0 1 CANCELLED +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service" +fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.1/unity-catalog/model-services/*" 404 0 1 NOT_FOUND +trace $CLI bundle deploy +trace print_requests.py --get //model-services | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/model_services/create_failure/test.toml b/acceptance/bundle/resources/model_services/create_failure/test.toml new file mode 100644 index 00000000000..695f2ead5c6 --- /dev/null +++ b/acceptance/bundle/resources/model_services/create_failure/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = true +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index c08b1467eb2..e854dc51fbf 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/apierr" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" ) // createTimeSkew is how much earlier than our first create request a found resource's create time @@ -117,6 +118,14 @@ func isUncertainRead(ctx context.Context, err error) bool { return retry.IsTransientRead(ctx, err) || isUncertainCreateError(ctx, err) } +// sdkTime converts a create time from an SDK response for createRecovery.createTime. +func sdkTime(t *sdktime.Time) (time.Time, bool) { + if t == nil { + return time.Time{}, false + } + return t.AsTime(), true +} + // recoveryError says why a resource found after a failed create request is not used, and wraps // that request's error so the bundle error output still shows its status code and details. The // 504s these requests get carry an empty message, so the message is left out when it is empty. diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8ec576a1f87..c5c8265bff4 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -45,13 +45,8 @@ func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (str get: func(ctx context.Context) (*ml.Feature, error) { return r.client.FeatureEngineering.GetFeature(ctx, ml.GetFeatureRequest{FullName: fullName}) }, - createTime: func(found *ml.Feature) (time.Time, bool) { - if found.CreatedAt == nil { - return time.Time{}, false - } - return found.CreatedAt.AsTime(), true - }, - create: create, + createTime: func(found *ml.Feature) (time.Time, bool) { return sdkTime(found.CreatedAt) }, + create: create, }, err) if err != nil { return "", nil, err diff --git a/bundle/direct/dresources/mcp_service.go b/bundle/direct/dresources/mcp_service.go index 06301797e07..b44309375e3 100644 --- a/bundle/direct/dresources/mcp_service.go +++ b/bundle/direct/dresources/mcp_service.go @@ -109,13 +109,8 @@ func (r *ResourceMcpService) DoCreate(ctx context.Context, config *resources.Mcp get: func(ctx context.Context) (*catalog.McpService, error) { return r.client.AiGateway.GetMcpService(ctx, catalog.GetMcpServiceRequest{Name: mcpServiceNamePrefix + id}) }, - createTime: func(found *catalog.McpService) (time.Time, bool) { - if found.CreateTime == nil { - return time.Time{}, false - } - return found.CreateTime.AsTime(), true - }, - create: create, + createTime: func(found *catalog.McpService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, }, err) if err != nil { return "", nil, err diff --git a/bundle/direct/dresources/model_provider_service.go b/bundle/direct/dresources/model_provider_service.go index 0eae8b9ddbb..c71a1c30818 100644 --- a/bundle/direct/dresources/model_provider_service.go +++ b/bundle/direct/dresources/model_provider_service.go @@ -109,13 +109,8 @@ func (r *ResourceModelProviderService) DoCreate(ctx context.Context, config *res get: func(ctx context.Context) (*catalog.ModelProviderService, error) { return r.client.AiGateway.GetModelProviderService(ctx, catalog.GetModelProviderServiceRequest{Name: modelProviderServiceNamePrefix + id}) }, - createTime: func(found *catalog.ModelProviderService) (time.Time, bool) { - if found.CreateTime == nil { - return time.Time{}, false - } - return found.CreateTime.AsTime(), true - }, - create: create, + createTime: func(found *catalog.ModelProviderService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, }, err) if err != nil { return "", nil, err diff --git a/bundle/direct/dresources/model_service.go b/bundle/direct/dresources/model_service.go index 7eb3860d191..7f6f8324aac 100644 --- a/bundle/direct/dresources/model_service.go +++ b/bundle/direct/dresources/model_service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -96,13 +97,30 @@ func (r *ResourceModelService) DoRead(ctx context.Context, id string) (*resource } func (r *ResourceModelService) DoCreate(ctx context.Context, config *resources.ModelServiceConfig) (string, *resources.ModelServiceConfig, error) { - resp, err := r.client.AiGateway.CreateModelService(ctx, catalog.CreateModelServiceRequest{ - Parent: config.Parent, - ModelServiceId: config.ModelServiceId, - ModelService: modelServiceBody(config), - }) + create := func(ctx context.Context) (*catalog.ModelService, error) { + return r.client.AiGateway.CreateModelService(ctx, catalog.CreateModelServiceRequest{ + Parent: config.Parent, + ModelServiceId: config.ModelServiceId, + ModelService: modelServiceBody(config), + }) + } + start := time.Now() + resp, err := create(ctx) if err != nil { - return "", nil, err + // The bundle chooses the full name (parent and model_service_id), and it is unique in the metastore. + id := strings.TrimPrefix(config.Parent, "schemas/") + "." + config.ModelServiceId + resp, err = recoverCreate(ctx, createRecovery[catalog.ModelService]{ + name: "model service " + id, + start: start, + get: func(ctx context.Context) (*catalog.ModelService, error) { + return r.client.AiGateway.GetModelService(ctx, catalog.GetModelServiceRequest{Name: modelServiceNamePrefix + id}) + }, + createTime: func(found *catalog.ModelService) (time.Time, bool) { return sdkTime(found.CreateTime) }, + create: create, + }, err) + if err != nil { + return "", nil, err + } } state, err := responseToModelServiceConfig(resp) if err != nil { diff --git a/libs/testserver/model_services.go b/libs/testserver/model_services.go index ce653543ac5..d9189b5a30e 100644 --- a/libs/testserver/model_services.go +++ b/libs/testserver/model_services.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -33,11 +35,22 @@ func (s *FakeWorkspace) ModelServicesCreate(req Request) Response { schema := strings.TrimPrefix(req.URL.Query().Get("parent"), "schemas/") key := schema + "." + req.URL.Query().Get("model_service_id") + if _, exists := s.ModelServices[key]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{ + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("ModelService '%s' already exists", key), + }, + } + } + ms.Name = "model-services/" + key ms.CreatedBy = s.CurrentUser().UserName ms.UpdatedBy = s.CurrentUser().UserName ms.EffectiveOwner = s.CurrentUser().UserName ms.MetastoreId = nextUUID() + ms.CreateTime = sdktime.New(time.Unix(0, nowNano()).UTC()) s.ModelServices[key] = ms return Response{