Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7782.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the MCP service, model service or model provider service that a create request created despite failing with a timeout, a 499 or a reset stream, instead of failing the deploy, in the direct deployment engine. Create recovery for features, serving endpoints and dashboards now also covers 499 and reset streams. ([#7782](https://github.com/databricks/cli/pull/7782))
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
bundle:
name: test-bundle

resources:
mcp_services:
mcp1:
parent: schemas/main.myschema
mcp_service_id: myservice
comment: COMMENT1
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@

=== The create request times out but the service was created: deploy uses the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created mcp_services.mcp1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //mcp-services
POST /api/2.1/unity-catalog/mcp-services
GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice

=== The create request is cancelled and the service was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.mcp_services.mcp1: retrying after 499 from POST /api/2.1/unity-catalog/mcp-services
Created mcp_services.mcp1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //mcp-services
POST /api/2.1/unity-catalog/mcp-services
GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice
POST /api/2.1/unity-catalog/mcp-services

=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.mcp_services.mcp1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/mcp-services
Created mcp_services.mcp1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //mcp-services
POST /api/2.1/unity-catalog/mcp-services
GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice
POST /api/2.1/unity-catalog/mcp-services
GET /api/2.1/unity-catalog/mcp-services/main.myschema.myservice
21 changes: 21 additions & 0 deletions acceptance/bundle/resources/mcp_services/create_failure/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
title "The create request times out but the service was created: deploy uses the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request is cancelled and the service was not created: deploy repeats the create"
fault.py "POST /api/2.1/unity-catalog/mcp-services" 499 0 1 CANCELLED
trace $CLI bundle deploy
trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/mcp-services" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.1/unity-catalog/mcp-services/*" 404 0 1 NOT_FOUND
trace $CLI bundle deploy
trace print_requests.py --get //mcp-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Cloud = false
RecordRequests = true
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
bundle:
name: test-bundle

resources:
model_provider_services:
mps1:
parent: schemas/main.myschema
model_provider_service_id: myservice
comment: COMMENT1
config:
provider_type: EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM
targets:
- model: gpt-4o
native_api_types:
- openai/v1/chat/completions
custom:
direct:
base_url: https://api.example.com/v1
api_key:
plaintext: dummy-api-key
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@

=== The create request times out but the service was created: deploy uses the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created model_provider_services.mps1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-provider-services
POST /api/2.1/unity-catalog/model-provider-services
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice

=== The create request is cancelled and the service was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_provider_services.mps1: retrying after 499 from POST /api/2.1/unity-catalog/model-provider-services
Created model_provider_services.mps1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-provider-services
POST /api/2.1/unity-catalog/model-provider-services
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice
POST /api/2.1/unity-catalog/model-provider-services

=== The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_provider_services.mps1: retrying after 499 from GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice
Created model_provider_services.mps1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-provider-services
POST /api/2.1/unity-catalog/model-provider-services
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice

=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_provider_services.mps1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-provider-services
Created model_provider_services.mps1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-provider-services
POST /api/2.1/unity-catalog/model-provider-services
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice
POST /api/2.1/unity-catalog/model-provider-services
GET /api/2.1/unity-catalog/model-provider-services/main.myschema.myservice
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
title "The create request times out but the service was created: deploy uses the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request is cancelled and the service was not created: deploy repeats the create"
fault.py "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED
trace $CLI bundle deploy
trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request is cancelled after creating the service and the first lookup is cancelled too: deploy repeats the lookup and uses the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 499 0 1 CANCELLED
fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 499 0 1 CANCELLED
trace $CLI bundle deploy
trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/model-provider-services" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.1/unity-catalog/model-provider-services/*" 404 0 1 NOT_FOUND
trace $CLI bundle deploy
trace print_requests.py --get //model-provider-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Cloud = false
RecordRequests = true
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
bundle:
name: test-bundle

resources:
model_services:
ms1:
parent: schemas/main.myschema
model_service_id: myservice
comment: COMMENT1
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@

=== The create request times out but the service was created: deploy uses the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created model_services.ms1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-services
POST /api/2.1/unity-catalog/model-services
GET /api/2.1/unity-catalog/model-services/main.myschema.myservice

=== The create request is cancelled and the service was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_services.ms1: retrying after 499 from POST /api/2.1/unity-catalog/model-services
Created model_services.ms1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-services
POST /api/2.1/unity-catalog/model-services
GET /api/2.1/unity-catalog/model-services/main.myschema.myservice
POST /api/2.1/unity-catalog/model-services

=== The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.model_services.ms1: retrying after 504 Gateway Timeout from POST /api/2.1/unity-catalog/model-services
Created model_services.ms1
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //model-services
POST /api/2.1/unity-catalog/model-services
GET /api/2.1/unity-catalog/model-services/main.myschema.myservice
POST /api/2.1/unity-catalog/model-services
GET /api/2.1/unity-catalog/model-services/main.myschema.myservice
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
title "The create request times out but the service was created: deploy uses the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //model-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request is cancelled and the service was not created: deploy repeats the create"
fault.py "POST /api/2.1/unity-catalog/model-services" 499 0 1 CANCELLED
trace $CLI bundle deploy
trace print_requests.py --get //model-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out after creating the service and the lookup misses it: the repeated create gets ALREADY_EXISTS and the next lookup finds the service"
fault.py --after-handler "POST /api/2.1/unity-catalog/model-services" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.1/unity-catalog/model-services/*" 404 0 1 NOT_FOUND
trace $CLI bundle deploy
trace print_requests.py --get //model-services | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Cloud = false
RecordRequests = true
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
50 changes: 45 additions & 5 deletions bundle/direct/dresources/create_recovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/apierr"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
)

// createTimeSkew is how much earlier than our first create request a found resource's create time
Expand Down Expand Up @@ -38,15 +39,15 @@ type createRecovery[T any] struct {
// repeat the request while it is missing. A repeat that fails because the resource exists means an
// earlier request has landed in the meantime, so the next look finds it.
//
// Errors that are not transient, and a done context, are returned as is.
// Errors that leave no doubt about the outcome, and a done context, are returned as is.
func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr error) (*T, error) {
if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) {
if ctx.Err() != nil || !isUncertainCreateError(ctx, createErr) {
return nil, createErr
}

lastErr := createErr
for attempt := 1; ; attempt++ {
found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) {
found, err := retry.With(ctx, isUncertainRead, func() (*T, error) {
return rc.get(ctx)
})
if ctx.Err() != nil {
Expand Down Expand Up @@ -80,13 +81,51 @@ func recoverCreate[T any](ctx context.Context, rc createRecovery[T], createErr e
if err == nil {
return created, nil
}
if !retry.IsTransient(ctx, err) && !isAlreadyExists(err) {
if !isUncertainCreateError(ctx, err) && !isAlreadyExists(err) {
return nil, err
}
lastErr = err
}
}

// isUncertainCreateError reports whether a create request failed in a way that leaves it unknown
// whether the resource was created. Besides the transient errors retry.IsTransient covers, these are:
// - 499 CANCELLED: the backend cancelled its own handling of the request (observed on model
// provider service creates, 2026-11), possibly after the resource was written;
// - an HTTP/2 stream reset, e.g. "response body: stream error: stream ID N; INTERNAL_ERROR"
// (observed on AI Gateway creates, 2026-11). It may come before or after the request was
// handled.
//
// Recovery looks the resource up before repeating the request, so treating these as uncertain
// cannot create a duplicate.
func isUncertainCreateError(ctx context.Context, err error) bool {
if retry.IsTransient(ctx, err) {
return true
}
if apiErr, ok := errors.AsType[*apierr.APIError](err); ok {
// Match the status, not apierr.ErrCancelled: a mapped error_code takes precedence over it.
return apiErr.StatusCode == statusCancelled
}
// net/http bundles its http2 types unexported, so match the message of http2StreamError.
return strings.Contains(err.Error(), "stream error: stream ID ")
}

// statusCancelled is the non-standard status the API uses for CANCELLED.
const statusCancelled = 499

// isUncertainRead reports whether the lookup after a failed create is worth repeating.
func isUncertainRead(ctx context.Context, err error) bool {
return retry.IsTransientRead(ctx, err) || isUncertainCreateError(ctx, err)
}

// sdkTime converts a create time from an SDK response for createRecovery.createTime.
func sdkTime(t *sdktime.Time) (time.Time, bool) {
if t == nil {
return time.Time{}, false
}
return t.AsTime(), true
}

// recoveryError says why a resource found after a failed create request is not used, and wraps
// that request's error so the bundle error output still shows its status code and details. The
// 504s these requests get carry an empty message, so the message is left out when it is empty.
Expand All @@ -105,7 +144,8 @@ func (e *recoveryError) Error() string {
func (e *recoveryError) Unwrap() error { return e.err }

// isAlreadyExists reports whether a create request failed because the resource exists. Features,
// dashboards and vector search endpoints answer with 409 ALREADY_EXISTS, serving endpoints with
// dashboards, vector search endpoints and AI Gateway services (checked for model provider services)
// answer with 409 ALREADY_EXISTS, serving endpoints with
// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11).
func isAlreadyExists(err error) bool {
return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists)
Expand Down
Loading
Loading