diff --git a/cmd/air/air.go b/cmd/air/air.go index 4154bab891c..584f4165c58 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -47,7 +47,7 @@ active runs with "databricks air list".`, wrapRunErrorWithDebugTip(runCommand) addToGroup(cmd, runGroupID, runCommand) addToGroup(cmd, manageGroupID, newGetCommand(), newListCommand(), newLogsCommand(), newCancelCommand()) - addToGroup(cmd, packageGroupID, newConvertToDabsCommand(), newImagesCommand()) + addToGroup(cmd, packageGroupID, newDeployCommand(), newConvertToDabsCommand(), newImagesCommand()) addToGroup(cmd, capacityGroupID, newPoolsCommand()) // Subcommands inherit this help func, so the footer is printed only for `air` itself. diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 11e4546d21d..1c877160270 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -18,7 +18,7 @@ func TestNewRegistersAllSubcommands(t *testing.T) { registered[c.Name()] = true } - want := []string{"run", "get", "list", "pools", "logs", "cancel", "convert-to-dabs", "images"} + want := []string{"run", "get", "list", "pools", "logs", "cancel", "deploy", "convert-to-dabs", "images"} for _, name := range want { assert.True(t, registered[name], "subcommand %q is not registered", name) } diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go new file mode 100644 index 00000000000..3d41ba6e990 --- /dev/null +++ b/cmd/air/deploy.go @@ -0,0 +1,535 @@ +package aircmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/spf13/cobra" +) + +// defaultDeployTimeout bounds the whole wait: staging a checkpoint's weights and +// starting the endpoint. +const defaultDeployTimeout = time.Hour + +// Errors air deploy wraps when it stops waiting. The server keeps working, and +// the error says how to continue. +var ( + errDeployInterrupted = errors.New("air deploy was interrupted") + errDeployTimedOut = errors.New("air deploy timed out") +) + +// JSON error codes of air deploy's own failures; API errors keep the server's. +const ( + codeInvalidArgs = "INVALID_ARGS" + codeInterrupted = "INTERRUPTED" + codeTimeout = "TIMEOUT" + codeStagingFailed = "STAGING_FAILED" + // codeEndpointNotReady: the endpoint serves the version but isn't ready, which + // running air deploy again doesn't change. + codeEndpointNotReady = "ENDPOINT_NOT_READY" + codeInternalError = "INTERNAL_ERROR" +) + +// apiErrorCodeRe matches an error code the server sent. The SDK fills ErrorCode +// with "" or "" when the response body has none. +var apiErrorCodeRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Z][A-Z_]*$`) }) + +// deployError carries the JSON error code and retryability of a failure whose +// message is err's. +type deployError struct { + code string + retryable bool + err error +} + +func (e *deployError) Error() string { return e.err.Error() } + +func (e *deployError) Unwrap() error { return e.err } + +func withCode(code string, retryable bool, err error) error { + return &deployError{code: code, retryable: retryable, err: err} +} + +// deployPlan is the deployment a dry run reports. +type deployPlan struct { + // Variant is the variant the file is read as: command, dedicated or registered. + Variant string `json:"variant"` + // Checkpoint is the checkpoint a new version is registered from. It is empty when + // the version is already registered. + Checkpoint string `json:"checkpoint,omitempty"` + // ModelVersion is the version served, empty when a new version is registered, or + // when the newest READY version is served, picked when deploying. + ModelVersion string `json:"model_version,omitempty"` + // ServedEntity is the entity the endpoint serves, as the endpoints API takes it. + ServedEntity any `json:"served_entity"` +} + +// deployResult is the JSON payload for `air deploy`. +type deployResult struct { + Status string `json:"status"` + DryRun bool `json:"dry_run,omitempty"` + Plan *deployPlan `json:"plan,omitempty"` + RegisteredModel string `json:"registered_model,omitempty"` + ModelVersion string `json:"model_version,omitempty"` + ModelID string `json:"model_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + // EnvironmentStagingFailed is set when the version's serving environment + // couldn't be prepared ahead of time, so the endpoint can be slower to start. + EnvironmentStagingFailed bool `json:"environment_staging_failed,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + EndpointURL string `json:"endpoint_url,omitempty"` +} + +// deployOptions are the `air deploy` flags. +type deployOptions struct { + file string + overrides []string + dryRun bool + // idempotencyKeySet records an explicit --idempotency-key, so an empty value + // (say, from an unset shell variable) fails instead of registering anew. + idempotencyKeySet bool + idempotencyKey string + retryFailedStaging bool + timeout time.Duration +} + +func newDeployCommand() *cobra.Command { + return newDeployCommandWithPollInterval(registrationPollInterval) +} + +func newDeployCommandWithPollInterval(pollInterval time.Duration) *cobra.Command { + var opts deployOptions + + cmd := &cobra.Command{ + Use: "deploy", + Args: root.NoArgs, + Short: "Register a model in Unity Catalog and serve it on an endpoint", + Long: `Register a model in Unity Catalog and serve it on a Model Serving endpoint. + +The deployment is described by a YAML file (see --file). Its fields say what is +served, and which variant the file is read as: + checkpoint and command a checkpoint in a volume, served by a command you give + checkpoint a checkpoint served with dedicated inference + neither a model version already registered in Unity Catalog + +For a checkpoint, air deploy registers a new version, waits until it is ready, +then creates or updates the endpoint to serve it and waits until the endpoint is +ready. For a registered version, it only creates or updates the endpoint. + +Registration is idempotent: run air deploy again with the idempotency key it +prints to continue an interrupted deployment. To look up a YAML field, pass +its path to -h as a separate argument, and see one YAML example per variant +with -h config: + + databricks air deploy -h config all fields, with examples + databricks air deploy -h config.compute one section + databricks air deploy -h config.checkpoint one field`, + Example: ` # Register a checkpoint and serve it + databricks air deploy -f deploy.yaml + + # Validate the config, and show the plan, without deploying + databricks air deploy -f deploy.yaml --dry-run + + # Continue a deployment that was interrupted + databricks air deploy -f deploy.yaml --idempotency-key 3f6c0b52-4c7e-4d0b-9b8e-0f7ad6a4c2d1`, + // Hidden until the registration API is available in every workspace. + Hidden: true, + } + + // cobra passes -h's positional args to the help func before Args/required-flag + // validation, so a config path documents a field without needing -f. + cmd.SetHelpFunc(func(c *cobra.Command, args []string) { + fields := c.Flags().Args() + if len(fields) == 0 { + // Parent() is nil for a detached command (unit tests). + if parent := c.Parent(); parent != nil { + parent.HelpFunc()(c, args) + return + } + _ = c.Usage() + return + } + field, err := resolveSchemaField(deploySchema(), fields[0]) + if err != nil { + c.PrintErrln("Error:", err) + return + } + renderConfigField(c.OutOrStdout(), deployCommandPath, field, nil) + }) + + // List flags in declaration order (most used first) rather than alphabetically. + cmd.Flags().SortFlags = false + cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to the deployment YAML config") + cmd.Flags().StringArrayVar(&opts.overrides, "override", nil, "Override a YAML field, e.g. compute.num_accelerators=4 (repeatable)") + cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Validate the config, and show the plan, without deploying") + cmd.Flags().StringVar(&opts.idempotencyKey, "idempotency-key", "", "Continue the registration that used this key, instead of registering another version") + cmd.Flags().BoolVar(&opts.retryFailedStaging, "retry-failed-staging", false, "Copy a checkpoint's weights again after copying them failed") + cmd.Flags().DurationVar(&opts.timeout, "timeout", defaultDeployTimeout, "How long to wait for the version and the endpoint to be ready") + _ = cmd.MarkFlagRequired("file") + + // --dry-run never authenticates, so it reports config mistakes without + // credentials. + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + if opts.dryRun { + return nil + } + return root.MustWorkspaceClient(cmd, args) + } + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + opts.idempotencyKeySet = cmd.Flags().Changed("idempotency-key") + err := runDeploy(cmd, opts, pollInterval) + if err == nil { + return nil + } + code, retryable := deployErrorCode(err) + kind := "PERMANENT" + if retryable { + kind = "TRANSIENT" + } + return renderError(cmd.Context(), cmd, code, kind, retryable, err) + } + return cmd +} + +func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duration) error { + ctx := cmd.Context() + cfg, err := loadDeployConfig(ctx, opts.file, opts.overrides) + if err != nil { + return withCode(codeInvalidArgs, false, err) + } + if err := validateDeployFlags(cfg, opts); err != nil { + return withCode(codeInvalidArgs, false, err) + } + + jsonOut := root.OutputType(cmd) == flags.OutputJSON + if opts.dryRun { + return renderDeployDryRun(ctx, cfg, jsonOut) + } + + w := cmdctx.WorkspaceClient(ctx) + ctx, cancel := context.WithTimeout(ctx, opts.timeout) + defer cancel() + ctx, stop := notifyInterrupt(ctx) + defer stop() + + result := deployResult{RegisteredModel: cfg.RegisteredModel, Endpoint: cfg.Endpoint} + if cfg.registers() { + // Fail on an endpoint air deploy won't change before copying any weights. + if err := checkEndpoint(ctx, w, cfg.Endpoint, cfg.RegisteredModel, cfg.variant() == variantDedicated); err != nil { + if stopped := deployStopped(ctx, err); stopped != nil { + return fmt.Errorf("%w while checking endpoint %s, before registering anything", stopped, cfg.Endpoint) + } + return err + } + result.IdempotencyKey = effectiveIdempotencyToken(opts.idempotencyKey, cfg.IdempotencyToken) + resp, err := registerDeployment(ctx, w, cfg, opts, result.IdempotencyKey, pollInterval, !jsonOut) + if err != nil { + return err + } + result.ModelVersion, result.ModelID = resp.ModelVersion, resp.ModelID + result.EnvironmentStagingFailed = resp.environmentState() == stagingStateFailed + if !jsonOut { + cmdio.LogString(ctx, fmt.Sprintf("Version %s of %s is ready.", stripControl(resp.ModelVersion), cfg.RegisteredModel)) + if result.EnvironmentStagingFailed { + cmdio.LogString(ctx, "Warning: the version's serving environment couldn't be prepared ahead of time, so the endpoint can be slower to start, or fail to start.") + } + } + } else { + version, err := resolveModelVersion(ctx, w, cfg) + if err != nil { + return err + } + result.ModelVersion = version + if !jsonOut && cfg.ModelVersion == nil { + cmdio.LogString(ctx, fmt.Sprintf("Serving version %s, the newest READY version of %s.", version, cfg.RegisteredModel)) + } + } + + entity, err := cfg.servedEntity(result.ModelVersion) + if err != nil { + return withCode(codeInvalidArgs, false, err) + } + _, err = serveVersion(ctx, w, cfg.Endpoint, entity, !jsonOut) + if err != nil { + resume := deployResumeCommand(w.Config.Profile, opts, result.IdempotencyKey) + if stopped := deployStopped(ctx, err); stopped != nil { + return fmt.Errorf("%w while waiting for endpoint %s, which keeps updating. To continue, run:\n %s", stopped, cfg.Endpoint, resume) + } + if errors.Is(err, errEndpointNotReady) { + return err + } + return fmt.Errorf("serving version %s of %s on endpoint %s failed: %w\n\nTo try again without registering another version, run:\n %s", result.ModelVersion, cfg.RegisteredModel, cfg.Endpoint, err, resume) + } + result.Status = "READY" + result.EndpointURL = endpointQueryURL(w.Config.Host, cfg.Endpoint, cfg.Task) + if jsonOut { + return renderEnvelope(ctx, result) + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Endpoint %s serves version %s of %s.\n", cfg.Endpoint, result.ModelVersion, cfg.RegisteredModel) + fmt.Fprintln(out, "Query it at: "+result.EndpointURL) + return nil +} + +// plan is the deployment a dry run reports. A dry run makes no request, so the version +// a new registration produces isn't known, and a registered model without +// model_version serves its newest READY version, picked when deploying. +func (c *deployConfig) plan() (deployPlan, error) { + p := deployPlan{Variant: c.variant()} + if c.registers() { + p.Checkpoint = *c.Checkpoint + } + if c.ModelVersion != nil { + p.ModelVersion = *c.ModelVersion + } + entity, err := c.servedEntity(p.ModelVersion) + if err != nil { + return deployPlan{}, err + } + if entity.Dedicated != nil { + p.ServedEntity = entity.dedicatedJSON() + } else { + p.ServedEntity = entity.servingInput() + } + return p, nil +} + +// renderDeployDryRun reports the plan of a valid config, without deploying. +func renderDeployDryRun(ctx context.Context, cfg *deployConfig, jsonOut bool) error { + plan, err := cfg.plan() + if err != nil { + return withCode(codeInvalidArgs, false, err) + } + if jsonOut { + return renderEnvelope(ctx, deployResult{Status: "DRY_RUN_OK", DryRun: true, Plan: &plan}) + } + cmdio.LogString(ctx, fmt.Sprintf("Dry run: %s.", cfg.readAs())) + switch { + case plan.Checkpoint != "": + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would register a new version of %s from %s, and serve it.", cfg.RegisteredModel, plan.Checkpoint)) + case plan.ModelVersion != "": + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve version %s of %s.", plan.ModelVersion, cfg.RegisteredModel)) + default: + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve the newest READY version of %s, picked when deploying.", cfg.RegisteredModel)) + } + served, err := json.MarshalIndent(plan.ServedEntity, " ", " ") + if err != nil { + return withCode(codeInternalError, false, err) + } + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve it on endpoint %s as:\n %s", cfg.Endpoint, served)) + cmdio.LogString(ctx, fmt.Sprintf("Dry run: configuration for endpoint %q is valid; not deploying.", cfg.Endpoint)) + return nil +} + +// resolveModelVersion returns the version of a registered model to serve: model_version +// when the config sets it, else the highest-numbered READY version of the model. +func resolveModelVersion(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig) (string, error) { + if cfg.ModelVersion != nil { + return *cfg.ModelVersion, nil + } + versions, err := w.ModelVersions.ListAll(ctx, catalog.ListModelVersionsRequest{FullName: cfg.RegisteredModel}) + if err != nil { + return "", fmt.Errorf("listing the versions of %s: %w", cfg.RegisteredModel, err) + } + newest := 0 + for _, v := range versions { + if v.Status == catalog.ModelVersionInfoStatusReady && v.Version > newest { + newest = v.Version + } + } + if newest == 0 { + return "", withCode(codeInvalidArgs, false, fmt.Errorf("%s has no READY version to serve; register and finalize a version, or set model_version", cfg.RegisteredModel)) + } + return strconv.Itoa(newest), nil +} + +// registerDeployment registers the version and makes it ready to serve, showing +// progress on a spinner when show is true. Its errors say how to continue. +func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { + // Printed to stderr in every output mode, before the first request, so a run + // that is killed can still be continued with this key. + cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) + req := cfg.registrationRequest(key) + req.RetryFailedStaging = opts.retryFailedStaging + + progress := func(*registerModelPackageResponse) {} + if show { + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + progress = func(resp *registerModelPackageResponse) { + sp.Update(registrationProgress(cfg, resp)) + } + } + resp, err := registerAndFinalize(ctx, w, req, pollInterval, progress) + if err == nil { + return resp, nil + } + resume := deployResumeCommand(w.Config.Profile, opts, key) + if errors.Is(err, errStagingFailed) { + return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying the weights in %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", *cfg.Checkpoint, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) + } + if stopped := deployStopped(ctx, err); stopped != nil { + continuing := "" + switch { + case resp == nil: + case resp.weightsState() == stagingStateSucceeded: + continuing = " The weights are copied, so continuing finalizes the version." + case resp.weightsState() != "": + continuing = " Copying the weights continues meanwhile." + } + return nil, fmt.Errorf("%w while registering a version of %s.%s To continue, run:\n %s", stopped, cfg.RegisteredModel, continuing, resume) + } + if _, ok := errors.AsType[*finalizeError](err); ok { + return nil, fmt.Errorf("finalizing version %s of %s failed: %w\n\nTo try again, run:\n %s", stripControl(resp.ModelVersion), cfg.RegisteredModel, err, resume) + } + failed := fmt.Errorf("registering a version of %s with idempotency key %s failed: %w", cfg.RegisteredModel, key, err) + // A retryable failure is retried with the same key, so no other version is + // registered. + if _, retryable := deployErrorCode(err); retryable { + return nil, fmt.Errorf("%w\n\nTo try again, run:\n %s", failed, resume) + } + return nil, failed +} + +// deployStopped returns why air deploy stopped waiting, or nil if it didn't. The +// SDK's endpoint poller reports a canceled context as its own timeout, so the +// context decides between Ctrl-C and --timeout. +func deployStopped(ctx context.Context, err error) error { + switch { + case errors.Is(ctx.Err(), context.Canceled): + return errDeployInterrupted + case errors.Is(ctx.Err(), context.DeadlineExceeded): + return errDeployTimedOut + } + if _, ok := errors.AsType[*retries.ErrTimedOut](err); ok { + return errDeployTimedOut + } + return nil +} + +// deployErrorCode classifies a failed deployment for the JSON error envelope. +// Failures that aren't the user's input or the server's answer, such as a lost +// connection or a failed endpoint update, are worth retrying with the printed +// command. +func deployErrorCode(err error) (code string, retryable bool) { + switch { + case errors.Is(err, errDeployInterrupted): + return codeInterrupted, true + case errors.Is(err, errDeployTimedOut): + return codeTimeout, true + } + if coded, ok := errors.AsType[*deployError](err); ok { + return coded.code, coded.retryable + } + if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { + retryable = apiErr.StatusCode == http.StatusTooManyRequests || apiErr.StatusCode >= http.StatusInternalServerError + if !apiErrorCodeRe().MatchString(apiErr.ErrorCode) { + return codeInternalError, retryable + } + return apiErr.ErrorCode, retryable + } + return codeInternalError, true +} + +// validateDeployFlags rejects flags the config's variant can't use. +func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { + if opts.timeout <= 0 { + return fmt.Errorf("--timeout must be positive, got %s", opts.timeout) + } + if opts.idempotencyKeySet { + if !cfg.registers() { + return fmt.Errorf("--idempotency-key applies only when air deploy registers a version; %s", cfg.readAs()) + } + if err := validateDeployIdempotencyKey(opts.idempotencyKey, "--idempotency-key"); err != nil { + return err + } + } + if opts.retryFailedStaging && !cfg.registers() { + return fmt.Errorf("--retry-failed-staging applies only to a checkpoint; %s", cfg.readAs()) + } + return nil +} + +// registrationProgress describes a registration response on the spinner. +func registrationProgress(cfg *deployConfig, resp *registerModelPackageResponse) string { + state := resp.weightsState() + switch state { + case "": + return fmt.Sprintf("Registering version %s of %s…", stripControl(resp.ModelVersion), cfg.RegisteredModel) + case stagingStateSucceeded: + return fmt.Sprintf("Finalizing version %s of %s…", stripControl(resp.ModelVersion), cfg.RegisteredModel) + } + return fmt.Sprintf("Copying the weights into version %s of %s (%s)…", stripControl(resp.ModelVersion), cfg.RegisteredModel, stripControl(state)) +} + +// serveVersion serves entity on the endpoint, showing the endpoint's state on a +// spinner when show is true. +func serveVersion(ctx context.Context, w *databricks.WorkspaceClient, name string, entity servedEntity, show bool) (*servingEndpoint, error) { + if !show { + return serveOnEndpoint(ctx, w, name, entity, nil) + } + cmdio.LogString(ctx, fmt.Sprintf("Serving version %s of %s on endpoint %s", entity.EntityVersion, entity.EntityName, name)) + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + return serveOnEndpoint(ctx, w, name, entity, func(endpoint *servingEndpoint) { + if endpoint.State != nil { + sp.Update(fmt.Sprintf("Waiting for endpoint %s (%s, %s)…", name, endpoint.State.Ready, endpoint.State.ConfigUpdate)) + } + }) +} + +// deployResumeCommand is the command that continues this deployment: the same +// config (by absolute path, so it runs from any directory), overrides and +// profile, with the registration's key when it has one. The overrides are part +// of the request the key was first used with, so they must be repeated. +func deployResumeCommand(profile string, opts deployOptions, key string) string { + file := opts.file + if abs, err := filepath.Abs(file); err == nil { + file = abs + } + args := []string{deployCommandPath, "-f", shellquote.BashArg(filepath.ToSlash(file))} + for _, override := range opts.overrides { + args = append(args, "--override", shellquote.BashArg(override)) + } + if key != "" { + args = append(args, "--idempotency-key", shellquote.BashArg(key)) + } + if profile != "" { + args = append(args, "-p", shellquote.BashArg(profile)) + } + return strings.Join(args, " ") +} + +// endpointQueryURL is the REST URL that queries the endpoint: an audio model's +// audio route, which takes a multipart upload, or invocations for any other model. +func endpointQueryURL(host, name string, task *string) string { + base := strings.TrimSuffix(host, "/") + "/serving-endpoints/" + name + if task != nil { + switch *task { + case taskAudioTranscriptions: + return base + "/audio/transcriptions" + case taskAudioTranslations: + return base + "/audio/translations" + } + } + return base + "/invocations" +} diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go new file mode 100644 index 00000000000..3c2af38a24a --- /dev/null +++ b/cmd/air/deploy_test.go @@ -0,0 +1,580 @@ +package aircmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// registeredNoVersionConfig serves the newest READY version of a registered model. +const registeredNoVersionConfig = `registered_model_name: main.models.qwen +endpoint_name: qwen-chat +` + +// modelVersionsJSON lists the versions of the test model: 4 is the newest READY one. +const modelVersionsJSON = `{"model_versions": [ + {"version": 2, "status": "READY"}, + {"version": 5, "status": "PENDING_REGISTRATION"}, + {"version": 4, "status": "READY"} +]}` + +// withDeadline returns the test's context with a deadline, which air deploy +// always sets from --timeout. +func withDeadline(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(fastRetries(t.Context()), time.Minute) + t.Cleanup(cancel) + return ctx +} + +// deployServer serves registerModelPackage with responses, the staging status +// and finalizeModelPackage with their defaults, and the serving endpoints API +// from endpoints. +func deployServer(t *testing.T, endpoints *fakeEndpoints, responses ...string) (*httptest.Server, *fakeModelServing) { + t.Helper() + fake := newFakeModelServing(t, responses...) + fake.endpoints = endpoints.ServeHTTP + return fake.start(), fake +} + +// deployServerWithVersions is deployServer, also serving the model's versions, as +// the Unity Catalog API lists them, from versionsJSON. +func deployServerWithVersions(t *testing.T, endpoints *fakeEndpoints, versionsJSON string) (*httptest.Server, *fakeModelServing) { + t.Helper() + fake := newFakeModelServing(t) + fake.endpoints = func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/versions") { + _, _ = w.Write([]byte(versionsJSON)) + return + } + endpoints.ServeHTTP(w, r) + } + return fake.start(), fake +} + +// runDeployCmd runs `air deploy` on config with the given flags. An empty srvURL +// runs it without a workspace client, as --dry-run does. +func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, config string, flagValues ...string) error { + t.Helper() + cmd := withOutput(newDeployCommandWithPollInterval(time.Millisecond), out) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", config))) + for i := 0; i < len(flagValues); i += 2 { + require.NoError(t, cmd.Flags().Set(flagValues[i], flagValues[i+1])) + } + + // JSON tests parse buf, so stderr's progress lines go elsewhere. + var stderr io.Writer = buf + if out == flags.OutputJSON { + stderr = io.Discard + } + ctx := cmdio.InContext(fastRetries(t.Context()), cmdio.NewIO(t.Context(), out, nil, buf, stderr, "", "")) + if srvURL != "" { + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + } + cmd.SetContext(ctx) + cmd.SetOut(buf) + return cmd.RunE(cmd, nil) +} + +func TestDeployCommandJSON(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints, stagingInProgress) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.NoError(t, err) + + var got struct { + Data deployResult `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, deployResult{ + Status: "READY", + RegisteredModel: "main.models.qwen", + ModelVersion: "3", + ModelID: "m-1", + IdempotencyKey: "key-1", + Endpoint: "qwen-chat", + EndpointURL: srv.URL + "/serving-endpoints/qwen-chat/invocations", + }, got.Data) + + bodies := fake.requests() + require.Len(t, bodies, 1) + assert.Equal(t, "key-1", bodies[0].IdempotencyKey) + assert.Equal(t, "/Volumes/main/models/weights/qwen3_8b", bodies[0].Checkpoint.VolumePath) + assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "main.models.qwen", ModelVersion: "3"}}, fake.finalizeRequests) + assert.Equal(t, []string{"create"}, endpoints.writes) + served := endpoints.endpoint.Config.ServedEntities[0] + assert.Equal(t, "3", served.EntityVersion) + assert.Equal(t, "GPU_MEDIUM", served.WorkloadType) + assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, served.EnvironmentVars) + // scale_to_zero_enabled: false is sent, not left to the server's default. + require.Len(t, endpoints.rawWrites, 1) + assert.Contains(t, endpoints.rawWrites[0], `"scale_to_zero_enabled":false`) +} + +func TestDeployCommandText(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) + out := buf.String() + assert.Contains(t, out, "Registering a version of main.models.qwen with idempotency key ") + assert.Contains(t, out, "Version 3 of main.models.qwen is ready.") + assert.Contains(t, out, "Serving version 3 of main.models.qwen on endpoint qwen-chat") + assert.Contains(t, out, "Endpoint qwen-chat serves version 3 of main.models.qwen.\n") + assert.Contains(t, out, "Query it at: "+srv.URL+"/serving-endpoints/qwen-chat/invocations\n") + assert.NotContains(t, out, "Warning") +} + +func TestDeployDedicatedRegistersAndCreatesTheDedicatedEndpoint(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, dedicatedDeployConfig, "idempotency-key", "key-1")) + + bodies := fake.requests() + require.Len(t, bodies, 1) + assert.Nil(t, bodies[0].Checkpoint) + assert.Equal(t, &dedicatedCheckpoint{VolumePath: "/Volumes/main/models/weights/qwen3_8b"}, bodies[0].DedicatedCheckpoint) + assert.Equal(t, []string{"create dedicated"}, endpoints.writes) + require.Len(t, endpoints.rawWrites, 1) + assert.Contains(t, endpoints.rawWrites[0], `"compute_profile":"COMPUTE_PROFILE_LOW_LATENCY"`) + assert.NotContains(t, endpoints.rawWrites[0], "scale_to_zero_enabled") + assert.Equal(t, "3", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) +} + +func TestDeployWarnsWhenTheEnvironmentFailedToStage(t *testing.T) { + environmentFailed := strings.Replace(stagingReady, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) + srv, _ := deployServer(t, newFakeEndpoints(t, nil), environmentFailed) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) + assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") +} + +func TestDeployUsesTheConfigTokenThenANewKey(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig+"idempotency_token: from-config\n")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig+"idempotency_token: from-config\n", "idempotency-key", "from-flag")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig)) + + bodies := fake.requests() + require.Len(t, bodies, 3) + assert.Equal(t, "from-config", bodies[0].IdempotencyKey) + assert.Equal(t, "from-flag", bodies[1].IdempotencyKey) + assert.Regexp(t, deployIdempotencyKeyRe(), bodies[2].IdempotencyKey) + assert.NotContains(t, []string{"from-config", "from-flag"}, bodies[2].IdempotencyKey) +} + +func TestDeployPrintsTheKeyBeforeRegisteringInJSONMode(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) + + var stdout, stderr bytes.Buffer + cmd := withOutput(newDeployCommandWithPollInterval(time.Millisecond), flags.OutputJSON) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", commandDeployConfig))) + require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + cmd.SetContext(cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))) + require.ErrorIs(t, cmd.RunE(cmd, nil), root.ErrAlreadyPrinted) + assert.Equal(t, "Registering a version of main.models.qwen with idempotency key key-1\n", stderr.String()) + assert.Contains(t, stdout.String(), `"code": "RESOURCE_CONFLICT"`) +} + +func TestDeploySendsRetryFailedStaging(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "retry-failed-staging", "true")) + bodies := fake.requests() + require.Len(t, bodies, 1) + assert.True(t, bodies[0].RetryFailedStaging) +} + +func TestDeployFailedStagingSaysHowToRetry(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingFailed) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "copying the weights in /Volumes/main/models/weights/qwen3_8b into version 3 of main.models.qwen failed") + assert.Contains(t, err.Error(), " --idempotency-key key-1 --retry-failed-staging") +} + +func TestDeployReportsRegistrationErrors(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + assert.ErrorContains(t, err, "registering a version of main.models.qwen with idempotency key key-1 failed: The idempotency key can't be used for this registration") +} + +func TestDeployReportsErrorsAsJSON(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "RESOURCE_CONFLICT", got.Error.Code) + assert.Equal(t, "PERMANENT", got.Error.Kind) + assert.False(t, got.Error.Retryable) + + buf.Reset() + err = runDeployCmd(t, flags.OutputJSON, &buf, "", "registered_model_name: x\n", "dry-run", "true") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "INVALID_ARGS", got.Error.Code) + assert.Contains(t, got.Error.Message, "registered_model_name must be a Unity Catalog model name") +} + +func TestDeployChecksTheEndpointBeforeRegistering(t *testing.T) { + other := testEntity("5") + other.EntityName = "main.models.llama" + srv, fake := deployServer(t, newFakeEndpoints(t, endpointServing(other, readyState)), stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig) + assert.ErrorContains(t, err, "endpoint qwen-chat serves version 5 of main.models.llama") + assert.Empty(t, fake.requests()) +} + +func TestDeployReportsAnEndpointThatIsNotReady(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + endpoints.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating} + srv, _ := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + // Running air deploy again doesn't make the endpoint ready, so nothing says to. + assert.Equal(t, "ENDPOINT_NOT_READY", got.Error.Code) + assert.False(t, got.Error.Retryable) + assert.NotContains(t, got.Error.Message, "run:") +} + +func TestDeployTimesOutWhileCopyingTheWeights(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) + fake.statuses = []string{statusInProgress} + + // Polls are spaced as the SDK's rate limiter allows, so the deadline, not the + // limiter, stops the wait. + var buf bytes.Buffer + cmd := withOutput(newDeployCommandWithPollInterval(100*time.Millisecond), flags.OutputJSON) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", commandDeployConfig))) + require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) + require.NoError(t, cmd.Flags().Set("timeout", "500ms")) + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &buf, io.Discard, "", "")) + cmd.SetContext(cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))) + require.ErrorIs(t, cmd.RunE(cmd, nil), root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "TIMEOUT", got.Error.Code) + assert.True(t, got.Error.Retryable) + assert.Contains(t, got.Error.Message, "air deploy timed out while registering a version of main.models.qwen. Copying the weights continues") + assert.Contains(t, got.Error.Message, " --idempotency-key key-1") + assert.Len(t, fake.requests(), 1) + assert.Empty(t, fake.finalizeRequests) +} + +func TestDeployFinalizeFailureSaysHowToRetry(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingSucceeded) + fake.finalizations = []string{notFinalizableResponse} + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "finalizing version 3 of main.models.qwen failed: Version 3 of model main.models.qwen can't be finalized yet") + assert.Contains(t, err.Error(), "To try again, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") +} + +func TestDeployWarnsWhenTheFinalizedEnvironmentFailedToStage(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) + fake.finalizations = []string{strings.Replace(finalizedResponse, `"environment_artifact": {"state": "IN_PROGRESS"}`, `"environment_artifact": {"state": "FAILED"}`, 1)} + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) + assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") +} + +func TestDeployRetryableRegistrationFailureSaysHowToRetry(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), serverErrorResponse) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed: Something went wrong.\n\nTo try again, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") + assert.Len(t, fake.requests(), retry.MaxRetries+1) +} + +func TestDeployEndpointFailureSaysHowToRetry(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + endpoints.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} + srv, _ := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "serving version 3 of main.models.qwen on endpoint qwen-chat failed: ") + assert.Contains(t, err.Error(), "To try again without registering another version, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") +} + +func TestDeployRegisteredVersionSkipsRegistration(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredDeployConfig)) + assert.Empty(t, fake.requests()) + assert.Equal(t, "7", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) + assert.Equal(t, "CPU", endpoints.endpoint.Config.ServedEntities[0].WorkloadType) + assert.NotContains(t, buf.String(), "idempotency_key") +} + +func TestDeployRegisteredPicksTheNewestReadyVersion(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServerWithVersions(t, endpoints, modelVersionsJSON) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, registeredNoVersionConfig)) + assert.Empty(t, fake.requests()) + assert.Equal(t, "4", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) + assert.Contains(t, buf.String(), "Serving version 4, the newest READY version of main.models.qwen.") +} + +func TestDeployRegisteredGivenVersionWinsOverTheNewestReady(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, _ := deployServerWithVersions(t, endpoints, modelVersionsJSON) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredDeployConfig)) + assert.Equal(t, "7", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) +} + +func TestDeployRegisteredWithoutAReadyVersionFails(t *testing.T) { + srv, _ := deployServerWithVersions(t, newFakeEndpoints(t, nil), `{"model_versions": [{"version": 1, "status": "PENDING_REGISTRATION"}]}`) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredNoVersionConfig) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "INVALID_ARGS", got.Error.Code) + assert.Contains(t, got.Error.Message, "main.models.qwen has no READY version to serve; register and finalize a version, or set model_version") +} + +func TestDeployDryRunNeedsNoWorkspace(t *testing.T) { + cmd := newDeployCommand() + require.NoError(t, cmd.Flags().Set("dry-run", "true")) + assert.NoError(t, cmd.PreRunE(cmd, nil)) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", commandDeployConfig, "dry-run", "true")) + out := buf.String() + assert.Contains(t, out, "Dry run: the file is read as a checkpoint served by command, because it sets checkpoint and command.") + assert.Contains(t, out, "Dry run: would register a new version of main.models.qwen from /Volumes/main/models/weights/qwen3_8b, and serve it.") + assert.Contains(t, out, `"workload_type": "GPU_MEDIUM"`) + assert.Contains(t, out, `Dry run: configuration for endpoint "qwen-chat" is valid; not deploying.`) + + buf.Reset() + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", commandDeployConfig, "dry-run", "true")) + var got struct { + Data deployResult `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "DRY_RUN_OK", got.Data.Status) + assert.True(t, got.Data.DryRun) + require.NotNil(t, got.Data.Plan) + assert.Equal(t, "command", got.Data.Plan.Variant) + assert.Equal(t, "/Volumes/main/models/weights/qwen3_8b", got.Data.Plan.Checkpoint) + served, ok := got.Data.Plan.ServedEntity.(map[string]any) + require.True(t, ok) + assert.Equal(t, "GPU_MEDIUM", served["workload_type"]) + assert.Equal(t, false, served["scale_to_zero_enabled"]) +} + +func TestDeployDryRunShowsEachVariant(t *testing.T) { + tests := []struct { + name string + config string + want []string + }{ + { + "dedicated", + dedicatedDeployConfig, + []string{ + "Dry run: the file is read as a dedicated checkpoint, because it sets checkpoint but not command.", + "would register a new version of main.models.qwen from /Volumes/main/models/weights/qwen3_8b, and serve it.", + `"compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"`, + `"provisioned_model_units": 600`, + }, + }, + { + "registered", + registeredDeployConfig, + []string{ + "Dry run: the file is read as a registered model, because it sets no checkpoint.", + "would serve version 7 of main.models.qwen.", + `"workload_type": "CPU"`, + }, + }, + { + "registered without a version", + registeredNoVersionConfig, + []string{"would serve the newest READY version of main.models.qwen, picked when deploying."}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", tt.config, "dry-run", "true")) + for _, want := range tt.want { + assert.Contains(t, buf.String(), want) + } + }) + } +} + +func TestDeployFlagValidation(t *testing.T) { + tests := []struct { + name string + config string + flagValues []string + wantErr string + }{ + {"key for a registered version", registeredDeployConfig, []string{"idempotency-key", "k"}, "--idempotency-key applies only when air deploy registers a version; the file is read as a registered model, because it sets no checkpoint"}, + {"bad key", commandDeployConfig, []string{"idempotency-key", "a/b"}, "--idempotency-key must be 1 to 128 letters"}, + {"retry for a registered version", registeredDeployConfig, []string{"retry-failed-staging", "true"}, "--retry-failed-staging applies only to a checkpoint; the file is read as a registered model"}, + {"zero timeout", commandDeployConfig, []string{"timeout", "0s"}, "--timeout must be positive"}, + {"empty key", commandDeployConfig, []string{"idempotency-key", ""}, `--idempotency-key must be 1 to 128 letters, digits, '.', '_' or '-', got ""`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, "", tt.config, append(tt.flagValues, "dry-run", "true")...) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployResumeCommand(t *testing.T) { + abs, err := filepath.Abs("my deploy.yaml") + require.NoError(t, err) + opts := deployOptions{file: "my deploy.yaml", overrides: []string{"compute.num_accelerators=4", "task=llm/v1/chat"}} + assert.Equal(t, + "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs))+" --override 'compute.num_accelerators=4' --override 'task=llm/v1/chat' --idempotency-key k-1 -p dev", + deployResumeCommand("dev", opts, "k-1")) + + abs, err = filepath.Abs("d.yaml") + require.NoError(t, err) + assert.Equal(t, "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs)), deployResumeCommand("", deployOptions{file: "d.yaml"}, "")) +} + +func TestDeployStopped(t *testing.T) { + assert.NoError(t, deployStopped(t.Context(), errors.New("boom"))) + + canceled, cancel := context.WithCancel(t.Context()) + cancel() + // The SDK's endpoint poller reports Ctrl-C as its own timeout. + assert.ErrorIs(t, deployStopped(canceled, &retries.ErrTimedOut{}), errDeployInterrupted) + assert.ErrorIs(t, deployStopped(canceled, context.Canceled), errDeployInterrupted) + + expired, cancelExpired := context.WithTimeout(t.Context(), -time.Second) + defer cancelExpired() + assert.ErrorIs(t, deployStopped(expired, context.DeadlineExceeded), errDeployTimedOut) + assert.ErrorIs(t, deployStopped(t.Context(), &retries.ErrTimedOut{}), errDeployTimedOut) +} + +func TestDeployErrorCode(t *testing.T) { + for _, tt := range []struct { + name string + err error + code string + retryable bool + }{ + {"interrupted", fmt.Errorf("x: %w", errDeployInterrupted), "INTERRUPTED", true}, + {"timed out", fmt.Errorf("x: %w", errDeployTimedOut), "TIMEOUT", true}, + {"invalid input", withCode(codeInvalidArgs, false, errors.New("registered_model_name must be a Unity Catalog model name")), "INVALID_ARGS", false}, + {"staging failed", withCode(codeStagingFailed, true, errors.New("copying failed")), "STAGING_FAILED", true}, + {"server unavailable", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "TEMPORARILY_UNAVAILABLE", StatusCode: 503}), "TEMPORARILY_UNAVAILABLE", true}, + {"bad request", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "INVALID_PARAMETER_VALUE", StatusCode: 400}), "INVALID_PARAMETER_VALUE", false}, + {"5xx without a body", &apierr.APIError{StatusCode: 502}, "INTERNAL_ERROR", true}, + {"error body without a code", &apierr.APIError{ErrorCode: "", StatusCode: 400}, "INTERNAL_ERROR", false}, + {"lost connection", errors.New("connection reset by peer"), "INTERNAL_ERROR", true}, + {"endpoint not ready", withCode(codeEndpointNotReady, false, errEndpointNotReady), "ENDPOINT_NOT_READY", false}, + } { + code, retryable := deployErrorCode(tt.err) + assert.Equal(t, tt.code, code, tt.name) + assert.Equal(t, tt.retryable, retryable, tt.name) + } +} + +func TestDeployHelpDocumentsAField(t *testing.T) { + var out strings.Builder + cmd := newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config.checkpoint"}) + require.NoError(t, cmd.Execute()) + assert.Contains(t, out.String(), "config.checkpoint\n Unity Catalog volume directory") + assert.Contains(t, out.String(), "Required: no") + + out.Reset() + cmd = newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config.compute.compute_profile"}) + require.NoError(t, cmd.Execute()) + assert.Contains(t, out.String(), "Required: when checkpoint is set without command") + + out.Reset() + cmd = newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config"}) + require.NoError(t, cmd.Execute()) + // The coding agent skill writes run YAML, so the deploy help leaves it out. + assert.NotContains(t, out.String(), "databricks-ai-runtime") +} + +func TestEndpointQueryURL(t *testing.T) { + task := func(s string) *string { return &s } + tests := []struct { + task *string + want string + }{ + {task("llm/v1/chat"), "https://h/serving-endpoints/ep/invocations"}, + {task(taskAudioTranscriptions), "https://h/serving-endpoints/ep/audio/transcriptions"}, + {task(taskAudioTranslations), "https://h/serving-endpoints/ep/audio/translations"}, + {nil, "https://h/serving-endpoints/ep/invocations"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, endpointQueryURL("https://h/", "ep", tt.task)) + } +} diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go new file mode 100644 index 00000000000..29449b4a688 --- /dev/null +++ b/cmd/air/deployconfig.go @@ -0,0 +1,537 @@ +package aircmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "maps" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "unicode/utf16" + + "go.yaml.in/yaml/v3" +) + +// This file defines the `air deploy` YAML schema and its validation. The fields a +// file sets select its variant, and each variant reads its own fields. Fields shared +// with `air run` (environment, env_variables, secrets, idempotency_token) keep their +// meaning; the server validates what needs the workspace, such as task names, +// runtimes and dependencies. + +// Variants of a deployment, named after what serves the checkpoint. +const ( + // variantCommand: a checkpoint served by the command the file sets. + variantCommand = "command" + // variantDedicated: a checkpoint Databricks serves with dedicated inference. + variantDedicated = "dedicated" + // variantRegistered: a model version already registered in Unity Catalog, which + // air deploy serves without registering another. + variantRegistered = "registered" +) + +// variantDescriptions and variantReasons say how a file was read, in errors about +// its fields: "the file is read as , because ". +var ( + variantDescriptions = map[string]string{ + variantCommand: "a checkpoint served by command", + variantDedicated: "a dedicated checkpoint", + variantRegistered: "a registered model", + } + variantReasons = map[string]string{ + variantCommand: "it sets checkpoint and command", + variantDedicated: "it sets checkpoint but not command", + variantRegistered: "it sets no checkpoint", + } +) + +// Audio tasks: their requests are multipart uploads of a recording, sent to the +// endpoint's audio route rather than to invocations. +const ( + taskAudioTranscriptions = "llm/v1/audio/transcriptions" + taskAudioTranslations = "llm/v1/audio/translations" +) + +// deployCommandPath is the command whose -h documents deployConfig. +const deployCommandPath = "databricks air deploy" + +// Serving compute a command or registered model gets when compute is omitted: one +// A10 for a command, CPU for a registered model. +const ( + defaultCommandWorkloadType = "GPU_MEDIUM" + defaultRegisteredWorkloadType = "CPU" + defaultWorkloadSize = "Small" +) + +// Limits the server enforces on a registration, checked here to fail early. +const ( + maxServingCommandChars = 32 * 1024 + maxVolumePathChars = 1024 + maxDependencies = 1000 +) + +// registeredModelRe matches .. with the characters the +// server accepts in each part. +var registeredModelRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile("^[^\\s./`]+\\.[^\\s./`]+\\.[^\\s./`]+$") }) + +// deployIdempotencyKeyRe matches the registration keys the server records. +var deployIdempotencyKeyRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`) }) + +// modelVersionRe matches a Unity Catalog model version number. +var modelVersionRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[1-9][0-9]*$`) }) + +// endpointNameRe matches a serving endpoint name: letters, digits, '-' and '_'. +// The name is part of the API's URL paths, so nothing else may reach a request. +var endpointNameRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_-]+$`) }) + +// workloadSizes are the concurrency tiers a served entity scales to. +var workloadSizes = []string{"Small", "Medium", "Large"} + +// servingShape is serving compute the server offers: an accelerator type, the number +// of accelerators, and the workload type that serves them. +type servingShape struct { + accelerator gpuType + count int + workloadType string +} + +// servingShapes lists the serving compute a command or registered model can use, in +// the order errors name them. +var servingShapes = []servingShape{ + {gpuType1xA10, 1, "GPU_MEDIUM"}, + {gpuType1xA10, 4, "MULTIGPU_MEDIUM"}, + {gpuType1xA10, 8, "GPU_MEDIUM_8"}, + {gpuType1xH100, 1, "GPU_XLARGE"}, + {gpuType8xH100, 8, "GPU_XLARGE_8"}, + {gpuType1xRTX6000, 1, "GPU_LARGE_RTX"}, +} + +// computeProfiles are the dedicated compute profiles, lowercase as the file names +// them. The API takes each as COMPUTE_PROFILE_ followed by its uppercase form. +var computeProfiles = []string{"low_latency", "fast", "throughput", "minimal_cost"} + +// deployConfig is the `air deploy` YAML schema: a model to serve on an endpoint, +// and the checkpoint to register as a version of it, if any. +type deployConfig struct { + RegisteredModel string `yaml:"registered_model_name" help:"Unity Catalog model, as ... A checkpoint is registered as a new version of it, creating the model if it doesn't exist. Without a checkpoint, the model must already have a version to serve." required:"yes" group:"Model"` + Checkpoint *string `yaml:"checkpoint" help:"Unity Catalog volume directory holding a checkpoint's weights, as /Volumes////. Model Serving copies it, with its subdirectories, into the version air deploy registers. Leave it out to serve a version that is already registered." group:"Model"` + Command *string `yaml:"command" help:"Shell command that starts a model server over the checkpoint's weights on port 8080, at most 32,768 characters. It runs from the directory where the checkpoint appears under its own name. Set it to serve the checkpoint with your own server; without it, Databricks serves the checkpoint with dedicated inference." group:"Model"` + Task *string `yaml:"task" help:"The task Model Serving routes to the command, such as llm/v1/chat, llm/v1/embeddings or llm/v1/audio/transcriptions." required:"when command is set" group:"Model"` + Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve the command on, and the packages to install. The model uses its Python version." required:"when command is set" group:"Model"` + ModelVersion *string `yaml:"model_version" help:"Version of registered_model to serve, e.g. 3. Without it, the newest READY version is served." group:"Model"` + + Endpoint string `yaml:"endpoint_name" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` + Compute *deployComputeConfig `yaml:"compute" help:"Compute to serve on. A command or registered model takes serving compute, and a dedicated checkpoint takes dedicated compute. Without compute, a command is served on one GPU_1xA10 with workload size Small, and a registered model on CPU with workload size Small." required:"when checkpoint is set without command" group:"Serving"` + EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables of the served model, as NAME: value. A name here cannot also appear in secrets. A command or registered model only." group:"Serving"` + Secrets map[string]string `yaml:"secrets" help:"Environment variables of the served model sourced from secrets, as NAME: scope/key. A command or registered model only." group:"Serving"` + + IdempotencyToken *string `yaml:"idempotency_token" help:"Key of the registration: 1 to 128 letters, digits, '.', '_' or '-'. Running air deploy again with the same key and unchanged inputs continues that registration instead of registering another version. Defaults to a new key, which air deploy prints. A checkpoint only." group:"Registration"` + MLflowExperimentPath *string `yaml:"mlflow_experiment_path" help:"Workspace path of the MLflow experiment that records the registered model (default: /Users//air-deployments). A checkpoint only." group:"Registration"` +} + +// deployEnvironmentConfig is the `environment` block of a command. +type deployEnvironmentConfig struct { + Version *stringOrInt `yaml:"version" required:"yes" help:"Serverless environment to serve on: 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment, as for air run. The model uses its Python version. Standard includes CUDA but not PyTorch; the Databricks AI environment adds PyTorch, Transformers and vLLM, and needs GPU compute. A command that doesn't install SGLang runs on the Databricks AI environment either way."` + Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000."` +} + +// deployComputeConfig is the `compute` block. A command or registered model reads +// the serving fields, and a dedicated checkpoint the dedicated fields. +type deployComputeConfig struct { + AcceleratorType *string `yaml:"accelerator_type" help:"Accelerator to serve on, as in air run: GPU_1xA10, GPU_1xH100, GPU_8xH100 or GPU_1xRTX6000. Each takes the num_accelerators listed for it: GPU_1xA10 takes 1, 4 or 8; GPU_1xH100 takes 1; GPU_8xH100 takes 8; GPU_1xRTX6000 takes 1." required:"when compute is set" group:"Serving compute"` + NumAccelerators *int `yaml:"num_accelerators" help:"Number of accelerators, from the pairs listed under accelerator_type." required:"when compute is set" group:"Serving compute"` + WorkloadSize *string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"when compute is set" group:"Serving compute"` + + ComputeProfile *string `yaml:"compute_profile" help:"Dedicated compute profile: low_latency, fast, throughput or minimal_cost, case-insensitive." required:"when checkpoint is set without command" group:"Dedicated compute"` + ProvisionedModelUnits *int `yaml:"provisioned_model_units" help:"Model units to provision, above 0." required:"when checkpoint is set without command" group:"Dedicated compute"` + ReservationTerm *string `yaml:"reservation_term" help:"Commitment term of the provisioned units, such as ONE_MONTH. Not supported yet: dedicated endpoints don't accept a commitment term, so a file that sets it is refused before anything is registered." group:"Dedicated compute"` +} + +// variantField is a field that only some variants use. +type variantField struct { + name string + set bool + users []string + // applies says what the field is for, as the error that rejects it names it. + applies string +} + +// variantFields lists the optional fields with the variants that use them, in the +// order their errors are reported. +func (c *deployConfig) variantFields() []variantField { + var compute deployComputeConfig + if c.Compute != nil { + compute = *c.Compute + } + servingVariants := []string{variantCommand, variantRegistered} + registeringVariants := []string{variantCommand, variantDedicated} + return []variantField{ + {"command", c.Command != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"task", c.Task != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"environment", c.Environment != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"model_version", c.ModelVersion != nil, []string{variantRegistered}, "a registered model"}, + {"env_variables", c.EnvVariables != nil, servingVariants, "a command or registered model"}, + {"secrets", c.Secrets != nil, servingVariants, "a command or registered model"}, + {"idempotency_token", c.IdempotencyToken != nil, registeringVariants, "a checkpoint, which air deploy registers"}, + {"mlflow_experiment_path", c.MLflowExperimentPath != nil, registeringVariants, "a checkpoint, which air deploy registers"}, + {"compute.accelerator_type", compute.AcceleratorType != nil, servingVariants, "a command or registered model"}, + {"compute.num_accelerators", compute.NumAccelerators != nil, servingVariants, "a command or registered model"}, + {"compute.workload_size", compute.WorkloadSize != nil, servingVariants, "a command or registered model"}, + {"compute.compute_profile", compute.ComputeProfile != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + {"compute.provisioned_model_units", compute.ProvisionedModelUnits != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + {"compute.reservation_term", compute.ReservationTerm != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + } +} + +// variant is the deployment the file describes, inferred from the fields it sets: a +// checkpoint with a command is served by that command, a checkpoint without one is +// served with dedicated inference, and no checkpoint means a registered version. +func (c *deployConfig) variant() string { + switch { + case c.Checkpoint == nil: + return variantRegistered + case c.Command != nil: + return variantCommand + default: + return variantDedicated + } +} + +// readAs says how the file was read and why, for errors about its fields. +func (c *deployConfig) readAs() string { + v := c.variant() + return fmt.Sprintf("the file is read as %s, because %s", variantDescriptions[v], variantReasons[v]) +} + +// registers reports whether air deploy registers a new version from the checkpoint. +func (c *deployConfig) registers() bool { + return c.Checkpoint != nil +} + +// validate runs structural validation over the whole config, returning the +// first failure. Fields are checked in declaration order to keep errors stable. +func (c *deployConfig) validate() error { + if !registeredModelRe().MatchString(c.RegisteredModel) { + return fmt.Errorf("registered_model_name must be a Unity Catalog model name as .., got %q", c.RegisteredModel) + } + if err := c.validateSource(); err != nil { + return err + } + if !endpointNameRe().MatchString(c.Endpoint) { + return fmt.Errorf("endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got %q", c.Endpoint) + } + if err := c.validateCompute(); err != nil { + return err + } + if err := validateEnvironmentVariables(c.EnvVariables, c.Secrets); err != nil { + return err + } + if c.IdempotencyToken != nil { + if err := validateDeployIdempotencyKey(*c.IdempotencyToken, "idempotency_token"); err != nil { + return err + } + } + if c.MLflowExperimentPath != nil && !strings.HasPrefix(*c.MLflowExperimentPath, "/") { + return fmt.Errorf("mlflow_experiment_path must be an absolute workspace path such as /Users//air-deployments, got %q", *c.MLflowExperimentPath) + } + return nil +} + +// validateSource checks that the fields the file sets fit its variant, and that the +// variant's own fields are valid. +func (c *deployConfig) validateSource() error { + if err := c.checkVariantFields(); err != nil { + return err + } + if c.Checkpoint != nil { + if err := validateVolumeDirectory("checkpoint", *c.Checkpoint); err != nil { + return err + } + } + switch c.variant() { + case variantCommand: + return c.validateCommandSource() + case variantRegistered: + if c.ModelVersion != nil && !modelVersionRe().MatchString(*c.ModelVersion) { + return fmt.Errorf("model_version must be a model version number such as 3, got %q", *c.ModelVersion) + } + } + return nil +} + +// checkVariantFields rejects a field the file's variant doesn't use, naming the +// field, the variant and why the file is read as that variant. +func (c *deployConfig) checkVariantFields() error { + variant := c.variant() + for _, f := range c.variantFields() { + if f.set && !slices.Contains(f.users, variant) { + return fmt.Errorf("%s can't be set: %s, and %s applies only to %s", f.name, c.readAs(), f.name, f.applies) + } + } + return nil +} + +// validateCommandSource checks the fields only a checkpoint served by command has. +func (c *deployConfig) validateCommandSource() error { + if c.Task == nil { + return errors.New("task is required for a checkpoint served by command: the task Model Serving routes to the command, such as llm/v1/chat") + } + if strings.TrimSpace(*c.Task) == "" { + return errors.New("task cannot be empty") + } + if strings.TrimSpace(*c.Command) == "" { + return errors.New("command cannot be empty") + } + if n := serverLength(*c.Command); n > maxServingCommandChars { + return fmt.Errorf("command must be at most %d characters, got %d", maxServingCommandChars, n) + } + if c.Environment == nil { + return errors.New("environment is required for a checkpoint served by command: the serverless environment and packages to serve it on") + } + return c.Environment.validate() +} + +// validateCompute checks the compute the variant reads. +func (c *deployConfig) validateCompute() error { + if c.variant() == variantDedicated { + _, err := c.resolveDedicatedCompute() + return err + } + _, _, err := c.resolveServingCompute() + return err +} + +// resolveServingCompute returns the workload type and size a command or registered +// model is served on. Without compute, the defaults apply; with it, all three fields +// are required and the accelerator and count must be a pair the server offers. +func (c *deployConfig) resolveServingCompute() (string, string, error) { + if c.Compute == nil { + if c.variant() == variantRegistered { + return defaultRegisteredWorkloadType, defaultWorkloadSize, nil + } + return defaultCommandWorkloadType, defaultWorkloadSize, nil + } + compute := c.Compute + switch { + case compute.AcceleratorType == nil: + return "", "", errors.New("compute.accelerator_type is required when compute is set, e.g. GPU_1xA10") + case compute.NumAccelerators == nil: + return "", "", errors.New("compute.num_accelerators is required when compute is set, e.g. 1") + case compute.WorkloadSize == nil: + return "", "", errors.New("compute.workload_size is required when compute is set: Small, Medium or Large") + } + if !slices.Contains(workloadSizes, *compute.WorkloadSize) { + return "", "", fmt.Errorf("compute.workload_size must be one of %s, got %q", strings.Join(workloadSizes, ", "), *compute.WorkloadSize) + } + accelerator, err := parseGPUType(*compute.AcceleratorType) + if err != nil { + return "", "", fmt.Errorf("compute.accelerator_type: %w; the supported pairs are %s", err, servingPairs()) + } + for _, shape := range servingShapes { + if shape.accelerator == accelerator && shape.count == *compute.NumAccelerators { + return shape.workloadType, *compute.WorkloadSize, nil + } + } + return "", "", fmt.Errorf("compute.accelerator_type and compute.num_accelerators must be a supported pair, got %s with %d; the supported pairs are %s", *compute.AcceleratorType, *compute.NumAccelerators, servingPairs()) +} + +// servingPairs lists the accelerator and count pairs serving compute supports. +func servingPairs() string { + pairs := make([]string, 0, len(servingShapes)) + for _, shape := range servingShapes { + pairs = append(pairs, fmt.Sprintf("%s x%d", shape.accelerator, shape.count)) + } + return strings.Join(pairs, ", ") +} + +// dedicatedCompute is the dedicated compute a dedicated served entity gets. +type dedicatedCompute struct { + ComputeProfile string + ProvisionedModelUnits int64 +} + +// resolveDedicatedCompute returns the dedicated compute of a dedicated checkpoint. The +// server has no default for it, so compute, its profile and its units are required. +func (c *deployConfig) resolveDedicatedCompute() (*dedicatedCompute, error) { + if c.Compute == nil { + return nil, errors.New("compute is required for a dedicated checkpoint: set compute.compute_profile and compute.provisioned_model_units") + } + compute := c.Compute + if compute.ComputeProfile == nil { + return nil, errors.New("compute.compute_profile is required for a dedicated checkpoint: one of low_latency, fast, throughput or minimal_cost") + } + profile := strings.ToLower(*compute.ComputeProfile) + if !slices.Contains(computeProfiles, profile) { + return nil, fmt.Errorf("compute.compute_profile must be one of %s, got %q", strings.Join(computeProfiles, ", "), *compute.ComputeProfile) + } + if compute.ProvisionedModelUnits == nil { + return nil, errors.New("compute.provisioned_model_units is required for a dedicated checkpoint: the model units to provision, above 0") + } + if *compute.ProvisionedModelUnits <= 0 { + return nil, fmt.Errorf("compute.provisioned_model_units must be above 0, got %d", *compute.ProvisionedModelUnits) + } + // The server refuses a commitment term for a dedicated endpoint, which it would + // only do after the weights are copied. + if compute.ReservationTerm != nil { + return nil, errors.New("compute.reservation_term isn't supported yet: dedicated endpoints don't accept a commitment term; leave it out") + } + return &dedicatedCompute{ + ComputeProfile: "COMPUTE_PROFILE_" + strings.ToUpper(profile), + ProvisionedModelUnits: int64(*compute.ProvisionedModelUnits), + }, nil +} + +// validateVolumeDirectory checks a Unity Catalog volume directory the server +// copies into the version. +func validateVolumeDirectory(field, v string) error { + if !strings.HasPrefix(v, "/Volumes/") || serverLength(v) > maxVolumePathChars { + return fmt.Errorf("%s must be a Unity Catalog volume directory as /Volumes////, of at most %d characters, got %q", field, maxVolumePathChars, v) + } + return nil +} + +// validate checks the environment version and dependency lines. +func (e *deployEnvironmentConfig) validate() error { + if e.Version == nil { + return errors.New("environment.version is required: the serverless environment to serve the command on, such as 6 or databricks_ai_v6") + } + if _, _, err := e.runtimeVersion(); err != nil { + return err + } + if len(e.Dependencies) > maxDependencies { + return fmt.Errorf("environment.dependencies can list at most %d packages, got %d", maxDependencies, len(e.Dependencies)) + } + for i, dependency := range e.Dependencies { + if strings.TrimSpace(dependency) == "" { + return fmt.Errorf("environment.dependencies[%d] cannot be empty", i) + } + } + return nil +} + +// validateDeployIdempotencyKey checks a registration key from source (the YAML +// field or the flag) against the server's rule. +func validateDeployIdempotencyKey(key, source string) error { + if !deployIdempotencyKeyRe().MatchString(key) { + return fmt.Errorf("%s must be 1 to 128 letters, digits, '.', '_' or '-', got %q", source, key) + } + return nil +} + +// serverLength is the length of s as the server's limits count it, in UTF-16 +// code units. +func serverLength(s string) int { + n := 0 + for _, r := range s { + n += utf16.RuneLen(r) + } + return n +} + +// runtimeVersion is environment.version as a version number, and whether it names +// the Databricks AI environment, as air run reads the same field. +func (e *deployEnvironmentConfig) runtimeVersion() (int, bool, error) { + if e.Version == nil { + return 0, false, errors.New("environment.version is required") + } + normalized, err := validateRuntimeVersion(e.Version.raw, "environment.version") + if err != nil { + return 0, false, err + } + ai := strings.HasPrefix(normalized, databricksAIPrefix) + major, err := strconv.Atoi(strings.TrimPrefix(normalized, databricksAIPrefix)) + if err != nil || major < 1 { + return 0, false, fmt.Errorf("environment.version must be a serverless environment version such as 6 or databricks_ai_v6, got %q", e.Version.raw) + } + return major, ai, nil +} + +// servedEnvironmentVars merges env_variables and secrets into the served +// entity's environment variables, with each secret as {{secrets/scope/key}}. +func (c *deployConfig) servedEnvironmentVars() map[string]string { + if len(c.EnvVariables) == 0 && len(c.Secrets) == 0 { + return nil + } + vars := maps.Clone(c.EnvVariables) + if vars == nil { + vars = map[string]string{} + } + for name, ref := range c.Secrets { + vars[name] = "{{secrets/" + ref + "}}" + } + return vars +} + +// deploySchema is the reflection walk over deployConfig, shared by +// `air deploy -h config.` and --override path validation. +func deploySchema() configField { + return configField{ + path: configHelpRoot, + help: `Deployment YAML schema. For details on a field, pass its path to -h, for example: + + databricks air deploy -h config.checkpoint + + A checkpoint served by a command of your own: + registered_model_name: main.serving.qwen30b_sglang + endpoint_name: qwen3-30b-sglang + checkpoint: /Volumes/main/weights/qwen3_30b_a3b + command: python -m sglang.launch_server --model-path qwen3_30b_a3b --attention-backend triton --sampling-backend pytorch --host 0.0.0.0 --port 8080 + task: llm/v1/chat + environment: + version: 6 + dependencies: [sglang==0.5.23] + compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 + workload_size: Small + + A checkpoint served with dedicated inference: + registered_model_name: main.serving.gemma4 + endpoint_name: gemma4 + checkpoint: /Volumes/main/weights/gemma4_e4b + compute: + compute_profile: low_latency + provisioned_model_units: 600 + + A version already registered in Unity Catalog: + registered_model_name: main.ml.churn + endpoint_name: churn + model_version: 3`, + children: describeStruct(reflect.TypeFor[deployConfig](), configHelpRoot), + } +} + +// loadDeployConfig reads a deployment YAML, applies --override entries, decodes +// it with unknown keys rejected and validates it. ctx is used only to log applied +// overrides. +func loadDeployConfig(ctx context.Context, path string, overrides []string) (*deployConfig, error) { + var ( + document *yaml.Node + validationYAML []byte + err error + ) + if len(overrides) == 0 { + document, validationYAML, err = readRunConfigDocument(path) + } else { + document, validationYAML, err = readConfigWithOverrides(ctx, path, overrides, deploySchema()) + } + if err != nil { + return nil, err + } + if len(document.Content) == 0 { + return nil, fmt.Errorf("config %s is empty", path) + } + + cfg, err := decodeConfigReader[deployConfig](bytes.NewReader(validationYAML), path) + if err != nil { + return nil, err + } + if err := cfg.validate(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go new file mode 100644 index 00000000000..c147a9f8920 --- /dev/null +++ b/cmd/air/deployconfig_test.go @@ -0,0 +1,519 @@ +package aircmd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// commandDeployConfig is a checkpoint served by a command. +const commandDeployConfig = `registered_model_name: main.models.qwen +checkpoint: /Volumes/main/models/weights/qwen3_8b +command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080 +task: llm/v1/chat +environment: + version: 6 + dependencies: [einops==0.8.0] +endpoint_name: qwen-chat +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 + workload_size: Small +secrets: + HF_TOKEN: llm/hf_token +env_variables: + LOG_LEVEL: info +` + +// dedicatedDeployConfig is a checkpoint served with dedicated inference. +const dedicatedDeployConfig = `registered_model_name: main.models.qwen +checkpoint: /Volumes/main/models/weights/qwen3_8b +endpoint_name: qwen-chat +compute: + compute_profile: low_latency + provisioned_model_units: 600 +` + +// registeredDeployConfig serves a version that is already registered. +const registeredDeployConfig = `registered_model_name: main.models.qwen +model_version: 7 +endpoint_name: qwen-chat +` + +func loadDeployConfigText(t *testing.T, text string, overrides ...string) (*deployConfig, error) { + t.Helper() + return loadDeployConfig(t.Context(), writeConfigFile(t, "deploy.yaml", text), overrides) +} + +func TestLoadDeployConfigVariantIsInferredFromTheFields(t *testing.T) { + for _, tt := range []struct { + name string + text string + variant string + }{ + {"checkpoint and command", commandDeployConfig, variantCommand}, + {"checkpoint without command", dedicatedDeployConfig, variantDedicated}, + {"no checkpoint", registeredDeployConfig, variantRegistered}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg, err := loadDeployConfigText(t, tt.text) + require.NoError(t, err) + assert.Equal(t, tt.variant, cfg.variant()) + assert.Equal(t, tt.variant != variantRegistered, cfg.registers()) + }) + } +} + +func TestLoadDeployConfigCommand(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig) + require.NoError(t, err) + + req := cfg.registrationRequest("key-1") + assert.Equal(t, registerModelPackageRequest{ + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/models/weights/qwen3_8b", + ServingCommand: "python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", + Task: "llm/v1/chat", + Environment: modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, + }, + }, req) + + entity, err := cfg.servedEntity("3") + require.NoError(t, err) + assert.Equal(t, servedEntity{ + EntityName: "main.models.qwen", + EntityVersion: "3", + WorkloadType: "GPU_MEDIUM", + WorkloadSize: "Small", + EnvironmentVars: map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, + }, entity) +} + +func TestLoadDeployConfigCommandOnDatabricksAI(t *testing.T) { + cfg, err := loadDeployConfigText(t, strings.Replace(commandDeployConfig, " version: 6\n", " version: databricks_ai_v6\n", 1)) + require.NoError(t, err) + req := cfg.registrationRequest("key-1") + assert.Equal(t, modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}, BaseEnvironment: "databricks-ai"}, req.Checkpoint.Environment) +} + +func TestLoadDeployConfigDedicated(t *testing.T) { + cfg, err := loadDeployConfigText(t, dedicatedDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") + require.NoError(t, err) + + req := cfg.registrationRequest("key-1") + assert.Equal(t, registerModelPackageRequest{ + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + DedicatedCheckpoint: &dedicatedCheckpoint{VolumePath: "/Volumes/main/models/weights/qwen3_8b"}, + ExperimentPath: "/Shared/qwen", + }, req) + + dedicated, err := cfg.resolveDedicatedCompute() + require.NoError(t, err) + assert.Equal(t, &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 600}, dedicated) +} + +func TestLoadDeployConfigRegisteredDefaultsToCPU(t *testing.T) { + cfg, err := loadDeployConfigText(t, registeredDeployConfig) + require.NoError(t, err) + assert.False(t, cfg.registers()) + + entity, err := cfg.servedEntity("7") + require.NoError(t, err) + assert.Equal(t, "CPU", entity.WorkloadType) + assert.Equal(t, "Small", entity.WorkloadSize) + assert.Nil(t, entity.EnvironmentVars) +} + +func TestDeployConfigCrossVariantFields(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + { + "task without a checkpoint", + registeredDeployConfig + "task: llm/v1/chat\n", + "task can't be set: the file is read as a registered model, because it sets no checkpoint, and task applies only to a checkpoint served by command", + }, + { + "command without a checkpoint", + registeredDeployConfig + "command: python serve.py\n", + "command can't be set: the file is read as a registered model, because it sets no checkpoint, and command applies only to a checkpoint served by command", + }, + { + "environment on a dedicated checkpoint", + dedicatedDeployConfig + "environment: {version: 6}\n", + "environment can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and environment applies only to a checkpoint served by command", + }, + { + "model_version with a checkpoint", + commandDeployConfig + "model_version: 3\n", + "model_version can't be set: the file is read as a checkpoint served by command, because it sets checkpoint and command, and model_version applies only to a registered model", + }, + { + "model_version on a dedicated checkpoint", + dedicatedDeployConfig + "model_version: 3\n", + "model_version can't be set: the file is read as a dedicated checkpoint", + }, + { + "secrets on a dedicated checkpoint", + dedicatedDeployConfig + "secrets: {HF_TOKEN: llm/hf_token}\n", + "secrets can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and secrets applies only to a command or registered model", + }, + { + "env_variables on a dedicated checkpoint", + dedicatedDeployConfig + "env_variables: {LOG_LEVEL: info}\n", + "env_variables can't be set: the file is read as a dedicated checkpoint", + }, + { + "idempotency_token without a checkpoint", + registeredDeployConfig + "idempotency_token: key-1\n", + "idempotency_token can't be set: the file is read as a registered model, because it sets no checkpoint, and idempotency_token applies only to a checkpoint, which air deploy registers", + }, + { + "mlflow_experiment_path without a checkpoint", + registeredDeployConfig + "mlflow_experiment_path: /Shared/qwen\n", + "mlflow_experiment_path can't be set: the file is read as a registered model", + }, + { + "serving compute on a dedicated checkpoint", + strings.Replace(dedicatedDeployConfig, " compute_profile: low_latency\n", " compute_profile: low_latency\n accelerator_type: GPU_1xA10\n", 1), + "compute.accelerator_type can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and compute.accelerator_type applies only to a command or registered model", + }, + { + "dedicated compute on a command", + strings.Replace(commandDeployConfig, " workload_size: Small\n", " workload_size: Small\n compute_profile: low_latency\n", 1), + "compute.compute_profile can't be set: the file is read as a checkpoint served by command, because it sets checkpoint and command, and compute.compute_profile applies only to a dedicated checkpoint", + }, + { + "reservation term on a command", + strings.Replace(commandDeployConfig, " workload_size: Small\n", " workload_size: Small\n reservation_term: ONE_MONTH\n", 1), + "compute.reservation_term can't be set: the file is read as a checkpoint served by command", + }, + { + "dedicated compute on a registered model", + registeredDeployConfig + "compute: {workload_size: Small, provisioned_model_units: 10}\n", + "compute.provisioned_model_units can't be set: the file is read as a registered model", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.config) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigRequiredFieldsPerVariant(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + {"command without a task", dropLine("task:")(commandDeployConfig), "task is required for a checkpoint served by command"}, + {"command without an environment", dropBlock("environment:")(commandDeployConfig), "environment is required for a checkpoint served by command"}, + {"dedicated without compute", dropBlock("compute:")(dedicatedDeployConfig), "compute is required for a dedicated checkpoint"}, + {"dedicated without a profile", dropLine(" compute_profile:")(dedicatedDeployConfig), "compute.compute_profile is required for a dedicated checkpoint"}, + {"dedicated without units", dropLine(" provisioned_model_units:")(dedicatedDeployConfig), "compute.provisioned_model_units is required for a dedicated checkpoint"}, + {"command compute without num_accelerators", dropLine(" num_accelerators:")(commandDeployConfig), "compute.num_accelerators is required when compute is set"}, + {"registered compute without accelerator", registeredDeployConfig + "compute: {workload_size: Small}\n", "compute.accelerator_type is required when compute is set"}, + {"no endpoint", dropLine("endpoint_name:")(commandDeployConfig), `endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, + {"no registered model", dropLine("registered_model_name:")(commandDeployConfig), `registered_model_name must be a Unity Catalog model name as .., got ""`}, + {"registered with a bad version", replaceFirst("model_version: 7", "model_version: v7")(registeredDeployConfig), "model_version must be a model version number such as 3"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.config) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// servingCompute is a config's serving compute for one accelerator, count and size. +func servingCompute(accelerator string, count int, size string) *deployComputeConfig { + return &deployComputeConfig{AcceleratorType: &accelerator, NumAccelerators: &count, WorkloadSize: &size} +} + +func TestDeployServingComputeTable(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + for _, tt := range []struct { + accelerator string + count int + workload string + }{ + {"GPU_1xA10", 1, "GPU_MEDIUM"}, + {"GPU_1xA10", 4, "MULTIGPU_MEDIUM"}, + {"GPU_1xA10", 8, "GPU_MEDIUM_8"}, + {"GPU_1xH100", 1, "GPU_XLARGE"}, + {"GPU_8xH100", 8, "GPU_XLARGE_8"}, + {"GPU_1xRTX6000", 1, "GPU_LARGE_RTX"}, + } { + cfg.Compute = servingCompute(tt.accelerator, tt.count, "Medium") + workload, size, err := cfg.resolveServingCompute() + require.NoError(t, err, tt.accelerator) + assert.Equal(t, tt.workload, workload, "%s x%d", tt.accelerator, tt.count) + assert.Equal(t, "Medium", size) + } +} + +func TestDeployServingComputeRejectsUnsupportedPairs(t *testing.T) { + const pairs = "GPU_1xA10 x1, GPU_1xA10 x4, GPU_1xA10 x8, GPU_1xH100 x1, GPU_8xH100 x8, GPU_1xRTX6000 x1" + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + for _, tt := range []struct { + accelerator string + count int + }{ + {"GPU_1xA10", 2}, + {"GPU_1xH100", 8}, + {"GPU_8xH100", 1}, + {"GPU_8xB300", 8}, + {"GPU_1xRTX6000", 2}, + } { + cfg.Compute = servingCompute(tt.accelerator, tt.count, "Small") + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, "must be a supported pair, got "+tt.accelerator+" with ", tt.accelerator) + assert.ErrorContains(t, err, "the supported pairs are "+pairs, tt.accelerator) + } + + cfg.Compute = servingCompute("GPU_9xA10", 1, "Small") + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, `compute.accelerator_type: "GPU_9xA10" is not a recognized accelerator type; the supported pairs are `+pairs) +} + +func TestDeployServingComputeDefaults(t *testing.T) { + command := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + workload, size, err := command.resolveServingCompute() + require.NoError(t, err) + assert.Equal(t, "GPU_MEDIUM", workload) + assert.Equal(t, "Small", size) + + registered := deployConfig{} + workload, size, err = registered.resolveServingCompute() + require.NoError(t, err) + assert.Equal(t, "CPU", workload) + assert.Equal(t, "Small", size) +} + +func TestDeployServingComputeRejectsBadSize(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve"), Compute: servingCompute("GPU_1xA10", 1, "Huge")} + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, `compute.workload_size must be one of Small, Medium, Large, got "Huge"`) +} + +func TestDeployDedicatedComputeMapping(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w")} + for profile, want := range map[string]string{ + "low_latency": "COMPUTE_PROFILE_LOW_LATENCY", + "FAST": "COMPUTE_PROFILE_FAST", + "Throughput": "COMPUTE_PROFILE_THROUGHPUT", + "minimal_cost": "COMPUTE_PROFILE_MINIMAL_COST", + } { + cfg.Compute = &deployComputeConfig{ComputeProfile: new(profile), ProvisionedModelUnits: new(600)} + dedicated, err := cfg.resolveDedicatedCompute() + require.NoError(t, err, profile) + assert.Equal(t, want, dedicated.ComputeProfile, profile) + } +} + +func TestDeployDedicatedComputeRejects(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w")} + for _, tt := range []struct { + name string + compute deployComputeConfig + wantErr string + }{ + {"unknown profile", deployComputeConfig{ComputeProfile: new("fastest"), ProvisionedModelUnits: new(600)}, `compute.compute_profile must be one of low_latency, fast, throughput, minimal_cost, got "fastest"`}, + {"zero units", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(0)}, "compute.provisioned_model_units must be above 0, got 0"}, + {"negative units", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(-1)}, "compute.provisioned_model_units must be above 0, got -1"}, + {"a reservation term", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(600), ReservationTerm: new("ONE_MONTH")}, "compute.reservation_term isn't supported yet"}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg.Compute = &tt.compute + _, err := cfg.resolveDedicatedCompute() + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigValidation(t *testing.T) { + tests := []struct { + name string + edit func(string) string + wantErr string + }{ + {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model_name must be a Unity Catalog model name"}, + {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model_name must be a Unity Catalog model name"}, + {"command without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required for a checkpoint served by command"}, + {"empty task", replaceFirst("task: llm/v1/chat", `task: " "`), "task cannot be empty"}, + {"path outside volumes", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "checkpoint must be a Unity Catalog volume directory"}, + {"path too long", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("a", 1100)), "checkpoint must be a Unity Catalog volume directory"}, + {"path too long in characters", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1020)), "checkpoint must be a Unity Catalog volume directory"}, + {"empty command", replaceFirst("command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", `command: " "`), "command cannot be empty"}, + {"command too long", replaceFirst("command: python", "command: python "+strings.Repeat("x", maxServingCommandChars)), "command must be at most 32768 characters"}, + {"AI environment before version 5", replaceFirst(" version: 6\n", " version: databricks_ai_v4\n"), "databricks_ai_v in environment.version requires AI Runtime version 5 or higher"}, + {"bad environment version", replaceFirst(" version: 6\n", " version: six\n"), "unsupported client image version"}, + {"zero environment version", replaceFirst(" version: 6\n", " version: 0\n"), "environment.version must be a serverless environment version such as 6 or databricks_ai_v6"}, + {"no environment version", replaceFirst(" version: 6\n", ""), "environment.version is required"}, + {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, + {"no environment", dropBlock("environment:"), "environment is required for a checkpoint served by command"}, + {"no endpoint", dropLine("endpoint_name:"), `endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, + {"endpoint with a path", replaceFirst("endpoint_name: qwen-chat", "endpoint_name: x/../../jobs"), "endpoint_name must be a serving endpoint name"}, + {"no workload size", dropLine(" workload_size:"), "compute.workload_size is required when compute is set"}, + {"lowercase workload size", replaceFirst("workload_size: Small", "workload_size: medium"), "compute.workload_size must be one of Small, Medium, Large"}, + {"unsupported accelerator pair", replaceFirst("num_accelerators: 1", "num_accelerators: 2"), "must be a supported pair, got GPU_1xA10 with 2"}, + {"secret without key", replaceFirst("llm/hf_token", "llm"), "expected format 'scope/key'"}, + {"env var and secret", replaceFirst("LOG_LEVEL: info", "HF_TOKEN: x"), `"HF_TOKEN" is set in both env_variables and secrets`}, + {"bad idempotency token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, + {"long idempotency token", appendText("idempotency_token: " + strings.Repeat("k", 129) + "\n"), "idempotency_token must be 1 to 128 letters"}, + {"relative experiment", appendText("mlflow_experiment_path: Users/me/x\n"), "mlflow_experiment_path must be an absolute workspace path"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.edit(commandDeployConfig)) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigDedicatedValidation(t *testing.T) { + tests := []struct { + name string + edit func(string) string + wantErr string + }{ + {"bad path", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "checkpoint must be a Unity Catalog volume directory"}, + {"bad endpoint", replaceFirst("endpoint_name: qwen-chat", "endpoint_name: a/b"), "endpoint_name must be a serving endpoint name"}, + {"bad token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, + {"unknown profile", replaceFirst("low_latency", "fastest"), "compute.compute_profile must be one of"}, + {"zero units", replaceFirst("provisioned_model_units: 600", "provisioned_model_units: 0"), "compute.provisioned_model_units must be above 0"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.edit(dedicatedDeployConfig)) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigCountsCharactersNotBytes(t *testing.T) { + // 1,000 two-byte characters fit the 1,024-character cap on a volume path. + _, err := loadDeployConfigText(t, replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1000))(commandDeployConfig)) + assert.NoError(t, err) +} + +func TestLoadDeployConfigOverride(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig, "compute.num_accelerators=4", "environment.version=5") + require.NoError(t, err) + entity, err := cfg.servedEntity("3") + require.NoError(t, err) + assert.Equal(t, "MULTIGPU_MEDIUM", entity.WorkloadType) + assert.Equal(t, "client.5-gpu", cfg.registrationRequest("k").Checkpoint.Environment.Runtime) +} + +func TestLoadDeployConfigOverrideIsValidatedAgainstTheVariant(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig, "compute.compute_profile=low_latency") + assert.ErrorContains(t, err, "compute.compute_profile can't be set: the file is read as a checkpoint served by command") +} + +func TestLoadDeployConfigRejectsUnknownOverride(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig, "compute.workers=2") + assert.ErrorContains(t, err, `"workers" is not a known field`) +} + +func TestLoadDeployConfigRejectsUnknownField(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig+"experiment_name: x\n") + assert.ErrorContains(t, err, "field experiment_name not found") +} + +func TestLoadDeployConfigRejectsOldShape(t *testing.T) { + _, err := loadDeployConfigText(t, "registered_model: main.models.qwen\nendpoint: qwen-chat\n") + assert.ErrorContains(t, err, "field registered_model not found") +} + +func TestLoadDeployConfigRejectsEmptyFile(t *testing.T) { + _, err := loadDeployConfigText(t, "") + assert.ErrorContains(t, err, "is empty") +} + +func TestDeploySchemaHelp(t *testing.T) { + root, err := resolveSchemaField(deploySchema(), "config") + require.NoError(t, err) + var out strings.Builder + renderConfigField(&out, deployCommandPath, root, nil) + help := out.String() + assert.Contains(t, help, " databricks air deploy -h config.checkpoint\n") + assert.Contains(t, help, " Model:\n") + assert.Contains(t, help, " Serving:\n") + assert.Contains(t, help, " Registration:\n") + assert.Contains(t, help, `For details on a field, run "databricks air deploy -h config.".`) + + field, err := resolveSchemaField(deploySchema(), "compute.compute_profile") + require.NoError(t, err) + assert.Equal(t, "when checkpoint is set without command", field.required) + + _, err = resolveSchemaField(deploySchema(), "checkpont") + assert.ErrorContains(t, err, `did you mean "checkpoint"?`) + + _, err = resolveSchemaField(deploySchema(), "secrets.HF_TOKEN") + assert.ErrorContains(t, err, "holds user-defined keys") +} + +// Every field a variant names must be a schema path, so the variant checks can't +// drift from the YAML. +func TestDeployVariantFieldsAreSchemaPaths(t *testing.T) { + var cfg deployConfig + for _, f := range cfg.variantFields() { + _, err := resolveSchemaField(deploySchema(), f.name) + assert.NoError(t, err, f.name) + } +} + +func replaceFirst(old, with string) func(string) string { + return func(s string) string { + if !strings.Contains(s, old) { + panic("fixture does not contain " + old) + } + return strings.Replace(s, old, with, 1) + } +} + +func appendText(extra string) func(string) string { + return func(s string) string { return s + extra } +} + +// dropLine removes the first line starting with prefix. +func dropLine(prefix string) func(string) string { + return func(s string) string { + lines := strings.SplitAfter(s, "\n") + for i, line := range lines { + if strings.HasPrefix(line, prefix) { + return strings.Join(append(lines[:i:i], lines[i+1:]...), "") + } + } + panic("fixture has no line starting with " + prefix) + } +} + +// dropBlock removes the top-level key starting with prefix and its indented lines. +func dropBlock(prefix string) func(string) string { + return func(s string) string { + var kept []string + dropping := false + for _, line := range strings.SplitAfter(s, "\n") { + switch { + case strings.HasPrefix(line, prefix): + dropping = true + case dropping && strings.HasPrefix(line, " "): + default: + dropping = false + kept = append(kept, line) + } + } + return strings.Join(kept, "") + } +} diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go new file mode 100644 index 00000000000..7c64bcc1cc5 --- /dev/null +++ b/cmd/air/deployendpoint.go @@ -0,0 +1,420 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "maps" + "net/http" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/serving" +) + +// servingEndpointsPath is the serving endpoints API. Dedicated endpoints are created +// and configured under dedicatedEndpointsPath. +const ( + servingEndpointsPath = "/api/2.0/serving-endpoints" + dedicatedEndpointsPath = servingEndpointsPath + "/pt" +) + +// servedEntity is the one entity air deploy serves on the endpoint: a version of the +// model on serving compute, or on dedicated compute when Dedicated is set. +type servedEntity struct { + EntityName string + EntityVersion string + // WorkloadType and WorkloadSize are the serving compute. + WorkloadType string + WorkloadSize string + // EnvironmentVars are the served model's environment variables. + EnvironmentVars map[string]string + // Dedicated is the dedicated compute, nil for serving compute. + Dedicated *dedicatedCompute +} + +// servedEntity is the entity the endpoint serves for version: the version the +// registration produced, or the one the config names. +func (c *deployConfig) servedEntity(version string) (servedEntity, error) { + entity := servedEntity{EntityName: c.RegisteredModel, EntityVersion: version, EnvironmentVars: c.servedEnvironmentVars()} + if c.variant() == variantDedicated { + dedicated, err := c.resolveDedicatedCompute() + if err != nil { + return servedEntity{}, err + } + entity.Dedicated = dedicated + return entity, nil + } + workloadType, workloadSize, err := c.resolveServingCompute() + if err != nil { + return servedEntity{}, err + } + entity.WorkloadType, entity.WorkloadSize = workloadType, workloadSize + return entity, nil +} + +// servingInput is the entity as the serving endpoints API takes it. scale_to_zero_enabled +// is sent even though it's false, since the server's default isn't known to be false. +func (e servedEntity) servingInput() serving.ServedEntityInput { + return serving.ServedEntityInput{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + WorkloadType: serving.ServingModelWorkloadType(e.WorkloadType), + WorkloadSize: e.WorkloadSize, + ScaleToZeroEnabled: false, + EnvironmentVars: e.EnvironmentVars, + ForceSendFields: []string{"ScaleToZeroEnabled"}, + } +} + +// dedicatedServedEntity is a dedicated served entity as the dedicated endpoints API +// takes it. The SDK has no compute_profile, so these requests are sent as JSON. The +// entity_version is empty in a dry run, before the registration produces one. +type dedicatedServedEntity struct { + EntityName string `json:"entity_name"` + EntityVersion string `json:"entity_version,omitempty"` + ProvisionedModelUnits int64 `json:"provisioned_model_units"` + ComputeProfile string `json:"compute_profile"` +} + +// dedicatedEndpointConfig is the config of a dedicated endpoint. +type dedicatedEndpointConfig struct { + ServedEntities []dedicatedServedEntity `json:"served_entities"` +} + +// dedicatedEndpointUpdate replaces the config of a dedicated endpoint. +type dedicatedEndpointUpdate struct { + Config dedicatedEndpointConfig `json:"config"` +} + +// dedicatedEndpointRequest creates a dedicated endpoint. +type dedicatedEndpointRequest struct { + Name string `json:"name"` + Config dedicatedEndpointConfig `json:"config"` +} + +// dedicatedJSON is the entity as the dedicated endpoints API takes it. +func (e servedEntity) dedicatedJSON() dedicatedServedEntity { + return dedicatedServedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + ProvisionedModelUnits: e.Dedicated.ProvisionedModelUnits, + ComputeProfile: e.Dedicated.ComputeProfile, + } +} + +// servingEndpoint is a serving endpoint as GET /api/2.0/serving-endpoints/{name} +// reports it, with the fields air deploy reads. It replaces the SDK's type, which has +// no compute_profile for a dedicated served entity. +type servingEndpoint struct { + Name string `json:"name"` + State *serving.EndpointState `json:"state,omitempty"` + Config *endpointConfig `json:"config,omitempty"` + PendingConfig *endpointConfig `json:"pending_config,omitempty"` +} + +// endpointConfig is the served entities of an endpoint's current or pending config. +type endpointConfig struct { + ServedEntities []observedEntity `json:"served_entities,omitempty"` +} + +// observedEntity is a served entity as the endpoint reports it: a dedicated one by +// its units and profile, and any other by its workload. +type observedEntity struct { + EntityName string `json:"entity_name"` + EntityVersion string `json:"entity_version"` + WorkloadType string `json:"workload_type"` + WorkloadSize string `json:"workload_size"` + ScaleToZeroEnabled bool `json:"scale_to_zero_enabled"` + EnvironmentVars map[string]string `json:"environment_vars"` + ProvisionedModelUnits int64 `json:"provisioned_model_units"` + ComputeProfile string `json:"compute_profile"` +} + +// endpointAPI calls the serving endpoints API for air deploy. Endpoints are read as +// JSON, and dedicated endpoints written so; serving compute is written through the SDK. +type endpointAPI struct { + w *databricks.WorkspaceClient + api *client.DatabricksClient + headers map[string]string +} + +func newEndpointAPI(w *databricks.WorkspaceClient) (*endpointAPI, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, withCode(codeInternalError, false, fmt.Errorf("failed to create API client: %w", err)) + } + return &endpointAPI{w: w, api: api, headers: auth.WorkspaceIDHeaders(w.Config)}, nil +} + +// read returns the endpoint named name, and the error the API gives when it doesn't +// exist. A transient failure is repeated. +func (e *endpointAPI) read(ctx context.Context, name string) (*servingEndpoint, error) { + return retry.With(ctx, retry.IsTransientRead, func() (*servingEndpoint, error) { + endpoint := &servingEndpoint{} + err := e.api.Do(ctx, http.MethodGet, servingEndpointsPath+"/"+name, e.headers, nil, nil, endpoint) + return endpoint, err + }) +} + +// get returns the endpoint named name, or nil if it doesn't exist. +func (e *endpointAPI) get(ctx context.Context, name string) (*servingEndpoint, error) { + endpoint, err := e.read(ctx, name) + if errors.Is(err, apierr.ErrResourceDoesNotExist) { + return nil, nil + } + return endpoint, err +} + +// create creates the endpoint named name with entity. It returns once the API accepts +// the request, not once the endpoint is up. +func (e *endpointAPI) create(ctx context.Context, name string, entity servedEntity) error { + if entity.Dedicated != nil { + request := dedicatedEndpointRequest{ + Name: name, + Config: dedicatedEndpointConfig{ServedEntities: []dedicatedServedEntity{entity.dedicatedJSON()}}, + } + return e.api.Do(ctx, http.MethodPost, dedicatedEndpointsPath, e.headers, nil, request, nil) + } + _, err := e.w.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ + Name: name, + Config: &serving.EndpointCoreConfigInput{ServedEntities: []serving.ServedEntityInput{entity.servingInput()}}, + }) + return err +} + +// update replaces the endpoint named name's served entity with entity. It returns once +// the API accepts the update. +func (e *endpointAPI) update(ctx context.Context, name string, entity servedEntity) error { + if entity.Dedicated != nil { + update := dedicatedEndpointUpdate{Config: dedicatedEndpointConfig{ServedEntities: []dedicatedServedEntity{entity.dedicatedJSON()}}} + return e.api.Do(ctx, http.MethodPut, dedicatedEndpointsPath+"/"+name+"/config", e.headers, nil, update, nil) + } + _, err := e.w.ServingEndpoints.UpdateConfig(ctx, serving.EndpointCoreConfigInput{ + Name: name, + ServedEntities: []serving.ServedEntityInput{entity.servingInput()}, + }) + return err +} + +// checkEndpoint fails when the endpoint named name, if it exists, serves +// something air deploy won't replace with a version of model, or is of the other +// kind than dedicated says. air deploy checks it before registering, so a wrong +// endpoint name fails before any weights are copied. +func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, model string, dedicated bool) error { + e, err := newEndpointAPI(w) + if err != nil { + return err + } + endpoint, err := e.get(ctx, name) + if err != nil || endpoint == nil { + return err + } + if err := checkReplaceable(endpoint, model); err != nil { + return err + } + return checkKind(endpoint, dedicated) +} + +// checkKind fails when the endpoint serves on dedicated compute and dedicated is +// false, or the reverse: the server converts neither kind of endpoint to the other. +// An endpoint that reports no served entity is taken as either kind. +func checkKind(endpoint *servingEndpoint, dedicated bool) error { + served := append(currentEntities(endpoint), pendingEntities(endpoint)...) + if len(served) == 0 { + return nil + } + isDedicated := slices.ContainsFunc(served, func(e observedEntity) bool { return e.ComputeProfile != "" }) + if isDedicated == dedicated { + return nil + } + kind, want := "serving compute", "dedicated compute" + if isDedicated { + kind, want = want, kind + } + return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves on %s, and this file serves on %s; an endpoint can't change between the two, so set endpoint_name to another name", stripControl(endpoint.Name), kind, want)) +} + +// checkReplaceable fails unless the endpoint serves at most one entity, a version +// of model, in its current and pending configs. Replacing anything else would +// remove another model or a traffic split the user didn't ask to change. The +// endpoint lists every served model under served_entities too (served_models +// mirrors it), so served_entities covers both. +func checkReplaceable(endpoint *servingEndpoint, model string) error { + for _, served := range [][]observedEntity{currentEntities(endpoint), pendingEntities(endpoint)} { + if len(served) > 1 || (len(served) == 1 && !strings.EqualFold(served[0].EntityName, model)) { + return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint_name to another name, or change this endpoint with databricks serving-endpoints update-config", stripControl(endpoint.Name), describeEntities(served), model)) + } + } + return nil +} + +// serveOnEndpoint makes the endpoint named name serve only entity and waits until +// it is ready, calling progress on every poll. It creates the endpoint if it +// doesn't exist. An endpoint that already serves entity isn't updated, and an +// update to entity that an interrupted air deploy started is waited for. +func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string, entity servedEntity, progress func(*servingEndpoint)) (*servingEndpoint, error) { + e, err := newEndpointAPI(w) + if err != nil { + return nil, err + } + endpoint, err := e.get(ctx, name) + if err != nil { + return nil, err + } + if endpoint == nil { + if err := e.create(ctx, name, entity); err != nil { + return nil, err + } + return waitServes(ctx, e, name, entity, progress) + } + if updating(endpoint) { + if servesOnly(pendingEntities(endpoint), entity) { + return waitServes(ctx, e, name, entity, progress) + } + // The endpoint takes no other update until this one finishes. + endpoint, err = waitNotUpdating(ctx, e, name, progress) + if err != nil { + return nil, err + } + } + if err := checkReplaceable(endpoint, entity.EntityName); err != nil { + return nil, err + } + if err := checkKind(endpoint, entity.Dedicated != nil); err != nil { + return nil, err + } + if servesOnly(currentEntities(endpoint), entity) { + return checkServes(endpoint, entity) + } + if err := e.update(ctx, name, entity); err != nil { + return nil, err + } + return waitServes(ctx, e, name, entity, progress) +} + +// waitNotUpdating waits until the endpoint has no update in progress, calling +// progress on every poll. Each poll repeats a transient failure, which ends the SDK's +// own wait: an endpoint can take many minutes to start. Creates and config updates +// aren't repeated this way, since a repeat after an applied write would fail. +func waitNotUpdating(ctx context.Context, e *endpointAPI, name string, progress func(*servingEndpoint)) (*servingEndpoint, error) { + return retries.Poll(ctx, untilDeadline(ctx), func() (*servingEndpoint, *retries.Err) { + endpoint, err := e.read(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if progress != nil { + progress(endpoint) + } + var state serving.EndpointStateConfigUpdate + if endpoint.State != nil { + state = endpoint.State.ConfigUpdate + } + switch state { + case serving.EndpointStateConfigUpdateNotUpdating: + return endpoint, nil + case serving.EndpointStateConfigUpdateUpdateFailed, serving.EndpointStateConfigUpdateUpdateCanceled: + return nil, retries.Halt(fmt.Errorf("the endpoint's update ended in %s; check its events and logs in the Serving UI", state)) + default: + return nil, retries.Continues(string(state)) + } + }) +} + +// waitServes waits until the endpoint's update finishes, then checks that it +// serves entity. +func waitServes(ctx context.Context, e *endpointAPI, name string, entity servedEntity, progress func(*servingEndpoint)) (*servingEndpoint, error) { + endpoint, err := waitNotUpdating(ctx, e, name, progress) + if err != nil { + return nil, err + } + return checkServes(endpoint, entity) +} + +// errEndpointNotReady reports an endpoint that serves the version but isn't ready. +// Running air deploy again doesn't make it ready; it reports success once the +// endpoint recovers, so the error isn't marked retryable. +var errEndpointNotReady = errors.New("endpoint isn't ready") + +// checkServes returns endpoint if it serves only entity and is ready to serve. +func checkServes(endpoint *servingEndpoint, entity servedEntity) (*servingEndpoint, error) { + name := stripControl(endpoint.Name) + if !servesOnly(currentEntities(endpoint), entity) { + return nil, fmt.Errorf("endpoint %s was changed to serve %s while air deploy waited for it", name, describeEntities(currentEntities(endpoint))) + } + if endpoint.State == nil || endpoint.State.Ready != serving.EndpointStateReadyReady { + return nil, withCode(codeEndpointNotReady, false, fmt.Errorf("%w: endpoint %s serves version %s of %s but isn't ready; check its events and logs in the Serving UI", errEndpointNotReady, name, entity.EntityVersion, entity.EntityName)) + } + return endpoint, nil +} + +func updating(endpoint *servingEndpoint) bool { + return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress +} + +func currentEntities(endpoint *servingEndpoint) []observedEntity { + if endpoint.Config == nil { + return nil + } + return endpoint.Config.ServedEntities +} + +func pendingEntities(endpoint *servingEndpoint) []observedEntity { + if endpoint.PendingConfig == nil { + return nil + } + return endpoint.PendingConfig.ServedEntities +} + +// describeEntities names served entities for an error, e.g. "version 2 of a.b.c". +func describeEntities(served []observedEntity) string { + if len(served) == 0 { + return "nothing" + } + names := make([]string, 0, len(served)) + for _, s := range served { + name := stripControl(s.EntityName) + if s.EntityVersion == "" { + names = append(names, name) + continue + } + names = append(names, fmt.Sprintf("version %s of %s", stripControl(s.EntityVersion), name)) + } + return strings.Join(names, " and ") +} + +// servesOnly reports whether served is exactly entity. A dedicated entity must match +// its units and compute profile; any other must match its workload, scale-to-zero +// setting and environment variables. +func servesOnly(served []observedEntity, entity servedEntity) bool { + if len(served) != 1 { + return false + } + s := served[0] + // Unity Catalog names are case-insensitive, and the endpoint reports them as + // Unity Catalog stores them. + if !strings.EqualFold(s.EntityName, entity.EntityName) || s.EntityVersion != entity.EntityVersion { + return false + } + if entity.Dedicated != nil { + return s.ProvisionedModelUnits == entity.Dedicated.ProvisionedModelUnits && s.ComputeProfile == entity.Dedicated.ComputeProfile + } + return s.WorkloadType == entity.WorkloadType && + s.WorkloadSize == entity.WorkloadSize && + !s.ScaleToZeroEnabled && + maps.Equal(s.EnvironmentVars, entity.EnvironmentVars) +} + +// untilDeadline is the time left before ctx's deadline, which air deploy always +// sets from --timeout. +func untilDeadline(ctx context.Context) time.Duration { + deadline, _ := ctx.Deadline() + return time.Until(deadline) +} diff --git a/cmd/air/deployendpoint_test.go b/cmd/air/deployendpoint_test.go new file mode 100644 index 00000000000..177d9b43234 --- /dev/null +++ b/cmd/air/deployendpoint_test.go @@ -0,0 +1,510 @@ +package aircmd + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeEndpoints is a stateful fake of the serving endpoints API for one endpoint. +// A create or config update is applied at once, and the endpoint then reports +// finalState (READY and NOT_UPDATING unless a test sets otherwise). +type fakeEndpoints struct { + t *testing.T + mu sync.Mutex + endpoint *servingEndpoint + finalState serving.EndpointState + // writes names every create and config update: "create" and "update" for serving + // compute, "create dedicated" and "update dedicated" for dedicated compute. + writes []string + // rawWrites holds the body of every create and config update, as sent. + rawWrites []string + // gets counts the GETs of an existing endpoint. Those numbered in failOnGets + // answer 500, and from readyAfterGets on (when set) the endpoint is ready. + gets int + failOnGets map[int]bool + readyAfterGets int +} + +func newFakeEndpoints(t *testing.T, existing *servingEndpoint) *fakeEndpoints { + return &fakeEndpoints{ + t: t, + endpoint: existing, + finalState: serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating}, + } +} + +func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.Method == http.MethodPost && r.URL.Path == servingEndpointsPath: + var req serving.CreateServingEndpoint + if !f.decode(w, r, &req) { + return + } + f.writes = append(f.writes, "create") + var entities []observedEntity + for _, e := range req.Config.ServedEntities { + entities = append(entities, observeServing(e)) + } + f.endpoint = &servingEndpoint{Name: req.Name, Config: &endpointConfig{ServedEntities: entities}, State: &f.finalState} + case r.Method == http.MethodPost && r.URL.Path == dedicatedEndpointsPath: + var req dedicatedEndpointRequest + if !f.decode(w, r, &req) { + return + } + f.writes = append(f.writes, "create dedicated") + f.endpoint = &servingEndpoint{Name: req.Name, Config: &endpointConfig{ServedEntities: observeDedicated(req.Config.ServedEntities)}, State: &f.finalState} + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, dedicatedEndpointsPath+"/"): + var req dedicatedEndpointUpdate + if !f.decode(w, r, &req) { + return + } + // As the server does, which reads the entities from the request's config. + if len(req.Config.ServedEntities) == 0 { + http.Error(w, `{"error_code": "INVALID_PARAMETER_VALUE", "message": "config is required for dedicated inference"}`, http.StatusBadRequest) + return + } + if f.updateConflicts(w) { + return + } + f.writes = append(f.writes, "update dedicated") + f.endpoint.Config = &endpointConfig{ServedEntities: observeDedicated(req.Config.ServedEntities)} + f.endpoint.PendingConfig = nil + f.endpoint.State = &f.finalState + case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/config"): + var req serving.EndpointCoreConfigInput + if !f.decode(w, r, &req) { + return + } + if f.updateConflicts(w) { + return + } + f.writes = append(f.writes, "update") + var entities []observedEntity + for _, e := range req.ServedEntities { + entities = append(entities, observeServing(e)) + } + f.endpoint.Config = &endpointConfig{ServedEntities: entities} + f.endpoint.PendingConfig = nil + f.endpoint.State = &f.finalState + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, servingEndpointsPath+"/"): + if f.endpoint != nil { + f.gets++ + if f.failOnGets[f.gets] { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + return + } + if f.readyAfterGets > 0 && f.gets >= f.readyAfterGets { + f.endpoint.State = &readyState + } + } + if f.endpoint == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "Endpoint not found"}`)) + return + } + default: + _, _ = w.Write([]byte(`{}`)) + return + } + assert.NoError(f.t, json.NewEncoder(w).Encode(f.endpoint)) +} + +// updateConflicts answers a config update that arrives while the endpoint is updating. +func (f *fakeEndpoints) updateConflicts(w http.ResponseWriter) bool { + if !updating(f.endpoint) { + return false + } + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The endpoint is being updated."}`)) + return true +} + +// decode reads a request body into v, failing the request (not the handler +// goroutine) when it can't. +func (f *fakeEndpoints) decode(w http.ResponseWriter, r *http.Request, v any) bool { + raw, err := io.ReadAll(r.Body) + if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, v)) { + w.WriteHeader(http.StatusInternalServerError) + return false + } + f.rawWrites = append(f.rawWrites, string(raw)) + return true +} + +// observeServing is a served entity with serving compute, as the endpoint reports it. +func observeServing(e serving.ServedEntityInput) observedEntity { + return observedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + WorkloadType: string(e.WorkloadType), + WorkloadSize: e.WorkloadSize, + EnvironmentVars: e.EnvironmentVars, + } +} + +// observeDedicated are dedicated served entities, as the endpoint reports them. +func observeDedicated(entities []dedicatedServedEntity) []observedEntity { + var out []observedEntity + for _, e := range entities { + out = append(out, observedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + ProvisionedModelUnits: e.ProvisionedModelUnits, + ComputeProfile: e.ComputeProfile, + }) + } + return out +} + +// observe is entity as the endpoint reports it once it serves it. +func observe(entity servedEntity) observedEntity { + if entity.Dedicated != nil { + return observedEntity{ + EntityName: entity.EntityName, + EntityVersion: entity.EntityVersion, + ProvisionedModelUnits: entity.Dedicated.ProvisionedModelUnits, + ComputeProfile: entity.Dedicated.ComputeProfile, + } + } + return observedEntity{ + EntityName: entity.EntityName, + EntityVersion: entity.EntityVersion, + WorkloadType: entity.WorkloadType, + WorkloadSize: entity.WorkloadSize, + EnvironmentVars: entity.EnvironmentVars, + } +} + +func (f *fakeEndpoints) start() *httptest.Server { + srv := httptest.NewServer(f) + f.t.Cleanup(srv.Close) + return srv +} + +// testEntity is a version of the test model on serving compute: a command's GPU_MEDIUM. +func testEntity(version string) servedEntity { + return servedEntity{ + EntityName: "main.models.qwen", + EntityVersion: version, + WorkloadType: "GPU_MEDIUM", + WorkloadSize: "Small", + EnvironmentVars: map[string]string{"HF_TOKEN": "{{secrets/llm/hf_token}}"}, + } +} + +// testDedicatedEntity is a version of the test model on dedicated compute. +func testDedicatedEntity(version string) servedEntity { + return servedEntity{ + EntityName: "main.models.qwen", + EntityVersion: version, + Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 600}, + } +} + +func endpointServing(entity servedEntity, state serving.EndpointState) *servingEndpoint { + return &servingEndpoint{ + Name: "qwen-chat", + Config: &endpointConfig{ServedEntities: []observedEntity{observe(entity)}}, + State: &state, + } +} + +var readyState = serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating} + +func TestServeOnEndpointCreatesAMissingEndpoint(t *testing.T) { + fake := newFakeEndpoints(t, nil) + srv := fake.start() + + endpoint, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create"}, fake.writes) + assert.Equal(t, "3", endpoint.Config.ServedEntities[0].EntityVersion) +} + +func TestServeOnEndpointUpdatesAnotherVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("2"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) +} + +func TestServeOnEndpointUpdatesChangedCompute(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), readyState)) + srv := fake.start() + entity := testEntity("3") + entity.WorkloadSize = "Medium" + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", entity, nil) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) +} + +func TestServeOnEndpointLeavesAServedVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), readyState)) + srv := fake.start() + // Unity Catalog reports names lowercased. + entity := testEntity("3") + entity.EntityName = "Main.Models.Qwen" + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", entity, nil) + require.NoError(t, err) + assert.Empty(t, fake.writes) +} + +func TestServeOnEndpointCreatesADedicatedEndpoint(t *testing.T) { + fake := newFakeEndpoints(t, nil) + srv := fake.start() + + endpoint, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testDedicatedEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create dedicated"}, fake.writes) + assert.Equal(t, "COMPUTE_PROFILE_LOW_LATENCY", endpoint.Config.ServedEntities[0].ComputeProfile) + require.Len(t, fake.rawWrites, 1) + assert.JSONEq(t, `{ + "name": "qwen-chat", + "config": {"served_entities": [{ + "entity_name": "main.models.qwen", + "entity_version": "3", + "provisioned_model_units": 600, + "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY" + }]} + }`, fake.rawWrites[0]) +} + +func TestServeOnEndpointUpdatesADedicatedEndpointWhenItsCapacityChanges(t *testing.T) { + tests := []struct { + name string + entity servedEntity + wantBody string + }{ + { + "units", + servedEntity{EntityName: "main.models.qwen", EntityVersion: "3", Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 900}}, + `{"config": {"served_entities": [{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 900, "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"}]}}`, + }, + { + "profile", + servedEntity{EntityName: "main.models.qwen", EntityVersion: "3", Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_FAST", ProvisionedModelUnits: 600}}, + `{"config": {"served_entities": [{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 600, "compute_profile": "COMPUTE_PROFILE_FAST"}]}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testDedicatedEntity("3"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", tt.entity, nil) + require.NoError(t, err) + assert.Equal(t, []string{"update dedicated"}, fake.writes) + require.Len(t, fake.rawWrites, 1) + assert.JSONEq(t, tt.wantBody, fake.rawWrites[0]) + }) + } +} + +func TestServeOnEndpointLeavesADedicatedVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testDedicatedEntity("3"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testDedicatedEntity("3"), nil) + require.NoError(t, err) + assert.Empty(t, fake.writes) +} + +func TestServeOnEndpointRefusesToChangeAnEndpointsKind(t *testing.T) { + tests := []struct { + name string + existing servedEntity + entity servedEntity + wantErr string + }{ + {"serving compute to dedicated", testEntity("3"), testDedicatedEntity("3"), "endpoint qwen-chat serves on serving compute, and this file serves on dedicated compute"}, + {"dedicated to serving compute", testDedicatedEntity("3"), testEntity("3"), "endpoint qwen-chat serves on dedicated compute, and this file serves on serving compute"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(tt.existing, readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", tt.entity, nil) + assert.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, fake.writes) + }) + } +} + +func TestCheckEndpointRefusesAnEndpointOfTheOtherKind(t *testing.T) { + tests := []struct { + name string + existing servedEntity + dedicated bool + wantErr string + }{ + {"dedicated file, serving endpoint", testEntity("2"), true, "serves on serving compute, and this file serves on dedicated compute"}, + {"serving file, dedicated endpoint", testDedicatedEntity("2"), false, "serves on dedicated compute, and this file serves on serving compute"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newFakeEndpoints(t, endpointServing(tt.existing, readyState)).start() + err := checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", tt.dedicated) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestCheckEndpointAllowsAnEndpointOfTheSameKind(t *testing.T) { + for name, tt := range map[string]struct { + existing servedEntity + dedicated bool + }{ + "serving": {testEntity("2"), false}, + "dedicated": {testDedicatedEntity("2"), true}, + } { + srv := newFakeEndpoints(t, endpointServing(tt.existing, readyState)).start() + assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", tt.dedicated), name) + } +} + +func TestServeOnEndpointWaitsForItsOwnPendingUpdate(t *testing.T) { + existing := endpointServing(testEntity("2"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) + existing.PendingConfig = &endpointConfig{ServedEntities: []observedEntity{observe(testEntity("3"))}} + fake := newFakeEndpoints(t, existing) + srv := fake.start() + + polls := 0 + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(*servingEndpoint) { + // The pending update finishes after the first poll. + polls++ + fake.mu.Lock() + defer fake.mu.Unlock() + fake.endpoint = endpointServing(testEntity("3"), readyState) + }) + require.NoError(t, err) + assert.Empty(t, fake.writes) + assert.Positive(t, polls) +} + +func TestServeOnEndpointWaitsForAnotherUpdateBeforeUpdating(t *testing.T) { + existing := endpointServing(testEntity("1"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) + existing.PendingConfig = &endpointConfig{ServedEntities: []observedEntity{observe(testEntity("2"))}} + fake := newFakeEndpoints(t, existing) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(endpoint *servingEndpoint) { + // The other update finishes after the first poll. + if updating(endpoint) { + fake.mu.Lock() + defer fake.mu.Unlock() + fake.endpoint = endpointServing(testEntity("2"), readyState) + } + }) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) + assert.Equal(t, "3", fake.endpoint.Config.ServedEntities[0].EntityVersion) +} + +func TestServeOnEndpointRefusesToReplaceOtherEntities(t *testing.T) { + other := testEntity("5") + other.EntityName = "main.models.llama" + split := endpointServing(testEntity("2"), readyState) + split.Config.ServedEntities = append(split.Config.ServedEntities, observe(testEntity("1"))) + tests := []struct { + name string + existing *servingEndpoint + wantErr string + }{ + {"another model", endpointServing(other, readyState), "endpoint qwen-chat serves version 5 of main.models.llama, and air deploy replaces only a single version of main.models.qwen"}, + {"a traffic split", split, "endpoint qwen-chat serves version 2 of main.models.qwen and version 1 of main.models.qwen"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, tt.existing) + w := newTestWorkspaceClient(t, fake.start().URL) + + assert.ErrorContains(t, checkEndpoint(withDeadline(t), w, "qwen-chat", "main.models.qwen", false), tt.wantErr) + _, err := serveOnEndpoint(withDeadline(t), w, "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, fake.writes) + }) + } +} + +func TestCheckEndpointAllowsAMissingEndpoint(t *testing.T) { + srv := newFakeEndpoints(t, nil).start() + assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", false)) +} + +func TestServeOnEndpointWaitsThroughTransientFailures(t *testing.T) { + fake := newFakeEndpoints(t, nil) + // The endpoint starts over three polls, and the first GET of each fails: more + // failures than one request may retry, spread over the wait. + fake.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress} + fake.failOnGets = map[int]bool{1: true, 3: true, 5: true} + fake.readyAfterGets = 6 + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create"}, fake.writes) + assert.Equal(t, 6, fake.gets) +} + +func TestServeOnEndpointReportsAFailedUpdate(t *testing.T) { + fake := newFakeEndpoints(t, nil) + fake.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, "UPDATE_FAILED") +} + +func TestServeOnEndpointReportsAnEndpointThatIsNotReady(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating})) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, "endpoint qwen-chat serves version 3 of main.models.qwen but isn't ready") +} + +func TestDedicatedEntityJSON(t *testing.T) { + entity := testDedicatedEntity("3") + raw, err := json.Marshal(entity.dedicatedJSON()) + require.NoError(t, err) + assert.JSONEq(t, `{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 600, "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"}`, string(raw)) +} + +func TestServesOnly(t *testing.T) { + entity := testEntity("3") + assert.True(t, servesOnly([]observedEntity{observe(entity)}, entity)) + assert.False(t, servesOnly(nil, entity)) + assert.False(t, servesOnly([]observedEntity{observe(entity), observe(testEntity("2"))}, entity)) + + changed := entity + changed.EnvironmentVars = map[string]string{"HF_TOKEN": "{{secrets/llm/other}}"} + assert.False(t, servesOnly([]observedEntity{observe(entity)}, changed)) + + scaled := observe(entity) + scaled.ScaleToZeroEnabled = true + assert.False(t, servesOnly([]observedEntity{scaled}, entity)) +} + +func TestServesOnlyDedicated(t *testing.T) { + entity := testDedicatedEntity("3") + assert.True(t, servesOnly([]observedEntity{observe(entity)}, entity)) + assert.False(t, servesOnly([]observedEntity{observe(testEntity("3"))}, entity), "serving compute isn't a dedicated entity") + assert.False(t, servesOnly([]observedEntity{observe(entity)}, testEntity("3")), "a dedicated entity isn't serving compute") +} diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go new file mode 100644 index 00000000000..d455ae47960 --- /dev/null +++ b/cmd/air/deployregister.go @@ -0,0 +1,316 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + "strconv" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" +) + +// registerModelPackagePath registers a model whose files Model Serving writes and +// starts copying a checkpoint's weights into the version. It runs as the caller and +// is idempotent per key: a repeat request with the same key and inputs continues +// the registration. It leaves the version pending. +const registerModelPackagePath = "/api/2.0/serving-endpoints:registerModelPackage" + +// stagingStatusPath reports the recorded staging progress of a model version, a +// read. +const stagingStatusPath = "/api/2.0/serving-endpoints:checkModelDeploymentStagingStatus" + +// finalizeModelPackagePath finalizes a registered version once its weights are +// copied, so it can be served. It is idempotent: a finalized version is returned +// as it is. +const finalizeModelPackagePath = "/api/2.0/serving-endpoints:finalizeModelPackage" + +// registrationPollInterval is how often air deploy checks on a checkpoint's +// weights while they are staged. +const registrationPollInterval = 15 * time.Second + +// ModelDeploymentStagingStatus states air deploy acts on. +const ( + // stagingStateNotStaged: no staging record exists for the version. + stagingStateNotStaged = "NOT_STAGED" + // stagingStateInProgress: an upload is queued, running or waiting to be retried. + stagingStateInProgress = "IN_PROGRESS" + // stagingStateSucceeded: the upload is recorded as done. + stagingStateSucceeded = "SUCCEEDED" + // stagingStateFailed: the upload failed and won't be retried automatically. + stagingStateFailed = "FAILED" +) + +// registerModelPackageRequest is the RegisterModelPackageRequest body. It sets +// exactly one of Checkpoint, a checkpoint served by command, and DedicatedCheckpoint, +// a checkpoint served with dedicated inference. +type registerModelPackageRequest struct { + RegisteredModel string `json:"registered_model"` + IdempotencyKey string `json:"idempotency_key"` + Checkpoint *modelPackageCheckpoint `json:"checkpoint,omitempty"` + DedicatedCheckpoint *dedicatedCheckpoint `json:"dedicated_checkpoint,omitempty"` + ExperimentPath string `json:"experiment_path,omitempty"` + RetryFailedStaging bool `json:"retry_failed_staging,omitempty"` +} + +// modelPackageCheckpoint is a checkpoint's weights in a volume and the command that +// serves them. +type modelPackageCheckpoint struct { + VolumePath string `json:"volume_path"` + ServingCommand string `json:"serving_command"` + Task string `json:"task"` + Environment modelPackageEnvironment `json:"environment"` +} + +// dedicatedCheckpoint is a checkpoint's weights in a volume, served with dedicated +// inference. +type dedicatedCheckpoint struct { + VolumePath string `json:"volume_path"` +} + +// baseEnvironmentDatabricksAI names the Databricks AI environment to the server. +const baseEnvironmentDatabricksAI = "databricks-ai" + +type modelPackageEnvironment struct { + Runtime string `json:"runtime"` + Dependencies []string `json:"dependencies,omitempty"` + BaseEnvironment string `json:"base_environment,omitempty"` +} + +// registerModelPackageResponse is the RegisterModelPackageResponse body. +type registerModelPackageResponse struct { + ModelID string `json:"model_id"` + RegisteredModel string `json:"registered_model"` + ModelVersion string `json:"model_version"` + Ready bool `json:"ready"` + StagingStatus *modelStagingStatus `json:"staging_status,omitempty"` +} + +type modelStagingStatus struct { + State string `json:"state,omitempty"` + ModelArtifact *artifactStagingStatus `json:"model_artifact,omitempty"` + EnvironmentArtifact *artifactStagingStatus `json:"environment_artifact,omitempty"` +} + +type artifactStagingStatus struct { + State string `json:"state,omitempty"` +} + +// finalizeModelPackageRequest is the FinalizeModelPackageRequest body. +type finalizeModelPackageRequest struct { + RegisteredModel string `json:"registered_model"` + ModelVersion string `json:"model_version"` +} + +// finalizeModelPackageResponse is the FinalizeModelPackageResponse body, the +// fields air deploy reads. +type finalizeModelPackageResponse struct { + ModelVersion string `json:"model_version"` + StagingStatus *modelStagingStatus `json:"staging_status,omitempty"` +} + +// stagingStatusRequest is the CheckModelDeploymentStagingStatusRequest body. +type stagingStatusRequest struct { + Model string `json:"model"` + ModelVersion string `json:"model_version"` +} + +// stagingStatusResponse is the CheckModelDeploymentStagingStatusResponse body. +type stagingStatusResponse struct { + Status *modelStagingStatus `json:"status,omitempty"` +} + +// weightsState is the staging state of a checkpoint's weights, or "" before a +// response reports it. +func (r *registerModelPackageResponse) weightsState() string { + if r.StagingStatus == nil || r.StagingStatus.ModelArtifact == nil { + return "" + } + return r.StagingStatus.ModelArtifact.State +} + +// environmentState is the staging state of the serving environment, or "" before +// a response reports it. +func (r *registerModelPackageResponse) environmentState() string { + if r.StagingStatus == nil || r.StagingStatus.EnvironmentArtifact == nil { + return "" + } + return r.StagingStatus.EnvironmentArtifact.State +} + +// errStagingFailed reports that Model Serving couldn't copy a checkpoint's weights +// into the version. Rerunning with --retry-failed-staging stages them again. +var errStagingFailed = errors.New("staging failed") + +// finalizeError is a failed request to finalize the registered version. +// Rerunning with the same key registers the same version again and retries it. +type finalizeError struct{ err error } + +func (e *finalizeError) Error() string { return e.err.Error() } + +func (e *finalizeError) Unwrap() error { return e.err } + +// registrationRequest builds the request that registers cfg's checkpoint under key. +func (c *deployConfig) registrationRequest(key string) registerModelPackageRequest { + req := registerModelPackageRequest{ + RegisteredModel: c.RegisteredModel, + IdempotencyKey: key, + } + if c.MLflowExperimentPath != nil { + req.ExperimentPath = *c.MLflowExperimentPath + } + if c.variant() == variantDedicated { + req.DedicatedCheckpoint = &dedicatedCheckpoint{VolumePath: *c.Checkpoint} + return req + } + req.Checkpoint = &modelPackageCheckpoint{ + VolumePath: *c.Checkpoint, + ServingCommand: *c.Command, + Task: *c.Task, + Environment: c.servingEnvironment(), + } + return req +} + +// servingEnvironment is the environment a command's checkpoint is registered with: +// its runtime, dependencies and base environment. A command is served on +// accelerators, so the runtime is the GPU one. The version was validated when the +// config loaded. +func (c *deployConfig) servingEnvironment() modelPackageEnvironment { + version, ai, _ := c.Environment.runtimeVersion() + environment := modelPackageEnvironment{ + Runtime: "client." + strconv.Itoa(version) + "-gpu", + Dependencies: c.Environment.Dependencies, + } + if ai { + environment.BaseEnvironment = baseEnvironmentDatabricksAI + } + return environment +} + +// registerAndFinalize registers req's version and makes it ready to serve, +// calling progress with the registration response and every staging update. The +// checkpoint's weights are copied into the version after registering: while they +// are, it polls the staging status, a read. It then finalizes the version; the +// server checks that the version is ready to finalize. A version that is already +// ready, such as one an earlier run finalized, is returned as it is. Each +// response's staging status replaces the last one, and a status without both +// artifacts' states is an error, not progress. On error it returns the last +// response it got, nil before the first one; errStagingFailed when staging +// failed, and a *finalizeError when finalizing did. +func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, withCode(codeInternalError, false, fmt.Errorf("failed to create API client: %w", err)) + } + p := modelServingPoller{api: apiClient, headers: auth.WorkspaceIDHeaders(w.Config), interval: interval} + var resp registerModelPackageResponse + if err := p.post(ctx, registerModelPackagePath, req, &resp); err != nil { + return nil, err + } + if resp.ModelVersion == "" { + return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned no version", req.RegisteredModel)) + } + if !modelVersionRe().MatchString(resp.ModelVersion) { + return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned %q, which isn't a version number", req.RegisteredModel, stripControl(resp.ModelVersion))) + } + if err := checkStagingStatus(resp.StagingStatus, req.RegisteredModel, resp.ModelVersion); err != nil { + return nil, err + } + progress(&resp) + if resp.Ready { + return &resp, nil + } + if err := p.waitForWeights(ctx, req.RegisteredModel, &resp, progress); err != nil { + return &resp, err + } + if resp.weightsState() == stagingStateFailed { + return &resp, errStagingFailed + } + var finalized finalizeModelPackageResponse + finalizeReq := finalizeModelPackageRequest{RegisteredModel: req.RegisteredModel, ModelVersion: resp.ModelVersion} + if err := p.post(ctx, finalizeModelPackagePath, finalizeReq, &finalized); err != nil { + return &resp, &finalizeError{err} + } + if finalized.ModelVersion != resp.ModelVersion { + return &resp, &finalizeError{withCode(codeInternalError, true, fmt.Errorf("the server finalized version %q instead", stripControl(finalized.ModelVersion)))} + } + if err := checkStagingStatus(finalized.StagingStatus, req.RegisteredModel, resp.ModelVersion); err != nil { + return &resp, &finalizeError{err} + } + resp.Ready = true + resp.StagingStatus = finalized.StagingStatus + return &resp, nil +} + +// knownStagingStates are the artifact states the staging status API reports. +var knownStagingStates = []string{stagingStateNotStaged, stagingStateInProgress, stagingStateSucceeded, stagingStateFailed} + +// checkStagingStatus checks that status, from a response about version of model, +// reports a known state for both the weights and the serving environment, as the +// server always does. Anything else is a response air deploy can't act on, so it +// fails rather than being read as progress until the timeout. +func checkStagingStatus(status *modelStagingStatus, model, version string) error { + var problem string + switch { + case status == nil: + problem = "it has no staging status" + case status.ModelArtifact == nil || status.ModelArtifact.State == "": + problem = "its staging status has no state for the weights" + case !slices.Contains(knownStagingStates, status.ModelArtifact.State): + problem = fmt.Sprintf("its staging status has an unknown state %q for the weights", stripControl(status.ModelArtifact.State)) + case status.EnvironmentArtifact == nil || status.EnvironmentArtifact.State == "": + problem = "its staging status has no state for the serving environment" + case !slices.Contains(knownStagingStates, status.EnvironmentArtifact.State): + problem = fmt.Sprintf("its staging status has an unknown state %q for the serving environment", stripControl(status.EnvironmentArtifact.State)) + default: + return nil + } + return withCode(codeInternalError, true, fmt.Errorf("air deploy can't read Model Serving's response about version %s of %s: %s", stripControl(version), model, problem)) +} + +// modelServingPoller sends the requests of a registration that can take many +// minutes. +type modelServingPoller struct { + api *client.DatabricksClient + headers map[string]string + interval time.Duration +} + +// waitForWeights polls the staging status of resp's version of model, replacing +// resp's status with each one, until its weights are copied, failed to copy, or +// have no staging record. resp's status was checked, and so is each new one. +func (p modelServingPoller) waitForWeights(ctx context.Context, model string, resp *registerModelPackageResponse, progress func(*registerModelPackageResponse)) error { + statusReq := stagingStatusRequest{Model: model, ModelVersion: resp.ModelVersion} + for resp.weightsState() == stagingStateInProgress { + if err := sleepOrCancel(ctx, p.interval); err != nil { + return err + } + var status stagingStatusResponse + if err := p.post(ctx, stagingStatusPath, statusReq, &status); err != nil { + return err + } + if err := checkStagingStatus(status.Status, model, resp.ModelVersion); err != nil { + return err + } + resp.StagingStatus = status.Status + progress(resp) + } + return nil +} + +// post sends a request, repeating it after a transient failure the SDK doesn't +// retry: a wait of many minutes outlasts the SDK's own retries. Every request air +// deploy sends here is safe to repeat: the registration is idempotent per key, +// the status is a read, and finalizing a finalized version returns it. +func (p modelServingPoller) post(ctx context.Context, path string, body, out any) error { + _, err := retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { + return struct{}{}, p.api.Do(ctx, http.MethodPost, path, p.headers, nil, body, out) + }) + return err +} diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go new file mode 100644 index 00000000000..98dbd6aff74 --- /dev/null +++ b/cmd/air/deployregister_test.go @@ -0,0 +1,440 @@ +package aircmd + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Canned fake responses that stand for an error rather than a body. +const ( + conflictResponse = "conflict" + serverErrorResponse = "server error" + notFinalizableResponse = "not finalizable" +) + +// fakeModelServing serves registerModelPackage with responses, the staging status +// API with statuses, and finalizeModelPackage with finalizations, each in turn and +// then repeating the last; statuses default to SUCCEEDED, and finalizations to the +// finalized version. It records every request. Other paths are served by endpoints +// when set, else get an empty object (the SDK's config probe). +type fakeModelServing struct { + t *testing.T + mu sync.Mutex + responses []string + statuses []string + finalizations []string + bodies []registerModelPackageRequest + statusRequests []stagingStatusRequest + finalizeRequests []finalizeModelPackageRequest + endpoints http.HandlerFunc +} + +func newFakeModelServing(t *testing.T, responses ...string) *fakeModelServing { + return &fakeModelServing{t: t, responses: responses} +} + +func (f *fakeModelServing) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case registerModelPackagePath: + var body registerModelPackageRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.bodies = append(f.bodies, body) + f.respond(w, f.responses, len(f.bodies)) + case stagingStatusPath: + var body stagingStatusRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.statusRequests = append(f.statusRequests, body) + statuses := f.statuses + if len(statuses) == 0 { + statuses = []string{statusSucceeded} + } + f.respond(w, statuses, len(f.statusRequests)) + case finalizeModelPackagePath: + var body finalizeModelPackageRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.finalizeRequests = append(f.finalizeRequests, body) + finalizations := f.finalizations + if len(finalizations) == 0 { + finalizations = []string{finalizedResponse} + } + f.respond(w, finalizations, len(f.finalizeRequests)) + default: + if f.endpoints != nil { + f.endpoints(w, r) + return + } + _, _ = w.Write([]byte(`{}`)) + } +} + +// decode reads a request body into v, failing the request (not the handler +// goroutine) when it can't. +func (f *fakeModelServing) decode(w http.ResponseWriter, r *http.Request, v any) bool { + raw, err := io.ReadAll(r.Body) + if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, v)) { + w.WriteHeader(http.StatusInternalServerError) + return false + } + return true +} + +// respond writes the nth (1-based) of responses, repeating the last. +func (f *fakeModelServing) respond(w http.ResponseWriter, responses []string, n int) { + switch body := responses[min(n, len(responses))-1]; body { + case conflictResponse: + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The idempotency key can't be used for this registration. Use a new key."}`)) + case serverErrorResponse: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + case notFinalizableResponse: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error_code": "BAD_REQUEST", "message": "Version 3 of model main.models.qwen can't be finalized yet: the checkpoint's weights haven't been copied into it."}`)) + default: + _, _ = w.Write([]byte(body)) + } +} + +func (f *fakeModelServing) start() *httptest.Server { + srv := httptest.NewServer(f) + f.t.Cleanup(srv.Close) + return srv +} + +func (f *fakeModelServing) requests() []registerModelPackageRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]registerModelPackageRequest(nil), f.bodies...) +} + +const ( + stagingInProgress = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingSucceeded = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingReady = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": true, + "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingFailed = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + + statusInProgress = `{"status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + statusSucceeded = `{"status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + statusFailed = `{"status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + + finalizedResponse = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", + "staging_status": {"state": "IN_PROGRESS", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "IN_PROGRESS"}}}` +) + +// fastRetries shortens the pause before air deploy repeats a failed request. +func fastRetries(ctx context.Context) context.Context { + return env.Set(ctx, retry.IntervalMsVariable, "1") +} + +func checkpointRequest() registerModelPackageRequest { + return registerModelPackageRequest{ + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/models/weights/qwen3_8b", + ServingCommand: "python -m vllm.entrypoints.openai.api_server --model qwen3_8b", + Task: "llm/v1/chat", + Environment: modelPackageEnvironment{Runtime: "client.6-gpu"}, + }, + } +} + +func registerTestCheckpoint(t *testing.T, fake *fakeModelServing, req registerModelPackageRequest) (*registerModelPackageResponse, []string, error) { + t.Helper() + srv := fake.start() + var states []string + resp, err := registerAndFinalize(fastRetries(t.Context()), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { + states = append(states, r.weightsState()) + }) + return resp, states, err +} + +func TestRegisterAndFinalizePollsStagingThenFinalizes(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + fake.statuses = []string{statusInProgress, statusSucceeded} + req := checkpointRequest() + req.RetryFailedStaging = true + + resp, states, err := registerTestCheckpoint(t, fake, req) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Equal(t, "3", resp.ModelVersion) + assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) + + // One registration; the status is polled for the registered version until the + // weights are copied, and the version is then finalized. + assert.Equal(t, []registerModelPackageRequest{req}, fake.requests()) + assert.Equal(t, []stagingStatusRequest{{Model: "main.models.qwen", ModelVersion: "3"}, {Model: "main.models.qwen", ModelVersion: "3"}}, fake.statusRequests) + assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "main.models.qwen", ModelVersion: "3"}}, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReportsTheFinalizedStagingStatus(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + // The environment is still being staged when the version is finalized. + assert.Equal(t, stagingStateInProgress, resp.environmentState()) + assert.Empty(t, fake.statusRequests) +} + +func TestRegisterAndFinalizeReturnsAReadyVersion(t *testing.T) { + fake := newFakeModelServing(t, stagingReady) + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReportsFailedStaging(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + fake.statuses = []string{statusFailed} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.ErrorIs(t, err, errStagingFailed) + assert.Equal(t, "3", resp.ModelVersion) + assert.Len(t, fake.requests(), 1) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReportsAFailureTheRegistrationReports(t *testing.T) { + fake := newFakeModelServing(t, stagingFailed) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.ErrorIs(t, err, errStagingFailed) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReturnsWhyTheServerWontFinalize(t *testing.T) { + notStaged := strings.ReplaceAll(stagingInProgress, "IN_PROGRESS", "NOT_STAGED") + fake := newFakeModelServing(t, notStaged) + fake.finalizations = []string{notFinalizableResponse} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, "haven't been copied into it") + assert.False(t, resp.Ready) + // A version with no staging record isn't polled; the server says what to do. + assert.Empty(t, fake.statusRequests) + assert.Len(t, fake.finalizeRequests, 1) +} + +// Staging statuses that break the contract: the server always reports a known state +// for both the weights and the serving environment. +var malformedStagingStatuses = map[string]struct { + status string + wantErr string +}{ + "no status": {`null`, "it has no staging status"}, + "no weights": {`{"state": "IN_PROGRESS", "environment_artifact": {"state": "SUCCEEDED"}}`, "its staging status has no state for the weights"}, + "weights without a state": {`{"state": "IN_PROGRESS", "model_artifact": {}, "environment_artifact": {"state": "SUCCEEDED"}}`, "its staging status has no state for the weights"}, + "unknown weights state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "COPYING"}, "environment_artifact": {"state": "SUCCEEDED"}}`, `unknown state "COPYING" for the weights`}, + "no environment": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}}`, "its staging status has no state for the serving environment"}, + "unknown environment state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "BUILDING"}}`, `unknown state "BUILDING" for the serving environment`}, + "control characters in the state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "X\u001b[31m"}, "environment_artifact": {"state": "SUCCEEDED"}}`, `unknown state "X" for the weights`}, +} + +func TestRegisterAndFinalizeRejectsAMalformedRegistrationStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, "staging_status": `+tt.status+`}`) + + resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "air deploy can't read Model Serving's response about version 3 of main.models.qwen: ") + assert.ErrorContains(t, err, tt.wantErr) + assert.Nil(t, resp) + assert.Empty(t, states) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) + }) + } +} + +func TestRegisterAndFinalizeRejectsAMalformedPolledStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + fake.statuses = []string{`{"status": ` + tt.status + `}`, statusSucceeded} + + resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, tt.wantErr) + code, retryable := deployErrorCode(err) + assert.Equal(t, codeInternalError, code) + assert.True(t, retryable) + // It stops at the malformed status instead of reading it as progress. + assert.Equal(t, []string{"IN_PROGRESS"}, states) + assert.Len(t, fake.statusRequests, 1) + assert.Empty(t, fake.finalizeRequests) + assert.False(t, resp.Ready) + }) + } +} + +func TestRegisterAndFinalizeRejectsAMalformedFinalizedStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + fake.finalizations = []string{`{"model_version": "3", "staging_status": ` + tt.status + `}`} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, tt.wantErr) + assert.False(t, resp.Ready) + }) + } +} + +func TestRegisterAndFinalizeUsesTheFinalizedStatusAsItIs(t *testing.T) { + environmentFailed := strings.Replace(stagingSucceeded, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) + fake := newFakeModelServing(t, environmentFailed) + fake.finalizations = []string{`{"model_version": "3", "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}`} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + // The finalize response's status replaces the registration's, not merged with it. + assert.Equal(t, stagingStateSucceeded, resp.environmentState()) +} + +func TestRegisterAndFinalizeRejectsAnotherFinalizedVersion(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + fake.finalizations = []string{`{"model_version": "4"}`} + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, `the server finalized version "4" instead`) +} + +func TestRegisterAndFinalizeRejectsARegistrationWithoutAVersion(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "ready": false}`) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "registering a version of main.models.qwen returned no version") + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeRejectsAVersionThatIsntANumber(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3; rm", "ready": false, "staging_status": {"model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}`) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, `registering a version of main.models.qwen returned "3; rm", which isn't a version number`) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeRetriesTransientErrors(t *testing.T) { + fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress) + fake.statuses = []string{serverErrorResponse, statusSucceeded} + fake.finalizations = []string{serverErrorResponse, finalizedResponse} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Len(t, fake.requests(), 2) + assert.Len(t, fake.statusRequests, 2) + assert.Len(t, fake.finalizeRequests, 2) +} + +func TestRegisterAndFinalizeGivesUpOnPersistentErrors(t *testing.T) { + fake := newFakeModelServing(t, serverErrorResponse) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "Something went wrong.") + assert.Len(t, fake.requests(), retry.MaxRetries+1) +} + +func TestRegisterAndFinalizeReturnsServerErrors(t *testing.T) { + fake := newFakeModelServing(t, conflictResponse) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "The idempotency key can't be used for this registration") + assert.Len(t, fake.requests(), 1) +} + +func TestRegisterAndFinalizeStopsWhenCanceled(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + srv := fake.start() + ctx, cancel := context.WithCancel(t.Context()) + + _, err := registerAndFinalize(ctx, newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Hour, func(*registerModelPackageResponse) { cancel() }) + assert.ErrorIs(t, err, context.Canceled) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegistrationRequestForCommandJSON(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") + require.NoError(t, err) + raw, err := json.Marshal(cfg.registrationRequest("k")) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "main.models.qwen", + "idempotency_key": "k", + "checkpoint": { + "volume_path": "/Volumes/main/models/weights/qwen3_8b", + "serving_command": "python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", + "task": "llm/v1/chat", + "environment": {"runtime": "client.6-gpu", "dependencies": ["einops==0.8.0"]} + }, + "experiment_path": "/Shared/qwen" + }`, string(raw)) +} + +func TestRegistrationRequestForDedicatedJSON(t *testing.T) { + cfg, err := loadDeployConfigText(t, dedicatedDeployConfig) + require.NoError(t, err) + raw, err := json.Marshal(cfg.registrationRequest("k")) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "main.models.qwen", + "idempotency_key": "k", + "dedicated_checkpoint": {"volume_path": "/Volumes/main/models/weights/qwen3_8b"} + }`, string(raw)) +} + +func TestRegisterModelPackageRequestJSON(t *testing.T) { + raw, err := json.Marshal(checkpointRequest()) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "main.models.qwen", + "idempotency_key": "key-1", + "checkpoint": { + "volume_path": "/Volumes/main/models/weights/qwen3_8b", + "serving_command": "python -m vllm.entrypoints.openai.api_server --model qwen3_8b", + "task": "llm/v1/chat", + "environment": {"runtime": "client.6-gpu"} + } + }`, string(raw)) +} diff --git a/cmd/air/run.go b/cmd/air/run.go index 20a7683461f..6492764d3c0 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -131,7 +131,7 @@ field, pass its path to -h as a separate argument: if field.path == "config.compute" || field.path == "config.compute.accelerator_type" { computeOptions = loadComputeOptionsForHelp(c, args) } - renderConfigField(c.OutOrStdout(), field, computeOptions) + renderConfigField(c.OutOrStdout(), runCommandPath, field, computeOptions) }) // List flags in declaration order (most used first) rather than alphabetically. @@ -170,7 +170,7 @@ field, pass its path to -h as a separate argument: } snapshotFileCount = &count } - idempotencyToken := effectiveIdempotencyToken(idempotencyKey, cfg) + idempotencyToken := effectiveIdempotencyToken(idempotencyKey, cfg.IdempotencyToken) ctx, validationErr := validateDryRunInWorkspace(ctx, cmd, args, cfg, idempotencyToken, validationTimeout) if err := validateIdempotencyToken(idempotencyToken); err != nil { return err diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index 78554b02183..7306c1ab3c6 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -119,18 +119,10 @@ func (c *runConfig) validate() error { } } - if err := validateSecretRefs(c.Secrets); err != nil { + if err := validateEnvironmentVariables(c.EnvVariables, c.Secrets); err != nil { return err } - // A name can't be both a plain env var and a secret: the precedence would be - // ambiguous and could leak the secret. Sorted for a stable error. - for _, name := range slices.Sorted(maps.Keys(c.EnvVariables)) { - if _, ok := c.Secrets[name]; ok { - return fmt.Errorf("%q is set in both env_variables and secrets; remove it from one", name) - } - } - if c.CodeSource != nil { if err := c.CodeSource.validate(); err != nil { return err @@ -335,6 +327,23 @@ func validateCommand(v string) error { return nil } +// validateEnvironmentVariables checks the env_variables and secrets of a config: +// secret references use the "scope/key" format, and no name is in both. +func validateEnvironmentVariables(envVariables, secrets map[string]string) error { + if err := validateSecretRefs(secrets); err != nil { + return err + } + + // A name can't be both a plain env var and a secret: the precedence would be + // ambiguous and could leak the secret. Sorted for a stable error. + for _, name := range slices.Sorted(maps.Keys(envVariables)) { + if _, ok := secrets[name]; ok { + return fmt.Errorf("%q is set in both env_variables and secrets; remove it from one", name) + } + } + return nil +} + // validateSecretRefs checks that secret references use the "scope/key" format. func validateSecretRefs(secrets map[string]string) error { for varName, ref := range secrets { @@ -608,13 +617,16 @@ func (p *permission) validate() error { // or bare `compute`). const configHelpRoot = "config" +// runCommandPath is the command whose -h documents runConfig. +const runCommandPath = "databricks air run" + // writeConfigFieldHelp resolves a dotted config path and writes its docs. func writeConfigFieldHelp(w io.Writer, path string) error { field, err := resolveConfigField(path) if err != nil { return err } - renderConfigField(w, field, nil) + renderConfigField(w, runCommandPath, field, nil) return nil } @@ -675,8 +687,13 @@ func configSchema() configField { // resolveConfigField resolves a dotted YAML path against the run config schema. // The leading "config." is optional; an empty path describes the whole schema. func resolveConfigField(path string) (configField, error) { + return resolveSchemaField(configSchema(), path) +} + +// resolveSchemaField resolves a dotted YAML path against a config schema rooted +// at root, as resolveConfigField does for the run config. +func resolveSchemaField(root configField, path string) (configField, error) { trimmed := strings.TrimPrefix(strings.TrimPrefix(path, configHelpRoot), ".") - root := configSchema() if trimmed == "" { return root, nil } @@ -867,7 +884,8 @@ func underlyingConfigStruct(t reflect.Type) reflect.Type { // renderConfigField writes a resolved field's documentation. An object lists its // immediate children; a leaf gets its type, required-ness, and description. -func renderConfigField(w io.Writer, f configField, computeOptions []computeOption) { +// command is the command whose -h documents the schema. +func renderConfigField(w io.Writer, command string, f configField, computeOptions []computeOption) { fmt.Fprintf(w, "%s\n", f.path) if f.help != "" { fmt.Fprintf(w, " %s\n", f.help) @@ -916,8 +934,9 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio fmt.Fprintf(w, " %-*s %s\n", width, configLeafName(c.path), configFieldSummary(c)) } } - fmt.Fprintf(w, "\nFor details on a field, run \"databricks air run -h %s.\".\n", f.path) - if f.path == configHelpRoot { + fmt.Fprintf(w, "\nFor details on a field, run \"%s -h %s.\".\n", command, f.path) + // The skill writes run YAML, so only `air run -h config` points at it. + if f.path == configHelpRoot && command == runCommandPath { fmt.Fprint(w, agentSkillHelp) } } diff --git a/cmd/air/runconfig_load.go b/cmd/air/runconfig_load.go index 2e05e1b50e6..e272bc8f096 100644 --- a/cmd/air/runconfig_load.go +++ b/cmd/air/runconfig_load.go @@ -18,10 +18,16 @@ import ( // decodeRunConfigReader decodes and unknown-key-checks a run YAML from r. path is // used only for error messages. func decodeRunConfigReader(r io.Reader, path string) (*runConfig, error) { + return decodeConfigReader[runConfig](r, path) +} + +// decodeConfigReader decodes and unknown-key-checks a YAML config from r into a +// T. path is used only for error messages. +func decodeConfigReader[T any](r io.Reader, path string) (*T, error) { dec := yaml.NewDecoder(r) dec.KnownFields(true) - var cfg runConfig + var cfg T if err := dec.Decode(&cfg); err != nil { if errors.Is(err, io.EOF) { return nil, fmt.Errorf("config %s is empty", path) @@ -59,30 +65,41 @@ func loadRunConfigWithOverrides(ctx context.Context, path string, overrides []st if len(overrides) == 0 { return loadRunConfig(path) } + document, validationYAML, err := readConfigWithOverrides(ctx, path, overrides, configSchema()) + if err != nil { + return nil, err + } + return finishRunConfigLoad(path, document, validationYAML) +} +// readConfigWithOverrides reads a YAML config and applies --override KEY=VALUE +// entries, checked against schema, to its ordered YAML tree. It returns the tree +// and its serialization, which the caller decodes with unknown keys rejected. +// ctx is used only to log applied overrides. +func readConfigWithOverrides(ctx context.Context, path string, overrides []string, schema configField) (*yaml.Node, []byte, error) { entries, err := parseOverrides(overrides) if err != nil { - return nil, err + return nil, nil, err } - if err := validateOverridePaths(entries); err != nil { - return nil, err + if err := validateOverridePaths(entries, schema); err != nil { + return nil, nil, err } document, _, err := readRunConfigDocument(path) if err != nil { - return nil, err + return nil, nil, err } if len(document.Content) == 0 { document.Kind = yaml.DocumentNode document.Content = []*yaml.Node{newMappingNode()} } if err := applyOverrides(ctx, document, entries); err != nil { - return nil, err + return nil, nil, err } validationYAML, err := yaml.Marshal(document) if err != nil { - return nil, fmt.Errorf("failed to serialize config %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to serialize config %s: %w", path, err) } - return finishRunConfigLoad(path, document, validationYAML) + return document, validationYAML, nil } func readRunConfigDocument(path string) (*yaml.Node, []byte, error) { diff --git a/cmd/air/runconfig_override.go b/cmd/air/runconfig_override.go index 3fbccd076cb..d7585c8214a 100644 --- a/cmd/air/runconfig_override.go +++ b/cmd/air/runconfig_override.go @@ -45,11 +45,10 @@ type overrideEntry struct { raw string } -// validateOverridePaths checks every dotted path against the runConfig schema +// validateOverridePaths checks every dotted path against a config schema // before mutation, so an error names the exact --override key rather than the // re-decode's Go-type language. -func validateOverridePaths(entries []overrideEntry) error { - schema := configSchema() +func validateOverridePaths(entries []overrideEntry, schema configField) error { for _, e := range entries { if err := checkOverridePath(strings.Split(e.path, "."), schema, e.path); err != nil { return err @@ -100,7 +99,7 @@ func applyOverrides(ctx context.Context, document *yaml.Node, entries []override root = document.Content[0] } if root.Kind != yaml.MappingNode { - return errors.New("run config must be a YAML mapping") + return errors.New("config must be a YAML mapping") } for _, e := range entries { diff --git a/cmd/air/runconfig_override_test.go b/cmd/air/runconfig_override_test.go index c0dc4937138..982f9ad74a0 100644 --- a/cmd/air/runconfig_override_test.go +++ b/cmd/air/runconfig_override_test.go @@ -90,7 +90,7 @@ func TestValidateOverridePaths(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validateOverridePaths([]overrideEntry{{path: tt.path, raw: "x"}}) + err := validateOverridePaths([]overrideEntry{{path: tt.path, raw: "x"}}, configSchema()) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index 74b4c187628..64413f5d789 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -1057,14 +1057,14 @@ func TestConfigSchemaSharedByHelpAndOverride(t *testing.T) { for _, p := range []string{"code_source.type", "code_source.snapshot.root_path"} { t.Run(p+" compatibility override", func(t *testing.T) { - require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}})) + require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}}, configSchema())) _, err := resolveConfigField(p) require.Error(t, err) }) } for _, p := range paths { t.Run(p, func(t *testing.T) { - require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}})) + require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}}, configSchema())) // The help path accepts the same field (free-form sub-paths resolve to // the map itself, which is the schema's most specific node). _, err := resolveConfigField(p) @@ -1075,7 +1075,7 @@ func TestConfigSchemaSharedByHelpAndOverride(t *testing.T) { } // A field unknown to one is unknown to the other. - require.Error(t, validateOverridePaths([]overrideEntry{{path: "compute.bogus", raw: "x"}})) + require.Error(t, validateOverridePaths([]overrideEntry{{path: "compute.bogus", raw: "x"}}, configSchema())) _, err := resolveConfigField("compute.bogus") require.Error(t, err) } diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index c06165dd2b4..8ee0beca466 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -270,10 +270,10 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) { // effectiveIdempotencyToken resolves the token sent to validation and submit: // the --idempotency-key flag wins, then the config's token, else a generated one. -func effectiveIdempotencyToken(flag string, cfg *runConfig) string { +func effectiveIdempotencyToken(flag string, configured *string) string { token := flag - if token == "" && cfg.IdempotencyToken != nil { - token = *cfg.IdempotencyToken + if token == "" && configured != nil { + token = *configured } if token == "" { token = uuid.New().String() @@ -345,7 +345,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } containers := submittedContainers(cfg, funcDir) - token := effectiveIdempotencyToken(idempotencyKey, cfg) + token := effectiveIdempotencyToken(idempotencyKey, cfg.IdempotencyToken) deps, _ := cfg.inlineDependencies() diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index e101bbc28af..daa87788e8b 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -33,9 +33,9 @@ func stubValidateConfig(server *testserver.Server) { func TestEffectiveIdempotencyToken(t *testing.T) { cfg := &runConfig{IdempotencyToken: new("from-config")} - assert.Equal(t, "from-flag", effectiveIdempotencyToken("from-flag", cfg)) - assert.Equal(t, "from-config", effectiveIdempotencyToken("", cfg)) - assert.NotEmpty(t, effectiveIdempotencyToken("", &runConfig{})) + assert.Equal(t, "from-flag", effectiveIdempotencyToken("from-flag", cfg.IdempotencyToken)) + assert.Equal(t, "from-config", effectiveIdempotencyToken("", cfg.IdempotencyToken)) + assert.NotEmpty(t, effectiveIdempotencyToken("", nil)) } func TestDlRuntimeImage(t *testing.T) { diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index c1dc9c5f89d..3a4f6ff6ddd 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -146,7 +146,9 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, if output, err := cmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) } - args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} + // -C makes ls-files list paths relative to repoPath; without it they are + // relative to the process's working directory, which may be a subdirectory. + args = []string{"-C", repoPath, "--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} } if len(includePaths) > 0 { diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index 068997e2671..37c7bdbfec3 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -324,3 +324,21 @@ func TestSnapshotFiles_RejectsIncludePathsMatchingNothing(t *testing.T) { assert.Contains(t, err.Error(), "contains no files") assert.Contains(t, err.Error(), "include_paths") } + +func TestSnapshotFilesOutsideGitFromASubdirectory(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, ".gitignore", "/creds.json\n") + writeRepoFile(t, dir, "creds.json", "{}") + writeRepoFile(t, dir, "agent.py", "x") + writeRepoFile(t, dir, "tools/a.py", "a") + // The listing must be relative to the root, not to the working directory. + t.Chdir(filepath.Join(dir, "tools")) + + files, err := snapshotFiles(t.Context(), dir, nil, false) + require.NoError(t, err) + var rels []string + for _, f := range files { + rels = append(rels, filepath.ToSlash(f.rel)) + } + assert.ElementsMatch(t, []string{".gitignore", "agent.py", "tools/a.py"}, rels) +}