diff --git a/.nextchanges/air/accelerator-pool-id.md b/.nextchanges/air/accelerator-pool-id.md new file mode 100644 index 00000000000..ca485e47e72 --- /dev/null +++ b/.nextchanges/air/accelerator-pool-id.md @@ -0,0 +1 @@ +* Breaking: `compute.pool_id` is no longer accepted in AIR workload YAML; use `compute.accelerator_pool_id`. ([#7698](https://github.com/databricks/cli/pull/7698)) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 5df22e08da8..fca01cb9674 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -113,12 +113,12 @@ config.compute GPU_1xRTX6000 - 1 accelerator per node Fields: - num_accelerators (required) Total number of GPUs to allocate. - accelerator_type (required) Which accelerator to run on, e.g. GPU_1xA10. + num_accelerators (required) Total number of GPUs to allocate. + accelerator_type (required) Which accelerator to run on, e.g. GPU_1xA10. - Provisioned capacity (optional): - pool_id GPU pool to run on, by id. - priority_class Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. + GPU Pools (optional): + accelerator_pool_id GPU Pool to run on, by ID. + priority_class Scheduling priority within the GPU Pool: NORMAL. For details on a field, run "databricks air run -h config.compute.". @@ -131,16 +131,16 @@ config.mlflow_artifact_location Type: string Required: no ->>> [CLI] air run -h config.compute.pool_id -config.compute.pool_id - GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'databricks air pools list'. +>>> [CLI] air run -h config.compute.accelerator_pool_id +config.compute.accelerator_pool_id + GPU Pool to run on, by ID. Must be 1-255 characters. List available pools with 'databricks air pools list'. Type: string Required: no >>> [CLI] air run -h config.compute.priority_class config.compute.priority_class - Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. Requires pool_id. + Scheduling priority within the GPU Pool: NORMAL. Requires accelerator_pool_id. Type: string Required: no @@ -231,9 +231,16 @@ config.compute.num_accelerators >>> [CLI] air run -h config.compute.acclerator_type Error: unknown config field "config.compute.acclerator_type"; did you mean "accelerator_type"? -fields under "config.compute" are: accelerator_type, num_accelerators, pool_id, priority_class +fields under "config.compute" are: accelerator_pool_id, accelerator_type, num_accelerators, priority_class === free-form map keys are not schema fields >>> [CLI] air run -h config.parameters.learning_rate Error: "config.parameters" holds user-defined keys, so "learning_rate" is not part of the schema; see "config.parameters" instead + +=== removed pool field is not in the schema + +>>> [CLI] air run -h config.compute.pool_id +Error: unknown config field "config.compute.pool_id"; did you mean "accelerator_pool_id"? + +fields under "config.compute" are: accelerator_pool_id, accelerator_type, num_accelerators, priority_class diff --git a/acceptance/air/config-help/script b/acceptance/air/config-help/script index 4f76006eeec..41d9d12a764 100644 --- a/acceptance/air/config-help/script +++ b/acceptance/air/config-help/script @@ -14,7 +14,7 @@ trace $CLI air run -h config.compute title "new submission fields are documented" trace $CLI air run -h config.mlflow_artifact_location -trace $CLI air run -h config.compute.pool_id +trace $CLI air run -h config.compute.accelerator_pool_id trace $CLI air run -h config.compute.priority_class trace $CLI air run -h config.environment.dependencies trace $CLI air run -h config.environment.version @@ -39,3 +39,6 @@ trace $CLI air run -h config.compute.acclerator_type title "free-form map keys are not schema fields" trace $CLI air run -h config.parameters.learning_rate + +title "removed pool field is not in the schema" +trace $CLI air run -h config.compute.pool_id diff --git a/acceptance/air/convert-to-dabs/train.yaml b/acceptance/air/convert-to-dabs/train.yaml index 63ea29014c2..0bd01bd530e 100644 --- a/acceptance/air/convert-to-dabs/train.yaml +++ b/acceptance/air/convert-to-dabs/train.yaml @@ -5,7 +5,7 @@ command: | compute: accelerator_type: GPU_1xA10 num_accelerators: 1 - pool_id: capacity-1 + accelerator_pool_id: capacity-1 priority_class: NORMAL mlflow_artifact_location: dbfs:/Volumes/main/default/artifacts environment: diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 83c7659f07d..fc81eb63c1f 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -25,7 +25,7 @@ Package and deploy: images Push container images for custom environments Provisioned capacity: - pools View provisioned GPU pools, if any (not required to run) + pools View provisioned GPU Pools, if any (not required to run) Flags: -h, --help[=true|false] help for air @@ -107,19 +107,19 @@ Global Flags: === pools help >>> [CLI] air pools --help -View GPU pools provisioned for your workspace. +View GPU Pools provisioned for your workspace. -A GPU pool is dedicated accelerator capacity that your Databricks account team +A GPU Pool is dedicated accelerator capacity that your Databricks account team provisions. Pools are optional: a run uses one only when its YAML sets -compute.pool_id. +compute.accelerator_pool_id. Usage: databricks air pools [flags] databricks air pools [command] Available Commands: - get Show a GPU pool and its accelerator usage - list List the GPU pools available to the current workspace + get Show a GPU Pool and its accelerator usage + list List the GPU Pools available to the current workspace Flags: -h, --help[=true|false] help for pools diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index ae716c0bdfd..17825adf7c4 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -14,7 +14,7 @@ Stream logs after submission using: >>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep {"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} -{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n accelerator_pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} === inline deps, artifact location, and capacity id reach the submit payload @@ -45,8 +45,8 @@ Stream logs after submission using: "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "compute": { "accelerator_count": 1, - "accelerator_type": "GPU_1xH100", - "provisioned_capacity_id": "capacity-123" + "accelerator_pool_id": "capacity-123", + "accelerator_type": "GPU_1xH100" } } ], diff --git a/acceptance/air/run-submit-deps/run.yaml b/acceptance/air/run-submit-deps/run.yaml index deb17acf080..010e3e93c42 100644 --- a/acceptance/air/run-submit-deps/run.yaml +++ b/acceptance/air/run-submit-deps/run.yaml @@ -3,7 +3,7 @@ command: python train.py compute: accelerator_type: GPU_1xH100 num_accelerators: 1 - pool_id: capacity-123 + accelerator_pool_id: capacity-123 environment: version: 5 dependencies: diff --git a/acceptance/air/run/invalid-priority.yaml b/acceptance/air/run/invalid-priority.yaml new file mode 100644 index 00000000000..e4cafb95290 --- /dev/null +++ b/acceptance/air/run/invalid-priority.yaml @@ -0,0 +1,7 @@ +experiment_name: smoke-test +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 + accelerator_pool_id: capacity-1 + priority_class: HIGH diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index e9ea843f2cf..3184a042cbe 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -108,3 +108,27 @@ Exit code: 1 Error: the AIR commands have moved out of experimental; use "databricks air" instead Exit code: 1 + +=== removed pool field is rejected + +>>> [CLI] air run -f removed-pool.yaml --dry-run +Error: invalid config removed-pool.yaml: line 6: unknown field "compute.pool_id"; did you mean "accelerator_pool_id"? Run "databricks air run -h config.compute" for the list of fields + +Tip: use the --debug flag to see more details and a trace of this error: + databricks --debug air run … + +=== unsupported priority is rejected + +>>> [CLI] air run -f invalid-priority.yaml --dry-run +Error: compute.priority_class: invalid priority_class "HIGH": valid values are: NORMAL + +Tip: use the --debug flag to see more details and a trace of this error: + databricks --debug air run … + +=== priority without a pool is rejected + +>>> [CLI] air run -f priority-without-pool.yaml --dry-run +Error: compute.priority_class requires compute.accelerator_pool_id — priority applies only within a GPU Pool + +Tip: use the --debug flag to see more details and a trace of this error: + databricks --debug air run … diff --git a/acceptance/air/run/priority-without-pool.yaml b/acceptance/air/run/priority-without-pool.yaml new file mode 100644 index 00000000000..8123b31b8b5 --- /dev/null +++ b/acceptance/air/run/priority-without-pool.yaml @@ -0,0 +1,6 @@ +experiment_name: smoke-test +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 + priority_class: NORMAL diff --git a/acceptance/air/run/removed-pool.yaml b/acceptance/air/run/removed-pool.yaml new file mode 100644 index 00000000000..bfee90c8402 --- /dev/null +++ b/acceptance/air/run/removed-pool.yaml @@ -0,0 +1,6 @@ +experiment_name: smoke-test +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 + pool_id: capacity-1 diff --git a/acceptance/air/run/script b/acceptance/air/run/script index 467133f342a..fb89b161496 100644 --- a/acceptance/air/run/script +++ b/acceptance/air/run/script @@ -47,3 +47,12 @@ errcode trace $CLI air run --dry-run title "experimental path directs users to the top-level command" errcode trace $CLI experimental air run -f valid.yaml --dry-run + +title "removed pool field is rejected" +musterr trace $CLI air run -f removed-pool.yaml --dry-run + +title "unsupported priority is rejected" +musterr trace $CLI air run -f invalid-priority.yaml --dry-run + +title "priority without a pool is rejected" +musterr trace $CLI air run -f priority-without-pool.yaml --dry-run diff --git a/cmd/air/compute.go b/cmd/air/compute.go index 97c0b3b7a29..f223d9fd4dc 100644 --- a/cmd/air/compute.go +++ b/cmd/air/compute.go @@ -39,16 +39,9 @@ const ( priorityClassNormal priorityClass = "NORMAL" ) -// priorityClasses lists every valid value, low to high. Used for validation -// error messages. -var priorityClasses = []priorityClass{priorityClassBestEffort, priorityClassNormal} - func validPriorityClassesHint() string { - names := make([]string, len(priorityClasses)) - for i, p := range priorityClasses { - names[i] = string(p) - } - return "valid values are: " + strings.Join(names, ", ") + // BEST_EFFORT remains accepted for compatibility with existing YAML. + return "valid values are: " + string(priorityClassNormal) } // parsePriorityClass normalizes a YAML priority_class to the enum's contract @@ -80,10 +73,10 @@ func gpusPerNode(g gpuType) (int, error) { // computeConfig is the `compute` block of the run YAML: which accelerators to // use and how many. type computeConfig struct { - NumAccelerators int `yaml:"num_accelerators" help:"Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See Docs below for GPUs per node." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` - AcceleratorType string `yaml:"accelerator_type" help:"Which accelerator to run on, e.g. GPU_1xA10. Matched case-sensitively." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` - PoolID *string `yaml:"pool_id" help:"GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'databricks air pools list'." group:"Provisioned capacity (optional)"` - PriorityClass *string `yaml:"priority_class" help:"Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. Requires pool_id." group:"Provisioned capacity (optional)"` + NumAccelerators int `yaml:"num_accelerators" help:"Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See Docs below for GPUs per node." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` + AcceleratorType string `yaml:"accelerator_type" help:"Which accelerator to run on, e.g. GPU_1xA10. Matched case-sensitively." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` + AcceleratorPoolID *string `yaml:"accelerator_pool_id" help:"GPU Pool to run on, by ID. Must be 1-255 characters. List available pools with 'databricks air pools list'." group:"GPU Pools (optional)"` + PriorityClass *string `yaml:"priority_class" help:"Scheduling priority within the GPU Pool: NORMAL. Requires accelerator_pool_id." group:"GPU Pools (optional)"` } // validate checks the compute block against the backend's constraints. @@ -105,15 +98,15 @@ func (c *computeConfig) validate() error { return fmt.Errorf("compute.num_accelerators for %s must be a multiple of %d, got %d", c.AcceleratorType, perNode, c.NumAccelerators) } - if c.PoolID != nil { - v := strings.TrimSpace(*c.PoolID) + if c.AcceleratorPoolID != nil { + v := strings.TrimSpace(*c.AcceleratorPoolID) if v == "" { - return errors.New("compute.pool_id cannot be empty") + return errors.New("compute.accelerator_pool_id cannot be empty") } if len(v) > 255 { - return fmt.Errorf("compute.pool_id must be 255 characters or less, got %d", len(v)) + return fmt.Errorf("compute.accelerator_pool_id must be 255 characters or less, got %d", len(v)) } - *c.PoolID = v + *c.AcceleratorPoolID = v } if c.PriorityClass != nil { @@ -123,8 +116,8 @@ func (c *computeConfig) validate() error { } // A priority class only ranks pending work within a pool, so it requires // one. - if c.PoolID == nil { - return errors.New("compute.priority_class requires compute.pool_id — priority applies only within a GPU pool") + if c.AcceleratorPoolID == nil { + return errors.New("compute.priority_class requires compute.accelerator_pool_id — priority applies only within a GPU Pool") } *c.PriorityClass = string(p) } diff --git a/cmd/air/compute_test.go b/cmd/air/compute_test.go index 9a8451b0112..2a15307521a 100644 --- a/cmd/air/compute_test.go +++ b/cmd/air/compute_test.go @@ -75,17 +75,18 @@ func TestParsePriorityClass(t *testing.T) { assert.Equal(t, priorityClassNormal, got) } - for _, in := range []string{"urgent", "", "high", "CRITICAL"} { + for _, in := range []string{"urgent", "", "HIGH", "high", "CRITICAL"} { _, err := parsePriorityClass(in) require.Error(t, err) - assert.Contains(t, err.Error(), "valid values are") + assert.ErrorContains(t, err, "valid values are: NORMAL") + assert.NotContains(t, err.Error(), "BEST_EFFORT") } } func TestComputeConfigValidateNormalizesPriorityClass(t *testing.T) { // A lower-case value is upper-cased in place so the submitted payload carries // the enum's contract value. - cfg := computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new("cap"), PriorityClass: new("best_effort")} + cfg := computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new("cap"), PriorityClass: new("best_effort")} require.NoError(t, cfg.validate()) assert.Equal(t, "BEST_EFFORT", *cfg.PriorityClass) } @@ -99,9 +100,9 @@ func TestComputeConfigValidate(t *testing.T) { {"single node", computeConfig{NumAccelerators: 8, AcceleratorType: "GPU_8xH100"}, ""}, {"multiple nodes", computeConfig{NumAccelerators: 16, AcceleratorType: "GPU_8xH100"}, ""}, {"single-gpu partitions", computeConfig{NumAccelerators: 3, AcceleratorType: "GPU_1xH100"}, ""}, - {"pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new(" capacity ")}, ""}, - {"empty pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new(" ")}, "cannot be empty"}, - {"long pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new(strings.Repeat("a", 256))}, "255 characters or less"}, + {"pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new(" capacity ")}, ""}, + {"empty pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new(" ")}, "cannot be empty"}, + {"long pool id", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new(strings.Repeat("a", 256))}, "255 characters or less"}, {"unknown type", computeConfig{NumAccelerators: 8, AcceleratorType: "b200"}, "accelerator_type"}, {"legacy type rejected", computeConfig{NumAccelerators: 8, AcceleratorType: "h100_80gb"}, "accelerator_type"}, {"non-positive count", computeConfig{NumAccelerators: 0, AcceleratorType: "GPU_1xH100"}, "must be positive"}, @@ -117,9 +118,9 @@ func TestComputeConfigValidate(t *testing.T) { {"B300 count twelve", computeConfig{NumAccelerators: 12, AcceleratorType: "GPU_8xB300"}, "multiple of 8"}, {"B300 zero", computeConfig{NumAccelerators: 0, AcceleratorType: "GPU_8xB300"}, "must be positive"}, {"B300 negative", computeConfig{NumAccelerators: -8, AcceleratorType: "GPU_8xB300"}, "must be positive"}, - {"priority class", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new("cap"), PriorityClass: new("normal")}, ""}, - {"priority class requires reservation", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PriorityClass: new("NORMAL")}, "requires compute.pool_id"}, - {"invalid priority class", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PoolID: new("cap"), PriorityClass: new("urgent")}, "invalid priority_class"}, + {"priority class", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new("cap"), PriorityClass: new("normal")}, ""}, + {"priority class requires reservation", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", PriorityClass: new("NORMAL")}, "requires compute.accelerator_pool_id"}, + {"invalid priority class", computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new("cap"), PriorityClass: new("urgent")}, "invalid priority_class"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -133,3 +134,37 @@ func TestComputeConfigValidate(t *testing.T) { }) } } + +func TestComputeConfigAcceleratorPool(t *testing.T) { + for _, tc := range []struct { + name string + poolID *string + want *string + wantErr string + }{ + {name: "omitted"}, + {name: "explicit", poolID: new("pool-1"), want: new("pool-1")}, + {name: "trimmed", poolID: new(" pool-1 "), want: new("pool-1")}, + {name: "empty", poolID: new(""), wantErr: "compute.accelerator_pool_id cannot be empty"}, + {name: "blank", poolID: new(" "), wantErr: "compute.accelerator_pool_id cannot be empty"}, + {name: "maximum length", poolID: new(strings.Repeat("x", 255)), want: new(strings.Repeat("x", 255))}, + {name: "too long", poolID: new(strings.Repeat("x", 256)), wantErr: "compute.accelerator_pool_id must be 255 characters or less"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: tc.poolID} + err := cfg.validate() + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, cfg.AcceleratorPoolID) + }) + } +} + +func TestComputeConfigPriorityWithAcceleratorPool(t *testing.T) { + cfg := computeConfig{NumAccelerators: 1, AcceleratorType: "GPU_1xH100", AcceleratorPoolID: new("pool-1"), PriorityClass: new("normal")} + require.NoError(t, cfg.validate()) + assert.Equal(t, "NORMAL", *cfg.PriorityClass) +} diff --git a/cmd/air/convert_to_dabs.go b/cmd/air/convert_to_dabs.go index 363bc07e4d7..1a861325131 100644 --- a/cmd/air/convert_to_dabs.go +++ b/cmd/air/convert_to_dabs.go @@ -302,8 +302,9 @@ func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourc "accelerator_type": nv(cfg.Compute.AcceleratorType, 1), "accelerator_count": nv(cfg.Compute.NumAccelerators, 2), } - if cfg.Compute.PoolID != nil { - compute["provisioned_capacity_id"] = nv(*cfg.Compute.PoolID, 3) + // Bundles still use the legacy field supported by their current schema. + if poolID := cfg.Compute.AcceleratorPoolID; poolID != nil { + compute["provisioned_capacity_id"] = nv(*poolID, 3) } deployment := map[string]dyn.Value{ "command_path": nv(localBundlePath(path.Join(generatedArtifactsDir, commandScriptName)), 1), diff --git a/cmd/air/convert_to_dabs_test.go b/cmd/air/convert_to_dabs_test.go index 1ac0a4f8c3c..0c71154b303 100644 --- a/cmd/air/convert_to_dabs_test.go +++ b/cmd/air/convert_to_dabs_test.go @@ -110,7 +110,7 @@ command: python train.py compute: accelerator_type: GPU_1xH100 num_accelerators: 1 - pool_id: capacity-1 + accelerator_pool_id: capacity-1 max_retries: 2 timeout_minutes: 30 mlflow_run_name: run-42 @@ -203,7 +203,7 @@ func TestConvertToDabsPreservesPoolPriorityAndUnityCatalogImage(t *testing.T) { cfg := minimalConfig var computeOptions string if tc.poolID != "" { - computeOptions += " pool_id: " + tc.poolID + "\n" + computeOptions += " accelerator_pool_id: " + tc.poolID + "\n" } if tc.priorityClass != "" { computeOptions += " priority_class: " + tc.priorityClass + "\n" diff --git a/cmd/air/pools.go b/cmd/air/pools.go index bdfd4488786..f49e1c086ba 100644 --- a/cmd/air/pools.go +++ b/cmd/air/pools.go @@ -20,9 +20,9 @@ import ( "github.com/spf13/cobra" ) -// A "GPU pool" is a pre-provisioned AI Runtime accelerator reservation — the -// same reservation a run targets via compute.provisioned_capacity_id. "GPU -// pool" (or "pool") is the product-facing name used across the Compute page and +// A "GPU Pool" is a pre-provisioned AI Runtime accelerator reservation — the +// same reservation a run targets via compute.accelerator_pool_id. "GPU +// Pool" (or "pool") is the product-facing name used across the Compute page and // this CLI. The backend still models and serves these as AiWorkflowService // ProvisionedCapacity resources, so the endpoint path and the wire structs // below keep the API's name; only the user-facing surface says "pool". @@ -58,7 +58,7 @@ func (n *apiInt64) UnmarshalJSON(b []byte) error { } // The wire structs below mirror the ProvisionedCapacity proto's JSON (the -// backend name for a GPU pool). +// backend name for a GPU Pool). type provisionedCapacity struct { Name string `json:"name"` Spec *provisionedCapacitySpec `json:"spec"` @@ -122,7 +122,7 @@ func poolAPIError(ctx context.Context, cmd *cobra.Command, action string, err er // specific than the generic 403 it may arrive as, so check it first. if apiErr, ok := errors.AsType[*apierr.APIError](err); ok && apiErr.ErrorCode == "FEATURE_DISABLED" { return renderError(ctx, cmd, "FEATURE_DISABLED", "PERMANENT", false, - errors.New("the GPU pools API is not enabled for this workspace")) + errors.New("the GPU Pools API is not enabled for this workspace")) } if errors.Is(err, apierr.ErrUnauthenticated) || errors.Is(err, apierr.ErrPermissionDenied) { return authError(ctx, cmd, err) @@ -184,12 +184,12 @@ func getPool(ctx context.Context, w *databricks.WorkspaceClient, id string) (*pr func newPoolsCommand() *cobra.Command { cmd := &cobra.Command{ Use: "pools", - Short: "View provisioned GPU pools, if any (not required to run)", - Long: `View GPU pools provisioned for your workspace. + Short: "View provisioned GPU Pools, if any (not required to run)", + Long: `View GPU Pools provisioned for your workspace. -A GPU pool is dedicated accelerator capacity that your Databricks account team +A GPU Pool is dedicated accelerator capacity that your Databricks account team provisions. Pools are optional: a run uses one only when its YAML sets -compute.pool_id.`, +compute.accelerator_pool_id.`, RunE: root.ReportUnknownSubcommand, } @@ -203,7 +203,7 @@ func newListPoolsCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list", Args: root.NoArgs, - Short: "List the GPU pools available to the current workspace", + Short: "List the GPU Pools available to the current workspace", } cmd.PreRunE = func(cmd *cobra.Command, args []string) error { @@ -220,7 +220,7 @@ func newListPoolsCommand() *cobra.Command { pools, err := listPools(ctx, w) if err != nil { - return poolAPIError(ctx, cmd, "list GPU pools", err) + return poolAPIError(ctx, cmd, "list GPU Pools", err) } data := poolListData{Rows: make([]poolRow, 0, len(pools))} @@ -242,8 +242,8 @@ func newGetPoolCommand() *cobra.Command { cmd := &cobra.Command{ Use: "get [POOL_ID]", Args: root.MaximumNArgs(1), - Short: "Show a GPU pool and its accelerator usage", - Long: `Show a GPU pool and its accelerator usage. + Short: "Show a GPU Pool and its accelerator usage", + Long: `Show a GPU Pool and its accelerator usage. POOL_ID may be omitted when the workspace has exactly one pool.`, } @@ -282,9 +282,9 @@ POOL_ID may be omitted when the workspace has exactly one pool.`, // ErrNotFound covers a plain 404 as well as RESOURCE_DOES_NOT_EXIST. if errors.Is(err, apierr.ErrNotFound) { return renderError(ctx, cmd, "NOT_FOUND", "NOT_FOUND", false, - fmt.Errorf("GPU pool %q not found: check the id with `air pools list`", id)) + fmt.Errorf("GPU Pool %q not found: check the id with `air pools list`", id)) } - return poolAPIError(ctx, cmd, fmt.Sprintf("get GPU pool %q", id), err) + return poolAPIError(ctx, cmd, fmt.Sprintf("get GPU Pool %q", id), err) } data := poolDetailFrom(*pool) @@ -304,21 +304,21 @@ POOL_ID may be omitted when the workspace has exactly one pool.`, func resolveSolePoolID(ctx context.Context, cmd *cobra.Command, w *databricks.WorkspaceClient) (string, error) { pools, err := listPools(ctx, w) if err != nil { - return "", poolAPIError(ctx, cmd, "list GPU pools", err) + return "", poolAPIError(ctx, cmd, "list GPU Pools", err) } switch len(pools) { case 1: return poolID(pools[0].Name), nil case 0: return "", renderError(ctx, cmd, "NOT_FOUND", "NOT_FOUND", false, - errors.New("no GPU pools found in this workspace")) + errors.New("no GPU Pools found in this workspace")) default: ids := make([]string, len(pools)) for i, p := range pools { ids[i] = poolID(p.Name) } return "", renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, - fmt.Errorf("workspace has %d GPU pools; specify one of: %s", len(pools), strings.Join(ids, ", "))) + fmt.Errorf("workspace has %d GPU Pools; specify one of: %s", len(pools), strings.Join(ids, ", "))) } } @@ -350,7 +350,7 @@ func poolDetailFrom(p provisionedCapacity) poolDetailData { // renderPoolTable prints the list as an aligned text table. func renderPoolTable(out io.Writer, rows []poolRow) { if len(rows) == 0 { - fmt.Fprintln(out, "No GPU pools found.") + fmt.Fprintln(out, "No GPU Pools found.") return } fmt.Fprintf(out, "%-40s %-14s %s\n", "ID", "ACCELERATOR", "RESERVED") diff --git a/cmd/air/pools_test.go b/cmd/air/pools_test.go index 0062c48f61a..93b457308c8 100644 --- a/cmd/air/pools_test.go +++ b/cmd/air/pools_test.go @@ -160,7 +160,7 @@ func TestListPoolsTextEmpty(t *testing.T) { cmd.SetOut(&buf) require.NoError(t, cmd.RunE(cmd, nil)) - assert.Contains(t, buf.String(), "No GPU pools found.") + assert.Contains(t, buf.String(), "No GPU Pools found.") } func TestGetPoolJSON(t *testing.T) { @@ -264,7 +264,7 @@ func TestGetPoolOmitIDNoPools(t *testing.T) { err := cmd.RunE(cmd, nil) require.Error(t, err) - assert.Contains(t, err.Error(), "no GPU pools found") + assert.Contains(t, err.Error(), "no GPU Pools found") } func TestGetPoolOmitIDMultipleErrors(t *testing.T) { @@ -334,7 +334,7 @@ func TestGetPoolNotFoundJSON(t *testing.T) { var got errorEnvelope require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) assert.Equal(t, "NOT_FOUND", got.Error.Code) - assert.Contains(t, got.Error.Message, `GPU pool "missing" not found`) + assert.Contains(t, got.Error.Message, `GPU Pool "missing" not found`) } func TestGetPoolPlain404(t *testing.T) { diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index 11a195bc21a..501f2d3ee9f 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -753,9 +753,13 @@ func visibleConfigChildNames(parent configField) []string { return names } -// closestConfigField returns the nearest candidate by edit distance, if one is -// close enough to be worth suggesting. +// closestConfigField suggests a renamed field or the nearest candidate by edit +// distance, if one is close enough to be worth suggesting. func closestConfigField(name string, candidates []string) (string, bool) { + // The renamed pool selector is too distant for the typo heuristic. + if name == "pool_id" && slices.Contains(candidates, "accelerator_pool_id") { + return "accelerator_pool_id", true + } best, bestDist := "", 0 for _, c := range candidates { d := configEditDistance(name, c) diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index 139504e1618..c46869192e8 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -807,6 +807,11 @@ func TestResolveConfigField_Errors(t *testing.T) { path: "config.compute.acclerator_type", wantParts: []string{`unknown config field "config.compute.acclerator_type"`, `did you mean "accelerator_type"?`, "accelerator_type, num_accelerators"}, }, + { + name: "removed pool selector suggests new name", + path: "config.compute.pool_id", + wantParts: []string{`unknown config field "config.compute.pool_id"`, `did you mean "accelerator_pool_id"?`}, + }, { name: "unknown top-level field lists siblings", path: "config.bogus", @@ -905,12 +910,13 @@ func TestWriteConfigFieldHelp(t *testing.T) { // Pool fields are set apart from the fields every run needs. var compute strings.Builder require.NoError(t, writeConfigFieldHelp(&compute, "config.compute")) - fields, capacity, found := strings.Cut(compute.String(), " Provisioned capacity (optional):\n") + fields, capacity, found := strings.Cut(compute.String(), " GPU Pools (optional):\n") require.True(t, found) - assert.Contains(t, fields, " Fields:\n num_accelerators (required) ") - assert.Contains(t, fields, " accelerator_type (required) ") + assert.Contains(t, fields, " Fields:\n num_accelerators (required) ") + assert.Contains(t, fields, " accelerator_type (required) ") assert.NotContains(t, fields, "pool_id") - assert.Contains(t, capacity, " pool_id ") + assert.NotContains(t, capacity, " pool_id ") + assert.Contains(t, capacity, " accelerator_pool_id ") assert.Contains(t, capacity, " priority_class ") require.Error(t, writeConfigFieldHelp(&strings.Builder{}, "config.nope")) @@ -1217,3 +1223,24 @@ func assertNoInlineDocsLinks(t *testing.T, f configField) { assertNoInlineDocsLinks(t, c) } } + +func TestLoadRunConfigAcceleratorPool(t *testing.T) { + for _, tc := range []struct { + name, fields, wantErr string + }{ + {name: "explicit", fields: " accelerator_pool_id: pool-1\n"}, + {name: "legacy rejected", fields: " pool_id: pool-1\n", wantErr: `unknown field "compute.pool_id"; did you mean "accelerator_pool_id"?`}, + {name: "alias rejected alongside selector", fields: " accelerator_pool_id: pool-1\n pool_id: pool-1\n", wantErr: `unknown field "compute.pool_id"; did you mean "accelerator_pool_id"?`}, + {name: "conflicting", fields: " accelerator_pool_id: pool-1\n pool_id: pool-2\n", wantErr: `unknown field "compute.pool_id"; did you mean "accelerator_pool_id"?`}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := loadRunConfig(writeConfig(t, minimalConfig+tc.fields)) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, new("pool-1"), cfg.Compute.AcceleratorPoolID) + }) + } +} diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index c06165dd2b4..bf6ceb760ee 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -222,8 +222,8 @@ func injectContainers(body map[string]any, containers []submittedContainer) erro // injectPoolFields sets the pool-only fields the SDK does not model onto the // decoded submit body: priority_class rides directly on the ai_runtime_task, -// while provisioned_capacity_id (the wire name for the pool) rides on the -// deployment's compute spec. Each is set only when non-empty. +// while accelerator_pool_id rides on the deployment's compute spec. +// Each is set only when non-empty. func injectPoolFields(body map[string]any, poolID, priorityClass string) error { aiRuntimeTask, err := aiRuntimeTaskFromSubmitBody(body) if err != nil { @@ -245,7 +245,7 @@ func injectPoolFields(body map[string]any, poolID, priorityClass string) error { if !ok { return errors.New("AIR submit payload is missing deployment compute") } - computeSpec["provisioned_capacity_id"] = poolID + computeSpec["accelerator_pool_id"] = poolID } return nil } @@ -435,11 +435,9 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run payload := buildSubmitPayload(cfg, commandPath, dlRuntimeImage(ctx, runtimeVersion), usagePolicyID, snap, deps) payload.IdempotencyToken = token - // The pool id is sent on the wire as provisioned_capacity_id (the backend's - // name for a GPU pool); only the user-facing YAML field is pool_id. poolID := "" - if cfg.Compute.PoolID != nil { - poolID = *cfg.Compute.PoolID + if selectedPoolID := cfg.Compute.AcceleratorPoolID; selectedPoolID != nil { + poolID = *selectedPoolID } priorityClass := "" if cfg.Compute.PriorityClass != nil { diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index e101bbc28af..2a5c739f4f5 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -162,7 +162,7 @@ func TestSubmitRun(t *testing.T) { assert.EqualValues(t, 0, task["max_retries"]) airTask := task["ai_runtime_task"].(map[string]any) // priority_class rides directly on the ai_runtime_task, next to - // provisioned_capacity_id on the deployment compute. + // accelerator_pool_id on the deployment compute. if tc.priorityClass == "" { assert.NotContains(t, airTask, "priority_class") } else { @@ -178,9 +178,9 @@ func TestSubmitRun(t *testing.T) { deployment := deployments[0].(map[string]any) compute := deployment["compute"].(map[string]any) if tc.poolID == "" { - assert.NotContains(t, compute, "provisioned_capacity_id") + assert.NotContains(t, compute, "accelerator_pool_id") } else { - assert.Equal(t, tc.poolID, compute["provisioned_capacity_id"]) + assert.Equal(t, tc.poolID, compute["accelerator_pool_id"]) } return jobs.SubmitRunResponse{RunId: 42} }) diff --git a/cmd/air/validateconfig.go b/cmd/air/validateconfig.go index 137347b985b..817bd7620d9 100644 --- a/cmd/air/validateconfig.go +++ b/cmd/air/validateconfig.go @@ -280,9 +280,7 @@ func validateConfigRequest(ctx context.Context, cfg *runConfig, commandPath stri if cfg.Compute != nil { compute["accelerator_type"] = cfg.Compute.AcceleratorType compute["accelerator_count"] = cfg.Compute.NumAccelerators - // Wire field stays provisioned_capacity_id (the backend name); the YAML - // field is pool_id. - putOpt(compute, "provisioned_capacity_id", cfg.Compute.PoolID) + putOpt(compute, "accelerator_pool_id", cfg.Compute.AcceleratorPoolID) } deployment := map[string]any{"compute": compute} if len(containers) == 0 { @@ -305,7 +303,7 @@ func validateConfigRequest(ctx context.Context, cfg *runConfig, commandPath stri } if cfg.Compute != nil { // priority_class rides on the ai_runtime_task (task-level), not the deployment - // compute where provisioned_capacity_id lives. + // compute where accelerator_pool_id lives. putOpt(task, "priority_class", cfg.Compute.PriorityClass) } putOpt(task, "mlflow_run", cfg.MLflowRunName) diff --git a/cmd/air/validateconfig_test.go b/cmd/air/validateconfig_test.go index 437f811a259..35a78ce918a 100644 --- a/cmd/air/validateconfig_test.go +++ b/cmd/air/validateconfig_test.go @@ -455,16 +455,16 @@ func TestValidateConfigRequestCarriesPriorityClass(t *testing.T) { srv := validateServer(t, http.StatusOK, `{}`, &gotReq) cfg := baseRunConfig() - cfg.Compute.PoolID = new("cap-8xh100-res") + cfg.Compute.AcceleratorPoolID = new("cap-8xh100-res") cfg.Compute.PriorityClass = new("NORMAL") _, err := preflightValidate(t.Context(), validationTestWorkspaceClient(t, srv.URL), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") require.NoError(t, err) task := gotReq["task"].(map[string]any) - // priority_class is task-level; provisioned_capacity_id stays on the compute spec. + // priority_class is task-level; accelerator_pool_id stays on the compute spec. assert.Equal(t, "NORMAL", task["priority_class"]) compute := task["deployments"].([]any)[0].(map[string]any)["compute"].(map[string]any) - assert.Equal(t, "cap-8xh100-res", compute["provisioned_capacity_id"]) + assert.Equal(t, "cap-8xh100-res", compute["accelerator_pool_id"]) } func TestValidateConfigRequestOmitsUnsetOptions(t *testing.T) { @@ -492,3 +492,29 @@ func TestValidateConfigRequestCarriesEnvironment(t *testing.T) { require.NoError(t, err) assert.JSONEq(t, `{"base_environment":"workspace-base-environments/databricks_ai_v6","dependencies":["torch==2.3.0","numpy"]}`, string(raw)) } + +func TestValidateConfigRequestPoolSelectors(t *testing.T) { + for _, tc := range []struct { + name string + poolID *string + want string + }{ + {name: "omitted"}, + {name: "explicit", poolID: new("pool-1"), want: "pool-1"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := baseRunConfig() + cfg.Compute.AcceleratorPoolID = tc.poolID + require.NoError(t, cfg.Compute.validate()) + req := validateConfigRequest(t.Context(), cfg, "/Workspace/Users/me/cmd.sh", nil, "token") + task := req["task"].(map[string]any) + compute := task["deployments"].([]any)[0].(map[string]any)["compute"].(map[string]any) + if tc.want == "" { + assert.NotContains(t, compute, "accelerator_pool_id") + } else { + assert.Equal(t, tc.want, compute["accelerator_pool_id"]) + } + assert.NotContains(t, compute, "provisioned_capacity_id") + }) + } +}