From c5f8b726e453fa97005eb01eaf5f25a41db99b03 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 18:02:26 +0000 Subject: [PATCH 1/8] air: Stop showing Max Retries 0 when runs/get omits the retry limit Jobs runs/get leaves max_retries out of every task run, and the SDK decodes the missing field as 0, so `air get` and the `air list` info pane showed "Max Retries 0" even for runs submitted with the default of 3 or an explicit limit . The server value is now used only when the response carries the key. For a one-time submit run, the limit is read from the run's uploaded training config: the configured value when it sets max_retries, and "3 (CLI default)" for ai_runtime_task runs that leave it unset, since the CLI always submits its default. Job runs, multi-container runs, legacy gen_ai_compute_task runs with no explicit value, and runs whose config can't be read show N/A. The derivation is resolveMaxRetries, which returns the limit and its source so `air get -o json` can expose the same value later. Co-authored-by: Isaac --- .nextchanges/air/get-max-retries.md | 1 + acceptance/air/get-ai-runtime/output.txt | 36 +++++- acceptance/air/get-ai-runtime/script | 7 ++ acceptance/air/get-ai-runtime/test.toml | 3 +- .../training_config_max_retries.yaml | 6 + acceptance/air/get/output.txt | 2 +- acceptance/air/get/test.toml | 6 +- cmd/air/format.go | 107 ++++++++++++++++-- cmd/air/format_test.go | 92 +++++++++++++++ cmd/air/get.go | 2 - cmd/air/list_detail.go | 3 + cmd/air/list_detail_test.go | 51 +++++++++ cmd/air/render.go | 24 ++-- cmd/air/render_test.go | 18 ++- 14 files changed, 327 insertions(+), 31 deletions(-) create mode 100644 .nextchanges/air/get-max-retries.md create mode 100644 acceptance/air/get-ai-runtime/training_config_max_retries.yaml diff --git a/.nextchanges/air/get-max-retries.md b/.nextchanges/air/get-max-retries.md new file mode 100644 index 00000000000..46f34b25cf3 --- /dev/null +++ b/.nextchanges/air/get-max-retries.md @@ -0,0 +1 @@ +* `air get` and the `air list` info pane no longer show `Max Retries 0` when the Jobs API leaves the task's retry limit out. For a run submitted with `air run`, the limit now comes from the run's uploaded config, shown as `3 (CLI default)` when the config doesn't set `max_retries`; it shows `N/A` when the limit can't be known (job runs, multi-container runs, legacy runs with no explicit value, or an unreadable config). diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index c84683e3f3e..57a09f7cd62 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -22,7 +22,7 @@ │ Status ● SUCCESS │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ -│ Max Retries 3 │ +│ Max Retries 3 (CLI default) │ │ Duration 12s │ │ Experiment my-exp │ │ MLflow Run my-run │ @@ -54,3 +54,37 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "est_percent_complete": null } } + +=== get (text, max_retries set in the config) + +>>> [CLI] air get 123 + +╭─ Configuration ────────────────────────────────────────────────╮ +│ │ +│ experiment_name: my-exp │ +│ compute: │ +│ accelerator_type: a10 │ +│ num_accelerators: 1 │ +│ max_retries: 1 │ +│ command: echo ok │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Status ● SUCCESS │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 1 │ +│ Duration 12s │ +│ Experiment my-exp │ +│ MLflow Run my-run │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment 4 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 diff --git a/acceptance/air/get-ai-runtime/script b/acceptance/air/get-ai-runtime/script index 7a98ebac655..3e6e210495a 100644 --- a/acceptance/air/get-ai-runtime/script +++ b/acceptance/air/get-ai-runtime/script @@ -9,3 +9,10 @@ trace $CLI air get 123 title "get (json)" trace $CLI air get 123 -o json + +# The fake runs/get, like the real one, has no max_retries on the task, so the +# limit comes from the uploaded config: the CLI default above, the set value here. +$CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries + +title "get (text, max_retries set in the config)" +trace $CLI air get 123 diff --git a/acceptance/air/get-ai-runtime/test.toml b/acceptance/air/get-ai-runtime/test.toml index 1879588b6c8..acf036047c2 100644 --- a/acceptance/air/get-ai-runtime/test.toml +++ b/acceptance/air/get-ai-runtime/test.toml @@ -12,11 +12,13 @@ Response.Body = '' # so the Environment cell instead comes from the run's environments[].spec (keyed # by the task's environment_key), which the typed SDK Run drops — `air get` reads # it with a raw request, so this route is hit twice (typed GetRun + that request). +# Like the real runs/get, the task carries no max_retries. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' { "run_id": 123, + "run_type": "SUBMIT_RUN", "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", "creator_user_name": "user@example.com", "start_time": 1700000000000, @@ -30,7 +32,6 @@ Response.Body = ''' "task_key": "train", "run_id": 456, "attempt_number": 0, - "max_retries": 3, "environment_key": "default", "ai_runtime_task": { "experiment": "my-exp", diff --git a/acceptance/air/get-ai-runtime/training_config_max_retries.yaml b/acceptance/air/get-ai-runtime/training_config_max_retries.yaml new file mode 100644 index 00000000000..a71625352e8 --- /dev/null +++ b/acceptance/air/get-ai-runtime/training_config_max_retries.yaml @@ -0,0 +1,6 @@ +experiment_name: my-exp +compute: + accelerator_type: a10 + num_accelerators: 1 +max_retries: 1 +command: echo ok diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 9c1fed61721..5283bbfd7f3 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -22,7 +22,7 @@ │ Status ● SUCCESS │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ -│ Max Retries 3 │ +│ Max Retries N/A │ │ Duration 12s │ │ Experiment my-exp │ │ MLflow Run my-run │ diff --git a/acceptance/air/get/test.toml b/acceptance/air/get/test.toml index 3545a34616e..dd4669e5542 100644 --- a/acceptance/air/get/test.toml +++ b/acceptance/air/get/test.toml @@ -4,12 +4,15 @@ Pattern = "HEAD /" Response.Body = '' -# A single GenAI-compute run with an experiment, GPUs, and a creator. +# A single GenAI-compute run with an experiment, GPUs, and a creator. Like the +# real runs/get, the task carries no max_retries, and its yaml_parameters leave +# max_retries unset, so the limit is unknown. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' { "run_id": 123, + "run_type": "SUBMIT_RUN", "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", "creator_user_name": "user@example.com", "start_time": 1700000000000, @@ -20,7 +23,6 @@ Response.Body = ''' "task_key": "train", "run_id": 456, "attempt_number": 0, - "max_retries": 3, "gen_ai_compute_task": { "mlflow_experiment_name": "/Users/user@example.com/my-exp", "compute": {"gpu_type": "GPU_1xA10", "num_gpus": 1}, diff --git a/cmd/air/format.go b/cmd/air/format.go index 68c956ad4c0..2f2614dd38a 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math" + "slices" "strconv" "strings" "time" @@ -359,15 +360,105 @@ func gpuDisplayName(gpuType string) string { return gpuType } -// maxRetries returns the configured retry limit for the run's latest task as a -// display string: "unlimited" for the backend's -1, otherwise the count. -func maxRetries(run *jobs.Run) string { +// maxRetriesSource says where a run's retry limit was read from. +type maxRetriesSource string + +const ( + // maxRetriesReported: runs/get returned max_retries on the task. + maxRetriesReported maxRetriesSource = "reported" + // maxRetriesConfigured: the run's uploaded config sets max_retries. + maxRetriesConfigured maxRetriesSource = "configured" + // maxRetriesCLIDefault: the config leaves max_retries unset, and the CLI + // submitted its default. + maxRetriesCLIDefault maxRetriesSource = "cli_default" +) + +// runMaxRetries is a run's retry limit and where it came from. Limit -1 means +// unlimited. +type runMaxRetries struct { + Limit int + Source maxRetriesSource +} + +// resolveMaxRetries returns the retry limit of the run's latest task, or nil +// when it can't be known. configRaw is the run's training config as uploaded +// (nil when it was not found or could not be downloaded). +// +// runs/get omits max_retries on task runs, and the SDK decodes the missing field +// as 0, so a value is trusted only when the response actually carried the key. +// Otherwise the limit is read from the config, and only for one-time submit +// runs: a job run (for example a bundle deployed with `air convert-to-dabs`) +// takes its retries from the job settings, which the config doesn't track. When +// the config leaves the key unset, the CLI default is assumed only for +// ai_runtime_task runs, because every CLI that submits that task type sends 3 +// (runsubmit.go always sends max_retries). Legacy gen_ai_compute_task runs may +// predate the Python CLI's switch from 0 to 3, so their unset limit is unknown. +func resolveMaxRetries(run *jobs.Run, configRaw []byte) *runMaxRetries { + if n, ok := reportedMaxRetries(run); ok { + return &runMaxRetries{Limit: n, Source: maxRetriesReported} + } + if run.RunType != jobs.RunTypeSubmitRun { + return nil + } + n, set, ok := configMaxRetries(configRaw) + switch { + case !ok: + return nil + case set: + return &runMaxRetries{Limit: n, Source: maxRetriesConfigured} + case aiRuntimeTaskOf(run) != nil: + return &runMaxRetries{Limit: defaultMaxRetries, Source: maxRetriesCLIDefault} + default: + return nil + } +} + +// maxRetriesDisplay renders a retry limit for the text view: "N/A" when it is +// unknown, "unlimited" for -1, and a "(CLI default)" note when the config left +// it unset. +func maxRetriesDisplay(m *runMaxRetries) string { + if m == nil { + return na + } + s := strconv.Itoa(m.Limit) + if m.Limit < 0 { + s = "unlimited" + } + if m.Source == maxRetriesCLIDefault { + s += " (CLI default)" + } + return s +} + +// reportedMaxRetries returns the latest task's max_retries if the runs/get +// response carried the key. The SDK records a decoded zero value in +// ForceSendFields, which tells an explicit 0 apart from a missing field. +func reportedMaxRetries(run *jobs.Run) (int, bool) { if len(run.Tasks) == 0 { - return "0" + return 0, false + } + t := run.Tasks[len(run.Tasks)-1] + if t.MaxRetries == 0 && !slices.Contains(t.ForceSendFields, "MaxRetries") { + return 0, false + } + return t.MaxRetries, true +} + +// configMaxRetries reads max_retries from a run's config YAML: the value, whether +// the key is set, and whether the config parsed at all. It decodes only that +// key so configs written by other CLI versions still parse. +func configMaxRetries(raw []byte) (n int, set, ok bool) { + if len(raw) == 0 { + return 0, false, false + } + var cfg struct { + MaxRetries *int `yaml:"max_retries"` + } + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return 0, false, false } - n := run.Tasks[len(run.Tasks)-1].MaxRetries - if n < 0 { - return "unlimited" + if cfg.MaxRetries == nil { + return 0, false, true } - return strconv.Itoa(n) + return *cfg.MaxRetries, true, true } diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index c76260d28cb..3b7edac72d0 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -1,7 +1,9 @@ package aircmd import ( + "encoding/json" "io" + "strconv" "testing" "github.com/databricks/cli/libs/cmdio" @@ -306,3 +308,93 @@ func TestTerminationReason(t *testing.T) { assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateSuccess, StateMessage: "done"}})) assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateCanceled, StateMessage: "cancelled"}})) } + +func TestMaxRetries(t *testing.T) { + submit := func(task jobs.RunTask) *jobs.Run { + return &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{task}} + } + aiRuntime := jobs.RunTask{AiRuntimeTask: &jobs.AiRuntimeTask{}} + reported := func(n int) jobs.RunTask { + return jobs.RunTask{MaxRetries: n, AiRuntimeTask: &jobs.AiRuntimeTask{}, ForceSendFields: []string{"MaxRetries"}} + } + legacy := jobs.RunTask{GenAiComputeTask: &jobs.GenAiComputeTask{}} + jobRun := &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{aiRuntime}} + tests := []struct { + name string + run *jobs.Run + config string + want string + }{ + {"submit, config sets 1", submit(aiRuntime), "max_retries: 1\n", "1"}, + {"submit, config sets 0", submit(aiRuntime), "max_retries: 0\n", "0"}, + {"submit, config sets unlimited", submit(aiRuntime), "max_retries: -1\n", "unlimited"}, + {"submit, config unset uses the CLI default", submit(aiRuntime), "experiment_name: e\n", "3 (CLI default)"}, + {"submit, no config", submit(aiRuntime), "", "N/A"}, + {"submit, unparsable config", submit(aiRuntime), "[not: a map", "N/A"}, + {"legacy submit, config unset is unknown", submit(legacy), "experiment_name: e\n", "N/A"}, + {"legacy submit, config sets 2", submit(legacy), "max_retries: 2\n", "2"}, + {"job run ignores the config", jobRun, "max_retries: 1\n", "N/A"}, + {"job run, config unset", jobRun, "experiment_name: e\n", "N/A"}, + {"run type unknown", &jobs.Run{Tasks: []jobs.RunTask{aiRuntime}}, "max_retries: 1\n", "N/A"}, + {"server reports 0 over the config", submit(reported(0)), "max_retries: 2\n", "0"}, + {"server reports 5 on a job run", &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{reported(5)}}, "", "5"}, + {"server reports unlimited", submit(reported(-1)), "", "unlimited"}, + {"no tasks", &jobs.Run{RunType: jobs.RunTypeSubmitRun}, "", "N/A"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw []byte + if tt.config != "" { + raw = []byte(tt.config) + } + assert.Equal(t, tt.want, maxRetriesDisplay(resolveMaxRetries(tt.run, raw))) + }) + } +} + +func TestResolveMaxRetriesSource(t *testing.T) { + run := &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{}}}} + assert.Equal(t, &runMaxRetries{Limit: 1, Source: maxRetriesConfigured}, resolveMaxRetries(run, []byte("max_retries: 1\n"))) + assert.Equal(t, &runMaxRetries{Limit: 3, Source: maxRetriesCLIDefault}, resolveMaxRetries(run, []byte("command: x\n"))) + run.Tasks[0].MaxRetries = 2 + assert.Equal(t, &runMaxRetries{Limit: 2, Source: maxRetriesReported}, resolveMaxRetries(run, nil)) +} + +// The runs/get bodies below are trimmed copies of real e2-dogfood responses: +// task runs carry no max_retries key at all. +func TestMaxRetriesFromDecodedRunGet(t *testing.T) { + decode := func(body string) *jobs.Run { + var run jobs.Run + require.NoError(t, json.Unmarshal([]byte(body), &run)) + return &run + } + display := func(run *jobs.Run, config []byte) string { + return maxRetriesDisplay(resolveMaxRetries(run, config)) + } + // Run 915071793879841: submitted with max_retries 1, failed, retried once. + aiRuntime := decode(`{"run_id": 915071793879841, "run_type": "SUBMIT_RUN", "tasks": [ + {"task_key": "t", "run_id": 996740861702356, "attempt_number": 0, "ai_runtime_task": {"experiment": "e"}}, + {"task_key": "t", "run_id": 286970869718720, "attempt_number": 1, "ai_runtime_task": {"experiment": "e"}}]}`) + assert.Equal(t, "1", display(aiRuntime, []byte("experiment_name: e\nmax_retries: 1\n"))) + assert.Equal(t, "3 (CLI default)", display(aiRuntime, []byte("experiment_name: e\n"))) + + // A multi-container run has no deployment-level command_path, so the config + // path is unknown and nothing is downloaded. + multi := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "t", "run_id": 2, + "ai_runtime_task": {"experiment": "e", "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]}}]}`) + var data getData + enrichFromAiRuntimeTask(multi, &data) + assert.Empty(t, data.TrainingConfigPath) + assert.Equal(t, "N/A", display(multi, nil)) + + // A legacy run as the Python CLI submitted it: yaml_parameters is the user's + // raw YAML, and max_retries is sent only when > 0. + legacyYAML := "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: python train.py\n" + legacy := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "train", "run_id": 2, "attempt_number": 0, + "gen_ai_compute_task": {"mlflow_experiment_name": "/Users/u/my-exp", "yaml_parameters": ` + strconv.Quote(legacyYAML) + `}}]}`) + _, raw := configYAML(t.Context(), nil, genAIComputeTask(legacy)) + assert.Equal(t, "N/A", display(legacy, raw)) + + explicitZero := decode(`{"run_type": "SUBMIT_RUN", "tasks": [{"max_retries": 0, "ai_runtime_task": {}}]}`) + assert.Equal(t, "0", display(explicitZero, []byte("experiment_name: e\n"))) +} diff --git a/cmd/air/get.go b/cmd/air/get.go index bac778ded51..963ec873f69 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -52,7 +52,6 @@ type getData struct { UserDisplay string `json:"-"` AcceleratorsDisplay string `json:"-"` EnvironmentDisplay string `json:"-"` - MaxRetriesDisplay string `json:"-"` // ProgressDisplay is the pre-rendered "Progress" cell ("45% · ~2h 15m left"), // set only for a running run with an estimable remaining time. ProgressDisplay string `json:"-"` @@ -292,6 +291,5 @@ func buildGetData(run *jobs.Run) getData { // version needs a raw GetRun read (the typed SDK Run drops it), so it is not // filled here alongside the fields read straight off the run. data.EnvironmentDisplay = na - data.MaxRetriesDisplay = maxRetries(run) return data } diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 59adaf93316..887fca8dd7d 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -24,6 +24,9 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceIDForRunLink(ctx, w)) + if findForEachTask(run) == nil && genAIComputeTask(run) == nil { + enrichFromAiRuntimeTask(run, &data) + } ids := mlflowIDs(ctx, w, run) if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 88939b9f363..5f880bc1481 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -145,3 +145,54 @@ func TestUnnamedOlderRetryLabelFitsTheListColumnAndIsWholeInTheInfoPane(t *testi assert.Equal(t, mlflowColWidth, lipgloss.Width(rows[0].MLflowLabel)) assert.Contains(t, info, mlflowRunID+" (retry 0)") } + +func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { + const configPath = "/Workspace/Users/me@example.com/.air/cli_launch/exp/exp_abc/training_config.yaml" + tests := []struct { + name string + runType string + config string // "" means the download fails + want string + }{ + {"configured limit", "SUBMIT_RUN", "experiment_name: exp\nmax_retries: 1\n", "Max Retries 1 "}, + {"unset limit", "SUBMIT_RUN", "experiment_name: exp\n", "Max Retries 3 (CLI default)"}, + {"config download fails", "SUBMIT_RUN", "", "Max Retries N/A"}, + {"job run", "JOB_RUN", "experiment_name: exp\nmax_retries: 1\n", "Max Retries N/A"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Given an ai_runtime run whose runs/get omits max_retries, as the + // Jobs API does. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "run_type": "` + tt.runType + `", + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [{"run_id": 456, "attempt_number": 0, "ai_runtime_task": {"experiment": "/Users/me@example.com/exp", + "deployments": [{"command_path": "/Workspace/Users/me@example.com/.air/cli_launch/exp/exp_abc/command.sh", + "compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]}}]}`)) + case "/api/2.0/workspace/export": + if tt.config != "" && r.URL.Query().Get("path") == configPath { + _, _ = w.Write([]byte(tt.config)) + return + } + w.WriteHeader(http.StatusNotFound) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + // When the list's info pane renders it. + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + + // Then it shows the same config box and limit as `air get`. + assert.Contains(t, got, tt.want) + assert.Contains(t, got, "1x A10") + if tt.config != "" { + assert.Contains(t, got, "experiment_name: exp") + } + }) + } +} diff --git a/cmd/air/render.go b/cmd/air/render.go index 755e8540fb9..acdaf7f7c78 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -104,13 +104,15 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl // with no serverless environment) stays "N/A". data.EnvironmentDisplay = orNA(aiRuntimeEnvironmentVersion(ctx, w, run.RunId)) + configBody, configRaw := resolveConfigYAML(ctx, w, run, data) + view := runView{ runID: data.RunID, dashboardURL: data.DashboardURL, status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, - maxRetries: data.MaxRetriesDisplay, + maxRetries: maxRetriesDisplay(resolveMaxRetries(run, configRaw)), duration: data.DurationDisplay, progress: data.ProgressDisplay, experiment: data.ExperimentDisplay, @@ -130,7 +132,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } var sections []string - if body := colorizeConfig(p, resolveConfigYAML(ctx, w, run, data)); body != "" { + if body := colorizeConfig(p, configBody); body != "" { sections = append(sections, renderBox(p, configBoxTitle, body)) } sections = append(sections, renderBox(p, metadataBoxTitle, renderFields(p, colorOn, view))) @@ -199,33 +201,35 @@ func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceCli } // resolveConfigYAML returns the config box body: from the downloaded config file -// when we have its path, else from the legacy task. -func resolveConfigYAML(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData) string { +// when we have its path, else from the legacy task. It also returns the raw +// config the body came from, or nil when the body is synthesized or empty. +func resolveConfigYAML(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData) (string, []byte) { if data.TrainingConfigPath != "" { if raw := downloadConfig(ctx, w, data.TrainingConfigPath); len(raw) > 0 { - return strings.TrimRight(reformatYAMLForDisplay(raw), "\n") + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } } if task := genAIComputeTask(run); task != nil { return configYAML(ctx, w, task) } - return "" + return "", nil } // configYAML returns the run's resolved training config as YAML for the box. The // full config (including the command/script) lives in the run's parameters, not // the structured task fields, so we prefer the inline parameters, then the // parameters file, and only synthesize a minimal config as a last resort. -func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.GenAiComputeTask) string { +func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.GenAiComputeTask) (string, []byte) { if task.YamlParameters != "" { - return strings.TrimRight(reformatYAMLForDisplay([]byte(task.YamlParameters)), "\n") + raw := []byte(task.YamlParameters) + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } if task.YamlParametersFilePath != "" { if raw := downloadConfig(ctx, w, task.YamlParametersFilePath); len(raw) > 0 { - return strings.TrimRight(reformatYAMLForDisplay(raw), "\n") + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } } - return synthConfigYAML(task) + return synthConfigYAML(task), nil } // downloadConfig fetches the run's training-config file, returning nil on diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index 66e5b09deb7..cd25947529a 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -28,7 +28,7 @@ func TestConfigYAML(t *testing.T) { task := &jobs.GenAiComputeTask{ YamlParameters: "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: \"for i in $(seq 1 3); do echo $i; done\\n\"\n", } - got := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) + got, _ := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) assert.Contains(t, got, "experiment_name: my-exp") assert.Contains(t, got, "accelerator_type: a10") assert.Contains(t, got, "command: |") @@ -41,7 +41,9 @@ func TestConfigYAML(t *testing.T) { Download(mock.Anything, "/Workspace/cfg.yaml"). Return(io.NopCloser(strings.NewReader("experiment_name: from-file\n")), nil) task := &jobs.GenAiComputeTask{YamlParametersFilePath: "/Workspace/cfg.yaml"} - assert.Equal(t, "experiment_name: from-file", configYAML(ctx, m.WorkspaceClient, task)) + got, raw := configYAML(ctx, m.WorkspaceClient, task) + assert.Equal(t, "experiment_name: from-file", got) + assert.Equal(t, "experiment_name: from-file\n", string(raw)) }) t.Run("falls back to a synthesized config", func(t *testing.T) { @@ -49,7 +51,7 @@ func TestConfigYAML(t *testing.T) { MlflowExperimentName: "/Users/me@example.com/exp", Compute: &jobs.ComputeConfig{GpuType: "a10", NumGpus: 1}, } - got := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) + got, _ := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) assert.Contains(t, got, "experiment_name: exp") assert.NotContains(t, got, "command") }) @@ -65,20 +67,24 @@ func TestResolveConfigYAML(t *testing.T) { Return(io.NopCloser(strings.NewReader("experiment_name: from-air\n")), nil) run := &jobs.Run{Tasks: []jobs.RunTask{{}}} data := &getData{TrainingConfigPath: "/Workspace/run/training_config.yaml"} - assert.Equal(t, "experiment_name: from-air", resolveConfigYAML(ctx, m.WorkspaceClient, run, data)) + got, raw := resolveConfigYAML(ctx, m.WorkspaceClient, run, data) + assert.Equal(t, "experiment_name: from-air", got) + assert.Equal(t, "experiment_name: from-air\n", string(raw)) }) t.Run("falls back to the legacy task when no path is set", func(t *testing.T) { run := &jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ MlflowExperimentName: "/Users/me@example.com/exp", }}}} - got := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) + got, _ := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) assert.Contains(t, got, "experiment_name: exp") }) t.Run("empty when neither source is present", func(t *testing.T) { run := &jobs.Run{Tasks: []jobs.RunTask{{}}} - assert.Empty(t, resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{})) + got, raw := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) + assert.Empty(t, got) + assert.Nil(t, raw) }) } From 0f83c804cea650ef67751f3e2f23ae9c8aefe2d1 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 18:21:05 +0000 Subject: [PATCH 2/8] air: Include run details in air get JSON Expose typed accelerator, environment, retry, user, MLflow and sweep details without changing existing JSON fields. Read and emit the stored config only when --include-config is passed, because it may contain plaintext environment variables and parameters. Without it, config-derived retry limits remain unknown. Bound config downloads before SDK response buffering and omit oversized files in text and JSON. Guarantee an empty array for sweep tasks when iteration lookup fails. Co-authored-by: Isaac --- .nextchanges/air/get-json-details.md | 1 + acceptance/air/get-ai-runtime/output.txt | 104 ++++++++++- acceptance/air/get-ai-runtime/script | 9 + acceptance/air/get-pending/output.txt | 13 +- acceptance/air/get/output.txt | 10 +- acceptance/air/help/output.txt | 3 +- cmd/air/format.go | 17 +- cmd/air/format_test.go | 14 -- cmd/air/get.go | 154 +++++++++++++-- cmd/air/get_test.go | 228 ++++++++++++++++++++++- cmd/air/list_detail_test.go | 25 +++ cmd/air/render.go | 95 +++++++--- cmd/air/render_test.go | 89 +++++++-- cmd/air/sweep.go | 31 +-- cmd/air/sweep_test.go | 4 + 15 files changed, 698 insertions(+), 99 deletions(-) create mode 100644 .nextchanges/air/get-json-details.md diff --git a/.nextchanges/air/get-json-details.md b/.nextchanges/air/get-json-details.md new file mode 100644 index 00000000000..e73de015bc6 --- /dev/null +++ b/.nextchanges/air/get-json-details.md @@ -0,0 +1 @@ +* `air get -o json` includes run details including flat `accelerator_type` and `num_accelerators`. Use `--include-config` to include stored config, which may contain plaintext `env_variables` and parameters; downloads over 1 MiB are omitted, and config-dependent `max_retries` is otherwise null. diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 57a09f7cd62..b63a4f08d40 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -51,7 +51,48 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null + } +} + +=== get (json, include config) + +>>> [CLI] air get 123 -o json --include-config +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": { + "limit": 3, + "source": "cli_default" + }, + "mlflow_run_name": "my-run", + "sweep": null, + "config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: |-\n for i in $(seq 1 10); do\n echo \"step $i\"\n done\n", + "config_source": "uploaded" } } @@ -88,3 +129,64 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get (json, max_retries set in the config) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null + } +} + +=== get (json, include config, max_retries set) + +>>> [CLI] air get 123 -o json --include-config +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": { + "limit": 1, + "source": "configured" + }, + "mlflow_run_name": "my-run", + "sweep": null, + "config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\nmax_retries: 1\ncommand: echo ok\n", + "config_source": "uploaded" + } +} diff --git a/acceptance/air/get-ai-runtime/script b/acceptance/air/get-ai-runtime/script index 3e6e210495a..a73d6742a95 100644 --- a/acceptance/air/get-ai-runtime/script +++ b/acceptance/air/get-ai-runtime/script @@ -10,9 +10,18 @@ trace $CLI air get 123 title "get (json)" trace $CLI air get 123 -o json +title "get (json, include config)" +trace $CLI air get 123 -o json --include-config + # The fake runs/get, like the real one, has no max_retries on the task, so the # limit comes from the uploaded config: the CLI default above, the set value here. $CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries title "get (text, max_retries set in the config)" trace $CLI air get 123 + +title "get (json, max_retries set in the config)" +trace $CLI air get 123 -o json + +title "get (json, include config, max_retries set)" +trace $CLI air get 123 -o json --include-config diff --git a/acceptance/air/get-pending/output.txt b/acceptance/air/get-pending/output.txt index 3a5f9a72eda..c838d560527 100644 --- a/acceptance/air/get-pending/output.txt +++ b/acceptance/air/get-pending/output.txt @@ -14,6 +14,17 @@ "mlflow_url": null, "status_message": "Waiting for accelerator compute capacity to become available...", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null } } diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 5283bbfd7f3..e82b0022e9f 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -51,7 +51,15 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null } } diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..5091bb160a3 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -96,7 +96,8 @@ Usage: databricks air get JOB_RUN_ID [flags] Flags: - -h, --help help for get + -h, --help help for get + --include-config Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters) Global Flags: --debug enable debug logging diff --git a/cmd/air/format.go b/cmd/air/format.go index 2f2614dd38a..d73c2d8a620 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -326,19 +326,6 @@ func stripExperimentUserPrefix(name string) string { return name } -// accelerators returns a short description of the GPUs the run uses, such as -// "8x H100", or an empty string if the run has no GPU compute attached. -func accelerators(run *jobs.Run) string { - if len(run.Tasks) == 0 { - return "" - } - task := run.Tasks[0].GenAiComputeTask - if task == nil || task.Compute == nil { - return "" - } - return acceleratorLabel(task.Compute.GpuType, task.Compute.NumGpus) -} - // acceleratorLabel renders a GPU count and type as "8x H100", "" for none, or // the count alone ("8x") when the type is unrecognized. func acceleratorLabel(gpuType string, count int) string { @@ -376,8 +363,8 @@ const ( // runMaxRetries is a run's retry limit and where it came from. Limit -1 means // unlimited. type runMaxRetries struct { - Limit int - Source maxRetriesSource + Limit int `json:"limit"` + Source maxRetriesSource `json:"source"` } // resolveMaxRetries returns the retry limit of the run's latest task, or nil diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 3b7edac72d0..8efd55bb0cc 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -213,20 +213,6 @@ func TestReformatYAMLForDisplay(t *testing.T) { assert.Equal(t, "\tnot: valid: yaml:", reformatYAMLForDisplay([]byte("\tnot: valid: yaml:"))) } -func TestAccelerators(t *testing.T) { - assert.Empty(t, accelerators(&jobs.Run{})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{}}})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{}, - }}})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 0}}, - }}})) - assert.Equal(t, "8x H100", accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 8, GpuType: "GPU_8xH100"}}, - }}})) -} - func TestAcceleratorLabel(t *testing.T) { assert.Empty(t, acceleratorLabel("GPU_8xH100", 0)) assert.Equal(t, "8x H100", acceleratorLabel("GPU_8xH100", 8)) diff --git a/cmd/air/get.go b/cmd/air/get.go index 963ec873f69..82ffeba0042 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -6,11 +6,13 @@ import ( "fmt" "path" "strconv" + "sync" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" @@ -41,11 +43,40 @@ type getData struct { ESTRemainingSeconds *int64 `json:"est_remaining_seconds"` ESTPercentComplete *int `json:"est_percent_complete"` - // The fields below are pre-rendered text-view cells, excluded from JSON - // (matching `air get --json`). Each shows "N/A" when its value is - // missing. The styled single-run renderer (render.go) consumes them; the - // Run ID, Status, and MLflow Run cells it draws are styled and hyperlinked - // there rather than stored here. + // The fields below carry the values of the text view's Configuration and + // Metadata boxes and its sweep summary. Missing values are null; config-based + // retry limits require opting in to reading the stored config. + User *string `json:"user"` + AcceleratorType *string `json:"accelerator_type"` + NumAccelerators *int `json:"num_accelerators"` + // EnvironmentVersion and BaseEnvironment come from the run's serverless + // environment spec; at most one is set. + EnvironmentVersion *string `json:"environment_version"` + BaseEnvironment *string `json:"base_environment"` + // MaxRetries is the run's retry limit and its source (see resolveMaxRetries). + MaxRetries *runMaxRetries `json:"max_retries"` + // Config is the stored run config the Configuration box shows, before display + // reformatting. ConfigSource names where it came from: "uploaded" (the + // training_config.yaml `air run` uploads after applying overrides and + // stripping comments), or "yaml_parameters" / "yaml_parameters_file" (legacy + // runs, a different schema). A config synthesized for display is not exposed. + Config *string `json:"-"` + ConfigSource *string `json:"-"` + // MLflowRunName is the MLflow run's name, which the MLflow Run cell shows. + MLflowRunName *string `json:"mlflow_run_name"` + // Sweep is set for foreach runs; its table replaces the single-run view. + Sweep *sweepInfo `json:"sweep"` + + // configBody is the Configuration box body, which may be synthesized for a + // legacy run without stored parameters. + configBody string + // detailsResolved records that enrichRunDetails has run. + detailsResolved bool + + // The fields below are pre-rendered text-view cells, excluded from JSON. + // Each shows "N/A" when its value is missing. The styled single-run renderer + // (render.go) consumes them; the Run ID, Status, and MLflow Run cells it draws + // are styled and hyperlinked there rather than stored here. SubmittedDisplay string `json:"-"` DurationDisplay string `json:"-"` ExperimentDisplay string `json:"-"` @@ -58,8 +89,6 @@ type getData struct { DisplayStatus string `json:"-"` // TrainingConfigPath is the run's config file, downloaded for the config box. TrainingConfigPath string `json:"-"` - // Sweep replaces the single-run view for foreach runs. - Sweep *sweepInfo `json:"-"` } // getTemplate is the text-mode layout for a sweep (foreach) run. Single runs are @@ -118,6 +147,7 @@ func credentialError(ctx context.Context, w *databricks.WorkspaceClient, err err // newGetCommand returns the `air get JOB_RUN_ID` command, which shows status, // configuration, and timing details for a specific run. func newGetCommand() *cobra.Command { + var includeConfig bool cmd := &cobra.Command{ Use: "get JOB_RUN_ID", Args: root.ExactArgs(1), @@ -130,6 +160,8 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, }, } + cmd.Flags().BoolVar(&includeConfig, "include-config", false, "Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters)") + // Resolve and authenticate the workspace client up front so an auth failure // fails fast here, before any run status or config is fetched or printed. // ErrAlreadyPrinted passes through (it was handled upstream); other failures @@ -144,6 +176,9 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + if includeConfig && root.OutputType(cmd) == flags.OutputText { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, errors.New("--include-config requires -o json")) + } w := cmdctx.WorkspaceClient(ctx) runID, err := strconv.ParseInt(args[0], 10, 64) @@ -206,11 +241,27 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, } if task := findForEachTask(run); task != nil { data.Sweep = buildSweepInfo(ctx, w, task) - } else if genAIComputeTask(run) == nil { - enrichFromAiRuntimeTask(run, &data) + } else { + if genAIComputeTask(run) == nil { + enrichFromAiRuntimeTask(run, &data) + } + enrichCtx := ctx + if root.OutputType(cmd) != flags.OutputText { + // A failed read already shows as a null field; keep stderr quiet + // for callers that merge it into stdout. + enrichCtx = withQuietEnrichment(ctx) + } + enrichRunDetails(enrichCtx, w, run, &data, ids, root.OutputType(cmd) == flags.OutputText || includeConfig) } if root.OutputType(cmd) != flags.OutputText { + if includeConfig { + return renderEnvelope(ctx, struct { + getData + Config *string `json:"config"` + ConfigSource *string `json:"config_source"` + }{data, data.Config, data.ConfigSource}) + } return renderEnvelope(ctx, data) } @@ -243,9 +294,7 @@ func enrichFromAiRuntimeTask(run *jobs.Run, data *getData) { if d.CommandPath != "" { data.TrainingConfigPath = path.Join(path.Dir(d.CommandPath), trainingConfigName) } - if a := acceleratorLabel(string(d.Compute.AcceleratorType), d.Compute.AcceleratorCount); a != "" { - data.AcceleratorsDisplay = a - } + setAccelerators(data, string(d.Compute.AcceleratorType), d.Compute.AcceleratorCount) } if task.Experiment != "" { exp := stripExperimentUserPrefix(task.Experiment) @@ -285,11 +334,90 @@ func buildGetData(run *jobs.Run) getData { if data.ExperimentName != nil { data.ExperimentDisplay = *data.ExperimentName } + data.User = nilIfEmpty(run.CreatorUserName) data.UserDisplay = orNA(run.CreatorUserName) - data.AcceleratorsDisplay = orNA(accelerators(run)) + data.AcceleratorsDisplay = na + accelType, count := jobCompute(run) + setAccelerators(&data, accelType, count) // EnvironmentDisplay is resolved at render time: the serverless environment // version needs a raw GetRun read (the typed SDK Run drops it), so it is not // filled here alongside the fields read straight off the run. data.EnvironmentDisplay = na return data } + +// setAccelerators records GPU compute in the same nullable shape as air list. +func setAccelerators(data *getData, accelType string, count int) { + data.AcceleratorType = nilIfEmpty(accelType) + if count != 0 { + data.NumAccelerators = new(count) + } else { + data.NumAccelerators = nil + } + data.AcceleratorsDisplay = orNA(acceleratorLabel(accelType, count)) +} + +// enrichRunDetails resolves the single-run values that need requests beyond the +// typed GetRun: the environment (raw GetRun), the stored config (download) and +// the retry limit and the MLflow run name. When readConfig is false, only a +// server-reported retry limit is available. Each read is best-effort. +func enrichRunDetails(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData, ids *mlflowIdentifiers, readConfig bool) { + data.detailsResolved = true + + // The reads are independent, so run them concurrently. + var ( + wg sync.WaitGroup + version, base string + body string + raw []byte + mlflowName string + ) + wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId) }) + if readConfig { + wg.Go(func() { body, raw = resolveConfigYAML(ctx, w, run, data) }) + } + if ids != nil { + wg.Go(func() { mlflowName = fetchMLflowRunName(ctx, w, ids.RunID) }) + } + wg.Wait() + + data.EnvironmentVersion = nilIfEmpty(version) + data.BaseEnvironment = nilIfEmpty(base) + data.EnvironmentDisplay = orNA(version) + data.configBody = body + if len(raw) > 0 { + cfg := string(raw) + data.Config = &cfg + data.ConfigSource = configSource(run, data) + } + data.MaxRetries = resolveMaxRetries(run, raw) + data.MLflowRunName = nilIfEmpty(mlflowName) +} + +// configSource names where resolveConfigYAML read a stored config from. +func configSource(run *jobs.Run, data *getData) *string { + source := "uploaded" + if task := genAIComputeTask(run); task != nil && data.TrainingConfigPath == "" { + source = "yaml_parameters_file" + if task.YamlParameters != "" { + source = "yaml_parameters" + } + } + return &source +} + +type quietEnrichmentKey struct{} + +// withQuietEnrichment demotes best-effort enrichment failures from warnings to +// debug logs. +func withQuietEnrichment(ctx context.Context) context.Context { + return context.WithValue(ctx, quietEnrichmentKey{}, true) +} + +func enrichmentWarnf(ctx context.Context, format string, v ...any) { + if quiet, _ := ctx.Value(quietEnrichmentKey{}).(bool); quiet { + log.Debugf(ctx, format, v...) + return + } + log.Warnf(ctx, format, v...) +} diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index c6e984c3b44..c70387f6d76 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -324,7 +324,9 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) { assert.JSONEq(t, `{ "run_id":"5","status":"FAILED","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, - "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null + "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null, + "user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null, + "max_retries":null,"mlflow_run_name":null,"sweep":null }`, string(body)) canceled := buildGetData(&jobs.Run{RunId: 6, State: &jobs.RunState{ResultState: jobs.RunResultStateCanceled, StateMessage: "user canceled"}}) @@ -346,7 +348,9 @@ func TestBuildGetDataWaitingForComputeStatus(t *testing.T) { assert.JSONEq(t, `{ "run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, - "est_remaining_seconds":null,"est_percent_complete":null + "est_remaining_seconds":null,"est_percent_complete":null, + "user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null, + "max_retries":null,"mlflow_run_name":null,"sweep":null }`, string(body)) } @@ -400,3 +404,223 @@ func TestBuildGetDataEmpty(t *testing.T) { assert.Equal(t, na, d.UserDisplay) assert.Equal(t, na, d.AcceleratorsDisplay) } + +func TestBuildGetDataMetadata(t *testing.T) { + t.Run("legacy task", func(t *testing.T) { + d := buildGetData(&jobs.Run{ + CreatorUserName: "me@example.com", + Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ + Compute: &jobs.ComputeConfig{GpuType: "GPU_8xH100", NumGpus: 8}, + }}}, + }) + assert.Equal(t, "me@example.com", *d.User) + assert.Equal(t, new("GPU_8xH100"), d.AcceleratorType) + assert.Equal(t, new(8), d.NumAccelerators) + assert.Equal(t, "8x H100", d.AcceleratorsDisplay) + }) + + t.Run("ai_runtime task", func(t *testing.T) { + d := buildGetData(&jobs.Run{Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{ + Deployments: []jobs.DeploymentSpec{{ + Compute: jobs.ComputeSpec{AcceleratorType: jobs.ComputeSpecAcceleratorTypeGpu1xA10, AcceleratorCount: 1}, + }}, + }}}}) + assert.Equal(t, new("GPU_1xA10"), d.AcceleratorType) + assert.Equal(t, new(1), d.NumAccelerators) + assert.Equal(t, "1x A10", d.AcceleratorsDisplay) + }) + + t.Run("no compute or creator stays null", func(t *testing.T) { + d := buildGetData(&jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ + Compute: &jobs.ComputeConfig{NumGpus: 0}, + }}}}) + assert.Nil(t, d.User) + assert.Nil(t, d.AcceleratorType) + assert.Nil(t, d.NumAccelerators) + assert.Equal(t, na, d.AcceleratorsDisplay) + assert.Equal(t, na, d.UserDisplay) + }) +} + +func TestGetAcceleratorJSONMatchesListWhenCountMissing(t *testing.T) { + run := &jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{GpuType: "GPU_1xA10"}}}}} + data := buildGetData(run) + encoded, err := json.Marshal(data) + require.NoError(t, err) + var fields map[string]any + require.NoError(t, json.Unmarshal(encoded, &fields)) + assert.Equal(t, "GPU_1xA10", fields["accelerator_type"]) + assert.Nil(t, fields["num_accelerators"]) +} + +func TestEnrichRunDetails(t *testing.T) { + const uploaded = "/Workspace/run/training_config.yaml" + submitAiRuntime := `"run_type": "SUBMIT_RUN", "tasks": [{"ai_runtime_task": {}}]` + tests := []struct { + name string + run string + config string // served at uploaded; empty means not found + mlflowName string // empty means the MLflow lookup fails + wantVersion *string + wantBase *string + wantEnvCell string + wantSource *string + wantRetries *runMaxRetries + wantBody string + }{ + { + name: "uploaded config and environment version", + run: `{"run_id": 1, "environments": [{"environment_key": "default", "spec": {"environment_version": "6"}}], ` + submitAiRuntime + `}`, + config: "max_retries: 1\n", + mlflowName: "my-run", + wantVersion: new("6"), wantEnvCell: "6", + wantSource: new("uploaded"), wantRetries: &runMaxRetries{Limit: 1, Source: maxRetriesConfigured}, wantBody: "max_retries: 1", + }, + { + name: "uploaded config without max_retries uses the CLI default", + run: `{"run_id": 1, ` + submitAiRuntime + `}`, + config: "experiment_name: e\n", + wantEnvCell: na, wantSource: new("uploaded"), + wantRetries: &runMaxRetries{Limit: 3, Source: maxRetriesCLIDefault}, wantBody: "experiment_name: e", + }, + { + name: "server-reported unlimited", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"max_retries": -1, "ai_runtime_task": {}}]}`, + wantEnvCell: na, wantRetries: &runMaxRetries{Limit: -1, Source: maxRetriesReported}, + }, + { + name: "job run with a readable config is unknown", + run: `{"run_id": 1, "run_type": "JOB_RUN", "tasks": [{"ai_runtime_task": {}}]}`, + config: "max_retries: 1\n", + wantEnvCell: na, wantSource: new("uploaded"), wantBody: "max_retries: 1", + }, + { + name: "base environment, config unreadable", + run: `{"run_id": 1, "environments": [{"environment_key": "default", "spec": {"base_environment": "workspace-base-environments/databricks_ai_v5"}}], ` + submitAiRuntime + `}`, + wantBase: new("workspace-base-environments/databricks_ai_v5"), wantEnvCell: na, + }, + { + name: "legacy inline parameters", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"yaml_parameters": "max_retries: 2\ncommand: echo hi\n"}}]}`, + wantEnvCell: na, wantSource: new("yaml_parameters"), + wantRetries: &runMaxRetries{Limit: 2, Source: maxRetriesConfigured}, wantBody: "max_retries: 2\ncommand: echo hi", + }, + { + name: "legacy inline parameters without max_retries are unknown", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"yaml_parameters": "command: echo hi\n"}}]}`, + wantEnvCell: na, wantSource: new("yaml_parameters"), wantBody: "command: echo hi", + }, + { + name: "legacy synthesized config is not exposed", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"mlflow_experiment_name": "exp"}}]}`, + wantEnvCell: na, wantBody: "experiment_name: exp", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(tt.run)) + case strings.HasPrefix(r.URL.Path, "/api/2.0/workspace/export") && tt.config != "": + _, _ = w.Write([]byte(tt.config)) + case r.URL.Path == "/api/2.0/mlflow/runs/get" && tt.mlflowName != "": + _, _ = w.Write([]byte(`{"run": {"info": {"run_name": "` + tt.mlflowName + `"}}}`)) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "not found"}`)) + } + })) + t.Cleanup(srv.Close) + w := newTestWorkspaceClient(t, srv.URL) + ctx := withQuietEnrichment(cmdio.MockDiscard(t.Context())) + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: 1}) + require.NoError(t, err) + data := buildGetData(run) + if genAIComputeTask(run) == nil { + data.TrainingConfigPath = uploaded + } + + enrichRunDetails(ctx, w, run, &data, &mlflowIdentifiers{ExperimentID: "e1", RunID: "r1"}, true) + + assert.Equal(t, tt.wantVersion, data.EnvironmentVersion) + assert.Equal(t, tt.wantBase, data.BaseEnvironment) + assert.Equal(t, tt.wantEnvCell, data.EnvironmentDisplay) + assert.Equal(t, tt.wantSource, data.ConfigSource) + assert.Equal(t, tt.wantSource == nil, data.Config == nil) + assert.Equal(t, tt.wantRetries, data.MaxRetries) + assert.Equal(t, tt.wantBody, data.configBody) + if tt.mlflowName == "" { + assert.Nil(t, data.MLflowRunName) + } else { + assert.Equal(t, tt.mlflowName, *data.MLflowRunName) + } + + // JSON is null exactly when the text cell is N/A, and both render the + // one resolved value. + body, err := json.Marshal(data) + require.NoError(t, err) + var got struct { + MaxRetries *runMaxRetries `json:"max_retries"` + } + require.NoError(t, json.Unmarshal(body, &got)) + assert.Equal(t, maxRetriesDisplay(data.MaxRetries) == na, got.MaxRetries == nil) + assert.Equal(t, maxRetriesDisplay(data.MaxRetries), maxRetriesDisplay(got.MaxRetries)) + }) + } +} + +func TestEnrichRunDetailsWithoutConfig(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _, _ = w.Write([]byte(`{"run_id":1}`)) + return + } + if strings.HasPrefix(r.URL.Path, "/api/2.0/workspace/export") { + t.Errorf("unexpected config download: %s", r.URL.Path) + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + w := newTestWorkspaceClient(t, srv.URL) + run := &jobs.Run{RunId: 1, RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{}}}} + data := buildGetData(run) + data.TrainingConfigPath = "/Workspace/run/training_config.yaml" + enrichRunDetails(withQuietEnrichment(t.Context()), w, run, &data, nil, false) + assert.Nil(t, data.Config) + assert.Nil(t, data.ConfigSource) + assert.Nil(t, data.MaxRetries) + + // Inline legacy parameters must not be parsed either without opt-in. + legacy := &jobs.Run{RunId: 1, RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{ + GenAiComputeTask: &jobs.GenAiComputeTask{YamlParameters: "max_retries: 2\n"}, + }}} + legacyData := buildGetData(legacy) + enrichRunDetails(withQuietEnrichment(t.Context()), w, legacy, &legacyData, nil, false) + assert.Nil(t, legacyData.Config) + assert.Nil(t, legacyData.MaxRetries) + + // Server-reported limits remain available without reading config. + legacy.Tasks[0].MaxRetries = 4 + enrichRunDetails(withQuietEnrichment(t.Context()), w, legacy, &legacyData, nil, false) + assert.Equal(t, &runMaxRetries{Limit: 4, Source: maxRetriesReported}, legacyData.MaxRetries) +} + +func TestSweepJSON(t *testing.T) { + exp := "exp" + body, err := json.Marshal(getData{RunID: "1", Sweep: &sweepInfo{ + Total: 2, Succeeded: 1, Active: 1, Completed: 1, + Tasks: []sweepTask{ + {TaskKey: "t0", RunID: "10", Status: "SUCCESS", Experiment: exp, ExperimentName: &exp}, + {TaskKey: "t1", RunID: "11", Status: "RUNNING"}, + }, + }}) + require.NoError(t, err) + var got struct { + Sweep json.RawMessage `json:"sweep"` + } + require.NoError(t, json.Unmarshal(body, &got)) + assert.JSONEq(t, `{"total":2,"succeeded":1,"failed":0,"active":1,"completed":1,"tasks":[ + {"task_key":"t0","run_id":"10","status":"SUCCESS","experiment_name":"exp"}, + {"task_key":"t1","run_id":"11","status":"RUNNING","experiment_name":null}]}`, string(got.Sweep)) +} diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 5f880bc1481..05eeb2a6ed2 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -196,3 +196,28 @@ func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { }) } } + +func TestRunDetailTextShowsEnvironmentAndLegacyConfig(t *testing.T) { + // The list picker's detail pane renders through runDetailText, not `air get`, + // so it must still resolve the environment and the full legacy config. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "run_type": "SUBMIT_RUN", "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "environments": [{"environment_key": "default", "spec": {"environment_version": "4"}}], + "tasks": [{"run_id": 456, "attempt_number": 0, "gen_ai_compute_task": { + "compute": {"gpu_type": "GPU_1xA10", "num_gpus": 1}, + "yaml_parameters": "experiment_name: my-exp\nmax_retries: 2\ncommand: echo legacy-command\n"}}]}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + assert.Contains(t, got, "command: echo legacy-command") + assert.Regexp(t, `Environment\s+4`, got) + assert.Regexp(t, `Max Retries\s+2 `, got) + assert.Regexp(t, `Accelerators\s+1x A10`, got) +} diff --git a/cmd/air/render.go b/cmd/air/render.go index acdaf7f7c78..6830f63680d 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -11,9 +11,10 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/muesli/termenv" "go.yaml.in/yaml/v3" @@ -99,12 +100,11 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl renderer, colorOn := cmdio.NewRenderer(ctx, out) p := newPalette(renderer) - // The serverless environment version lives on the run's environments[].spec, - // which the typed SDK Run drops, so read it with a raw request. Empty (a run - // with no serverless environment) stays "N/A". - data.EnvironmentDisplay = orNA(aiRuntimeEnvironmentVersion(ctx, w, run.RunId)) - - configBody, configRaw := resolveConfigYAML(ctx, w, run, data) + // The list detail pane (list_detail.go) renders through here without going + // through `air get`, which resolves these up front. + if !data.detailsResolved { + enrichRunDetails(ctx, w, run, data, ids, true) + } view := runView{ runID: data.RunID, @@ -112,7 +112,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, - maxRetries: maxRetriesDisplay(resolveMaxRetries(run, configRaw)), + maxRetries: maxRetriesDisplay(data.MaxRetries), duration: data.DurationDisplay, progress: data.ProgressDisplay, experiment: data.ExperimentDisplay, @@ -126,13 +126,17 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } if ids != nil { + name := "" + if data.MLflowRunName != nil { + name = *data.MLflowRunName + } // The box grows to fit its fields, so the label is shown whole. - view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID, 0) + ids.retryNote() + view.mlflowLabel = mlflowRunLabel(name, ids.RunID, 0) + ids.retryNote() view.mlflowURL = mlflowRunURL(w.Config.Host, ids) } var sections []string - if body := colorizeConfig(p, configBody); body != "" { + if body := colorizeConfig(p, data.configBody); body != "" { sections = append(sections, renderBox(p, configBoxTitle, body)) } sections = append(sections, renderBox(p, metadataBoxTitle, renderFields(p, colorOn, view))) @@ -166,22 +170,24 @@ func genAIComputeTask(run *jobs.Run) *jobs.GenAiComputeTask { return run.Tasks[0].GenAiComputeTask } -// aiRuntimeEnvironmentVersion returns the serverless environment version an -// ai_runtime run used (e.g. "4"), read from the Jobs GetRun response's -// environments[] entry keyed by aiRuntimeEnvironmentKey. The typed SDK Run has -// no environments field, so the value is fetched with a raw request. Best-effort: -// returns "" (logged) on any error or when the run declares no environment. -func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceClient, runID int64) string { +// aiRuntimeEnvironment returns the serverless environment an ai_runtime run used, +// read from the Jobs GetRun response's environments[] entry keyed by +// aiRuntimeEnvironmentKey: a version (e.g. "4") or, for Databricks AI runtimes, a +// base environment. The typed SDK Run has no environments field, so the value is +// fetched with a raw request. Best-effort: returns empty strings (logged) on any +// error or when the run declares no environment. +func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (version, baseEnvironment string) { apiClient, err := client.New(w.Config) if err != nil { - log.Warnf(ctx, "air get: could not create client to read environment: %v", err) - return "" + enrichmentWarnf(ctx, "air get: could not create client to read environment: %v", err) + return "", "" } var resp struct { Environments []struct { EnvironmentKey string `json:"environment_key"` Spec struct { EnvironmentVersion string `json:"environment_version"` + BaseEnvironment string `json:"base_environment"` } `json:"spec"` } `json:"environments"` } @@ -189,15 +195,15 @@ func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceCli // run_id is passed as the request, mirroring the other raw calls in this package. query := map[string]any{"run_id": runID} if err := apiClient.Do(ctx, http.MethodGet, jobsRunsGetPath, auth.WorkspaceIDHeaders(w.Config), nil, query, &resp); err != nil { - log.Warnf(ctx, "air get: could not read environment for run %d: %v", runID, err) - return "" + enrichmentWarnf(ctx, "air get: could not read environment for run %d: %v", runID, err) + return "", "" } for _, e := range resp.Environments { if e.EnvironmentKey == aiRuntimeEnvironmentKey { - return e.Spec.EnvironmentVersion + return e.Spec.EnvironmentVersion, e.Spec.BaseEnvironment } } - return "" + return "", "" } // resolveConfigYAML returns the config box body: from the downloaded config file @@ -232,18 +238,53 @@ func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.G return synthConfigYAML(task), nil } -// downloadConfig fetches the run's training-config file, returning nil on -// failure (logged as a warning). Best-effort, like the rest of the enrichment. +// configDownloadTransport limits the body before SDK response handling, which +// otherwise buffers JSON responses and Workspace.Download buffers every response. +type configDownloadTransport struct{ base http.RoundTripper } + +func (t configDownloadTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + if err != nil { + return resp, err + } + resp.Body = struct { + io.Reader + io.Closer + }{io.LimitReader(resp.Body, maxConfigYAMLBytes+1), resp.Body} + return resp, nil +} + +// downloadConfig fetches at most 1 MiB; oversized and unreadable configs are omitted. func downloadConfig(ctx context.Context, w *databricks.WorkspaceClient, path string) []byte { - r, err := w.Workspace.Download(ctx, path) + cfg, err := config.HTTPClientConfigFromConfig(w.Config) if err != nil { - log.Warnf(ctx, "air get: could not download training config %s: %v", path, err) + enrichmentWarnf(ctx, "air get: could not create client to download config: %v", err) + return nil + } + transport := cfg.Transport + if transport == nil { + transport = http.DefaultTransport + } + cfg.Transport = configDownloadTransport{base: transport} + apiClient := httpclient.NewApiClient(cfg) + var r io.ReadCloser + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/workspace/export", + httpclient.WithRequestHeaders(auth.WorkspaceIDHeaders(w.Config)), + httpclient.WithRequestHeader("Content-Type", "application/json"), + httpclient.WithRequestData(map[string]any{"path": path, "direct_download": true}), + httpclient.WithResponseUnmarshal(&r)) + if err != nil { + enrichmentWarnf(ctx, "air get: could not download training config %s: %v", path, err) return nil } defer r.Close() content, err := io.ReadAll(r) if err != nil { - log.Warnf(ctx, "air get: could not read training config %s: %v", path, err) + enrichmentWarnf(ctx, "air get: could not read training config %s: %v", path, err) + return nil + } + if len(content) > maxConfigYAMLBytes { + enrichmentWarnf(ctx, "air get: training config %s exceeds the %d-byte limit", path, maxConfigYAMLBytes) return nil } return content diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index cd25947529a..10dd316b69a 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -2,16 +2,18 @@ package aircmd import ( "io" + "net/http" + "net/http/httptest" "strings" "testing" "github.com/charmbracelet/lipgloss" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/muesli/termenv" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) // asciiPalette returns a palette whose styles emit no escape codes, so render @@ -21,6 +23,26 @@ func asciiPalette(t *testing.T) palette { return newPalette(r) } +// A local HTTP server exercises the SDK export path without a live workspace. +func configTestClient(t *testing.T, path, content string) *databricks.WorkspaceClient { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.0/workspace/export" { + assert.Equal(t, path, r.URL.Query().Get("path")) + assert.Equal(t, "true", r.URL.Query().Get("direct_download")) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.Empty(t, body) + _, _ = io.WriteString(w, content) + return + } + _, _ = io.WriteString(w, `{}`) + })) + t.Cleanup(srv.Close) + return newTestWorkspaceClient(t, srv.URL) +} + func TestConfigYAML(t *testing.T) { ctx := t.Context() @@ -36,12 +58,9 @@ func TestConfigYAML(t *testing.T) { }) t.Run("downloads the parameters file when there are no inline parameters", func(t *testing.T) { - m := mocks.NewMockWorkspaceClient(t) - m.GetMockWorkspaceAPI().EXPECT(). - Download(mock.Anything, "/Workspace/cfg.yaml"). - Return(io.NopCloser(strings.NewReader("experiment_name: from-file\n")), nil) + w := configTestClient(t, "/Workspace/cfg.yaml", "experiment_name: from-file\n") task := &jobs.GenAiComputeTask{YamlParametersFilePath: "/Workspace/cfg.yaml"} - got, raw := configYAML(ctx, m.WorkspaceClient, task) + got, raw := configYAML(ctx, w, task) assert.Equal(t, "experiment_name: from-file", got) assert.Equal(t, "experiment_name: from-file\n", string(raw)) }) @@ -57,17 +76,65 @@ func TestConfigYAML(t *testing.T) { }) } +func TestDownloadConfigSizeLimit(t *testing.T) { + for _, tc := range []struct { + name string + size int + wantNil bool + }{ + {"at limit", maxConfigYAMLBytes, false}, + {"above limit", maxConfigYAMLBytes + 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := configTestClient(t, "/Workspace/cfg.yaml", strings.Repeat("a", tc.size)) + got := downloadConfig(withQuietEnrichment(t.Context()), w, "/Workspace/cfg.yaml") + if tc.wantNil { + assert.Nil(t, got) + } else { + assert.Len(t, got, tc.size) + } + }) + } +} + +type configCountingReader struct { + reader io.Reader + bytesRead int +} + +func (r *configCountingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.bytesRead += n + return n, err +} + +func TestDownloadConfigLimitsSDKResponseReads(t *testing.T) { + for _, contentType := range []string{"application/json", "application/octet-stream"} { + t.Run(contentType, func(t *testing.T) { + body := &configCountingReader{reader: strings.NewReader(strings.Repeat("x", 4*maxConfigYAMLBytes))} + // The transport simulates a large workspace export without a live service. + transport := validationRoundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/api/2.0/workspace/export" { + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{contentType}}, Body: io.NopCloser(body), Request: req}, nil + }) + w := validationTestWorkspaceClientWithTransport(t, transport) + got := downloadConfig(withQuietEnrichment(t.Context()), w, "/Workspace/cfg.yaml") + assert.Nil(t, got) + assert.LessOrEqual(t, body.bytesRead, maxConfigYAMLBytes+1) + }) + } +} + func TestResolveConfigYAML(t *testing.T) { ctx := t.Context() t.Run("downloads the training config path for new runs", func(t *testing.T) { - m := mocks.NewMockWorkspaceClient(t) - m.GetMockWorkspaceAPI().EXPECT(). - Download(mock.Anything, "/Workspace/run/training_config.yaml"). - Return(io.NopCloser(strings.NewReader("experiment_name: from-air\n")), nil) + w := configTestClient(t, "/Workspace/run/training_config.yaml", "experiment_name: from-air\n") run := &jobs.Run{Tasks: []jobs.RunTask{{}}} data := &getData{TrainingConfigPath: "/Workspace/run/training_config.yaml"} - got, raw := resolveConfigYAML(ctx, m.WorkspaceClient, run, data) + got, raw := resolveConfigYAML(ctx, w, run, data) assert.Equal(t, "experiment_name: from-air", got) assert.Equal(t, "experiment_name: from-air\n", string(raw)) }) diff --git a/cmd/air/sweep.go b/cmd/air/sweep.go index b346f43f1b6..ba25114e28e 100644 --- a/cmd/air/sweep.go +++ b/cmd/air/sweep.go @@ -10,22 +10,25 @@ import ( ) // sweepInfo summarizes a "foreach" run, which fans a single config out into many -// iterations (a hyperparameter sweep). It is shown only in text output. +// iterations (a hyperparameter sweep). type sweepInfo struct { - Total int - Succeeded int - Failed int - Active int - Completed int - Tasks []sweepTask + Total int `json:"total"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + Active int `json:"active"` + Completed int `json:"completed"` + Tasks []sweepTask `json:"tasks"` } // sweepTask is one iteration of a sweep. type sweepTask struct { - TaskKey string - RunID string - Status string - Experiment string + TaskKey string `json:"task_key"` + RunID string `json:"run_id"` + Status string `json:"status"` + // ExperimentName is null when the iteration names no experiment; Experiment + // is the same value for the text table, which shows it blank. + ExperimentName *string `json:"experiment_name"` + Experiment string `json:"-"` } // findForEachTask returns the run's foreach task if it has one, or nil. A run is @@ -44,7 +47,7 @@ func findForEachTask(run *jobs.Run) *jobs.RunTask { // iterations require a second lookup. If that lookup fails we still return the // counts (logging the failure) so the user sees the summary. func buildSweepInfo(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.RunTask) *sweepInfo { - info := &sweepInfo{} + info := &sweepInfo{Tasks: []sweepTask{}} if task.ForEachTask.Stats != nil && task.ForEachTask.Stats.TaskRunStats != nil { stats := task.ForEachTask.Stats.TaskRunStats info.Total = stats.TotalIterations @@ -68,7 +71,9 @@ func buildSweepInfo(ctx context.Context, w *databricks.WorkspaceClient, task *jo Status: runStatus(it.State), } if it.GenAiComputeTask != nil && it.GenAiComputeTask.MlflowExperimentName != "" { - row.Experiment = stripExperimentUserPrefix(it.GenAiComputeTask.MlflowExperimentName) + exp := stripExperimentUserPrefix(it.GenAiComputeTask.MlflowExperimentName) + row.Experiment = exp + row.ExperimentName = &exp } info.Tasks = append(info.Tasks, row) } diff --git a/cmd/air/sweep_test.go b/cmd/air/sweep_test.go index 10134c0df42..f5987236380 100644 --- a/cmd/air/sweep_test.go +++ b/cmd/air/sweep_test.go @@ -1,6 +1,7 @@ package aircmd import ( + "encoding/json" "testing" "github.com/databricks/databricks-sdk-go/apierr" @@ -77,5 +78,8 @@ func TestBuildSweepInfo(t *testing.T) { info := buildSweepInfo(ctx, m.WorkspaceClient, sweepTaskFixture()) assert.Equal(t, 4, info.Total) assert.Empty(t, info.Tasks) + encoded, err := json.Marshal(info) + require.NoError(t, err) + assert.JSONEq(t, `{"total":4,"succeeded":1,"failed":1,"active":2,"completed":2,"tasks":[]}`, string(encoded)) }) } From 6059ed203fdf8cbf8deeef296863ed51fb2b471b Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:48:05 +0000 Subject: [PATCH 3/8] air: link the run details changelog fragment to the pull request Co-authored-by: Isaac --- .nextchanges/air/get-json-details.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/get-json-details.md b/.nextchanges/air/get-json-details.md index e73de015bc6..ccb20f01ecf 100644 --- a/.nextchanges/air/get-json-details.md +++ b/.nextchanges/air/get-json-details.md @@ -1 +1 @@ -* `air get -o json` includes run details including flat `accelerator_type` and `num_accelerators`. Use `--include-config` to include stored config, which may contain plaintext `env_variables` and parameters; downloads over 1 MiB are omitted, and config-dependent `max_retries` is otherwise null. +* `air get -o json` includes run details including flat `accelerator_type` and `num_accelerators`. Use `--include-config` to include stored config, which may contain plaintext `env_variables` and parameters; downloads over 1 MiB are omitted, and config-dependent `max_retries` is otherwise null. ([#7624](https://github.com/databricks/cli/pull/7624)) From 56727e6740165b276969def6b19f1cce7cdef200 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:48:11 +0000 Subject: [PATCH 4/8] air: link the max retries changelog fragment to the pull request Co-authored-by: Isaac --- .nextchanges/air/get-max-retries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/get-max-retries.md b/.nextchanges/air/get-max-retries.md index 46f34b25cf3..64792857e5c 100644 --- a/.nextchanges/air/get-max-retries.md +++ b/.nextchanges/air/get-max-retries.md @@ -1 +1 @@ -* `air get` and the `air list` info pane no longer show `Max Retries 0` when the Jobs API leaves the task's retry limit out. For a run submitted with `air run`, the limit now comes from the run's uploaded config, shown as `3 (CLI default)` when the config doesn't set `max_retries`; it shows `N/A` when the limit can't be known (job runs, multi-container runs, legacy runs with no explicit value, or an unreadable config). +* `air get` and the `air list` info pane no longer show `Max Retries 0` when the Jobs API leaves the task's retry limit out. For a run submitted with `air run`, the limit now comes from the run's uploaded config, shown as `3 (CLI default)` when the config doesn't set `max_retries`; it shows `N/A` when the limit can't be known (job runs, multi-container runs, legacy runs with no explicit value, or an unreadable config). ([#7623](https://github.com/databricks/cli/pull/7623)) From d3bdec7bef7103b6c62114cb2c5ce05dea3cadce Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:53:48 +0000 Subject: [PATCH 5/8] air: use neutral wording for the runs/get sample source Co-authored-by: Isaac --- cmd/air/format_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 3b7edac72d0..ee2bbe9de14 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -360,7 +360,7 @@ func TestResolveMaxRetriesSource(t *testing.T) { assert.Equal(t, &runMaxRetries{Limit: 2, Source: maxRetriesReported}, resolveMaxRetries(run, nil)) } -// The runs/get bodies below are trimmed copies of real e2-dogfood responses: +// The runs/get bodies below are trimmed copies of real workspace responses: // task runs carry no max_retries key at all. func TestMaxRetriesFromDecodedRunGet(t *testing.T) { decode := func(body string) *jobs.Run { From 9186e20d4ef54c6de6e33464572461f58042c7de Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:55:02 +0000 Subject: [PATCH 6/8] air: use synthetic run IDs in the retry-limit unit test Co-authored-by: Isaac --- cmd/air/format_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index ee2bbe9de14..04c74c07dd1 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -371,10 +371,10 @@ func TestMaxRetriesFromDecodedRunGet(t *testing.T) { display := func(run *jobs.Run, config []byte) string { return maxRetriesDisplay(resolveMaxRetries(run, config)) } - // Run 915071793879841: submitted with max_retries 1, failed, retried once. - aiRuntime := decode(`{"run_id": 915071793879841, "run_type": "SUBMIT_RUN", "tasks": [ - {"task_key": "t", "run_id": 996740861702356, "attempt_number": 0, "ai_runtime_task": {"experiment": "e"}}, - {"task_key": "t", "run_id": 286970869718720, "attempt_number": 1, "ai_runtime_task": {"experiment": "e"}}]}`) + // Run 111111111111111: submitted with max_retries 1, failed, retried once. + aiRuntime := decode(`{"run_id": 111111111111111, "run_type": "SUBMIT_RUN", "tasks": [ + {"task_key": "t", "run_id": 222222222222222, "attempt_number": 0, "ai_runtime_task": {"experiment": "e"}}, + {"task_key": "t", "run_id": 333333333333333, "attempt_number": 1, "ai_runtime_task": {"experiment": "e"}}]}`) assert.Equal(t, "1", display(aiRuntime, []byte("experiment_name: e\nmax_retries: 1\n"))) assert.Equal(t, "3 (CLI default)", display(aiRuntime, []byte("experiment_name: e\n"))) From cb575dd9843195fcf72e70729d0c715fd7e04965 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Sat, 14 Nov 2026 00:15:25 +0000 Subject: [PATCH 7/8] air: normalize CRLF in the reported run config The config downloaded for the JSON output is reported verbatim, so a config uploaded from a Windows checkout (CRLF) produced different JSON than the same config uploaded from Unix (LF). Normalize the line endings for the machine-readable field; the text display already reformats the config. Co-authored-by: Isaac --- cmd/air/get.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/air/get.go b/cmd/air/get.go index 82ffeba0042..76664aaf24d 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -6,6 +6,7 @@ import ( "fmt" "path" "strconv" + "strings" "sync" "github.com/databricks/cli/cmd/root" @@ -386,7 +387,9 @@ func enrichRunDetails(ctx context.Context, w *databricks.WorkspaceClient, run *j data.EnvironmentDisplay = orNA(version) data.configBody = body if len(raw) > 0 { - cfg := string(raw) + // Normalize CRLF so the machine-readable output is identical for the + // same config uploaded from Windows and Unix checkouts. + cfg := strings.ReplaceAll(string(raw), "\r\n", "\n") data.Config = &cfg data.ConfigSource = configSource(run, data) } From 4d6c177dee5157d989a38bc0d9de72f42f47c86e Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Mon, 16 Nov 2026 18:42:02 +0000 Subject: [PATCH 8/8] air: update the control-chars golden for the config-derived retry limit The fixture's fake run has no max_retries on its task, which the Jobs API leaves out, so the reported limit is now N/A instead of 0. Co-authored-by: Isaac --- acceptance/air/control-chars/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/air/control-chars/output.txt b/acceptance/air/control-chars/output.txt index b320060ce29..3362d6aa1ce 100644 --- a/acceptance/air/control-chars/output.txt +++ b/acceptance/air/control-chars/output.txt @@ -9,7 +9,7 @@ │ Status ● SUCCESS │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ -│ Max Retries 0 │ +│ Max Retries N/A │ │ Duration 12s │ │ Experiment my-exp │ │ MLflow Run my-run │