From 08a6bf900f78309addc90cfc8957c99f261170d1 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:32:20 +0000 Subject: [PATCH 1/9] air: find the AI Runtime task anywhere in a multi-task job in `air list` and `air get` Multi-task AIR jobs, such as the official DAB example that runs a notebook prep task before its ai_runtime_task, were omitted from `air list` and showed N/A experiment and accelerators in `air get`, because both read tasks[0]. - Discover the first AIR task in task order: an ai_runtime_task, a legacy gen_ai_compute_task with a training script, or a foreach wrapping one. Non-AIR tasks no longer feed status, timing or the MLflow lookup. - A job with several AIR tasks lists one row per task. Rows share the run_id; JSON output gains task_key. Each row reads its own task runs. - `air get` gains --task-key (default: the first AIR task), its JSON gains task_key, and the text view adds a Task line when the run has several. - The list detail pane opens the selected row's task; `cancel --all` cancels a multi-row run once. - The index-path row cache stores one row per AIR task (air-list-runs-v3). Verified against a real run of universe ai-compute/dabs/examples/03_multitask_dag on e2-dogfood (run 47834518189346). Co-authored-by: Isaac --- acceptance/air/control-chars/output.txt | 1 + acceptance/air/get-ai-runtime/output.txt | 4 + acceptance/air/get-mlflow-retry/output.txt | 1 + acceptance/air/get-multitask/out.test.toml | 1 + acceptance/air/get-multitask/output.txt | 139 +++++++++++++++++ acceptance/air/get-multitask/script | 17 +++ acceptance/air/get-multitask/test.toml | 84 +++++++++++ acceptance/air/get-pending/output.txt | 1 + acceptance/air/get/output.txt | 1 + acceptance/air/help/output.txt | 6 + acceptance/air/list-index-fallback/output.txt | 1 + acceptance/air/list/output.txt | 21 ++- acceptance/air/list/test.toml | 34 ++++- cmd/air/cancel.go | 17 +++ cmd/air/format.go | 14 +- cmd/air/format_test.go | 18 +++ cmd/air/get.go | 68 ++++++++- cmd/air/get_test.go | 99 +++++++++++- cmd/air/joblist.go | 120 ++++++++++++--- cmd/air/joblist_test.go | 142 ++++++++++++++++++ cmd/air/list.go | 14 +- cmd/air/list_cache.go | 55 ++++--- cmd/air/list_cache_test.go | 52 ++++++- cmd/air/list_detail.go | 11 +- cmd/air/list_detail_test.go | 39 ++++- cmd/air/list_format.go | 42 +++++- cmd/air/list_index.go | 29 ++-- cmd/air/list_test.go | 76 +++++++++- cmd/air/list_tui.go | 9 +- cmd/air/render.go | 19 ++- 30 files changed, 1032 insertions(+), 103 deletions(-) create mode 100644 acceptance/air/get-multitask/out.test.toml create mode 100644 acceptance/air/get-multitask/output.txt create mode 100644 acceptance/air/get-multitask/script create mode 100644 acceptance/air/get-multitask/test.toml diff --git a/acceptance/air/control-chars/output.txt b/acceptance/air/control-chars/output.txt index f5d95c03e06..524712a6cb1 100644 --- a/acceptance/air/control-chars/output.txt +++ b/acceptance/air/control-chars/output.txt @@ -38,6 +38,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "runs": [ { "run_id": "123", + "task_key": "train", "run_name": "qwen\u001b[2J-train", "user": "user\u001b[31m@example.com", "status": "SUCCESS", diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 034d307bd1f..051a9c8371f 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -43,6 +43,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -73,6 +74,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -142,6 +144,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -172,6 +175,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/get-mlflow-retry/output.txt b/acceptance/air/get-mlflow-retry/output.txt index 17e49f3febf..e4fb6bb7be0 100644 --- a/acceptance/air/get-mlflow-retry/output.txt +++ b/acceptance/air/get-mlflow-retry/output.txt @@ -7,6 +7,7 @@ "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "FAILED", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/get-multitask/out.test.toml b/acceptance/air/get-multitask/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-multitask/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-multitask/output.txt b/acceptance/air/get-multitask/output.txt new file mode 100644 index 00000000000..ee5a8fa2f17 --- /dev/null +++ b/acceptance/air/get-multitask/output.txt @@ -0,0 +1,139 @@ + +=== get (text): the first AI Runtime task, not the notebook prep task + +>>> [CLI] air get 123 + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Task train │ +│ Status ● SUCCESS │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 0 │ +│ Duration 10s │ +│ Experiment air-byot-multitask-dag │ +│ MLflow Run matmul-smoke │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment 5 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get --task-key eval (text) + +>>> [CLI] air get 123 --task-key eval + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Task eval │ +│ Status ● SUCCESS │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 2 │ +│ Duration 8s │ +│ Experiment eval-exp │ +│ MLflow Run matmul-smoke │ +│ User user@example.com │ +│ Accelerators 8x H100 │ +│ Environment 5 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 10, + "attempt_number": 0, + "experiment_name": "air-byot-multitask-dag", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "mlflow_experiment_id": "exp1", + "mlflow_run_id": "run1", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "5", + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": "matmul-smoke", + "sweep": null + } +} + +=== get --task-key eval (json) + +>>> [CLI] air get 123 --task-key eval -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "eval", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 8, + "attempt_number": 0, + "experiment_name": "eval-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "mlflow_experiment_id": "exp1", + "mlflow_run_id": "run1", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_8xH100", + "num_accelerators": 8, + "environment_version": "5", + "base_environment": null, + "max_retries": { + "limit": 2, + "source": "reported" + }, + "mlflow_run_name": "matmul-smoke", + "sweep": null + } +} + +=== get --task-key prep: not an AI Runtime task + +>>> [CLI] air get 123 --task-key prep +Error: run 123 has no AI Runtime task "prep"; its AI Runtime tasks are: train, eval + +Exit code: 1 + +=== get --task-key prep (json) + +>>> [CLI] air get 123 --task-key prep -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "error": { + "code": "INVALID_ARGS", + "kind": "PERMANENT", + "message": "run 123 has no AI Runtime task \"prep\"; its AI Runtime tasks are: train, eval", + "retryable": false + } +} + +Exit code: 1 diff --git a/acceptance/air/get-multitask/script b/acceptance/air/get-multitask/script new file mode 100644 index 00000000000..d4d4505995f --- /dev/null +++ b/acceptance/air/get-multitask/script @@ -0,0 +1,17 @@ +title "get (text): the first AI Runtime task, not the notebook prep task" +trace $CLI air get 123 + +title "get --task-key eval (text)" +trace $CLI air get 123 --task-key eval + +title "get (json)" +trace $CLI air get 123 -o json + +title "get --task-key eval (json)" +trace $CLI air get 123 --task-key eval -o json + +title "get --task-key prep: not an AI Runtime task" +errcode trace $CLI air get 123 --task-key prep + +title "get --task-key prep (json)" +errcode trace $CLI air get 123 --task-key prep -o json diff --git a/acceptance/air/get-multitask/test.toml b/acceptance/air/get-multitask/test.toml new file mode 100644 index 00000000000..751f09abb17 --- /dev/null +++ b/acceptance/air/get-multitask/test.toml @@ -0,0 +1,84 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. + +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A multi-task job in the shape of the official DAB example (universe +# ai-compute/dabs/examples/03_multitask_dag): a notebook prep task first, then the +# ai_runtime_task that depends on it, plus a second ai_runtime_task. `air get` +# must read the AI Runtime task wherever it sits, not tasks[0]. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "environments": [ + {"environment_key": "default", "spec": {"environment_version": "5"}} + ], + "tasks": [ + { + "task_key": "prep", + "run_id": 455, + "attempt_number": 0, + "start_time": 1700000000000, + "end_time": 1700000002000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "notebook_task": {"notebook_path": "/Workspace/Users/user@example.com/prep"} + }, + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "start_time": 1700000002000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "depends_on": [{"task_key": "prep"}], + "environment_key": "default", + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [ + {"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}} + ] + } + }, + { + "task_key": "eval", + "run_id": 457, + "attempt_number": 0, + "max_retries": 2, + "start_time": 1700000012000, + "end_time": 1700000020000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "depends_on": [{"task_key": "train"}], + "environment_key": "default", + "ai_runtime_task": { + "experiment": "/Users/user@example.com/eval-exp", + "deployments": [ + {"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 8}} + ] + } + } + ] +} +''' + +# MLflow identifiers for the deep link, fetched for the selected task's run. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}} +''' + +[[Server]] +Pattern = "GET /api/2.0/mlflow/runs/get" +Response.Body = ''' +{"run": {"info": {"run_name": "matmul-smoke"}}} +''' diff --git a/acceptance/air/get-pending/output.txt b/acceptance/air/get-pending/output.txt index 295de735ce5..930980364f2 100644 --- a/acceptance/air/get-pending/output.txt +++ b/acceptance/air/get-pending/output.txt @@ -5,6 +5,7 @@ "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "PENDING", "started_at": null, "duration_seconds": null, diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index ef11028e860..6c85755a318 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -43,6 +43,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 6f75b4afcc6..1a649d62749 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -92,12 +92,18 @@ Show a run's status, configuration, and timing. JOB_RUN_ID is the job run ID returned by "databricks air run". +A job may run other tasks around its AI Runtime task (for example a notebook +that prepares data first); the view shows the first AI Runtime task in task +order. When a job has several AI Runtime tasks, pass --task-key to show another +one. + Usage: databricks air get JOB_RUN_ID [flags] Flags: -h, --help[=true|false] help for get --include-config[=true|false] Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters) + --task-key string Show this AI Runtime task of the run (default: the first in task order) Global Flags: --debug[=true|false] enable debug logging diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt index 8cc9da7c7fd..3651c47ad0a 100644 --- a/acceptance/air/list-index-fallback/output.txt +++ b/acceptance/air/list-index-fallback/output.txt @@ -14,6 +14,7 @@ Warn: air list: the AI Runtime run index is unavailable (failed to list training "runs": [ { "run_id": "334747067049496", + "task_key": "", "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 5f76708bf45..5270a44ce52 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -2,8 +2,9 @@ === list (text) >>> [CLI] air list - Run ID Experiment Status Started Duration Progress MLflow User Accelerators - 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + Run ID Experiment Status Started Duration Progress MLflow User Accelerators + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + 334747067049500 air-byot-multitask-dag ● SUCCESS [TIMESTAMP] 20s - qwen-train-001 [USERNAME] 1x A10 === list (json) @@ -15,6 +16,7 @@ "runs": [ { "run_id": "334747067049496", + "task_key": "", "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", @@ -25,6 +27,20 @@ "accelerator_type": "GPU_8xH100", "num_accelerators": 8, "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049496?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049500", + "task_key": "train", + "run_name": "air-byot-multitask-dag", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "air-byot-multitask-dag", + "duration_seconds": 20, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049500?o=[WORKSPACE_ID]" } ] } @@ -46,6 +62,7 @@ "runs": [ { "run_id": "334747067049496", + "task_key": "", "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/test.toml b/acceptance/air/list/test.toml index 9d5ca13df25..9ae1417ce39 100644 --- a/acceptance/air/list/test.toml +++ b/acceptance/air/list/test.toml @@ -10,8 +10,10 @@ Response.Body = '' # `air list` reads Jobs runs/list directly (expand_tasks), filtering to AIR runs. # The current runs carry an ai_runtime_task (not modeled by the typed SDK), which # is why the CLI parses the response raw. Both runs belong to the default user -# (tester@databricks.com, from the built-in scim/v2/Me handler). The second run -# has no AI task and must be filtered out. +# (tester@databricks.com, from the built-in scim/v2/Me handler). The second run is +# the official multi-task DAB shape, a notebook prep task before the +# ai_runtime_task, and must be listed from its AIR task. The third run has no AI +# task and must be filtered out. [[Server]] Pattern = "GET /api/2.2/jobs/runs/list" Response.Body = ''' @@ -32,6 +34,34 @@ Response.Body = ''' } }] }, + { + "run_id": 334747067049500, + "run_name": "air-byot-multitask-dag", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608700000, + "end_time": 1717608730000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [ + { + "task_key": "prep", + "run_id": 334747067049501, + "start_time": 1717608700000, + "end_time": 1717608705000, + "notebook_task": {"notebook_path": "/Workspace/Users/tester@databricks.com/prep"} + }, + { + "task_key": "train", + "run_id": 334747067049502, + "start_time": 1717608710000, + "end_time": 1717608730000, + "depends_on": [{"task_key": "prep"}], + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + } + ] + }, { "run_id": 999000999000, "run_name": "not-an-air-run", diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index 0ca93c793cd..36ad20fe309 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -106,6 +106,9 @@ all active runs submitted by the current user. fmt.Errorf("failed to list active runs: %w", err)) } + // A job with several AI runtime tasks lists one row per task; the + // run is cancelled (and previewed) once. + rows = firstRowPerRun(rows) runIDs = make([]string, 0, len(rows)) for i := range rows { if rows[i].RunID != "" { @@ -215,6 +218,20 @@ func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, s return err } +// firstRowPerRun keeps the first row of each run id, in order. +func firstRowPerRun(rows []listRow) []listRow { + seen := make(map[string]bool, len(rows)) + unique := make([]listRow, 0, len(rows)) + for _, row := range rows { + if seen[row.RunID] { + continue + } + seen[row.RunID] = true + unique = append(unique, row) + } + return unique +} + // displayCancelPreview shows the runs that `cancel --all` is about to terminate. func displayCancelPreview(ctx context.Context, rows []listRow, host string) { var sb strings.Builder diff --git a/cmd/air/format.go b/cmd/air/format.go index d73c2d8a620..2e8a112f682 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -120,10 +120,11 @@ func runWaitingForCompute(run *jobs.Run) bool { if run.State.LifeCycleState == jobs.RunLifeCycleStatePending { return true } - if run.State.LifeCycleState != jobs.RunLifeCycleStateRunning || len(run.Tasks) == 0 || run.Tasks[0].State == nil { + task := leadTask(run) + if run.State.LifeCycleState != jobs.RunLifeCycleStateRunning || task == nil || task.State == nil { return false } - switch run.Tasks[0].State.LifeCycleState { + switch task.State.LifeCycleState { case jobs.RunLifeCycleStatePending, jobs.RunLifeCycleStateQueued, jobs.RunLifeCycleStateWaitingForRetry, jobs.RunLifeCycleStateBlocked: return true @@ -157,8 +158,8 @@ func terminationReason(run *jobs.Run) *string { return &message } } - if len(run.Tasks) > 0 && run.Tasks[0].State != nil { - if message := strings.TrimSpace(run.Tasks[0].State.StateMessage); message != "" { + if task := leadTask(run); task != nil && task.State != nil { + if message := strings.TrimSpace(task.State.StateMessage); message != "" { return &message } } @@ -300,10 +301,11 @@ func latestAttemptNumber(run *jobs.Run) int { // "/Users/me@example.com/my-experiment"); we strip that prefix so users see just // the experiment name they chose. func experimentName(run *jobs.Run) *string { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - task := run.Tasks[0].GenAiComputeTask + task := t.GenAiComputeTask if task == nil || task.MlflowExperimentName == "" { return nil } diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 306e0bc3e1c..230aa4ba02d 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -384,3 +384,21 @@ func TestMaxRetriesFromDecodedRunGet(t *testing.T) { explicitZero := decode(`{"run_type": "SUBMIT_RUN", "tasks": [{"max_retries": 0, "ai_runtime_task": {}}]}`) assert.Equal(t, "0", display(explicitZero, []byte("experiment_name: e\n"))) } + +func TestLegacyExperimentAndAcceleratorsSkipNonAirTasks(t *testing.T) { + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + {TaskKey: "train", RunId: 52, GenAiComputeTask: &jobs.GenAiComputeTask{ + TrainingScriptPath: "/train.py", + MlflowExperimentName: "/Users/me@example.com/legacy-exp", + Compute: &jobs.ComputeConfig{GpuType: "H100", NumGpus: 8}, + }}, + }} + name := experimentName(run) + require.NotNil(t, name) + assert.Equal(t, "legacy-exp", *name) + gpu, n := jobCompute(run) + assert.Equal(t, "H100", gpu) + assert.Equal(t, 8, n) + assert.Equal(t, "8x H100", buildGetData(run).AcceleratorsDisplay) +} diff --git a/cmd/air/get.go b/cmd/air/get.go index 7a4de046fef..3d8f080f64e 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -26,7 +26,10 @@ import ( // the machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable text view. type getData struct { - RunID string `json:"run_id"` + RunID string `json:"run_id"` + // TaskKey is the AI runtime task shown (see selectAirTask), or null for a + // run without one. + TaskKey *string `json:"task_key"` Status string `json:"status"` StartedAt *string `json:"started_at"` DurationSeconds *int64 `json:"duration_seconds"` @@ -88,6 +91,9 @@ type getData struct { // Each shows "N/A" when its value is missing. The styled single-run renderer // (render.go) consumes them; the Run ID, Status, and MLflow Run cells it draws // are styled and hyperlinked there rather than stored here. + // TaskDisplay is the "Task" cell, set only when the run has several AI + // runtime tasks, so the view says which one it shows. + TaskDisplay string `json:"-"` SubmittedDisplay string `json:"-"` DurationDisplay string `json:"-"` ExperimentDisplay string `json:"-"` @@ -159,19 +165,26 @@ func credentialError(ctx context.Context, w *databricks.WorkspaceClient, err err // configuration, and timing details for a specific run. func newGetCommand() *cobra.Command { var includeConfig bool + var taskKey string cmd := &cobra.Command{ Use: "get JOB_RUN_ID", Args: root.ExactArgs(1), Short: "Show a run's status, configuration, and timing", Long: `Show a run's status, configuration, and timing. -JOB_RUN_ID is the job run ID returned by "databricks air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run". + +A job may run other tasks around its AI Runtime task (for example a notebook +that prepares data first); the view shows the first AI Runtime task in task +order. When a job has several AI Runtime tasks, pass --task-key to show another +one.`, Annotations: map[string]string{ "template": getTemplate, }, } cmd.Flags().BoolVar(&includeConfig, "include-config", false, "Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters)") + cmd.Flags().StringVar(&taskKey, "task-key", "", "Show this AI Runtime task of the run (default: the first in task order)") // Resolve and authenticate the workspace client up front so an auth failure // fails fast here, before any run status or config is fetched or printed. @@ -212,6 +225,14 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, fmt.Errorf("failed to get status for run %d: %w", runID, err)) } + // From here on, run holds only the selected AI runtime task's task runs, + // so the status, timing, MLflow and config lookups all read that task. + fullRun := run + run, err = selectAirTask(run, taskKey) + if err != nil { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) + } + // Prefer the configured workspace ID; only look it up when it is unset. id, err := auth.ResolveWorkspaceID(ctx, w) if err != nil { @@ -225,6 +246,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, } data := buildGetData(run) + annotateTask(fullRun, run, &data) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) @@ -295,6 +317,43 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, return cmd } +// selectAirTask narrows run (see taskRun) to the AI runtime task named by key, +// or to its first AI runtime task in task order when key is empty, so the rest +// of the command reads that task alone. A run with no AI runtime task is +// returned unchanged when no key is given. The error names the run's AI runtime +// tasks when key is not one of them. +func selectAirTask(run *jobs.Run, key string) (*jobs.Run, error) { + keys := airTaskKeys(run) + if key == "" { + if len(keys) == 0 { + return run, nil + } + key = keys[0] + } + if narrowed := taskRun(run, key); narrowed != nil { + return narrowed, nil + } + if len(keys) == 0 { + return nil, fmt.Errorf("run %d has no AI Runtime task %q; it has no AI Runtime tasks", run.RunId, key) + } + return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) +} + +// annotateTask records which AI runtime task of full the view shows: selected +// is full narrowed to it (see selectAirTask). The text cell is set only when +// full has several AI runtime tasks. +func annotateTask(full, selected *jobs.Run, data *getData) { + t := leadTask(selected) + if t == nil { + return + } + key := t.TaskKey + data.TaskKey = &key + if len(airTaskKeys(full)) > 1 { + data.TaskDisplay = key + } +} + // enrichFromAiRuntimeTask fills the config path, experiment, and accelerators // from the run's ai_runtime_task. Best-effort: empty fields leave the existing // "N/A" fallbacks in place. @@ -319,10 +378,11 @@ func enrichFromAiRuntimeTask(run *jobs.Run, data *getData) { // aiRuntimeTaskOf returns the run's first ai_runtime_task, or nil. func aiRuntimeTaskOf(run *jobs.Run) *jobs.AiRuntimeTask { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - return run.Tasks[0].AiRuntimeTask + return t.AiRuntimeTask } // buildGetData extracts the fields we display from a run. The text-view cells diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index dc6eb3636c0..05f72a1cf8b 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -428,7 +428,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("an older retry's mlflow run names the retry", func(t *testing.T) { data := getRunJSON(t, srv, "123") assert.JSONEq(t, `{ - "run_id":"123","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"123","task_key":"train","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":2,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/123?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs", @@ -444,7 +444,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("the latest attempt's mlflow run has no mlflow_retry key", func(t *testing.T) { data := getRunJSON(t, srv, "124") assert.JSONEq(t, `{ - "run_id":"124","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"124","task_key":"train","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":0,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/124?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs", @@ -539,7 +539,7 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) { body, err := json.Marshal(failed) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"5","status":"FAILED","started_at":null,"duration_seconds":null, + "run_id":"5","task_key":null,"status":"FAILED","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, "mlflow_experiment_id":null,"mlflow_run_id":null, "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null, @@ -564,7 +564,7 @@ func TestBuildGetDataWaitingForComputeStatus(t *testing.T) { require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null, + "run_id":"8","task_key":null,"status":"PENDING","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, "mlflow_experiment_id":null,"mlflow_run_id":null, "est_remaining_seconds":null,"est_percent_complete":null, @@ -843,3 +843,94 @@ func TestSweepJSON(t *testing.T) { {"task_key":"t0","run_id":"10","status":"SUCCESS","experiment_name":"exp"}, {"task_key":"t1","run_id":"11","status":"RUNNING","experiment_name":null}]}`, string(got.Sweep)) } + +func TestAiRuntimeTaskOfSkipsNonAirTasks(t *testing.T) { + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + aiRuntimeTask("train", 52, "/Users/me@example.com/exp", "GPU_1xA10", 1), + }} + task := aiRuntimeTaskOf(run) + require.NotNil(t, task) + assert.Equal(t, "/Users/me@example.com/exp", task.Experiment) +} + +// multiTaskGetServer serves runs/get for a job with a notebook prep task and two +// ai_runtime_tasks, and stubs every other request. +func multiTaskGetServer(t *testing.T) *httptest.Server { + t.Helper() + run := jobs.Run{ + RunId: 5, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, + Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + aiRuntimeTask("train", 52, "/Users/me@example.com/exp-train", "GPU_1xA10", 1), + aiRuntimeTask("eval", 53, "/Users/me@example.com/exp-eval", "GPU_8xH100", 8), + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _ = json.NewEncoder(w).Encode(run) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +func runGetJSONTask(t *testing.T, w *databricks.WorkspaceClient, taskKey string) (string, error) { + t.Helper() + var buf bytes.Buffer + ctx := cmdctx.SetWorkspaceClient(t.Context(), w) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &buf, &buf, "", "")) + cmd := withOutput(newGetCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + if taskKey != "" { + require.NoError(t, cmd.Flags().Set("task-key", taskKey)) + } + err := cmd.RunE(cmd, []string{"5"}) + return buf.String(), err +} + +func TestGetRunSelectsAirTask(t *testing.T) { + srv := multiTaskGetServer(t) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token", WorkspaceID: "67890"}) + require.NoError(t, err) + + type got struct { + Data struct { + TaskKey *string `json:"task_key"` + ExperimentName *string `json:"experiment_name"` + } `json:"data"` + } + + t.Run("defaults to the first AIR task, skipping the notebook", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "") + require.NoError(t, err) + var g got + require.NoError(t, json.Unmarshal([]byte(out), &g)) + require.NotNil(t, g.Data.TaskKey) + assert.Equal(t, "train", *g.Data.TaskKey) + require.NotNil(t, g.Data.ExperimentName) + assert.Equal(t, "exp-train", *g.Data.ExperimentName) + }) + + t.Run("--task-key selects a later AIR task", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "eval") + require.NoError(t, err) + var g got + require.NoError(t, json.Unmarshal([]byte(out), &g)) + require.NotNil(t, g.Data.TaskKey) + assert.Equal(t, "eval", *g.Data.TaskKey) + require.NotNil(t, g.Data.ExperimentName) + assert.Equal(t, "exp-eval", *g.Data.ExperimentName) + }) + + t.Run("--task-key must name an AIR task", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "prep") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var e errorEnvelope + require.NoError(t, json.Unmarshal([]byte(out), &e)) + assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `run 5 has no AI Runtime task "prep"; its AI Runtime tasks are: train, eval`}, e.Error) + }) +} diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index af696f24331..85849fc0d09 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "slices" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -16,20 +17,100 @@ import ( // of run ids from the AiTrainingService index. const hydrateConcurrency = 16 -// firstTask returns the run's first task, unwrapping a foreach sweep to the -// iterated task, or nil when the run has no tasks. -func firstTask(r *jobs.Run) *jobs.RunTask { - if len(r.Tasks) == 0 { +// isAirTask reports whether a task run is an AI runtime workload: an +// ai_runtime_task, or a legacy gen_ai_compute_task with a training script. A +// foreach task counts when the task it iterates is one. +func isAirTask(t *jobs.RunTask) bool { + if t.ForEachTask != nil { + inner := t.ForEachTask.Task + return inner.AiRuntimeTask != nil || + (inner.GenAiComputeTask != nil && inner.GenAiComputeTask.TrainingScriptPath != "") + } + return t.AiRuntimeTask != nil || + (t.GenAiComputeTask != nil && t.GenAiComputeTask.TrainingScriptPath != "") +} + +// firstAirTaskIndex returns the index in r.Tasks of the run's first AI runtime +// task in task order, or -1 when it has none. A job may put other tasks before +// it (the official multi-task DAB runs a notebook prep step first), so the +// first task is not inspected directly. +func firstAirTaskIndex(r *jobs.Run) int { + for i := range r.Tasks { + if isAirTask(&r.Tasks[i]) { + return i + } + } + return -1 +} + +// airTaskKeys returns the task keys of the run's AI runtime tasks in task +// order, each once: a retried task appears in r.Tasks once per attempt. +func airTaskKeys(r *jobs.Run) []string { + var keys []string + for i := range r.Tasks { + t := &r.Tasks[i] + if isAirTask(t) && !slices.Contains(keys, t.TaskKey) { + keys = append(keys, t.TaskKey) + } + } + return keys +} + +// taskRun returns a copy of r narrowed to the task runs (every attempt, in +// order) of the AI runtime task with the given key, so the display helpers, +// which read a run's tasks, report that task alone. It returns nil when key is +// not one of the run's AI runtime tasks. +func taskRun(r *jobs.Run, key string) *jobs.Run { + if !slices.Contains(airTaskKeys(r), key) { + return nil + } + narrowed := *r + narrowed.Tasks = nil + for i := range r.Tasks { + if r.Tasks[i].TaskKey == key { + narrowed.Tasks = append(narrowed.Tasks, r.Tasks[i]) + } + } + return &narrowed +} + +// airWrapperTask returns the run's first AI runtime task run as the Jobs API +// reports it (for a sweep, the foreach wrapper, which carries the state and +// timing), or nil when the run has none. +func airWrapperTask(r *jobs.Run) *jobs.RunTask { + i := firstAirTaskIndex(r) + if i < 0 { return nil } - t := &r.Tasks[0] - if t.AiRuntimeTask != nil || t.GenAiComputeTask != nil { + return &r.Tasks[i] +} + +// leadTask returns the task run whose state and timing stand for the run: its +// first AI runtime task (for a sweep, the foreach wrapper), else, for a run +// without one, its first task; nil when the run has no tasks. +func leadTask(r *jobs.Run) *jobs.RunTask { + if t := airWrapperTask(r); t != nil { return t } + if len(r.Tasks) == 0 { + return nil + } + return &r.Tasks[0] +} + +// firstTask returns the run's first AI runtime task, unwrapping a foreach sweep +// to the iterated task. A run without one falls back to its first task, so a +// legacy run's fields are still read; nil when the run has no tasks. +func firstTask(r *jobs.Run) *jobs.RunTask { + t := leadTask(r) + if t == nil { + return nil + } if t.ForEachTask != nil { // The foreach inner task is a jobs.Task; expose the AI fields via a // synthetic RunTask so callers can treat both shapes uniformly. return &jobs.RunTask{ + TaskKey: t.TaskKey, AiRuntimeTask: t.ForEachTask.Task.AiRuntimeTask, GenAiComputeTask: t.ForEachTask.Task.GenAiComputeTask, } @@ -37,20 +118,16 @@ func firstTask(r *jobs.Run) *jobs.RunTask { return t } -// isAirRun reports whether a run is an AI runtime workload: an ai_runtime_task, -// or a legacy gen_ai_compute_task with a training script. +// isAirRun reports whether a run is an AI runtime workload: it has an +// ai_runtime_task, or a legacy gen_ai_compute_task with a training script. func isAirRun(r *jobs.Run) bool { - t := firstTask(r) - if t == nil { - return false - } - return t.AiRuntimeTask != nil || - (t.GenAiComputeTask != nil && t.GenAiComputeTask.TrainingScriptPath != "") + return firstAirTaskIndex(r) >= 0 } -// isSweep reports whether the run's first task fans out into iterations. +// isSweep reports whether the run's AI runtime task fans out into iterations. func isSweep(r *jobs.Run) bool { - return len(r.Tasks) > 0 && r.Tasks[0].ForEachTask != nil + t := leadTask(r) + return t != nil && t.ForEachTask != nil } // retryTasks returns the run's task runs with only the fields that pick its @@ -96,14 +173,13 @@ func jobCompute(r *jobs.Run) (string, int) { } // jobTiming returns the run's start and end times (epoch ms), preferring the -// first task's window so a run reports its task attempt rather than the wrapper. +// AI runtime task's window so a run reports its task attempt rather than the +// wrapper. func jobTiming(r *jobs.Run) (startMillis, endMillis int64) { startMillis, endMillis = r.StartTime, r.EndTime - if len(r.Tasks) > 0 { - if t := r.Tasks[0]; t.StartTime > 0 { - startMillis = t.StartTime - endMillis = t.EndTime - } + if t := leadTask(r); t != nil && t.StartTime > 0 { + startMillis = t.StartTime + endMillis = t.EndTime } return startMillis, endMillis } diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 33c69970dbc..2e53d89d425 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -119,3 +119,145 @@ func TestHydrateJobRunsFailsOnOtherErrors(t *testing.T) { }) } } + +// notebookTask builds a non-AIR task run, like the prep step that precedes the +// ai_runtime_task in the official multi-task DAB example. +func notebookTask(key string, runID int64) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, NotebookTask: &jobs.NotebookTask{NotebookPath: "/prep"}} +} + +// aiRuntimeTask builds a direct ai_runtime_task task run. +func aiRuntimeTask(key string, runID int64, experiment, accelType string, count int) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, AiRuntimeTask: &jobs.AiRuntimeTask{ + Experiment: experiment, + Deployments: []jobs.DeploymentSpec{{ + Compute: jobs.ComputeSpec{AcceleratorType: jobs.ComputeSpecAcceleratorType(accelType), AcceleratorCount: count}, + }}, + }} +} + +// forEachAiRuntimeTask builds a foreach (sweep) task whose iterated task is an +// ai_runtime_task. +func forEachAiRuntimeTask(key string, runID int64, experiment string) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, ForEachTask: &jobs.RunForEachTask{ + Task: jobs.Task{AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: experiment}}, + }} +} + +func TestAirTaskSelection(t *testing.T) { + tests := []struct { + name string + tasks []jobs.RunTask + air bool + keys []string + experiment string + sweep bool + }{ + { + name: "AIR task first", + tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "exp-train", "GPU_1xA10", 1), notebookTask("report", 12)}, + air: true, + keys: []string{"train"}, + experiment: "exp-train", + }, + { + name: "AIR task second", + tasks: []jobs.RunTask{notebookTask("prep", 11), aiRuntimeTask("train", 12, "exp-train", "GPU_1xA10", 1)}, + air: true, + keys: []string{"train"}, + experiment: "exp-train", + }, + { + name: "two AIR tasks", + tasks: []jobs.RunTask{aiRuntimeTask("a", 11, "exp-a", "GPU_1xA10", 1), aiRuntimeTask("b", 12, "exp-b", "GPU_8xH100", 8)}, + air: true, + keys: []string{"a", "b"}, + experiment: "exp-a", + }, + { + name: "foreach AIR task", + tasks: []jobs.RunTask{forEachAiRuntimeTask("sweep", 11, "exp-sweep")}, + air: true, + keys: []string{"sweep"}, + experiment: "exp-sweep", + sweep: true, + }, + { + name: "foreach AIR task after a notebook", + tasks: []jobs.RunTask{notebookTask("prep", 11), forEachAiRuntimeTask("sweep", 12, "exp-sweep")}, + air: true, + keys: []string{"sweep"}, + experiment: "exp-sweep", + sweep: true, + }, + { + name: "retries share one key", + tasks: []jobs.RunTask{notebookTask("prep", 11), aiRuntimeTask("train", 12, "exp-train", "GPU_1xA10", 1), aiRuntimeTask("train", 13, "exp-train", "GPU_1xA10", 1)}, + air: true, keys: []string{"train"}, experiment: "exp-train", + }, + { + name: "no AIR task", + tasks: []jobs.RunTask{notebookTask("prep", 11)}, + }, + { + name: "no tasks", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := &jobs.Run{RunId: 1, Tasks: tt.tasks} + assert.Equal(t, tt.air, isAirRun(run), "isAirRun") + assert.Equal(t, tt.keys, airTaskKeys(run), "airTaskKeys") + assert.Equal(t, tt.experiment, jobExperiment(run), "jobExperiment") + assert.Equal(t, tt.sweep, isSweep(run), "isSweep") + if len(tt.tasks) == 0 { + assert.Nil(t, firstTask(run), "firstTask") + } + }) + } +} + +func TestTaskRunNarrowsToOneAirTask(t *testing.T) { + run := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + notebookTask("prep", 11), + aiRuntimeTask("a", 12, "exp-a", "GPU_1xA10", 1), + {TaskKey: "a", RunId: 13, AttemptNumber: 1, AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: "exp-a"}}, + aiRuntimeTask("b", 14, "exp-b", "GPU_8xH100", 8), + }, + } + + a := taskRun(run, "a") + require.NotNil(t, a) + assert.Equal(t, int64(1), a.RunId) + assert.Equal(t, run.State, a.State) + require.Len(t, a.Tasks, 2, "both attempts of task a, in order") + assert.Equal(t, int64(12), a.Tasks[0].RunId) + assert.Equal(t, int64(13), a.Tasks[1].RunId) + assert.Equal(t, "exp-a", jobExperiment(a)) + + b := taskRun(run, "b") + require.NotNil(t, b) + assert.Equal(t, "exp-b", jobExperiment(b)) + gpu, n := jobCompute(b) + assert.Equal(t, "GPU_8xH100", gpu) + assert.Equal(t, 8, n) + + assert.Len(t, run.Tasks, 4, "the source run is left intact") + assert.Nil(t, taskRun(run, "prep"), "a non-AIR task is not selectable") + assert.Nil(t, taskRun(run, "missing")) +} + +func TestJobTimingUsesTheAirTask(t *testing.T) { + prep := notebookTask("prep", 11) + prep.StartTime, prep.EndTime = 100, 150 + train := aiRuntimeTask("train", 12, "exp", "GPU_1xA10", 1) + train.StartTime, train.EndTime = 200, 260 + run := &jobs.Run{RunId: 1, StartTime: 90, EndTime: 300, Tasks: []jobs.RunTask{prep, train}} + + start, end := jobTiming(run) + assert.Equal(t, int64(200), start) + assert.Equal(t, int64(260), end) +} diff --git a/cmd/air/list.go b/cmd/air/list.go index 9a5a85b0cbc..a4d2fed681d 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -44,7 +44,10 @@ type listData struct { // machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable table. type listRow struct { - RunID string `json:"run_id"` + RunID string `json:"run_id"` + // TaskKey names the AI runtime task the row shows. A job with several AI + // runtime tasks has one row per task, all with the same RunID. + TaskKey string `json:"task_key"` RunName string `json:"run_name"` User string `json:"user"` Status string `json:"status"` @@ -75,6 +78,7 @@ type listRow struct { type listedRun struct { row listRow runID int64 + taskKey string tasks []jobs.RunTask // see retryTasks terminal bool } @@ -415,10 +419,12 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { if s.userFilter != "" && run.CreatorUserName != s.userFilter { continue } - if !s.filters.matches(run) { - continue + // One row per AI runtime task, each filtered on its own task. + for _, e := range buildListRows(run, s.host, s.workspaceID) { + if s.filters.matchesFields(e.fields) { + entries = append(entries, e.entry) + } } - entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, tasks: retryTasks(run), terminal: isTerminal(run)}) } return entries, nil } diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index 941327e93b1..898c3747fa7 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -14,7 +14,7 @@ import ( // TTL matches AICM's ~60-day retention, after which the run drops out of the // index anyway. const ( - listCacheComponent = "air-list-runs-v2" + listCacheComponent = "air-list-runs-v3" listCacheTTL = 60 * 24 * time.Hour ) @@ -25,11 +25,18 @@ type listCacheKey struct { RunID int64 `json:"run_id"` } -// cachedRun is the persisted value: every listRow field (including the +// cachedRuns is the persisted value for a run: one cachedRun per AI runtime +// task (see buildListRows). +type cachedRuns struct { + Rows []cachedRun `json:"rows"` +} + +// cachedRun is one persisted list row: every listRow field (including the // table-only columns, which listRow tags json:"-" and so wouldn't survive a // direct marshal), the filter inputs, and the submit time. type cachedRun struct { RunID string `json:"run_id"` + TaskKey string `json:"task_key"` RunName string `json:"run_name"` User string `json:"user"` Status string `json:"status"` @@ -56,7 +63,7 @@ type cachedRun struct { func (c cachedRun) toRow() listRow { return listRow{ - RunID: c.RunID, RunName: c.RunName, User: c.User, Status: c.Status, + RunID: c.RunID, TaskKey: c.TaskKey, RunName: c.RunName, User: c.User, Status: c.Status, StartedAt: c.StartedAt, IsSweep: c.IsSweep, ExperimentName: c.ExperimentName, DurationSeconds: c.DurationSeconds, AcceleratorType: c.AcceleratorType, NumAccelerators: c.NumAccelerators, DashboardURL: c.DashboardURL, Experiment: c.Experiment, Duration: c.Duration, MLflowURL: c.MLflowURL, MLflowLabel: c.MLflowLabel, @@ -66,7 +73,7 @@ func (c cachedRun) toRow() listRow { func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun { return cachedRun{ - RunID: r.RunID, RunName: r.RunName, User: r.User, Status: r.Status, + RunID: r.RunID, TaskKey: r.TaskKey, RunName: r.RunName, User: r.User, Status: r.Status, StartedAt: r.StartedAt, IsSweep: r.IsSweep, ExperimentName: r.ExperimentName, DurationSeconds: r.DurationSeconds, AcceleratorType: r.AcceleratorType, NumAccelerators: r.NumAccelerators, DashboardURL: r.DashboardURL, Experiment: r.Experiment, Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel, @@ -81,27 +88,39 @@ func newListCache(ctx context.Context) *cache.Cache { return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil) } -// cachedRow returns the cached row, filter fields, and task runs for a run. -// Entries written before task runs were cached are treated as misses. -func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, []jobs.RunTask, bool) { - entry, ok := cache.Get[cachedRun](ctx, c, listCacheKey{Host: host, RunID: runID}) - if !ok || len(entry.Tasks) == 0 { - return listRow{}, filterFields{}, nil, false +// cachedRows returns the cached rows of a run, one per AI runtime task. An +// entry with a row that lacks task runs is treated as a miss. +func cachedRows(ctx context.Context, c *cache.Cache, host string, runID int64) ([]cachedRun, bool) { + entry, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) + if !ok || len(entry.Rows) == 0 { + return nil, false } - return entry.toRow(), entry.Fields, entry.Tasks, true + for _, row := range entry.Rows { + if len(row.Tasks) == 0 { + return nil, false + } + } + return entry.Rows, true } -// putRow caches a terminal run's base row and filter fields under its submit time. -func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { - cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) +// putRows caches a terminal run's rows. +func putRows(ctx context.Context, c *cache.Cache, host string, runID int64, rows []cachedRun) { + cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRuns{Rows: rows}) } -// updateCachedRow replaces the display row while preserving cache metadata. -func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) { +// updateCachedRow replaces the display row of the run's task while preserving +// cache metadata. A run or task that is not cached is left alone. +func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, taskKey string, row listRow) { key := listCacheKey{Host: host, RunID: runID} - entry, ok := cache.Get[cachedRun](ctx, c, key) + entry, ok := cache.Get[cachedRuns](ctx, c, key) if !ok { return } - cache.Put(ctx, c, key, cachedRunFromRow(row, entry.Tasks, entry.Fields, entry.SubmitTimeMs)) + for i, cached := range entry.Rows { + if cached.TaskKey == taskKey { + entry.Rows[i] = cachedRunFromRow(row, cached.Tasks, cached.Fields, cached.SubmitTimeMs) + cache.Put(ctx, c, key, entry) + return + } + } } diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 00c821ac010..71001e00d87 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -1,6 +1,7 @@ package aircmd import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -12,39 +13,74 @@ import ( "github.com/stretchr/testify/require" ) +// putRow caches a single-row run, the common case in these tests. +func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { + putRows(ctx, c, host, runID, []cachedRun{cachedRunFromRow(row, tasks, fields, submitTimeMs)}) +} + func TestListCacheRoundTrip(t *testing.T) { t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) ctx := t.Context() c := newListCache(ctx) - _, _, _, ok := cachedRow(ctx, c, "https://host.test", 42) + _, ok := cachedRows(ctx, c, "https://host.test", 42) require.False(t, ok, "miss before write") row := listRow{ - RunID: "42", Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), + RunID: "42", TaskKey: "train", Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), AcceleratorType: new("GPU_1xA10"), NumAccelerators: new(1), DashboardURL: "https://host.test/jobs/runs/42?o=1", Experiment: "exp", } fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1} tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}} putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields) - got, gotFields, gotTasks, ok := cachedRow(ctx, c, "https://host.test", 42) + got, ok := cachedRows(ctx, c, "https://host.test", 42) require.True(t, ok, "hit after write") - assert.Equal(t, row, got) - assert.Equal(t, fields, gotFields) + require.Len(t, got, 1) + assert.Equal(t, row, got[0].toRow()) + assert.Equal(t, fields, got[0].Fields) + gotTasks := got[0].Tasks for i := range gotTasks { gotTasks[i].ForceSendFields = nil // set by the JSON round trip } assert.Equal(t, tasks, gotTasks) // Different host is a different key. - _, _, _, ok = cachedRow(ctx, c, "https://other.test", 42) + _, ok = cachedRows(ctx, c, "https://other.test", 42) assert.False(t, ok) - // Entries from previous cache formats have no task runs and must be refreshed. + // Entries from previous cache formats have no rows and must be refreshed. cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430}) - _, _, _, ok = cachedRow(ctx, c, "https://host.test", 43) + _, ok = cachedRows(ctx, c, "https://host.test", 43) assert.False(t, ok) + cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 44}, cachedRuns{Rows: []cachedRun{{RunID: "44"}}}) + _, ok = cachedRows(ctx, c, "https://host.test", 44) + assert.False(t, ok, "a row without task runs is a miss") +} + +func TestListCacheKeepsOneRowPerAirTask(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + ctx := t.Context() + c := newListCache(ctx) + host := "https://host.test" + + putRows(ctx, c, host, 42, []cachedRun{ + cachedRunFromRow(listRow{RunID: "42", TaskKey: "train", Experiment: "exp-a"}, []jobs.RunTask{{RunId: 420, TaskKey: "train"}}, filterFields{Experiment: "exp-a"}, 1000), + cachedRunFromRow(listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b"}, []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}, filterFields{Experiment: "exp-b"}, 1000), + }) + + // The MLflow link of one task updates that row alone. + updateCachedRow(ctx, c, host, 42, "eval", listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b", MLflowURL: "https://host.test/ml/x"}) + updateCachedRow(ctx, c, host, 42, "missing", listRow{RunID: "42", TaskKey: "missing"}) + + got, ok := cachedRows(ctx, c, host, 42) + require.True(t, ok) + require.Len(t, got, 2) + assert.Equal(t, "train", got[0].TaskKey) + assert.Empty(t, got[0].MLflowURL) + assert.Equal(t, "eval", got[1].TaskKey) + assert.Equal(t, "https://host.test/ml/x", got[1].MLflowURL) + assert.Equal(t, int64(1000), got[1].SubmitTimeMs, "cache metadata is preserved") } func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) { diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 5f77665f5ca..3179bc135f6 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -12,8 +12,9 @@ import ( ) // runDetailText fetches a run and renders the same styled view as `air get` into -// a string, for the list picker's detail pane. -func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { +// a string, for the list picker's detail pane. taskKey selects the AI runtime +// task shown, as `air get --task-key` does; "" means the first. +func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int64, taskKey string) (string, error) { run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { @@ -21,8 +22,14 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } return "", fmt.Errorf("failed to fetch run: %w", err) } + fullRun := run + run, err = selectAirTask(run, taskKey) + if err != nil { + return "", err + } data := buildGetData(run) + annotateTask(fullRun, run, &data) // As in `air get`: an ai_runtime_task run keeps its experiment, accelerators // and config path on the task. if genAIComputeTask(run) == nil { diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index b53d492bc6c..2e2eca05d5a 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -83,7 +83,7 @@ func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { currentUser: selfUser, limit: 10, fetchMLflow: true, }).next(10) require.NoError(t, err) - info, err := runDetailText(ctx, w, 7) + info, err := runDetailText(ctx, w, 7, "") require.NoError(t, err) logs, err := runLogsSnapshot(ctx, w, 7) require.NoError(t, err) @@ -135,7 +135,7 @@ func TestUnnamedOlderRetryLabelFitsTheListColumnAndIsWholeInTheInfoPane(t *testi currentUser: selfUser, limit: 10, fetchMLflow: true, }).next(10) require.NoError(t, err) - info, err := runDetailText(ctx, w, 7) + info, err := runDetailText(ctx, w, 7, "") require.NoError(t, err) // Then the list row keeps the end of the ID and the whole retry note within @@ -184,7 +184,7 @@ func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { t.Cleanup(srv.Close) // When the list's info pane renders it. - got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) // Then it shows the same config box and limit as `air get`. @@ -214,7 +214,7 @@ func TestRunDetailTextShowsEnvironmentAndLegacyConfig(t *testing.T) { })) t.Cleanup(srv.Close) - got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) assert.Contains(t, got, "command: echo legacy-command") assert.Regexp(t, `Environment\s+4`, got) @@ -253,7 +253,7 @@ func TestRunDetailTextShowsAiRuntimeTaskFields(t *testing.T) { t.Cleanup(srv.Close) ctx := cmdio.MockDiscard(t.Context()) - info, err := runDetailText(ctx, newTestWorkspaceClient(t, srv.URL), 7) + info, err := runDetailText(ctx, newTestWorkspaceClient(t, srv.URL), 7, "") require.NoError(t, err) // The pane shows what the list row and `air get` show, with the experiment @@ -263,3 +263,32 @@ func TestRunDetailTextShowsAiRuntimeTaskFields(t *testing.T) { assert.Contains(t, info, "Accelerators 1x A10") assert.Contains(t, info, "command: python train.py") } + +func TestRunDetailTextShowsTheSelectedAirTask(t *testing.T) { + run := jobs.Run{RunId: 7, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, Tasks: []jobs.RunTask{ + notebookTask("prep", 71), + aiRuntimeTask("train", 72, "/Users/me@example.com/exp-train", "GPU_1xA10", 1), + aiRuntimeTask("eval", 73, "/Users/me@example.com/exp-eval", "GPU_8xH100", 8), + }} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _ = json.NewEncoder(w).Encode(run) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + ctx := cmdio.MockDiscard(t.Context()) + w := newTestWorkspaceClient(t, srv.URL) + + info, err := runDetailText(ctx, w, 7, "") + require.NoError(t, err) + assert.Contains(t, info, "Experiment exp-train", "the default is the first AIR task, not the notebook") + assert.Contains(t, info, "Task train") + + info, err = runDetailText(ctx, w, 7, "eval") + require.NoError(t, err) + assert.Contains(t, info, "Experiment exp-eval") + assert.Contains(t, info, "Accelerators 8x H100") + assert.Contains(t, info, "Task eval") +} diff --git a/cmd/air/list_format.go b/cmd/air/list_format.go index cda11b49a58..a828fab6312 100644 --- a/cmd/air/list_format.go +++ b/cmd/air/list_format.go @@ -6,10 +6,45 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" ) -// buildListRow extracts the columns shown for one run. Optional cells fall back -// to "-"; MLflowURL starts as "-" and setMLflowLinks fills it in for text output. -// host and workspaceID are used for building dashboard URLs. +// listedRunFields is a list entry with the task-filter inputs it was built from. +type listedRunFields struct { + entry listedRun + fields filterFields +} + +// buildListRows builds one list entry per AI runtime task of the run, in task +// order, each read from that task's own task runs (see taskRun). A run without +// an AI runtime task yields a single entry read from the run as is. +func buildListRows(run *jobs.Run, host string, workspaceID int64) []listedRunFields { + keys := airTaskKeys(run) + if len(keys) == 0 { + return []listedRunFields{listedRunFor(run, host, workspaceID)} + } + entries := make([]listedRunFields, 0, len(keys)) + for _, key := range keys { + entries = append(entries, listedRunFor(taskRun(run, key), host, workspaceID)) + } + return entries +} + +// listedRunFor builds the list entry for a run (or a run narrowed to one task). +func listedRunFor(run *jobs.Run, host string, workspaceID int64) listedRunFields { + row := buildListRow(run, host, workspaceID) + return listedRunFields{ + entry: listedRun{row: row, runID: run.RunId, taskKey: row.TaskKey, tasks: retryTasks(run), terminal: isTerminal(run)}, + fields: filterFieldsFromRun(run), + } +} + +// buildListRow extracts the columns shown for one run, read from its first AI +// runtime task. Optional cells fall back to "-"; MLflowURL starts as "-" and +// setMLflowLinks fills it in for text output. host and workspaceID are used for +// building dashboard URLs. func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { + taskKey := "" + if t := airWrapperTask(run); t != nil { + taskKey = t.TaskKey + } experiment := jobExperiment(run) experimentName := nilIfEmpty(experiment) if experiment == "" { @@ -38,6 +73,7 @@ func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { return listRow{ RunID: strconv.FormatInt(run.RunId, 10), + TaskKey: taskKey, RunName: run.RunName, User: run.CreatorUserName, Status: displayRunStatus(run), diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index d9cf6a568bc..824d105343c 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -105,9 +105,11 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { // The cache key excludes the filter, so a cached row is still run through the // active filter; a non-matching hit is dropped, not re-fetched. for _, id := range ids { - if row, fields, tasks, ok := cachedRow(s.ctx, s.cache, host, id); ok { - if s.filters.matchesFields(fields) { - rows = append(rows, listedRun{row: row, runID: id, tasks: tasks, terminal: true}) + if cached, ok := cachedRows(s.ctx, s.cache, host, id); ok { + for _, c := range cached { + if s.filters.matchesFields(c.Fields) { + rows = append(rows, listedRun{row: c.toRow(), runID: id, taskKey: c.TaskKey, tasks: c.Tasks, terminal: true}) + } } continue } @@ -119,16 +121,19 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { return nil, err } for _, run := range runs { - fields := filterFieldsFromRun(run) - if !s.filters.matchesFields(fields) { - continue + // One row per AI runtime task; the whole set is cached so a later lookup + // by run id can re-filter every task. + entries := buildListRows(run, s.host, s.workspaceID) + start, _ := jobTiming(run) + cached := make([]cachedRun, 0, len(entries)) + for _, e := range entries { + cached = append(cached, cachedRunFromRow(e.entry.row, e.entry.tasks, e.fields, start)) + if s.filters.matchesFields(e.fields) { + rows = append(rows, e.entry) + } } - row := buildListRow(run, s.host, s.workspaceID) - tasks := retryTasks(run) - rows = append(rows, listedRun{row: row, runID: run.RunId, tasks: tasks, terminal: isTerminal(run)}) if isTerminal(run) { - start, _ := jobTiming(run) - putRow(s.ctx, s.cache, host, run.RunId, tasks, start, row, fields) + putRows(s.ctx, s.cache, host, run.RunId, cached) } } @@ -145,7 +150,7 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { func (s *indexStrategy) cacheMLflowLinks(entries []listedRun) { for _, entry := range entries { if entry.row.MLflowURL != "" && entry.row.MLflowURL != "-" { - updateCachedRow(s.ctx, s.cache, s.host, entry.runID, entry.row) + updateCachedRow(s.ctx, s.cache, s.host, entry.runID, entry.taskKey, entry.row) } } } diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index f3af7fd9aa1..b91c28843b6 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -242,7 +242,7 @@ func TestBuildListRow(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"123","run_name":"run-123","user":"me@example.com","status":"SUCCESS", + "run_id":"123","task_key":"","run_name":"run-123","user":"me@example.com","status":"SUCCESS", "started_at":"2023-11-14T22:13:20+00:00","is_sweep":false, "experiment_name":"exp","duration_seconds":12, "accelerator_type":"GPU_8xH100","num_accelerators":8, @@ -276,7 +276,7 @@ func TestBuildListRowWaitingForComputeStatus(t *testing.T) { assert.Equal(t, "PENDING", row.Status) assert.Equal(t, listRowStatus(row), row.Status) assert.JSONEq(t, `{ - "run_id":"123","run_name":"run-123","user":"me@example.com","status":"PENDING", + "run_id":"123","task_key":"","run_name":"run-123","user":"me@example.com","status":"PENDING", "started_at":null,"is_sweep":false,"experiment_name":"exp","duration_seconds":null, "accelerator_type":"GPU_8xH100","num_accelerators":8, "dashboard_url":"https://example.test/jobs/runs/123?o=456" @@ -298,7 +298,7 @@ func TestBuildListRowMissingValues(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"7","run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, + "run_id":"7","task_key":"","run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, "experiment_name":null,"duration_seconds":null,"accelerator_type":null,"num_accelerators":null, "dashboard_url":"https://example.test/jobs/runs/7" }`, string(body)) @@ -502,3 +502,73 @@ func TestListValidatesAcceleratorOnlyAfterNoMatches(t *testing.T) { }) } } + +func TestBuildListRowsOnePerAirTask(t *testing.T) { + run := jobs.Run{RunId: 7, RunName: "dag", CreatorUserName: "me@example.com", Tasks: []jobs.RunTask{ + notebookTask("prep", 71), + aiRuntimeTask("train", 72, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("eval", 73, "/Users/me@example.com/exp-b", "GPU_8xH100", 8), + }} + + rows := buildListRows(&run, "https://example.test", 0) + require.Len(t, rows, 2) + assert.Equal(t, "7", rows[0].entry.row.RunID) + assert.Equal(t, "train", rows[0].entry.row.TaskKey) + assert.Equal(t, "exp-a", rows[0].entry.row.Experiment) + assert.Equal(t, "1x A10", rows[0].entry.row.Accelerators) + assert.Equal(t, []int64{72}, taskRunIDs(rows[0].entry.tasks), "the row's MLflow lookup sees only its own task runs") + assert.Equal(t, "7", rows[1].entry.row.RunID) + assert.Equal(t, "eval", rows[1].entry.row.TaskKey) + assert.Equal(t, "exp-b", rows[1].entry.row.Experiment) + assert.Equal(t, "8x H100", rows[1].entry.row.Accelerators) + assert.Equal(t, []int64{73}, taskRunIDs(rows[1].entry.tasks)) +} + +func taskRunIDs(tasks []jobs.RunTask) []int64 { + ids := make([]int64, len(tasks)) + for i, t := range tasks { + ids[i] = t.RunId + } + return ids +} + +func TestListAirRunsAirTaskAfterNotebook(t *testing.T) { + // The official multi-task DAB example: a notebook prep task, then the + // ai_runtime_task that depends on it. + dag := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/air-byot-multitask-dag") + dag.Tasks[0].TaskKey, dag.Tasks[0].RunId = "train", 12 + dag.Tasks = []jobs.RunTask{notebookTask("prep", 11), dag.Tasks[0]} + srv := runsServer(t, runsListBody(t, "", dag)) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, "1", rows[0].RunID) + assert.Equal(t, "train", rows[0].TaskKey) + assert.Equal(t, "air-byot-multitask-dag", rows[0].Experiment) + assert.Equal(t, "1x A10", rows[0].Accelerators) +} + +func TestListAirRunsOneRowPerAirTask(t *testing.T) { + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_8xH100", 8), + } + srv := runsServer(t, runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xH100", 1, "/Users/me@example.com/exp-c"))) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ + activeOnly: true, + filters: listFilters{Experiment: "exp-b"}, + }).next(10) + require.NoError(t, err) + require.Len(t, rows, 1, "the task filter applies per AIR task") + assert.Equal(t, "1", rows[0].RunID) + assert.Equal(t, "b", rows[0].TaskKey) + + rows, err = newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 3) + assert.Equal(t, []string{"1", "1", "2"}, []string{rows[0].RunID, rows[1].RunID, rows[2].RunID}) + assert.Equal(t, []string{"a", "b", ""}, []string{rows[0].TaskKey, rows[1].TaskKey, rows[2].TaskKey}) +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 6798244c621..0ba01ed8f90 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -291,7 +291,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Run Details" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunDetail(runID) + return m, m.fetchRunDetail(runID, m.rows[m.cursor].TaskKey) } case "L", "l": // Open the logs snapshot pane; the fetch fills it in. @@ -440,12 +440,13 @@ func hiddenNote(hidden []string, names bool) string { } } -// fetchRunDetail returns a tea.Cmd that fetches run details in the background. -func (m listModel) fetchRunDetail(runID int64) tea.Cmd { +// fetchRunDetail returns a tea.Cmd that fetches the details of the run's task +// in the background. +func (m listModel) fetchRunDetail(runID int64, taskKey string) tea.Cmd { ctx := m.fetcher.ctx w := m.fetcher.w return func() tea.Msg { - body, err := runDetailText(ctx, w, runID) + body, err := runDetailText(ctx, w, runID, taskKey) if err != nil { return detailMsg{title: "Run Details", err: err} } diff --git a/cmd/air/render.go b/cmd/air/render.go index b8f0043d906..70f66ec7b5f 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -75,6 +75,9 @@ func newPalette(r *lipgloss.Renderer) palette { type runView struct { runID string dashboardURL string + // task is the selected AI runtime task's key, shown only when the run has + // several AI runtime tasks to tell apart. + task string status string submitted string retries int @@ -109,6 +112,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl view := runView{ runID: data.RunID, dashboardURL: data.DashboardURL, + task: data.TaskDisplay, status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, @@ -162,12 +166,14 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } } -// genAIComputeTask returns the run's first GenAI-compute task, or nil. +// genAIComputeTask returns the run's first AI runtime task's GenAI-compute +// task, or nil. func genAIComputeTask(run *jobs.Run) *jobs.GenAiComputeTask { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - return run.Tasks[0].GenAiComputeTask + return t.GenAiComputeTask } // aiRuntimeEnvironment returns the serverless environment an ai_runtime run used, @@ -447,12 +453,17 @@ func renderFields(p palette, colorOn bool, v runView) string { status := statusStyle(p, v.status).Render("● " + v.status) rows := []string{ field(p, "Run ID", link(colorOn, p.blue, v.runID, v.dashboardURL)), + } + if v.task != "" { + rows = append(rows, field(p, "Task", p.n12.Render(stripControl(v.task)))) + } + rows = append(rows, field(p, "Status", status), field(p, "Submitted", p.n12.Render(v.submitted)), field(p, "Retries", p.n12.Render(strconv.Itoa(v.retries))), field(p, "Max Retries", p.n12.Render(v.maxRetries)), field(p, "Duration", p.n12.Render(v.duration)), - } + ) // Progress is shown only for a running run we could estimate; amber marks it // as live, in-progress information. if v.progress != "" { From 624085729a54ac2ef0b663feba74673057bbae0c Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:37:41 +0000 Subject: [PATCH 2/9] air: add changelog fragment for #7632 Co-authored-by: Isaac --- .nextchanges/air/multitask-air-tasks.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/air/multitask-air-tasks.md diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md new file mode 100644 index 00000000000..a8cb3da5d2d --- /dev/null +++ b/.nextchanges/air/multitask-air-tasks.md @@ -0,0 +1 @@ +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task; a job with several AI Runtime tasks lists one row per task, `air get` gains `--task-key`, and both commands report `task_key` in JSON output. ([#7632](https://github.com/databricks/cli/pull/7632)) From a69ac1c25cd22220c73c0f99777a2dc9471cee46 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:37:44 +0000 Subject: [PATCH 3/9] air: report each AI Runtime task's own status and count --limit in runs Address review on #7632: - A run narrowed to one AI Runtime task now reports that task's latest attempt's state, so a succeeded train next to a failed eval, or an AI Runtime task skipped after a failed prep notebook, no longer shows the run's aggregate. While the run is still active after a failed attempt, the run's state stands, since the task may be waiting to retry. - `--limit` counts runs on both the scan and index paths. - `task_key` is null for a run without an AI Runtime task, matching `air list`. - The `air list` logs pane notes when its logs belong to another task than the selected row's. - `cancel --all` previews every AI Runtime task of each run, and its help says a cancel stops the whole run. - Tests for per-task status with differing results, the run-counted limit, `cancel --all` on a multi-task run, and the logs note. Co-authored-by: Isaac --- .nextchanges/air/multitask-air-tasks.md | 2 +- acceptance/air/get-mlflow-retry/output.txt | 2 +- acceptance/air/get-multitask/output.txt | 7 +- acceptance/air/get-multitask/test.toml | 10 ++- acceptance/air/get/output.txt | 2 +- acceptance/air/help/output.txt | 5 +- cmd/air/cancel.go | 37 ++++---- cmd/air/cancel_test.go | 53 +++++++++++- cmd/air/format.go | 14 +++- cmd/air/get.go | 15 ++-- cmd/air/get_test.go | 26 +++++- cmd/air/joblist.go | 53 +++++++++--- cmd/air/joblist_test.go | 98 ++++++++++++++++++++++ cmd/air/list.go | 21 ++++- cmd/air/list_detail.go | 33 +++++++- cmd/air/list_detail_test.go | 14 +++- cmd/air/list_format.go | 2 +- cmd/air/list_index.go | 6 +- cmd/air/list_test.go | 28 +++++++ cmd/air/list_tui.go | 6 +- 20 files changed, 372 insertions(+), 62 deletions(-) diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md index a8cb3da5d2d..3ca69fcbe9f 100644 --- a/.nextchanges/air/multitask-air-tasks.md +++ b/.nextchanges/air/multitask-air-tasks.md @@ -1 +1 @@ -* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task; a job with several AI Runtime tasks lists one row per task, `air get` gains `--task-key`, and both commands report `task_key` in JSON output. ([#7632](https://github.com/databricks/cli/pull/7632)) +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task; a job with several AI Runtime tasks lists one row per task (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs), each row and `air get --task-key` report that task's own status, `air get` gains `--task-key`, and both commands report `task_key` in JSON output. ([#7632](https://github.com/databricks/cli/pull/7632)) diff --git a/acceptance/air/get-mlflow-retry/output.txt b/acceptance/air/get-mlflow-retry/output.txt index e4fb6bb7be0..f46b8e1f124 100644 --- a/acceptance/air/get-mlflow-retry/output.txt +++ b/acceptance/air/get-mlflow-retry/output.txt @@ -7,7 +7,7 @@ "ts": "[TIMESTAMP]", "data": { "run_id": "123", - "task_key": "train", + "task_key": null, "status": "FAILED", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/get-multitask/output.txt b/acceptance/air/get-multitask/output.txt index ee5a8fa2f17..f6e1bda3a50 100644 --- a/acceptance/air/get-multitask/output.txt +++ b/acceptance/air/get-multitask/output.txt @@ -31,7 +31,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 │ │ │ Run ID 123 │ │ Task eval │ -│ Status ● SUCCESS │ +│ Status ● FAILED │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ │ Max Retries 2 │ @@ -44,6 +44,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 │ │ ╰────────────────────────────────────────────────────────────────╯ +Failure reason: eval: loss is NaN + Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 @@ -90,7 +92,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "data": { "run_id": "123", "task_key": "eval", - "status": "SUCCESS", + "status": "FAILED", "started_at": "[TIMESTAMP]", "duration_seconds": 8, "attempt_number": 0, @@ -99,6 +101,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "mlflow_experiment_id": "exp1", "mlflow_run_id": "run1", + "termination_reason": "eval: loss is NaN", "est_remaining_seconds": null, "est_percent_complete": null, "user": "user@example.com", diff --git a/acceptance/air/get-multitask/test.toml b/acceptance/air/get-multitask/test.toml index 751f09abb17..dc3d357dc1e 100644 --- a/acceptance/air/get-multitask/test.toml +++ b/acceptance/air/get-multitask/test.toml @@ -8,7 +8,9 @@ Response.Body = '' # A multi-task job in the shape of the official DAB example (universe # ai-compute/dabs/examples/03_multitask_dag): a notebook prep task first, then the # ai_runtime_task that depends on it, plus a second ai_runtime_task. `air get` -# must read the AI Runtime task wherever it sits, not tasks[0]. +# must read the AI Runtime task wherever it sits, not tasks[0]. The tasks end +# differently (train succeeds, eval fails, so the run fails): each view must +# report its own task's status, not the run's. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' @@ -17,8 +19,8 @@ Response.Body = ''' "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", "creator_user_name": "user@example.com", "start_time": 1700000000000, - "end_time": 1700000012000, - "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "end_time": 1700000020000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task eval failed"}, "environments": [ {"environment_key": "default", "spec": {"environment_version": "5"}} ], @@ -56,7 +58,7 @@ Response.Body = ''' "max_retries": 2, "start_time": 1700000012000, "end_time": 1700000020000, - "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "eval: loss is NaN"}, "depends_on": [{"task_key": "train"}], "environment_key": "default", "ai_runtime_task": { diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 6c85755a318..64054d265aa 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -43,7 +43,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", - "task_key": "train", + "task_key": null, "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 1a649d62749..7f6898a4fa1 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -77,7 +77,7 @@ Flags: --all-users[=true|false] Show runs from all users --filter stringArray Filter runs by KEY=VALUE (repeatable) -h, --help[=true|false] help for list - --limit int Maximum number of runs to show (default 20) + --limit int Maximum number of runs to show; a run with several AI Runtime tasks shows one row per task (default 20) Global Flags: --debug[=true|false] enable debug logging @@ -232,6 +232,9 @@ Global Flags: Cancel one or more runs by job run ID, or use --all to cancel all active runs submitted by the current user. +Cancelling a run stops all of its tasks: every AI Runtime task in the job, and +any other task such as a notebook that prepares data. + --all prompts for confirmation. Use --yes to skip the prompt. Usage: diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index 36ad20fe309..41b311104a5 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -47,6 +47,9 @@ func newCancelCommand() *cobra.Command { Long: `Cancel one or more runs by job run ID, or use --all to cancel all active runs submitted by the current user. +Cancelling a run stops all of its tasks: every AI Runtime task in the job, and +any other task such as a notebook that prepares data. + --all prompts for confirmation. Use --yes to skip the prompt.`, } @@ -107,8 +110,9 @@ all active runs submitted by the current user. } // A job with several AI runtime tasks lists one row per task; the - // run is cancelled (and previewed) once. - rows = firstRowPerRun(rows) + // run is cancelled (and previewed) once, naming all its tasks. + var taskKeys map[string][]string + rows, taskKeys = firstRowPerRun(rows) runIDs = make([]string, 0, len(rows)) for i := range rows { if rows[i].RunID != "" { @@ -126,7 +130,7 @@ all active runs submitted by the current user. } if !yes { - displayCancelPreview(ctx, rows, host) + displayCancelPreview(ctx, rows, taskKeys, host) confirmed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nCancel %d run(s) in %s?", len(runIDs), host)) if err != nil { return err @@ -218,35 +222,40 @@ func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, s return err } -// firstRowPerRun keeps the first row of each run id, in order. -func firstRowPerRun(rows []listRow) []listRow { - seen := make(map[string]bool, len(rows)) +// firstRowPerRun keeps the first row of each run id, in order, and returns the +// task keys of every row by run id, so the preview can name all the AI runtime +// tasks a cancel stops. +func firstRowPerRun(rows []listRow) ([]listRow, map[string][]string) { + taskKeys := make(map[string][]string, len(rows)) unique := make([]listRow, 0, len(rows)) for _, row := range rows { - if seen[row.RunID] { - continue + if _, seen := taskKeys[row.RunID]; !seen { + unique = append(unique, row) + taskKeys[row.RunID] = nil + } + if row.TaskKey != "" { + taskKeys[row.RunID] = append(taskKeys[row.RunID], row.TaskKey) } - seen[row.RunID] = true - unique = append(unique, row) } - return unique + return unique, taskKeys } // displayCancelPreview shows the runs that `cancel --all` is about to terminate. -func displayCancelPreview(ctx context.Context, rows []listRow, host string) { +func displayCancelPreview(ctx context.Context, rows []listRow, taskKeys map[string][]string, host string) { var sb strings.Builder fmt.Fprintf(&sb, "\nWorkspace: %s\n", host) fmt.Fprintf(&sb, "Found %d active run(s) to cancel:\n\n", len(rows)) tw := tabwriter.NewWriter(&sb, 0, 0, 2, ' ', 0) - fmt.Fprintln(tw, "Run ID\tExperiment\tStarted") + fmt.Fprintln(tw, "Run ID\tExperiment\tAI Runtime tasks\tStarted") for i := range rows { experiment := orNA(rows[i].Experiment) started := na if rows[i].StartedAt != nil { started = *rows[i].StartedAt } - fmt.Fprintf(tw, "%s\t%s\t%s\n", rows[i].RunID, experiment, started) + tasks := orNA(strings.Join(taskKeys[rows[i].RunID], ", ")) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", rows[i].RunID, experiment, tasks, started) } tw.Flush() diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index 6f9166d1db3..a5e57e4709c 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -325,7 +325,8 @@ func TestDisplayCancelPreview(t *testing.T) { {RunID: "111", Experiment: "exp-a", StartedAt: &started}, {RunID: "222"}, // no experiment or start time -> N/A } - displayCancelPreview(ctx, rows, "https://my-workspace.cloud.databricks.test") + taskKeys := map[string][]string{"111": {"train", "eval"}} + displayCancelPreview(ctx, rows, taskKeys, "https://my-workspace.cloud.databricks.test") out := buf.String() assert.Contains(t, out, "Workspace: https://my-workspace.cloud.databricks.test") @@ -333,6 +334,56 @@ func TestDisplayCancelPreview(t *testing.T) { assert.Contains(t, out, "Run ID") assert.Contains(t, out, "111") assert.Contains(t, out, "exp-a") + assert.Contains(t, out, "AI Runtime tasks") + assert.Contains(t, out, "train, eval") assert.Contains(t, out, "222") assert.Contains(t, out, na) } + +func TestFirstRowPerRun(t *testing.T) { + rows := []listRow{ + {RunID: "1", TaskKey: "train"}, + {RunID: "1", TaskKey: "eval"}, + {RunID: "2"}, + {RunID: "3", TaskKey: "sweep"}, + } + unique, taskKeys := firstRowPerRun(rows) + assert.Equal(t, []listRow{{RunID: "1", TaskKey: "train"}, {RunID: "2"}, {RunID: "3", TaskKey: "sweep"}}, unique) + assert.Equal(t, map[string][]string{"1": {"train", "eval"}, "2": nil, "3": {"sweep"}}, taskKeys) +} + +func TestCancelAllCancelsAMultiTaskRunOnce(t *testing.T) { + // Run 111 has two AI Runtime tasks after a notebook, so air list shows it as + // two rows; cancel --all must cancel it once and name both tasks. + multi := airBaseRun(111, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + multi.Tasks = []jobs.RunTask{ + notebookTask("prep", 1110), + aiRuntimeTask("train", 1111, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("eval", 1112, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + single := airBaseRun(222, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c") + list := runsListBody(t, "", multi, single) + var cancelled []int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/list": + _, _ = w.Write([]byte(list)) + case "/api/2.2/jobs/runs/cancel": + var req jobs.CancelRun + assert.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + cancelled = append(cancelled, req.RunId) + _, _ = w.Write([]byte(`{}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var buf bytes.Buffer + require.NoError(t, runCancelAll(t, newTestWorkspaceClient(t, srv.URL), flags.OutputText, strings.NewReader("y\n"), &buf)) + out := buf.String() + assert.Contains(t, out, "Found 2 active run(s) to cancel:") + assert.Contains(t, out, "train, eval") + assert.Contains(t, out, "Cancel 2 run(s)") + assert.ElementsMatch(t, []int64{111, 222}, cancelled, "each run is cancelled exactly once") +} diff --git a/cmd/air/format.go b/cmd/air/format.go index 2e8a112f682..d43fec0e0f7 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -113,18 +113,28 @@ func displayRunStatus(run *jobs.Run) string { return status } +// runWaitingForCompute reports whether a run has not started training yet: it +// is pending itself, or running while its lead task waits. A run narrowed to one +// task (see taskRun) carries that task's state, so the task's waiting states +// count at run level too. func runWaitingForCompute(run *jobs.Run) bool { if run.State == nil || run.State.ResultState != "" { return false } - if run.State.LifeCycleState == jobs.RunLifeCycleStatePending { + if isWaitingState(run.State.LifeCycleState) { return true } task := leadTask(run) if run.State.LifeCycleState != jobs.RunLifeCycleStateRunning || task == nil || task.State == nil { return false } - switch task.State.LifeCycleState { + return isWaitingState(task.State.LifeCycleState) +} + +// isWaitingState reports whether a run or task lifecycle state means it has not +// started running yet. +func isWaitingState(s jobs.RunLifeCycleState) bool { + switch s { case jobs.RunLifeCycleStatePending, jobs.RunLifeCycleStateQueued, jobs.RunLifeCycleStateWaitingForRetry, jobs.RunLifeCycleStateBlocked: return true diff --git a/cmd/air/get.go b/cmd/air/get.go index 3d8f080f64e..1a768302ca7 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -246,7 +246,7 @@ one.`, } data := buildGetData(run) - annotateTask(fullRun, run, &data) + annotateTask(run, len(airTaskKeys(fullRun)), &data) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) @@ -339,17 +339,18 @@ func selectAirTask(run *jobs.Run, key string) (*jobs.Run, error) { return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) } -// annotateTask records which AI runtime task of full the view shows: selected -// is full narrowed to it (see selectAirTask). The text cell is set only when -// full has several AI runtime tasks. -func annotateTask(full, selected *jobs.Run, data *getData) { - t := leadTask(selected) +// annotateTask records which AI runtime task the view shows: selected is the +// run narrowed to it (see selectAirTask), and airTasks is how many AI runtime +// tasks the whole run has. TaskKey stays null for a run without one; the text +// cell is set only when there are several to tell apart. +func annotateTask(selected *jobs.Run, airTasks int, data *getData) { + t := airTask(selected) if t == nil { return } key := t.TaskKey data.TaskKey = &key - if len(airTaskKeys(full)) > 1 { + if airTasks > 1 { data.TaskDisplay = key } } diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index 05f72a1cf8b..f86f76d913d 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -428,7 +428,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("an older retry's mlflow run names the retry", func(t *testing.T) { data := getRunJSON(t, srv, "123") assert.JSONEq(t, `{ - "run_id":"123","task_key":"train","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"123","task_key":null,"status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":2,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/123?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs", @@ -444,7 +444,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("the latest attempt's mlflow run has no mlflow_retry key", func(t *testing.T) { data := getRunJSON(t, srv, "124") assert.JSONEq(t, `{ - "run_id":"124","task_key":"train","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"124","task_key":null,"status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":0,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/124?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs", @@ -934,3 +934,25 @@ func TestGetRunSelectsAirTask(t *testing.T) { assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `run 5 has no AI Runtime task "prep"; its AI Runtime tasks are: train, eval`}, e.Error) }) } + +func TestAnnotateTaskIsNullWithoutAnAirTask(t *testing.T) { + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}} + selected, err := selectAirTask(run, "") + require.NoError(t, err) + var data getData + annotateTask(selected, len(airTaskKeys(run)), &data) + assert.Nil(t, data.TaskKey, "matches air list, which reports no task, and --task-key prep, which errors") + assert.Empty(t, data.TaskDisplay) + + two := &jobs.Run{RunId: 6, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 61, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 62, "e", "GPU_1xA10", 1), + }} + selected, err = selectAirTask(two, "eval") + require.NoError(t, err) + data = getData{} + annotateTask(selected, len(airTaskKeys(two)), &data) + require.NotNil(t, data.TaskKey) + assert.Equal(t, "eval", *data.TaskKey) + assert.Equal(t, "eval", data.TaskDisplay) +} diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 85849fc0d09..2d062a64306 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -58,8 +58,9 @@ func airTaskKeys(r *jobs.Run) []string { // taskRun returns a copy of r narrowed to the task runs (every attempt, in // order) of the AI runtime task with the given key, so the display helpers, -// which read a run's tasks, report that task alone. It returns nil when key is -// not one of the run's AI runtime tasks. +// which read a run's tasks, report that task alone. Its state is that task's +// (see taskState). It returns nil when key is not one of the run's AI runtime +// tasks. func taskRun(r *jobs.Run, key string) *jobs.Run { if !slices.Contains(airTaskKeys(r), key) { return nil @@ -71,13 +72,37 @@ func taskRun(r *jobs.Run, key string) *jobs.Run { narrowed.Tasks = append(narrowed.Tasks, r.Tasks[i]) } } + latest := &narrowed.Tasks[len(narrowed.Tasks)-1] + if state := taskState(r, latest); state != r.State { + narrowed.State = state + narrowed.Status = latest.Status + } return &narrowed } -// airWrapperTask returns the run's first AI runtime task run as the Jobs API -// reports it (for a sweep, the foreach wrapper, which carries the state and +// taskState returns the state to report for the task whose latest attempt is +// latest: that attempt's state, so one task's result doesn't stand for another +// (a successful train next to a failed eval, or an AI runtime task skipped +// because its prep task failed). It keeps the run's state when the task has +// none, and while the run is still active after a failed attempt, since the +// task may be waiting to retry and runs/get doesn't reliably report how many +// retries are left. +func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { + s := latest.State + if s == nil { + return r.State + } + failed := isTerminalState(s) && s.ResultState != jobs.RunResultStateSuccess + if failed && !isTerminal(r) { + return r.State + } + return s +} + +// airTask returns the task run of the run's first AI runtime task as the Jobs +// API reports it (for a sweep, the foreach wrapper, which carries the state and // timing), or nil when the run has none. -func airWrapperTask(r *jobs.Run) *jobs.RunTask { +func airTask(r *jobs.Run) *jobs.RunTask { i := firstAirTaskIndex(r) if i < 0 { return nil @@ -85,11 +110,11 @@ func airWrapperTask(r *jobs.Run) *jobs.RunTask { return &r.Tasks[i] } -// leadTask returns the task run whose state and timing stand for the run: its -// first AI runtime task (for a sweep, the foreach wrapper), else, for a run -// without one, its first task; nil when the run has no tasks. +// leadTask is airTask with a fallback to the first task for a run without an AI +// runtime task (such as a legacy gen_ai_compute_task without a training +// script), whose fields `air get` still shows; nil when the run has no tasks. func leadTask(r *jobs.Run) *jobs.RunTask { - if t := airWrapperTask(r); t != nil { + if t := airTask(r); t != nil { return t } if len(r.Tasks) == 0 { @@ -187,10 +212,12 @@ func jobTiming(r *jobs.Run) (startMillis, endMillis int64) { // isTerminal reports whether a run has finished and its details are immutable, // so its row is safe to cache. func isTerminal(r *jobs.Run) bool { - if r.State == nil { - return false - } - switch r.State.LifeCycleState { + return r.State != nil && isTerminalState(r.State) +} + +// isTerminalState reports whether a run or task state is final. +func isTerminalState(s *jobs.RunState) bool { + switch s.LifeCycleState { case jobs.RunLifeCycleStateTerminated, jobs.RunLifeCycleStateInternalError, jobs.RunLifeCycleStateSkipped: return true default: diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 2e53d89d425..79f6bc239a5 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -261,3 +261,101 @@ func TestJobTimingUsesTheAirTask(t *testing.T) { assert.Equal(t, int64(200), start) assert.Equal(t, int64(260), end) } + +// finished returns a task run whose latest attempt ended in result. +func finished(task jobs.RunTask, life jobs.RunLifeCycleState, result jobs.RunResultState) jobs.RunTask { + task.State = &jobs.RunState{LifeCycleState: life, ResultState: result} + return task +} + +func TestTaskRunReportsTheTasksOwnState(t *testing.T) { + failedRun := &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + tests := []struct { + name string + runState *jobs.RunState + tasks []jobs.RunTask + key string + want string + }{ + { + name: "succeeded task next to a failed one", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + }, + key: "train", + want: "SUCCESS", + }, + { + name: "the failed task", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + }, + key: "eval", + want: "FAILED", + }, + { + name: "skipped because the prep notebook failed", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed), + }, + key: "train", + want: "UPSTREAM_FAILED", + }, + { + name: "latest attempt is the one reported", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + func() jobs.RunTask { + t := finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess) + t.AttemptNumber = 1 + return t + }(), + }, + key: "train", + want: "SUCCESS", + }, + { + name: "a failed attempt in an active run may be retried, so the run's state stands", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + }, + key: "train", + want: "RUNNING", + }, + { + name: "a queued task in a running run is waiting for compute", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + }, + key: "train", + want: "PENDING", + }, + { + name: "a task without a state falls back to the run", + runState: failedRun, + tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1)}, + key: "train", + want: "FAILED", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := &jobs.Run{RunId: 1, State: tt.runState, Tasks: tt.tasks} + narrowed := taskRun(run, tt.key) + require.NotNil(t, narrowed) + assert.Equal(t, tt.want, displayRunStatus(narrowed)) + assert.Equal(t, tt.want, buildListRow(narrowed, "https://example.test", 0).Status) + assert.Same(t, tt.runState, run.State, "the source run keeps its state") + }) + } +} diff --git a/cmd/air/list.go b/cmd/air/list.go index a4d2fed681d..8448160bd6d 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -136,7 +136,7 @@ Filters: cmd.PreRunE = root.MustWorkspaceClient - cmd.Flags().IntVar(&limit, "limit", 20, "Maximum number of runs to show") + cmd.Flags().IntVar(&limit, "limit", 20, "Maximum number of runs to show; a run with several AI Runtime tasks shows one row per task") cmd.Flags().BoolVar(&allStatus, "all-status", false, "Show runs in all states (default: active only)") cmd.Flags().BoolVar(&allUsers, "all-users", false, "Show runs from all users") cmd.Flags().StringArrayVar(&filters, "filter", nil, "Filter runs by KEY=VALUE (repeatable)") @@ -396,9 +396,12 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l } } +// next returns the rows of the next want matching runs. A run with several AI +// runtime tasks yields one row per task, so want counts runs, not rows. func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { var entries []listedRun - for len(entries) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { + runs := 0 + for runs < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { base, err := s.iter.Next(s.ctx) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) @@ -420,15 +423,29 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { continue } // One row per AI runtime task, each filtered on its own task. + matched := false for _, e := range buildListRows(run, s.host, s.workspaceID) { if s.filters.matchesFields(e.fields) { entries = append(entries, e.entry) + matched = true } } + if matched { + runs++ + } } return entries, nil } +// countRuns returns how many distinct runs the entries belong to. +func countRuns(entries []listedRun) int { + seen := make(map[int64]bool, len(entries)) + for _, e := range entries { + seen[e.runID] = true + } + return len(seen) +} + func (s *jobsScanStrategy) done() bool { return s.scanned >= s.maxScan || !s.iter.HasNext(s.ctx) } diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 3179bc135f6..213f4126b16 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -29,7 +29,7 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } data := buildGetData(run) - annotateTask(fullRun, run, &data) + annotateTask(run, len(airTaskKeys(fullRun)), &data) // As in `air get`: an ai_runtime_task run keeps its experiment, accelerators // and config path on the task. if genAIComputeTask(run) == nil { @@ -51,8 +51,10 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } // runLogsSnapshot fetches a one-shot tail of a run's logs into a string, for the -// list picker's detail pane, followed by the run's MLflow logs link. -func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { +// list picker's detail pane, followed by the run's MLflow logs link. taskKey is +// the selected row's task; the pane notes when the logs belong to another task +// (see logTaskNote). +func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64, taskKey string) (string, error) { run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { @@ -70,6 +72,10 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i } var buf bytes.Buffer + if note := logTaskNote(run, taskKey); note != "" { + fmt.Fprintln(&buf, note) + fmt.Fprintln(&buf) + } if _, err := fetchLogs(ctx, w, &buf, req, status); err != nil { return "", fmt.Errorf("failed to fetch logs: %w", err) } @@ -78,3 +84,24 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i } return buf.String(), nil } + +// logTaskNote returns a note for the logs pane when the logs it shows are not +// those of taskKey, the selected row's task: logs are read at run level from +// the run's latest task (see resolveLogAttempt), so on a run with several AI +// runtime tasks they can belong to another one. It returns "" when they match +// or no task was selected. +func logTaskNote(run *jobs.Run, taskKey string) string { + if taskKey == "" { + return "" + } + _, taskRunID, err := resolveLogAttempt(run, -1) + if err != nil { + return "" + } + for _, t := range run.Tasks { + if t.RunId == taskRunID && t.TaskKey != taskKey { + return fmt.Sprintf("Note: these are the logs of task %q, the run's latest task, not of the selected task %q. Logs are not yet available per task.", t.TaskKey, taskKey) + } + } + return "" +} diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 2e2eca05d5a..6c39ca907be 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -33,7 +33,7 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { })) t.Cleanup(srv.Close) - got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ "Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+ @@ -85,7 +85,7 @@ func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { require.NoError(t, err) info, err := runDetailText(ctx, w, 7, "") require.NoError(t, err) - logs, err := runLogsSnapshot(ctx, w, 7) + logs, err := runLogsSnapshot(ctx, w, 7, "") require.NoError(t, err) // Then each links retry 1's MLflow run and names that retry. @@ -292,3 +292,13 @@ func TestRunDetailTextShowsTheSelectedAirTask(t *testing.T) { assert.Contains(t, info, "Accelerators 8x H100") assert.Contains(t, info, "Task eval") } + +func TestLogTaskNote(t *testing.T) { + run := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 72, "e", "GPU_1xA10", 1), + }} + assert.Equal(t, `Note: these are the logs of task "eval", the run's latest task, not of the selected task "train". Logs are not yet available per task.`, logTaskNote(run, "train")) + assert.Empty(t, logTaskNote(run, "eval"), "the logs are the selected task's") + assert.Empty(t, logTaskNote(run, ""), "no task selected") +} diff --git a/cmd/air/list_format.go b/cmd/air/list_format.go index a828fab6312..518d8848287 100644 --- a/cmd/air/list_format.go +++ b/cmd/air/list_format.go @@ -42,7 +42,7 @@ func listedRunFor(run *jobs.Run, host string, workspaceID int64) listedRunFields // building dashboard URLs. func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { taskKey := "" - if t := airWrapperTask(run); t != nil { + if t := airTask(run); t != nil { taskKey = t.TaskKey } experiment := jobExperiment(run) diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index 824d105343c..aa779e1a82e 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -70,9 +70,11 @@ func (s *indexStrategy) next(want int) ([]listedRun, error) { want = min(want, s.limit-s.returned) } + // want counts runs, as on the scan path: a run with several AI runtime + // tasks hydrates into one row per task. var entries []listedRun - for len(entries) < want && s.pos < len(s.ids) { - end := min(s.pos+want-len(entries), len(s.ids)) + for runs := 0; runs < want && s.pos < len(s.ids); runs = countRuns(entries) { + end := min(s.pos+want-runs, len(s.ids)) batch := s.ids[s.pos:end] s.pos = end diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index b91c28843b6..6cbd4438b9f 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -572,3 +572,31 @@ func TestListAirRunsOneRowPerAirTask(t *testing.T) { assert.Equal(t, []string{"1", "1", "2"}, []string{rows[0].RunID, rows[1].RunID, rows[2].RunID}) assert.Equal(t, []string{"a", "b", ""}, []string{rows[0].TaskKey, rows[1].TaskKey, rows[2].TaskKey}) } + +func TestListAirRunsLimitCountsRunsNotRows(t *testing.T) { + // The first run has two AI Runtime tasks, so it lists as two rows; --limit 1 + // still shows that whole run, and --limit 2 also shows the next run. + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + body := runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c")) + + for _, tt := range []struct { + limit int + want []string + }{ + {1, []string{"1", "1"}}, + {2, []string{"1", "1", "2"}}, + } { + srv := runsServer(t, body) + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(tt.limit) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + assert.Equal(t, tt.want, ids, "limit %d", tt.limit) + } +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 0ba01ed8f90..4155bbaa20d 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -300,7 +300,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Logs Snapshot" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunLogs(runID) + return m, m.fetchRunLogs(runID, m.rows[m.cursor].TaskKey) } } m.offset = m.clampedOffset() @@ -455,11 +455,11 @@ func (m listModel) fetchRunDetail(runID int64, taskKey string) tea.Cmd { } // fetchRunLogs returns a tea.Cmd that fetches a log snapshot in the background. -func (m listModel) fetchRunLogs(runID int64) tea.Cmd { +func (m listModel) fetchRunLogs(runID int64, taskKey string) tea.Cmd { ctx := m.fetcher.ctx w := m.fetcher.w return func() tea.Msg { - body, err := runLogsSnapshot(ctx, w, runID) + body, err := runLogsSnapshot(ctx, w, runID, taskKey) if err != nil { return detailMsg{title: "Logs Snapshot", err: err} } From 2e0e1d5d820289c2490c90eca8b56ae586ae583b Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:59:05 +0000 Subject: [PATCH 4/9] air: write the AI Runtime task lookup and per-run grouping once Address review on #7632: - `airTaskKeys` is the one scan for AI Runtime tasks; `isAirRun` is built on it, and readers take `Tasks[0]` of the run narrowed by `taskRun` (`leadTask`), so `airTask` and `firstAirTaskIndex` are gone. `isAirTask` unwraps a foreach once (`taskSpec`) instead of checking twice. Callers outside `get` and `list` that read an unnarrowed run go through `defaultTaskRun`. - `selectAirTask` checks the key once with early returns; `taskRun` no longer re-checks it. The "no AI Runtime tasks" error no longer repeats. - `air get` and the detail pane share `taskView`. - Every list path passes runs as groups (`runGroup`): `--limit` counts groups, the index path sorts groups so a run's rows stay together, filtering happens in `matchingGroup`, the cache stores and reads a run's entries in one place, and `cancel --all` takes each group's first row and task keys. `countRuns` and `firstRowPerRun` are gone. Co-authored-by: Isaac --- cmd/air/cancel.go | 31 ++++----- cmd/air/cancel_test.go | 17 +++-- cmd/air/format_test.go | 1 + cmd/air/get.go | 50 ++++++++------- cmd/air/get_test.go | 42 ++++++++---- cmd/air/joblist.go | 127 +++++++++++++++++-------------------- cmd/air/joblist_test.go | 17 +++-- cmd/air/list.go | 81 ++++++++++++----------- cmd/air/list_cache.go | 30 ++++++--- cmd/air/list_cache_test.go | 40 +++++++----- cmd/air/list_detail.go | 5 +- cmd/air/list_format.go | 57 +++++++++++++---- cmd/air/list_index.go | 61 ++++++++---------- cmd/air/list_index_test.go | 26 ++++++++ cmd/air/list_test.go | 4 +- cmd/air/logdownload.go | 5 +- 16 files changed, 340 insertions(+), 254 deletions(-) diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index 41b311104a5..6095bf9cd8e 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -103,7 +103,7 @@ any other task such as a notebook that prepares data. // Fetch every active run (up to the scan bound) so --all cancels all // of them, not just the first page. fetcher := newRunFetcher(ctx, w, listQuery{activeOnly: true, userFilter: me.UserName}) - rows, err := fetcher.next(maxListScan) + runs, err := fetcher.nextRuns(maxListScan) if err != nil { return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, fmt.Errorf("failed to list active runs: %w", err)) @@ -111,8 +111,7 @@ any other task such as a notebook that prepares data. // A job with several AI runtime tasks lists one row per task; the // run is cancelled (and previewed) once, naming all its tasks. - var taskKeys map[string][]string - rows, taskKeys = firstRowPerRun(rows) + rows, taskKeys := cancelPreviewRows(runs) runIDs = make([]string, 0, len(rows)) for i := range rows { if rows[i].RunID != "" { @@ -222,22 +221,20 @@ func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, s return err } -// firstRowPerRun keeps the first row of each run id, in order, and returns the -// task keys of every row by run id, so the preview can name all the AI runtime -// tasks a cancel stops. -func firstRowPerRun(rows []listRow) ([]listRow, map[string][]string) { - taskKeys := make(map[string][]string, len(rows)) - unique := make([]listRow, 0, len(rows)) - for _, row := range rows { - if _, seen := taskKeys[row.RunID]; !seen { - unique = append(unique, row) - taskKeys[row.RunID] = nil - } - if row.TaskKey != "" { - taskKeys[row.RunID] = append(taskKeys[row.RunID], row.TaskKey) +// cancelPreviewRows returns the first row of each run, for the preview and the +// cancel, and the AI runtime task keys of each run by run id. +func cancelPreviewRows(runs [][]listRow) ([]listRow, map[string][]string) { + rows := make([]listRow, 0, len(runs)) + taskKeys := make(map[string][]string, len(runs)) + for _, run := range runs { + rows = append(rows, run[0]) + for _, r := range run { + if r.TaskKey != "" { + taskKeys[r.RunID] = append(taskKeys[r.RunID], r.TaskKey) + } } } - return unique, taskKeys + return rows, taskKeys } // displayCancelPreview shows the runs that `cancel --all` is about to terminate. diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index a5e57e4709c..f6db226803b 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -340,16 +340,15 @@ func TestDisplayCancelPreview(t *testing.T) { assert.Contains(t, out, na) } -func TestFirstRowPerRun(t *testing.T) { - rows := []listRow{ - {RunID: "1", TaskKey: "train"}, - {RunID: "1", TaskKey: "eval"}, - {RunID: "2"}, - {RunID: "3", TaskKey: "sweep"}, +func TestCancelPreviewRows(t *testing.T) { + runs := [][]listRow{ + {{RunID: "1", TaskKey: "train"}, {RunID: "1", TaskKey: "eval"}}, + {{RunID: "2"}}, + {{RunID: "3", TaskKey: "sweep"}}, } - unique, taskKeys := firstRowPerRun(rows) - assert.Equal(t, []listRow{{RunID: "1", TaskKey: "train"}, {RunID: "2"}, {RunID: "3", TaskKey: "sweep"}}, unique) - assert.Equal(t, map[string][]string{"1": {"train", "eval"}, "2": nil, "3": {"sweep"}}, taskKeys) + rows, taskKeys := cancelPreviewRows(runs) + assert.Equal(t, []listRow{{RunID: "1", TaskKey: "train"}, {RunID: "2"}, {RunID: "3", TaskKey: "sweep"}}, rows) + assert.Equal(t, map[string][]string{"1": {"train", "eval"}, "3": {"sweep"}}, taskKeys) } func TestCancelAllCancelsAMultiTaskRunOnce(t *testing.T) { diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 230aa4ba02d..0afb7e7ffe1 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -394,6 +394,7 @@ func TestLegacyExperimentAndAcceleratorsSkipNonAirTasks(t *testing.T) { Compute: &jobs.ComputeConfig{GpuType: "H100", NumGpus: 8}, }}, }} + run = defaultTaskRun(run) name := experimentName(run) require.NotNil(t, name) assert.Equal(t, "legacy-exp", *name) diff --git a/cmd/air/get.go b/cmd/air/get.go index 1a768302ca7..533f90d39bb 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "slices" "strconv" "strings" "sync" @@ -227,8 +228,7 @@ one.`, // From here on, run holds only the selected AI runtime task's task runs, // so the status, timing, MLflow and config lookups all read that task. - fullRun := run - run, err = selectAirTask(run, taskKey) + run, data, err := taskView(run, taskKey) if err != nil { return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) } @@ -245,8 +245,6 @@ one.`, fmt.Errorf("configured workspace_id %q is not a numeric workspace ID", id)) } - data := buildGetData(run) - annotateTask(run, len(airTaskKeys(fullRun)), &data) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) @@ -324,35 +322,39 @@ one.`, // tasks when key is not one of them. func selectAirTask(run *jobs.Run, key string) (*jobs.Run, error) { keys := airTaskKeys(run) - if key == "" { - if len(keys) == 0 { + if len(keys) == 0 { + if key == "" { return run, nil } + return nil, fmt.Errorf("run %d has no AI Runtime tasks", run.RunId) + } + if key == "" { key = keys[0] } - if narrowed := taskRun(run, key); narrowed != nil { - return narrowed, nil + if !slices.Contains(keys, key) { + return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) } - if len(keys) == 0 { - return nil, fmt.Errorf("run %d has no AI Runtime task %q; it has no AI Runtime tasks", run.RunId, key) - } - return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) + return taskRun(run, key), nil } -// annotateTask records which AI runtime task the view shows: selected is the -// run narrowed to it (see selectAirTask), and airTasks is how many AI runtime -// tasks the whole run has. TaskKey stays null for a run without one; the text -// cell is set only when there are several to tell apart. -func annotateTask(selected *jobs.Run, airTasks int, data *getData) { - t := airTask(selected) - if t == nil { - return +// taskView narrows run to the AI runtime task named by key (see selectAirTask) +// and builds its getData, recording the task shown: the shared start of +// `air get` and the list picker's detail pane, so the two stay in step. +// TaskKey stays null for a run without an AI runtime task; the Task cell is +// set only when the run has several to tell apart. +func taskView(run *jobs.Run, key string) (*jobs.Run, getData, error) { + selected, err := selectAirTask(run, key) + if err != nil { + return nil, getData{}, err } - key := t.TaskKey - data.TaskKey = &key - if airTasks > 1 { - data.TaskDisplay = key + data := buildGetData(selected) + if k := airTaskKey(selected); k != "" { + data.TaskKey = &k + if len(airTaskKeys(run)) > 1 { + data.TaskDisplay = k + } } + return selected, data, nil } // enrichFromAiRuntimeTask fills the config path, experiment, and accelerators diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index f86f76d913d..0e57e2f6b96 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -849,7 +849,7 @@ func TestAiRuntimeTaskOfSkipsNonAirTasks(t *testing.T) { notebookTask("prep", 51), aiRuntimeTask("train", 52, "/Users/me@example.com/exp", "GPU_1xA10", 1), }} - task := aiRuntimeTaskOf(run) + task := aiRuntimeTaskOf(defaultTaskRun(run)) require.NotNil(t, task) assert.Equal(t, "/Users/me@example.com/exp", task.Experiment) } @@ -935,24 +935,44 @@ func TestGetRunSelectsAirTask(t *testing.T) { }) } -func TestAnnotateTaskIsNullWithoutAnAirTask(t *testing.T) { +func TestTaskViewTaskKey(t *testing.T) { run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}} - selected, err := selectAirTask(run, "") + _, data, err := taskView(run, "") require.NoError(t, err) - var data getData - annotateTask(selected, len(airTaskKeys(run)), &data) assert.Nil(t, data.TaskKey, "matches air list, which reports no task, and --task-key prep, which errors") assert.Empty(t, data.TaskDisplay) - two := &jobs.Run{RunId: 6, Tasks: []jobs.RunTask{ - aiRuntimeTask("train", 61, "e", "GPU_1xA10", 1), - aiRuntimeTask("eval", 62, "e", "GPU_1xA10", 1), + single := &jobs.Run{RunId: 6, Tasks: []jobs.RunTask{notebookTask("prep", 60), aiRuntimeTask("train", 61, "e", "GPU_1xA10", 1)}} + _, data, err = taskView(single, "") + require.NoError(t, err) + require.NotNil(t, data.TaskKey) + assert.Equal(t, "train", *data.TaskKey) + assert.Empty(t, data.TaskDisplay, "one AI Runtime task needs no Task line") + + two := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 72, "e", "GPU_1xA10", 1), }} - selected, err = selectAirTask(two, "eval") + selected, data, err := taskView(two, "eval") require.NoError(t, err) - data = getData{} - annotateTask(selected, len(airTaskKeys(two)), &data) + assert.Equal(t, "eval", selected.Tasks[0].TaskKey) require.NotNil(t, data.TaskKey) assert.Equal(t, "eval", *data.TaskKey) assert.Equal(t, "eval", data.TaskDisplay) } + +func TestSelectAirTaskErrors(t *testing.T) { + _, err := selectAirTask(&jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}}, "prep") + assert.EqualError(t, err, "run 5 has no AI Runtime tasks") + + two := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ + notebookTask("prep", 70), + aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), + }} + _, err = selectAirTask(two, "prep") + assert.EqualError(t, err, `run 7 has no AI Runtime task "prep"; its AI Runtime tasks are: train`) + + run, err := selectAirTask(&jobs.Run{RunId: 8, Tasks: []jobs.RunTask{notebookTask("prep", 80)}}, "") + require.NoError(t, err) + assert.Len(t, run.Tasks, 1, "a run without an AI Runtime task is shown as is") +} diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 2d062a64306..3c743751f7f 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -17,34 +17,27 @@ import ( // of run ids from the AiTrainingService index. const hydrateConcurrency = 16 -// isAirTask reports whether a task run is an AI runtime workload: an -// ai_runtime_task, or a legacy gen_ai_compute_task with a training script. A -// foreach task counts when the task it iterates is one. -func isAirTask(t *jobs.RunTask) bool { +// taskSpec returns the AI task definitions a task run runs: its own, or for a +// foreach sweep those of the task it iterates. +func taskSpec(t *jobs.RunTask) (*jobs.AiRuntimeTask, *jobs.GenAiComputeTask) { if t.ForEachTask != nil { - inner := t.ForEachTask.Task - return inner.AiRuntimeTask != nil || - (inner.GenAiComputeTask != nil && inner.GenAiComputeTask.TrainingScriptPath != "") + return t.ForEachTask.Task.AiRuntimeTask, t.ForEachTask.Task.GenAiComputeTask } - return t.AiRuntimeTask != nil || - (t.GenAiComputeTask != nil && t.GenAiComputeTask.TrainingScriptPath != "") + return t.AiRuntimeTask, t.GenAiComputeTask } -// firstAirTaskIndex returns the index in r.Tasks of the run's first AI runtime -// task in task order, or -1 when it has none. A job may put other tasks before -// it (the official multi-task DAB runs a notebook prep step first), so the -// first task is not inspected directly. -func firstAirTaskIndex(r *jobs.Run) int { - for i := range r.Tasks { - if isAirTask(&r.Tasks[i]) { - return i - } - } - return -1 +// isAirTask reports whether a task run is an AI runtime workload: an +// ai_runtime_task, or a legacy gen_ai_compute_task with a training script, +// directly or as the task a foreach sweep iterates. +func isAirTask(t *jobs.RunTask) bool { + ai, genAI := taskSpec(t) + return ai != nil || (genAI != nil && genAI.TrainingScriptPath != "") } // airTaskKeys returns the task keys of the run's AI runtime tasks in task -// order, each once: a retried task appears in r.Tasks once per attempt. +// order, each once: a retried task appears in r.Tasks once per attempt. It is +// the one scan for AI runtime tasks; everything after it reads a run narrowed +// to one of them (see taskRun), whose task is Tasks[0]. func airTaskKeys(r *jobs.Run) []string { var keys []string for i := range r.Tasks { @@ -56,15 +49,17 @@ func airTaskKeys(r *jobs.Run) []string { return keys } +// isAirRun reports whether a run is an AI runtime workload: it has at least +// one AI runtime task. +func isAirRun(r *jobs.Run) bool { + return len(airTaskKeys(r)) > 0 +} + // taskRun returns a copy of r narrowed to the task runs (every attempt, in -// order) of the AI runtime task with the given key, so the display helpers, -// which read a run's tasks, report that task alone. Its state is that task's -// (see taskState). It returns nil when key is not one of the run's AI runtime -// tasks. +// order) of the task with the given key, one of airTaskKeys(r), so the display +// helpers, which read a run's tasks, report that task alone. Its state is that +// task's (see taskState). func taskRun(r *jobs.Run, key string) *jobs.Run { - if !slices.Contains(airTaskKeys(r), key) { - return nil - } narrowed := *r narrowed.Tasks = nil for i := range r.Tasks { @@ -72,14 +67,25 @@ func taskRun(r *jobs.Run, key string) *jobs.Run { narrowed.Tasks = append(narrowed.Tasks, r.Tasks[i]) } } - latest := &narrowed.Tasks[len(narrowed.Tasks)-1] - if state := taskState(r, latest); state != r.State { - narrowed.State = state - narrowed.Status = latest.Status + if n := len(narrowed.Tasks); n > 0 { + latest := &narrowed.Tasks[n-1] + if state := taskState(r, latest); state != r.State { + narrowed.State = state + narrowed.Status = latest.Status + } } return &narrowed } +// defaultTaskRun returns r narrowed to its first AI runtime task in task order, +// or r as is when it has none: the run `air get` shows without --task-key. +func defaultTaskRun(r *jobs.Run) *jobs.Run { + if keys := airTaskKeys(r); len(keys) > 0 { + return taskRun(r, keys[0]) + } + return r +} + // taskState returns the state to report for the task whose latest attempt is // latest: that attempt's state, so one task's result doesn't stand for another // (a successful train next to a failed eval, or an AI runtime task skipped @@ -99,57 +105,40 @@ func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { return s } -// airTask returns the task run of the run's first AI runtime task as the Jobs -// API reports it (for a sweep, the foreach wrapper, which carries the state and -// timing), or nil when the run has none. -func airTask(r *jobs.Run) *jobs.RunTask { - i := firstAirTaskIndex(r) - if i < 0 { - return nil - } - return &r.Tasks[i] -} - -// leadTask is airTask with a fallback to the first task for a run without an AI -// runtime task (such as a legacy gen_ai_compute_task without a training -// script), whose fields `air get` still shows; nil when the run has no tasks. +// leadTask returns the task run a run reports: Tasks[0], which is the AI +// runtime task once the run is narrowed (see taskRun). A run without an AI +// runtime task is not narrowed, so its first task stands in, as it always has; +// nil when the run has no tasks. func leadTask(r *jobs.Run) *jobs.RunTask { - if t := airTask(r); t != nil { - return t - } if len(r.Tasks) == 0 { return nil } return &r.Tasks[0] } -// firstTask returns the run's first AI runtime task, unwrapping a foreach sweep -// to the iterated task. A run without one falls back to its first task, so a -// legacy run's fields are still read; nil when the run has no tasks. +// firstTask returns the run's lead task (see leadTask), unwrapping a foreach +// sweep to the iterated task; nil when the run has no tasks. func firstTask(r *jobs.Run) *jobs.RunTask { t := leadTask(r) - if t == nil { - return nil - } - if t.ForEachTask != nil { - // The foreach inner task is a jobs.Task; expose the AI fields via a - // synthetic RunTask so callers can treat both shapes uniformly. - return &jobs.RunTask{ - TaskKey: t.TaskKey, - AiRuntimeTask: t.ForEachTask.Task.AiRuntimeTask, - GenAiComputeTask: t.ForEachTask.Task.GenAiComputeTask, - } + if t == nil || t.ForEachTask == nil { + return t } - return t + // The foreach inner task is a jobs.Task; expose the AI fields via a + // synthetic RunTask so callers can treat both shapes uniformly. + ai, genAI := taskSpec(t) + return &jobs.RunTask{TaskKey: t.TaskKey, AiRuntimeTask: ai, GenAiComputeTask: genAI} } -// isAirRun reports whether a run is an AI runtime workload: it has an -// ai_runtime_task, or a legacy gen_ai_compute_task with a training script. -func isAirRun(r *jobs.Run) bool { - return firstAirTaskIndex(r) >= 0 +// airTaskKey returns the key of the AI runtime task a narrowed run shows, or "" +// for a run without one. +func airTaskKey(r *jobs.Run) string { + if t := leadTask(r); t != nil && isAirTask(t) { + return t.TaskKey + } + return "" } -// isSweep reports whether the run's AI runtime task fans out into iterations. +// isSweep reports whether the run's lead task fans out into iterations. func isSweep(r *jobs.Run) bool { t := leadTask(r) return t != nil && t.ForEachTask != nil diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 79f6bc239a5..000503d0114 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -208,8 +208,10 @@ func TestAirTaskSelection(t *testing.T) { run := &jobs.Run{RunId: 1, Tasks: tt.tasks} assert.Equal(t, tt.air, isAirRun(run), "isAirRun") assert.Equal(t, tt.keys, airTaskKeys(run), "airTaskKeys") - assert.Equal(t, tt.experiment, jobExperiment(run), "jobExperiment") - assert.Equal(t, tt.sweep, isSweep(run), "isSweep") + // Readers take the run narrowed to its AI Runtime task. + shown := defaultTaskRun(run) + assert.Equal(t, tt.experiment, jobExperiment(shown), "jobExperiment") + assert.Equal(t, tt.sweep, isSweep(shown), "isSweep") if len(tt.tasks) == 0 { assert.Nil(t, firstTask(run), "firstTask") } @@ -246,8 +248,13 @@ func TestTaskRunNarrowsToOneAirTask(t *testing.T) { assert.Equal(t, 8, n) assert.Len(t, run.Tasks, 4, "the source run is left intact") - assert.Nil(t, taskRun(run, "prep"), "a non-AIR task is not selectable") - assert.Nil(t, taskRun(run, "missing")) + + first := defaultTaskRun(run) + assert.Equal(t, "a", first.Tasks[0].TaskKey, "the default is the first AI Runtime task, not the notebook") + assert.Equal(t, "a", airTaskKey(first)) + notebookOnly := &jobs.Run{RunId: 2, Tasks: []jobs.RunTask{notebookTask("prep", 21)}} + assert.Same(t, notebookOnly, defaultTaskRun(notebookOnly), "a run without an AI Runtime task is not narrowed") + assert.Empty(t, airTaskKey(notebookOnly)) } func TestJobTimingUsesTheAirTask(t *testing.T) { @@ -257,7 +264,7 @@ func TestJobTimingUsesTheAirTask(t *testing.T) { train.StartTime, train.EndTime = 200, 260 run := &jobs.Run{RunId: 1, StartTime: 90, EndTime: 300, Tasks: []jobs.RunTask{prep, train}} - start, end := jobTiming(run) + start, end := jobTiming(defaultTaskRun(run)) assert.Equal(t, int64(200), start) assert.Equal(t, int64(260), end) } diff --git a/cmd/air/list.go b/cmd/air/list.go index 8448160bd6d..a3def34937d 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -261,8 +261,8 @@ func validateEmptyAcceleratorFilter(ctx context.Context, w *databricks.Workspace // jobsScanStrategy pages runs/list; indexStrategy hydrates the AiTrainingService // index. The fetcher wraps whichever is chosen. type listStrategy interface { - // next returns up to want more matching runs (already row-built + task id). - next(want int) ([]listedRun, error) + // next returns up to want more matching runs, each as its group of rows. + next(want int) ([]runGroup, error) // done reports whether the source has no more runs to yield. done() bool // truncated reports whether a safety cap stopped the scan short of the end. @@ -322,15 +322,29 @@ func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQ // text output, and projects it to rows. It sets exhausted once the strategy is // drained so the interactive table knows to stop paging. func (f *runFetcher) next(want int) ([]listRow, error) { - entries, err := f.strategy.next(want) + groups, err := f.nextRuns(want) + if err != nil { + return nil, err + } + var rows []listRow + for _, g := range groups { + rows = append(rows, g...) + } + return rows, nil +} + +// nextRuns is next with the rows kept grouped by run: one slice per run, one +// row per AI runtime task. +func (f *runFetcher) nextRuns(want int) ([][]listRow, error) { + groups, err := f.strategy.next(want) if err != nil { return nil, err } f.exhausted = f.strategy.done() - if len(entries) > 0 { + if len(groups) > 0 { f.matchedAny = true } - if len(entries) == 0 && f.exhausted && !f.matchedAny && f.validateEmpty != nil && !f.validatedEmpty { + if len(groups) == 0 && f.exhausted && !f.matchedAny && f.validateEmpty != nil && !f.validatedEmpty { f.validatedEmpty = true if err := f.validateEmpty(); err != nil { return nil, err @@ -338,19 +352,28 @@ func (f *runFetcher) next(want int) ([]listRow, error) { } // MLflow links appear only in the text table, so the per-run get-output - // lookups are skipped for JSON output (which omits the column anyway). + // lookups are skipped for JSON output (which omits the column anyway). They + // run over all rows at once, then land back in their groups. if f.fetchMLflow { - setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries) + var flat []listedRun + for _, g := range groups { + flat = append(flat, g.entries...) + } + setMLflowLinks(f.ctx, f.w, f.w.Config.Host, flat) if s, ok := f.strategy.(*indexStrategy); ok { - s.cacheMLflowLinks(entries) + s.cacheMLflowLinks(flat) + } + for _, g := range groups { + n := copy(g.entries, flat) + flat = flat[n:] } } - rows := make([]listRow, len(entries)) - for i, e := range entries { - rows[i] = e.row + out := make([][]listRow, len(groups)) + for i, g := range groups { + out[i] = g.rows() } - return rows, nil + return out, nil } // jobsScanStrategy pages Jobs runs/list, keeping the AIR runs that match the user @@ -396,12 +419,11 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l } } -// next returns the rows of the next want matching runs. A run with several AI -// runtime tasks yields one row per task, so want counts runs, not rows. -func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { - var entries []listedRun - runs := 0 - for runs < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { +// next returns the next want matching runs, each as its group of rows (see +// runGroup). +func (s *jobsScanStrategy) next(want int) ([]runGroup, error) { + var groups []runGroup + for len(groups) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { base, err := s.iter.Next(s.ctx) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) @@ -422,28 +444,11 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { if s.userFilter != "" && run.CreatorUserName != s.userFilter { continue } - // One row per AI runtime task, each filtered on its own task. - matched := false - for _, e := range buildListRows(run, s.host, s.workspaceID) { - if s.filters.matchesFields(e.fields) { - entries = append(entries, e.entry) - matched = true - } - } - if matched { - runs++ + if g, ok := matchingGroup(run.RunId, buildTaskEntries(run, s.host, s.workspaceID), s.filters); ok { + groups = append(groups, g) } } - return entries, nil -} - -// countRuns returns how many distinct runs the entries belong to. -func countRuns(entries []listedRun) int { - seen := make(map[int64]bool, len(entries)) - for _, e := range entries { - seen[e.runID] = true - } - return len(seen) + return groups, nil } func (s *jobsScanStrategy) done() bool { diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index 898c3747fa7..10581323043 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -26,7 +26,7 @@ type listCacheKey struct { } // cachedRuns is the persisted value for a run: one cachedRun per AI runtime -// task (see buildListRows). +// task, before filtering (see buildTaskEntries), so a hit can be re-filtered. type cachedRuns struct { Rows []cachedRun `json:"rows"` } @@ -88,23 +88,33 @@ func newListCache(ctx context.Context) *cache.Cache { return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil) } -// cachedRows returns the cached rows of a run, one per AI runtime task. An -// entry with a row that lacks task runs is treated as a miss. -func cachedRows(ctx context.Context, c *cache.Cache, host string, runID int64) ([]cachedRun, bool) { - entry, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) - if !ok || len(entry.Rows) == 0 { +// cachedEntries returns the cached entries of a run, one per AI runtime task. +// An entry whose row lacks task runs is treated as a miss. +func cachedEntries(ctx context.Context, c *cache.Cache, host string, runID int64) ([]taskEntry, bool) { + cached, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) + if !ok || len(cached.Rows) == 0 { return nil, false } - for _, row := range entry.Rows { + entries := make([]taskEntry, len(cached.Rows)) + for i, row := range cached.Rows { if len(row.Tasks) == 0 { return nil, false } + entries[i] = taskEntry{ + entry: listedRun{row: row.toRow(), runID: runID, taskKey: row.TaskKey, tasks: row.Tasks, terminal: true}, + fields: row.Fields, + } } - return entry.Rows, true + return entries, true } -// putRows caches a terminal run's rows. -func putRows(ctx context.Context, c *cache.Cache, host string, runID int64, rows []cachedRun) { +// putEntries caches a terminal run's entries, all of them, so a later hit can +// be re-filtered on every task. +func putEntries(ctx context.Context, c *cache.Cache, host string, runID int64, entries []taskEntry, submitTimeMs int64) { + rows := make([]cachedRun, len(entries)) + for i, e := range entries { + rows[i] = cachedRunFromRow(e.entry.row, e.entry.tasks, e.fields, submitTimeMs) + } cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRuns{Rows: rows}) } diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 71001e00d87..243e2722012 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -15,7 +15,15 @@ import ( // putRow caches a single-row run, the common case in these tests. func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { - putRows(ctx, c, host, runID, []cachedRun{cachedRunFromRow(row, tasks, fields, submitTimeMs)}) + putEntries(ctx, c, host, runID, []taskEntry{{entry: listedRun{row: row, tasks: tasks}, fields: fields}}, submitTimeMs) +} + +// rawCached reads the persisted value for a run, to check what was stored. +func rawCached(t *testing.T, ctx context.Context, c *cache.Cache, host string, runID int64) cachedRuns { + t.Helper() + got, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) + require.True(t, ok) + return got } func TestListCacheRoundTrip(t *testing.T) { @@ -23,7 +31,7 @@ func TestListCacheRoundTrip(t *testing.T) { ctx := t.Context() c := newListCache(ctx) - _, ok := cachedRows(ctx, c, "https://host.test", 42) + _, ok := cachedEntries(ctx, c, "https://host.test", 42) require.False(t, ok, "miss before write") row := listRow{ @@ -34,27 +42,30 @@ func TestListCacheRoundTrip(t *testing.T) { tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}} putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields) - got, ok := cachedRows(ctx, c, "https://host.test", 42) + got, ok := cachedEntries(ctx, c, "https://host.test", 42) require.True(t, ok, "hit after write") require.Len(t, got, 1) - assert.Equal(t, row, got[0].toRow()) - assert.Equal(t, fields, got[0].Fields) - gotTasks := got[0].Tasks + assert.Equal(t, row, got[0].entry.row) + assert.Equal(t, fields, got[0].fields) + assert.Equal(t, int64(42), got[0].entry.runID) + assert.True(t, got[0].entry.terminal) + assert.Equal(t, int64(1700000000000), rawCached(t, ctx, c, "https://host.test", 42).Rows[0].SubmitTimeMs) + gotTasks := got[0].entry.tasks for i := range gotTasks { gotTasks[i].ForceSendFields = nil // set by the JSON round trip } assert.Equal(t, tasks, gotTasks) // Different host is a different key. - _, ok = cachedRows(ctx, c, "https://other.test", 42) + _, ok = cachedEntries(ctx, c, "https://other.test", 42) assert.False(t, ok) // Entries from previous cache formats have no rows and must be refreshed. cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430}) - _, ok = cachedRows(ctx, c, "https://host.test", 43) + _, ok = cachedEntries(ctx, c, "https://host.test", 43) assert.False(t, ok) cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 44}, cachedRuns{Rows: []cachedRun{{RunID: "44"}}}) - _, ok = cachedRows(ctx, c, "https://host.test", 44) + _, ok = cachedEntries(ctx, c, "https://host.test", 44) assert.False(t, ok, "a row without task runs is a miss") } @@ -64,17 +75,16 @@ func TestListCacheKeepsOneRowPerAirTask(t *testing.T) { c := newListCache(ctx) host := "https://host.test" - putRows(ctx, c, host, 42, []cachedRun{ - cachedRunFromRow(listRow{RunID: "42", TaskKey: "train", Experiment: "exp-a"}, []jobs.RunTask{{RunId: 420, TaskKey: "train"}}, filterFields{Experiment: "exp-a"}, 1000), - cachedRunFromRow(listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b"}, []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}, filterFields{Experiment: "exp-b"}, 1000), - }) + putEntries(ctx, c, host, 42, []taskEntry{ + {entry: listedRun{row: listRow{RunID: "42", TaskKey: "train", Experiment: "exp-a"}, tasks: []jobs.RunTask{{RunId: 420, TaskKey: "train"}}}, fields: filterFields{Experiment: "exp-a"}}, + {entry: listedRun{row: listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b"}, tasks: []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}}, fields: filterFields{Experiment: "exp-b"}}, + }, 1000) // The MLflow link of one task updates that row alone. updateCachedRow(ctx, c, host, 42, "eval", listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b", MLflowURL: "https://host.test/ml/x"}) updateCachedRow(ctx, c, host, 42, "missing", listRow{RunID: "42", TaskKey: "missing"}) - got, ok := cachedRows(ctx, c, host, 42) - require.True(t, ok) + got := rawCached(t, ctx, c, host, 42).Rows require.Len(t, got, 2) assert.Equal(t, "train", got[0].TaskKey) assert.Empty(t, got[0].MLflowURL) diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 213f4126b16..5c4015c28e8 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -22,14 +22,11 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } return "", fmt.Errorf("failed to fetch run: %w", err) } - fullRun := run - run, err = selectAirTask(run, taskKey) + run, data, err := taskView(run, taskKey) if err != nil { return "", err } - data := buildGetData(run) - annotateTask(run, len(airTaskKeys(fullRun)), &data) // As in `air get`: an ai_runtime_task run keeps its experiment, accelerators // and config path on the task. if genAIComputeTask(run) == nil { diff --git a/cmd/air/list_format.go b/cmd/air/list_format.go index 518d8848287..aff014f110f 100644 --- a/cmd/air/list_format.go +++ b/cmd/air/list_format.go @@ -6,45 +6,74 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" ) -// listedRunFields is a list entry with the task-filter inputs it was built from. -type listedRunFields struct { +// taskEntry is one task's list entry with the task-filter inputs it was built +// from, before any filter is applied. +type taskEntry struct { entry listedRun fields filterFields } -// buildListRows builds one list entry per AI runtime task of the run, in task +// runGroup is one run's list entries that pass the filters: one per AI runtime +// task, in task order (a run without one has a single entry). Every list path +// passes runs along as groups, so --limit counts runs, sorting keeps a run's +// rows together, and cancel and the cache act on the whole run. +type runGroup struct { + runID int64 + entries []listedRun +} + +// buildTaskEntries builds one entry per AI runtime task of the run, in task // order, each read from that task's own task runs (see taskRun). A run without // an AI runtime task yields a single entry read from the run as is. -func buildListRows(run *jobs.Run, host string, workspaceID int64) []listedRunFields { +func buildTaskEntries(run *jobs.Run, host string, workspaceID int64) []taskEntry { keys := airTaskKeys(run) if len(keys) == 0 { - return []listedRunFields{listedRunFor(run, host, workspaceID)} + return []taskEntry{taskEntryFor(run, host, workspaceID)} } - entries := make([]listedRunFields, 0, len(keys)) + entries := make([]taskEntry, 0, len(keys)) for _, key := range keys { - entries = append(entries, listedRunFor(taskRun(run, key), host, workspaceID)) + entries = append(entries, taskEntryFor(taskRun(run, key), host, workspaceID)) } return entries } -// listedRunFor builds the list entry for a run (or a run narrowed to one task). -func listedRunFor(run *jobs.Run, host string, workspaceID int64) listedRunFields { +// taskEntryFor builds the entry for a run narrowed to one task (or a run +// without an AI runtime task). +func taskEntryFor(run *jobs.Run, host string, workspaceID int64) taskEntry { row := buildListRow(run, host, workspaceID) - return listedRunFields{ + return taskEntry{ entry: listedRun{row: row, runID: run.RunId, taskKey: row.TaskKey, tasks: retryTasks(run), terminal: isTerminal(run)}, fields: filterFieldsFromRun(run), } } +// matchingGroup returns the run's entries that pass f, each filtered on its own +// task, or ok false when none do. +func matchingGroup(runID int64, all []taskEntry, f listFilters) (runGroup, bool) { + g := runGroup{runID: runID} + for _, e := range all { + if f.matchesFields(e.fields) { + g.entries = append(g.entries, e.entry) + } + } + return g, len(g.entries) > 0 +} + +// rows returns the group's display rows. +func (g runGroup) rows() []listRow { + rows := make([]listRow, len(g.entries)) + for i, e := range g.entries { + rows[i] = e.row + } + return rows +} + // buildListRow extracts the columns shown for one run, read from its first AI // runtime task. Optional cells fall back to "-"; MLflowURL starts as "-" and // setMLflowLinks fills it in for text output. host and workspaceID are used for // building dashboard URLs. func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { - taskKey := "" - if t := airTask(run); t != nil { - taskKey = t.TaskKey - } + taskKey := airTaskKey(run) experiment := jobExperiment(run) experimentName := nilIfEmpty(experiment) if experiment == "" { diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index aa779e1a82e..70a6b56409b 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -60,7 +60,7 @@ func (s *indexStrategy) load() error { return nil } -func (s *indexStrategy) next(want int) ([]listedRun, error) { +func (s *indexStrategy) next(want int) ([]runGroup, error) { if !s.loaded { if err := s.load(); err != nil { return nil, err @@ -71,21 +71,21 @@ func (s *indexStrategy) next(want int) ([]listedRun, error) { } // want counts runs, as on the scan path: a run with several AI runtime - // tasks hydrates into one row per task. - var entries []listedRun - for runs := 0; runs < want && s.pos < len(s.ids); runs = countRuns(entries) { - end := min(s.pos+want-runs, len(s.ids)) + // tasks hydrates into one group of rows. + var groups []runGroup + for len(groups) < want && s.pos < len(s.ids) { + end := min(s.pos+want-len(groups), len(s.ids)) batch := s.ids[s.pos:end] s.pos = end - rows, err := s.hydrate(batch) + hydrated, err := s.hydrate(batch) if err != nil { return nil, err } - entries = append(entries, rows...) + groups = append(groups, hydrated...) } - s.returned += len(entries) - return entries, nil + s.returned += len(groups) + return groups, nil } func (s *indexStrategy) done() bool { @@ -99,19 +99,17 @@ func (s *indexStrategy) truncated() bool { return false } // without a network call and fetching the rest via runs/get. Freshly hydrated // terminal runs are cached. Results keep the input (newest-first) order, then // the batch is re-sorted by start time since concurrent hydration reorders it. -func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { +func (s *indexStrategy) hydrate(ids []int64) ([]runGroup, error) { host := s.w.Config.Host - rows := make([]listedRun, 0, len(ids)) + groups := make([]runGroup, 0, len(ids)) var toFetch []int64 - // The cache key excludes the filter, so a cached row is still run through the + // The cache key excludes the filter, so a cached run is still run through the // active filter; a non-matching hit is dropped, not re-fetched. for _, id := range ids { - if cached, ok := cachedRows(s.ctx, s.cache, host, id); ok { - for _, c := range cached { - if s.filters.matchesFields(c.Fields) { - rows = append(rows, listedRun{row: c.toRow(), runID: id, taskKey: c.TaskKey, tasks: c.Tasks, terminal: true}) - } + if entries, ok := cachedEntries(s.ctx, s.cache, host, id); ok { + if g, ok := matchingGroup(id, entries, s.filters); ok { + groups = append(groups, g) } continue } @@ -123,28 +121,23 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { return nil, err } for _, run := range runs { - // One row per AI runtime task; the whole set is cached so a later lookup - // by run id can re-filter every task. - entries := buildListRows(run, s.host, s.workspaceID) - start, _ := jobTiming(run) - cached := make([]cachedRun, 0, len(entries)) - for _, e := range entries { - cached = append(cached, cachedRunFromRow(e.entry.row, e.entry.tasks, e.fields, start)) - if s.filters.matchesFields(e.fields) { - rows = append(rows, e.entry) - } - } + entries := buildTaskEntries(run, s.host, s.workspaceID) if isTerminal(run) { - putRows(s.ctx, s.cache, host, run.RunId, cached) + start, _ := jobTiming(defaultTaskRun(run)) + putEntries(s.ctx, s.cache, host, run.RunId, entries, start) + } + if g, ok := matchingGroup(run.RunId, entries, s.filters); ok { + groups = append(groups, g) } } - // Concurrent hydration reorders runs, so re-sort the batch newest-first. The - // ISO start timestamp sorts lexicographically; a missing time ("") sorts last. - slices.SortStableFunc(rows, func(a, b listedRun) int { - return cmp.Compare(rowStartKey(b.row), rowStartKey(a.row)) + // Concurrent hydration reorders runs, so re-sort the batch newest-first by + // each run's first row, keeping a run's rows together. The ISO start + // timestamp sorts lexicographically; a missing time ("") sorts last. + slices.SortStableFunc(groups, func(a, b runGroup) int { + return cmp.Compare(rowStartKey(b.entries[0].row), rowStartKey(a.entries[0].row)) }) - return rows, nil + return groups, nil } // cacheMLflowLinks writes successful MLflow enrichment back to terminal rows diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index c7495f2be66..edb5a5969ef 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -382,3 +382,29 @@ func TestListSelfUserFilterUsesIndex(t *testing.T) { assert.Contains(t, stdout.String(), `"run_id": "7"`) assert.Empty(t, warnings.String()) } + +func TestIndexStrategyKeepsARunsRowsTogether(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 7's two AI Runtime tasks start before and after run 8. Sorting rows + // would split run 7 around run 8; sorting runs keeps its rows together, and + // --limit counts it once. + multi := indexRun(7, 3000_000) + train, eval := multi.Tasks[0], multi.Tasks[0] + train.TaskKey, train.RunId, train.StartTime = "train", 70, 1000_000 + eval.TaskKey, eval.RunId, eval.StartTime = "eval", 71, 3000_000 + multi.Tasks = []jobs.RunTask{train, eval} + refs := []workflowRef{{jobRunID: 7, submitTimeMs: 1000_000}, {jobRunID: 8, submitTimeMs: 2000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{7: multi, 8: indexRun(8, 2000_000)}, nil, nil) + + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{currentUser: selfUser, limit: 10}) + runs, err := f.nextRuns(10) + require.NoError(t, err) + got := make([][]string, len(runs)) + for i, run := range runs { + for _, r := range run { + got[i] = append(got[i], r.RunID+"/"+r.TaskKey) + } + } + assert.Equal(t, [][]string{{"8/"}, {"7/train", "7/eval"}}, got) +} diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index 6cbd4438b9f..448734f6608 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -503,14 +503,14 @@ func TestListValidatesAcceleratorOnlyAfterNoMatches(t *testing.T) { } } -func TestBuildListRowsOnePerAirTask(t *testing.T) { +func TestBuildTaskEntriesOnePerAirTask(t *testing.T) { run := jobs.Run{RunId: 7, RunName: "dag", CreatorUserName: "me@example.com", Tasks: []jobs.RunTask{ notebookTask("prep", 71), aiRuntimeTask("train", 72, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), aiRuntimeTask("eval", 73, "/Users/me@example.com/exp-b", "GPU_8xH100", 8), }} - rows := buildListRows(&run, "https://example.test", 0) + rows := buildTaskEntries(&run, "https://example.test", 0) require.Len(t, rows, 2) assert.Equal(t, "7", rows[0].entry.row.RunID) assert.Equal(t, "train", rows[0].entry.row.TaskKey) diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index a33a0d92ab9..8ab5f244254 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -29,9 +29,10 @@ func validateNodeIndex(node, numNodes int) error { return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1) } -// resolveNodeCount returns how many nodes a run used. +// resolveNodeCount returns how many nodes a run used, read from its first AI +// runtime task (see defaultTaskRun). func resolveNodeCount(run *jobs.Run) (int, error) { - accelType, count := jobCompute(run) + accelType, count := jobCompute(defaultTaskRun(run)) if accelType == "" { return 0, fmt.Errorf("run %d has no AI runtime compute config", run.RunId) } From 0b6c20b1248ba227a8c81e20fe49e88db1cac3f0 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:17:32 +0000 Subject: [PATCH 5/9] air: keep single-task runs as they were and report each task's own environment, timing and nodes Review follow-ups for multi-task AI Runtime jobs: - A run whose only task is the AI Runtime task keeps its run-level state, status and timing, so single-task `air run` submissions report as before (termination prefix, run-level TIMEDOUT). - A skipped task (upstream failed, upstream cancelled, excluded) keeps its own state, and a task of a multi-task run reports only its own timing, never the whole run's. - Only a pending run, or a running run whose task waits to start, reports as waiting for compute; queued, blocked and waiting-for-retry runs report that state, as on main. - `air get` reads the environment keyed by the selected task's environment_key instead of `default`. - `air logs --download-to` and `--node` validation read the node count from the task the logs come from. - `air list` names the task in the Run ID column for a run with several AI Runtime tasks, reports `task_key` as null without one (matching `air get`), prints `"runs": []` for an empty list, and sorts a run by its first started task. - The list fixture comment no longer implies the default scan reaches bundle-deployed (JOB_RUN) jobs. Co-authored-by: Isaac --- .nextchanges/air/multitask-air-tasks.md | 2 +- acceptance/air/get-multitask/output.txt | 4 +- acceptance/air/get-multitask/test.toml | 5 +- acceptance/air/list-index-fallback/output.txt | 2 +- acceptance/air/list/output.txt | 4 +- acceptance/air/list/test.toml | 10 +- cmd/air/cancel.go | 4 +- cmd/air/cancel_test.go | 6 +- cmd/air/format.go | 13 +- cmd/air/get.go | 2 +- cmd/air/joblist.go | 26 +++- cmd/air/joblist_test.go | 111 +++++++++++++++++- cmd/air/list.go | 32 ++++- cmd/air/list_cache.go | 4 +- cmd/air/list_cache_test.go | 10 +- cmd/air/list_format.go | 11 +- cmd/air/list_index.go | 20 +++- cmd/air/list_index_test.go | 2 +- cmd/air/list_test.go | 59 ++++++++-- cmd/air/list_tui.go | 4 +- cmd/air/list_tui_render.go | 4 +- cmd/air/list_tui_test.go | 14 +++ cmd/air/logdownload.go | 25 +++- cmd/air/logdownload_test.go | 21 +++- cmd/air/logs.go | 2 +- cmd/air/render.go | 28 ++++- cmd/air/render_test.go | 22 +++- 27 files changed, 367 insertions(+), 80 deletions(-) diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md index 3ca69fcbe9f..b01f12d1b5a 100644 --- a/.nextchanges/air/multitask-air-tasks.md +++ b/.nextchanges/air/multitask-air-tasks.md @@ -1 +1 @@ -* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task; a job with several AI Runtime tasks lists one row per task (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs), each row and `air get --task-key` report that task's own status, `air get` gains `--task-key`, and both commands report `task_key` in JSON output. ([#7632](https://github.com/databricks/cli/pull/7632)) +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing, and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before. ([#7632](https://github.com/databricks/cli/pull/7632)) diff --git a/acceptance/air/get-multitask/output.txt b/acceptance/air/get-multitask/output.txt index f6e1bda3a50..8292b018c7a 100644 --- a/acceptance/air/get-multitask/output.txt +++ b/acceptance/air/get-multitask/output.txt @@ -40,7 +40,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 │ MLflow Run matmul-smoke │ │ User user@example.com │ │ Accelerators 8x H100 │ -│ Environment 5 │ +│ Environment 6 │ │ │ ╰────────────────────────────────────────────────────────────────╯ @@ -107,7 +107,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "user": "user@example.com", "accelerator_type": "GPU_8xH100", "num_accelerators": 8, - "environment_version": "5", + "environment_version": "6", "base_environment": null, "max_retries": { "limit": 2, diff --git a/acceptance/air/get-multitask/test.toml b/acceptance/air/get-multitask/test.toml index dc3d357dc1e..5d971327d24 100644 --- a/acceptance/air/get-multitask/test.toml +++ b/acceptance/air/get-multitask/test.toml @@ -22,7 +22,8 @@ Response.Body = ''' "end_time": 1700000020000, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task eval failed"}, "environments": [ - {"environment_key": "default", "spec": {"environment_version": "5"}} + {"environment_key": "default", "spec": {"environment_version": "5"}}, + {"environment_key": "eval_env", "spec": {"environment_version": "6"}} ], "tasks": [ { @@ -60,7 +61,7 @@ Response.Body = ''' "end_time": 1700000020000, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "eval: loss is NaN"}, "depends_on": [{"task_key": "train"}], - "environment_key": "default", + "environment_key": "eval_env", "ai_runtime_task": { "experiment": "/Users/user@example.com/eval-exp", "deployments": [ diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt index 3651c47ad0a..72512ee9cbc 100644 --- a/acceptance/air/list-index-fallback/output.txt +++ b/acceptance/air/list-index-fallback/output.txt @@ -14,7 +14,7 @@ Warn: air list: the AI Runtime run index is unavailable (failed to list training "runs": [ { "run_id": "334747067049496", - "task_key": "", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 5270a44ce52..55f989531a1 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -16,7 +16,7 @@ "runs": [ { "run_id": "334747067049496", - "task_key": "", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", @@ -62,7 +62,7 @@ "runs": [ { "run_id": "334747067049496", - "task_key": "", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/test.toml b/acceptance/air/list/test.toml index 9ae1417ce39..87598660582 100644 --- a/acceptance/air/list/test.toml +++ b/acceptance/air/list/test.toml @@ -10,10 +10,12 @@ Response.Body = '' # `air list` reads Jobs runs/list directly (expand_tasks), filtering to AIR runs. # The current runs carry an ai_runtime_task (not modeled by the typed SDK), which # is why the CLI parses the response raw. Both runs belong to the default user -# (tester@databricks.com, from the built-in scim/v2/Me handler). The second run is -# the official multi-task DAB shape, a notebook prep task before the -# ai_runtime_task, and must be listed from its AIR task. The third run has no AI -# task and must be filtered out. +# (tester@databricks.com, from the built-in scim/v2/Me handler). The second run +# has the shape of the official multi-task DAB (a notebook prep task before the +# ai_runtime_task) and must be listed from its AIR task. The third run has no AI +# task and must be filtered out. This stub ignores the query: the real scan asks +# for run_type=SUBMIT_RUN, so a bundle-deployed job, which runs as a JOB_RUN, is +# reached only through the run index (`--all-status`), not this scan. [[Server]] Pattern = "GET /api/2.2/jobs/runs/list" Response.Body = ''' diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index 6095bf9cd8e..276bf82da2b 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -229,8 +229,8 @@ func cancelPreviewRows(runs [][]listRow) ([]listRow, map[string][]string) { for _, run := range runs { rows = append(rows, run[0]) for _, r := range run { - if r.TaskKey != "" { - taskKeys[r.RunID] = append(taskKeys[r.RunID], r.TaskKey) + if k := r.taskKey(); k != "" { + taskKeys[r.RunID] = append(taskKeys[r.RunID], k) } } } diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index f6db226803b..b0989437448 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -342,12 +342,12 @@ func TestDisplayCancelPreview(t *testing.T) { func TestCancelPreviewRows(t *testing.T) { runs := [][]listRow{ - {{RunID: "1", TaskKey: "train"}, {RunID: "1", TaskKey: "eval"}}, + {{RunID: "1", TaskKey: new("train")}, {RunID: "1", TaskKey: new("eval")}}, {{RunID: "2"}}, - {{RunID: "3", TaskKey: "sweep"}}, + {{RunID: "3", TaskKey: new("sweep")}}, } rows, taskKeys := cancelPreviewRows(runs) - assert.Equal(t, []listRow{{RunID: "1", TaskKey: "train"}, {RunID: "2"}, {RunID: "3", TaskKey: "sweep"}}, rows) + assert.Equal(t, []listRow{{RunID: "1", TaskKey: new("train")}, {RunID: "2"}, {RunID: "3", TaskKey: new("sweep")}}, rows) assert.Equal(t, map[string][]string{"1": {"train", "eval"}, "3": {"sweep"}}, taskKeys) } diff --git a/cmd/air/format.go b/cmd/air/format.go index d43fec0e0f7..89d717c3aeb 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -113,15 +113,16 @@ func displayRunStatus(run *jobs.Run) string { return status } -// runWaitingForCompute reports whether a run has not started training yet: it -// is pending itself, or running while its lead task waits. A run narrowed to one -// task (see taskRun) carries that task's state, so the task's waiting states -// count at run level too. +// runWaitingForCompute reports whether a run is waiting for accelerator +// compute: it is pending itself, or running while its lead task waits to start. +// A run that is queued, blocked on an upstream task or waiting to retry at run +// level reports that state instead; for a task of a multi-task run, which +// carries the task's own state (see taskRun), that is the task's. func runWaitingForCompute(run *jobs.Run) bool { if run.State == nil || run.State.ResultState != "" { return false } - if isWaitingState(run.State.LifeCycleState) { + if run.State.LifeCycleState == jobs.RunLifeCycleStatePending { return true } task := leadTask(run) @@ -131,7 +132,7 @@ func runWaitingForCompute(run *jobs.Run) bool { return isWaitingState(task.State.LifeCycleState) } -// isWaitingState reports whether a run or task lifecycle state means it has not +// isWaitingState reports whether a task lifecycle state means it has not // started running yet. func isWaitingState(s jobs.RunLifeCycleState) bool { switch s { diff --git a/cmd/air/get.go b/cmd/air/get.go index 533f90d39bb..4116b3a03d6 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -449,7 +449,7 @@ func enrichRunDetails(ctx context.Context, w *databricks.WorkspaceClient, run *j raw []byte mlflowName string ) - wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId) }) + wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId, taskEnvironmentKey(run)) }) if readConfig { wg.Go(func() { body, raw = resolveConfigYAML(ctx, w, run, data) }) } diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 3c743751f7f..6d14b8f7dfa 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -57,22 +57,29 @@ func isAirRun(r *jobs.Run) bool { // taskRun returns a copy of r narrowed to the task runs (every attempt, in // order) of the task with the given key, one of airTaskKeys(r), so the display -// helpers, which read a run's tasks, report that task alone. Its state is that -// task's (see taskState). +// helpers, which read a run's tasks, report that task alone. When the run has +// other tasks too, the copy also takes that task's state (see taskState) and +// drops the run's own start and end times, so a task that hasn't started (or +// was skipped) reports no timing rather than the whole run's. A run whose only +// task is this one keeps its run-level state and timing, as before. func taskRun(r *jobs.Run, key string) *jobs.Run { narrowed := *r narrowed.Tasks = nil + others := false for i := range r.Tasks { if r.Tasks[i].TaskKey == key { narrowed.Tasks = append(narrowed.Tasks, r.Tasks[i]) + } else { + others = true } } - if n := len(narrowed.Tasks); n > 0 { + if n := len(narrowed.Tasks); n > 0 && others { latest := &narrowed.Tasks[n-1] if state := taskState(r, latest); state != r.State { narrowed.State = state narrowed.Status = latest.Status } + narrowed.StartTime, narrowed.EndTime = 0, 0 } return &narrowed } @@ -92,19 +99,28 @@ func defaultTaskRun(r *jobs.Run) *jobs.Run { // because its prep task failed). It keeps the run's state when the task has // none, and while the run is still active after a failed attempt, since the // task may be waiting to retry and runs/get doesn't reliably report how many -// retries are left. +// retries are left. A skipped task never retries, so it keeps its own state. func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { s := latest.State if s == nil { return r.State } - failed := isTerminalState(s) && s.ResultState != jobs.RunResultStateSuccess + failed := isTerminalState(s) && s.ResultState != jobs.RunResultStateSuccess && !isSkippedState(s) if failed && !isTerminal(r) { return r.State } return s } +// isSkippedState reports whether a task was skipped without running: an +// upstream task failed or was cancelled, or a condition excluded it. +func isSkippedState(s *jobs.RunState) bool { + skipped := s.ResultState == jobs.RunResultStateUpstreamFailed || + s.ResultState == jobs.RunResultStateUpstreamCanceled || + s.ResultState == jobs.RunResultStateExcluded + return skipped || s.LifeCycleState == jobs.RunLifeCycleStateSkipped +} + // leadTask returns the task run a run reports: Tasks[0], which is the AI // runtime task once the run is narrowed (see taskRun). A run without an AI // runtime task is not narrowed, so its first task stands in, as it always has; diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 000503d0114..13ca0febb24 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -338,15 +338,55 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { want: "RUNNING", }, { - name: "a queued task in a running run is waiting for compute", + name: "a pending task in a running run is waiting for compute", runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, tasks: []jobs.RunTask{ finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), - finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStatePending, ""), }, key: "train", want: "PENDING", }, + { + name: "a queued task is reported as queued", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + }, + key: "train", + want: "QUEUED", + }, + { + name: "a task blocked on its upstream task is reported as blocked", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateRunning, ""), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateBlocked, ""), + }, + key: "train", + want: "BLOCKED", + }, + { + name: "a skipped task keeps its state while another branch still runs", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateRunning, ""), + finished(notebookTask("prep_eval", 12), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(aiRuntimeTask("eval", 13, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed), + }, + key: "eval", + want: "UPSTREAM_FAILED", + }, + { + name: "the run's only task keeps the run's state", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateTimedout}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + }, + key: "train", + want: "TIMEDOUT", + }, { name: "a task without a state falls back to the run", runState: failedRun, @@ -366,3 +406,70 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { }) } } + +func TestTaskRunOfTheOnlyTaskKeepsTheRunsStatus(t *testing.T) { + // A single-task `air run` submission: the run's termination message carries + // the "Task failed with message:" prefix, which must survive. + train := finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed) + train.Status = &jobs.RunStatus{TerminationDetails: &jobs.TerminationDetails{Message: "OOM"}} + run := &jobs.Run{ + RunId: 1, + StartTime: 100, + EndTime: 300, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed}, + Status: &jobs.RunStatus{TerminationDetails: &jobs.TerminationDetails{Message: "Task train failed with message: OOM"}}, + Tasks: []jobs.RunTask{train}, + } + narrowed := taskRun(run, "train") + require.NotNil(t, terminationReason(narrowed)) + assert.Equal(t, "Task train failed with message: OOM", *terminationReason(narrowed)) + assert.Equal(t, int64(100), narrowed.StartTime, "the run's timing stands in for a task without its own") +} + +func TestTaskRunOfASkippedTaskHasNoTiming(t *testing.T) { + train := finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed) + train.StartTime, train.EndTime = 200_000, 260_000 + eval := finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed) + run := &jobs.Run{ + RunId: 1, + StartTime: 100_000, + EndTime: 300_000, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed}, + Tasks: []jobs.RunTask{train, eval}, + } + + skipped := taskRun(run, "eval") + assert.Nil(t, startedAt(skipped), "a task that never started reports no start, not the run's") + assert.Nil(t, durationSeconds(skipped)) + + ran := taskRun(run, "train") + require.NotNil(t, durationSeconds(ran)) + assert.Equal(t, int64(60), *durationSeconds(ran), "train reports its own 60s, not the run's 200s") +} + +func TestRunLevelWaitingStatesAreReportedAsSuch(t *testing.T) { + for _, life := range []jobs.RunLifeCycleState{ + jobs.RunLifeCycleStateQueued, jobs.RunLifeCycleStateBlocked, jobs.RunLifeCycleStateWaitingForRetry, + } { + t.Run(string(life), func(t *testing.T) { + run := &jobs.Run{RunId: 1, State: &jobs.RunState{LifeCycleState: life}, Tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1)}} + assert.False(t, runWaitingForCompute(run)) + assert.Equal(t, string(life), displayRunStatus(run)) + assert.NotEqual(t, waitingForComputeStatus, detailedDisplayRunStatus(run, "")) + }) + } + + pending := &jobs.Run{RunId: 1, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStatePending}} + assert.Equal(t, waitingForComputeStatus, detailedDisplayRunStatus(pending, "")) + + // The notebook-first DAB shape: train waits on prep, not on compute. + blocked := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateRunning, ""), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateBlocked, ""), + }, + } + assert.Equal(t, "BLOCKED", detailedDisplayRunStatus(taskRun(blocked, "train"), "")) +} diff --git a/cmd/air/list.go b/cmd/air/list.go index a3def34937d..4e84fc12c3c 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -45,9 +45,12 @@ type listData struct { // human-readable table. type listRow struct { RunID string `json:"run_id"` - // TaskKey names the AI runtime task the row shows. A job with several AI - // runtime tasks has one row per task, all with the same RunID. - TaskKey string `json:"task_key"` + // TaskKey names the AI runtime task the row shows, or is null for a run + // without one (as in `air get`). A job with several AI runtime tasks has one + // row per task, all with the same RunID; ShowTask marks those rows so the + // table names the task. + TaskKey *string `json:"task_key"` + ShowTask bool `json:"-"` RunName string `json:"run_name"` User string `json:"user"` Status string `json:"status"` @@ -73,6 +76,25 @@ type listRow struct { Accelerators string `json:"-"` } +// taskKey returns the AI runtime task key the row shows, or "" for a run +// without one. +func (r listRow) taskKey() string { + if r.TaskKey == nil { + return "" + } + return *r.TaskKey +} + +// runIDLabel is the Run ID cell: the run ID, followed by the task key when the +// run has several AI runtime tasks, so each of its rows names the task to pass +// to `air get --task-key`. +func (r listRow) runIDLabel() string { + if r.ShowTask && r.TaskKey != nil { + return r.RunID + " " + *r.TaskKey + } + return r.RunID +} + // listedRun pairs a row with its parent run id and task runs, so the MLflow link // can be fetched after the run has been filtered in and cached afterward. type listedRun struct { @@ -326,7 +348,9 @@ func (f *runFetcher) next(want int) ([]listRow, error) { if err != nil { return nil, err } - var rows []listRow + // Start from an empty slice so a list with no matching runs prints + // "runs": [] rather than null. + rows := make([]listRow, 0, len(groups)) for _, g := range groups { rows = append(rows, g...) } diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index 10581323043..bc72a5f92ed 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -63,7 +63,7 @@ type cachedRun struct { func (c cachedRun) toRow() listRow { return listRow{ - RunID: c.RunID, TaskKey: c.TaskKey, RunName: c.RunName, User: c.User, Status: c.Status, + RunID: c.RunID, TaskKey: nilIfEmpty(c.TaskKey), RunName: c.RunName, User: c.User, Status: c.Status, StartedAt: c.StartedAt, IsSweep: c.IsSweep, ExperimentName: c.ExperimentName, DurationSeconds: c.DurationSeconds, AcceleratorType: c.AcceleratorType, NumAccelerators: c.NumAccelerators, DashboardURL: c.DashboardURL, Experiment: c.Experiment, Duration: c.Duration, MLflowURL: c.MLflowURL, MLflowLabel: c.MLflowLabel, @@ -73,7 +73,7 @@ func (c cachedRun) toRow() listRow { func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun { return cachedRun{ - RunID: r.RunID, TaskKey: r.TaskKey, RunName: r.RunName, User: r.User, Status: r.Status, + RunID: r.RunID, TaskKey: r.taskKey(), RunName: r.RunName, User: r.User, Status: r.Status, StartedAt: r.StartedAt, IsSweep: r.IsSweep, ExperimentName: r.ExperimentName, DurationSeconds: r.DurationSeconds, AcceleratorType: r.AcceleratorType, NumAccelerators: r.NumAccelerators, DashboardURL: r.DashboardURL, Experiment: r.Experiment, Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel, diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 243e2722012..c4a543b90d4 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -35,7 +35,7 @@ func TestListCacheRoundTrip(t *testing.T) { require.False(t, ok, "miss before write") row := listRow{ - RunID: "42", TaskKey: "train", Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), + RunID: "42", TaskKey: new("train"), Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), AcceleratorType: new("GPU_1xA10"), NumAccelerators: new(1), DashboardURL: "https://host.test/jobs/runs/42?o=1", Experiment: "exp", } fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1} @@ -76,13 +76,13 @@ func TestListCacheKeepsOneRowPerAirTask(t *testing.T) { host := "https://host.test" putEntries(ctx, c, host, 42, []taskEntry{ - {entry: listedRun{row: listRow{RunID: "42", TaskKey: "train", Experiment: "exp-a"}, tasks: []jobs.RunTask{{RunId: 420, TaskKey: "train"}}}, fields: filterFields{Experiment: "exp-a"}}, - {entry: listedRun{row: listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b"}, tasks: []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}}, fields: filterFields{Experiment: "exp-b"}}, + {entry: listedRun{row: listRow{RunID: "42", TaskKey: new("train"), Experiment: "exp-a"}, tasks: []jobs.RunTask{{RunId: 420, TaskKey: "train"}}}, fields: filterFields{Experiment: "exp-a"}}, + {entry: listedRun{row: listRow{RunID: "42", TaskKey: new("eval"), Experiment: "exp-b"}, tasks: []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}}, fields: filterFields{Experiment: "exp-b"}}, }, 1000) // The MLflow link of one task updates that row alone. - updateCachedRow(ctx, c, host, 42, "eval", listRow{RunID: "42", TaskKey: "eval", Experiment: "exp-b", MLflowURL: "https://host.test/ml/x"}) - updateCachedRow(ctx, c, host, 42, "missing", listRow{RunID: "42", TaskKey: "missing"}) + updateCachedRow(ctx, c, host, 42, "eval", listRow{RunID: "42", TaskKey: new("eval"), Experiment: "exp-b", MLflowURL: "https://host.test/ml/x"}) + updateCachedRow(ctx, c, host, 42, "missing", listRow{RunID: "42", TaskKey: new("missing")}) got := rawCached(t, ctx, c, host, 42).Rows require.Len(t, got, 2) diff --git a/cmd/air/list_format.go b/cmd/air/list_format.go index aff014f110f..ba2ca349a70 100644 --- a/cmd/air/list_format.go +++ b/cmd/air/list_format.go @@ -42,18 +42,21 @@ func buildTaskEntries(run *jobs.Run, host string, workspaceID int64) []taskEntry func taskEntryFor(run *jobs.Run, host string, workspaceID int64) taskEntry { row := buildListRow(run, host, workspaceID) return taskEntry{ - entry: listedRun{row: row, runID: run.RunId, taskKey: row.TaskKey, tasks: retryTasks(run), terminal: isTerminal(run)}, + entry: listedRun{row: row, runID: run.RunId, taskKey: airTaskKey(run), tasks: retryTasks(run), terminal: isTerminal(run)}, fields: filterFieldsFromRun(run), } } // matchingGroup returns the run's entries that pass f, each filtered on its own -// task, or ok false when none do. +// task, or ok false when none do. When the run has several AI runtime tasks, +// every row is marked to show its task, even if the filter leaves one. func matchingGroup(runID int64, all []taskEntry, f listFilters) (runGroup, bool) { g := runGroup{runID: runID} for _, e := range all { if f.matchesFields(e.fields) { - g.entries = append(g.entries, e.entry) + entry := e.entry + entry.row.ShowTask = len(all) > 1 + g.entries = append(g.entries, entry) } } return g, len(g.entries) > 0 @@ -102,7 +105,7 @@ func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { return listRow{ RunID: strconv.FormatInt(run.RunId, 10), - TaskKey: taskKey, + TaskKey: nilIfEmpty(taskKey), RunName: run.RunName, User: run.CreatorUserName, Status: displayRunStatus(run), diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index 70a6b56409b..7704c9a5344 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -131,11 +131,11 @@ func (s *indexStrategy) hydrate(ids []int64) ([]runGroup, error) { } } - // Concurrent hydration reorders runs, so re-sort the batch newest-first by - // each run's first row, keeping a run's rows together. The ISO start - // timestamp sorts lexicographically; a missing time ("") sorts last. + // Concurrent hydration reorders runs, so re-sort the batch newest-first, + // keeping a run's rows together. The ISO start timestamp sorts + // lexicographically; a missing time ("") sorts last. slices.SortStableFunc(groups, func(a, b runGroup) int { - return cmp.Compare(rowStartKey(b.entries[0].row), rowStartKey(a.entries[0].row)) + return cmp.Compare(groupStartKey(b), groupStartKey(a)) }) return groups, nil } @@ -150,6 +150,18 @@ func (s *indexStrategy) cacheMLflowLinks(entries []listedRun) { } } +// groupStartKey returns the start key of a run's first row that has started: a +// task that hasn't started or was skipped has none (see taskRun), and shouldn't +// sort its whole run last. +func groupStartKey(g runGroup) string { + for _, e := range g.entries { + if k := rowStartKey(e.row); k != "" { + return k + } + } + return "" +} + // rowStartKey returns a row's ISO start timestamp for ordering, or "" when the // run hasn't started (which sorts last under descending comparison). func rowStartKey(r listRow) string { diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index edb5a5969ef..078577ac3df 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -403,7 +403,7 @@ func TestIndexStrategyKeepsARunsRowsTogether(t *testing.T) { got := make([][]string, len(runs)) for i, run := range runs { for _, r := range run { - got[i] = append(got[i], r.RunID+"/"+r.TaskKey) + got[i] = append(got[i], r.RunID+"/"+r.taskKey()) } } assert.Equal(t, [][]string{{"8/"}, {"7/train", "7/eval"}}, got) diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index 448734f6608..3571a7ebc42 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -242,7 +242,7 @@ func TestBuildListRow(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"123","task_key":"","run_name":"run-123","user":"me@example.com","status":"SUCCESS", + "run_id":"123","task_key":null,"run_name":"run-123","user":"me@example.com","status":"SUCCESS", "started_at":"2023-11-14T22:13:20+00:00","is_sweep":false, "experiment_name":"exp","duration_seconds":12, "accelerator_type":"GPU_8xH100","num_accelerators":8, @@ -276,7 +276,7 @@ func TestBuildListRowWaitingForComputeStatus(t *testing.T) { assert.Equal(t, "PENDING", row.Status) assert.Equal(t, listRowStatus(row), row.Status) assert.JSONEq(t, `{ - "run_id":"123","task_key":"","run_name":"run-123","user":"me@example.com","status":"PENDING", + "run_id":"123","task_key":null,"run_name":"run-123","user":"me@example.com","status":"PENDING", "started_at":null,"is_sweep":false,"experiment_name":"exp","duration_seconds":null, "accelerator_type":"GPU_8xH100","num_accelerators":8, "dashboard_url":"https://example.test/jobs/runs/123?o=456" @@ -298,7 +298,7 @@ func TestBuildListRowMissingValues(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"7","task_key":"","run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, + "run_id":"7","task_key":null,"run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, "experiment_name":null,"duration_seconds":null,"accelerator_type":null,"num_accelerators":null, "dashboard_url":"https://example.test/jobs/runs/7" }`, string(body)) @@ -513,12 +513,12 @@ func TestBuildTaskEntriesOnePerAirTask(t *testing.T) { rows := buildTaskEntries(&run, "https://example.test", 0) require.Len(t, rows, 2) assert.Equal(t, "7", rows[0].entry.row.RunID) - assert.Equal(t, "train", rows[0].entry.row.TaskKey) + assert.Equal(t, "train", rows[0].entry.row.taskKey()) assert.Equal(t, "exp-a", rows[0].entry.row.Experiment) assert.Equal(t, "1x A10", rows[0].entry.row.Accelerators) assert.Equal(t, []int64{72}, taskRunIDs(rows[0].entry.tasks), "the row's MLflow lookup sees only its own task runs") assert.Equal(t, "7", rows[1].entry.row.RunID) - assert.Equal(t, "eval", rows[1].entry.row.TaskKey) + assert.Equal(t, "eval", rows[1].entry.row.taskKey()) assert.Equal(t, "exp-b", rows[1].entry.row.Experiment) assert.Equal(t, "8x H100", rows[1].entry.row.Accelerators) assert.Equal(t, []int64{73}, taskRunIDs(rows[1].entry.tasks)) @@ -544,7 +544,7 @@ func TestListAirRunsAirTaskAfterNotebook(t *testing.T) { require.NoError(t, err) require.Len(t, rows, 1) assert.Equal(t, "1", rows[0].RunID) - assert.Equal(t, "train", rows[0].TaskKey) + assert.Equal(t, "train", rows[0].taskKey()) assert.Equal(t, "air-byot-multitask-dag", rows[0].Experiment) assert.Equal(t, "1x A10", rows[0].Accelerators) } @@ -564,13 +564,13 @@ func TestListAirRunsOneRowPerAirTask(t *testing.T) { require.NoError(t, err) require.Len(t, rows, 1, "the task filter applies per AIR task") assert.Equal(t, "1", rows[0].RunID) - assert.Equal(t, "b", rows[0].TaskKey) + assert.Equal(t, "b", rows[0].taskKey()) rows, err = newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) require.NoError(t, err) require.Len(t, rows, 3) assert.Equal(t, []string{"1", "1", "2"}, []string{rows[0].RunID, rows[1].RunID, rows[2].RunID}) - assert.Equal(t, []string{"a", "b", ""}, []string{rows[0].TaskKey, rows[1].TaskKey, rows[2].TaskKey}) + assert.Equal(t, []string{"a", "b", ""}, []string{rows[0].taskKey(), rows[1].taskKey(), rows[2].taskKey()}) } func TestListAirRunsLimitCountsRunsNotRows(t *testing.T) { @@ -600,3 +600,46 @@ func TestListAirRunsLimitCountsRunsNotRows(t *testing.T) { assert.Equal(t, tt.want, ids, "limit %d", tt.limit) } } + +func TestListAirRunsMarksTheRowsOfAMultiTaskRun(t *testing.T) { + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + srv := runsServer(t, runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c"))) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 3) + assert.Equal(t, []bool{true, true, false}, []bool{rows[0].ShowTask, rows[1].ShowTask, rows[2].ShowTask}) + + // A filter that leaves one of the two tasks still names it. + rows, err = newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ + activeOnly: true, + filters: listFilters{Experiment: "exp-b"}, + }).next(10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.True(t, rows[0].ShowTask) + assert.Equal(t, "1 b", rows[0].runIDLabel()) +} + +func TestListJSONEmptyListIsAnArray(t *testing.T) { + srv := runsServer(t, runsListBody(t, "")) + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + b, err := json.Marshal(listData{Rows: rows}) + require.NoError(t, err) + assert.JSONEq(t, `{"runs": []}`, string(b)) +} + +func TestListRowTaskKeyIsNullWithoutAnAirTask(t *testing.T) { + notebookOnly := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{notebookTask("prep", 71)}} + b, err := json.Marshal(buildListRow(notebookOnly, "https://example.test", 0)) + require.NoError(t, err) + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + assert.Contains(t, got, "task_key") + assert.Nil(t, got["task_key"], "matches air get, which reports null") +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 4155bbaa20d..1faf358e23e 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -291,7 +291,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Run Details" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunDetail(runID, m.rows[m.cursor].TaskKey) + return m, m.fetchRunDetail(runID, m.rows[m.cursor].taskKey()) } case "L", "l": // Open the logs snapshot pane; the fetch fills it in. @@ -300,7 +300,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Logs Snapshot" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunLogs(runID, m.rows[m.cursor].TaskKey) + return m, m.fetchRunLogs(runID, m.rows[m.cursor].taskKey()) } } m.offset = m.clampedOffset() diff --git a/cmd/air/list_tui_render.go b/cmd/air/list_tui_render.go index 98b75347f1e..c145087d830 100644 --- a/cmd/air/list_tui_render.go +++ b/cmd/air/list_tui_render.go @@ -89,7 +89,7 @@ func computeListCols(rows []listRow) listCols { accel: len("Accelerators"), } for _, r := range rows { - c.runID = max(c.runID, lipgloss.Width(r.RunID)) + c.runID = max(c.runID, lipgloss.Width(r.runIDLabel())) c.experiment = min(columnCap, max(c.experiment, lipgloss.Width(r.Experiment))) c.status = max(c.status, lipgloss.Width("● "+listRowStatus(r))) c.started = max(c.started, lipgloss.Width(startedDisplay(r))) @@ -283,7 +283,7 @@ func (s listStyles) renderRow(cols listCols, r listRow, selected, links bool) st cell := func(col listColumn) string { switch col { case columnRunID: - return s.cell(base, r.RunID, cols.runID, fg(colRunID), false, runIDLink != "", runIDLink) + return s.cell(base, r.runIDLabel(), cols.runID, fg(colRunID), false, runIDLink != "", runIDLink) case columnExperiment: return s.cell(base, r.Experiment, cols.experiment, fg(colN11), false, experimentLink != "", experimentLink) case columnStatus: diff --git a/cmd/air/list_tui_test.go b/cmd/air/list_tui_test.go index 19806e1d1ec..1a727c657ab 100644 --- a/cmd/air/list_tui_test.go +++ b/cmd/air/list_tui_test.go @@ -812,3 +812,17 @@ func TestListModelEnterOpensMLflowLogsWhenColumnHidden(t *testing.T) { cmd() assert.Contains(t, stderr.String(), "https://h/ml/experiments/E/runs/04c41514fbb0/logs") } + +func TestRenderRowNamesTheTaskOfAMultiTaskRun(t *testing.T) { + r, _ := cmdio.NewRenderer(cmdio.MockDiscard(t.Context()), io.Discard) + styles := newListStyles(r) + train := listRow{RunID: "1", TaskKey: new("train"), ShowTask: true, Status: "SUCCESS", Duration: "-", Accelerators: "-"} + eval := listRow{RunID: "1", TaskKey: new("eval"), ShowTask: true, Status: "SUCCESS", Duration: "-", Accelerators: "-"} + single := listRow{RunID: "2", TaskKey: new("train"), Status: "SUCCESS", Duration: "-", Accelerators: "-"} + cols := computeListCols([]listRow{train, eval, single}) + + assert.Contains(t, styles.renderRow(cols, train, false, false), "1 train") + assert.Contains(t, styles.renderRow(cols, eval, false, false), "1 eval") + assert.NotContains(t, styles.renderRow(cols, single, false, false), "train", "a single-task run shows only its run ID") + assert.GreaterOrEqual(t, cols.runID, len("1 train"), "the column is sized for the task label") +} diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index 8ab5f244254..4d4080b961f 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -29,10 +29,10 @@ func validateNodeIndex(node, numNodes int) error { return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1) } -// resolveNodeCount returns how many nodes a run used, read from its first AI -// runtime task (see defaultTaskRun). -func resolveNodeCount(run *jobs.Run) (int, error) { - accelType, count := jobCompute(defaultTaskRun(run)) +// resolveNodeCount returns how many nodes the task whose logs are read for the +// requested retry used (see logTaskRun), so the node indices match the logs. +func resolveNodeCount(run *jobs.Run, requestedAttempt int) (int, error) { + accelType, count := jobCompute(logTaskRun(run, requestedAttempt)) if accelType == "" { return 0, fmt.Errorf("run %d has no AI runtime compute config", run.RunId) } @@ -55,12 +55,27 @@ func resolveNodeCount(run *jobs.Run) (int, error) { return count / perNode, nil } +// logTaskRun returns run narrowed to the AI runtime task whose logs `air logs` +// reads for the requested retry: the task run resolveLogAttempt picks. It falls +// back to the first AI runtime task (see defaultTaskRun) when that task run is +// not an AI runtime task or the retry doesn't resolve. +func logTaskRun(run *jobs.Run, requestedAttempt int) *jobs.Run { + if _, taskRunID, err := resolveLogAttempt(run, requestedAttempt); err == nil { + for i := range run.Tasks { + if t := &run.Tasks[i]; t.RunId == taskRunID && isAirTask(t) { + return taskRun(run, t.TaskKey) + } + } + } + return defaultTaskRun(run) +} + // downloadLogs writes each node's logs to /logs/node_.log and // prints a summary. An explicit --node downloads only that node; otherwise all of // them. Logs come from MLflow artifacts, since Bricklens only streams. The // returned bool is the run's outcome, so the exit code matches the streaming path. func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) { - numNodes, err := resolveNodeCount(run) + numNodes, err := resolveNodeCount(run, req.attempt) if err != nil { return false, err } diff --git a/cmd/air/logdownload_test.go b/cmd/air/logdownload_test.go index 63c307b2f58..391778e4375 100644 --- a/cmd/air/logdownload_test.go +++ b/cmd/air/logdownload_test.go @@ -48,23 +48,23 @@ func TestResolveNodeCount(t *testing.T) { {"GPU_8xB300", 16, 2}, } for _, tt := range tests { - n, err := resolveNodeCount(airRunWithCompute(tt.accelType, tt.count)) + n, err := resolveNodeCount(airRunWithCompute(tt.accelType, tt.count), -1) require.NoError(t, err) assert.Equal(t, tt.want, n) } // A run with no AI runtime compute errors. - _, err := resolveNodeCount(&jobs.Run{RunId: 1}) + _, err := resolveNodeCount(&jobs.Run{RunId: 1}, -1) require.Error(t, err) // A count that isn't a whole number of nodes can't be mapped to node indices, // so it errors rather than truncating to 0. - _, err = resolveNodeCount(airRunWithCompute("GPU_8xH100", 4)) + _, err = resolveNodeCount(airRunWithCompute("GPU_8xH100", 4), -1) require.Error(t, err) assert.Contains(t, err.Error(), "not a multiple of 8") // A zero count is reported as such, not as a missing config. - _, err = resolveNodeCount(airRunWithCompute("GPU_1xA10", 0)) + _, err = resolveNodeCount(airRunWithCompute("GPU_1xA10", 0), -1) require.Error(t, err) } @@ -514,3 +514,16 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) { assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)") assert.NotContains(t, stdout.String(), "No logs available") } + +func TestResolveNodeCountReadsTheTaskTheLogsComeFrom(t *testing.T) { + // A 1-node train before a 2-node eval: the logs come from eval, the last + // task, so the node count must too. + run := &jobs.Run{RunId: 1, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 2), + }} + n, err := resolveNodeCount(run, -1) + require.NoError(t, err) + assert.Equal(t, 2, n) + assert.NoError(t, validateNodeIndex(1, n), "--node 1 is a valid eval node") +} diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 2a6d83f2e4c..3b60caf15f6 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -160,7 +160,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { status := projectRunStatus(run) if req.nodeSet { - if numNodes, err := resolveNodeCount(run); err == nil { + if numNodes, err := resolveNodeCount(run, req.attempt); err == nil { if err := validateNodeIndex(req.node, numNodes); err != nil { return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) } diff --git a/cmd/air/render.go b/cmd/air/render.go index 70f66ec7b5f..915c14c6921 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -176,13 +176,13 @@ func genAIComputeTask(run *jobs.Run) *jobs.GenAiComputeTask { return t.GenAiComputeTask } -// aiRuntimeEnvironment returns the serverless environment an ai_runtime run used, -// read from the Jobs GetRun response's environments[] entry keyed by -// aiRuntimeEnvironmentKey: a version (e.g. "4") or, for Databricks AI runtimes, a -// base environment. The typed SDK Run has no environments field, so the value is +// aiRuntimeEnvironment returns the serverless environment an ai_runtime task +// used, read from the Jobs GetRun response's environments[] entry keyed by +// envKey (see taskEnvironmentKey): a version (e.g. "4") or, for Databricks AI +// runtimes, a base environment. The typed SDK Run has no environments field, so the value is // fetched with a raw request. Best-effort: returns empty strings (logged) on any // error or when the run declares no environment. -func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (version, baseEnvironment string) { +func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64, envKey string) (version, baseEnvironment string) { apiClient, err := client.New(w.Config) if err != nil { enrichmentWarnf(ctx, "air get: could not create client to read environment: %v", err) @@ -205,13 +205,29 @@ func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, ru return "", "" } for _, e := range resp.Environments { - if e.EnvironmentKey == aiRuntimeEnvironmentKey { + if e.EnvironmentKey == envKey { return e.Spec.EnvironmentVersion, e.Spec.BaseEnvironment } } return "", "" } +// taskEnvironmentKey returns the environment key of the run's lead task (the +// selected AI runtime task once the run is narrowed, see taskRun), unwrapping a +// foreach sweep to its iterated task. It falls back to aiRuntimeEnvironmentKey, +// the key `air run` submits, when the task names none. +func taskEnvironmentKey(run *jobs.Run) string { + if t := leadTask(run); t != nil { + if t.ForEachTask != nil && t.ForEachTask.Task.EnvironmentKey != "" { + return t.ForEachTask.Task.EnvironmentKey + } + if t.EnvironmentKey != "" { + return t.EnvironmentKey + } + } + return aiRuntimeEnvironmentKey +} + // environmentLabel is the text shown for the Environment cell: the environment // version, else the ID of a workspace base environment, or "" for neither. // buildRunEnvironment submits a databricks_ai_v* image as a base environment diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index db1af3bfb4b..a01967a2830 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -106,7 +106,7 @@ func TestAiRuntimeEnvironment(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { srv := runGetServer(t, `{"run_id": 5, "environments": [{"environment_key": "default", "spec": `+tt.spec+`}]}`) - version, base := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5) + version, base := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5, aiRuntimeEnvironmentKey) assert.Equal(t, tt.wantVersion, version) assert.Equal(t, tt.wantBase, base) assert.Equal(t, tt.wantLabel, environmentLabel(version, base)) @@ -322,3 +322,23 @@ func TestStatusStyleSelectors(t *testing.T) { assert.True(t, isFailedStatus("TIMEDOUT")) assert.False(t, isFailedStatus("RUNNING")) } + +func TestAiRuntimeEnvironmentUsesTheSelectedTasksKey(t *testing.T) { + srv := runGetServer(t, `{"run_id": 5, "environments": [ + {"environment_key": "default", "spec": {"environment_version": "4"}}, + {"environment_key": "eval_env", "spec": {"environment_version": "6"}} + ]}`) + train := aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1) + eval := aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1) + eval.EnvironmentKey = "eval_env" + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{train, eval}} + + assert.Equal(t, aiRuntimeEnvironmentKey, taskEnvironmentKey(taskRun(run, "train")), "a task without a key uses air run's") + assert.Equal(t, "eval_env", taskEnvironmentKey(taskRun(run, "eval"))) + + version, _ := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5, taskEnvironmentKey(taskRun(run, "eval"))) + assert.Equal(t, "6", version, "--task-key eval shows eval's environment, not default's") + + sweep := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{{TaskKey: "sweep", ForEachTask: &jobs.RunForEachTask{Task: jobs.Task{EnvironmentKey: "eval_env"}}}}} + assert.Equal(t, "eval_env", taskEnvironmentKey(sweep), "a sweep uses its iterated task's key") +} From ddfaf6c65665bebcb0a272e43160df68c16c6bb4 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:24:15 +0000 Subject: [PATCH 6/9] air: cover the multi-task review fixes in acceptance tests - acceptance/air/get-run-state: a single-task run that TIMEDOUT after its task succeeded reports the run's TIMEDOUT state and termination message. - acceptance/air/list: a run with two AI Runtime tasks lists as two rows that name their task and report their own status, and `air list -o json` with no matching runs prints "runs": []. - Drop the unit tests these and the existing get-multitask and cancel cases now cover, which called helpers directly. - selectAirTask starts from defaultTaskRun for an empty key, and taskRun drops a guard its callers make unnecessary. Co-authored-by: Isaac --- acceptance/air/get-run-state/out.test.toml | 1 + acceptance/air/get-run-state/output.txt | 35 +++++++++++++++ acceptance/air/get-run-state/script | 2 + acceptance/air/get-run-state/test.toml | 51 ++++++++++++++++++++++ acceptance/air/list/output.txt | 47 ++++++++++++++++++-- acceptance/air/list/script | 3 ++ acceptance/air/list/test.toml | 39 ++++++++++++++++- cmd/air/cancel_test.go | 11 ----- cmd/air/get.go | 9 ++-- cmd/air/get_test.go | 26 ----------- cmd/air/joblist.go | 4 +- cmd/air/joblist_test.go | 19 -------- cmd/air/list_test.go | 19 -------- cmd/air/list_tui_test.go | 14 ------ 14 files changed, 178 insertions(+), 102 deletions(-) create mode 100644 acceptance/air/get-run-state/out.test.toml create mode 100644 acceptance/air/get-run-state/output.txt create mode 100644 acceptance/air/get-run-state/script create mode 100644 acceptance/air/get-run-state/test.toml diff --git a/acceptance/air/get-run-state/out.test.toml b/acceptance/air/get-run-state/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-run-state/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-run-state/output.txt b/acceptance/air/get-run-state/output.txt new file mode 100644 index 00000000000..4769155138b --- /dev/null +++ b/acceptance/air/get-run-state/output.txt @@ -0,0 +1,35 @@ + +=== get (json): the run's TIMEDOUT state and message, not its task's SUCCESS + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "TIMEDOUT", + "started_at": "[TIMESTAMP]", + "duration_seconds": 3500, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "mlflow_experiment_id": null, + "mlflow_run_id": null, + "termination_reason": "Task train timed out after 3600 seconds", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null + } +} diff --git a/acceptance/air/get-run-state/script b/acceptance/air/get-run-state/script new file mode 100644 index 00000000000..83a3b597051 --- /dev/null +++ b/acceptance/air/get-run-state/script @@ -0,0 +1,2 @@ +title "get (json): the run's TIMEDOUT state and message, not its task's SUCCESS" +trace $CLI air get 123 -o json diff --git a/acceptance/air/get-run-state/test.toml b/acceptance/air/get-run-state/test.toml new file mode 100644 index 00000000000..d6fb7958a8e --- /dev/null +++ b/acceptance/air/get-run-state/test.toml @@ -0,0 +1,51 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A single-task `air run` submission whose run state differs from its only +# task's: the run hit its timeout after the task had already succeeded. A run +# whose only task is the AI Runtime task keeps its run-level state and +# termination message (with its "Task ..." prefix); only a task of a +# multi-task run reports its own (see get-multitask). +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700003600000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "TIMEDOUT", "state_message": "Run timed out"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "Task train timed out after 3600 seconds"}}, + "tasks": [ + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "start_time": 1700000000000, + "end_time": 1700003500000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "SUCCESS", "type": "SUCCESS", "message": ""}}, + "ai_runtime_task": { + "experiment": "my-exp", + "deployments": [ + { + "command_path": "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/command.sh", + "compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1} + } + ] + } + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{} +''' diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 55f989531a1..7e25bb5d7a1 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -2,9 +2,11 @@ === list (text) >>> [CLI] air list - Run ID Experiment Status Started Duration Progress MLflow User Accelerators - 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 - 334747067049500 air-byot-multitask-dag ● SUCCESS [TIMESTAMP] 20s - qwen-train-001 [USERNAME] 1x A10 + Run ID Experiment Status Started Duration Progress MLflow User Accelerators + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + 334747067049500 air-byot-multitask-dag ● SUCCESS [TIMESTAMP] 20s - qwen-train-001 [USERNAME] 1x A10 + 334747067049510 train train-then-eval ● SUCCESS [TIMESTAMP] 40s - qwen-train-001 [USERNAME] 1x A10 + 334747067049510 eval train-then-eval ● FAILED [TIMESTAMP] 15s - qwen-train-001 [USERNAME] 1x A10 === list (json) @@ -41,11 +43,50 @@ "accelerator_type": "GPU_1xA10", "num_accelerators": 1, "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049500?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049510", + "task_key": "train", + "run_name": "train-then-eval", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "train-then-eval", + "duration_seconds": 40, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049510?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049510", + "task_key": "eval", + "run_name": "train-then-eval", + "user": "[USERNAME]", + "status": "FAILED", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "train-then-eval", + "duration_seconds": 15, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049510?o=[WORKSPACE_ID]" } ] } } +=== list (json): no matching runs is an empty array + +>>> [CLI] air list --filter experiment=no-such-experiment -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "runs": [] + } +} + === list --all-status (text, via AiTrainingService index) >>> [CLI] air list --all-status diff --git a/acceptance/air/list/script b/acceptance/air/list/script index d22fca13ad6..49cca4b79dd 100644 --- a/acceptance/air/list/script +++ b/acceptance/air/list/script @@ -4,6 +4,9 @@ trace $CLI air list title "list (json)" trace $CLI air list -o json +title "list (json): no matching runs is an empty array" +trace $CLI air list --filter 'experiment=no-such-experiment' -o json + title "list --all-status (text, via AiTrainingService index)" trace $CLI air list --all-status diff --git a/acceptance/air/list/test.toml b/acceptance/air/list/test.toml index 87598660582..009458a44e4 100644 --- a/acceptance/air/list/test.toml +++ b/acceptance/air/list/test.toml @@ -12,8 +12,10 @@ Response.Body = '' # is why the CLI parses the response raw. Both runs belong to the default user # (tester@databricks.com, from the built-in scim/v2/Me handler). The second run # has the shape of the official multi-task DAB (a notebook prep task before the -# ai_runtime_task) and must be listed from its AIR task. The third run has no AI -# task and must be filtered out. This stub ignores the query: the real scan asks +# ai_runtime_task) and must be listed from its AIR task. The third run has two +# AI Runtime tasks that ended differently: it lists as two rows, each naming its +# task and reporting its own status. The fourth run has no AI task and must be +# filtered out. This stub ignores the query: the real scan asks # for run_type=SUBMIT_RUN, so a bundle-deployed job, which runs as a JOB_RUN, is # reached only through the run index (`--all-status`), not this scan. [[Server]] @@ -64,6 +66,39 @@ Response.Body = ''' } ] }, + { + "run_id": 334747067049510, + "run_name": "train-then-eval", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608600000, + "end_time": 1717608660000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [ + { + "task_key": "train", + "run_id": 334747067049511, + "start_time": 1717608600000, + "end_time": 1717608640000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "ai_runtime_task": { + "experiment": "train-then-eval", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + }, + { + "task_key": "eval", + "run_id": 334747067049512, + "start_time": 1717608645000, + "end_time": 1717608660000, + "depends_on": [{"task_key": "train"}], + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "ai_runtime_task": { + "experiment": "train-then-eval", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + } + ] + }, { "run_id": 999000999000, "run_name": "not-an-air-run", diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index b0989437448..4203451662f 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -340,17 +340,6 @@ func TestDisplayCancelPreview(t *testing.T) { assert.Contains(t, out, na) } -func TestCancelPreviewRows(t *testing.T) { - runs := [][]listRow{ - {{RunID: "1", TaskKey: new("train")}, {RunID: "1", TaskKey: new("eval")}}, - {{RunID: "2"}}, - {{RunID: "3", TaskKey: new("sweep")}}, - } - rows, taskKeys := cancelPreviewRows(runs) - assert.Equal(t, []listRow{{RunID: "1", TaskKey: new("train")}, {RunID: "2"}, {RunID: "3", TaskKey: new("sweep")}}, rows) - assert.Equal(t, map[string][]string{"1": {"train", "eval"}, "3": {"sweep"}}, taskKeys) -} - func TestCancelAllCancelsAMultiTaskRunOnce(t *testing.T) { // Run 111 has two AI Runtime tasks after a notebook, so air list shows it as // two rows; cancel --all must cancel it once and name both tasks. diff --git a/cmd/air/get.go b/cmd/air/get.go index 4116b3a03d6..89123da7cc6 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -321,16 +321,13 @@ one.`, // returned unchanged when no key is given. The error names the run's AI runtime // tasks when key is not one of them. func selectAirTask(run *jobs.Run, key string) (*jobs.Run, error) { + if key == "" { + return defaultTaskRun(run), nil + } keys := airTaskKeys(run) if len(keys) == 0 { - if key == "" { - return run, nil - } return nil, fmt.Errorf("run %d has no AI Runtime tasks", run.RunId) } - if key == "" { - key = keys[0] - } if !slices.Contains(keys, key) { return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) } diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index 0e57e2f6b96..5fd29635d2b 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -935,32 +935,6 @@ func TestGetRunSelectsAirTask(t *testing.T) { }) } -func TestTaskViewTaskKey(t *testing.T) { - run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}} - _, data, err := taskView(run, "") - require.NoError(t, err) - assert.Nil(t, data.TaskKey, "matches air list, which reports no task, and --task-key prep, which errors") - assert.Empty(t, data.TaskDisplay) - - single := &jobs.Run{RunId: 6, Tasks: []jobs.RunTask{notebookTask("prep", 60), aiRuntimeTask("train", 61, "e", "GPU_1xA10", 1)}} - _, data, err = taskView(single, "") - require.NoError(t, err) - require.NotNil(t, data.TaskKey) - assert.Equal(t, "train", *data.TaskKey) - assert.Empty(t, data.TaskDisplay, "one AI Runtime task needs no Task line") - - two := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ - aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), - aiRuntimeTask("eval", 72, "e", "GPU_1xA10", 1), - }} - selected, data, err := taskView(two, "eval") - require.NoError(t, err) - assert.Equal(t, "eval", selected.Tasks[0].TaskKey) - require.NotNil(t, data.TaskKey) - assert.Equal(t, "eval", *data.TaskKey) - assert.Equal(t, "eval", data.TaskDisplay) -} - func TestSelectAirTaskErrors(t *testing.T) { _, err := selectAirTask(&jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}}, "prep") assert.EqualError(t, err, "run 5 has no AI Runtime tasks") diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 6d14b8f7dfa..833f143d1a1 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -73,8 +73,8 @@ func taskRun(r *jobs.Run, key string) *jobs.Run { others = true } } - if n := len(narrowed.Tasks); n > 0 && others { - latest := &narrowed.Tasks[n-1] + if others { + latest := &narrowed.Tasks[len(narrowed.Tasks)-1] if state := taskState(r, latest); state != r.State { narrowed.State = state narrowed.Status = latest.Status diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 13ca0febb24..9ffa72049e8 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -407,25 +407,6 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { } } -func TestTaskRunOfTheOnlyTaskKeepsTheRunsStatus(t *testing.T) { - // A single-task `air run` submission: the run's termination message carries - // the "Task failed with message:" prefix, which must survive. - train := finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed) - train.Status = &jobs.RunStatus{TerminationDetails: &jobs.TerminationDetails{Message: "OOM"}} - run := &jobs.Run{ - RunId: 1, - StartTime: 100, - EndTime: 300, - State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed}, - Status: &jobs.RunStatus{TerminationDetails: &jobs.TerminationDetails{Message: "Task train failed with message: OOM"}}, - Tasks: []jobs.RunTask{train}, - } - narrowed := taskRun(run, "train") - require.NotNil(t, terminationReason(narrowed)) - assert.Equal(t, "Task train failed with message: OOM", *terminationReason(narrowed)) - assert.Equal(t, int64(100), narrowed.StartTime, "the run's timing stands in for a task without its own") -} - func TestTaskRunOfASkippedTaskHasNoTiming(t *testing.T) { train := finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed) train.StartTime, train.EndTime = 200_000, 260_000 diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index 3571a7ebc42..847d1368a92 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -624,22 +624,3 @@ func TestListAirRunsMarksTheRowsOfAMultiTaskRun(t *testing.T) { assert.True(t, rows[0].ShowTask) assert.Equal(t, "1 b", rows[0].runIDLabel()) } - -func TestListJSONEmptyListIsAnArray(t *testing.T) { - srv := runsServer(t, runsListBody(t, "")) - rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) - require.NoError(t, err) - b, err := json.Marshal(listData{Rows: rows}) - require.NoError(t, err) - assert.JSONEq(t, `{"runs": []}`, string(b)) -} - -func TestListRowTaskKeyIsNullWithoutAnAirTask(t *testing.T) { - notebookOnly := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{notebookTask("prep", 71)}} - b, err := json.Marshal(buildListRow(notebookOnly, "https://example.test", 0)) - require.NoError(t, err) - var got map[string]any - require.NoError(t, json.Unmarshal(b, &got)) - assert.Contains(t, got, "task_key") - assert.Nil(t, got["task_key"], "matches air get, which reports null") -} diff --git a/cmd/air/list_tui_test.go b/cmd/air/list_tui_test.go index 1a727c657ab..19806e1d1ec 100644 --- a/cmd/air/list_tui_test.go +++ b/cmd/air/list_tui_test.go @@ -812,17 +812,3 @@ func TestListModelEnterOpensMLflowLogsWhenColumnHidden(t *testing.T) { cmd() assert.Contains(t, stderr.String(), "https://h/ml/experiments/E/runs/04c41514fbb0/logs") } - -func TestRenderRowNamesTheTaskOfAMultiTaskRun(t *testing.T) { - r, _ := cmdio.NewRenderer(cmdio.MockDiscard(t.Context()), io.Discard) - styles := newListStyles(r) - train := listRow{RunID: "1", TaskKey: new("train"), ShowTask: true, Status: "SUCCESS", Duration: "-", Accelerators: "-"} - eval := listRow{RunID: "1", TaskKey: new("eval"), ShowTask: true, Status: "SUCCESS", Duration: "-", Accelerators: "-"} - single := listRow{RunID: "2", TaskKey: new("train"), Status: "SUCCESS", Duration: "-", Accelerators: "-"} - cols := computeListCols([]listRow{train, eval, single}) - - assert.Contains(t, styles.renderRow(cols, train, false, false), "1 train") - assert.Contains(t, styles.renderRow(cols, eval, false, false), "1 eval") - assert.NotContains(t, styles.renderRow(cols, single, false, false), "train", "a single-task run shows only its run ID") - assert.GreaterOrEqual(t, cols.runID, len("1 train"), "the column is sized for the task label") -} From e5def32e0686c1a252862e5ed09c801ccebe9cb8 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:10:34 +0000 Subject: [PATCH 7/9] air: resolve the log retry once and read the sweep environment key through firstTask Follow-ups from review: - downloadLogs resolves the retry once and passes that task run to resolveNodeCount; logTaskRun becomes a plain lookup by task run ID, and runLogs's --node check goes through nodeCountForRetry. - firstTask's synthetic foreach task carries EnvironmentKey, so taskEnvironmentKey reads firstTask(run).EnvironmentKey instead of unwrapping the sweep a third time. - acceptance/air/get-multitask-skipped: prep fails and train is skipped, so the golden shows UPSTREAM_FAILED and no start time or duration. - The run-level waiting-state test asserts the exact status. Co-authored-by: Isaac --- .../air/get-multitask-skipped/out.test.toml | 1 + .../air/get-multitask-skipped/output.txt | 56 +++++++++++++++++++ acceptance/air/get-multitask-skipped/script | 5 ++ .../air/get-multitask-skipped/test.toml | 53 ++++++++++++++++++ cmd/air/joblist.go | 13 +++-- cmd/air/joblist_test.go | 2 +- cmd/air/logdownload.go | 46 ++++++++------- cmd/air/logdownload_test.go | 10 ++-- cmd/air/logs.go | 2 +- cmd/air/render.go | 15 ++--- 10 files changed, 163 insertions(+), 40 deletions(-) create mode 100644 acceptance/air/get-multitask-skipped/out.test.toml create mode 100644 acceptance/air/get-multitask-skipped/output.txt create mode 100644 acceptance/air/get-multitask-skipped/script create mode 100644 acceptance/air/get-multitask-skipped/test.toml diff --git a/acceptance/air/get-multitask-skipped/out.test.toml b/acceptance/air/get-multitask-skipped/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-multitask-skipped/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-multitask-skipped/output.txt b/acceptance/air/get-multitask-skipped/output.txt new file mode 100644 index 00000000000..3d03bd68031 --- /dev/null +++ b/acceptance/air/get-multitask-skipped/output.txt @@ -0,0 +1,56 @@ + +=== get (text): the skipped AI Runtime task's own status, with no duration + +>>> [CLI] air get 123 + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Status ● UPSTREAM_FAILED │ +│ Submitted N/A │ +│ Retries 0 │ +│ Max Retries 0 │ +│ Duration N/A │ +│ Experiment air-byot-multitask-dag │ +│ MLflow Run N/A │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment N/A │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] + +=== get (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "UPSTREAM_FAILED", + "started_at": null, + "duration_seconds": null, + "attempt_number": 0, + "experiment_name": "air-byot-multitask-dag", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "mlflow_experiment_id": null, + "mlflow_run_id": null, + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null + } +} diff --git a/acceptance/air/get-multitask-skipped/script b/acceptance/air/get-multitask-skipped/script new file mode 100644 index 00000000000..d1ec551de0a --- /dev/null +++ b/acceptance/air/get-multitask-skipped/script @@ -0,0 +1,5 @@ +title "get (text): the skipped AI Runtime task's own status, with no duration" +trace $CLI air get 123 + +title "get (json)" +trace $CLI air get 123 -o json diff --git a/acceptance/air/get-multitask-skipped/test.toml b/acceptance/air/get-multitask-skipped/test.toml new file mode 100644 index 00000000000..869255d9464 --- /dev/null +++ b/acceptance/air/get-multitask-skipped/test.toml @@ -0,0 +1,53 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# The notebook-first DAB shape where the prep notebook fails, so the +# ai_runtime_task that depends on it is skipped (UPSTREAM_FAILED) and never +# starts. `air get` must report the task's own UPSTREAM_FAILED, not the run's +# FAILED, and no start time or duration rather than the run's. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700000005000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task prep failed"}, + "tasks": [ + { + "task_key": "prep", + "run_id": 455, + "attempt_number": 0, + "start_time": 1700000000000, + "end_time": 1700000005000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "prep: table not found"}, + "notebook_task": {"notebook_path": "/Workspace/Users/user@example.com/prep"} + }, + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "state": {"life_cycle_state": "SKIPPED", "result_state": "UPSTREAM_FAILED", "state_message": "Upstream task(s) failed."}, + "depends_on": [{"task_key": "prep"}], + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [ + {"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}} + ] + } + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{} +''' diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 833f143d1a1..21020c60fc5 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -61,7 +61,7 @@ func isAirRun(r *jobs.Run) bool { // other tasks too, the copy also takes that task's state (see taskState) and // drops the run's own start and end times, so a task that hasn't started (or // was skipped) reports no timing rather than the whole run's. A run whose only -// task is this one keeps its run-level state and timing, as before. +// task is this one keeps its run-level state and timing. func taskRun(r *jobs.Run, key string) *jobs.Run { narrowed := *r narrowed.Tasks = nil @@ -139,10 +139,15 @@ func firstTask(r *jobs.Run) *jobs.RunTask { if t == nil || t.ForEachTask == nil { return t } - // The foreach inner task is a jobs.Task; expose the AI fields via a - // synthetic RunTask so callers can treat both shapes uniformly. + // The foreach inner task is a jobs.Task; expose the fields callers read via + // a synthetic RunTask so they can treat both shapes uniformly. ai, genAI := taskSpec(t) - return &jobs.RunTask{TaskKey: t.TaskKey, AiRuntimeTask: ai, GenAiComputeTask: genAI} + return &jobs.RunTask{ + TaskKey: t.TaskKey, + AiRuntimeTask: ai, + GenAiComputeTask: genAI, + EnvironmentKey: t.ForEachTask.Task.EnvironmentKey, + } } // airTaskKey returns the key of the AI runtime task a narrowed run shows, or "" diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 9ffa72049e8..7565639438f 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -436,7 +436,7 @@ func TestRunLevelWaitingStatesAreReportedAsSuch(t *testing.T) { run := &jobs.Run{RunId: 1, State: &jobs.RunState{LifeCycleState: life}, Tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1)}} assert.False(t, runWaitingForCompute(run)) assert.Equal(t, string(life), displayRunStatus(run)) - assert.NotEqual(t, waitingForComputeStatus, detailedDisplayRunStatus(run, "")) + assert.Equal(t, string(life), detailedDisplayRunStatus(run, "")) }) } diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index 4d4080b961f..d774deda235 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -29,10 +29,10 @@ func validateNodeIndex(node, numNodes int) error { return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1) } -// resolveNodeCount returns how many nodes the task whose logs are read for the -// requested retry used (see logTaskRun), so the node indices match the logs. -func resolveNodeCount(run *jobs.Run, requestedAttempt int) (int, error) { - accelType, count := jobCompute(logTaskRun(run, requestedAttempt)) +// resolveNodeCount returns how many nodes a task run used: the task the logs +// come from (see logTaskRun), so the node indices match the logs. +func resolveNodeCount(run *jobs.Run) (int, error) { + accelType, count := jobCompute(run) if accelType == "" { return 0, fmt.Errorf("run %d has no AI runtime compute config", run.RunId) } @@ -55,27 +55,39 @@ func resolveNodeCount(run *jobs.Run, requestedAttempt int) (int, error) { return count / perNode, nil } -// logTaskRun returns run narrowed to the AI runtime task whose logs `air logs` -// reads for the requested retry: the task run resolveLogAttempt picks. It falls -// back to the first AI runtime task (see defaultTaskRun) when that task run is -// not an AI runtime task or the retry doesn't resolve. -func logTaskRun(run *jobs.Run, requestedAttempt int) *jobs.Run { - if _, taskRunID, err := resolveLogAttempt(run, requestedAttempt); err == nil { - for i := range run.Tasks { - if t := &run.Tasks[i]; t.RunId == taskRunID && isAirTask(t) { - return taskRun(run, t.TaskKey) - } +// logTaskRun returns run narrowed to the AI runtime task whose task run is +// taskRunID, the one resolveLogAttempt picked for the logs. It falls back to the +// first AI runtime task (see defaultTaskRun) when that task run is not an AI +// runtime task. +func logTaskRun(run *jobs.Run, taskRunID int64) *jobs.Run { + for i := range run.Tasks { + if t := &run.Tasks[i]; t.RunId == taskRunID && isAirTask(t) { + return taskRun(run, t.TaskKey) } } return defaultTaskRun(run) } +// nodeCountForRetry returns how many nodes the task whose logs are read for the +// requested retry used. +func nodeCountForRetry(run *jobs.Run, requestedAttempt int) (int, error) { + _, taskRunID, err := resolveLogAttempt(run, requestedAttempt) + if err != nil { + return 0, err + } + return resolveNodeCount(logTaskRun(run, taskRunID)) +} + // downloadLogs writes each node's logs to /logs/node_.log and // prints a summary. An explicit --node downloads only that node; otherwise all of // them. Logs come from MLflow artifacts, since Bricklens only streams. The // returned bool is the run's outcome, so the exit code matches the streaming path. func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) { - numNodes, err := resolveNodeCount(run, req.attempt) + attempt, taskRunID, err := resolveLogAttempt(run, req.attempt) + if err != nil { + return false, err + } + numNodes, err := resolveNodeCount(logTaskRun(run, taskRunID)) if err != nil { return false, err } @@ -91,10 +103,6 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri // A run with no logs is reported the same way as on the streaming path, so // the message and exit code agree between them. - attempt, taskRunID, err := resolveLogAttempt(run, req.attempt) - if err != nil { - return false, err - } ids := mlflowIDsForTask(ctx, w, taskRunID) if ids == nil || ids.RunID == "" { emitNoLogs(out, req, status) diff --git a/cmd/air/logdownload_test.go b/cmd/air/logdownload_test.go index 391778e4375..c1464f2b6e3 100644 --- a/cmd/air/logdownload_test.go +++ b/cmd/air/logdownload_test.go @@ -48,23 +48,23 @@ func TestResolveNodeCount(t *testing.T) { {"GPU_8xB300", 16, 2}, } for _, tt := range tests { - n, err := resolveNodeCount(airRunWithCompute(tt.accelType, tt.count), -1) + n, err := resolveNodeCount(airRunWithCompute(tt.accelType, tt.count)) require.NoError(t, err) assert.Equal(t, tt.want, n) } // A run with no AI runtime compute errors. - _, err := resolveNodeCount(&jobs.Run{RunId: 1}, -1) + _, err := resolveNodeCount(&jobs.Run{RunId: 1}) require.Error(t, err) // A count that isn't a whole number of nodes can't be mapped to node indices, // so it errors rather than truncating to 0. - _, err = resolveNodeCount(airRunWithCompute("GPU_8xH100", 4), -1) + _, err = resolveNodeCount(airRunWithCompute("GPU_8xH100", 4)) require.Error(t, err) assert.Contains(t, err.Error(), "not a multiple of 8") // A zero count is reported as such, not as a missing config. - _, err = resolveNodeCount(airRunWithCompute("GPU_1xA10", 0), -1) + _, err = resolveNodeCount(airRunWithCompute("GPU_1xA10", 0)) require.Error(t, err) } @@ -522,7 +522,7 @@ func TestResolveNodeCountReadsTheTaskTheLogsComeFrom(t *testing.T) { aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 2), }} - n, err := resolveNodeCount(run, -1) + n, err := nodeCountForRetry(run, -1) require.NoError(t, err) assert.Equal(t, 2, n) assert.NoError(t, validateNodeIndex(1, n), "--node 1 is a valid eval node") diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 3b60caf15f6..72d4ab62df5 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -160,7 +160,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { status := projectRunStatus(run) if req.nodeSet { - if numNodes, err := resolveNodeCount(run, req.attempt); err == nil { + if numNodes, err := nodeCountForRetry(run, req.attempt); err == nil { if err := validateNodeIndex(req.node, numNodes); err != nil { return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) } diff --git a/cmd/air/render.go b/cmd/air/render.go index 915c14c6921..3adf063489a 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -213,17 +213,12 @@ func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, ru } // taskEnvironmentKey returns the environment key of the run's lead task (the -// selected AI runtime task once the run is narrowed, see taskRun), unwrapping a -// foreach sweep to its iterated task. It falls back to aiRuntimeEnvironmentKey, -// the key `air run` submits, when the task names none. +// selected AI runtime task once the run is narrowed, see taskRun; a sweep's +// iterated task, see firstTask). It falls back to aiRuntimeEnvironmentKey, the +// key `air run` submits, when the task names none. func taskEnvironmentKey(run *jobs.Run) string { - if t := leadTask(run); t != nil { - if t.ForEachTask != nil && t.ForEachTask.Task.EnvironmentKey != "" { - return t.ForEachTask.Task.EnvironmentKey - } - if t.EnvironmentKey != "" { - return t.EnvironmentKey - } + if t := firstTask(run); t != nil && t.EnvironmentKey != "" { + return t.EnvironmentKey } return aiRuntimeEnvironmentKey } From 309a022f278b5e2db0d0c1c1950cb76b630d8946 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:59:27 +0000 Subject: [PATCH 8/9] air: keep index order for runs with an unstarted task and report a queued task as pending Review follow-ups: - The index path keeps a batch in the index's order (newest submission first) instead of re-sorting by row start time, which a task that hasn't started or was skipped no longer has. Such a run now lists where main lists it, fresh or cached. - A task of a multi-task run that is pending, queued or waiting to retry reports PENDING with the waiting-for-compute message, as a single-task run does; a task blocked on an upstream task still reports BLOCKED. - The per-task state cases that had a single task key get a sibling task, so they reach taskState, and get-multitask carries status.termination_details at run and task level, so the task's own failure reason is pinned. - The changelog and `air get` help state that a task whose latest attempt failed reads RUNNING while another task of the run is still active. Co-authored-by: Isaac --- .nextchanges/air/multitask-air-tasks.md | 2 +- acceptance/air/get-multitask/output.txt | 4 +-- acceptance/air/get-multitask/test.toml | 3 ++ acceptance/air/help/output.txt | 4 ++- cmd/air/format.go | 5 ++-- cmd/air/get.go | 4 ++- cmd/air/joblist.go | 6 ++++ cmd/air/joblist_test.go | 26 +++++++++++++---- cmd/air/list_index.go | 38 +++++++------------------ cmd/air/list_index_test.go | 31 ++++++++++++++++++++ 10 files changed, 84 insertions(+), 39 deletions(-) diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md index b01f12d1b5a..ae8825c7c3e 100644 --- a/.nextchanges/air/multitask-air-tasks.md +++ b/.nextchanges/air/multitask-air-tasks.md @@ -1 +1 @@ -* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing, and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before. ([#7632](https://github.com/databricks/cli/pull/7632)) +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing (except that a task whose latest attempt failed shows the run's RUNNING, with no failure reason, while another task of the run is still active, since it may yet be retried), and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before. ([#7632](https://github.com/databricks/cli/pull/7632)) diff --git a/acceptance/air/get-multitask/output.txt b/acceptance/air/get-multitask/output.txt index 8292b018c7a..3c2f87e5363 100644 --- a/acceptance/air/get-multitask/output.txt +++ b/acceptance/air/get-multitask/output.txt @@ -44,7 +44,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 │ │ ╰────────────────────────────────────────────────────────────────╯ -Failure reason: eval: loss is NaN +Failure reason: eval: loss is NaN (step 120) Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 @@ -101,7 +101,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "mlflow_experiment_id": "exp1", "mlflow_run_id": "run1", - "termination_reason": "eval: loss is NaN", + "termination_reason": "eval: loss is NaN (step 120)", "est_remaining_seconds": null, "est_percent_complete": null, "user": "user@example.com", diff --git a/acceptance/air/get-multitask/test.toml b/acceptance/air/get-multitask/test.toml index 5d971327d24..e39241dfd66 100644 --- a/acceptance/air/get-multitask/test.toml +++ b/acceptance/air/get-multitask/test.toml @@ -21,6 +21,7 @@ Response.Body = ''' "start_time": 1700000000000, "end_time": 1700000020000, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task eval failed"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "Task eval failed with message: eval: loss is NaN"}}, "environments": [ {"environment_key": "default", "spec": {"environment_version": "5"}}, {"environment_key": "eval_env", "spec": {"environment_version": "6"}} @@ -43,6 +44,7 @@ Response.Body = ''' "start_time": 1700000002000, "end_time": 1700000012000, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "SUCCESS", "type": "SUCCESS", "message": ""}}, "depends_on": [{"task_key": "prep"}], "environment_key": "default", "ai_runtime_task": { @@ -60,6 +62,7 @@ Response.Body = ''' "start_time": 1700000012000, "end_time": 1700000020000, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "eval: loss is NaN"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "eval: loss is NaN (step 120)"}}, "depends_on": [{"task_key": "train"}], "environment_key": "eval_env", "ai_runtime_task": { diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7f6898a4fa1..d2b8736fa3d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -95,7 +95,9 @@ JOB_RUN_ID is the job run ID returned by "databricks air run". A job may run other tasks around its AI Runtime task (for example a notebook that prepares data first); the view shows the first AI Runtime task in task order. When a job has several AI Runtime tasks, pass --task-key to show another -one. +one. The status is that task's own, except that a task whose latest attempt +failed shows RUNNING while another task of the run is still active, since it +may yet be retried. Usage: databricks air get JOB_RUN_ID [flags] diff --git a/cmd/air/format.go b/cmd/air/format.go index 89d717c3aeb..718c6f34c45 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -116,8 +116,9 @@ func displayRunStatus(run *jobs.Run) string { // runWaitingForCompute reports whether a run is waiting for accelerator // compute: it is pending itself, or running while its lead task waits to start. // A run that is queued, blocked on an upstream task or waiting to retry at run -// level reports that state instead; for a task of a multi-task run, which -// carries the task's own state (see taskRun), that is the task's. +// level reports that state instead. A task of a multi-task run carries its own +// state (see taskRun): PENDING while it waits to start, BLOCKED while it waits +// on an upstream task. func runWaitingForCompute(run *jobs.Run) bool { if run.State == nil || run.State.ResultState != "" { return false diff --git a/cmd/air/get.go b/cmd/air/get.go index 89123da7cc6..46b3e3e1e6f 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -178,7 +178,9 @@ JOB_RUN_ID is the job run ID returned by "databricks air run". A job may run other tasks around its AI Runtime task (for example a notebook that prepares data first); the view shows the first AI Runtime task in task order. When a job has several AI Runtime tasks, pass --task-key to show another -one.`, +one. The status is that task's own, except that a task whose latest attempt +failed shows RUNNING while another task of the run is still active, since it +may yet be retried.`, Annotations: map[string]string{ "template": getTemplate, }, diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 21020c60fc5..6023d673bc1 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -100,11 +100,17 @@ func defaultTaskRun(r *jobs.Run) *jobs.Run { // none, and while the run is still active after a failed attempt, since the // task may be waiting to retry and runs/get doesn't reliably report how many // retries are left. A skipped task never retries, so it keeps its own state. +// A task waiting to start (pending, queued or waiting to retry, but not blocked +// on an upstream task) reports PENDING, as a single-task run whose task waits +// does (see runWaitingForCompute). func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { s := latest.State if s == nil { return r.State } + if s.LifeCycleState != jobs.RunLifeCycleStateBlocked && isWaitingState(s.LifeCycleState) { + return &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStatePending, StateMessage: s.StateMessage} + } failed := isTerminalState(s) && s.ResultState != jobs.RunResultStateSuccess && !isSkippedState(s) if failed && !isTerminal(r) { return r.State diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 7565639438f..5c07484a4ea 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -318,6 +318,7 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { name: "latest attempt is the one reported", runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, tasks: []jobs.RunTask{ + finished(notebookTask("prep", 10), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), func() jobs.RunTask { t := finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess) @@ -333,6 +334,7 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, tasks: []jobs.RunTask{ finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), }, key: "train", want: "RUNNING", @@ -348,14 +350,14 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { want: "PENDING", }, { - name: "a queued task is reported as queued", + name: "a queued task in a running run is waiting for compute", runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, tasks: []jobs.RunTask{ finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), }, key: "train", - want: "QUEUED", + want: "PENDING", }, { name: "a task blocked on its upstream task is reported as blocked", @@ -390,9 +392,12 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { { name: "a task without a state falls back to the run", runState: failedRun, - tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1)}, - key: "train", - want: "FAILED", + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 10), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), + }, + key: "train", + want: "FAILED", }, } for _, tt := range tests { @@ -453,4 +458,15 @@ func TestRunLevelWaitingStatesAreReportedAsSuch(t *testing.T) { }, } assert.Equal(t, "BLOCKED", detailedDisplayRunStatus(taskRun(blocked, "train"), "")) + + // The same task queued for compute reads like a single-task run's wait. + queued := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + }, + } + assert.Equal(t, waitingForComputeStatus, detailedDisplayRunStatus(taskRun(queued, "train"), "")) } diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index 7704c9a5344..d4f03e9069c 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -97,8 +97,8 @@ func (s *indexStrategy) truncated() bool { return false } // hydrate turns a batch of run ids into rows, serving cached terminal rows // without a network call and fetching the rest via runs/get. Freshly hydrated -// terminal runs are cached. Results keep the input (newest-first) order, then -// the batch is re-sorted by start time since concurrent hydration reorders it. +// terminal runs are cached. Results keep the input order: the index's, +// newest submission first. func (s *indexStrategy) hydrate(ids []int64) ([]runGroup, error) { host := s.w.Config.Host @@ -131,11 +131,16 @@ func (s *indexStrategy) hydrate(ids []int64) ([]runGroup, error) { } } - // Concurrent hydration reorders runs, so re-sort the batch newest-first, - // keeping a run's rows together. The ISO start timestamp sorts - // lexicographically; a missing time ("") sorts last. + // Cache hits come first and concurrent hydration reorders the rest, so put + // the batch back in the index's order, keeping a run's rows together. It is + // the order runs were submitted in, which a task that hasn't started or was + // skipped (and so has no start time, see taskRun) can't disturb. + pos := make(map[int64]int, len(ids)) + for i, id := range ids { + pos[id] = i + } slices.SortStableFunc(groups, func(a, b runGroup) int { - return cmp.Compare(groupStartKey(b), groupStartKey(a)) + return cmp.Compare(pos[a.runID], pos[b.runID]) }) return groups, nil } @@ -149,24 +154,3 @@ func (s *indexStrategy) cacheMLflowLinks(entries []listedRun) { } } } - -// groupStartKey returns the start key of a run's first row that has started: a -// task that hasn't started or was skipped has none (see taskRun), and shouldn't -// sort its whole run last. -func groupStartKey(g runGroup) string { - for _, e := range g.entries { - if k := rowStartKey(e.row); k != "" { - return k - } - } - return "" -} - -// rowStartKey returns a row's ISO start timestamp for ordering, or "" when the -// run hasn't started (which sorts last under descending comparison). -func rowStartKey(r listRow) string { - if r.StartedAt == nil { - return "" - } - return *r.StartedAt -} diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index 078577ac3df..3a0a9a98189 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -408,3 +408,34 @@ func TestIndexStrategyKeepsARunsRowsTogether(t *testing.T) { } assert.Equal(t, [][]string{{"8/"}, {"7/train", "7/eval"}}, got) } + +func TestIndexStrategyKeepsIndexOrderForARunWithNoStartedTask(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 9, the newest submission, failed in prep, so its AI Runtime task was + // skipped and has no start time. It must still list first, both on the first + // fetch and once it is served from the cache. + skipped := indexRun(9, 0) + skipped.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + train := skipped.Tasks[0] + train.TaskKey, train.RunId, train.StartTime, train.EndTime = "train", 91, 0, 0 + train.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateSkipped, ResultState: jobs.RunResultStateUpstreamFailed} + prep := notebookTask("prep", 90) + prep.StartTime, prep.EndTime = 3000_000, 3001_000 + prep.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + skipped.Tasks = []jobs.RunTask{prep, train} + refs := []workflowRef{{jobRunID: 8, submitTimeMs: 2000_000}, {jobRunID: 9, submitTimeMs: 3000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{9: skipped, 8: indexRun(8, 2000_000)}, nil, nil) + + for _, pass := range []string{"fresh", "cached"} { + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{currentUser: selfUser, limit: 10}) + rows, err := f.next(10) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + assert.Equal(t, []string{"9", "8"}, ids, pass) + assert.Nil(t, rows[0].StartedAt, "%s: the skipped task reports no start", pass) + } +} From bf8d601b21afd29a5832e70523b0a7314c7e6004 Mon Sep 17 00:00:00 2001 From: Dima Kotlyarov <4886215+dimk0@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:43:05 +0000 Subject: [PATCH 9/9] air: keep a cancelled task's state and pin the index-path order Review follow-ups: - Only an attempt Jobs may retry (failed, timed out, internal error) makes a task of an active run read the run's RUNNING; a cancelled task, like a skipped one, keeps its own state. - Tests pin the index path's order: a newer active run lists above cached older ones, and a retried run lists by submission time. The changelog notes that `air list --all-status` now orders runs by submission time. Co-authored-by: Isaac --- .nextchanges/air/multitask-air-tasks.md | 2 +- cmd/air/joblist.go | 28 ++++++++------- cmd/air/joblist_test.go | 20 +++++++++++ cmd/air/list_index_test.go | 45 +++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md index ae8825c7c3e..e6c65106386 100644 --- a/.nextchanges/air/multitask-air-tasks.md +++ b/.nextchanges/air/multitask-air-tasks.md @@ -1 +1 @@ -* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing (except that a task whose latest attempt failed shows the run's RUNNING, with no failure reason, while another task of the run is still active, since it may yet be retried), and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before. ([#7632](https://github.com/databricks/cli/pull/7632)) +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing (except that a task whose latest attempt failed shows the run's RUNNING, with no failure reason, while another task of the run is still active, since it may yet be retried), and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before, except that `air list --all-status` now orders runs by submission time, as the run index pages them, rather than by their latest attempt's start. ([#7632](https://github.com/databricks/cli/pull/7632)) diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index 6023d673bc1..2f7438996d2 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -97,9 +97,11 @@ func defaultTaskRun(r *jobs.Run) *jobs.Run { // latest: that attempt's state, so one task's result doesn't stand for another // (a successful train next to a failed eval, or an AI runtime task skipped // because its prep task failed). It keeps the run's state when the task has -// none, and while the run is still active after a failed attempt, since the -// task may be waiting to retry and runs/get doesn't reliably report how many -// retries are left. A skipped task never retries, so it keeps its own state. +// none, and while the run is still active after an attempt that may be +// retried (one that failed, timed out or hit an internal error), since the task +// may be waiting to retry and runs/get doesn't reliably report how many retries +// are left. A cancelled or skipped task is never retried, so it keeps its own +// state. // A task waiting to start (pending, queued or waiting to retry, but not blocked // on an upstream task) reports PENDING, as a single-task run whose task waits // does (see runWaitingForCompute). @@ -111,20 +113,22 @@ func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { if s.LifeCycleState != jobs.RunLifeCycleStateBlocked && isWaitingState(s.LifeCycleState) { return &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStatePending, StateMessage: s.StateMessage} } - failed := isTerminalState(s) && s.ResultState != jobs.RunResultStateSuccess && !isSkippedState(s) - if failed && !isTerminal(r) { + if mayRetry(s) && !isTerminal(r) { return r.State } return s } -// isSkippedState reports whether a task was skipped without running: an -// upstream task failed or was cancelled, or a condition excluded it. -func isSkippedState(s *jobs.RunState) bool { - skipped := s.ResultState == jobs.RunResultStateUpstreamFailed || - s.ResultState == jobs.RunResultStateUpstreamCanceled || - s.ResultState == jobs.RunResultStateExcluded - return skipped || s.LifeCycleState == jobs.RunLifeCycleStateSkipped +// mayRetry reports whether a finished task attempt is one Jobs may retry: it +// failed, timed out (with retry_on_timeout) or hit an internal error. A +// successful, cancelled or skipped attempt is final. +func mayRetry(s *jobs.RunState) bool { + if !isTerminalState(s) { + return false + } + return s.ResultState == jobs.RunResultStateFailed || + s.ResultState == jobs.RunResultStateTimedout || + s.LifeCycleState == jobs.RunLifeCycleStateInternalError } // leadTask returns the task run a run reports: Tasks[0], which is the AI diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 5c07484a4ea..9acf8304859 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -380,6 +380,26 @@ func TestTaskRunReportsTheTasksOwnState(t *testing.T) { key: "eval", want: "UPSTREAM_FAILED", }, + { + name: "a cancelled task keeps its state while another task still runs", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateCanceled), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), + }, + key: "train", + want: "CANCELED", + }, + { + name: "a timed-out attempt in an active run may be retried, so the run's state stands", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateTimedout), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), + }, + key: "train", + want: "RUNNING", + }, { name: "the run's only task keeps the run's state", runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateTimedout}, diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index 3a0a9a98189..955aea31e1a 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -439,3 +439,48 @@ func TestIndexStrategyKeepsIndexOrderForARunWithNoStartedTask(t *testing.T) { assert.Nil(t, rows[0].StartedAt, "%s: the skipped task reports no start", pass) } } + +// indexOrder fetches one page through the index path and returns its run IDs. +func indexOrder(t *testing.T, url string) []string { + t.Helper() + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, url), listQuery{currentUser: selfUser, limit: 10}) + rows, err := f.next(10) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + return ids +} + +func TestIndexStrategyListsANewActiveRunAboveCachedOlderOnes(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Runs 8 and 9 are terminal, so the first fetch caches them; run 10 is still + // running, so it is fetched every time. Cache hits are collected before the + // fetched runs, and the batch must still come back newest submission first. + active := indexRun(10, 4000_000) + active.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning} + active.Tasks[0].EndTime = 0 + refs := []workflowRef{{jobRunID: 8, submitTimeMs: 2000_000}, {jobRunID: 9, submitTimeMs: 3000_000}, {jobRunID: 10, submitTimeMs: 4000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{8: indexRun(8, 2000_000), 9: indexRun(9, 3000_000), 10: active}, nil, nil) + + assert.Equal(t, []string{"10", "9", "8"}, indexOrder(t, srv.URL), "fresh") + assert.Equal(t, []string{"10", "9", "8"}, indexOrder(t, srv.URL), "8 and 9 from the cache") +} + +func TestIndexStrategyListsARetriedRunBySubmission(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 7 was submitted before run 8 but its retry started after run 8. The + // index pages runs by submission time, so a batch keeps that order rather + // than moving run 7 up by its latest attempt's start. + retried := indexRun(7, 1000_000) + second := retried.Tasks[0] + second.RunId, second.AttemptNumber, second.StartTime, second.EndTime = 71, 1, 5000_000, 5001_000 + retried.Tasks = append(retried.Tasks, second) + refs := []workflowRef{{jobRunID: 7, submitTimeMs: 1000_000}, {jobRunID: 8, submitTimeMs: 2000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{7: retried, 8: indexRun(8, 2000_000)}, nil, nil) + + assert.Equal(t, []string{"8", "7"}, indexOrder(t, srv.URL)) +}