diff --git a/.nextchanges/air/multitask-air-tasks.md b/.nextchanges/air/multitask-air-tasks.md new file mode 100644 index 00000000000..e6c65106386 --- /dev/null +++ b/.nextchanges/air/multitask-air-tasks.md @@ -0,0 +1 @@ +* `air list` and `air get` now read the AI Runtime task wherever it sits in a multi-task job (for example after a notebook prep task) instead of the first task. A job with several AI Runtime tasks lists one row per task, named in the table's Run ID column (so `run_id` is no longer unique in `air list -o json`, and `--limit` still counts runs); each such row and `air get --task-key` report that task's own status and timing (except that a task whose latest attempt failed shows the run's RUNNING, with no failure reason, while another task of the run is still active, since it may yet be retried), and `air get` shows the selected task's environment. `air get` gains `--task-key`, and both commands report `task_key` in JSON output (`null` for a run without an AI Runtime task). Runs with a single task report as before, except that `air list --all-status` now orders runs by submission time, as the run index pages them, rather than by their latest attempt's start. ([#7632](https://github.com/databricks/cli/pull/7632)) diff --git a/acceptance/air/control-chars/output.txt b/acceptance/air/control-chars/output.txt index f5d95c03e06..524712a6cb1 100644 --- a/acceptance/air/control-chars/output.txt +++ b/acceptance/air/control-chars/output.txt @@ -38,6 +38,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "runs": [ { "run_id": "123", + "task_key": "train", "run_name": "qwen\u001b[2J-train", "user": "user\u001b[31m@example.com", "status": "SUCCESS", diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 034d307bd1f..051a9c8371f 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -43,6 +43,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -73,6 +74,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -142,6 +144,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, @@ -172,6 +175,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/get-mlflow-retry/output.txt b/acceptance/air/get-mlflow-retry/output.txt index 17e49f3febf..f46b8e1f124 100644 --- a/acceptance/air/get-mlflow-retry/output.txt +++ b/acceptance/air/get-mlflow-retry/output.txt @@ -7,6 +7,7 @@ "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": null, "status": "FAILED", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/get-multitask-skipped/out.test.toml b/acceptance/air/get-multitask-skipped/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-multitask-skipped/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-multitask-skipped/output.txt b/acceptance/air/get-multitask-skipped/output.txt new file mode 100644 index 00000000000..3d03bd68031 --- /dev/null +++ b/acceptance/air/get-multitask-skipped/output.txt @@ -0,0 +1,56 @@ + +=== get (text): the skipped AI Runtime task's own status, with no duration + +>>> [CLI] air get 123 + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Status ● UPSTREAM_FAILED │ +│ Submitted N/A │ +│ Retries 0 │ +│ Max Retries 0 │ +│ Duration N/A │ +│ Experiment air-byot-multitask-dag │ +│ MLflow Run N/A │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment N/A │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] + +=== get (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "UPSTREAM_FAILED", + "started_at": null, + "duration_seconds": null, + "attempt_number": 0, + "experiment_name": "air-byot-multitask-dag", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "mlflow_experiment_id": null, + "mlflow_run_id": null, + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null + } +} diff --git a/acceptance/air/get-multitask-skipped/script b/acceptance/air/get-multitask-skipped/script new file mode 100644 index 00000000000..d1ec551de0a --- /dev/null +++ b/acceptance/air/get-multitask-skipped/script @@ -0,0 +1,5 @@ +title "get (text): the skipped AI Runtime task's own status, with no duration" +trace $CLI air get 123 + +title "get (json)" +trace $CLI air get 123 -o json diff --git a/acceptance/air/get-multitask-skipped/test.toml b/acceptance/air/get-multitask-skipped/test.toml new file mode 100644 index 00000000000..869255d9464 --- /dev/null +++ b/acceptance/air/get-multitask-skipped/test.toml @@ -0,0 +1,53 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# The notebook-first DAB shape where the prep notebook fails, so the +# ai_runtime_task that depends on it is skipped (UPSTREAM_FAILED) and never +# starts. `air get` must report the task's own UPSTREAM_FAILED, not the run's +# FAILED, and no start time or duration rather than the run's. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700000005000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task prep failed"}, + "tasks": [ + { + "task_key": "prep", + "run_id": 455, + "attempt_number": 0, + "start_time": 1700000000000, + "end_time": 1700000005000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "prep: table not found"}, + "notebook_task": {"notebook_path": "/Workspace/Users/user@example.com/prep"} + }, + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "state": {"life_cycle_state": "SKIPPED", "result_state": "UPSTREAM_FAILED", "state_message": "Upstream task(s) failed."}, + "depends_on": [{"task_key": "prep"}], + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [ + {"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}} + ] + } + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{} +''' diff --git a/acceptance/air/get-multitask/out.test.toml b/acceptance/air/get-multitask/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-multitask/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-multitask/output.txt b/acceptance/air/get-multitask/output.txt new file mode 100644 index 00000000000..3c2f87e5363 --- /dev/null +++ b/acceptance/air/get-multitask/output.txt @@ -0,0 +1,142 @@ + +=== get (text): the first AI Runtime task, not the notebook prep task + +>>> [CLI] air get 123 + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Task train │ +│ Status ● SUCCESS │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 0 │ +│ Duration 10s │ +│ Experiment air-byot-multitask-dag │ +│ MLflow Run matmul-smoke │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment 5 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get --task-key eval (text) + +>>> [CLI] air get 123 --task-key eval + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Task eval │ +│ Status ● FAILED │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 2 │ +│ Duration 8s │ +│ Experiment eval-exp │ +│ MLflow Run matmul-smoke │ +│ User user@example.com │ +│ Accelerators 8x H100 │ +│ Environment 6 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Failure reason: eval: loss is NaN (step 120) + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 10, + "attempt_number": 0, + "experiment_name": "air-byot-multitask-dag", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "mlflow_experiment_id": "exp1", + "mlflow_run_id": "run1", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "5", + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": "matmul-smoke", + "sweep": null + } +} + +=== get --task-key eval (json) + +>>> [CLI] air get 123 --task-key eval -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "eval", + "status": "FAILED", + "started_at": "[TIMESTAMP]", + "duration_seconds": 8, + "attempt_number": 0, + "experiment_name": "eval-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "mlflow_experiment_id": "exp1", + "mlflow_run_id": "run1", + "termination_reason": "eval: loss is NaN (step 120)", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_8xH100", + "num_accelerators": 8, + "environment_version": "6", + "base_environment": null, + "max_retries": { + "limit": 2, + "source": "reported" + }, + "mlflow_run_name": "matmul-smoke", + "sweep": null + } +} + +=== get --task-key prep: not an AI Runtime task + +>>> [CLI] air get 123 --task-key prep +Error: run 123 has no AI Runtime task "prep"; its AI Runtime tasks are: train, eval + +Exit code: 1 + +=== get --task-key prep (json) + +>>> [CLI] air get 123 --task-key prep -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "error": { + "code": "INVALID_ARGS", + "kind": "PERMANENT", + "message": "run 123 has no AI Runtime task \"prep\"; its AI Runtime tasks are: train, eval", + "retryable": false + } +} + +Exit code: 1 diff --git a/acceptance/air/get-multitask/script b/acceptance/air/get-multitask/script new file mode 100644 index 00000000000..d4d4505995f --- /dev/null +++ b/acceptance/air/get-multitask/script @@ -0,0 +1,17 @@ +title "get (text): the first AI Runtime task, not the notebook prep task" +trace $CLI air get 123 + +title "get --task-key eval (text)" +trace $CLI air get 123 --task-key eval + +title "get (json)" +trace $CLI air get 123 -o json + +title "get --task-key eval (json)" +trace $CLI air get 123 --task-key eval -o json + +title "get --task-key prep: not an AI Runtime task" +errcode trace $CLI air get 123 --task-key prep + +title "get --task-key prep (json)" +errcode trace $CLI air get 123 --task-key prep -o json diff --git a/acceptance/air/get-multitask/test.toml b/acceptance/air/get-multitask/test.toml new file mode 100644 index 00000000000..e39241dfd66 --- /dev/null +++ b/acceptance/air/get-multitask/test.toml @@ -0,0 +1,90 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. + +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A multi-task job in the shape of the official DAB example (universe +# ai-compute/dabs/examples/03_multitask_dag): a notebook prep task first, then the +# ai_runtime_task that depends on it, plus a second ai_runtime_task. `air get` +# must read the AI Runtime task wherever it sits, not tasks[0]. The tasks end +# differently (train succeeds, eval fails, so the run fails): each view must +# report its own task's status, not the run's. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700000020000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Task eval failed"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "Task eval failed with message: eval: loss is NaN"}}, + "environments": [ + {"environment_key": "default", "spec": {"environment_version": "5"}}, + {"environment_key": "eval_env", "spec": {"environment_version": "6"}} + ], + "tasks": [ + { + "task_key": "prep", + "run_id": 455, + "attempt_number": 0, + "start_time": 1700000000000, + "end_time": 1700000002000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "notebook_task": {"notebook_path": "/Workspace/Users/user@example.com/prep"} + }, + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "start_time": 1700000002000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "SUCCESS", "type": "SUCCESS", "message": ""}}, + "depends_on": [{"task_key": "prep"}], + "environment_key": "default", + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [ + {"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}} + ] + } + }, + { + "task_key": "eval", + "run_id": 457, + "attempt_number": 0, + "max_retries": 2, + "start_time": 1700000012000, + "end_time": 1700000020000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "eval: loss is NaN"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "eval: loss is NaN (step 120)"}}, + "depends_on": [{"task_key": "train"}], + "environment_key": "eval_env", + "ai_runtime_task": { + "experiment": "/Users/user@example.com/eval-exp", + "deployments": [ + {"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 8}} + ] + } + } + ] +} +''' + +# MLflow identifiers for the deep link, fetched for the selected task's run. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}} +''' + +[[Server]] +Pattern = "GET /api/2.0/mlflow/runs/get" +Response.Body = ''' +{"run": {"info": {"run_name": "matmul-smoke"}}} +''' diff --git a/acceptance/air/get-pending/output.txt b/acceptance/air/get-pending/output.txt index 295de735ce5..930980364f2 100644 --- a/acceptance/air/get-pending/output.txt +++ b/acceptance/air/get-pending/output.txt @@ -5,6 +5,7 @@ "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": "train", "status": "PENDING", "started_at": null, "duration_seconds": null, diff --git a/acceptance/air/get-run-state/out.test.toml b/acceptance/air/get-run-state/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-run-state/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-run-state/output.txt b/acceptance/air/get-run-state/output.txt new file mode 100644 index 00000000000..4769155138b --- /dev/null +++ b/acceptance/air/get-run-state/output.txt @@ -0,0 +1,35 @@ + +=== get (json): the run's TIMEDOUT state and message, not its task's SUCCESS + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "task_key": "train", + "status": "TIMEDOUT", + "started_at": "[TIMESTAMP]", + "duration_seconds": 3500, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "mlflow_experiment_id": null, + "mlflow_run_id": null, + "termination_reason": "Task train timed out after 3600 seconds", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null + } +} diff --git a/acceptance/air/get-run-state/script b/acceptance/air/get-run-state/script new file mode 100644 index 00000000000..83a3b597051 --- /dev/null +++ b/acceptance/air/get-run-state/script @@ -0,0 +1,2 @@ +title "get (json): the run's TIMEDOUT state and message, not its task's SUCCESS" +trace $CLI air get 123 -o json diff --git a/acceptance/air/get-run-state/test.toml b/acceptance/air/get-run-state/test.toml new file mode 100644 index 00000000000..d6fb7958a8e --- /dev/null +++ b/acceptance/air/get-run-state/test.toml @@ -0,0 +1,51 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A single-task `air run` submission whose run state differs from its only +# task's: the run hit its timeout after the task had already succeeded. A run +# whose only task is the AI Runtime task keeps its run-level state and +# termination message (with its "Task ..." prefix); only a task of a +# multi-task run reports its own (see get-multitask). +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "start_time": 1700000000000, + "end_time": 1700003600000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "TIMEDOUT", "state_message": "Run timed out"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "RUN_EXECUTION_ERROR", "type": "CLIENT_ERROR", "message": "Task train timed out after 3600 seconds"}}, + "tasks": [ + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "start_time": 1700000000000, + "end_time": 1700003500000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "status": {"state": "TERMINATED", "termination_details": {"code": "SUCCESS", "type": "SUCCESS", "message": ""}}, + "ai_runtime_task": { + "experiment": "my-exp", + "deployments": [ + { + "command_path": "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/command.sh", + "compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1} + } + ] + } + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{} +''' diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index ef11028e860..64054d265aa 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -43,6 +43,7 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "ts": "[TIMESTAMP]", "data": { "run_id": "123", + "task_key": null, "status": "SUCCESS", "started_at": "[TIMESTAMP]", "duration_seconds": 12, diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 6f75b4afcc6..d2b8736fa3d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -77,7 +77,7 @@ Flags: --all-users[=true|false] Show runs from all users --filter stringArray Filter runs by KEY=VALUE (repeatable) -h, --help[=true|false] help for list - --limit int Maximum number of runs to show (default 20) + --limit int Maximum number of runs to show; a run with several AI Runtime tasks shows one row per task (default 20) Global Flags: --debug[=true|false] enable debug logging @@ -92,12 +92,20 @@ Show a run's status, configuration, and timing. JOB_RUN_ID is the job run ID returned by "databricks air run". +A job may run other tasks around its AI Runtime task (for example a notebook +that prepares data first); the view shows the first AI Runtime task in task +order. When a job has several AI Runtime tasks, pass --task-key to show another +one. The status is that task's own, except that a task whose latest attempt +failed shows RUNNING while another task of the run is still active, since it +may yet be retried. + Usage: databricks air get JOB_RUN_ID [flags] Flags: -h, --help[=true|false] help for get --include-config[=true|false] Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters) + --task-key string Show this AI Runtime task of the run (default: the first in task order) Global Flags: --debug[=true|false] enable debug logging @@ -226,6 +234,9 @@ Global Flags: Cancel one or more runs by job run ID, or use --all to cancel all active runs submitted by the current user. +Cancelling a run stops all of its tasks: every AI Runtime task in the job, and +any other task such as a notebook that prepares data. + --all prompts for confirmation. Use --yes to skip the prompt. Usage: diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt index 8cc9da7c7fd..72512ee9cbc 100644 --- a/acceptance/air/list-index-fallback/output.txt +++ b/acceptance/air/list-index-fallback/output.txt @@ -14,6 +14,7 @@ Warn: air list: the AI Runtime run index is unavailable (failed to list training "runs": [ { "run_id": "334747067049496", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 5f76708bf45..7e25bb5d7a1 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -2,8 +2,11 @@ === list (text) >>> [CLI] air list - Run ID Experiment Status Started Duration Progress MLflow User Accelerators - 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + Run ID Experiment Status Started Duration Progress MLflow User Accelerators + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + 334747067049500 air-byot-multitask-dag ● SUCCESS [TIMESTAMP] 20s - qwen-train-001 [USERNAME] 1x A10 + 334747067049510 train train-then-eval ● SUCCESS [TIMESTAMP] 40s - qwen-train-001 [USERNAME] 1x A10 + 334747067049510 eval train-then-eval ● FAILED [TIMESTAMP] 15s - qwen-train-001 [USERNAME] 1x A10 === list (json) @@ -15,6 +18,7 @@ "runs": [ { "run_id": "334747067049496", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", @@ -25,11 +29,64 @@ "accelerator_type": "GPU_8xH100", "num_accelerators": 8, "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049496?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049500", + "task_key": "train", + "run_name": "air-byot-multitask-dag", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "air-byot-multitask-dag", + "duration_seconds": 20, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049500?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049510", + "task_key": "train", + "run_name": "train-then-eval", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "train-then-eval", + "duration_seconds": 40, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049510?o=[WORKSPACE_ID]" + }, + { + "run_id": "334747067049510", + "task_key": "eval", + "run_name": "train-then-eval", + "user": "[USERNAME]", + "status": "FAILED", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "train-then-eval", + "duration_seconds": 15, + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049510?o=[WORKSPACE_ID]" } ] } } +=== list (json): no matching runs is an empty array + +>>> [CLI] air list --filter experiment=no-such-experiment -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "runs": [] + } +} + === list --all-status (text, via AiTrainingService index) >>> [CLI] air list --all-status @@ -46,6 +103,7 @@ "runs": [ { "run_id": "334747067049496", + "task_key": null, "run_name": "qwen-train", "user": "[USERNAME]", "status": "SUCCESS", diff --git a/acceptance/air/list/script b/acceptance/air/list/script index d22fca13ad6..49cca4b79dd 100644 --- a/acceptance/air/list/script +++ b/acceptance/air/list/script @@ -4,6 +4,9 @@ trace $CLI air list title "list (json)" trace $CLI air list -o json +title "list (json): no matching runs is an empty array" +trace $CLI air list --filter 'experiment=no-such-experiment' -o json + title "list --all-status (text, via AiTrainingService index)" trace $CLI air list --all-status diff --git a/acceptance/air/list/test.toml b/acceptance/air/list/test.toml index 9d5ca13df25..009458a44e4 100644 --- a/acceptance/air/list/test.toml +++ b/acceptance/air/list/test.toml @@ -11,7 +11,13 @@ Response.Body = '' # The current runs carry an ai_runtime_task (not modeled by the typed SDK), which # is why the CLI parses the response raw. Both runs belong to the default user # (tester@databricks.com, from the built-in scim/v2/Me handler). The second run -# has no AI task and must be filtered out. +# has the shape of the official multi-task DAB (a notebook prep task before the +# ai_runtime_task) and must be listed from its AIR task. The third run has two +# AI Runtime tasks that ended differently: it lists as two rows, each naming its +# task and reporting its own status. The fourth run has no AI task and must be +# filtered out. This stub ignores the query: the real scan asks +# for run_type=SUBMIT_RUN, so a bundle-deployed job, which runs as a JOB_RUN, is +# reached only through the run index (`--all-status`), not this scan. [[Server]] Pattern = "GET /api/2.2/jobs/runs/list" Response.Body = ''' @@ -32,6 +38,67 @@ Response.Body = ''' } }] }, + { + "run_id": 334747067049500, + "run_name": "air-byot-multitask-dag", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608700000, + "end_time": 1717608730000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [ + { + "task_key": "prep", + "run_id": 334747067049501, + "start_time": 1717608700000, + "end_time": 1717608705000, + "notebook_task": {"notebook_path": "/Workspace/Users/tester@databricks.com/prep"} + }, + { + "task_key": "train", + "run_id": 334747067049502, + "start_time": 1717608710000, + "end_time": 1717608730000, + "depends_on": [{"task_key": "prep"}], + "ai_runtime_task": { + "experiment": "air-byot-multitask-dag", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + } + ] + }, + { + "run_id": 334747067049510, + "run_name": "train-then-eval", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608600000, + "end_time": 1717608660000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [ + { + "task_key": "train", + "run_id": 334747067049511, + "start_time": 1717608600000, + "end_time": 1717608640000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "ai_runtime_task": { + "experiment": "train-then-eval", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + }, + { + "task_key": "eval", + "run_id": 334747067049512, + "start_time": 1717608645000, + "end_time": 1717608660000, + "depends_on": [{"task_key": "train"}], + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "ai_runtime_task": { + "experiment": "train-then-eval", + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + } + ] + }, { "run_id": 999000999000, "run_name": "not-an-air-run", diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index 0ca93c793cd..276bf82da2b 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -47,6 +47,9 @@ func newCancelCommand() *cobra.Command { Long: `Cancel one or more runs by job run ID, or use --all to cancel all active runs submitted by the current user. +Cancelling a run stops all of its tasks: every AI Runtime task in the job, and +any other task such as a notebook that prepares data. + --all prompts for confirmation. Use --yes to skip the prompt.`, } @@ -100,12 +103,15 @@ all active runs submitted by the current user. // Fetch every active run (up to the scan bound) so --all cancels all // of them, not just the first page. fetcher := newRunFetcher(ctx, w, listQuery{activeOnly: true, userFilter: me.UserName}) - rows, err := fetcher.next(maxListScan) + runs, err := fetcher.nextRuns(maxListScan) if err != nil { return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, fmt.Errorf("failed to list active runs: %w", err)) } + // A job with several AI runtime tasks lists one row per task; the + // run is cancelled (and previewed) once, naming all its tasks. + rows, taskKeys := cancelPreviewRows(runs) runIDs = make([]string, 0, len(rows)) for i := range rows { if rows[i].RunID != "" { @@ -123,7 +129,7 @@ all active runs submitted by the current user. } if !yes { - displayCancelPreview(ctx, rows, host) + displayCancelPreview(ctx, rows, taskKeys, host) confirmed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nCancel %d run(s) in %s?", len(runIDs), host)) if err != nil { return err @@ -215,21 +221,38 @@ func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, s return err } +// cancelPreviewRows returns the first row of each run, for the preview and the +// cancel, and the AI runtime task keys of each run by run id. +func cancelPreviewRows(runs [][]listRow) ([]listRow, map[string][]string) { + rows := make([]listRow, 0, len(runs)) + taskKeys := make(map[string][]string, len(runs)) + for _, run := range runs { + rows = append(rows, run[0]) + for _, r := range run { + if k := r.taskKey(); k != "" { + taskKeys[r.RunID] = append(taskKeys[r.RunID], k) + } + } + } + return rows, taskKeys +} + // displayCancelPreview shows the runs that `cancel --all` is about to terminate. -func displayCancelPreview(ctx context.Context, rows []listRow, host string) { +func displayCancelPreview(ctx context.Context, rows []listRow, taskKeys map[string][]string, host string) { var sb strings.Builder fmt.Fprintf(&sb, "\nWorkspace: %s\n", host) fmt.Fprintf(&sb, "Found %d active run(s) to cancel:\n\n", len(rows)) tw := tabwriter.NewWriter(&sb, 0, 0, 2, ' ', 0) - fmt.Fprintln(tw, "Run ID\tExperiment\tStarted") + fmt.Fprintln(tw, "Run ID\tExperiment\tAI Runtime tasks\tStarted") for i := range rows { experiment := orNA(rows[i].Experiment) started := na if rows[i].StartedAt != nil { started = *rows[i].StartedAt } - fmt.Fprintf(tw, "%s\t%s\t%s\n", rows[i].RunID, experiment, started) + tasks := orNA(strings.Join(taskKeys[rows[i].RunID], ", ")) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", rows[i].RunID, experiment, tasks, started) } tw.Flush() diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index 6f9166d1db3..4203451662f 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -325,7 +325,8 @@ func TestDisplayCancelPreview(t *testing.T) { {RunID: "111", Experiment: "exp-a", StartedAt: &started}, {RunID: "222"}, // no experiment or start time -> N/A } - displayCancelPreview(ctx, rows, "https://my-workspace.cloud.databricks.test") + taskKeys := map[string][]string{"111": {"train", "eval"}} + displayCancelPreview(ctx, rows, taskKeys, "https://my-workspace.cloud.databricks.test") out := buf.String() assert.Contains(t, out, "Workspace: https://my-workspace.cloud.databricks.test") @@ -333,6 +334,44 @@ func TestDisplayCancelPreview(t *testing.T) { assert.Contains(t, out, "Run ID") assert.Contains(t, out, "111") assert.Contains(t, out, "exp-a") + assert.Contains(t, out, "AI Runtime tasks") + assert.Contains(t, out, "train, eval") assert.Contains(t, out, "222") assert.Contains(t, out, na) } + +func TestCancelAllCancelsAMultiTaskRunOnce(t *testing.T) { + // Run 111 has two AI Runtime tasks after a notebook, so air list shows it as + // two rows; cancel --all must cancel it once and name both tasks. + multi := airBaseRun(111, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + multi.Tasks = []jobs.RunTask{ + notebookTask("prep", 1110), + aiRuntimeTask("train", 1111, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("eval", 1112, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + single := airBaseRun(222, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c") + list := runsListBody(t, "", multi, single) + var cancelled []int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/list": + _, _ = w.Write([]byte(list)) + case "/api/2.2/jobs/runs/cancel": + var req jobs.CancelRun + assert.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + cancelled = append(cancelled, req.RunId) + _, _ = w.Write([]byte(`{}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var buf bytes.Buffer + require.NoError(t, runCancelAll(t, newTestWorkspaceClient(t, srv.URL), flags.OutputText, strings.NewReader("y\n"), &buf)) + out := buf.String() + assert.Contains(t, out, "Found 2 active run(s) to cancel:") + assert.Contains(t, out, "train, eval") + assert.Contains(t, out, "Cancel 2 run(s)") + assert.ElementsMatch(t, []int64{111, 222}, cancelled, "each run is cancelled exactly once") +} diff --git a/cmd/air/format.go b/cmd/air/format.go index d73c2d8a620..718c6f34c45 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -113,6 +113,12 @@ func displayRunStatus(run *jobs.Run) string { return status } +// runWaitingForCompute reports whether a run is waiting for accelerator +// compute: it is pending itself, or running while its lead task waits to start. +// A run that is queued, blocked on an upstream task or waiting to retry at run +// level reports that state instead. A task of a multi-task run carries its own +// state (see taskRun): PENDING while it waits to start, BLOCKED while it waits +// on an upstream task. func runWaitingForCompute(run *jobs.Run) bool { if run.State == nil || run.State.ResultState != "" { return false @@ -120,10 +126,17 @@ func runWaitingForCompute(run *jobs.Run) bool { if run.State.LifeCycleState == jobs.RunLifeCycleStatePending { return true } - if run.State.LifeCycleState != jobs.RunLifeCycleStateRunning || len(run.Tasks) == 0 || run.Tasks[0].State == nil { + task := leadTask(run) + if run.State.LifeCycleState != jobs.RunLifeCycleStateRunning || task == nil || task.State == nil { return false } - switch run.Tasks[0].State.LifeCycleState { + return isWaitingState(task.State.LifeCycleState) +} + +// isWaitingState reports whether a task lifecycle state means it has not +// started running yet. +func isWaitingState(s jobs.RunLifeCycleState) bool { + switch s { case jobs.RunLifeCycleStatePending, jobs.RunLifeCycleStateQueued, jobs.RunLifeCycleStateWaitingForRetry, jobs.RunLifeCycleStateBlocked: return true @@ -157,8 +170,8 @@ func terminationReason(run *jobs.Run) *string { return &message } } - if len(run.Tasks) > 0 && run.Tasks[0].State != nil { - if message := strings.TrimSpace(run.Tasks[0].State.StateMessage); message != "" { + if task := leadTask(run); task != nil && task.State != nil { + if message := strings.TrimSpace(task.State.StateMessage); message != "" { return &message } } @@ -300,10 +313,11 @@ func latestAttemptNumber(run *jobs.Run) int { // "/Users/me@example.com/my-experiment"); we strip that prefix so users see just // the experiment name they chose. func experimentName(run *jobs.Run) *string { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - task := run.Tasks[0].GenAiComputeTask + task := t.GenAiComputeTask if task == nil || task.MlflowExperimentName == "" { return nil } diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 306e0bc3e1c..0afb7e7ffe1 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -384,3 +384,22 @@ func TestMaxRetriesFromDecodedRunGet(t *testing.T) { explicitZero := decode(`{"run_type": "SUBMIT_RUN", "tasks": [{"max_retries": 0, "ai_runtime_task": {}}]}`) assert.Equal(t, "0", display(explicitZero, []byte("experiment_name: e\n"))) } + +func TestLegacyExperimentAndAcceleratorsSkipNonAirTasks(t *testing.T) { + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + {TaskKey: "train", RunId: 52, GenAiComputeTask: &jobs.GenAiComputeTask{ + TrainingScriptPath: "/train.py", + MlflowExperimentName: "/Users/me@example.com/legacy-exp", + Compute: &jobs.ComputeConfig{GpuType: "H100", NumGpus: 8}, + }}, + }} + run = defaultTaskRun(run) + name := experimentName(run) + require.NotNil(t, name) + assert.Equal(t, "legacy-exp", *name) + gpu, n := jobCompute(run) + assert.Equal(t, "H100", gpu) + assert.Equal(t, 8, n) + assert.Equal(t, "8x H100", buildGetData(run).AcceleratorsDisplay) +} diff --git a/cmd/air/get.go b/cmd/air/get.go index 7a4de046fef..46b3e3e1e6f 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "slices" "strconv" "strings" "sync" @@ -26,7 +27,10 @@ import ( // the machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable text view. type getData struct { - RunID string `json:"run_id"` + RunID string `json:"run_id"` + // TaskKey is the AI runtime task shown (see selectAirTask), or null for a + // run without one. + TaskKey *string `json:"task_key"` Status string `json:"status"` StartedAt *string `json:"started_at"` DurationSeconds *int64 `json:"duration_seconds"` @@ -88,6 +92,9 @@ type getData struct { // Each shows "N/A" when its value is missing. The styled single-run renderer // (render.go) consumes them; the Run ID, Status, and MLflow Run cells it draws // are styled and hyperlinked there rather than stored here. + // TaskDisplay is the "Task" cell, set only when the run has several AI + // runtime tasks, so the view says which one it shows. + TaskDisplay string `json:"-"` SubmittedDisplay string `json:"-"` DurationDisplay string `json:"-"` ExperimentDisplay string `json:"-"` @@ -159,19 +166,28 @@ func credentialError(ctx context.Context, w *databricks.WorkspaceClient, err err // configuration, and timing details for a specific run. func newGetCommand() *cobra.Command { var includeConfig bool + var taskKey string cmd := &cobra.Command{ Use: "get JOB_RUN_ID", Args: root.ExactArgs(1), Short: "Show a run's status, configuration, and timing", Long: `Show a run's status, configuration, and timing. -JOB_RUN_ID is the job run ID returned by "databricks air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run". + +A job may run other tasks around its AI Runtime task (for example a notebook +that prepares data first); the view shows the first AI Runtime task in task +order. When a job has several AI Runtime tasks, pass --task-key to show another +one. The status is that task's own, except that a task whose latest attempt +failed shows RUNNING while another task of the run is still active, since it +may yet be retried.`, Annotations: map[string]string{ "template": getTemplate, }, } cmd.Flags().BoolVar(&includeConfig, "include-config", false, "Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters)") + cmd.Flags().StringVar(&taskKey, "task-key", "", "Show this AI Runtime task of the run (default: the first in task order)") // Resolve and authenticate the workspace client up front so an auth failure // fails fast here, before any run status or config is fetched or printed. @@ -212,6 +228,13 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, fmt.Errorf("failed to get status for run %d: %w", runID, err)) } + // From here on, run holds only the selected AI runtime task's task runs, + // so the status, timing, MLflow and config lookups all read that task. + run, data, err := taskView(run, taskKey) + if err != nil { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) + } + // Prefer the configured workspace ID; only look it up when it is unset. id, err := auth.ResolveWorkspaceID(ctx, w) if err != nil { @@ -224,7 +247,6 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, fmt.Errorf("configured workspace_id %q is not a numeric workspace ID", id)) } - data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) @@ -295,6 +317,45 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, return cmd } +// selectAirTask narrows run (see taskRun) to the AI runtime task named by key, +// or to its first AI runtime task in task order when key is empty, so the rest +// of the command reads that task alone. A run with no AI runtime task is +// returned unchanged when no key is given. The error names the run's AI runtime +// tasks when key is not one of them. +func selectAirTask(run *jobs.Run, key string) (*jobs.Run, error) { + if key == "" { + return defaultTaskRun(run), nil + } + keys := airTaskKeys(run) + if len(keys) == 0 { + return nil, fmt.Errorf("run %d has no AI Runtime tasks", run.RunId) + } + if !slices.Contains(keys, key) { + return nil, fmt.Errorf("run %d has no AI Runtime task %q; its AI Runtime tasks are: %s", run.RunId, key, strings.Join(keys, ", ")) + } + return taskRun(run, key), nil +} + +// taskView narrows run to the AI runtime task named by key (see selectAirTask) +// and builds its getData, recording the task shown: the shared start of +// `air get` and the list picker's detail pane, so the two stay in step. +// TaskKey stays null for a run without an AI runtime task; the Task cell is +// set only when the run has several to tell apart. +func taskView(run *jobs.Run, key string) (*jobs.Run, getData, error) { + selected, err := selectAirTask(run, key) + if err != nil { + return nil, getData{}, err + } + data := buildGetData(selected) + if k := airTaskKey(selected); k != "" { + data.TaskKey = &k + if len(airTaskKeys(run)) > 1 { + data.TaskDisplay = k + } + } + return selected, data, nil +} + // enrichFromAiRuntimeTask fills the config path, experiment, and accelerators // from the run's ai_runtime_task. Best-effort: empty fields leave the existing // "N/A" fallbacks in place. @@ -319,10 +380,11 @@ func enrichFromAiRuntimeTask(run *jobs.Run, data *getData) { // aiRuntimeTaskOf returns the run's first ai_runtime_task, or nil. func aiRuntimeTaskOf(run *jobs.Run) *jobs.AiRuntimeTask { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - return run.Tasks[0].AiRuntimeTask + return t.AiRuntimeTask } // buildGetData extracts the fields we display from a run. The text-view cells @@ -386,7 +448,7 @@ func enrichRunDetails(ctx context.Context, w *databricks.WorkspaceClient, run *j raw []byte mlflowName string ) - wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId) }) + wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId, taskEnvironmentKey(run)) }) if readConfig { wg.Go(func() { body, raw = resolveConfigYAML(ctx, w, run, data) }) } diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index dc6eb3636c0..5fd29635d2b 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -428,7 +428,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("an older retry's mlflow run names the retry", func(t *testing.T) { data := getRunJSON(t, srv, "123") assert.JSONEq(t, `{ - "run_id":"123","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"123","task_key":null,"status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":2,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/123?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs", @@ -444,7 +444,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) { t.Run("the latest attempt's mlflow run has no mlflow_retry key", func(t *testing.T) { data := getRunJSON(t, srv, "124") assert.JSONEq(t, `{ - "run_id":"124","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", + "run_id":"124","task_key":null,"status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", "duration_seconds":12,"attempt_number":0,"experiment_name":null, "dashboard_url":"`+host+`/jobs/runs/124?o=9999", "mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs", @@ -539,7 +539,7 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) { body, err := json.Marshal(failed) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"5","status":"FAILED","started_at":null,"duration_seconds":null, + "run_id":"5","task_key":null,"status":"FAILED","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, "mlflow_experiment_id":null,"mlflow_run_id":null, "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null, @@ -564,7 +564,7 @@ func TestBuildGetDataWaitingForComputeStatus(t *testing.T) { require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null, + "run_id":"8","task_key":null,"status":"PENDING","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, "mlflow_experiment_id":null,"mlflow_run_id":null, "est_remaining_seconds":null,"est_percent_complete":null, @@ -843,3 +843,110 @@ func TestSweepJSON(t *testing.T) { {"task_key":"t0","run_id":"10","status":"SUCCESS","experiment_name":"exp"}, {"task_key":"t1","run_id":"11","status":"RUNNING","experiment_name":null}]}`, string(got.Sweep)) } + +func TestAiRuntimeTaskOfSkipsNonAirTasks(t *testing.T) { + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + aiRuntimeTask("train", 52, "/Users/me@example.com/exp", "GPU_1xA10", 1), + }} + task := aiRuntimeTaskOf(defaultTaskRun(run)) + require.NotNil(t, task) + assert.Equal(t, "/Users/me@example.com/exp", task.Experiment) +} + +// multiTaskGetServer serves runs/get for a job with a notebook prep task and two +// ai_runtime_tasks, and stubs every other request. +func multiTaskGetServer(t *testing.T) *httptest.Server { + t.Helper() + run := jobs.Run{ + RunId: 5, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, + Tasks: []jobs.RunTask{ + notebookTask("prep", 51), + aiRuntimeTask("train", 52, "/Users/me@example.com/exp-train", "GPU_1xA10", 1), + aiRuntimeTask("eval", 53, "/Users/me@example.com/exp-eval", "GPU_8xH100", 8), + }, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _ = json.NewEncoder(w).Encode(run) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +func runGetJSONTask(t *testing.T, w *databricks.WorkspaceClient, taskKey string) (string, error) { + t.Helper() + var buf bytes.Buffer + ctx := cmdctx.SetWorkspaceClient(t.Context(), w) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &buf, &buf, "", "")) + cmd := withOutput(newGetCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + if taskKey != "" { + require.NoError(t, cmd.Flags().Set("task-key", taskKey)) + } + err := cmd.RunE(cmd, []string{"5"}) + return buf.String(), err +} + +func TestGetRunSelectsAirTask(t *testing.T) { + srv := multiTaskGetServer(t) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token", WorkspaceID: "67890"}) + require.NoError(t, err) + + type got struct { + Data struct { + TaskKey *string `json:"task_key"` + ExperimentName *string `json:"experiment_name"` + } `json:"data"` + } + + t.Run("defaults to the first AIR task, skipping the notebook", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "") + require.NoError(t, err) + var g got + require.NoError(t, json.Unmarshal([]byte(out), &g)) + require.NotNil(t, g.Data.TaskKey) + assert.Equal(t, "train", *g.Data.TaskKey) + require.NotNil(t, g.Data.ExperimentName) + assert.Equal(t, "exp-train", *g.Data.ExperimentName) + }) + + t.Run("--task-key selects a later AIR task", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "eval") + require.NoError(t, err) + var g got + require.NoError(t, json.Unmarshal([]byte(out), &g)) + require.NotNil(t, g.Data.TaskKey) + assert.Equal(t, "eval", *g.Data.TaskKey) + require.NotNil(t, g.Data.ExperimentName) + assert.Equal(t, "exp-eval", *g.Data.ExperimentName) + }) + + t.Run("--task-key must name an AIR task", func(t *testing.T) { + out, err := runGetJSONTask(t, w, "prep") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var e errorEnvelope + require.NoError(t, json.Unmarshal([]byte(out), &e)) + assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `run 5 has no AI Runtime task "prep"; its AI Runtime tasks are: train, eval`}, e.Error) + }) +} + +func TestSelectAirTaskErrors(t *testing.T) { + _, err := selectAirTask(&jobs.Run{RunId: 5, Tasks: []jobs.RunTask{notebookTask("prep", 51)}}, "prep") + assert.EqualError(t, err, "run 5 has no AI Runtime tasks") + + two := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ + notebookTask("prep", 70), + aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), + }} + _, err = selectAirTask(two, "prep") + assert.EqualError(t, err, `run 7 has no AI Runtime task "prep"; its AI Runtime tasks are: train`) + + run, err := selectAirTask(&jobs.Run{RunId: 8, Tasks: []jobs.RunTask{notebookTask("prep", 80)}}, "") + require.NoError(t, err) + assert.Len(t, run.Tasks, 1, "a run without an AI Runtime task is shown as is") +} diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index af696f24331..2f7438996d2 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "slices" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -16,41 +17,162 @@ import ( // of run ids from the AiTrainingService index. const hydrateConcurrency = 16 -// firstTask returns the run's first task, unwrapping a foreach sweep to the -// iterated task, or nil when the run has no tasks. -func firstTask(r *jobs.Run) *jobs.RunTask { +// taskSpec returns the AI task definitions a task run runs: its own, or for a +// foreach sweep those of the task it iterates. +func taskSpec(t *jobs.RunTask) (*jobs.AiRuntimeTask, *jobs.GenAiComputeTask) { + if t.ForEachTask != nil { + return t.ForEachTask.Task.AiRuntimeTask, t.ForEachTask.Task.GenAiComputeTask + } + return t.AiRuntimeTask, t.GenAiComputeTask +} + +// isAirTask reports whether a task run is an AI runtime workload: an +// ai_runtime_task, or a legacy gen_ai_compute_task with a training script, +// directly or as the task a foreach sweep iterates. +func isAirTask(t *jobs.RunTask) bool { + ai, genAI := taskSpec(t) + return ai != nil || (genAI != nil && genAI.TrainingScriptPath != "") +} + +// airTaskKeys returns the task keys of the run's AI runtime tasks in task +// order, each once: a retried task appears in r.Tasks once per attempt. It is +// the one scan for AI runtime tasks; everything after it reads a run narrowed +// to one of them (see taskRun), whose task is Tasks[0]. +func airTaskKeys(r *jobs.Run) []string { + var keys []string + for i := range r.Tasks { + t := &r.Tasks[i] + if isAirTask(t) && !slices.Contains(keys, t.TaskKey) { + keys = append(keys, t.TaskKey) + } + } + return keys +} + +// isAirRun reports whether a run is an AI runtime workload: it has at least +// one AI runtime task. +func isAirRun(r *jobs.Run) bool { + return len(airTaskKeys(r)) > 0 +} + +// taskRun returns a copy of r narrowed to the task runs (every attempt, in +// order) of the task with the given key, one of airTaskKeys(r), so the display +// helpers, which read a run's tasks, report that task alone. When the run has +// other tasks too, the copy also takes that task's state (see taskState) and +// drops the run's own start and end times, so a task that hasn't started (or +// was skipped) reports no timing rather than the whole run's. A run whose only +// task is this one keeps its run-level state and timing. +func taskRun(r *jobs.Run, key string) *jobs.Run { + narrowed := *r + narrowed.Tasks = nil + others := false + for i := range r.Tasks { + if r.Tasks[i].TaskKey == key { + narrowed.Tasks = append(narrowed.Tasks, r.Tasks[i]) + } else { + others = true + } + } + if others { + latest := &narrowed.Tasks[len(narrowed.Tasks)-1] + if state := taskState(r, latest); state != r.State { + narrowed.State = state + narrowed.Status = latest.Status + } + narrowed.StartTime, narrowed.EndTime = 0, 0 + } + return &narrowed +} + +// defaultTaskRun returns r narrowed to its first AI runtime task in task order, +// or r as is when it has none: the run `air get` shows without --task-key. +func defaultTaskRun(r *jobs.Run) *jobs.Run { + if keys := airTaskKeys(r); len(keys) > 0 { + return taskRun(r, keys[0]) + } + return r +} + +// taskState returns the state to report for the task whose latest attempt is +// latest: that attempt's state, so one task's result doesn't stand for another +// (a successful train next to a failed eval, or an AI runtime task skipped +// because its prep task failed). It keeps the run's state when the task has +// none, and while the run is still active after an attempt that may be +// retried (one that failed, timed out or hit an internal error), since the task +// may be waiting to retry and runs/get doesn't reliably report how many retries +// are left. A cancelled or skipped task is never retried, so it keeps its own +// state. +// A task waiting to start (pending, queued or waiting to retry, but not blocked +// on an upstream task) reports PENDING, as a single-task run whose task waits +// does (see runWaitingForCompute). +func taskState(r *jobs.Run, latest *jobs.RunTask) *jobs.RunState { + s := latest.State + if s == nil { + return r.State + } + if s.LifeCycleState != jobs.RunLifeCycleStateBlocked && isWaitingState(s.LifeCycleState) { + return &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStatePending, StateMessage: s.StateMessage} + } + if mayRetry(s) && !isTerminal(r) { + return r.State + } + return s +} + +// mayRetry reports whether a finished task attempt is one Jobs may retry: it +// failed, timed out (with retry_on_timeout) or hit an internal error. A +// successful, cancelled or skipped attempt is final. +func mayRetry(s *jobs.RunState) bool { + if !isTerminalState(s) { + return false + } + return s.ResultState == jobs.RunResultStateFailed || + s.ResultState == jobs.RunResultStateTimedout || + s.LifeCycleState == jobs.RunLifeCycleStateInternalError +} + +// leadTask returns the task run a run reports: Tasks[0], which is the AI +// runtime task once the run is narrowed (see taskRun). A run without an AI +// runtime task is not narrowed, so its first task stands in, as it always has; +// nil when the run has no tasks. +func leadTask(r *jobs.Run) *jobs.RunTask { if len(r.Tasks) == 0 { return nil } - t := &r.Tasks[0] - if t.AiRuntimeTask != nil || t.GenAiComputeTask != nil { + return &r.Tasks[0] +} + +// firstTask returns the run's lead task (see leadTask), unwrapping a foreach +// sweep to the iterated task; nil when the run has no tasks. +func firstTask(r *jobs.Run) *jobs.RunTask { + t := leadTask(r) + if t == nil || t.ForEachTask == nil { return t } - if t.ForEachTask != nil { - // The foreach inner task is a jobs.Task; expose the AI fields via a - // synthetic RunTask so callers can treat both shapes uniformly. - return &jobs.RunTask{ - AiRuntimeTask: t.ForEachTask.Task.AiRuntimeTask, - GenAiComputeTask: t.ForEachTask.Task.GenAiComputeTask, - } + // The foreach inner task is a jobs.Task; expose the fields callers read via + // a synthetic RunTask so they can treat both shapes uniformly. + ai, genAI := taskSpec(t) + return &jobs.RunTask{ + TaskKey: t.TaskKey, + AiRuntimeTask: ai, + GenAiComputeTask: genAI, + EnvironmentKey: t.ForEachTask.Task.EnvironmentKey, } - return t } -// isAirRun reports whether a run is an AI runtime workload: an ai_runtime_task, -// or a legacy gen_ai_compute_task with a training script. -func isAirRun(r *jobs.Run) bool { - t := firstTask(r) - if t == nil { - return false +// airTaskKey returns the key of the AI runtime task a narrowed run shows, or "" +// for a run without one. +func airTaskKey(r *jobs.Run) string { + if t := leadTask(r); t != nil && isAirTask(t) { + return t.TaskKey } - return t.AiRuntimeTask != nil || - (t.GenAiComputeTask != nil && t.GenAiComputeTask.TrainingScriptPath != "") + return "" } -// isSweep reports whether the run's first task fans out into iterations. +// isSweep reports whether the run's lead task fans out into iterations. func isSweep(r *jobs.Run) bool { - return len(r.Tasks) > 0 && r.Tasks[0].ForEachTask != nil + t := leadTask(r) + return t != nil && t.ForEachTask != nil } // retryTasks returns the run's task runs with only the fields that pick its @@ -96,14 +218,13 @@ func jobCompute(r *jobs.Run) (string, int) { } // jobTiming returns the run's start and end times (epoch ms), preferring the -// first task's window so a run reports its task attempt rather than the wrapper. +// AI runtime task's window so a run reports its task attempt rather than the +// wrapper. func jobTiming(r *jobs.Run) (startMillis, endMillis int64) { startMillis, endMillis = r.StartTime, r.EndTime - if len(r.Tasks) > 0 { - if t := r.Tasks[0]; t.StartTime > 0 { - startMillis = t.StartTime - endMillis = t.EndTime - } + if t := leadTask(r); t != nil && t.StartTime > 0 { + startMillis = t.StartTime + endMillis = t.EndTime } return startMillis, endMillis } @@ -111,10 +232,12 @@ func jobTiming(r *jobs.Run) (startMillis, endMillis int64) { // isTerminal reports whether a run has finished and its details are immutable, // so its row is safe to cache. func isTerminal(r *jobs.Run) bool { - if r.State == nil { - return false - } - switch r.State.LifeCycleState { + return r.State != nil && isTerminalState(r.State) +} + +// isTerminalState reports whether a run or task state is final. +func isTerminalState(s *jobs.RunState) bool { + switch s.LifeCycleState { case jobs.RunLifeCycleStateTerminated, jobs.RunLifeCycleStateInternalError, jobs.RunLifeCycleStateSkipped: return true default: diff --git a/cmd/air/joblist_test.go b/cmd/air/joblist_test.go index 33c69970dbc..9acf8304859 100644 --- a/cmd/air/joblist_test.go +++ b/cmd/air/joblist_test.go @@ -119,3 +119,374 @@ func TestHydrateJobRunsFailsOnOtherErrors(t *testing.T) { }) } } + +// notebookTask builds a non-AIR task run, like the prep step that precedes the +// ai_runtime_task in the official multi-task DAB example. +func notebookTask(key string, runID int64) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, NotebookTask: &jobs.NotebookTask{NotebookPath: "/prep"}} +} + +// aiRuntimeTask builds a direct ai_runtime_task task run. +func aiRuntimeTask(key string, runID int64, experiment, accelType string, count int) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, AiRuntimeTask: &jobs.AiRuntimeTask{ + Experiment: experiment, + Deployments: []jobs.DeploymentSpec{{ + Compute: jobs.ComputeSpec{AcceleratorType: jobs.ComputeSpecAcceleratorType(accelType), AcceleratorCount: count}, + }}, + }} +} + +// forEachAiRuntimeTask builds a foreach (sweep) task whose iterated task is an +// ai_runtime_task. +func forEachAiRuntimeTask(key string, runID int64, experiment string) jobs.RunTask { + return jobs.RunTask{TaskKey: key, RunId: runID, ForEachTask: &jobs.RunForEachTask{ + Task: jobs.Task{AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: experiment}}, + }} +} + +func TestAirTaskSelection(t *testing.T) { + tests := []struct { + name string + tasks []jobs.RunTask + air bool + keys []string + experiment string + sweep bool + }{ + { + name: "AIR task first", + tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "exp-train", "GPU_1xA10", 1), notebookTask("report", 12)}, + air: true, + keys: []string{"train"}, + experiment: "exp-train", + }, + { + name: "AIR task second", + tasks: []jobs.RunTask{notebookTask("prep", 11), aiRuntimeTask("train", 12, "exp-train", "GPU_1xA10", 1)}, + air: true, + keys: []string{"train"}, + experiment: "exp-train", + }, + { + name: "two AIR tasks", + tasks: []jobs.RunTask{aiRuntimeTask("a", 11, "exp-a", "GPU_1xA10", 1), aiRuntimeTask("b", 12, "exp-b", "GPU_8xH100", 8)}, + air: true, + keys: []string{"a", "b"}, + experiment: "exp-a", + }, + { + name: "foreach AIR task", + tasks: []jobs.RunTask{forEachAiRuntimeTask("sweep", 11, "exp-sweep")}, + air: true, + keys: []string{"sweep"}, + experiment: "exp-sweep", + sweep: true, + }, + { + name: "foreach AIR task after a notebook", + tasks: []jobs.RunTask{notebookTask("prep", 11), forEachAiRuntimeTask("sweep", 12, "exp-sweep")}, + air: true, + keys: []string{"sweep"}, + experiment: "exp-sweep", + sweep: true, + }, + { + name: "retries share one key", + tasks: []jobs.RunTask{notebookTask("prep", 11), aiRuntimeTask("train", 12, "exp-train", "GPU_1xA10", 1), aiRuntimeTask("train", 13, "exp-train", "GPU_1xA10", 1)}, + air: true, keys: []string{"train"}, experiment: "exp-train", + }, + { + name: "no AIR task", + tasks: []jobs.RunTask{notebookTask("prep", 11)}, + }, + { + name: "no tasks", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := &jobs.Run{RunId: 1, Tasks: tt.tasks} + assert.Equal(t, tt.air, isAirRun(run), "isAirRun") + assert.Equal(t, tt.keys, airTaskKeys(run), "airTaskKeys") + // Readers take the run narrowed to its AI Runtime task. + shown := defaultTaskRun(run) + assert.Equal(t, tt.experiment, jobExperiment(shown), "jobExperiment") + assert.Equal(t, tt.sweep, isSweep(shown), "isSweep") + if len(tt.tasks) == 0 { + assert.Nil(t, firstTask(run), "firstTask") + } + }) + } +} + +func TestTaskRunNarrowsToOneAirTask(t *testing.T) { + run := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + notebookTask("prep", 11), + aiRuntimeTask("a", 12, "exp-a", "GPU_1xA10", 1), + {TaskKey: "a", RunId: 13, AttemptNumber: 1, AiRuntimeTask: &jobs.AiRuntimeTask{Experiment: "exp-a"}}, + aiRuntimeTask("b", 14, "exp-b", "GPU_8xH100", 8), + }, + } + + a := taskRun(run, "a") + require.NotNil(t, a) + assert.Equal(t, int64(1), a.RunId) + assert.Equal(t, run.State, a.State) + require.Len(t, a.Tasks, 2, "both attempts of task a, in order") + assert.Equal(t, int64(12), a.Tasks[0].RunId) + assert.Equal(t, int64(13), a.Tasks[1].RunId) + assert.Equal(t, "exp-a", jobExperiment(a)) + + b := taskRun(run, "b") + require.NotNil(t, b) + assert.Equal(t, "exp-b", jobExperiment(b)) + gpu, n := jobCompute(b) + assert.Equal(t, "GPU_8xH100", gpu) + assert.Equal(t, 8, n) + + assert.Len(t, run.Tasks, 4, "the source run is left intact") + + first := defaultTaskRun(run) + assert.Equal(t, "a", first.Tasks[0].TaskKey, "the default is the first AI Runtime task, not the notebook") + assert.Equal(t, "a", airTaskKey(first)) + notebookOnly := &jobs.Run{RunId: 2, Tasks: []jobs.RunTask{notebookTask("prep", 21)}} + assert.Same(t, notebookOnly, defaultTaskRun(notebookOnly), "a run without an AI Runtime task is not narrowed") + assert.Empty(t, airTaskKey(notebookOnly)) +} + +func TestJobTimingUsesTheAirTask(t *testing.T) { + prep := notebookTask("prep", 11) + prep.StartTime, prep.EndTime = 100, 150 + train := aiRuntimeTask("train", 12, "exp", "GPU_1xA10", 1) + train.StartTime, train.EndTime = 200, 260 + run := &jobs.Run{RunId: 1, StartTime: 90, EndTime: 300, Tasks: []jobs.RunTask{prep, train}} + + start, end := jobTiming(defaultTaskRun(run)) + assert.Equal(t, int64(200), start) + assert.Equal(t, int64(260), end) +} + +// finished returns a task run whose latest attempt ended in result. +func finished(task jobs.RunTask, life jobs.RunLifeCycleState, result jobs.RunResultState) jobs.RunTask { + task.State = &jobs.RunState{LifeCycleState: life, ResultState: result} + return task +} + +func TestTaskRunReportsTheTasksOwnState(t *testing.T) { + failedRun := &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + tests := []struct { + name string + runState *jobs.RunState + tasks []jobs.RunTask + key string + want string + }{ + { + name: "succeeded task next to a failed one", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + }, + key: "train", + want: "SUCCESS", + }, + { + name: "the failed task", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + }, + key: "eval", + want: "FAILED", + }, + { + name: "skipped because the prep notebook failed", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed), + }, + key: "train", + want: "UPSTREAM_FAILED", + }, + { + name: "latest attempt is the one reported", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 10), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + func() jobs.RunTask { + t := finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess) + t.AttemptNumber = 1 + return t + }(), + }, + key: "train", + want: "SUCCESS", + }, + { + name: "a failed attempt in an active run may be retried, so the run's state stands", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), + }, + key: "train", + want: "RUNNING", + }, + { + name: "a pending task in a running run is waiting for compute", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStatePending, ""), + }, + key: "train", + want: "PENDING", + }, + { + name: "a queued task in a running run is waiting for compute", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + }, + key: "train", + want: "PENDING", + }, + { + name: "a task blocked on its upstream task is reported as blocked", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateRunning, ""), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateBlocked, ""), + }, + key: "train", + want: "BLOCKED", + }, + { + name: "a skipped task keeps its state while another branch still runs", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateRunning, ""), + finished(notebookTask("prep_eval", 12), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + finished(aiRuntimeTask("eval", 13, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed), + }, + key: "eval", + want: "UPSTREAM_FAILED", + }, + { + name: "a cancelled task keeps its state while another task still runs", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateCanceled), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), + }, + key: "train", + want: "CANCELED", + }, + { + name: "a timed-out attempt in an active run may be retried, so the run's state stands", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateTimedout), + finished(notebookTask("report", 12), jobs.RunLifeCycleStateRunning, ""), + }, + key: "train", + want: "RUNNING", + }, + { + name: "the run's only task keeps the run's state", + runState: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateTimedout}, + tasks: []jobs.RunTask{ + finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + }, + key: "train", + want: "TIMEDOUT", + }, + { + name: "a task without a state falls back to the run", + runState: failedRun, + tasks: []jobs.RunTask{ + finished(notebookTask("prep", 10), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed), + aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), + }, + key: "train", + want: "FAILED", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := &jobs.Run{RunId: 1, State: tt.runState, Tasks: tt.tasks} + narrowed := taskRun(run, tt.key) + require.NotNil(t, narrowed) + assert.Equal(t, tt.want, displayRunStatus(narrowed)) + assert.Equal(t, tt.want, buildListRow(narrowed, "https://example.test", 0).Status) + assert.Same(t, tt.runState, run.State, "the source run keeps its state") + }) + } +} + +func TestTaskRunOfASkippedTaskHasNoTiming(t *testing.T) { + train := finished(aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateFailed) + train.StartTime, train.EndTime = 200_000, 260_000 + eval := finished(aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateSkipped, jobs.RunResultStateUpstreamFailed) + run := &jobs.Run{ + RunId: 1, + StartTime: 100_000, + EndTime: 300_000, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed}, + Tasks: []jobs.RunTask{train, eval}, + } + + skipped := taskRun(run, "eval") + assert.Nil(t, startedAt(skipped), "a task that never started reports no start, not the run's") + assert.Nil(t, durationSeconds(skipped)) + + ran := taskRun(run, "train") + require.NotNil(t, durationSeconds(ran)) + assert.Equal(t, int64(60), *durationSeconds(ran), "train reports its own 60s, not the run's 200s") +} + +func TestRunLevelWaitingStatesAreReportedAsSuch(t *testing.T) { + for _, life := range []jobs.RunLifeCycleState{ + jobs.RunLifeCycleStateQueued, jobs.RunLifeCycleStateBlocked, jobs.RunLifeCycleStateWaitingForRetry, + } { + t.Run(string(life), func(t *testing.T) { + run := &jobs.Run{RunId: 1, State: &jobs.RunState{LifeCycleState: life}, Tasks: []jobs.RunTask{aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1)}} + assert.False(t, runWaitingForCompute(run)) + assert.Equal(t, string(life), displayRunStatus(run)) + assert.Equal(t, string(life), detailedDisplayRunStatus(run, "")) + }) + } + + pending := &jobs.Run{RunId: 1, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStatePending}} + assert.Equal(t, waitingForComputeStatus, detailedDisplayRunStatus(pending, "")) + + // The notebook-first DAB shape: train waits on prep, not on compute. + blocked := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateRunning, ""), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateBlocked, ""), + }, + } + assert.Equal(t, "BLOCKED", detailedDisplayRunStatus(taskRun(blocked, "train"), "")) + + // The same task queued for compute reads like a single-task run's wait. + queued := &jobs.Run{ + RunId: 1, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{ + finished(notebookTask("prep", 11), jobs.RunLifeCycleStateTerminated, jobs.RunResultStateSuccess), + finished(aiRuntimeTask("train", 12, "e", "GPU_1xA10", 1), jobs.RunLifeCycleStateQueued, ""), + }, + } + assert.Equal(t, waitingForComputeStatus, detailedDisplayRunStatus(taskRun(queued, "train"), "")) +} diff --git a/cmd/air/list.go b/cmd/air/list.go index 9a5a85b0cbc..4e84fc12c3c 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -44,7 +44,13 @@ type listData struct { // machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable table. type listRow struct { - RunID string `json:"run_id"` + RunID string `json:"run_id"` + // TaskKey names the AI runtime task the row shows, or is null for a run + // without one (as in `air get`). A job with several AI runtime tasks has one + // row per task, all with the same RunID; ShowTask marks those rows so the + // table names the task. + TaskKey *string `json:"task_key"` + ShowTask bool `json:"-"` RunName string `json:"run_name"` User string `json:"user"` Status string `json:"status"` @@ -70,11 +76,31 @@ type listRow struct { Accelerators string `json:"-"` } +// taskKey returns the AI runtime task key the row shows, or "" for a run +// without one. +func (r listRow) taskKey() string { + if r.TaskKey == nil { + return "" + } + return *r.TaskKey +} + +// runIDLabel is the Run ID cell: the run ID, followed by the task key when the +// run has several AI runtime tasks, so each of its rows names the task to pass +// to `air get --task-key`. +func (r listRow) runIDLabel() string { + if r.ShowTask && r.TaskKey != nil { + return r.RunID + " " + *r.TaskKey + } + return r.RunID +} + // listedRun pairs a row with its parent run id and task runs, so the MLflow link // can be fetched after the run has been filtered in and cached afterward. type listedRun struct { row listRow runID int64 + taskKey string tasks []jobs.RunTask // see retryTasks terminal bool } @@ -132,7 +158,7 @@ Filters: cmd.PreRunE = root.MustWorkspaceClient - cmd.Flags().IntVar(&limit, "limit", 20, "Maximum number of runs to show") + cmd.Flags().IntVar(&limit, "limit", 20, "Maximum number of runs to show; a run with several AI Runtime tasks shows one row per task") cmd.Flags().BoolVar(&allStatus, "all-status", false, "Show runs in all states (default: active only)") cmd.Flags().BoolVar(&allUsers, "all-users", false, "Show runs from all users") cmd.Flags().StringArrayVar(&filters, "filter", nil, "Filter runs by KEY=VALUE (repeatable)") @@ -257,8 +283,8 @@ func validateEmptyAcceleratorFilter(ctx context.Context, w *databricks.Workspace // jobsScanStrategy pages runs/list; indexStrategy hydrates the AiTrainingService // index. The fetcher wraps whichever is chosen. type listStrategy interface { - // next returns up to want more matching runs (already row-built + task id). - next(want int) ([]listedRun, error) + // next returns up to want more matching runs, each as its group of rows. + next(want int) ([]runGroup, error) // done reports whether the source has no more runs to yield. done() bool // truncated reports whether a safety cap stopped the scan short of the end. @@ -318,15 +344,31 @@ func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQ // text output, and projects it to rows. It sets exhausted once the strategy is // drained so the interactive table knows to stop paging. func (f *runFetcher) next(want int) ([]listRow, error) { - entries, err := f.strategy.next(want) + groups, err := f.nextRuns(want) + if err != nil { + return nil, err + } + // Start from an empty slice so a list with no matching runs prints + // "runs": [] rather than null. + rows := make([]listRow, 0, len(groups)) + for _, g := range groups { + rows = append(rows, g...) + } + return rows, nil +} + +// nextRuns is next with the rows kept grouped by run: one slice per run, one +// row per AI runtime task. +func (f *runFetcher) nextRuns(want int) ([][]listRow, error) { + groups, err := f.strategy.next(want) if err != nil { return nil, err } f.exhausted = f.strategy.done() - if len(entries) > 0 { + if len(groups) > 0 { f.matchedAny = true } - if len(entries) == 0 && f.exhausted && !f.matchedAny && f.validateEmpty != nil && !f.validatedEmpty { + if len(groups) == 0 && f.exhausted && !f.matchedAny && f.validateEmpty != nil && !f.validatedEmpty { f.validatedEmpty = true if err := f.validateEmpty(); err != nil { return nil, err @@ -334,19 +376,28 @@ func (f *runFetcher) next(want int) ([]listRow, error) { } // MLflow links appear only in the text table, so the per-run get-output - // lookups are skipped for JSON output (which omits the column anyway). + // lookups are skipped for JSON output (which omits the column anyway). They + // run over all rows at once, then land back in their groups. if f.fetchMLflow { - setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries) + var flat []listedRun + for _, g := range groups { + flat = append(flat, g.entries...) + } + setMLflowLinks(f.ctx, f.w, f.w.Config.Host, flat) if s, ok := f.strategy.(*indexStrategy); ok { - s.cacheMLflowLinks(entries) + s.cacheMLflowLinks(flat) + } + for _, g := range groups { + n := copy(g.entries, flat) + flat = flat[n:] } } - rows := make([]listRow, len(entries)) - for i, e := range entries { - rows[i] = e.row + out := make([][]listRow, len(groups)) + for i, g := range groups { + out[i] = g.rows() } - return rows, nil + return out, nil } // jobsScanStrategy pages Jobs runs/list, keeping the AIR runs that match the user @@ -392,9 +443,11 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l } } -func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { - var entries []listedRun - for len(entries) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { +// next returns the next want matching runs, each as its group of rows (see +// runGroup). +func (s *jobsScanStrategy) next(want int) ([]runGroup, error) { + var groups []runGroup + for len(groups) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { base, err := s.iter.Next(s.ctx) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) @@ -415,12 +468,11 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { if s.userFilter != "" && run.CreatorUserName != s.userFilter { continue } - if !s.filters.matches(run) { - continue + if g, ok := matchingGroup(run.RunId, buildTaskEntries(run, s.host, s.workspaceID), s.filters); ok { + groups = append(groups, g) } - entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, tasks: retryTasks(run), terminal: isTerminal(run)}) } - return entries, nil + return groups, nil } func (s *jobsScanStrategy) done() bool { diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index 941327e93b1..bc72a5f92ed 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -14,7 +14,7 @@ import ( // TTL matches AICM's ~60-day retention, after which the run drops out of the // index anyway. const ( - listCacheComponent = "air-list-runs-v2" + listCacheComponent = "air-list-runs-v3" listCacheTTL = 60 * 24 * time.Hour ) @@ -25,11 +25,18 @@ type listCacheKey struct { RunID int64 `json:"run_id"` } -// cachedRun is the persisted value: every listRow field (including the +// cachedRuns is the persisted value for a run: one cachedRun per AI runtime +// task, before filtering (see buildTaskEntries), so a hit can be re-filtered. +type cachedRuns struct { + Rows []cachedRun `json:"rows"` +} + +// cachedRun is one persisted list row: every listRow field (including the // table-only columns, which listRow tags json:"-" and so wouldn't survive a // direct marshal), the filter inputs, and the submit time. type cachedRun struct { RunID string `json:"run_id"` + TaskKey string `json:"task_key"` RunName string `json:"run_name"` User string `json:"user"` Status string `json:"status"` @@ -56,7 +63,7 @@ type cachedRun struct { func (c cachedRun) toRow() listRow { return listRow{ - RunID: c.RunID, RunName: c.RunName, User: c.User, Status: c.Status, + RunID: c.RunID, TaskKey: nilIfEmpty(c.TaskKey), RunName: c.RunName, User: c.User, Status: c.Status, StartedAt: c.StartedAt, IsSweep: c.IsSweep, ExperimentName: c.ExperimentName, DurationSeconds: c.DurationSeconds, AcceleratorType: c.AcceleratorType, NumAccelerators: c.NumAccelerators, DashboardURL: c.DashboardURL, Experiment: c.Experiment, Duration: c.Duration, MLflowURL: c.MLflowURL, MLflowLabel: c.MLflowLabel, @@ -66,7 +73,7 @@ func (c cachedRun) toRow() listRow { func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun { return cachedRun{ - RunID: r.RunID, RunName: r.RunName, User: r.User, Status: r.Status, + RunID: r.RunID, TaskKey: r.taskKey(), RunName: r.RunName, User: r.User, Status: r.Status, StartedAt: r.StartedAt, IsSweep: r.IsSweep, ExperimentName: r.ExperimentName, DurationSeconds: r.DurationSeconds, AcceleratorType: r.AcceleratorType, NumAccelerators: r.NumAccelerators, DashboardURL: r.DashboardURL, Experiment: r.Experiment, Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel, @@ -81,27 +88,49 @@ func newListCache(ctx context.Context) *cache.Cache { return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil) } -// cachedRow returns the cached row, filter fields, and task runs for a run. -// Entries written before task runs were cached are treated as misses. -func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, []jobs.RunTask, bool) { - entry, ok := cache.Get[cachedRun](ctx, c, listCacheKey{Host: host, RunID: runID}) - if !ok || len(entry.Tasks) == 0 { - return listRow{}, filterFields{}, nil, false +// cachedEntries returns the cached entries of a run, one per AI runtime task. +// An entry whose row lacks task runs is treated as a miss. +func cachedEntries(ctx context.Context, c *cache.Cache, host string, runID int64) ([]taskEntry, bool) { + cached, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) + if !ok || len(cached.Rows) == 0 { + return nil, false + } + entries := make([]taskEntry, len(cached.Rows)) + for i, row := range cached.Rows { + if len(row.Tasks) == 0 { + return nil, false + } + entries[i] = taskEntry{ + entry: listedRun{row: row.toRow(), runID: runID, taskKey: row.TaskKey, tasks: row.Tasks, terminal: true}, + fields: row.Fields, + } } - return entry.toRow(), entry.Fields, entry.Tasks, true + return entries, true } -// putRow caches a terminal run's base row and filter fields under its submit time. -func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { - cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) +// putEntries caches a terminal run's entries, all of them, so a later hit can +// be re-filtered on every task. +func putEntries(ctx context.Context, c *cache.Cache, host string, runID int64, entries []taskEntry, submitTimeMs int64) { + rows := make([]cachedRun, len(entries)) + for i, e := range entries { + rows[i] = cachedRunFromRow(e.entry.row, e.entry.tasks, e.fields, submitTimeMs) + } + cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRuns{Rows: rows}) } -// updateCachedRow replaces the display row while preserving cache metadata. -func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) { +// updateCachedRow replaces the display row of the run's task while preserving +// cache metadata. A run or task that is not cached is left alone. +func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, taskKey string, row listRow) { key := listCacheKey{Host: host, RunID: runID} - entry, ok := cache.Get[cachedRun](ctx, c, key) + entry, ok := cache.Get[cachedRuns](ctx, c, key) if !ok { return } - cache.Put(ctx, c, key, cachedRunFromRow(row, entry.Tasks, entry.Fields, entry.SubmitTimeMs)) + for i, cached := range entry.Rows { + if cached.TaskKey == taskKey { + entry.Rows[i] = cachedRunFromRow(row, cached.Tasks, cached.Fields, cached.SubmitTimeMs) + cache.Put(ctx, c, key, entry) + return + } + } } diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 00c821ac010..c4a543b90d4 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -1,6 +1,7 @@ package aircmd import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -12,39 +13,84 @@ import ( "github.com/stretchr/testify/require" ) +// putRow caches a single-row run, the common case in these tests. +func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { + putEntries(ctx, c, host, runID, []taskEntry{{entry: listedRun{row: row, tasks: tasks}, fields: fields}}, submitTimeMs) +} + +// rawCached reads the persisted value for a run, to check what was stored. +func rawCached(t *testing.T, ctx context.Context, c *cache.Cache, host string, runID int64) cachedRuns { + t.Helper() + got, ok := cache.Get[cachedRuns](ctx, c, listCacheKey{Host: host, RunID: runID}) + require.True(t, ok) + return got +} + func TestListCacheRoundTrip(t *testing.T) { t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) ctx := t.Context() c := newListCache(ctx) - _, _, _, ok := cachedRow(ctx, c, "https://host.test", 42) + _, ok := cachedEntries(ctx, c, "https://host.test", 42) require.False(t, ok, "miss before write") row := listRow{ - RunID: "42", Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), + RunID: "42", TaskKey: new("train"), Status: "SUCCESS", ExperimentName: new("exp"), DurationSeconds: new(int64(12)), AcceleratorType: new("GPU_1xA10"), NumAccelerators: new(1), DashboardURL: "https://host.test/jobs/runs/42?o=1", Experiment: "exp", } fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1} tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}} putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields) - got, gotFields, gotTasks, ok := cachedRow(ctx, c, "https://host.test", 42) + got, ok := cachedEntries(ctx, c, "https://host.test", 42) require.True(t, ok, "hit after write") - assert.Equal(t, row, got) - assert.Equal(t, fields, gotFields) + require.Len(t, got, 1) + assert.Equal(t, row, got[0].entry.row) + assert.Equal(t, fields, got[0].fields) + assert.Equal(t, int64(42), got[0].entry.runID) + assert.True(t, got[0].entry.terminal) + assert.Equal(t, int64(1700000000000), rawCached(t, ctx, c, "https://host.test", 42).Rows[0].SubmitTimeMs) + gotTasks := got[0].entry.tasks for i := range gotTasks { gotTasks[i].ForceSendFields = nil // set by the JSON round trip } assert.Equal(t, tasks, gotTasks) // Different host is a different key. - _, _, _, ok = cachedRow(ctx, c, "https://other.test", 42) + _, ok = cachedEntries(ctx, c, "https://other.test", 42) assert.False(t, ok) - // Entries from previous cache formats have no task runs and must be refreshed. + // Entries from previous cache formats have no rows and must be refreshed. cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430}) - _, _, _, ok = cachedRow(ctx, c, "https://host.test", 43) + _, ok = cachedEntries(ctx, c, "https://host.test", 43) assert.False(t, ok) + cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 44}, cachedRuns{Rows: []cachedRun{{RunID: "44"}}}) + _, ok = cachedEntries(ctx, c, "https://host.test", 44) + assert.False(t, ok, "a row without task runs is a miss") +} + +func TestListCacheKeepsOneRowPerAirTask(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + ctx := t.Context() + c := newListCache(ctx) + host := "https://host.test" + + putEntries(ctx, c, host, 42, []taskEntry{ + {entry: listedRun{row: listRow{RunID: "42", TaskKey: new("train"), Experiment: "exp-a"}, tasks: []jobs.RunTask{{RunId: 420, TaskKey: "train"}}}, fields: filterFields{Experiment: "exp-a"}}, + {entry: listedRun{row: listRow{RunID: "42", TaskKey: new("eval"), Experiment: "exp-b"}, tasks: []jobs.RunTask{{RunId: 421, TaskKey: "eval"}}}, fields: filterFields{Experiment: "exp-b"}}, + }, 1000) + + // The MLflow link of one task updates that row alone. + updateCachedRow(ctx, c, host, 42, "eval", listRow{RunID: "42", TaskKey: new("eval"), Experiment: "exp-b", MLflowURL: "https://host.test/ml/x"}) + updateCachedRow(ctx, c, host, 42, "missing", listRow{RunID: "42", TaskKey: new("missing")}) + + got := rawCached(t, ctx, c, host, 42).Rows + require.Len(t, got, 2) + assert.Equal(t, "train", got[0].TaskKey) + assert.Empty(t, got[0].MLflowURL) + assert.Equal(t, "eval", got[1].TaskKey) + assert.Equal(t, "https://host.test/ml/x", got[1].MLflowURL) + assert.Equal(t, int64(1000), got[1].SubmitTimeMs, "cache metadata is preserved") } func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) { diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 5f77665f5ca..5c4015c28e8 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -12,8 +12,9 @@ import ( ) // runDetailText fetches a run and renders the same styled view as `air get` into -// a string, for the list picker's detail pane. -func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { +// a string, for the list picker's detail pane. taskKey selects the AI runtime +// task shown, as `air get --task-key` does; "" means the first. +func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int64, taskKey string) (string, error) { run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { @@ -21,8 +22,11 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } return "", fmt.Errorf("failed to fetch run: %w", err) } + run, data, err := taskView(run, taskKey) + if err != nil { + return "", err + } - data := buildGetData(run) // As in `air get`: an ai_runtime_task run keeps its experiment, accelerators // and config path on the task. if genAIComputeTask(run) == nil { @@ -44,8 +48,10 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } // runLogsSnapshot fetches a one-shot tail of a run's logs into a string, for the -// list picker's detail pane, followed by the run's MLflow logs link. -func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { +// list picker's detail pane, followed by the run's MLflow logs link. taskKey is +// the selected row's task; the pane notes when the logs belong to another task +// (see logTaskNote). +func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64, taskKey string) (string, error) { run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { @@ -63,6 +69,10 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i } var buf bytes.Buffer + if note := logTaskNote(run, taskKey); note != "" { + fmt.Fprintln(&buf, note) + fmt.Fprintln(&buf) + } if _, err := fetchLogs(ctx, w, &buf, req, status); err != nil { return "", fmt.Errorf("failed to fetch logs: %w", err) } @@ -71,3 +81,24 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i } return buf.String(), nil } + +// logTaskNote returns a note for the logs pane when the logs it shows are not +// those of taskKey, the selected row's task: logs are read at run level from +// the run's latest task (see resolveLogAttempt), so on a run with several AI +// runtime tasks they can belong to another one. It returns "" when they match +// or no task was selected. +func logTaskNote(run *jobs.Run, taskKey string) string { + if taskKey == "" { + return "" + } + _, taskRunID, err := resolveLogAttempt(run, -1) + if err != nil { + return "" + } + for _, t := range run.Tasks { + if t.RunId == taskRunID && t.TaskKey != taskKey { + return fmt.Sprintf("Note: these are the logs of task %q, the run's latest task, not of the selected task %q. Logs are not yet available per task.", t.TaskKey, taskKey) + } + } + return "" +} diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index b53d492bc6c..6c39ca907be 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -33,7 +33,7 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { })) t.Cleanup(srv.Close) - got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ "Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+ @@ -83,9 +83,9 @@ func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { currentUser: selfUser, limit: 10, fetchMLflow: true, }).next(10) require.NoError(t, err) - info, err := runDetailText(ctx, w, 7) + info, err := runDetailText(ctx, w, 7, "") require.NoError(t, err) - logs, err := runLogsSnapshot(ctx, w, 7) + logs, err := runLogsSnapshot(ctx, w, 7, "") require.NoError(t, err) // Then each links retry 1's MLflow run and names that retry. @@ -135,7 +135,7 @@ func TestUnnamedOlderRetryLabelFitsTheListColumnAndIsWholeInTheInfoPane(t *testi currentUser: selfUser, limit: 10, fetchMLflow: true, }).next(10) require.NoError(t, err) - info, err := runDetailText(ctx, w, 7) + info, err := runDetailText(ctx, w, 7, "") require.NoError(t, err) // Then the list row keeps the end of the ID and the whole retry note within @@ -184,7 +184,7 @@ func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { t.Cleanup(srv.Close) // When the list's info pane renders it. - got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) // Then it shows the same config box and limit as `air get`. @@ -214,7 +214,7 @@ func TestRunDetailTextShowsEnvironmentAndLegacyConfig(t *testing.T) { })) t.Cleanup(srv.Close) - got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123, "") require.NoError(t, err) assert.Contains(t, got, "command: echo legacy-command") assert.Regexp(t, `Environment\s+4`, got) @@ -253,7 +253,7 @@ func TestRunDetailTextShowsAiRuntimeTaskFields(t *testing.T) { t.Cleanup(srv.Close) ctx := cmdio.MockDiscard(t.Context()) - info, err := runDetailText(ctx, newTestWorkspaceClient(t, srv.URL), 7) + info, err := runDetailText(ctx, newTestWorkspaceClient(t, srv.URL), 7, "") require.NoError(t, err) // The pane shows what the list row and `air get` show, with the experiment @@ -263,3 +263,42 @@ func TestRunDetailTextShowsAiRuntimeTaskFields(t *testing.T) { assert.Contains(t, info, "Accelerators 1x A10") assert.Contains(t, info, "command: python train.py") } + +func TestRunDetailTextShowsTheSelectedAirTask(t *testing.T) { + run := jobs.Run{RunId: 7, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateSuccess}, Tasks: []jobs.RunTask{ + notebookTask("prep", 71), + aiRuntimeTask("train", 72, "/Users/me@example.com/exp-train", "GPU_1xA10", 1), + aiRuntimeTask("eval", 73, "/Users/me@example.com/exp-eval", "GPU_8xH100", 8), + }} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _ = json.NewEncoder(w).Encode(run) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + ctx := cmdio.MockDiscard(t.Context()) + w := newTestWorkspaceClient(t, srv.URL) + + info, err := runDetailText(ctx, w, 7, "") + require.NoError(t, err) + assert.Contains(t, info, "Experiment exp-train", "the default is the first AIR task, not the notebook") + assert.Contains(t, info, "Task train") + + info, err = runDetailText(ctx, w, 7, "eval") + require.NoError(t, err) + assert.Contains(t, info, "Experiment exp-eval") + assert.Contains(t, info, "Accelerators 8x H100") + assert.Contains(t, info, "Task eval") +} + +func TestLogTaskNote(t *testing.T) { + run := &jobs.Run{RunId: 7, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 71, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 72, "e", "GPU_1xA10", 1), + }} + assert.Equal(t, `Note: these are the logs of task "eval", the run's latest task, not of the selected task "train". Logs are not yet available per task.`, logTaskNote(run, "train")) + assert.Empty(t, logTaskNote(run, "eval"), "the logs are the selected task's") + assert.Empty(t, logTaskNote(run, ""), "no task selected") +} diff --git a/cmd/air/list_format.go b/cmd/air/list_format.go index cda11b49a58..ba2ca349a70 100644 --- a/cmd/air/list_format.go +++ b/cmd/air/list_format.go @@ -6,10 +6,77 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" ) -// buildListRow extracts the columns shown for one run. Optional cells fall back -// to "-"; MLflowURL starts as "-" and setMLflowLinks fills it in for text output. -// host and workspaceID are used for building dashboard URLs. +// taskEntry is one task's list entry with the task-filter inputs it was built +// from, before any filter is applied. +type taskEntry struct { + entry listedRun + fields filterFields +} + +// runGroup is one run's list entries that pass the filters: one per AI runtime +// task, in task order (a run without one has a single entry). Every list path +// passes runs along as groups, so --limit counts runs, sorting keeps a run's +// rows together, and cancel and the cache act on the whole run. +type runGroup struct { + runID int64 + entries []listedRun +} + +// buildTaskEntries builds one entry per AI runtime task of the run, in task +// order, each read from that task's own task runs (see taskRun). A run without +// an AI runtime task yields a single entry read from the run as is. +func buildTaskEntries(run *jobs.Run, host string, workspaceID int64) []taskEntry { + keys := airTaskKeys(run) + if len(keys) == 0 { + return []taskEntry{taskEntryFor(run, host, workspaceID)} + } + entries := make([]taskEntry, 0, len(keys)) + for _, key := range keys { + entries = append(entries, taskEntryFor(taskRun(run, key), host, workspaceID)) + } + return entries +} + +// taskEntryFor builds the entry for a run narrowed to one task (or a run +// without an AI runtime task). +func taskEntryFor(run *jobs.Run, host string, workspaceID int64) taskEntry { + row := buildListRow(run, host, workspaceID) + return taskEntry{ + entry: listedRun{row: row, runID: run.RunId, taskKey: airTaskKey(run), tasks: retryTasks(run), terminal: isTerminal(run)}, + fields: filterFieldsFromRun(run), + } +} + +// matchingGroup returns the run's entries that pass f, each filtered on its own +// task, or ok false when none do. When the run has several AI runtime tasks, +// every row is marked to show its task, even if the filter leaves one. +func matchingGroup(runID int64, all []taskEntry, f listFilters) (runGroup, bool) { + g := runGroup{runID: runID} + for _, e := range all { + if f.matchesFields(e.fields) { + entry := e.entry + entry.row.ShowTask = len(all) > 1 + g.entries = append(g.entries, entry) + } + } + return g, len(g.entries) > 0 +} + +// rows returns the group's display rows. +func (g runGroup) rows() []listRow { + rows := make([]listRow, len(g.entries)) + for i, e := range g.entries { + rows[i] = e.row + } + return rows +} + +// buildListRow extracts the columns shown for one run, read from its first AI +// runtime task. Optional cells fall back to "-"; MLflowURL starts as "-" and +// setMLflowLinks fills it in for text output. host and workspaceID are used for +// building dashboard URLs. func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { + taskKey := airTaskKey(run) experiment := jobExperiment(run) experimentName := nilIfEmpty(experiment) if experiment == "" { @@ -38,6 +105,7 @@ func buildListRow(run *jobs.Run, host string, workspaceID int64) listRow { return listRow{ RunID: strconv.FormatInt(run.RunId, 10), + TaskKey: nilIfEmpty(taskKey), RunName: run.RunName, User: run.CreatorUserName, Status: displayRunStatus(run), diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index d9cf6a568bc..d4f03e9069c 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -60,7 +60,7 @@ func (s *indexStrategy) load() error { return nil } -func (s *indexStrategy) next(want int) ([]listedRun, error) { +func (s *indexStrategy) next(want int) ([]runGroup, error) { if !s.loaded { if err := s.load(); err != nil { return nil, err @@ -70,20 +70,22 @@ func (s *indexStrategy) next(want int) ([]listedRun, error) { want = min(want, s.limit-s.returned) } - var entries []listedRun - for len(entries) < want && s.pos < len(s.ids) { - end := min(s.pos+want-len(entries), len(s.ids)) + // want counts runs, as on the scan path: a run with several AI runtime + // tasks hydrates into one group of rows. + var groups []runGroup + for len(groups) < want && s.pos < len(s.ids) { + end := min(s.pos+want-len(groups), len(s.ids)) batch := s.ids[s.pos:end] s.pos = end - rows, err := s.hydrate(batch) + hydrated, err := s.hydrate(batch) if err != nil { return nil, err } - entries = append(entries, rows...) + groups = append(groups, hydrated...) } - s.returned += len(entries) - return entries, nil + s.returned += len(groups) + return groups, nil } func (s *indexStrategy) done() bool { @@ -95,19 +97,19 @@ func (s *indexStrategy) truncated() bool { return false } // hydrate turns a batch of run ids into rows, serving cached terminal rows // without a network call and fetching the rest via runs/get. Freshly hydrated -// terminal runs are cached. Results keep the input (newest-first) order, then -// the batch is re-sorted by start time since concurrent hydration reorders it. -func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { +// terminal runs are cached. Results keep the input order: the index's, +// newest submission first. +func (s *indexStrategy) hydrate(ids []int64) ([]runGroup, error) { host := s.w.Config.Host - rows := make([]listedRun, 0, len(ids)) + groups := make([]runGroup, 0, len(ids)) var toFetch []int64 - // The cache key excludes the filter, so a cached row is still run through the + // The cache key excludes the filter, so a cached run is still run through the // active filter; a non-matching hit is dropped, not re-fetched. for _, id := range ids { - if row, fields, tasks, ok := cachedRow(s.ctx, s.cache, host, id); ok { - if s.filters.matchesFields(fields) { - rows = append(rows, listedRun{row: row, runID: id, tasks: tasks, terminal: true}) + if entries, ok := cachedEntries(s.ctx, s.cache, host, id); ok { + if g, ok := matchingGroup(id, entries, s.filters); ok { + groups = append(groups, g) } continue } @@ -119,25 +121,28 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { return nil, err } for _, run := range runs { - fields := filterFieldsFromRun(run) - if !s.filters.matchesFields(fields) { - continue - } - row := buildListRow(run, s.host, s.workspaceID) - tasks := retryTasks(run) - rows = append(rows, listedRun{row: row, runID: run.RunId, tasks: tasks, terminal: isTerminal(run)}) + entries := buildTaskEntries(run, s.host, s.workspaceID) if isTerminal(run) { - start, _ := jobTiming(run) - putRow(s.ctx, s.cache, host, run.RunId, tasks, start, row, fields) + start, _ := jobTiming(defaultTaskRun(run)) + putEntries(s.ctx, s.cache, host, run.RunId, entries, start) + } + if g, ok := matchingGroup(run.RunId, entries, s.filters); ok { + groups = append(groups, g) } } - // Concurrent hydration reorders runs, so re-sort the batch newest-first. The - // ISO start timestamp sorts lexicographically; a missing time ("") sorts last. - slices.SortStableFunc(rows, func(a, b listedRun) int { - return cmp.Compare(rowStartKey(b.row), rowStartKey(a.row)) + // Cache hits come first and concurrent hydration reorders the rest, so put + // the batch back in the index's order, keeping a run's rows together. It is + // the order runs were submitted in, which a task that hasn't started or was + // skipped (and so has no start time, see taskRun) can't disturb. + pos := make(map[int64]int, len(ids)) + for i, id := range ids { + pos[id] = i + } + slices.SortStableFunc(groups, func(a, b runGroup) int { + return cmp.Compare(pos[a.runID], pos[b.runID]) }) - return rows, nil + return groups, nil } // cacheMLflowLinks writes successful MLflow enrichment back to terminal rows @@ -145,16 +150,7 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { func (s *indexStrategy) cacheMLflowLinks(entries []listedRun) { for _, entry := range entries { if entry.row.MLflowURL != "" && entry.row.MLflowURL != "-" { - updateCachedRow(s.ctx, s.cache, s.host, entry.runID, entry.row) + updateCachedRow(s.ctx, s.cache, s.host, entry.runID, entry.taskKey, entry.row) } } } - -// rowStartKey returns a row's ISO start timestamp for ordering, or "" when the -// run hasn't started (which sorts last under descending comparison). -func rowStartKey(r listRow) string { - if r.StartedAt == nil { - return "" - } - return *r.StartedAt -} diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index c7495f2be66..955aea31e1a 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -382,3 +382,105 @@ func TestListSelfUserFilterUsesIndex(t *testing.T) { assert.Contains(t, stdout.String(), `"run_id": "7"`) assert.Empty(t, warnings.String()) } + +func TestIndexStrategyKeepsARunsRowsTogether(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 7's two AI Runtime tasks start before and after run 8. Sorting rows + // would split run 7 around run 8; sorting runs keeps its rows together, and + // --limit counts it once. + multi := indexRun(7, 3000_000) + train, eval := multi.Tasks[0], multi.Tasks[0] + train.TaskKey, train.RunId, train.StartTime = "train", 70, 1000_000 + eval.TaskKey, eval.RunId, eval.StartTime = "eval", 71, 3000_000 + multi.Tasks = []jobs.RunTask{train, eval} + refs := []workflowRef{{jobRunID: 7, submitTimeMs: 1000_000}, {jobRunID: 8, submitTimeMs: 2000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{7: multi, 8: indexRun(8, 2000_000)}, nil, nil) + + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{currentUser: selfUser, limit: 10}) + runs, err := f.nextRuns(10) + require.NoError(t, err) + got := make([][]string, len(runs)) + for i, run := range runs { + for _, r := range run { + got[i] = append(got[i], r.RunID+"/"+r.taskKey()) + } + } + assert.Equal(t, [][]string{{"8/"}, {"7/train", "7/eval"}}, got) +} + +func TestIndexStrategyKeepsIndexOrderForARunWithNoStartedTask(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 9, the newest submission, failed in prep, so its AI Runtime task was + // skipped and has no start time. It must still list first, both on the first + // fetch and once it is served from the cache. + skipped := indexRun(9, 0) + skipped.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + train := skipped.Tasks[0] + train.TaskKey, train.RunId, train.StartTime, train.EndTime = "train", 91, 0, 0 + train.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateSkipped, ResultState: jobs.RunResultStateUpstreamFailed} + prep := notebookTask("prep", 90) + prep.StartTime, prep.EndTime = 3000_000, 3001_000 + prep.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated, ResultState: jobs.RunResultStateFailed} + skipped.Tasks = []jobs.RunTask{prep, train} + refs := []workflowRef{{jobRunID: 8, submitTimeMs: 2000_000}, {jobRunID: 9, submitTimeMs: 3000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{9: skipped, 8: indexRun(8, 2000_000)}, nil, nil) + + for _, pass := range []string{"fresh", "cached"} { + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{currentUser: selfUser, limit: 10}) + rows, err := f.next(10) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + assert.Equal(t, []string{"9", "8"}, ids, pass) + assert.Nil(t, rows[0].StartedAt, "%s: the skipped task reports no start", pass) + } +} + +// indexOrder fetches one page through the index path and returns its run IDs. +func indexOrder(t *testing.T, url string) []string { + t.Helper() + f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, url), listQuery{currentUser: selfUser, limit: 10}) + rows, err := f.next(10) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + return ids +} + +func TestIndexStrategyListsANewActiveRunAboveCachedOlderOnes(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Runs 8 and 9 are terminal, so the first fetch caches them; run 10 is still + // running, so it is fetched every time. Cache hits are collected before the + // fetched runs, and the batch must still come back newest submission first. + active := indexRun(10, 4000_000) + active.State = &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning} + active.Tasks[0].EndTime = 0 + refs := []workflowRef{{jobRunID: 8, submitTimeMs: 2000_000}, {jobRunID: 9, submitTimeMs: 3000_000}, {jobRunID: 10, submitTimeMs: 4000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{8: indexRun(8, 2000_000), 9: indexRun(9, 3000_000), 10: active}, nil, nil) + + assert.Equal(t, []string{"10", "9", "8"}, indexOrder(t, srv.URL), "fresh") + assert.Equal(t, []string{"10", "9", "8"}, indexOrder(t, srv.URL), "8 and 9 from the cache") +} + +func TestIndexStrategyListsARetriedRunBySubmission(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // Run 7 was submitted before run 8 but its retry started after run 8. The + // index pages runs by submission time, so a batch keeps that order rather + // than moving run 7 up by its latest attempt's start. + retried := indexRun(7, 1000_000) + second := retried.Tasks[0] + second.RunId, second.AttemptNumber, second.StartTime, second.EndTime = 71, 1, 5000_000, 5001_000 + retried.Tasks = append(retried.Tasks, second) + refs := []workflowRef{{jobRunID: 7, submitTimeMs: 1000_000}, {jobRunID: 8, submitTimeMs: 2000_000}} + srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{7: retried, 8: indexRun(8, 2000_000)}, nil, nil) + + assert.Equal(t, []string{"8", "7"}, indexOrder(t, srv.URL)) +} diff --git a/cmd/air/list_test.go b/cmd/air/list_test.go index f3af7fd9aa1..847d1368a92 100644 --- a/cmd/air/list_test.go +++ b/cmd/air/list_test.go @@ -242,7 +242,7 @@ func TestBuildListRow(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"123","run_name":"run-123","user":"me@example.com","status":"SUCCESS", + "run_id":"123","task_key":null,"run_name":"run-123","user":"me@example.com","status":"SUCCESS", "started_at":"2023-11-14T22:13:20+00:00","is_sweep":false, "experiment_name":"exp","duration_seconds":12, "accelerator_type":"GPU_8xH100","num_accelerators":8, @@ -276,7 +276,7 @@ func TestBuildListRowWaitingForComputeStatus(t *testing.T) { assert.Equal(t, "PENDING", row.Status) assert.Equal(t, listRowStatus(row), row.Status) assert.JSONEq(t, `{ - "run_id":"123","run_name":"run-123","user":"me@example.com","status":"PENDING", + "run_id":"123","task_key":null,"run_name":"run-123","user":"me@example.com","status":"PENDING", "started_at":null,"is_sweep":false,"experiment_name":"exp","duration_seconds":null, "accelerator_type":"GPU_8xH100","num_accelerators":8, "dashboard_url":"https://example.test/jobs/runs/123?o=456" @@ -298,7 +298,7 @@ func TestBuildListRowMissingValues(t *testing.T) { body, err := json.Marshal(row) require.NoError(t, err) assert.JSONEq(t, `{ - "run_id":"7","run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, + "run_id":"7","task_key":null,"run_name":"","user":"","status":"UNKNOWN","started_at":null,"is_sweep":false, "experiment_name":null,"duration_seconds":null,"accelerator_type":null,"num_accelerators":null, "dashboard_url":"https://example.test/jobs/runs/7" }`, string(body)) @@ -502,3 +502,125 @@ func TestListValidatesAcceleratorOnlyAfterNoMatches(t *testing.T) { }) } } + +func TestBuildTaskEntriesOnePerAirTask(t *testing.T) { + run := jobs.Run{RunId: 7, RunName: "dag", CreatorUserName: "me@example.com", Tasks: []jobs.RunTask{ + notebookTask("prep", 71), + aiRuntimeTask("train", 72, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("eval", 73, "/Users/me@example.com/exp-b", "GPU_8xH100", 8), + }} + + rows := buildTaskEntries(&run, "https://example.test", 0) + require.Len(t, rows, 2) + assert.Equal(t, "7", rows[0].entry.row.RunID) + assert.Equal(t, "train", rows[0].entry.row.taskKey()) + assert.Equal(t, "exp-a", rows[0].entry.row.Experiment) + assert.Equal(t, "1x A10", rows[0].entry.row.Accelerators) + assert.Equal(t, []int64{72}, taskRunIDs(rows[0].entry.tasks), "the row's MLflow lookup sees only its own task runs") + assert.Equal(t, "7", rows[1].entry.row.RunID) + assert.Equal(t, "eval", rows[1].entry.row.taskKey()) + assert.Equal(t, "exp-b", rows[1].entry.row.Experiment) + assert.Equal(t, "8x H100", rows[1].entry.row.Accelerators) + assert.Equal(t, []int64{73}, taskRunIDs(rows[1].entry.tasks)) +} + +func taskRunIDs(tasks []jobs.RunTask) []int64 { + ids := make([]int64, len(tasks)) + for i, t := range tasks { + ids[i] = t.RunId + } + return ids +} + +func TestListAirRunsAirTaskAfterNotebook(t *testing.T) { + // The official multi-task DAB example: a notebook prep task, then the + // ai_runtime_task that depends on it. + dag := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/air-byot-multitask-dag") + dag.Tasks[0].TaskKey, dag.Tasks[0].RunId = "train", 12 + dag.Tasks = []jobs.RunTask{notebookTask("prep", 11), dag.Tasks[0]} + srv := runsServer(t, runsListBody(t, "", dag)) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, "1", rows[0].RunID) + assert.Equal(t, "train", rows[0].taskKey()) + assert.Equal(t, "air-byot-multitask-dag", rows[0].Experiment) + assert.Equal(t, "1x A10", rows[0].Accelerators) +} + +func TestListAirRunsOneRowPerAirTask(t *testing.T) { + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_8xH100", 8), + } + srv := runsServer(t, runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xH100", 1, "/Users/me@example.com/exp-c"))) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ + activeOnly: true, + filters: listFilters{Experiment: "exp-b"}, + }).next(10) + require.NoError(t, err) + require.Len(t, rows, 1, "the task filter applies per AIR task") + assert.Equal(t, "1", rows[0].RunID) + assert.Equal(t, "b", rows[0].taskKey()) + + rows, err = newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 3) + assert.Equal(t, []string{"1", "1", "2"}, []string{rows[0].RunID, rows[1].RunID, rows[2].RunID}) + assert.Equal(t, []string{"a", "b", ""}, []string{rows[0].taskKey(), rows[1].taskKey(), rows[2].taskKey()}) +} + +func TestListAirRunsLimitCountsRunsNotRows(t *testing.T) { + // The first run has two AI Runtime tasks, so it lists as two rows; --limit 1 + // still shows that whole run, and --limit 2 also shows the next run. + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + body := runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c")) + + for _, tt := range []struct { + limit int + want []string + }{ + {1, []string{"1", "1"}}, + {2, []string{"1", "1", "2"}}, + } { + srv := runsServer(t, body) + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(tt.limit) + require.NoError(t, err) + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.RunID + } + assert.Equal(t, tt.want, ids, "limit %d", tt.limit) + } +} + +func TestListAirRunsMarksTheRowsOfAMultiTaskRun(t *testing.T) { + two := airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-a") + two.Tasks = []jobs.RunTask{ + aiRuntimeTask("a", 11, "/Users/me@example.com/exp-a", "GPU_1xA10", 1), + aiRuntimeTask("b", 12, "/Users/me@example.com/exp-b", "GPU_1xA10", 1), + } + srv := runsServer(t, runsListBody(t, "", two, airBaseRun(2, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp-c"))) + + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{activeOnly: true}).next(10) + require.NoError(t, err) + require.Len(t, rows, 3) + assert.Equal(t, []bool{true, true, false}, []bool{rows[0].ShowTask, rows[1].ShowTask, rows[2].ShowTask}) + + // A filter that leaves one of the two tasks still names it. + rows, err = newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ + activeOnly: true, + filters: listFilters{Experiment: "exp-b"}, + }).next(10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.True(t, rows[0].ShowTask) + assert.Equal(t, "1 b", rows[0].runIDLabel()) +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 6798244c621..1faf358e23e 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -291,7 +291,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Run Details" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunDetail(runID) + return m, m.fetchRunDetail(runID, m.rows[m.cursor].taskKey()) } case "L", "l": // Open the logs snapshot pane; the fetch fills it in. @@ -300,7 +300,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailLoading = true m.detailTitle = "Logs Snapshot" runID, _ := parseRunID(m.rows[m.cursor].RunID) - return m, m.fetchRunLogs(runID) + return m, m.fetchRunLogs(runID, m.rows[m.cursor].taskKey()) } } m.offset = m.clampedOffset() @@ -440,12 +440,13 @@ func hiddenNote(hidden []string, names bool) string { } } -// fetchRunDetail returns a tea.Cmd that fetches run details in the background. -func (m listModel) fetchRunDetail(runID int64) tea.Cmd { +// fetchRunDetail returns a tea.Cmd that fetches the details of the run's task +// in the background. +func (m listModel) fetchRunDetail(runID int64, taskKey string) tea.Cmd { ctx := m.fetcher.ctx w := m.fetcher.w return func() tea.Msg { - body, err := runDetailText(ctx, w, runID) + body, err := runDetailText(ctx, w, runID, taskKey) if err != nil { return detailMsg{title: "Run Details", err: err} } @@ -454,11 +455,11 @@ func (m listModel) fetchRunDetail(runID int64) tea.Cmd { } // fetchRunLogs returns a tea.Cmd that fetches a log snapshot in the background. -func (m listModel) fetchRunLogs(runID int64) tea.Cmd { +func (m listModel) fetchRunLogs(runID int64, taskKey string) tea.Cmd { ctx := m.fetcher.ctx w := m.fetcher.w return func() tea.Msg { - body, err := runLogsSnapshot(ctx, w, runID) + body, err := runLogsSnapshot(ctx, w, runID, taskKey) if err != nil { return detailMsg{title: "Logs Snapshot", err: err} } diff --git a/cmd/air/list_tui_render.go b/cmd/air/list_tui_render.go index 98b75347f1e..c145087d830 100644 --- a/cmd/air/list_tui_render.go +++ b/cmd/air/list_tui_render.go @@ -89,7 +89,7 @@ func computeListCols(rows []listRow) listCols { accel: len("Accelerators"), } for _, r := range rows { - c.runID = max(c.runID, lipgloss.Width(r.RunID)) + c.runID = max(c.runID, lipgloss.Width(r.runIDLabel())) c.experiment = min(columnCap, max(c.experiment, lipgloss.Width(r.Experiment))) c.status = max(c.status, lipgloss.Width("● "+listRowStatus(r))) c.started = max(c.started, lipgloss.Width(startedDisplay(r))) @@ -283,7 +283,7 @@ func (s listStyles) renderRow(cols listCols, r listRow, selected, links bool) st cell := func(col listColumn) string { switch col { case columnRunID: - return s.cell(base, r.RunID, cols.runID, fg(colRunID), false, runIDLink != "", runIDLink) + return s.cell(base, r.runIDLabel(), cols.runID, fg(colRunID), false, runIDLink != "", runIDLink) case columnExperiment: return s.cell(base, r.Experiment, cols.experiment, fg(colN11), false, experimentLink != "", experimentLink) case columnStatus: diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index a33a0d92ab9..d774deda235 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -29,7 +29,8 @@ func validateNodeIndex(node, numNodes int) error { return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1) } -// resolveNodeCount returns how many nodes a run used. +// resolveNodeCount returns how many nodes a task run used: the task the logs +// come from (see logTaskRun), so the node indices match the logs. func resolveNodeCount(run *jobs.Run) (int, error) { accelType, count := jobCompute(run) if accelType == "" { @@ -54,12 +55,39 @@ func resolveNodeCount(run *jobs.Run) (int, error) { return count / perNode, nil } +// logTaskRun returns run narrowed to the AI runtime task whose task run is +// taskRunID, the one resolveLogAttempt picked for the logs. It falls back to the +// first AI runtime task (see defaultTaskRun) when that task run is not an AI +// runtime task. +func logTaskRun(run *jobs.Run, taskRunID int64) *jobs.Run { + for i := range run.Tasks { + if t := &run.Tasks[i]; t.RunId == taskRunID && isAirTask(t) { + return taskRun(run, t.TaskKey) + } + } + return defaultTaskRun(run) +} + +// nodeCountForRetry returns how many nodes the task whose logs are read for the +// requested retry used. +func nodeCountForRetry(run *jobs.Run, requestedAttempt int) (int, error) { + _, taskRunID, err := resolveLogAttempt(run, requestedAttempt) + if err != nil { + return 0, err + } + return resolveNodeCount(logTaskRun(run, taskRunID)) +} + // downloadLogs writes each node's logs to /logs/node_.log and // prints a summary. An explicit --node downloads only that node; otherwise all of // them. Logs come from MLflow artifacts, since Bricklens only streams. The // returned bool is the run's outcome, so the exit code matches the streaming path. func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) { - numNodes, err := resolveNodeCount(run) + attempt, taskRunID, err := resolveLogAttempt(run, req.attempt) + if err != nil { + return false, err + } + numNodes, err := resolveNodeCount(logTaskRun(run, taskRunID)) if err != nil { return false, err } @@ -75,10 +103,6 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri // A run with no logs is reported the same way as on the streaming path, so // the message and exit code agree between them. - attempt, taskRunID, err := resolveLogAttempt(run, req.attempt) - if err != nil { - return false, err - } ids := mlflowIDsForTask(ctx, w, taskRunID) if ids == nil || ids.RunID == "" { emitNoLogs(out, req, status) diff --git a/cmd/air/logdownload_test.go b/cmd/air/logdownload_test.go index 63c307b2f58..c1464f2b6e3 100644 --- a/cmd/air/logdownload_test.go +++ b/cmd/air/logdownload_test.go @@ -514,3 +514,16 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) { assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)") assert.NotContains(t, stdout.String(), "No logs available") } + +func TestResolveNodeCountReadsTheTaskTheLogsComeFrom(t *testing.T) { + // A 1-node train before a 2-node eval: the logs come from eval, the last + // task, so the node count must too. + run := &jobs.Run{RunId: 1, Tasks: []jobs.RunTask{ + aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1), + aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 2), + }} + n, err := nodeCountForRetry(run, -1) + require.NoError(t, err) + assert.Equal(t, 2, n) + assert.NoError(t, validateNodeIndex(1, n), "--node 1 is a valid eval node") +} diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 2a6d83f2e4c..72d4ab62df5 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -160,7 +160,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { status := projectRunStatus(run) if req.nodeSet { - if numNodes, err := resolveNodeCount(run); err == nil { + if numNodes, err := nodeCountForRetry(run, req.attempt); err == nil { if err := validateNodeIndex(req.node, numNodes); err != nil { return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) } diff --git a/cmd/air/render.go b/cmd/air/render.go index b8f0043d906..3adf063489a 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -75,6 +75,9 @@ func newPalette(r *lipgloss.Renderer) palette { type runView struct { runID string dashboardURL string + // task is the selected AI runtime task's key, shown only when the run has + // several AI runtime tasks to tell apart. + task string status string submitted string retries int @@ -109,6 +112,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl view := runView{ runID: data.RunID, dashboardURL: data.DashboardURL, + task: data.TaskDisplay, status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, @@ -162,21 +166,23 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } } -// genAIComputeTask returns the run's first GenAI-compute task, or nil. +// genAIComputeTask returns the run's first AI runtime task's GenAI-compute +// task, or nil. func genAIComputeTask(run *jobs.Run) *jobs.GenAiComputeTask { - if len(run.Tasks) == 0 { + t := firstTask(run) + if t == nil { return nil } - return run.Tasks[0].GenAiComputeTask + return t.GenAiComputeTask } -// aiRuntimeEnvironment returns the serverless environment an ai_runtime run used, -// read from the Jobs GetRun response's environments[] entry keyed by -// aiRuntimeEnvironmentKey: a version (e.g. "4") or, for Databricks AI runtimes, a -// base environment. The typed SDK Run has no environments field, so the value is +// aiRuntimeEnvironment returns the serverless environment an ai_runtime task +// used, read from the Jobs GetRun response's environments[] entry keyed by +// envKey (see taskEnvironmentKey): a version (e.g. "4") or, for Databricks AI +// runtimes, a base environment. The typed SDK Run has no environments field, so the value is // fetched with a raw request. Best-effort: returns empty strings (logged) on any // error or when the run declares no environment. -func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (version, baseEnvironment string) { +func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64, envKey string) (version, baseEnvironment string) { apiClient, err := client.New(w.Config) if err != nil { enrichmentWarnf(ctx, "air get: could not create client to read environment: %v", err) @@ -199,13 +205,24 @@ func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, ru return "", "" } for _, e := range resp.Environments { - if e.EnvironmentKey == aiRuntimeEnvironmentKey { + if e.EnvironmentKey == envKey { return e.Spec.EnvironmentVersion, e.Spec.BaseEnvironment } } return "", "" } +// taskEnvironmentKey returns the environment key of the run's lead task (the +// selected AI runtime task once the run is narrowed, see taskRun; a sweep's +// iterated task, see firstTask). It falls back to aiRuntimeEnvironmentKey, the +// key `air run` submits, when the task names none. +func taskEnvironmentKey(run *jobs.Run) string { + if t := firstTask(run); t != nil && t.EnvironmentKey != "" { + return t.EnvironmentKey + } + return aiRuntimeEnvironmentKey +} + // environmentLabel is the text shown for the Environment cell: the environment // version, else the ID of a workspace base environment, or "" for neither. // buildRunEnvironment submits a databricks_ai_v* image as a base environment @@ -447,12 +464,17 @@ func renderFields(p palette, colorOn bool, v runView) string { status := statusStyle(p, v.status).Render("● " + v.status) rows := []string{ field(p, "Run ID", link(colorOn, p.blue, v.runID, v.dashboardURL)), + } + if v.task != "" { + rows = append(rows, field(p, "Task", p.n12.Render(stripControl(v.task)))) + } + rows = append(rows, field(p, "Status", status), field(p, "Submitted", p.n12.Render(v.submitted)), field(p, "Retries", p.n12.Render(strconv.Itoa(v.retries))), field(p, "Max Retries", p.n12.Render(v.maxRetries)), field(p, "Duration", p.n12.Render(v.duration)), - } + ) // Progress is shown only for a running run we could estimate; amber marks it // as live, in-progress information. if v.progress != "" { diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index db1af3bfb4b..a01967a2830 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -106,7 +106,7 @@ func TestAiRuntimeEnvironment(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { srv := runGetServer(t, `{"run_id": 5, "environments": [{"environment_key": "default", "spec": `+tt.spec+`}]}`) - version, base := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5) + version, base := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5, aiRuntimeEnvironmentKey) assert.Equal(t, tt.wantVersion, version) assert.Equal(t, tt.wantBase, base) assert.Equal(t, tt.wantLabel, environmentLabel(version, base)) @@ -322,3 +322,23 @@ func TestStatusStyleSelectors(t *testing.T) { assert.True(t, isFailedStatus("TIMEDOUT")) assert.False(t, isFailedStatus("RUNNING")) } + +func TestAiRuntimeEnvironmentUsesTheSelectedTasksKey(t *testing.T) { + srv := runGetServer(t, `{"run_id": 5, "environments": [ + {"environment_key": "default", "spec": {"environment_version": "4"}}, + {"environment_key": "eval_env", "spec": {"environment_version": "6"}} + ]}`) + train := aiRuntimeTask("train", 11, "e", "GPU_1xA10", 1) + eval := aiRuntimeTask("eval", 12, "e", "GPU_1xA10", 1) + eval.EnvironmentKey = "eval_env" + run := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{train, eval}} + + assert.Equal(t, aiRuntimeEnvironmentKey, taskEnvironmentKey(taskRun(run, "train")), "a task without a key uses air run's") + assert.Equal(t, "eval_env", taskEnvironmentKey(taskRun(run, "eval"))) + + version, _ := aiRuntimeEnvironment(t.Context(), newTestWorkspaceClient(t, srv.URL), 5, taskEnvironmentKey(taskRun(run, "eval"))) + assert.Equal(t, "6", version, "--task-key eval shows eval's environment, not default's") + + sweep := &jobs.Run{RunId: 5, Tasks: []jobs.RunTask{{TaskKey: "sweep", ForEachTask: &jobs.RunForEachTask{Task: jobs.Task{EnvironmentKey: "eval_env"}}}}} + assert.Equal(t, "eval_env", taskEnvironmentKey(sweep), "a sweep uses its iterated task's key") +}