Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7636.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air get -o json` now includes `mlflow_run_id` and `mlflow_experiment_id`, so a run can be handed to MLflow-based steps without parsing `mlflow_url`. They identify the MLflow run `mlflow_url` links to, including one from an earlier retry that `mlflow_retry` names. ([#7636](https://github.com/databricks/cli/pull/7636))
8 changes: 8 additions & 0 deletions acceptance/air/get-ai-runtime/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"mlflow_experiment_id": "exp1",
"mlflow_run_id": "run1",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
Expand Down Expand Up @@ -78,6 +80,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"mlflow_experiment_id": "exp1",
"mlflow_run_id": "run1",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
Expand Down Expand Up @@ -145,6 +149,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"mlflow_experiment_id": "exp1",
"mlflow_run_id": "run1",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
Expand Down Expand Up @@ -173,6 +179,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"mlflow_experiment_id": "exp1",
"mlflow_run_id": "run1",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
Expand Down
2 changes: 2 additions & 0 deletions acceptance/air/get-mlflow-retry/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
"experiment_name": null,
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": null,
"mlflow_experiment_id": null,
"mlflow_run_id": null,
"termination_reason": "Out of capacity",
"est_remaining_seconds": null,
"est_percent_complete": null,
Expand Down
2 changes: 2 additions & 0 deletions acceptance/air/get-pending/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": null,
"mlflow_experiment_id": null,
"mlflow_run_id": null,
"status_message": "Waiting for accelerator compute capacity to become available...",
"est_remaining_seconds": null,
"est_percent_complete": null,
Expand Down
2 changes: 2 additions & 0 deletions acceptance/air/get/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"mlflow_experiment_id": "exp1",
"mlflow_run_id": "run1",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
Expand Down
9 changes: 9 additions & 0 deletions cmd/air/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ type getData struct {
ExperimentName *string `json:"experiment_name"`
DashboardURL string `json:"dashboard_url"`
MLflowURL *string `json:"mlflow_url"`
// MLflowExperimentID and MLflowRunID identify the MLflow run MLflowURL links
// to. For a finished run whose last retry never started an MLflow run, they
// come from the newest earlier retry that has one, which MLflowRetry names.
// Like MLflowURL, they are null whenever no MLflow run is found, including
// when a get-output call fails.
MLflowExperimentID *string `json:"mlflow_experiment_id"`
MLflowRunID *string `json:"mlflow_run_id"`
// MLflowRetry is the retry the MLflow IDs above came from when it isn't the
// run's latest (see mlflowIDsFromRetries), else absent.
MLflowRetry *int `json:"mlflow_retry,omitempty"`
Expand Down Expand Up @@ -224,6 +231,8 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`,
if ids != nil {
url := mlflowLogsURL(w.Config.Host, ids)
data.MLflowURL = &url
data.MLflowExperimentID = &ids.ExperimentID
data.MLflowRunID = &ids.RunID
data.MLflowRetry = ids.OlderRetry
// A remaining-time estimate only makes sense while the run is still
// training; a terminal run is either done or stopped mid-progress.
Expand Down
121 changes: 114 additions & 7 deletions cmd/air/get_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"errors"
"maps"
"net/http"
"net/http/httptest"
"strings"
Expand Down Expand Up @@ -178,14 +179,25 @@ func TestGetRunNotFoundJSON(t *testing.T) {
assert.Equal(t, jsonError{Code: "NOT_FOUND", Kind: "NOT_FOUND", Message: "run 5 not found: check the run ID and that it is a job run ID"}, got.Error)
}

// getRunServer serves a terminal runs/get and a /Me that reports workspace
// 12345 in the X-Databricks-Org-Id header. *meCalls counts /Me requests.
func getRunServer(t *testing.T, meCalls *atomic.Int32) *httptest.Server {
const succeededRun = `{"run_id": 5, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}}`

// getRunServer serves run from runs/get, outputs[task run ID] from
// runs/get-output (a 500 for a task run with no entry), and a /Me that reports
// workspace 12345 in the X-Databricks-Org-Id header. *meCalls counts /Me
// requests.
func getRunServer(t *testing.T, meCalls *atomic.Int32, run string, outputs map[string]string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(`{"run_id": 5, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}}`))
_, _ = w.Write([]byte(run))
case r.URL.Path == "/api/2.2/jobs/runs/get-output":
output, ok := outputs[r.URL.Query().Get("run_id")]
if !ok {
w.WriteHeader(http.StatusInternalServerError)
output = `{"error_code": "INTERNAL_ERROR", "message": "get-output failed"}`
}
_, _ = w.Write([]byte(output))
case strings.HasSuffix(r.URL.Path, "/scim/v2/Me"):
meCalls.Add(1)
w.Header().Set("X-Databricks-Org-Id", "12345")
Expand Down Expand Up @@ -261,7 +273,7 @@ func runGetJSON(t *testing.T, w *databricks.WorkspaceClient) (string, error) {

func TestGetRunURLUsesConfiguredWorkspaceID(t *testing.T) {
var meCalls atomic.Int32
srv := getRunServer(t, &meCalls)
srv := getRunServer(t, &meCalls, succeededRun, nil)
w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token", WorkspaceID: "67890"})
require.NoError(t, err)

Expand All @@ -275,7 +287,7 @@ func TestGetRunURLLooksUpUnsetWorkspaceID(t *testing.T) {
// The SDK also reads the workspace ID from the environment.
t.Setenv("DATABRICKS_WORKSPACE_ID", "")
var meCalls atomic.Int32
srv := getRunServer(t, &meCalls)
srv := getRunServer(t, &meCalls, succeededRun, nil)

out, err := runGetJSON(t, newTestWorkspaceClient(t, srv.URL))
require.NoError(t, err)
Expand All @@ -285,7 +297,7 @@ func TestGetRunURLLooksUpUnsetWorkspaceID(t *testing.T) {

func TestGetRunRejectsNonNumericWorkspaceID(t *testing.T) {
var meCalls atomic.Int32
srv := getRunServer(t, &meCalls)
srv := getRunServer(t, &meCalls, succeededRun, nil)
w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token", WorkspaceID: "not-a-number"})
require.NoError(t, err)

Expand All @@ -296,6 +308,97 @@ func TestGetRunRejectsNonNumericWorkspaceID(t *testing.T) {
assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `configured workspace_id "not-a-number" is not a numeric workspace ID`}, got.Error)
}

func TestGetRunJSONMLflowIDs(t *testing.T) {
const (
succeededTaskRun = `{"run_id": 5, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, "tasks": [{"run_id": 50}]}`
outputWithIDs = `{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`
)
tests := []struct {
name string
run string
outputs map[string]string
// wantExperimentID and wantRunID are the IDs the JSON names, or empty
// when mlflow_url and both IDs must be null.
wantExperimentID string
wantRunID string
// wantRetry is the retry mlflow_retry names, or nil when the key must be
// absent.
wantRetry *int
}{
{
name: "set for a finished run",
run: succeededTaskRun,
outputs: map[string]string{"50": outputWithIDs},
wantExperimentID: "E1",
wantRunID: "R1",
},
{
name: "set while the run is running",
run: `{"run_id": 5, "state": {"life_cycle_state": "RUNNING"}, "tasks": [{"run_id": 50}]}`,
outputs: map[string]string{"50": outputWithIDs},
wantExperimentID: "E1",
wantRunID: "R1",
},
{
name: "from an earlier retry when the last retry never started",
run: `{"run_id": 5, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"},
"tasks": [{"task_key": "train", "run_id": 50}, {"task_key": "train", "run_id": 51, "attempt_number": 1}]}`,
outputs: map[string]string{
"50": `{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R0"}}`,
"51": `{}`,
},
wantExperimentID: "E1",
wantRunID: "R0",
wantRetry: new(0),
},
{
name: "null when the run has no MLflow run",
run: succeededTaskRun,
outputs: map[string]string{"50": `{}`},
},
{
name: "null when the output has only an experiment ID",
run: succeededTaskRun,
outputs: map[string]string{"50": `{"ai_runtime_task_output": {"mlflow_experiment_id": "E1"}}`},
},
{
// No output for task run 50, so its get-output fails.
name: "null when get-output fails",
run: succeededTaskRun,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
srv := getRunServer(t, new(atomic.Int32), tc.run, tc.outputs)
w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token", WorkspaceID: "67890"})
require.NoError(t, err)

out, err := runGetJSON(t, w)
require.NoError(t, err)
// Decode into a map, which tells a missing key from null.
var got struct {
Data map[string]any `json:"data"`
}
require.NoError(t, json.Unmarshal([]byte(out), &got))
maps.DeleteFunc(got.Data, func(key string, _ any) bool { return !strings.HasPrefix(key, "mlflow_") })
want := map[string]any{"mlflow_url": nil, "mlflow_experiment_id": nil, "mlflow_run_id": nil}
if tc.wantRunID != "" {
want = map[string]any{
"mlflow_url": srv.URL + "/ml/experiments/" + tc.wantExperimentID + "/runs/" + tc.wantRunID + "/logs",
"mlflow_experiment_id": tc.wantExperimentID,
"mlflow_run_id": tc.wantRunID,
}
}
// getRunServer serves no MLflow run, so mlflow_run_name stays null.
want["mlflow_run_name"] = nil
if tc.wantRetry != nil {
want["mlflow_retry"] = float64(*tc.wantRetry)
}
assert.Equal(t, want, got.Data)
})
}
}

// getRunJSON runs `air get RUN_ID -o json` against srv and returns the decoded
// data payload.
func getRunJSON(t *testing.T, srv *httptest.Server, runID string) map[string]any {
Expand Down Expand Up @@ -329,6 +432,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) {
"duration_seconds":12,"attempt_number":2,"experiment_name":null,
"dashboard_url":"`+host+`/jobs/runs/123?o=9999",
"mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs",
"mlflow_experiment_id":"E1","mlflow_run_id":"R0",
"mlflow_retry":0,
"est_remaining_seconds":null,"est_percent_complete":null,
"user":null,"accelerator_type":null,"num_accelerators":null,
Expand All @@ -344,6 +448,7 @@ func TestGetRunMLflowRetryJSON(t *testing.T) {
"duration_seconds":12,"attempt_number":0,"experiment_name":null,
"dashboard_url":"`+host+`/jobs/runs/124?o=9999",
"mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs",
"mlflow_experiment_id":"E1","mlflow_run_id":"R1",
"est_remaining_seconds":null,"est_percent_complete":null,
"user":null,"accelerator_type":null,"num_accelerators":null,
"environment_version":null,"base_environment":null,
Expand Down Expand Up @@ -436,6 +541,7 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) {
assert.JSONEq(t, `{
"run_id":"5","status":"FAILED","started_at":null,"duration_seconds":null,
"attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null,
"mlflow_experiment_id":null,"mlflow_run_id":null,
"termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null,
"user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null,
"max_retries":null,"mlflow_run_name":null,"sweep":null
Expand All @@ -460,6 +566,7 @@ func TestBuildGetDataWaitingForComputeStatus(t *testing.T) {
assert.JSONEq(t, `{
"run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null,
"attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null,
"mlflow_experiment_id":null,"mlflow_run_id":null,
"est_remaining_seconds":null,"est_percent_complete":null,
"user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null,
"max_retries":null,"mlflow_run_name":null,"sweep":null
Expand Down
Loading