diff --git a/.nextchanges/air/logs-json-waiting.md b/.nextchanges/air/logs-json-waiting.md new file mode 100644 index 00000000000..bf0d941d8ae --- /dev/null +++ b/.nextchanges/air/logs-json-waiting.md @@ -0,0 +1 @@ +* `databricks air logs -o json` and `databricks air run --watch -o json` now emit a `WAITING` event, for example `{"type":"WAITING","message":"Waiting for accelerator compute capacity..."}`, when the reported waiting status changes. ([#7695](https://github.com/databricks/cli/pull/7695)) diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 6d18adb29f2..ea79f82908e 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -480,8 +480,8 @@ func TestLogsCommandRepeatsRunLinkAfterStreaming(t *testing.T) { assert.Contains(t, buf.String(), `"line":"step 2"`) assert.NotContains(t, buf.String(), "View job run at:") } else { - // stderr shares buf and is not a TTY, so the waiting status prints as a line. - assert.Equal(t, "Waiting for run to start (node 0)...\nstep 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String()) + // The first fetch already returns logs, so no waiting status prints. + assert.Equal(t, "step 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String()) } }) } diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index 79ed5e18b3e..88b2651bd78 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -399,6 +399,12 @@ func (st *bricklensStreamer) logWaitingStatus(msg string) { cmdio.LogProgress(st.ctx, msg) } +// printWaitingEvent reports a waiting status as a WAITING JSONL event in place of +// the spinner, which -o json never shows. +func (st *bricklensStreamer) printWaitingEvent(msg string) { + printWaitingEvent(st.out, msg) +} + // reportStatusChange fires onStatusChange when the run's display state differs // from the last reported one. func (st *bricklensStreamer) reportStatusChange() { @@ -424,11 +430,14 @@ func (st *bricklensStreamer) run() (bool, error) { return st.drainStatic(st.req.toSeconds(st.status)) } - // Show a "waiting for run to start" spinner on stderr while the run has not - // yet produced logs, so a provisioning run doesn't look hung. Suppressed in - // --json mode and auto-degraded to nothing on a non-interactive terminal. - // The first emitted log line stops it via onFirstLog (before any stdout write). - if !st.req.jsonOutput { + // Report the waiting status while the run has not yet produced logs, so a + // provisioning run doesn't look hung: a spinner on an interactive stderr, one + // line per status change on a non-interactive stderr, and WAITING events in + // --json mode. The first emitted log line stops the spinner via onFirstLog + // (before any stdout write). + if st.req.jsonOutput { + st.updateSpinner = st.printWaitingEvent + } else { sp := cmdio.NewSpinner(st.ctx) defer sp.Close() st.onFirstLog = sp.Close @@ -478,19 +487,6 @@ func (st *bricklensStreamer) run() (bool, error) { terminal := st.status.terminal() toSec := st.req.toSeconds(st.status) - // While waiting on a still-active run with no logs yet, refresh the spinner - // with the server-set status (throttled), so a run stuck waiting for compute - // shows why rather than a generic "waiting" message. - if !terminal && !st.firstLogSeen && st.updateSpinner != nil { - if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 { - if desired := st.waitingSpinnerText(); desired != lastSpinnerText { - st.updateSpinner(desired) - lastSpinnerText = desired - } - } - statusRefreshCounter++ - } - // `air logs` starts an active attachment with a bounded tail; subsequent // polls follow live with dedup so the overlap does not re-print it. var emitted int @@ -533,6 +529,20 @@ func (st *bricklensStreamer) run() (bool, error) { return st.status.succeeded(), nil } if !st.firstLogSeen { + // The fetch above found no logs for this still-active run, so refresh the + // waiting status with the server-set message (throttled): a run stuck + // waiting for compute shows why rather than a generic message. Doing this + // after the fetch means attaching to a run that already has logs never + // reports it as waiting. + if st.updateSpinner != nil { + if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 { + if desired := st.waitingSpinnerText(); desired != lastSpinnerText { + st.updateSpinner(desired) + lastSpinnerText = desired + } + } + statusRefreshCounter++ + } emptyStreamPolls++ if emptyStreamPolls%bricklensEmptyMLflowProbeEveryNPolls == 0 && mlflowLogsExist(st.ctx, st.w, st.req) { log.Debugf(st.ctx, "air logs: MLflow artifacts exist for run %d; falling back to MLflow log stream", st.req.runID) diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index fb6735da646..a1a457658d4 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -166,6 +166,28 @@ func printStatusEvent(out io.Writer, current, previous string) { fmt.Fprintln(out, string(b)) } +// waitingEvent is a JSONL event emitted while a run has produced no logs yet, +// each time the reason it is waiting changes, so a -o json consumer can tell a +// provisioning run from a hung command. +type waitingEvent struct { + Type string `json:"type"` + TS string `json:"ts"` + Message string `json:"message"` +} + +// printWaitingEvent writes a WAITING JSONL event. +func printWaitingEvent(out io.Writer, message string) { + b, err := json.Marshal(waitingEvent{ + Type: "WAITING", + TS: time.Now().UTC().Format(time.RFC3339), + Message: message, + }) + if err != nil { + return + } + fmt.Fprintln(out, string(b)) +} + // terminalEvent is the closing envelope `air run --watch -o json` emits after // streaming, carrying the run's terminal status. type terminalEvent struct { diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 5bb60caa344..290b72fdace 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -477,6 +477,91 @@ func TestStreamBricklensPrintsWaitingStatusWithoutTTY(t *testing.T) { assert.Contains(t, stdout.String(), "training") } +func TestStreamBricklensJSONEmitsWaitingEvent(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + var runGets int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + if runGets < 4 { + _, _ = w.Write([]byte(`{"log_records":[]}`)) + return + } + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets++ + if runGets < 4 { + _, _ = w.Write([]byte(`{"run_id":1,"state":{"life_cycle_state":"PENDING"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + return + } + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var stdout, stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout, + logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true}, + logRunStatus{lifeCycleState: "PENDING", startTimeMs: 10_000}) + require.NoError(t, err) + assert.True(t, ok) + + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + require.GreaterOrEqual(t, len(lines), 2) + var first waitingEvent + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "WAITING", first.Type) + assert.Equal(t, "Waiting for GPU capacity...", first.Message) + assert.Contains(t, lines[len(lines)-1], `"line":"training"`) + assert.Equal(t, 1, strings.Count(stdout.String(), `"type":"WAITING"`), "one event per distinct status") + assert.Empty(t, stderr.String()) +} + +func TestStreamBricklensJSONActiveRunWithLogsSkipsWaitingEvent(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + var runGets int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets++ + if runGets < 3 { + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"RUNNING"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + return + } + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var stdout, stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout, + logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true, boundInitialLogs: true}, + logRunStatus{lifeCycleState: "RUNNING", startTimeMs: 10_000}) + require.NoError(t, err) + assert.True(t, ok) + + assert.Contains(t, stdout.String(), `"line":"training"`) + assert.NotContains(t, stdout.String(), `"type":"WAITING"`, "a run that already has logs is not waiting") +} + func TestEmitLogLineJSON(t *testing.T) { var buf bytes.Buffer emitLogLine(&buf, logRequest{node: 2, jsonOutput: true}, "hello") @@ -812,7 +897,9 @@ func TestStreamBricklensActiveEmptyProbesMLflow(t *testing.T) { logRequest{runID: 123, attempt: -1, tailLines: -1, jsonOutput: true}, logRunStatus{lifeCycleState: "RUNNING"}) require.ErrorIs(t, err, errBricklensFeatureDisabled) - assert.Empty(t, out.String()) + // Only the WAITING status event precedes the fallback; no log lines. + assert.NotContains(t, out.String(), `"line"`) + assert.Contains(t, out.String(), `"type":"WAITING"`) } func TestFetchLogsFallsBackToMLflowWhenBricklensEmpty(t *testing.T) {