From 39655d4239cc16beea3d3f4a28393827e5dfade9 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 16:48:23 +0000 Subject: [PATCH 1/2] air logs: emit WAITING events in -o json mode `-o json` never shows the waiting spinner, so `air logs -o json` and `air run --watch -o json` printed nothing until the first log line and a provisioning run looked hung. Emit a WAITING JSONL event each time the waiting status changes, the same signal text mode gets from the spinner (or, since #7657, from a stderr line when stderr is not a terminal). Follow-up to review feedback on #7657. Co-authored-by: Isaac --- .nextchanges/air/logs-json-waiting.md | 1 + cmd/air/logstream.go | 14 ++++++-- cmd/air/logstream_support.go | 22 ++++++++++++ cmd/air/logstream_test.go | 52 ++++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 .nextchanges/air/logs-json-waiting.md diff --git a/.nextchanges/air/logs-json-waiting.md b/.nextchanges/air/logs-json-waiting.md new file mode 100644 index 00000000000..bf720481f0f --- /dev/null +++ b/.nextchanges/air/logs-json-waiting.md @@ -0,0 +1 @@ +* `databricks air logs -o json` and `databricks air run --watch -o json` now emit a `WAITING` event, for example `{"type":"WAITING","message":"Waiting for accelerator compute capacity..."}`, each time the waiting status changes before the first log line, instead of printing nothing. ([#7695](https://github.com/databricks/cli/pull/7695)) diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index 79ed5e18b3e..52d2562397e 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -399,6 +399,12 @@ func (st *bricklensStreamer) logWaitingStatus(msg string) { cmdio.LogProgress(st.ctx, msg) } +// printWaitingEvent reports a waiting status as a WAITING JSONL event in place of +// the spinner, which -o json never shows. +func (st *bricklensStreamer) printWaitingEvent(msg string) { + printWaitingEvent(st.out, msg) +} + // reportStatusChange fires onStatusChange when the run's display state differs // from the last reported one. func (st *bricklensStreamer) reportStatusChange() { @@ -425,10 +431,12 @@ func (st *bricklensStreamer) run() (bool, error) { } // Show a "waiting for run to start" spinner on stderr while the run has not - // yet produced logs, so a provisioning run doesn't look hung. Suppressed in - // --json mode and auto-degraded to nothing on a non-interactive terminal. + // yet produced logs, so a provisioning run doesn't look hung. --json mode gets + // WAITING events instead, and it auto-degrades to nothing on a non-interactive terminal. // The first emitted log line stops it via onFirstLog (before any stdout write). - if !st.req.jsonOutput { + if st.req.jsonOutput { + st.updateSpinner = st.printWaitingEvent + } else { sp := cmdio.NewSpinner(st.ctx) defer sp.Close() st.onFirstLog = sp.Close diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index fb6735da646..a1a457658d4 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -166,6 +166,28 @@ func printStatusEvent(out io.Writer, current, previous string) { fmt.Fprintln(out, string(b)) } +// waitingEvent is a JSONL event emitted while a run has produced no logs yet, +// each time the reason it is waiting changes, so a -o json consumer can tell a +// provisioning run from a hung command. +type waitingEvent struct { + Type string `json:"type"` + TS string `json:"ts"` + Message string `json:"message"` +} + +// printWaitingEvent writes a WAITING JSONL event. +func printWaitingEvent(out io.Writer, message string) { + b, err := json.Marshal(waitingEvent{ + Type: "WAITING", + TS: time.Now().UTC().Format(time.RFC3339), + Message: message, + }) + if err != nil { + return + } + fmt.Fprintln(out, string(b)) +} + // terminalEvent is the closing envelope `air run --watch -o json` emits after // streaming, carrying the run's terminal status. type terminalEvent struct { diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 5bb60caa344..8aa659d4075 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -477,6 +477,54 @@ func TestStreamBricklensPrintsWaitingStatusWithoutTTY(t *testing.T) { assert.Contains(t, stdout.String(), "training") } +func TestStreamBricklensJSONEmitsWaitingEvent(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + var runGets int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + if runGets < 4 { + _, _ = w.Write([]byte(`{"log_records":[]}`)) + return + } + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets++ + if runGets < 4 { + _, _ = w.Write([]byte(`{"run_id":1,"state":{"life_cycle_state":"PENDING"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + return + } + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var stdout, stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout, + logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true}, + logRunStatus{lifeCycleState: "PENDING", startTimeMs: 10_000}) + require.NoError(t, err) + assert.True(t, ok) + + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + require.GreaterOrEqual(t, len(lines), 2) + var first waitingEvent + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "WAITING", first.Type) + assert.Equal(t, "Waiting for GPU capacity...", first.Message) + assert.Contains(t, lines[len(lines)-1], `"line":"training"`) + assert.Equal(t, 1, strings.Count(stdout.String(), `"type":"WAITING"`), "one event per distinct status") + assert.Empty(t, stderr.String()) +} + func TestEmitLogLineJSON(t *testing.T) { var buf bytes.Buffer emitLogLine(&buf, logRequest{node: 2, jsonOutput: true}, "hello") @@ -812,7 +860,9 @@ func TestStreamBricklensActiveEmptyProbesMLflow(t *testing.T) { logRequest{runID: 123, attempt: -1, tailLines: -1, jsonOutput: true}, logRunStatus{lifeCycleState: "RUNNING"}) require.ErrorIs(t, err, errBricklensFeatureDisabled) - assert.Empty(t, out.String()) + // Only the WAITING status event precedes the fallback; no log lines. + assert.NotContains(t, out.String(), `"line"`) + assert.Contains(t, out.String(), `"type":"WAITING"`) } func TestFetchLogsFallsBackToMLflowWhenBricklensEmpty(t *testing.T) { From 7cc3aab3add5f0c3c0422d13ca1a88780e4839fd Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 19:21:27 +0000 Subject: [PATCH 2/2] air logs: report waiting only after a fetch finds no logs Address review: attaching to an active run that already has logs no longer emits WAITING (or the non-TTY waiting line) before the first LOG. Also fix the stale spinner comment and the changelog wording. Co-authored-by: Isaac --- .nextchanges/air/logs-json-waiting.md | 2 +- cmd/air/logs_test.go | 4 +-- cmd/air/logstream.go | 36 ++++++++++++++------------ cmd/air/logstream_test.go | 37 +++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/.nextchanges/air/logs-json-waiting.md b/.nextchanges/air/logs-json-waiting.md index bf720481f0f..bf0d941d8ae 100644 --- a/.nextchanges/air/logs-json-waiting.md +++ b/.nextchanges/air/logs-json-waiting.md @@ -1 +1 @@ -* `databricks air logs -o json` and `databricks air run --watch -o json` now emit a `WAITING` event, for example `{"type":"WAITING","message":"Waiting for accelerator compute capacity..."}`, each time the waiting status changes before the first log line, instead of printing nothing. ([#7695](https://github.com/databricks/cli/pull/7695)) +* `databricks air logs -o json` and `databricks air run --watch -o json` now emit a `WAITING` event, for example `{"type":"WAITING","message":"Waiting for accelerator compute capacity..."}`, when the reported waiting status changes. ([#7695](https://github.com/databricks/cli/pull/7695)) diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 6d18adb29f2..ea79f82908e 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -480,8 +480,8 @@ func TestLogsCommandRepeatsRunLinkAfterStreaming(t *testing.T) { assert.Contains(t, buf.String(), `"line":"step 2"`) assert.NotContains(t, buf.String(), "View job run at:") } else { - // stderr shares buf and is not a TTY, so the waiting status prints as a line. - assert.Equal(t, "Waiting for run to start (node 0)...\nstep 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String()) + // The first fetch already returns logs, so no waiting status prints. + assert.Equal(t, "step 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String()) } }) } diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index 52d2562397e..88b2651bd78 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -430,10 +430,11 @@ func (st *bricklensStreamer) run() (bool, error) { return st.drainStatic(st.req.toSeconds(st.status)) } - // Show a "waiting for run to start" spinner on stderr while the run has not - // yet produced logs, so a provisioning run doesn't look hung. --json mode gets - // WAITING events instead, and it auto-degrades to nothing on a non-interactive terminal. - // The first emitted log line stops it via onFirstLog (before any stdout write). + // Report the waiting status while the run has not yet produced logs, so a + // provisioning run doesn't look hung: a spinner on an interactive stderr, one + // line per status change on a non-interactive stderr, and WAITING events in + // --json mode. The first emitted log line stops the spinner via onFirstLog + // (before any stdout write). if st.req.jsonOutput { st.updateSpinner = st.printWaitingEvent } else { @@ -486,19 +487,6 @@ func (st *bricklensStreamer) run() (bool, error) { terminal := st.status.terminal() toSec := st.req.toSeconds(st.status) - // While waiting on a still-active run with no logs yet, refresh the spinner - // with the server-set status (throttled), so a run stuck waiting for compute - // shows why rather than a generic "waiting" message. - if !terminal && !st.firstLogSeen && st.updateSpinner != nil { - if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 { - if desired := st.waitingSpinnerText(); desired != lastSpinnerText { - st.updateSpinner(desired) - lastSpinnerText = desired - } - } - statusRefreshCounter++ - } - // `air logs` starts an active attachment with a bounded tail; subsequent // polls follow live with dedup so the overlap does not re-print it. var emitted int @@ -541,6 +529,20 @@ func (st *bricklensStreamer) run() (bool, error) { return st.status.succeeded(), nil } if !st.firstLogSeen { + // The fetch above found no logs for this still-active run, so refresh the + // waiting status with the server-set message (throttled): a run stuck + // waiting for compute shows why rather than a generic message. Doing this + // after the fetch means attaching to a run that already has logs never + // reports it as waiting. + if st.updateSpinner != nil { + if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 { + if desired := st.waitingSpinnerText(); desired != lastSpinnerText { + st.updateSpinner(desired) + lastSpinnerText = desired + } + } + statusRefreshCounter++ + } emptyStreamPolls++ if emptyStreamPolls%bricklensEmptyMLflowProbeEveryNPolls == 0 && mlflowLogsExist(st.ctx, st.w, st.req) { log.Debugf(st.ctx, "air logs: MLflow artifacts exist for run %d; falling back to MLflow log stream", st.req.runID) diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 8aa659d4075..290b72fdace 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -525,6 +525,43 @@ func TestStreamBricklensJSONEmitsWaitingEvent(t *testing.T) { assert.Empty(t, stderr.String()) } +func TestStreamBricklensJSONActiveRunWithLogsSkipsWaitingEvent(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + var runGets int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets++ + if runGets < 3 { + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"RUNNING"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + return + } + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + var stdout, stderr bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout, + logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true, boundInitialLogs: true}, + logRunStatus{lifeCycleState: "RUNNING", startTimeMs: 10_000}) + require.NoError(t, err) + assert.True(t, ok) + + assert.Contains(t, stdout.String(), `"line":"training"`) + assert.NotContains(t, stdout.String(), `"type":"WAITING"`, "a run that already has logs is not waiting") +} + func TestEmitLogLineJSON(t *testing.T) { var buf bytes.Buffer emitLogLine(&buf, logRequest{node: 2, jsonOutput: true}, "hello")