Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/logs-json-waiting.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air logs -o json` and `databricks air run --watch -o json` now emit a `WAITING` event, for example `{"type":"WAITING","message":"Waiting for accelerator compute capacity..."}`, when the reported waiting status changes. ([#7695](https://github.com/databricks/cli/pull/7695))
4 changes: 2 additions & 2 deletions cmd/air/logs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -480,8 +480,8 @@ func TestLogsCommandRepeatsRunLinkAfterStreaming(t *testing.T) {
assert.Contains(t, buf.String(), `"line":"step 2"`)
assert.NotContains(t, buf.String(), "View job run at:")
} else {
// stderr shares buf and is not a TTY, so the waiting status prints as a line.
assert.Equal(t, "Waiting for run to start (node 0)...\nstep 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String())
// The first fetch already returns logs, so no waiting status prints.
assert.Equal(t, "step 1\nstep 2\n\nView job run at: "+srv.URL+"/jobs/runs/777\n", buf.String())
}
})
}
Expand Down
46 changes: 28 additions & 18 deletions cmd/air/logstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,12 @@ func (st *bricklensStreamer) logWaitingStatus(msg string) {
cmdio.LogProgress(st.ctx, msg)
}

// printWaitingEvent reports a waiting status as a WAITING JSONL event in place of
// the spinner, which -o json never shows.
func (st *bricklensStreamer) printWaitingEvent(msg string) {
printWaitingEvent(st.out, msg)
}

// reportStatusChange fires onStatusChange when the run's display state differs
// from the last reported one.
func (st *bricklensStreamer) reportStatusChange() {
Expand All @@ -424,11 +430,14 @@ func (st *bricklensStreamer) run() (bool, error) {
return st.drainStatic(st.req.toSeconds(st.status))
}

// Show a "waiting for run to start" spinner on stderr while the run has not
// yet produced logs, so a provisioning run doesn't look hung. Suppressed in
// --json mode and auto-degraded to nothing on a non-interactive terminal.
// The first emitted log line stops it via onFirstLog (before any stdout write).
if !st.req.jsonOutput {
// Report the waiting status while the run has not yet produced logs, so a
// provisioning run doesn't look hung: a spinner on an interactive stderr, one
// line per status change on a non-interactive stderr, and WAITING events in
// --json mode. The first emitted log line stops the spinner via onFirstLog
// (before any stdout write).
if st.req.jsonOutput {
st.updateSpinner = st.printWaitingEvent

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This hook runs before the first log fetch, so attaching to an active run with existing logs emits WAITING immediately before LOG. Emit WAITING only after a fetch confirms no logs are available, and cover the existing-logs case.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 7cc3aab. The waiting-status update now runs after the poll's fetch, inside the !firstLogSeen branch, so attaching to an active run that already has logs never emits WAITING. This also fixes the same ordering in the non-TTY text path from #7657, which printed "Waiting for run to start" before existing logs; TestLogsCommandRepeatsRunLinkAfterStreaming had that baked in and is updated. New test TestStreamBricklensJSONActiveRunWithLogsSkipsWaitingEvent covers the existing-logs case and fails on the old ordering.

} else {
sp := cmdio.NewSpinner(st.ctx)
defer sp.Close()
st.onFirstLog = sp.Close
Expand Down Expand Up @@ -478,19 +487,6 @@ func (st *bricklensStreamer) run() (bool, error) {
terminal := st.status.terminal()
toSec := st.req.toSeconds(st.status)

// While waiting on a still-active run with no logs yet, refresh the spinner
// with the server-set status (throttled), so a run stuck waiting for compute
// shows why rather than a generic "waiting" message.
if !terminal && !st.firstLogSeen && st.updateSpinner != nil {
if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 {
if desired := st.waitingSpinnerText(); desired != lastSpinnerText {
st.updateSpinner(desired)
lastSpinnerText = desired
}
}
statusRefreshCounter++
}

// `air logs` starts an active attachment with a bounded tail; subsequent
// polls follow live with dedup so the overlap does not re-print it.
var emitted int
Expand Down Expand Up @@ -533,6 +529,20 @@ func (st *bricklensStreamer) run() (bool, error) {
return st.status.succeeded(), nil
}
if !st.firstLogSeen {
// The fetch above found no logs for this still-active run, so refresh the
// waiting status with the server-set message (throttled): a run stuck
// waiting for compute shows why rather than a generic message. Doing this
// after the fetch means attaching to a run that already has logs never
// reports it as waiting.
if st.updateSpinner != nil {
if statusRefreshCounter%statusMessageRefreshEveryNPolls == 0 {
if desired := st.waitingSpinnerText(); desired != lastSpinnerText {
st.updateSpinner(desired)
lastSpinnerText = desired
}
}
statusRefreshCounter++
}
emptyStreamPolls++
if emptyStreamPolls%bricklensEmptyMLflowProbeEveryNPolls == 0 && mlflowLogsExist(st.ctx, st.w, st.req) {
log.Debugf(st.ctx, "air logs: MLflow artifacts exist for run %d; falling back to MLflow log stream", st.req.runID)
Expand Down
22 changes: 22 additions & 0 deletions cmd/air/logstream_support.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,28 @@ func printStatusEvent(out io.Writer, current, previous string) {
fmt.Fprintln(out, string(b))
}

// waitingEvent is a JSONL event emitted while a run has produced no logs yet,
// each time the reason it is waiting changes, so a -o json consumer can tell a
// provisioning run from a hung command.
type waitingEvent struct {
Type string `json:"type"`
TS string `json:"ts"`
Message string `json:"message"`
}

// printWaitingEvent writes a WAITING JSONL event.
func printWaitingEvent(out io.Writer, message string) {
b, err := json.Marshal(waitingEvent{
Type: "WAITING",
TS: time.Now().UTC().Format(time.RFC3339),
Message: message,
})
if err != nil {
return
}
fmt.Fprintln(out, string(b))
}

// terminalEvent is the closing envelope `air run --watch -o json` emits after
// streaming, carrying the run's terminal status.
type terminalEvent struct {
Expand Down
89 changes: 88 additions & 1 deletion cmd/air/logstream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,91 @@ func TestStreamBricklensPrintsWaitingStatusWithoutTTY(t *testing.T) {
assert.Contains(t, stdout.String(), "training")
}

func TestStreamBricklensJSONEmitsWaitingEvent(t *testing.T) {
original := bricklensPollInterval
bricklensPollInterval = time.Millisecond
t.Cleanup(func() { bricklensPollInterval = original })

var runGets int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/logs"):
if runGets < 4 {
_, _ = w.Write([]byte(`{"log_records":[]}`))
return
}
_, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get":
runGets++
if runGets < 4 {
_, _ = w.Write([]byte(`{"run_id":1,"state":{"life_cycle_state":"PENDING"},"tasks":[{"run_id":2,"attempt_number":0}]}`))
return
}
_, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get-output":
_, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)

var stdout, stderr bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", ""))
ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout,
logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true},
logRunStatus{lifeCycleState: "PENDING", startTimeMs: 10_000})
require.NoError(t, err)
assert.True(t, ok)

lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")
require.GreaterOrEqual(t, len(lines), 2)
var first waitingEvent
require.NoError(t, json.Unmarshal([]byte(lines[0]), &first))
assert.Equal(t, "WAITING", first.Type)
assert.Equal(t, "Waiting for GPU capacity...", first.Message)
assert.Contains(t, lines[len(lines)-1], `"line":"training"`)
assert.Equal(t, 1, strings.Count(stdout.String(), `"type":"WAITING"`), "one event per distinct status")
assert.Empty(t, stderr.String())
}

func TestStreamBricklensJSONActiveRunWithLogsSkipsWaitingEvent(t *testing.T) {
original := bricklensPollInterval
bricklensPollInterval = time.Millisecond
t.Cleanup(func() { bricklensPollInterval = original })

var runGets int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/logs"):
_, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"training"}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get":
runGets++
if runGets < 3 {
_, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"RUNNING"},"tasks":[{"run_id":2,"attempt_number":0}]}`))
return
}
_, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get-output":
_, _ = w.Write([]byte(`{"ai_runtime_task_output":{"status_message":"STATUS: Waiting for GPU capacity"}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)

var stdout, stderr bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", ""))
ok, err := streamBricklensLogs(ctx, newTestWorkspaceClient(t, srv.URL), &stdout,
logRequest{runID: 1, attempt: -1, tailLines: -1, jsonOutput: true, boundInitialLogs: true},
logRunStatus{lifeCycleState: "RUNNING", startTimeMs: 10_000})
require.NoError(t, err)
assert.True(t, ok)

assert.Contains(t, stdout.String(), `"line":"training"`)
assert.NotContains(t, stdout.String(), `"type":"WAITING"`, "a run that already has logs is not waiting")
}

func TestEmitLogLineJSON(t *testing.T) {
var buf bytes.Buffer
emitLogLine(&buf, logRequest{node: 2, jsonOutput: true}, "hello")
Expand Down Expand Up @@ -812,7 +897,9 @@ func TestStreamBricklensActiveEmptyProbesMLflow(t *testing.T) {
logRequest{runID: 123, attempt: -1, tailLines: -1, jsonOutput: true},
logRunStatus{lifeCycleState: "RUNNING"})
require.ErrorIs(t, err, errBricklensFeatureDisabled)
assert.Empty(t, out.String())
// Only the WAITING status event precedes the fallback; no log lines.
assert.NotContains(t, out.String(), `"line"`)
assert.Contains(t, out.String(), `"type":"WAITING"`)
}

func TestFetchLogsFallsBackToMLflowWhenBricklensEmpty(t *testing.T) {
Expand Down
Loading