From 0fb5422faa97944b1b56d97a0570cfe014f29b26 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 07:18:37 +0000 Subject: [PATCH 1/3] air logs: wait for the log index to settle on a just-ended run Bricklens keeps indexing a run's records for a short while after it ends. `air logs` on a run that already ended reads the tail once, so a read right after the end could drop lines from the middle of the log; an agent saw epochs 3, 6 and 10 of 10 missing while --download-to had all of them. When the run ended in the last two minutes, re-read the tail until two reads agree (at most three extra reads) before printing. Older runs are unchanged. Co-authored-by: Isaac --- .nextchanges/air/logs-late-records.md | 1 + cmd/air/logs_test.go | 54 ++++++++++++++++++++ cmd/air/logstream.go | 72 +++++++++++++++++++++------ 3 files changed, 112 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/air/logs-late-records.md diff --git a/.nextchanges/air/logs-late-records.md b/.nextchanges/air/logs-late-records.md new file mode 100644 index 00000000000..6f08d7ceebd --- /dev/null +++ b/.nextchanges/air/logs-late-records.md @@ -0,0 +1 @@ +* `databricks air logs` on a run that ended in the last two minutes now waits for the log index to settle before printing, instead of sometimes dropping lines that were indexed late. diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 224df3e374b..67dd52ec799 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -500,3 +500,57 @@ func TestRunLogsPreservesSubmittedRunURL(t *testing.T) { require.NoError(t, err) assert.True(t, strings.HasSuffix(buf.String(), "\n\nView job run at: "+runURL+"\n"), buf.String()) } + +// lateIndexedRunServer serves a run that ended endedAgo ago. Its first tail read +// misses a record that Bricklens indexes late; later reads include it. +func lateIndexedRunServer(t *testing.T, endedAgo time.Duration, logReads *int) *httptest.Server { + t.Helper() + end := time.Now().Add(-endedAgo).UnixMilli() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + *logReads++ + if *logReads == 1 { + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"b","time_unix_nano":20000000000,"body":"epoch 2"},{"record_id":"a","time_unix_nano":10000000000,"body":"epoch 1"}]}`)) + return + } + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"b","time_unix_nano":20000000000,"body":"epoch 2"},{"record_id":"late","time_unix_nano":15000000000,"body":"epoch 1.5"},{"record_id":"a","time_unix_nano":10000000000,"body":"epoch 1"}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + fmt.Fprintf(w, `{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`, end) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestLogsJustEndedRunWaitsForLateRecords(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + for _, tc := range []struct { + name string + endedAgo time.Duration + want string + }{ + {"just ended", 5 * time.Second, "epoch 1\nepoch 1.5\nepoch 2\n"}, + // An old run's index has settled, so it gets a single read. + {"ended long ago", time.Hour, "epoch 1\nepoch 2\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + var logReads int + srv := lateIndexedRunServer(t, tc.endedAgo, &logReads) + var buf bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL)) + cmd := withOutput(newLogsCommand(), flags.OutputText) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + + require.NoError(t, cmd.RunE(cmd, []string{"1"})) + assert.Equal(t, tc.want, buf.String()) + }) + } +} diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index a431ecb6af1..05290a7f67a 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -38,6 +38,9 @@ const ( bricklensLogLookback = 30 * time.Second // A terminal job can have records that are not searchable yet. bricklensTerminalEmptyPolls = 3 + // Bricklens keeps indexing a run's records for a short while after it ends, so + // a tail read right after the end can miss lines from the middle of the log. + bricklensIndexSettleWindow = 2 * time.Minute ) // statusMessageType tags a client-facing message packed into @@ -200,6 +203,11 @@ func (s logRunStatus) terminal() bool { return terminalLifeCycleStates[s.lifeCycleState] || terminalResultStates[s.resultState] } +// endedWithin reports whether a terminal run ended less than d before now. +func (s logRunStatus) endedWithin(now time.Time, d time.Duration) bool { + return s.terminal() && s.endTimeMs > 0 && now.Sub(time.UnixMilli(s.endTimeMs)) < d +} + func (s logRunStatus) succeeded() bool { return s.resultState == "SUCCESS" } @@ -581,9 +589,55 @@ func (req logRequest) tailTarget() int { // returns records newest-first, so it pages until it has `target`, keeps the // newest `target`, and reverses to chronological order. func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error { + collected, err := st.fetchTail(toSec) + if err != nil { + return err + } + if st.status.endedWithin(time.Now(), bricklensIndexSettleWindow) { + collected, err = st.settleTail(toSec, collected) + if err != nil { + return err + } + } + + // collected is newest first; print oldest -> newest. + for _, c := range slices.Backward(collected) { + st.emit(c.Body) + if remember { + st.seen.add(c) + st.lastNano = max(st.lastNano, c.nano()) + } + } + if remember && st.lastNano != 0 { + st.fromSec = max(st.streamStartSec, st.lastNano/1_000_000_000-int64(bricklensLogLookback/time.Second)) + } + return nil +} + +// settleTail re-reads the tail of a run that just ended until two reads return +// the same number of records, so lines Bricklens indexes late are not dropped. +func (st *bricklensStreamer) settleTail(toSec int64, collected []logRecord) ([]logRecord, error) { + for range bricklensTerminalEmptyPolls { + if err := sleepOrCancel(st.ctx, bricklensPollInterval); err != nil { + return nil, err + } + next, err := st.fetchTail(toSec) + if err != nil { + return nil, err + } + if len(next) == len(collected) { + return next, nil + } + collected = next + } + return collected, nil +} + +// fetchTail returns up to the tail target's newest records, newest first. +func (st *bricklensStreamer) fetchTail(toSec int64) ([]logRecord, error) { target := st.req.tailTarget() if target <= 0 { - return nil + return nil, nil } var collected []logRecord @@ -591,7 +645,7 @@ func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error { for len(collected) < target { resp, err := st.requestPage(pageToken, toSec, target, false) if err != nil { - return err + return nil, err } collected = append(collected, resp.LogRecords...) pageToken = resp.NextPageToken @@ -599,22 +653,10 @@ func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error { break } } - - // Keep the newest `target`, then reverse to print oldest -> newest. if len(collected) > target { collected = collected[:target] } - for _, c := range slices.Backward(collected) { - st.emit(c.Body) - if remember { - st.seen.add(c) - st.lastNano = max(st.lastNano, c.nano()) - } - } - if remember && st.lastNano != 0 { - st.fromSec = max(st.streamStartSec, st.lastNano/1_000_000_000-int64(bricklensLogLookback/time.Second)) - } - return nil + return collected, nil } // drainPages exhausts all pages from the current from-second in ascending order. From 77a1c6fcf042495ec07d0822eb14866a414d6abd Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 07:18:42 +0000 Subject: [PATCH 2/3] Add PR link to changelog fragment Co-authored-by: Isaac --- .nextchanges/air/logs-late-records.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/logs-late-records.md b/.nextchanges/air/logs-late-records.md index 6f08d7ceebd..94eeec04615 100644 --- a/.nextchanges/air/logs-late-records.md +++ b/.nextchanges/air/logs-late-records.md @@ -1 +1 @@ -* `databricks air logs` on a run that ended in the last two minutes now waits for the log index to settle before printing, instead of sometimes dropping lines that were indexed late. +* `databricks air logs` on a run that ended in the last two minutes now waits for the log index to settle before printing, instead of sometimes dropping lines that were indexed late. ([#7663](https://github.com/databricks/cli/pull/7663)) From ad4cd8a302c8cd1ea248d9f9ef2f3ae56db83679 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 20:42:20 +0000 Subject: [PATCH 3/3] air logs: settle on record identity and the read attempt's end time Address review: a full tail can shift without changing length, so compare record identities between reads. And key the settle window to the end time of the attempt being read, so --retry N on an attempt that ended long ago reads once even if the run just ended. Co-authored-by: Isaac --- cmd/air/logs_test.go | 70 ++++++++++++++++++++++++++++++++++++++++++++ cmd/air/logstream.go | 31 ++++++++++++++++---- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 67dd52ec799..0c94e9d9e02 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -525,6 +525,76 @@ func lateIndexedRunServer(t *testing.T, endedAgo time.Duration, logReads *int) * return srv } +// snapshotRunServer serves a terminal run whose successive log reads return +// snapshots in order, repeating the last one. +func snapshotRunServer(t *testing.T, runJSON string, snapshots []string, logReads *int) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = w.Write([]byte(snapshots[min(*logReads, len(snapshots)-1)])) + *logReads++ + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(runJSON)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func runLogsCommand(t *testing.T, srvURL string, args ...string) string { + t.Helper() + var buf bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + cmd := withOutput(newLogsCommand(), flags.OutputText) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + require.NoError(t, cmd.ParseFlags(args[1:])) + require.NoError(t, cmd.RunE(cmd, args[:1])) + return buf.String() +} + +func TestLogsJustEndedFullTailSettlesOnRecordIdentity(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + rec := func(id string, sec int) string { + return fmt.Sprintf(`{"record_id":%q,"time_unix_nano":%d,"body":%q}`, id, sec*1_000_000_000, id) + } + page := func(recs ...string) string { return `{"log_records":[` + strings.Join(recs, ",") + `]}` } + // A full --tail 2 window shifts as late records arrive without changing length. + snapshots := []string{ + page(rec("B", 20), rec("A", 10)), + page(rec("C", 30), rec("B", 20)), + page(rec("D", 40), rec("C", 30)), + } + runJSON := fmt.Sprintf(`{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`, time.Now().Add(-5*time.Second).UnixMilli()) + var logReads int + srv := snapshotRunServer(t, runJSON, snapshots, &logReads) + + assert.Equal(t, "C\nD\n", runLogsCommand(t, srv.URL, "1", "--tail", "2")) +} + +func TestLogsPastRetrySkipsSettleWhenOnlyRunJustEnded(t *testing.T) { + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + + now := time.Now() + // Attempt 0 ended an hour ago; attempt 1 (and so the run) just ended. + runJSON := fmt.Sprintf(`{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0,"end_time":%d},{"run_id":3,"attempt_number":1,"end_time":%d}]}`, + now.Add(-5*time.Second).UnixMilli(), now.Add(-time.Hour).UnixMilli(), now.Add(-5*time.Second).UnixMilli()) + var logReads int + srv := snapshotRunServer(t, runJSON, []string{`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"retry 0 log"}]}`}, &logReads) + + assert.Equal(t, "retry 0 log\n", runLogsCommand(t, srv.URL, "1", "--retry", "0")) + assert.Equal(t, 1, logReads, "a past attempt that ended long ago is read once") +} + func TestLogsJustEndedRunWaitsForLateRecords(t *testing.T) { original := bricklensPollInterval bricklensPollInterval = time.Millisecond diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index 05290a7f67a..482d6515d32 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -132,6 +132,8 @@ type logRunStatus struct { endTimeMs int64 // latestAttempt is the highest attempt_number across the run's tasks. latestAttempt int + // attemptEndTimeMs maps each finished attempt_number to its task's end time. + attemptEndTimeMs map[int]int64 } // retryTracker reports each retry once across either Jobs transition: @@ -203,9 +205,18 @@ func (s logRunStatus) terminal() bool { return terminalLifeCycleStates[s.lifeCycleState] || terminalResultStates[s.resultState] } -// endedWithin reports whether a terminal run ended less than d before now. -func (s logRunStatus) endedWithin(now time.Time, d time.Duration) bool { - return s.terminal() && s.endTimeMs > 0 && now.Sub(time.UnixMilli(s.endTimeMs)) < d +// attemptEndedWithin reports whether the given attempt (-1 for the latest) of a +// terminal run ended less than d before now. A past attempt uses its own task's +// end time: it may have ended long before the run did. +func (s logRunStatus) attemptEndedWithin(attempt int, now time.Time, d time.Duration) bool { + if !s.terminal() { + return false + } + endMs := s.endTimeMs + if attempt >= 0 && attempt != s.latestAttempt { + endMs = s.attemptEndTimeMs[attempt] + } + return endMs > 0 && now.Sub(time.UnixMilli(endMs)) < d } func (s logRunStatus) succeeded() bool { @@ -262,6 +273,12 @@ func projectRunStatus(run *jobs.Run) logRunStatus { } for i := range run.Tasks { task := &run.Tasks[i] + if task.EndTime > 0 { + if s.attemptEndTimeMs == nil { + s.attemptEndTimeMs = map[int]int64{} + } + s.attemptEndTimeMs[task.AttemptNumber] = max(s.attemptEndTimeMs[task.AttemptNumber], task.EndTime) + } if i > 0 && task.AttemptNumber <= s.latestAttempt { continue } @@ -593,7 +610,7 @@ func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error { if err != nil { return err } - if st.status.endedWithin(time.Now(), bricklensIndexSettleWindow) { + if st.status.attemptEndedWithin(st.req.attempt, time.Now(), bricklensIndexSettleWindow) { collected, err = st.settleTail(toSec, collected) if err != nil { return err @@ -615,7 +632,9 @@ func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error { } // settleTail re-reads the tail of a run that just ended until two reads return -// the same number of records, so lines Bricklens indexes late are not dropped. +// the same records, so lines Bricklens indexes late are not dropped. Comparing +// identities rather than counts matters once the tail is full: a late record +// then shifts the window without changing its length. func (st *bricklensStreamer) settleTail(toSec int64, collected []logRecord) ([]logRecord, error) { for range bricklensTerminalEmptyPolls { if err := sleepOrCancel(st.ctx, bricklensPollInterval); err != nil { @@ -625,7 +644,7 @@ func (st *bricklensStreamer) settleTail(toSec int64, collected []logRecord) ([]l if err != nil { return nil, err } - if len(next) == len(collected) { + if slices.EqualFunc(next, collected, func(a, b logRecord) bool { return seenRecordKey(a) == seenRecordKey(b) }) { return next, nil } collected = next