Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/logs-late-records.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air logs` on a run that ended in the last two minutes now waits for the log index to settle before printing, instead of sometimes dropping lines that were indexed late. ([#7663](https://github.com/databricks/cli/pull/7663))
124 changes: 124 additions & 0 deletions cmd/air/logs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -500,3 +500,127 @@ func TestRunLogsPreservesSubmittedRunURL(t *testing.T) {
require.NoError(t, err)
assert.True(t, strings.HasSuffix(buf.String(), "\n\nView job run at: "+runURL+"\n"), buf.String())
}

// lateIndexedRunServer serves a run that ended endedAgo ago. Its first tail read
// misses a record that Bricklens indexes late; later reads include it.
func lateIndexedRunServer(t *testing.T, endedAgo time.Duration, logReads *int) *httptest.Server {
t.Helper()
end := time.Now().Add(-endedAgo).UnixMilli()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/logs"):
*logReads++
if *logReads == 1 {
_, _ = w.Write([]byte(`{"log_records":[{"record_id":"b","time_unix_nano":20000000000,"body":"epoch 2"},{"record_id":"a","time_unix_nano":10000000000,"body":"epoch 1"}]}`))
return
}
_, _ = w.Write([]byte(`{"log_records":[{"record_id":"b","time_unix_nano":20000000000,"body":"epoch 2"},{"record_id":"late","time_unix_nano":15000000000,"body":"epoch 1.5"},{"record_id":"a","time_unix_nano":10000000000,"body":"epoch 1"}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get":
fmt.Fprintf(w, `{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`, end)
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)
return srv
}

// snapshotRunServer serves a terminal run whose successive log reads return
// snapshots in order, repeating the last one.
func snapshotRunServer(t *testing.T, runJSON string, snapshots []string, logReads *int) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/logs"):
_, _ = w.Write([]byte(snapshots[min(*logReads, len(snapshots)-1)]))
*logReads++
case r.URL.Path == "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(runJSON))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)
return srv
}

func runLogsCommand(t *testing.T, srvURL string, args ...string) string {
t.Helper()
var buf bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &buf, &buf, "", ""))
ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL))
cmd := withOutput(newLogsCommand(), flags.OutputText)
cmd.SetContext(ctx)
cmd.SetOut(&buf)
require.NoError(t, cmd.ParseFlags(args[1:]))
require.NoError(t, cmd.RunE(cmd, args[:1]))
return buf.String()
}

func TestLogsJustEndedFullTailSettlesOnRecordIdentity(t *testing.T) {
original := bricklensPollInterval
bricklensPollInterval = time.Millisecond
t.Cleanup(func() { bricklensPollInterval = original })

rec := func(id string, sec int) string {
return fmt.Sprintf(`{"record_id":%q,"time_unix_nano":%d,"body":%q}`, id, sec*1_000_000_000, id)
}
page := func(recs ...string) string { return `{"log_records":[` + strings.Join(recs, ",") + `]}` }
// A full --tail 2 window shifts as late records arrive without changing length.
snapshots := []string{
page(rec("B", 20), rec("A", 10)),
page(rec("C", 30), rec("B", 20)),
page(rec("D", 40), rec("C", 30)),
}
runJSON := fmt.Sprintf(`{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0}]}`, time.Now().Add(-5*time.Second).UnixMilli())
var logReads int
srv := snapshotRunServer(t, runJSON, snapshots, &logReads)

assert.Equal(t, "C\nD\n", runLogsCommand(t, srv.URL, "1", "--tail", "2"))
}

func TestLogsPastRetrySkipsSettleWhenOnlyRunJustEnded(t *testing.T) {
original := bricklensPollInterval
bricklensPollInterval = time.Millisecond
t.Cleanup(func() { bricklensPollInterval = original })

now := time.Now()
// Attempt 0 ended an hour ago; attempt 1 (and so the run) just ended.
runJSON := fmt.Sprintf(`{"run_id":1,"start_time":10000,"end_time":%d,"state":{"life_cycle_state":"TERMINATED","result_state":"SUCCESS"},"tasks":[{"run_id":2,"attempt_number":0,"end_time":%d},{"run_id":3,"attempt_number":1,"end_time":%d}]}`,
now.Add(-5*time.Second).UnixMilli(), now.Add(-time.Hour).UnixMilli(), now.Add(-5*time.Second).UnixMilli())
var logReads int
srv := snapshotRunServer(t, runJSON, []string{`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"retry 0 log"}]}`}, &logReads)

assert.Equal(t, "retry 0 log\n", runLogsCommand(t, srv.URL, "1", "--retry", "0"))
assert.Equal(t, 1, logReads, "a past attempt that ended long ago is read once")
}

func TestLogsJustEndedRunWaitsForLateRecords(t *testing.T) {
original := bricklensPollInterval
bricklensPollInterval = time.Millisecond
t.Cleanup(func() { bricklensPollInterval = original })

for _, tc := range []struct {
name string
endedAgo time.Duration
want string
}{
{"just ended", 5 * time.Second, "epoch 1\nepoch 1.5\nepoch 2\n"},
// An old run's index has settled, so it gets a single read.
{"ended long ago", time.Hour, "epoch 1\nepoch 2\n"},
} {
t.Run(tc.name, func(t *testing.T) {
var logReads int
srv := lateIndexedRunServer(t, tc.endedAgo, &logReads)
var buf bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &buf, &buf, "", ""))
ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))
cmd := withOutput(newLogsCommand(), flags.OutputText)
cmd.SetContext(ctx)
cmd.SetOut(&buf)

require.NoError(t, cmd.RunE(cmd, []string{"1"}))
assert.Equal(t, tc.want, buf.String())
})
}
}
91 changes: 76 additions & 15 deletions cmd/air/logstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ const (
bricklensLogLookback = 30 * time.Second
// A terminal job can have records that are not searchable yet.
bricklensTerminalEmptyPolls = 3
// Bricklens keeps indexing a run's records for a short while after it ends, so
// a tail read right after the end can miss lines from the middle of the log.
bricklensIndexSettleWindow = 2 * time.Minute
)

// statusMessageType tags a client-facing message packed into
Expand Down Expand Up @@ -129,6 +132,8 @@ type logRunStatus struct {
endTimeMs int64
// latestAttempt is the highest attempt_number across the run's tasks.
latestAttempt int
// attemptEndTimeMs maps each finished attempt_number to its task's end time.
attemptEndTimeMs map[int]int64
}

// retryTracker reports each retry once across either Jobs transition:
Expand Down Expand Up @@ -200,6 +205,20 @@ func (s logRunStatus) terminal() bool {
return terminalLifeCycleStates[s.lifeCycleState] || terminalResultStates[s.resultState]
}

// attemptEndedWithin reports whether the given attempt (-1 for the latest) of a
// terminal run ended less than d before now. A past attempt uses its own task's
// end time: it may have ended long before the run did.
func (s logRunStatus) attemptEndedWithin(attempt int, now time.Time, d time.Duration) bool {
if !s.terminal() {
return false
}
endMs := s.endTimeMs
if attempt >= 0 && attempt != s.latestAttempt {
endMs = s.attemptEndTimeMs[attempt]
}
return endMs > 0 && now.Sub(time.UnixMilli(endMs)) < d
}

func (s logRunStatus) succeeded() bool {
return s.resultState == "SUCCESS"
}
Expand Down Expand Up @@ -254,6 +273,12 @@ func projectRunStatus(run *jobs.Run) logRunStatus {
}
for i := range run.Tasks {
task := &run.Tasks[i]
if task.EndTime > 0 {
if s.attemptEndTimeMs == nil {
s.attemptEndTimeMs = map[int]int64{}
}
s.attemptEndTimeMs[task.AttemptNumber] = max(s.attemptEndTimeMs[task.AttemptNumber], task.EndTime)
}
if i > 0 && task.AttemptNumber <= s.latestAttempt {
continue
}
Expand Down Expand Up @@ -581,40 +606,76 @@ func (req logRequest) tailTarget() int {
// returns records newest-first, so it pages until it has `target`, keeps the
// newest `target`, and reverses to chronological order.
func (st *bricklensStreamer) drainTail(toSec int64, remember bool) error {
collected, err := st.fetchTail(toSec)
if err != nil {
return err
}
if st.status.attemptEndedWithin(st.req.attempt, time.Now(), bricklensIndexSettleWindow) {
collected, err = st.settleTail(toSec, collected)
if err != nil {
return err
}
}

// collected is newest first; print oldest -> newest.
for _, c := range slices.Backward(collected) {
st.emit(c.Body)
if remember {
st.seen.add(c)
st.lastNano = max(st.lastNano, c.nano())
}
}
if remember && st.lastNano != 0 {
st.fromSec = max(st.streamStartSec, st.lastNano/1_000_000_000-int64(bricklensLogLookback/time.Second))
}
return nil
}

// settleTail re-reads the tail of a run that just ended until two reads return
// the same records, so lines Bricklens indexes late are not dropped. Comparing
// identities rather than counts matters once the tail is full: a late record
// then shifts the window without changing its length.
func (st *bricklensStreamer) settleTail(toSec int64, collected []logRecord) ([]logRecord, error) {
for range bricklensTerminalEmptyPolls {
if err := sleepOrCancel(st.ctx, bricklensPollInterval); err != nil {
return nil, err
}
next, err := st.fetchTail(toSec)
if err != nil {
return nil, err
}
if slices.EqualFunc(next, collected, func(a, b logRecord) bool { return seenRecordKey(a) == seenRecordKey(b) }) {
return next, nil
}
collected = next
}
return collected, nil
}

// fetchTail returns up to the tail target's newest records, newest first.
func (st *bricklensStreamer) fetchTail(toSec int64) ([]logRecord, error) {
target := st.req.tailTarget()
if target <= 0 {
return nil
return nil, nil
}

var collected []logRecord
var pageToken string
for len(collected) < target {
resp, err := st.requestPage(pageToken, toSec, target, false)
if err != nil {
return err
return nil, err
}
collected = append(collected, resp.LogRecords...)
pageToken = resp.NextPageToken
if pageToken == "" {
break
}
}

// Keep the newest `target`, then reverse to print oldest -> newest.
if len(collected) > target {
collected = collected[:target]
}
for _, c := range slices.Backward(collected) {
st.emit(c.Body)
if remember {
st.seen.add(c)
st.lastNano = max(st.lastNano, c.nano())
}
}
if remember && st.lastNano != 0 {
st.fromSec = max(st.streamStartSec, st.lastNano/1_000_000_000-int64(bricklensLogLookback/time.Second))
}
return nil
return collected, nil
}

// drainPages exhausts all pages from the current from-second in ascending order.
Expand Down
Loading