diff --git a/.nextchanges/air/logs-follow.md b/.nextchanges/air/logs-follow.md new file mode 100644 index 00000000000..0f5dbd97520 --- /dev/null +++ b/.nextchanges/air/logs-follow.md @@ -0,0 +1 @@ +* **Behavior change:** `databricks air logs` now prints the logs available now and exits, also for an active run, matching `kubectl logs`, `docker logs` and `databricks apps logs`. Pass `--follow` (`-f`) to stream new lines until the run completes; `--tail N --follow` prints the last N lines first. `databricks air run --watch` still submits and streams. ([#7658](https://github.com/databricks/cli/pull/7658)) diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 83c7659f07d..8d3d5f0bea6 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -5,8 +5,8 @@ Run GPU workloads on Databricks serverless compute. Describe a workload in a YAML file and submit it with "databricks air run". -Follow a run with "databricks air logs" and "databricks air get", or see your -active runs with "databricks air list". +Follow a run with "databricks air logs --follow" and "databricks air get", or +see your active runs with "databricks air list". Usage: databricks air [command] @@ -18,7 +18,7 @@ Manage runs: cancel Cancel one or more runs get Show a run's status, configuration, and timing list List your active runs - logs Stream or fetch a run's logs + logs Print or follow a run's logs Package and deploy: convert-to-dabs Convert a workload YAML into a Databricks Asset Bundle @@ -192,26 +192,30 @@ Global Flags: === logs help >>> [CLI] air logs --help -Stream logs from an active run until it completes, or fetch logs from a -completed run. +Print a run's logs and exit, or follow an active run with --follow. -JOB_RUN_ID is the job run ID returned by "databricks air run". +JOB_RUN_ID is the job run ID returned by "databricks air run". By default this +prints the logs available now: the last 10000 lines, or --tail N lines. With +--follow it prints them and then streams new lines until the run completes. Usage: databricks air logs JOB_RUN_ID [flags] Examples: databricks air logs 123456789 + databricks air logs 123456789 --tail 50 + databricks air logs 123456789 --follow databricks air logs 123456789 --node 1 --retry 0 databricks air logs 123456789 --download-to ./logs Flags: - --download-to string Download all logs to this directory instead of printing; not combinable with --tail or --minutes - -h, --help[=true|false] help for logs - --minutes int Fetch only logs from the last N minutes - --node int Fetch logs from this node - --retry int View logs from a specific retry attempt; -1 means latest (default -1) - --tail int Print the last N existing lines before following, or from a completed run (default 10000) + --download-to string Download all logs to this directory instead of printing; not combinable with --tail, --minutes, or --follow + -f, --follow[=true|false] Stream new lines until the run completes + -h, --help[=true|false] help for logs + --minutes int Fetch only logs from the last N minutes + --node int Fetch logs from this node + --retry int View logs from a specific retry attempt; -1 means latest (default -1) + --tail int Print the last N lines (default 10000); with --follow, before streaming new ones Global Flags: --debug[=true|false] enable debug logging diff --git a/acceptance/air/run-submit-databricks-ai/output.txt b/acceptance/air/run-submit-databricks-ai/output.txt index 5e19f717249..202f4936f21 100644 --- a/acceptance/air/run-submit-databricks-ai/output.txt +++ b/acceptance/air/run-submit-databricks-ai/output.txt @@ -8,7 +8,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === submit includes the Databricks AI environment version and omits dependencies diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index ae716c0bdfd..36f5f50014f 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -8,7 +8,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === only config + command are uploaded; no requirements.yaml diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 3e1f5e69ed3..41b41d3169a 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -11,7 +11,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === the uploaded config preserves source order and contains overrides diff --git a/acceptance/air/run-submit/output.txt b/acceptance/air/run-submit/output.txt index 1ca32b0df45..57cd811815c 100644 --- a/acceptance/air/run-submit/output.txt +++ b/acceptance/air/run-submit/output.txt @@ -9,7 +9,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === the ai_runtime_task and additive job + experiment permission grants @@ -116,7 +116,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === the B300 accelerator and runtime reach the submit payload @@ -167,7 +167,7 @@ View job run at: [DATABRICKS_URL]/jobs/runs/555?o=[WORKSPACE_ID] Tip: use --watch when submitting a run to stream logs to your terminal. Stream logs after submission using: - databricks air logs 555 + databricks air logs 555 --follow === the RTX6000 accelerator and runtime reach the submit payload diff --git a/cmd/air/air.go b/cmd/air/air.go index 07f4c9281a5..7bb15f1e995 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -37,8 +37,8 @@ func buildCommand(ctx context.Context, cmd *cobra.Command) { cmd.Long = `Run GPU workloads on Databricks serverless compute. Describe a workload in a YAML file and submit it with "databricks air run". -Follow a run with "databricks air logs" and "databricks air get", or see your -active runs with "databricks air list".` +Follow a run with "databricks air logs --follow" and "databricks air get", or +see your active runs with "databricks air list".` cmd.AddGroup( &cobra.Group{ID: runGroupID, Title: "Run workloads:"}, diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 2a6d83f2e4c..07950874809 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -26,26 +26,31 @@ func newLogsCommand() *cobra.Command { retry int downloadTo string review bool + follow bool ) cmd := &cobra.Command{ Use: "logs JOB_RUN_ID", Args: root.ExactArgs(1), - Short: "Stream or fetch a run's logs", - Long: `Stream logs from an active run until it completes, or fetch logs from a -completed run. + Short: "Print or follow a run's logs", + Long: `Print a run's logs and exit, or follow an active run with --follow. -JOB_RUN_ID is the job run ID returned by "databricks air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run". By default this +prints the logs available now: the last 10000 lines, or --tail N lines. With +--follow it prints them and then streams new lines until the run completes.`, Example: ` databricks air logs 123456789 + databricks air logs 123456789 --tail 50 + databricks air logs 123456789 --follow databricks air logs 123456789 --node 1 --retry 0 databricks air logs 123456789 --download-to ./logs`, } cmd.Flags().IntVar(&node, "node", 0, "Fetch logs from this node") - cmd.Flags().IntVar(&tail, "tail", 0, "Print the last N existing lines before following, or from a completed run (default 10000)") + cmd.Flags().IntVar(&tail, "tail", 0, "Print the last N lines (default 10000); with --follow, before streaming new ones") cmd.Flags().IntVar(&minutes, "minutes", 0, "Fetch only logs from the last N minutes") cmd.Flags().IntVar(&retry, "retry", -1, "View logs from a specific retry attempt; -1 means latest") - cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing; not combinable with --tail or --minutes") + cmd.Flags().BoolVarP(&follow, "follow", "f", false, "Stream new lines until the run completes") + cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing; not combinable with --tail, --minutes, or --follow") cmd.Flags().BoolVar(&review, "review", false, "Download logs from all nodes and filter for error signatures") cmd.Flags().MarkHidden("review") @@ -75,6 +80,11 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, errors.New("--download-to writes complete logs, so it cannot be combined with --tail or --minutes")) } + if downloadTo != "" && follow { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, + errors.New("--download-to writes the logs available now, so it cannot be combined with --follow")) + } + // --tail (line tail) and --minutes (time window) answer the same question // two ways, so reject both together rather than silently honoring one. if tail > 0 && minutes > 0 { @@ -112,9 +122,10 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, // Only the streaming path prints resume guidance, so only it catches the // interrupt: without this a Ctrl-C kills the process before - // handleWatchResult runs. Download and JSON keep the default handling. + // handleWatchResult runs. Printing, download and JSON keep the default + // handling. streamCtx := ctx - if downloadTo == "" && root.OutputType(cmd) != flags.OutputJSON { + if follow && root.OutputType(cmd) != flags.OutputJSON { var stop context.CancelFunc streamCtx, stop = notifyInterrupt(ctx) defer stop() @@ -130,6 +141,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, boundInitialLogs: minutes == 0, downloadTo: downloadTo, jsonOutput: root.OutputType(cmd) == flags.OutputJSON, + printOnce: !follow, }) if downloadTo != "" || root.OutputType(cmd) == flags.OutputJSON { return err @@ -186,8 +198,11 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { return nil } - // A past retry of an active run has immutable logs: render once, don't follow. - if req.attempt >= 0 && req.attempt < status.latestAttempt && !status.terminal() { + // Without --follow an active run's logs print once. A past retry of an active + // run has immutable logs, so it renders once either way. A completed run keeps + // the regular path, which already prints and exits. + pastRetry := req.attempt >= 0 && req.attempt < status.latestAttempt + if (req.printOnce || pastRetry) && !status.terminal() { req.staticView = true } diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index f89b1b8b9d8..769ef34abd3 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -400,12 +400,13 @@ func TestLogsCommandPrintsGuidanceWhenStreamingIsInterrupted(t *testing.T) { cmd := withOutput(newLogsCommand(), flags.OutputText) cmd.SetContext(ctx) cmd.SetOut(&buf) + require.NoError(t, cmd.Flags().Set("follow", "true")) err := cmd.RunE(cmd, []string{"5"}) require.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Contains(t, buf.String(), "View job run at: "+srv.URL+"/jobs/runs/5?o=42\n\nStreaming logs interrupted.") assert.Contains(t, buf.String(), "To check status:\ndatabricks air get 5 -p 'team profile'") - assert.Contains(t, buf.String(), "To resume streaming logs:\ndatabricks air logs 5 -p 'team profile'") + assert.Contains(t, buf.String(), "To resume streaming logs:\ndatabricks air logs 5 --follow -p 'team profile'") } // activeRunPastRetryServer serves a still-RUNNING run with two attempts and a @@ -436,6 +437,43 @@ func activeRunPastRetryServer(t *testing.T, getRunHits *int) *httptest.Server { return srv } +func TestLogsPrintsActiveRunOnceByDefault(t *testing.T) { + var runGets, logReads int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets++ + _, _ = w.Write([]byte(`{"run_id":1,"start_time":10000,"state":{"life_cycle_state":"RUNNING"},"tasks":[{"run_id":2,"attempt_number":0}]}`)) + case strings.HasSuffix(r.URL.Path, "/logs"): + logReads++ + _, _ = w.Write([]byte(`{"log_records":[{"record_id":"a","time_unix_nano":20000000000,"body":"epoch 1"}]}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + // Without --follow an active run prints what is there and exits 0, with no + // run link (that only frames a stream). + assert.Equal(t, "epoch 1\n", runLogsCommand(t, srv.URL, "1", "--tail", "50")) + assert.Equal(t, 1, runGets, "no status polling after the initial resolve") + assert.Equal(t, 1, logReads) +} + +func TestLogsRejectsFollowWithDownload(t *testing.T) { + var buf bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &buf, &buf, "", "")) + cmd := withOutput(newLogsCommand(), flags.OutputText) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + cmd.SetErr(&buf) + require.NoError(t, cmd.ParseFlags([]string{"--follow", "--download-to", t.TempDir()})) + + err := cmd.RunE(cmd, []string{"1"}) + require.Error(t, err) + assert.Contains(t, err.Error()+buf.String(), "cannot be combined with --follow") +} + func TestLogsPastRetryOfActiveRunIsStatic(t *testing.T) { var getRunHits int srv := activeRunPastRetryServer(t, &getRunHits) @@ -469,6 +507,7 @@ func TestLogsCommandRepeatsRunLinkAfterStreaming(t *testing.T) { cmd := withOutput(newLogsCommand(), output) cmd.SetContext(ctx) cmd.SetOut(&buf) + require.NoError(t, cmd.Flags().Set("follow", "true")) err := cmd.RunE(cmd, []string{"777"}) if result == "FAILED" { diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index e7af29cb19e..1e994e1ec46 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -106,10 +106,11 @@ type logRequest struct { tailLines int // downloadTo, when set, writes logs to that directory instead of stdout. downloadTo string - // staticView renders a one-shot tail instead of following the run. Set for a - // past retry of an active run: that attempt's logs are immutable, so streaming - // would poll forever waiting for the run (not the attempt) to finish. + // staticView renders snapshots and immutable past retries without following an active run. staticView bool + // printOnce prints an active run's available logs and exits instead of + // following it (air logs without --follow). + printOnce bool // boundInitialLogs limits existing output before following an active run. boundInitialLogs bool jsonOutput bool @@ -457,8 +458,6 @@ func (st *bricklensStreamer) run() (bool, error) { st.fromSec = st.req.fromSeconds(st.status, now) st.streamStartSec = st.fromSec - // A past retry's logs are immutable: render a one-shot tail rather than - // following the still-active run, which would poll forever. if st.req.staticView { return st.drainStatic(st.req.toSeconds(st.status)) } @@ -520,8 +519,7 @@ func (st *bricklensStreamer) run() (bool, error) { terminal := st.status.terminal() toSec := st.req.toSeconds(st.status) - // `air logs` starts an active attachment with a bounded tail; subsequent - // polls follow live with dedup so the overlap does not re-print it. + // Remember an active attachment's initial tail so overlapping polls do not reprint records. var emitted int var err error if firstIteration && (terminal || st.req.boundInitialLogs) { diff --git a/cmd/air/run.go b/cmd/air/run.go index 7551d1b661a..8417fa94045 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -320,7 +320,7 @@ func loadComputeOptionsForHelp(cmd *cobra.Command, args []string) []computeOptio } func airLogsCommand(profile, runID string) string { - args := []string{"databricks", "air", "logs", shellquote.BashArg(runID)} + args := []string{"databricks", "air", "logs", shellquote.BashArg(runID), "--follow"} if profile != "" { args = append(args, "-p", shellquote.BashArg(profile)) } diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index cedae60e672..78f4deb3129 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -81,7 +81,7 @@ func TestRunSubmitTextOutput(t *testing.T) { assert.Contains(t, out, "/jobs/runs/555") assert.Contains(t, out, "Tip: use --watch when submitting a run to stream logs to your terminal.") assert.Contains(t, out, "Stream logs after submission using:") - assert.Contains(t, out, "databricks air logs 555") + assert.Contains(t, out, "databricks air logs 555 --follow") assert.NotContains(t, out, "View MLflow run at:") assert.Zero(t, counts.runGet.Load(), "bare submission must not poll runs/get") assert.Zero(t, counts.runGetOutput.Load(), "bare submission must not poll runs/get-output") @@ -104,13 +104,13 @@ func TestRunSubmitTextOutputIncludesProfileInLogsCommand(t *testing.T) { err := runSubmitCmdWithProfile(t, flags.OutputText, &buf, srv.URL, "team profile") require.NoError(t, err) - assert.Contains(t, buf.String(), "databricks air logs 555 -p 'team profile'") + assert.Contains(t, buf.String(), "databricks air logs 555 --follow -p 'team profile'") } func TestAirLogsCommand(t *testing.T) { - assert.Equal(t, "databricks air logs 123", airLogsCommand("", "123")) - assert.Equal(t, "databricks air logs 123 -p profile-name", airLogsCommand("profile-name", "123")) - assert.Equal(t, "databricks air logs 123 -p 'team profile'", airLogsCommand("team profile", "123")) + assert.Equal(t, "databricks air logs 123 --follow", airLogsCommand("", "123")) + assert.Equal(t, "databricks air logs 123 --follow -p profile-name", airLogsCommand("profile-name", "123")) + assert.Equal(t, "databricks air logs 123 --follow -p 'team profile'", airLogsCommand("team profile", "123")) } func TestAirGetCommand(t *testing.T) { diff --git a/cmd/air/run_watch_test.go b/cmd/air/run_watch_test.go index 13c21923057..210cd33f3b2 100644 --- a/cmd/air/run_watch_test.go +++ b/cmd/air/run_watch_test.go @@ -214,7 +214,7 @@ func TestHandleWatchResultPrintsProfileAwareResumeCommand(t *testing.T) { out := buf.String() assert.Contains(t, out, "Streaming logs interrupted.") assert.Contains(t, out, "To check status:\ndatabricks air get 777 -p 'team profile'") - assert.Contains(t, out, "To resume streaming logs:\ndatabricks air logs 777 -p 'team profile'") + assert.Contains(t, out, "To resume streaming logs:\ndatabricks air logs 777 --follow -p 'team profile'") assert.NotContains(t, out, "The workload was not canceled") }