From a65db32a2b92983f85e84a5971933e77004d0941 Mon Sep 17 00:00:00 2001 From: "ethan.ma" Date: Mon, 16 Nov 2026 18:48:32 +0000 Subject: [PATCH 1/2] air: Disable MLflow artifact log fallback by default --- .nextchanges/air/bricklens-logs-only.md | 1 + cmd/air/logdownload.go | 94 ++++++++++++++++++-- cmd/air/logdownload_test.go | 112 ++++++++++++++++++++++++ cmd/air/logs.go | 18 +++- cmd/air/logs_test.go | 26 +++++- cmd/air/logstream.go | 38 ++++---- cmd/air/logstream_test.go | 54 ++++++++++++ cmd/air/run_watch_test.go | 1 + 8 files changed, 316 insertions(+), 28 deletions(-) create mode 100644 .nextchanges/air/bricklens-logs-only.md diff --git a/.nextchanges/air/bricklens-logs-only.md b/.nextchanges/air/bricklens-logs-only.md new file mode 100644 index 00000000000..2faabab41b8 --- /dev/null +++ b/.nextchanges/air/bricklens-logs-only.md @@ -0,0 +1 @@ +* Use Bricklens for AIR log streaming and downloads by default; temporarily restore the legacy MLflow artifact fallback and downloads with `DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true`. diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index a33a0d92ab9..69540a43c1b 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -10,10 +10,13 @@ import ( "path/filepath" "slices" "sync" + "time" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/jobs" "golang.org/x/sync/errgroup" ) @@ -56,7 +59,7 @@ func resolveNodeCount(run *jobs.Run) (int, error) { // downloadLogs writes each node's logs to /logs/node_.log and // prints a summary. An explicit --node downloads only that node; otherwise all of -// them. Logs come from MLflow artifacts, since Bricklens only streams. The +// them. Logs come from Bricklens unless legacy artifact downloads are enabled. The // returned bool is the run's outcome, so the exit code matches the streaming path. func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) { numNodes, err := resolveNodeCount(run) @@ -79,10 +82,14 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri if err != nil { return false, err } - ids := mlflowIDsForTask(ctx, w, taskRunID) - if ids == nil || ids.RunID == "" { - emitNoLogs(out, req, status) - return status.downloadOutcome(), nil + var mlflowRunID string + if mlflowLogFallbackEnabled(ctx) { + ids := mlflowIDsForTask(ctx, w, taskRunID) + if ids == nil || ids.RunID == "" { + emitNoLogs(out, req, status) + return status.downloadOutcome(), nil + } + mlflowRunID = ids.RunID } dir, err := filepath.Abs(req.downloadTo) @@ -95,7 +102,16 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri return false, fmt.Errorf("failed to create %s: %w", dir, err) } - nodeLogs, failures, err := downloadAllNodeLogs(ctx, w, ids.RunID, dir, nodes, attempt) + var nodeLogs map[int]string + var failures map[int]string + if mlflowLogFallbackEnabled(ctx) { + nodeLogs, failures, err = downloadAllNodeLogs(ctx, w, mlflowRunID, dir, nodes, attempt) + } else { + req.attempt = attempt + nodeLogs, failures, err = downloadNodes(ctx, nodes, func(ctx context.Context, node int) (string, error) { + return downloadBricklensNodeLog(ctx, w, dir, node, req, status) + }) + } if err != nil { return false, err } @@ -139,13 +155,20 @@ func downloadAllNodeLogs(ctx context.Context, w *databricks.WorkspaceClient, mlf return nil, nil, err } + return downloadNodes(ctx, nodes, func(ctx context.Context, node int) (string, error) { + return downloadNodeLog(ctx, w, mlflowRunID, node, attemptDir, withAttempt, dir) + }) +} + +// downloadNodes bounds concurrency and retains partial files alongside their errors. +func downloadNodes(ctx context.Context, nodes []int, download func(context.Context, int) (string, error)) (map[int]string, map[int]string, error) { paths := make([]string, len(nodes)) reasons := make([]string, len(nodes)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(downloadConcurrency) for i, node := range nodes { g.Go(func() error { - path, err := downloadNodeLog(gctx, w, mlflowRunID, node, attemptDir, withAttempt, dir) + path, err := download(gctx, node) switch { case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): // Interrupting the command must not look like a node with no logs. @@ -269,3 +292,60 @@ func sortedNodeKeys(m map[int]string) []int { slices.Sort(keys) return keys } + +// downloadBricklensNodeLog pages oldest-first without a tail cap. Keep a partial +// file on fetch failure so the caller can report it as incomplete. +func downloadBricklensNodeLog(ctx context.Context, w *databricks.WorkspaceClient, dir string, node int, req logRequest, status logRunStatus) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + outPath := filepath.Join(dir, "logs", fmt.Sprintf("node_%d.log", node)) + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return "", err + } + f, err := os.Create(outPath) + if err != nil { + return "", err + } + req.node = node + st := bricklensStreamer{ + ctx: ctx, + w: w, + apiClient: apiClient, + headers: auth.WorkspaceIDHeaders(w.Config), + req: req, + fromSec: req.fromSeconds(status, time.Now()), + } + var written bool + var pageToken string + for { + var resp *bricklensLogsResponse + resp, err = st.requestPage(pageToken, req.toSeconds(status), 0, true) + if err != nil { + break + } + for _, rec := range resp.LogRecords { + if _, err = fmt.Fprintln(f, rec.Body); err != nil { + break + } + written = true + } + if err != nil || resp.NextPageToken == "" { + break + } + pageToken = resp.NextPageToken + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if ctx.Err() != nil { + os.Remove(outPath) + return "", ctx.Err() + } + if !written { + os.Remove(outPath) + return "", err + } + return outPath, err +} diff --git a/cmd/air/logdownload_test.go b/cmd/air/logdownload_test.go index 63c307b2f58..c1180f88c8b 100644 --- a/cmd/air/logdownload_test.go +++ b/cmd/air/logdownload_test.go @@ -3,11 +3,14 @@ package aircmd import ( "bytes" "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" "path" "path/filepath" + "strconv" "strings" "testing" @@ -148,6 +151,7 @@ func fullDownloadServer(t *testing.T) *httptest.Server { } func TestDownloadLogsAllNodes(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) dir := t.TempDir() @@ -160,6 +164,7 @@ func TestDownloadLogsAllNodes(t *testing.T) { } func TestDownloadLogsSingleNode(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) dir := t.TempDir() @@ -172,6 +177,7 @@ func TestDownloadLogsSingleNode(t *testing.T) { } func TestDownloadLogsExplicitNodeZero(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) dir := t.TempDir() @@ -462,6 +468,7 @@ func TestDownloadOutcomeTreatsActiveRunAsSuccess(t *testing.T) { } func TestDownloadLogsActiveRunExitsZero(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) @@ -500,6 +507,7 @@ func allNodesFailServer(t *testing.T) *httptest.Server { } func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, allNodesFailServer(t).URL) var stdout bytes.Buffer @@ -514,3 +522,107 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) { assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)") assert.NotContains(t, stdout.String(), "No logs available") } + +func TestDownloadLogsUsesBricklens(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "") + for _, nodeSet := range []bool{false, true} { + t.Run(strconv.FormatBool(nodeSet), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = w.Write([]byte(`{}`)) + return + } + assert.Equal(t, "/api/2.0/ai-training/workflows/by-run-id/123/logs", r.URL.Path) + q := r.URL.Query() + assert.Equal(t, "true", q.Get("ascending")) + assert.Equal(t, "0", q.Get("ref.attempt_number")) + assert.Empty(t, q.Get("page_size"), "downloads must not use the tail cap") + if q.Get("page_token") == "" { + records := make([]logRecord, defaultCompletedRunTailLines+1) + for i := range records { + records[i].Body = "node " + q.Get("filter.node_index") + } + assert.NoError(t, json.NewEncoder(w).Encode(bricklensLogsResponse{LogRecords: records, NextPageToken: "next"})) + } else { + assert.Equal(t, "next", q.Get("page_token")) + _, _ = w.Write([]byte(`{"log_records":[{"body":"last line"}]}`)) + } + })) + t.Cleanup(srv.Close) + dir := t.TempDir() + success, err := downloadLogs(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), &bytes.Buffer{}, + logRequest{runID: 123, attempt: -1, downloadTo: dir, node: 1, nodeSet: nodeSet}, + logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2)) + require.NoError(t, err) + assert.True(t, success) + for node := range 2 { + file := filepath.Join(dir, "logs", fmt.Sprintf("node_%d.log", node)) + if nodeSet && node == 0 { + assert.NoFileExists(t, file) + continue + } + data, err := os.ReadFile(file) + require.NoError(t, err) + assert.Equal(t, strings.Repeat(fmt.Sprintf("node %d\n", node), defaultCompletedRunTailLines+1)+"last line\n", string(data)) + } + }) + } +} + +func TestDownloadBricklensNodeLogReportsFetchFailures(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "") + for _, partial := range []bool{false, true} { + t.Run(strconv.FormatBool(partial), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = w.Write([]byte(`{}`)) + return + } + if partial && r.URL.Query().Get("page_token") == "" { + _, _ = w.Write([]byte(`{"log_records":[{"body":"first page"}],"next_page_token":"next"}`)) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error_code":"FEATURE_DISABLED","message":"logs disabled"}`)) + })) + t.Cleanup(srv.Close) + dir := t.TempDir() + file, err := downloadBricklensNodeLog(t.Context(), newTestWorkspaceClient(t, srv.URL), dir, 0, + logRequest{runID: 123, attempt: 2}, logRunStatus{resultState: "SUCCESS"}) + require.ErrorIs(t, err, errBricklensFeatureDisabled) + assert.Contains(t, err.Error(), "logs disabled") + if partial { + data, err := os.ReadFile(file) + require.NoError(t, err) + assert.Equal(t, "first page\n", string(data)) + } else { + assert.Empty(t, file) + assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log")) + } + }) + } +} + +func TestDownloadBricklensNodeLogCancellationRemovesPartialFile(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Query().Get("page_token") == "" { + _, _ = w.Write([]byte(`{"log_records":[{"body":"first page"}],"next_page_token":"next"}`)) + } else { + cancel() + <-r.Context().Done() + } + })) + t.Cleanup(srv.Close) + dir := t.TempDir() + file, err := downloadBricklensNodeLog(ctx, newTestWorkspaceClient(t, srv.URL), dir, 0, + logRequest{runID: 123, attempt: 0}, logRunStatus{resultState: "SUCCESS"}) + require.ErrorIs(t, err, context.Canceled) + assert.Empty(t, file) + assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log")) +} diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 949c08d13ef..64068861274 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -35,7 +36,10 @@ func newLogsCommand() *cobra.Command { Long: `Stream logs from an active run until it completes, or fetch logs from a completed run. -JOB_RUN_ID is the job run ID returned by "databricks air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run". + +Logs are read from Bricklens. To temporarily restore the legacy MLflow artifact +fallback and artifact downloads, set DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true.`, Example: ` databricks air logs 123456789 databricks air logs 123456789 --node 1 --retry 0 databricks air logs 123456789 --download-to ./logs`, @@ -241,14 +245,20 @@ func resolveLogAttempt(run *jobs.Run, requested int) (int, int64, error) { return 0, 0, fmt.Errorf("run %d has no task for retry %d", run.RunId, attempt) } -// fetchLogs serves logs from Bricklens, falling back to MLflow when Bricklens -// returns errBricklensFeatureDisabled. +// mlflowLogFallbackEnabled temporarily preserves access to logs from the retired +// artifact ingestion path while customers migrate to Bricklens. +func mlflowLogFallbackEnabled(ctx context.Context) bool { + enabled, _ := env.GetBool(ctx, "DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK") + return enabled +} + +// fetchLogs serves logs from Bricklens, with an opt-in legacy MLflow fallback. func fetchLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus) (bool, error) { if req.attempt < 0 && !status.terminal() && req.retryTracker == nil { req.retryTracker = newRetryTracker(status) } success, err := streamBricklensLogs(ctx, w, out, req, status) - if errors.Is(err, errBricklensFeatureDisabled) { + if errors.Is(err, errBricklensFeatureDisabled) && mlflowLogFallbackEnabled(ctx) { return mlflowLogFallback(ctx, w, out, req, status) } return success, err diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 6d18adb29f2..f93b4fa94a1 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -306,6 +306,7 @@ func mlflowFallbackServer(t *testing.T) *httptest.Server { } func TestLogsFallsBackToMLflow(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") srv := mlflowFallbackServer(t) var buf bytes.Buffer ctx := cmdctx.SetWorkspaceClient(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL)) @@ -322,6 +323,7 @@ func TestLogsFallsBackToMLflow(t *testing.T) { } func TestFetchLogsRetryNotificationDeduplicatedAcrossFallback(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") oldRetryInterval := retryCheckInterval retryCheckInterval = time.Millisecond t.Cleanup(func() { retryCheckInterval = oldRetryInterval }) @@ -360,7 +362,7 @@ func TestFetchLogsRetryNotificationDeduplicatedAcrossFallback(t *testing.T) { } maxRetries := 3 var out bytes.Buffer - success, err := fetchLogs(t.Context(), newTestWorkspaceClient(t, srv.URL), &out, logRequest{ + success, err := fetchLogs(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), &out, logRequest{ runID: 5, attempt: -1, tailLines: -1, @@ -501,3 +503,25 @@ func TestRunLogsPreservesSubmittedRunURL(t *testing.T) { require.NoError(t, err) assert.True(t, strings.HasSuffix(buf.String(), "\n\nView job run at: "+runURL+"\n"), buf.String()) } + +func TestFetchLogsMLflowFallbackRequiresOptIn(t *testing.T) { + for _, value := range []string{"", "false", "invalid", "true", "1"} { + t.Run(value, func(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", value) + srv := mlflowFallbackServer(t) + var out bytes.Buffer + success, err := fetchLogs(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), &out, + logRequest{runID: 5, attempt: -1, tailLines: -1}, + logRunStatus{lifeCycleState: "TERMINATED", resultState: "SUCCESS"}) + if value == "true" || value == "1" { + require.NoError(t, err) + assert.True(t, success) + assert.Equal(t, "line one\nline two\n", out.String()) + } else { + require.ErrorIs(t, err, errBricklensFeatureDisabled) + assert.False(t, success) + assert.Empty(t, out.String()) + } + }) + } +} diff --git a/cmd/air/logstream.go b/cmd/air/logstream.go index 79ed5e18b3e..1e4fa8c65a7 100644 --- a/cmd/air/logstream.go +++ b/cmd/air/logstream.go @@ -21,7 +21,7 @@ import ( const ( // maxTransientFailures is how many consecutive Bricklens failures to tolerate - // before falling back to MLflow. + // before returning an error. maxTransientFailures = 5 // defaultCompletedRunTailLines caps a completed run's output when neither // --tail nor --minutes is set. @@ -76,12 +76,10 @@ var retryCheckInterval = 3 * time.Second // bricklensPollInterval is independent of slower retry and discovery backoffs. var bricklensPollInterval = time.Second -// errBricklensFeatureDisabled signals the caller to fall back to MLflow: Bricklens -// is gated off (FEATURE_DISABLED), not deployed (ENDPOINT_NOT_FOUND / 404), -// persistently failing, or served every request successfully but never returned a -// record for a run whose logs may still be in MLflow. The flag is evaluated -// server-side. -var errBricklensFeatureDisabled = errors.New("bricklens logs unavailable; falling back to mlflow") +// errBricklensFeatureDisabled marks an unavailable Bricklens endpoint. When the +// legacy fallback is enabled, empty streams and exhausted retries also use this +// sentinel to hand off to MLflow. +var errBricklensFeatureDisabled = errors.New("bricklens logs unavailable") // logRequest describes what to fetch, shared by both backends so they honor the // same flags. windowMinutes and tailLines are mutually exclusive. @@ -523,16 +521,19 @@ func (st *bricklensStreamer) run() (bool, error) { continue } } - if !st.firstLogSeen { + if !st.firstLogSeen && mlflowLogFallbackEnabled(st.ctx) { // A successful but empty Bricklens stream isn't proof the run has no - // logs; they may be in MLflow (as --download-to reads). Fall back - // there, which owns the real no-logs report and the same exit code. + // logs; older runs may have artifacts. The opt-in legacy path owns + // the no-logs report and exit code in that case. return false, errBricklensFeatureDisabled } + if !st.firstLogSeen { + emitNoLogs(st.out, st.req, st.status) + } log.Infof(st.ctx, "air logs: run %d finished in state %s", st.req.runID, st.status.displayState()) return st.status.succeeded(), nil } - if !st.firstLogSeen { + if !st.firstLogSeen && mlflowLogFallbackEnabled(st.ctx) { emptyStreamPolls++ if emptyStreamPolls%bricklensEmptyMLflowProbeEveryNPolls == 0 && mlflowLogsExist(st.ctx, st.w, st.req) { log.Debugf(st.ctx, "air logs: MLflow artifacts exist for run %d; falling back to MLflow log stream", st.req.runID) @@ -570,12 +571,15 @@ func (st *bricklensStreamer) drainStatic(toSec int64) (bool, error) { if err := st.drainTail(toSec, false); err != nil { return false, err } - if !st.firstLogSeen { + if !st.firstLogSeen && mlflowLogFallbackEnabled(st.ctx) { // An empty Bricklens tail doesn't mean the attempt has no logs; fall back to // MLflow, which holds the immutable per-attempt artifacts. See the terminal // branch in run. return false, errBricklensFeatureDisabled } + if !st.firstLogSeen { + emitNoLogs(st.out, st.req, st.status) + } return st.status.downloadOutcome(), nil } @@ -675,7 +679,7 @@ func (st *bricklensStreamer) drainPages(toSec int64, suppressInitialHistory bool } // requestPage fetches one page, retrying transient failures up to -// maxTransientFailures before falling back to MLflow. A feature-gated response or +// maxTransientFailures before returning an error. A feature-gated response or // genuine not-found returns immediately. func (st *bricklensStreamer) requestPage(pageToken string, toSec int64, pageSize int, ascending bool) (*bricklensLogsResponse, error) { q := bricklensLogsQuery{ @@ -697,15 +701,17 @@ func (st *bricklensStreamer) requestPage(pageToken string, toSec int64, pageSize switch classified := classifyLogError(err); { case errors.Is(classified, errBricklensFeatureDisabled): - return nil, errBricklensFeatureDisabled + return nil, fmt.Errorf("%w: %w", errBricklensFeatureDisabled, err) case classified != nil: return nil, classified } transientFailures++ if transientFailures >= maxTransientFailures { - log.Debugf(st.ctx, "air logs: bricklens failed %d times; falling back to mlflow", maxTransientFailures) - return nil, errBricklensFeatureDisabled + if mlflowLogFallbackEnabled(st.ctx) { + return nil, errBricklensFeatureDisabled + } + return nil, fmt.Errorf("bricklens logs failed after %d attempts: %w", maxTransientFailures, err) } log.Debugf(st.ctx, "air logs: bricklens transient failure (%d/%d): %v", transientFailures, maxTransientFailures, err) if err := sleepOrCancel(st.ctx, retryCheckInterval); err != nil { diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 5bb60caa344..59b03267776 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -640,6 +640,7 @@ func TestEmitNoLogs(t *testing.T) { } func TestRequestPageRetriesThenFallsBack(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") // Shrink the retry wait so the transient-failure loop runs fast. orig := retryCheckInterval retryCheckInterval = time.Millisecond @@ -728,6 +729,7 @@ func emptyLogsServer(t *testing.T) *httptest.Server { } func TestStreamBricklensEmptyFallsBackToMLflow(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") // Bricklens served every request but returned no record. That is not proof the // run has no logs (they may be in MLflow), so the streamer must hand off via // errBricklensFeatureDisabled and emit nothing itself, rather than reporting @@ -781,6 +783,7 @@ func TestStreamBricklensTerminalWithRecordsDoesNotFallBack(t *testing.T) { } func TestStreamBricklensActiveEmptyProbesMLflow(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") oldInterval := retryCheckInterval retryCheckInterval = time.Millisecond t.Cleanup(func() { retryCheckInterval = oldInterval }) @@ -816,6 +819,7 @@ func TestStreamBricklensActiveEmptyProbesMLflow(t *testing.T) { } func TestFetchLogsFallsBackToMLflowWhenBricklensEmpty(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") // End-to-end repro: a terminal SUCCESS run whose Bricklens stream is empty but // whose logs are in MLflow. The print path must fall back to MLflow and print // them, exactly as --download-to already does. @@ -882,3 +886,53 @@ func TestSleepOrCancel(t *testing.T) { cancel() require.ErrorIs(t, sleepOrCancel(ctx, time.Hour), context.Canceled) } + +func TestStreamBricklensEmptyWithoutMLflow(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "") + original := bricklensPollInterval + bricklensPollInterval = time.Millisecond + t.Cleanup(func() { bricklensPollInterval = original }) + for _, result := range []string{"SUCCESS", "CANCELED"} { + t.Run(result, func(t *testing.T) { + polls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/databricks-config" { + _, _ = w.Write([]byte(`{}`)) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + polls++ + _, _ = w.Write([]byte(`{"log_records":[]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + if polls < bricklensEmptyMLflowProbeEveryNPolls+1 { + _, _ = w.Write([]byte(`{"state":{"life_cycle_state":"RUNNING"}}`)) + } else { + _, _ = w.Write([]byte(`{"state":{"life_cycle_state":"TERMINATED","result_state":"` + result + `"}}`)) + } + default: + t.Errorf("unexpected request: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + t.Cleanup(srv.Close) + var out bytes.Buffer + success, err := streamBricklensLogs(t.Context(), newTestWorkspaceClient(t, srv.URL), &out, + logRequest{runID: 123, attempt: -1, tailLines: -1, jsonOutput: true}, logRunStatus{lifeCycleState: "RUNNING"}) + require.NoError(t, err) + assert.Equal(t, result == "SUCCESS", success) + assert.Contains(t, out.String(), "No logs available for run 123") + assert.Greater(t, polls, bricklensEmptyMLflowProbeEveryNPolls) + }) + } +} + +func TestStreamBricklensStaticEmptyWithoutMLflow(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "") + var out bytes.Buffer + success, err := streamBricklensLogs(t.Context(), newTestWorkspaceClient(t, emptyLogsServer(t).URL), &out, + logRequest{runID: 123, attempt: 0, tailLines: -1, staticView: true}, logRunStatus{lifeCycleState: "RUNNING"}) + require.NoError(t, err) + assert.True(t, success) + assert.Contains(t, out.String(), "No logs available yet for run 123") +} diff --git a/cmd/air/run_watch_test.go b/cmd/air/run_watch_test.go index 13c21923057..3d8245585ad 100644 --- a/cmd/air/run_watch_test.go +++ b/cmd/air/run_watch_test.go @@ -255,6 +255,7 @@ func TestRunWatchDryRunSkipsSubmit(t *testing.T) { } func TestRunWatchJSONMLflowFallbackTerminalEnvelope(t *testing.T) { + t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true") // Regression: through the MLflow fallback the terminal status must come from // the run's actual state, not the onStatusChange callback (which is // Bricklens-only), so a SUCCESS run isn't mislabeled FAILED in the envelope. From 34969f2a621dd0082eec337e8a6e93de4facd13d Mon Sep 17 00:00:00 2001 From: "ethan.ma" Date: Mon, 16 Nov 2026 18:50:01 +0000 Subject: [PATCH 2/2] air: Link log fallback cleanup release note --- .nextchanges/air/bricklens-logs-only.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/bricklens-logs-only.md b/.nextchanges/air/bricklens-logs-only.md index 2faabab41b8..dbc0511d5c9 100644 --- a/.nextchanges/air/bricklens-logs-only.md +++ b/.nextchanges/air/bricklens-logs-only.md @@ -1 +1 @@ -* Use Bricklens for AIR log streaming and downloads by default; temporarily restore the legacy MLflow artifact fallback and downloads with `DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true`. +* Use Bricklens for AIR log streaming and downloads by default; temporarily restore the legacy MLflow artifact fallback and downloads with `DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true`. ([#7697](https://github.com/databricks/cli/pull/7697))