Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/bricklens-logs-only.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use Bricklens for AIR log streaming and downloads by default; temporarily restore the legacy MLflow artifact fallback and downloads with `DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true`. ([#7697](https://github.com/databricks/cli/pull/7697))
94 changes: 87 additions & 7 deletions cmd/air/logdownload.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@ import (
"path/filepath"
"slices"
"sync"
"time"

"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/client"
"github.com/databricks/databricks-sdk-go/service/jobs"
"golang.org/x/sync/errgroup"
)
Expand Down Expand Up @@ -56,7 +59,7 @@ func resolveNodeCount(run *jobs.Run) (int, error) {

// downloadLogs writes each node's logs to <downloadTo>/logs/node_<n>.log and
// prints a summary. An explicit --node downloads only that node; otherwise all of
// them. Logs come from MLflow artifacts, since Bricklens only streams. The
// them. Logs come from Bricklens unless legacy artifact downloads are enabled. The
// returned bool is the run's outcome, so the exit code matches the streaming path.
func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) {
numNodes, err := resolveNodeCount(run)
Expand All @@ -79,10 +82,14 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri
if err != nil {
return false, err
}
ids := mlflowIDsForTask(ctx, w, taskRunID)
if ids == nil || ids.RunID == "" {
emitNoLogs(out, req, status)
return status.downloadOutcome(), nil
var mlflowRunID string
if mlflowLogFallbackEnabled(ctx) {
ids := mlflowIDsForTask(ctx, w, taskRunID)
if ids == nil || ids.RunID == "" {
emitNoLogs(out, req, status)
return status.downloadOutcome(), nil
}
mlflowRunID = ids.RunID
}

dir, err := filepath.Abs(req.downloadTo)
Expand All @@ -95,7 +102,16 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri
return false, fmt.Errorf("failed to create %s: %w", dir, err)
}

nodeLogs, failures, err := downloadAllNodeLogs(ctx, w, ids.RunID, dir, nodes, attempt)
var nodeLogs map[int]string
var failures map[int]string
if mlflowLogFallbackEnabled(ctx) {
nodeLogs, failures, err = downloadAllNodeLogs(ctx, w, mlflowRunID, dir, nodes, attempt)
} else {
req.attempt = attempt
nodeLogs, failures, err = downloadNodes(ctx, nodes, func(ctx context.Context, node int) (string, error) {
return downloadBricklensNodeLog(ctx, w, dir, node, req, status)
})
}
if err != nil {
return false, err
}
Expand Down Expand Up @@ -139,13 +155,20 @@ func downloadAllNodeLogs(ctx context.Context, w *databricks.WorkspaceClient, mlf
return nil, nil, err
}

return downloadNodes(ctx, nodes, func(ctx context.Context, node int) (string, error) {
return downloadNodeLog(ctx, w, mlflowRunID, node, attemptDir, withAttempt, dir)
})
}

// downloadNodes bounds concurrency and retains partial files alongside their errors.
func downloadNodes(ctx context.Context, nodes []int, download func(context.Context, int) (string, error)) (map[int]string, map[int]string, error) {
paths := make([]string, len(nodes))
reasons := make([]string, len(nodes))
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(downloadConcurrency)
for i, node := range nodes {
g.Go(func() error {
path, err := downloadNodeLog(gctx, w, mlflowRunID, node, attemptDir, withAttempt, dir)
path, err := download(gctx, node)
switch {
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
// Interrupting the command must not look like a node with no logs.
Expand Down Expand Up @@ -269,3 +292,60 @@ func sortedNodeKeys(m map[int]string) []int {
slices.Sort(keys)
return keys
}

// downloadBricklensNodeLog pages oldest-first without a tail cap. Keep a partial
// file on fetch failure so the caller can report it as incomplete.
func downloadBricklensNodeLog(ctx context.Context, w *databricks.WorkspaceClient, dir string, node int, req logRequest, status logRunStatus) (string, error) {
apiClient, err := client.New(w.Config)
if err != nil {
return "", fmt.Errorf("failed to create API client: %w", err)
}
outPath := filepath.Join(dir, "logs", fmt.Sprintf("node_%d.log", node))
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
return "", err
}
f, err := os.Create(outPath)
if err != nil {
return "", err
}
req.node = node
st := bricklensStreamer{
ctx: ctx,
w: w,
apiClient: apiClient,
headers: auth.WorkspaceIDHeaders(w.Config),
req: req,
fromSec: req.fromSeconds(status, time.Now()),
}
var written bool
var pageToken string
for {
var resp *bricklensLogsResponse
resp, err = st.requestPage(pageToken, req.toSeconds(status), 0, true)
if err != nil {
break
}
for _, rec := range resp.LogRecords {
if _, err = fmt.Fprintln(f, rec.Body); err != nil {
break
}
written = true
}
if err != nil || resp.NextPageToken == "" {
break
}
pageToken = resp.NextPageToken
}
if closeErr := f.Close(); err == nil {
err = closeErr
}
if ctx.Err() != nil {
os.Remove(outPath)
return "", ctx.Err()
}
if !written {
os.Remove(outPath)
return "", err
}
return outPath, err
}
112 changes: 112 additions & 0 deletions cmd/air/logdownload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package aircmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"testing"

Expand Down Expand Up @@ -148,6 +151,7 @@ func fullDownloadServer(t *testing.T) *httptest.Server {
}

func TestDownloadLogsAllNodes(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true")
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()
Expand All @@ -160,6 +164,7 @@ func TestDownloadLogsAllNodes(t *testing.T) {
}

func TestDownloadLogsSingleNode(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true")
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()
Expand All @@ -172,6 +177,7 @@ func TestDownloadLogsSingleNode(t *testing.T) {
}

func TestDownloadLogsExplicitNodeZero(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true")
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()
Expand Down Expand Up @@ -462,6 +468,7 @@ func TestDownloadOutcomeTreatsActiveRunAsSuccess(t *testing.T) {
}

func TestDownloadLogsActiveRunExitsZero(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true")
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)

Expand Down Expand Up @@ -500,6 +507,7 @@ func allNodesFailServer(t *testing.T) *httptest.Server {
}

func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "true")
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, allNodesFailServer(t).URL)
var stdout bytes.Buffer
Expand All @@ -514,3 +522,107 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) {
assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)")
assert.NotContains(t, stdout.String(), "No logs available")
}

func TestDownloadLogsUsesBricklens(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "")
for _, nodeSet := range []bool{false, true} {
t.Run(strconv.FormatBool(nodeSet), func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/.well-known/databricks-config" {
_, _ = w.Write([]byte(`{}`))
return
}
assert.Equal(t, "/api/2.0/ai-training/workflows/by-run-id/123/logs", r.URL.Path)
q := r.URL.Query()
assert.Equal(t, "true", q.Get("ascending"))
assert.Equal(t, "0", q.Get("ref.attempt_number"))
assert.Empty(t, q.Get("page_size"), "downloads must not use the tail cap")
if q.Get("page_token") == "" {
records := make([]logRecord, defaultCompletedRunTailLines+1)
for i := range records {
records[i].Body = "node " + q.Get("filter.node_index")
}
assert.NoError(t, json.NewEncoder(w).Encode(bricklensLogsResponse{LogRecords: records, NextPageToken: "next"}))
} else {
assert.Equal(t, "next", q.Get("page_token"))
_, _ = w.Write([]byte(`{"log_records":[{"body":"last line"}]}`))
}
}))
t.Cleanup(srv.Close)
dir := t.TempDir()
success, err := downloadLogs(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), &bytes.Buffer{},
logRequest{runID: 123, attempt: -1, downloadTo: dir, node: 1, nodeSet: nodeSet},
logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.True(t, success)
for node := range 2 {
file := filepath.Join(dir, "logs", fmt.Sprintf("node_%d.log", node))
if nodeSet && node == 0 {
assert.NoFileExists(t, file)
continue
}
data, err := os.ReadFile(file)
require.NoError(t, err)
assert.Equal(t, strings.Repeat(fmt.Sprintf("node %d\n", node), defaultCompletedRunTailLines+1)+"last line\n", string(data))
}
})
}
}

func TestDownloadBricklensNodeLogReportsFetchFailures(t *testing.T) {
t.Setenv("DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK", "")
for _, partial := range []bool{false, true} {
t.Run(strconv.FormatBool(partial), func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/.well-known/databricks-config" {
_, _ = w.Write([]byte(`{}`))
return
}
if partial && r.URL.Query().Get("page_token") == "" {
_, _ = w.Write([]byte(`{"log_records":[{"body":"first page"}],"next_page_token":"next"}`))
return
}
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"error_code":"FEATURE_DISABLED","message":"logs disabled"}`))
}))
t.Cleanup(srv.Close)
dir := t.TempDir()
file, err := downloadBricklensNodeLog(t.Context(), newTestWorkspaceClient(t, srv.URL), dir, 0,
logRequest{runID: 123, attempt: 2}, logRunStatus{resultState: "SUCCESS"})
require.ErrorIs(t, err, errBricklensFeatureDisabled)
assert.Contains(t, err.Error(), "logs disabled")
if partial {
data, err := os.ReadFile(file)
require.NoError(t, err)
assert.Equal(t, "first page\n", string(data))
} else {
assert.Empty(t, file)
assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log"))
}
})
}
}

func TestDownloadBricklensNodeLogCancellationRemovesPartialFile(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
t.Cleanup(cancel)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/.well-known/databricks-config" {
_, _ = w.Write([]byte(`{}`))
return
}
if r.URL.Query().Get("page_token") == "" {
_, _ = w.Write([]byte(`{"log_records":[{"body":"first page"}],"next_page_token":"next"}`))
} else {
cancel()
<-r.Context().Done()
}
}))
t.Cleanup(srv.Close)
dir := t.TempDir()
file, err := downloadBricklensNodeLog(ctx, newTestWorkspaceClient(t, srv.URL), dir, 0,
logRequest{runID: 123, attempt: 0}, logRunStatus{resultState: "SUCCESS"})
require.ErrorIs(t, err, context.Canceled)
assert.Empty(t, file)
assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log"))
}
18 changes: 14 additions & 4 deletions cmd/air/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
Expand All @@ -35,7 +36,10 @@ func newLogsCommand() *cobra.Command {
Long: `Stream logs from an active run until it completes, or fetch logs from a
completed run.

JOB_RUN_ID is the job run ID returned by "databricks air run".`,
JOB_RUN_ID is the job run ID returned by "databricks air run".

Logs are read from Bricklens. To temporarily restore the legacy MLflow artifact
fallback and artifact downloads, set DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK=true.`,
Example: ` databricks air logs 123456789
databricks air logs 123456789 --node 1 --retry 0
databricks air logs 123456789 --download-to ./logs`,
Expand Down Expand Up @@ -241,14 +245,20 @@ func resolveLogAttempt(run *jobs.Run, requested int) (int, int64, error) {
return 0, 0, fmt.Errorf("run %d has no task for retry %d", run.RunId, attempt)
}

// fetchLogs serves logs from Bricklens, falling back to MLflow when Bricklens
// returns errBricklensFeatureDisabled.
// mlflowLogFallbackEnabled temporarily preserves access to logs from the retired
// artifact ingestion path while customers migrate to Bricklens.
func mlflowLogFallbackEnabled(ctx context.Context) bool {
enabled, _ := env.GetBool(ctx, "DATABRICKS_AIR_ENABLE_MLFLOW_LOG_FALLBACK")
return enabled
}

// fetchLogs serves logs from Bricklens, with an opt-in legacy MLflow fallback.
func fetchLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus) (bool, error) {
if req.attempt < 0 && !status.terminal() && req.retryTracker == nil {
req.retryTracker = newRetryTracker(status)
}
success, err := streamBricklensLogs(ctx, w, out, req, status)
if errors.Is(err, errBricklensFeatureDisabled) {
if errors.Is(err, errBricklensFeatureDisabled) && mlflowLogFallbackEnabled(ctx) {
return mlflowLogFallback(ctx, w, out, req, status)
}
return success, err
Expand Down
Loading
Loading