Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7627.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air list --all-users` and `air list --filter user=` cache finished runs' MLflow links, so repeat listings are about 3x faster. ([#7627](https://github.com/databricks/cli/pull/7627))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the default list is server-side ActiveOnly, so finished rows only appear with --all-status. Could the entry mention --all-status rather than --all-users / --filter user=?

10 changes: 10 additions & 0 deletions cmd/air/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"sync"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cache"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/log"
Expand Down Expand Up @@ -271,10 +272,17 @@ func (f *runFetcher) next(want int) ([]listRow, error) {
// MLflow links appear only in the text table, so the per-run get-output
// lookups are skipped for JSON output (which omits the column anyway).
if f.fetchMLflow {
scan, isScan := f.strategy.(*jobsScanStrategy)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

problem: The scan's caching is hard to find. Someone reading or changing jobsScanStrategy won't see that it caches, because the cache reads and writes happen here in the shared runFetcher.next, behind scan-only type checks. The index path keeps all of its caching inside its own strategy.

suggested fix: Move it into the strategy the way the index path does: apply cached links in jobsScanStrategy.next() where the row is built, and write them back from a (s *jobsScanStrategy) cacheMLflowLinks method called next to the index one.

if isScan {
applyCachedMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries)
}
setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries)
if s, ok := f.strategy.(*indexStrategy); ok {
s.cacheMLflowLinks(entries)
}
if isScan {
putMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries)
}
}

rows := make([]listRow, len(entries))
Expand All @@ -296,6 +304,7 @@ type jobsScanStrategy struct {
filters listFilters
host string
workspaceID int64
mlflowCache *cache.Cache // terminal runs' MLflow links

scanned int
}
Expand All @@ -319,6 +328,7 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l
userFilter: q.userFilter,
currentUser: q.currentUser,
filters: q.filters,
mlflowCache: newScanMLflowCache(ctx),
host: w.Config.Host,
workspaceID: q.workspaceID,
}
Expand Down
61 changes: 61 additions & 0 deletions cmd/air/list_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package aircmd

import (
"context"
"slices"
"time"

"github.com/databricks/cli/libs/cache"
Expand Down Expand Up @@ -96,6 +97,66 @@ func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks
cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs))
}

// The Jobs scan builds its rows from runs/list on every call, so it caches only
// the MLflow links of terminal runs, which cost a get-output and an MLflow
// request per row. It uses its own component so its entries never stand in for
// the index path's full rows.
const listScanMLflowCacheComponent = "air-list-scan-mlflow-v1"

// cachedMLflowLink is a terminal run's MLflow link fields, with the task runs
// they were resolved from.
type cachedMLflowLink struct {
MLflowURL string `json:"mlflow_url"`
MLflowLabel string `json:"mlflow_label"`
ExperimentURL string `json:"experiment_url"`
Tasks []jobs.RunTask `json:"tasks"`
}

func newScanMLflowCache(ctx context.Context) *cache.Cache {
return cache.NewCache(ctx, listScanMLflowCacheComponent, listCacheTTL, nil)
}

// applyCachedMLflowLinks fills terminal entries' MLflow fields from the cache.
// An entry is used only while the run's task runs and attempts still match the
// ones it was resolved from, so a run repaired after it was cached is refetched.
func applyCachedMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) {
for i := range entries {
if !entries[i].terminal {
continue
}
link, ok := cache.Get[cachedMLflowLink](ctx, c, listCacheKey{Host: host, RunID: entries[i].runID})
if !ok || !sameTaskRuns(link.Tasks, entries[i].tasks) {
continue
}
entries[i].row.MLflowURL = link.MLflowURL
entries[i].row.MLflowLabel = link.MLflowLabel
entries[i].row.ExperimentURL = link.ExperimentURL
}
}

// putMLflowLinks caches terminal entries that have an MLflow link.
func putMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) {
for _, e := range entries {
if !e.terminal || e.row.MLflowURL == "" || e.row.MLflowURL == "-" {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: a failed MLflow name lookup gets cached here. fetchMLflowRunName (mlflow.go:146) returns "" on any error and mlflowRunLabel then falls back to the run ID, so the ID is cached as the label for up to 60 days. The index path already does the same (list_index.go:146), so this isn't new, but this PR extends it to the scan path. Could we skip the put when the name lookup failed (e.g. a nameKnown flag on listedRun)? A test where GetRun errors would pin it.

continue
}
cache.Put(ctx, c, listCacheKey{Host: host, RunID: e.runID}, cachedMLflowLink{
MLflowURL: e.row.MLflowURL,
MLflowLabel: e.row.MLflowLabel,
ExperimentURL: e.row.ExperimentURL,
Tasks: e.tasks,
})
}
}

// sameTaskRuns reports whether two retryTasks lists name the same task runs and
// attempts.
func sameTaskRuns(a, b []jobs.RunTask) bool {
return slices.EqualFunc(a, b, func(x, y jobs.RunTask) bool {
return x.RunId == y.RunId && x.TaskKey == y.TaskKey && x.AttemptNumber == y.AttemptNumber
})
}

// updateCachedRow replaces the display row while preserving cache metadata.
func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) {
key := listCacheKey{Host: host, RunID: runID}
Expand Down
66 changes: 66 additions & 0 deletions cmd/air/list_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,72 @@ func TestIndexStrategyCachesMLflowEnrichment(t *testing.T) {
assert.Equal(t, []string{"70"}, outputRunIDs, "get-output must use the task run id")
}

func TestJobsScanCachesTerminalMLflowLinks(t *testing.T) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the repair and active-run cases are well covered. Two cases are missing: a terminal run whose first lookup has no link (-) shouldn't be cached and should resolve once the link appears, and a corrupted cache file should fall back to a refetch.

t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir())

// scanRun is a runs/list AIR run; one task run per attempt, newest last.
scanRun := func(id int64, state jobs.RunLifeCycleState, attempts ...int) jobs.BaseRun {
r := airBaseRun(id, "me@example.com", "GPU_1xA10", 1, "exp")
r.State = &jobs.RunState{LifeCycleState: state}
task := r.Tasks[0]
r.Tasks = nil
for _, a := range attempts {
task.RunId = id*10 + int64(a)
task.AttemptNumber = a
r.Tasks = append(r.Tasks, task)
}
return r
}
var runs []jobs.BaseRun
var outputHits, mlflowHits int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/2.2/jobs/runs/list":
_, _ = w.Write([]byte(runsListBody(t, "", runs...)))
case "/api/2.2/jobs/runs/get-output":
outputHits++
_, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"exp1","mlflow_run_id":"m-` + r.URL.Query().Get("run_id") + `"}}`))
case "/api/2.0/mlflow/runs/get":
mlflowHits++
_, _ = w.Write([]byte(`{"run":{"info":{"run_name":"training-run"}}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)
list := func() []listRow {
rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", fetchMLflow: true, limit: 10,
}).next(10)
require.NoError(t, err)
require.Len(t, rows, 1)
return rows
}

// A terminal run's links are resolved once, then served from the cache.
runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0)}
first := list()
assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-70/logs", first[0].MLflowURL)
assert.Equal(t, first, list())
assert.Equal(t, 1, outputHits)
assert.Equal(t, 1, mlflowHits)

// A repaired run has a new attempt, so its cached links are stale.
runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0, 1)}
assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-71/logs", list()[0].MLflowURL)
assert.Equal(t, 2, outputHits)

// An active run is never served from the cache.
runs = []jobs.BaseRun{scanRun(8, jobs.RunLifeCycleStateRunning, 0)}
list()
list()
assert.Equal(t, 4, outputHits)

// The scan's entries never stand in for the index path's full rows.
_, _, _, ok := cachedRow(t.Context(), newListCache(t.Context()), srv.URL, 7)
assert.False(t, ok)
}

func TestIsTerminal(t *testing.T) {
assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}}))
assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateInternalError}}))
Expand Down
Loading