Repository navigation
air list: cache terminal runs' MLflow links on the Jobs scan path #7627
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| * `air list --all-users` and `air list --filter user=` cache finished runs' MLflow links, so repeat listings are about 3x faster. ([#7627](https://github.com/databricks/cli/pull/7627)) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ import ( | |
| "sync" | ||
|
|
||
| "github.com/databricks/cli/cmd/root" | ||
| "github.com/databricks/cli/libs/cache" | ||
| "github.com/databricks/cli/libs/cmdctx" | ||
| "github.com/databricks/cli/libs/flags" | ||
| "github.com/databricks/cli/libs/log" | ||
|
|
@@ -271,10 +272,17 @@ func (f *runFetcher) next(want int) ([]listRow, error) { | |
| // MLflow links appear only in the text table, so the per-run get-output | ||
| // lookups are skipped for JSON output (which omits the column anyway). | ||
| if f.fetchMLflow { | ||
| scan, isScan := f.strategy.(*jobsScanStrategy) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. problem: The scan's caching is hard to find. Someone reading or changing suggested fix: Move it into the strategy the way the index path does: apply cached links in |
||
| if isScan { | ||
| applyCachedMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries) | ||
| } | ||
| setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries) | ||
| if s, ok := f.strategy.(*indexStrategy); ok { | ||
| s.cacheMLflowLinks(entries) | ||
| } | ||
| if isScan { | ||
| putMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries) | ||
| } | ||
| } | ||
|
|
||
| rows := make([]listRow, len(entries)) | ||
|
|
@@ -296,6 +304,7 @@ type jobsScanStrategy struct { | |
| filters listFilters | ||
| host string | ||
| workspaceID int64 | ||
| mlflowCache *cache.Cache // terminal runs' MLflow links | ||
|
|
||
| scanned int | ||
| } | ||
|
|
@@ -319,6 +328,7 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l | |
| userFilter: q.userFilter, | ||
| currentUser: q.currentUser, | ||
| filters: q.filters, | ||
| mlflowCache: newScanMLflowCache(ctx), | ||
| host: w.Config.Host, | ||
| workspaceID: q.workspaceID, | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ package aircmd | |
|
|
||
| import ( | ||
| "context" | ||
| "slices" | ||
| "time" | ||
|
|
||
| "github.com/databricks/cli/libs/cache" | ||
|
|
@@ -96,6 +97,66 @@ func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks | |
| cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) | ||
| } | ||
|
|
||
| // The Jobs scan builds its rows from runs/list on every call, so it caches only | ||
| // the MLflow links of terminal runs, which cost a get-output and an MLflow | ||
| // request per row. It uses its own component so its entries never stand in for | ||
| // the index path's full rows. | ||
| const listScanMLflowCacheComponent = "air-list-scan-mlflow-v1" | ||
|
|
||
| // cachedMLflowLink is a terminal run's MLflow link fields, with the task runs | ||
| // they were resolved from. | ||
| type cachedMLflowLink struct { | ||
| MLflowURL string `json:"mlflow_url"` | ||
| MLflowLabel string `json:"mlflow_label"` | ||
| ExperimentURL string `json:"experiment_url"` | ||
| Tasks []jobs.RunTask `json:"tasks"` | ||
| } | ||
|
|
||
| func newScanMLflowCache(ctx context.Context) *cache.Cache { | ||
| return cache.NewCache(ctx, listScanMLflowCacheComponent, listCacheTTL, nil) | ||
| } | ||
|
|
||
| // applyCachedMLflowLinks fills terminal entries' MLflow fields from the cache. | ||
| // An entry is used only while the run's task runs and attempts still match the | ||
| // ones it was resolved from, so a run repaired after it was cached is refetched. | ||
| func applyCachedMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) { | ||
| for i := range entries { | ||
| if !entries[i].terminal { | ||
| continue | ||
| } | ||
| link, ok := cache.Get[cachedMLflowLink](ctx, c, listCacheKey{Host: host, RunID: entries[i].runID}) | ||
| if !ok || !sameTaskRuns(link.Tasks, entries[i].tasks) { | ||
| continue | ||
| } | ||
| entries[i].row.MLflowURL = link.MLflowURL | ||
| entries[i].row.MLflowLabel = link.MLflowLabel | ||
| entries[i].row.ExperimentURL = link.ExperimentURL | ||
| } | ||
| } | ||
|
|
||
| // putMLflowLinks caches terminal entries that have an MLflow link. | ||
| func putMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) { | ||
| for _, e := range entries { | ||
| if !e.terminal || e.row.MLflowURL == "" || e.row.MLflowURL == "-" { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: a failed MLflow name lookup gets cached here. |
||
| continue | ||
| } | ||
| cache.Put(ctx, c, listCacheKey{Host: host, RunID: e.runID}, cachedMLflowLink{ | ||
| MLflowURL: e.row.MLflowURL, | ||
| MLflowLabel: e.row.MLflowLabel, | ||
| ExperimentURL: e.row.ExperimentURL, | ||
| Tasks: e.tasks, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // sameTaskRuns reports whether two retryTasks lists name the same task runs and | ||
| // attempts. | ||
| func sameTaskRuns(a, b []jobs.RunTask) bool { | ||
| return slices.EqualFunc(a, b, func(x, y jobs.RunTask) bool { | ||
| return x.RunId == y.RunId && x.TaskKey == y.TaskKey && x.AttemptNumber == y.AttemptNumber | ||
| }) | ||
| } | ||
|
|
||
| // updateCachedRow replaces the display row while preserving cache metadata. | ||
| func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) { | ||
| key := listCacheKey{Host: host, RunID: runID} | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -171,6 +171,72 @@ func TestIndexStrategyCachesMLflowEnrichment(t *testing.T) { | |
| assert.Equal(t, []string{"70"}, outputRunIDs, "get-output must use the task run id") | ||
| } | ||
|
|
||
| func TestJobsScanCachesTerminalMLflowLinks(t *testing.T) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: the repair and active-run cases are well covered. Two cases are missing: a terminal run whose first lookup has no link ( |
||
| t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) | ||
|
|
||
| // scanRun is a runs/list AIR run; one task run per attempt, newest last. | ||
| scanRun := func(id int64, state jobs.RunLifeCycleState, attempts ...int) jobs.BaseRun { | ||
| r := airBaseRun(id, "me@example.com", "GPU_1xA10", 1, "exp") | ||
| r.State = &jobs.RunState{LifeCycleState: state} | ||
| task := r.Tasks[0] | ||
| r.Tasks = nil | ||
| for _, a := range attempts { | ||
| task.RunId = id*10 + int64(a) | ||
| task.AttemptNumber = a | ||
| r.Tasks = append(r.Tasks, task) | ||
| } | ||
| return r | ||
| } | ||
| var runs []jobs.BaseRun | ||
| var outputHits, mlflowHits int | ||
| srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| switch r.URL.Path { | ||
| case "/api/2.2/jobs/runs/list": | ||
| _, _ = w.Write([]byte(runsListBody(t, "", runs...))) | ||
| case "/api/2.2/jobs/runs/get-output": | ||
| outputHits++ | ||
| _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"exp1","mlflow_run_id":"m-` + r.URL.Query().Get("run_id") + `"}}`)) | ||
| case "/api/2.0/mlflow/runs/get": | ||
| mlflowHits++ | ||
| _, _ = w.Write([]byte(`{"run":{"info":{"run_name":"training-run"}}}`)) | ||
| default: | ||
| _, _ = w.Write([]byte(`{}`)) | ||
| } | ||
| })) | ||
| t.Cleanup(srv.Close) | ||
| list := func() []listRow { | ||
| rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ | ||
| userFilter: "me@example.com", fetchMLflow: true, limit: 10, | ||
| }).next(10) | ||
| require.NoError(t, err) | ||
| require.Len(t, rows, 1) | ||
| return rows | ||
| } | ||
|
|
||
| // A terminal run's links are resolved once, then served from the cache. | ||
| runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0)} | ||
| first := list() | ||
| assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-70/logs", first[0].MLflowURL) | ||
| assert.Equal(t, first, list()) | ||
| assert.Equal(t, 1, outputHits) | ||
| assert.Equal(t, 1, mlflowHits) | ||
|
|
||
| // A repaired run has a new attempt, so its cached links are stale. | ||
| runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0, 1)} | ||
| assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-71/logs", list()[0].MLflowURL) | ||
| assert.Equal(t, 2, outputHits) | ||
|
|
||
| // An active run is never served from the cache. | ||
| runs = []jobs.BaseRun{scanRun(8, jobs.RunLifeCycleStateRunning, 0)} | ||
| list() | ||
| list() | ||
| assert.Equal(t, 4, outputHits) | ||
|
|
||
| // The scan's entries never stand in for the index path's full rows. | ||
| _, _, _, ok := cachedRow(t.Context(), newListCache(t.Context()), srv.URL, 7) | ||
| assert.False(t, ok) | ||
| } | ||
|
|
||
| func TestIsTerminal(t *testing.T) { | ||
| assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}})) | ||
| assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateInternalError}})) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: the default list is server-side
ActiveOnly, so finished rows only appear with--all-status. Could the entry mention--all-statusrather than--all-users/--filter user=?