Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7743.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air list --all-status` now fills `--limit` past purged or inaccessible indexed runs. ([#7743](https://github.com/databricks/cli/pull/7743))
8 changes: 0 additions & 8 deletions cmd/air/list_filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,6 @@ import (
// supportedFilterKeys are the keys accepted by `air list --filter KEY=VALUE`.
var supportedFilterKeys = []string{"accelerator_type", "experiment", "num_accelerators", "user"}

// hasTaskFilter reports whether any filter is applied to a run's task fields
// (experiment or accelerators), i.e. matched after a run is fetched rather than
// while scanning. The index path uses this to skip its newest-N truncation, so a
// dropped match doesn't shrink the result below --limit.
func (f listFilters) hasTaskFilter() bool {
return f.Experiment != "" || f.AcceleratorType != "" || f.NumAccelerators != nil
}

// listFilters holds the parsed `--filter` values for `air list`.
type listFilters struct {
// User is an exact creator-email match
Expand Down
23 changes: 11 additions & 12 deletions cmd/air/list_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

// indexStrategy serves the caller's own runs from the AiTrainingService index:
// it fetches every run id up front (cheap id+timestamp pairs), orders them
// newest-first, keeps the newest `limit`, then hydrates them into full rows in
// newest-first, then hydrates them into full rows in
// want-sized batches via Jobs runs/get. Terminal rows are cached so repeat calls
// skip the network. Unlike the Jobs scan it can't lazy-page (it must sort the
// whole id set first), but it still yields in batches so the table paints early.
Expand All @@ -25,9 +25,10 @@ type indexStrategy struct {
host string
workspaceID int64

ids []int64 // newest-first run ids to hydrate, resolved on first next()
pos int
loaded bool
ids []int64 // newest-first run ids to hydrate, resolved on first next()
pos int
returned int
loaded bool
}

func newIndexStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery, limit int) *indexStrategy {
Expand All @@ -51,12 +52,6 @@ func (s *indexStrategy) load() error {
return err
}
slices.SortFunc(refs, func(a, b workflowRef) int { return cmp.Compare(b.submitTimeMs, a.submitTimeMs) })
// Keep only the newest `limit` ids so hydration is bounded — but skip that when
// a task filter is active, since it drops matches post-hydration and we'd
// otherwise return fewer than `limit`. The caller stops pulling at `limit`.
if s.limit > 0 && len(refs) > s.limit && !s.filters.hasTaskFilter() {
refs = refs[:s.limit]
}
s.ids = make([]int64, len(refs))
for i, r := range refs {
s.ids[i] = r.jobRunID
Expand All @@ -71,6 +66,9 @@ func (s *indexStrategy) next(want int) ([]listedRun, error) {
return nil, err
}
}
if s.limit > 0 {
want = min(want, s.limit-s.returned)
}

var entries []listedRun
for len(entries) < want && s.pos < len(s.ids) {
Expand All @@ -84,14 +82,15 @@ func (s *indexStrategy) next(want int) ([]listedRun, error) {
}
entries = append(entries, rows...)
}
s.returned += len(entries)
return entries, nil
}

func (s *indexStrategy) done() bool {
return s.loaded && s.pos >= len(s.ids)
return s.loaded && ((s.limit > 0 && s.returned >= s.limit) || s.pos >= len(s.ids))
}

// truncated is always false: the index path is bounded by limit, not a scan cap.
// truncated is always false: the index path stops at the requested limit, not a scan cap.
func (s *indexStrategy) truncated() bool { return false }

// hydrate turns a batch of run ids into rows, serving cached terminal rows
Expand Down
19 changes: 19 additions & 0 deletions cmd/air/list_index_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,25 @@ func TestIndexStrategyDropsForbiddenAndMissing(t *testing.T) {
assert.Equal(t, "3", rows[1].RunID)
}

func TestIndexStrategyFillsLimitPastMissingRun(t *testing.T) {
refs := []workflowRef{
{jobRunID: 1, submitTimeMs: 3000_000},
{jobRunID: 2, submitTimeMs: 2000_000},
}
runs := map[int64]jobs.Run{2: indexRun(2, 2000_000)}
srv, _ := indexAndGetServer(t, refs, runs, nil, map[int64]bool{1: true})
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
currentUser: selfUser,
limit: 1,
})
rows, err := f.next(1)
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, "2", rows[0].RunID)
}

func TestIndexStrategyPropagatesServerError(t *testing.T) {
refs := []workflowRef{{jobRunID: 1, submitTimeMs: 1000_000}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down
Loading