From 30a84ca418b303ff6b4aa37b6fb7d236ba1b9efb Mon Sep 17 00:00:00 2001 From: SE agent Date: Wed, 11 Nov 2026 13:30:51 +0000 Subject: [PATCH] feat(ann): filter-aware candidate retrieval (#3) --- internal/brain/ann/ann.go | 161 +++++++++--- internal/brain/ann/ann_filter_test.go | 213 ++++++++++++++++ internal/brain/ann_filter_e2e_test.go | 336 +++++++++++++++++++++++++ internal/brain/ann_integration_test.go | 9 +- internal/brain/ann_rollout_test.go | 5 +- internal/brain/annbridge.go | 25 +- internal/brain/anntool.go | 24 +- internal/brain/bench_gate_test.go | 3 +- internal/brain/ingest_race_test.go | 2 +- internal/brain/ingest_soak_test.go | 7 +- internal/brain/rank/evalq.go | 31 ++- internal/brain/rank/filter.go | 101 ++++++++ internal/brain/rank/filter_test.go | 248 ++++++++++++++++++ internal/brain/rank/query.go | 13 + internal/brain/rank/rank.go | 41 +-- internal/brain/read.go | 140 ++++++++++- internal/brain/search.go | 114 +++++++-- 17 files changed, 1353 insertions(+), 120 deletions(-) create mode 100644 internal/brain/ann/ann_filter_test.go create mode 100644 internal/brain/ann_filter_e2e_test.go create mode 100644 internal/brain/rank/filter.go create mode 100644 internal/brain/rank/filter_test.go diff --git a/internal/brain/ann/ann.go b/internal/brain/ann/ann.go index b2527db..0a77448 100644 --- a/internal/brain/ann/ann.go +++ b/internal/brain/ann/ann.go @@ -3,6 +3,7 @@ package ann import ( "bufio" "encoding/binary" + "errors" "fmt" "io" "math" @@ -15,9 +16,21 @@ import ( ) // Row is one leaf vector to index, keyed by the brain leaf id (string). +// +// Since issue #3 a row also carries the leaf metadata that search filters on +// (root, source, D24 validity interval) so retrieval can skip ineligible rows +// before the top-k cut. A rare root (facts = 0.09% of the live corpus) would +// otherwise never survive a limit*3 candidate window. The fields are optional +// for backward compatibility with WAL lines written before v2: empty Root +// matches no root filter, and empty intervals mean "legacy leaf, always +// active", exactly like rank.Filter. type Row struct { - ID string - Vec []float32 + ID string + Vec []float32 + Root string + Source string + ValidFrom string + ValidTo string } // Result is one nearest-neighbor hit with a cosine similarity in [0,1] @@ -376,7 +389,7 @@ func (ix *Index) applyDelLocked(id string) bool { // query. A zero query vector or an empty index yields no hits (the caller // falls back to a linear scan / FTS-only ranking). func (ix *Index) Search(vec []float32, k int) []Result { - return ix.search(float64Vec(vec), k) + return ix.search(float64Vec(vec), k, nil) } // Search64 is Search for the float64 query vector the embedding daemon @@ -386,10 +399,24 @@ func (ix *Index) Search(vec []float32, k int) []Result { // the true neighbors (probe #204: float32 query rounding reordered dense // top-30 and broke recall@5 vs baseline). func (ix *Index) Search64(q []float64, k int) []Result { - return ix.search(q, k) + return ix.search(q, k, nil) } -func (ix *Index) search(q []float64, k int) []Result { +// SearchFiltered is Search64 restricted to the rows pred accepts (nil pred = +// every row). The filter is applied *inside* the cell scan, so ineligible rows +// never consume a top-k slot (issue #3). +// +// Eligibility is sparse by nature (facts are 0.09% of the corpus), so when the +// configured NProbe cells do not yield k eligible rows the probe set doubles +// until it does or the whole index has been scanned. Filtered retrieval is +// therefore an exact filtered pass in the worst case instead of a silently +// empty answer — and with a rare filter that worst case is the normal case. +// pred receives a copy of the row and must not retain it. +func (ix *Index) SearchFiltered(q []float64, k int, pred func(Row) bool) []Result { + return ix.search(q, k, pred) +} + +func (ix *Index) search(q []float64, k int, pred func(Row) bool) []Result { ix.mu.RLock() defer ix.mu.RUnlock() if k <= 0 || len(ix.centroids) == 0 || zeroNorm64(q) { @@ -405,22 +432,41 @@ func (ix *Index) search(q []float64, k int) []Result { cells[i] = cs{idx: i, sim: cosine64(q, c)} } sort.Slice(cells, func(i, j int) bool { return cells[i].sim > cells[j].sim }) - nprobe := ix.params.NProbe - if nprobe > len(cells) { - nprobe = len(cells) - } // Collect candidates from the probed cells and rank by exact cosine. type cand struct { r Row sim float64 } var out []cand - for p := 0; p < nprobe; p++ { - for _, r := range ix.lists[cells[p].idx] { - if _, dead := ix.tombstones[r.ID]; dead { - continue // deleted leaf: must not occupy a top-k slot (#4) + probe := ix.params.NProbe + if probe > len(cells) { + probe = len(cells) + } + if probe < 1 { + probe = 1 + } + scanned := 0 + for { + for ; scanned < probe; scanned++ { + for _, r := range ix.lists[cells[scanned].idx] { + if _, dead := ix.tombstones[r.ID]; dead { + continue // deleted leaf: must not occupy a top-k slot (#4) + } + if pred != nil && !pred(r) { + continue + } + out = append(out, cand{r: r, sim: cosine64(q, r.Vec)}) } - out = append(out, cand{r: r, sim: cosine64(q, r.Vec)}) + } + // Unfiltered search keeps the fixed NProbe budget (#204 recall gate); + // a filter widens the probe set until it has k eligible rows or the + // whole index is covered. + if pred == nil || len(out) >= k || probe >= len(cells) { + break + } + probe *= 2 + if probe > len(cells) { + probe = len(cells) } } // Deterministic ranking: descending similarity, ties broken by id — @@ -565,11 +611,29 @@ func (ix *Index) SaveTo(path string) error { // snapshot format (little-endian, fixed layout so 320MB loads fast): // -// magic "2DPHANN1" (8 bytes) +// magic "2DPHANN2" (8 bytes) // dim int32, nlist int32, nprobe int32 // centroids: nlist * (dim * float32) -// cells: nlist * { count int32, count * (idLen int32, id, dim * float32) } -const snapshotMagic = "2DPHANN1" +// cells: nlist * { count int32, count * row } +// row: idLen int32, id, root, source, validFrom, validTo (each len int32 + bytes), +// dim * float32 +// +// v2 (issue #3) added the four metadata strings per row: filter-aware retrieval +// (root/repo/as-of) reads them without a graph point lookup. A v1 snapshot +// ("2DPHANN1") carries no metadata and is refused with ErrStaleFormat — the +// caller falls back to the linear scan and the wave's ann-build rebuilds it; +// reading v1 as v2 would mis-place every row boundary and drop all filters. +const ( + snapshotMagic = "2DPHANN2" + snapshotMagicV1 = "2DPHANN1" + // FormatVersion is the current snapshot format; surfaced by + // bin/brain/ann.go stats so a rollout can see whether a rebuild happened. + FormatVersion = 2 +) + +// ErrStaleFormat means the snapshot was written by an older index version and +// must be rebuilt: callers fall back to the linear scan until then. +var ErrStaleFormat = errors.New("ann snapshot: stale format") func (ix *Index) saveSnapshot() error { if err := os.MkdirAll(filepath.Dir(ix.path), 0o755); err != nil { @@ -635,11 +699,13 @@ func (ix *Index) writeSnapshot(w io.Writer) error { if _, dead := ix.tombstones[r.ID]; dead { continue } - if err := binary.Write(w, binary.LittleEndian, int32(len(r.ID))); err != nil { + if err := writeString(w, r.ID); err != nil { return err } - if _, err := w.Write([]byte(r.ID)); err != nil { - return err + for _, s := range []string{r.Root, r.Source, r.ValidFrom, r.ValidTo} { + if err := writeString(w, s); err != nil { + return err + } } for _, v := range r.Vec { if err := binary.Write(w, binary.LittleEndian, v); err != nil { @@ -651,6 +717,33 @@ func (ix *Index) writeSnapshot(w io.Writer) error { return nil } +// metaStringMax bounds one metadata string read from a snapshot (root/source +// names and ISO days are short; anything bigger is a corrupt offset). +const metaStringMax = 4096 + +func writeString(w io.Writer, s string) error { + if err := binary.Write(w, binary.LittleEndian, int32(len(s))); err != nil { + return err + } + _, err := io.WriteString(w, s) + return err +} + +func readString(r io.Reader, max int) (string, error) { + var n int32 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return "", err + } + if n < 0 || n > int32(max) { + return "", fmt.Errorf("ann snapshot: bad string len %d", n) + } + b := make([]byte, n) + if _, err := io.ReadFull(r, b); err != nil { + return "", err + } + return string(b), nil +} + // compactLocked drops tombstoned rows from the cells, rebuilds idCell and // clears the tombstone set. It runs after the snapshot landed, so a failure // mid-save leaves the WAL + tombstones intact for the next attempt. One @@ -703,7 +796,11 @@ func (ix *Index) readSnapshot(r io.Reader) error { if _, err := io.ReadFull(r, magic); err != nil { return fmt.Errorf("ann snapshot: %w", err) } - if string(magic) != snapshotMagic { + switch string(magic) { + case snapshotMagic: + case snapshotMagicV1: + return fmt.Errorf("%w: v1 snapshot carries no filter metadata (current v%d) — rebuild the index", ErrStaleFormat, FormatVersion) + default: return fmt.Errorf("ann snapshot: bad magic %q", magic) } var hdr [3]int32 @@ -739,23 +836,25 @@ func (ix *Index) readSnapshot(r io.Reader) error { } cell := make([]Row, 0, cnt) for j := 0; j < int(cnt); j++ { - var idLen int32 - if err := binary.Read(r, binary.LittleEndian, &idLen); err != nil { + id, err := readString(r, 1<<16) + if err != nil { return err } - if idLen < 0 || idLen > 1<<16 { - return fmt.Errorf("ann snapshot: bad id len %d", idLen) - } - id := make([]byte, idLen) - if _, err := io.ReadFull(r, id); err != nil { - return err + var meta [4]string + for k := range meta { + if meta[k], err = readString(r, metaStringMax); err != nil { + return err + } } vec := make([]float32, dim) if err := binary.Read(r, binary.LittleEndian, vec); err != nil { return err } - cell = append(cell, Row{ID: string(id), Vec: vec}) - ix.idCell[string(id)] = c + cell = append(cell, Row{ + ID: id, Vec: vec, + Root: meta[0], Source: meta[1], ValidFrom: meta[2], ValidTo: meta[3], + }) + ix.idCell[id] = c } ix.lists[c] = cell } diff --git a/internal/brain/ann/ann_filter_test.go b/internal/brain/ann/ann_filter_test.go new file mode 100644 index 0000000..5bf9505 --- /dev/null +++ b/internal/brain/ann/ann_filter_test.go @@ -0,0 +1,213 @@ +package ann + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" +) + +// filterRows returns n rows: n-1 info rows in one tight cluster around e0 plus +// one facts row in a second cluster around e1, so "root == facts" is a rare +// slice exactly like the live corpus (39 facts in 43k leafs). +func filterRows(n int) ([]Row, []float32) { + rows := make([]Row, 0, n) + for i := 0; i < n-1; i++ { + v := make([]float32, 256) + v[0] = 1 + v[8+(i%16)] = float32(i%7) * 0.001 + rows = append(rows, Row{ID: "info-" + itoa(i), Vec: v, Root: "info", + Source: "docs/info-" + itoa(i) + ".md", ValidFrom: "", ValidTo: ""}) + } + fact := make([]float32, 256) + fact[1] = 1 + rows = append(rows, Row{ID: "fact-1", Vec: fact, Root: "facts", + Source: "office-assistant/AGENTS.md x 2dph/AGENTS.md", + ValidFrom: "2025-01-01", ValidTo: "2025-12-31"}) + + q := make([]float32, 256) + q[0] = 1 + return rows, q +} + +func rootIs(root string) func(Row) bool { + return func(r Row) bool { return r.Root == root } +} + +// TestSearchFilteredHitsRareRoot — an unfiltered top-1 is an info leaf; the +// filtered search returns the facts leaf that the candidate window would drop +// (the defect of issue #3, at the index level). +func TestSearchFilteredHitsRareRoot(t *testing.T) { + rows, q := filterRows(200) + ix := New(Params{RngSeed: 3, NList: 4, NProbe: 4}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + top := ix.Search(q, 1) + if len(top) != 1 || top[0].ID != "info-0" { + t.Fatalf("unfiltered top-1 = %v, want the info leaf (fixture sanity)", top) + } + got := ix.SearchFiltered(float64Vec(q), 1, rootIs("facts")) + if len(got) != 1 || got[0].ID != "fact-1" { + t.Fatalf("filtered top-1 = %v, want fact-1", got) + } +} + +// TestSearchFilteredEscalatesProbeSet — with NProbe=1 only one cell is probed +// by default; a rare root in a far cell must widen the probe set until it is +// covered (the in-index form of the exact-scan fallback). pred sees every row +// exactly once, so the call count proves the escalation reached all 200 rows. +func TestSearchFilteredEscalatesProbeSet(t *testing.T) { + rows, q := filterRows(200) + ix := New(Params{RngSeed: 3, NList: 4, NProbe: 1}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + // Sanity: NProbe=1 really restricts the unfiltered search to one cell. + if got := ix.Search(q, 200); len(got) >= len(rows) { + t.Fatalf("unfiltered search returned all %d rows, NProbe=1 not applied", len(got)) + } + seen := 0 + got := ix.SearchFiltered(float64Vec(q), 1, func(r Row) bool { + seen++ + return r.Root == "facts" + }) + if len(got) != 1 || got[0].ID != "fact-1" { + t.Fatalf("filtered top-1 = %v, want fact-1 (probe escalation)", got) + } + if seen != len(rows) { + t.Fatalf("pred saw %d rows, want all %d (probe set must widen)", seen, len(rows)) + } +} + +// TestSearchFilteredNoMatchTerminates — a filter that matches nothing must end +// with an empty result once every cell has been probed, not spin. +func TestSearchFilteredNoMatchTerminates(t *testing.T) { + rows, q := filterRows(120) + ix := New(Params{RngSeed: 5, NList: 4, NProbe: 1}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + got := ix.SearchFiltered(float64Vec(q), 5, rootIs("nothing")) + if len(got) != 0 { + t.Fatalf("filtered search for an absent root = %v, want empty", got) + } +} + +// TestSearchUnfilteredUnaffected — nil pred mirrors Search64, so the #204 +// unfiltered recall gate keeps measuring the same candidate set. +func TestSearchUnfilteredUnaffected(t *testing.T) { + rows, q := filterRows(120) + ix := New(Params{RngSeed: 7, NList: 4, NProbe: 2}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + want := ix.Search64(float64Vec(q), 10) + got := ix.Search(q, 10) + if len(got) != len(want) { + t.Fatalf("Search = %d hits, Search64 = %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("hit %d: Search %+v != Search64 %+v", i, got[i], want[i]) + } + } +} + +// TestSnapshotV2KeepsFilterMetadata — the filter fields survive save/open, so +// a restarted serve filters without a graph point lookup. +func TestSnapshotV2KeepsFilterMetadata(t *testing.T) { + rows, q := filterRows(150) + ix := New(Params{RngSeed: 11, NList: 4, NProbe: 4}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + path := filepath.Join(t.TempDir(), "vector.ann") + if err := ix.SaveTo(path); err != nil { + t.Fatalf("save: %v", err) + } + ix2, err := Open(path, Params{RngSeed: 11, NList: 4, NProbe: 4}) + if err != nil { + t.Fatalf("open: %v", err) + } + defer ix2.Close() + got := ix2.SearchFiltered(float64Vec(q), 1, func(r Row) bool { + return r.Root == "facts" && r.Source == "office-assistant/AGENTS.md x 2dph/AGENTS.md" && + r.ValidFrom == "2025-01-01" && r.ValidTo == "2025-12-31" + }) + if len(got) != 1 || got[0].ID != "fact-1" { + t.Fatalf("metadata lost across the snapshot: %v", got) + } + if err := ix2.Upsert([]Row{{ID: "facts-2", Vec: q, Root: "facts", Source: "2dph/AGENTS.md"}}); err != nil { + t.Fatalf("upsert: %v", err) + } + if got := ix2.SearchFiltered(float64Vec(q), 2, rootIs("facts")); len(got) != 2 { + t.Fatalf("upserted row not filtered in: %v", got) + } +} + +// TestSnapshotV1Rejected — the format bump (issue #3): a v1 snapshot carries no +// filter metadata. Reading it as v2 would mis-place every row boundary, so Open +// must fail with ErrStaleFormat and let the caller rebuild or fall back. +func TestSnapshotV1Rejected(t *testing.T) { + path := filepath.Join(t.TempDir(), "vector.ann") + var buf bytes.Buffer + buf.WriteString(snapshotMagicV1) + hdr := [3]int32{int32(Defaults().Dim), 1, int32(Defaults().NProbe)} + if err := binary.Write(&buf, binary.LittleEndian, &hdr); err != nil { + t.Fatal(err) + } + buf.Write(make([]byte, int(hdr[0])*4)) // one centroid + buf.Write(make([]byte, 4)) // one empty cell + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + _, err := Open(path, Defaults()) + if err == nil { + t.Fatal("v1 snapshot must be rejected") + } + if !errors.Is(err, ErrStaleFormat) { + t.Fatalf("err = %v, want ErrStaleFormat", err) + } +} + +// TestWALV1RowsReplayWithoutMetadata — WAL lines written before the format bump +// stay readable (JSON, optional fields): they replay as legacy rows with an +// empty root, matching no root filter and staying as-of eligible (D24). +func TestWALV1RowsReplayWithoutMetadata(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "vector.ann") + rows, _ := filterRows(20) + ix := New(Params{RngSeed: 13, NList: 4, NProbe: 4}) + if err := ix.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + if err := ix.SaveTo(path); err != nil { + t.Fatalf("save: %v", err) + } + vec := make([]float32, 256) + vec[2] = 1 + line, err := json.Marshal(map[string]any{"ID": "legacy-1", "Vec": vec}) // v1 shape + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path+".wal", append(line, '\n'), 0o644); err != nil { + t.Fatal(err) + } + ix2, err := Open(path, Params{RngSeed: 13, NList: 4, NProbe: 4}) + if err != nil { + t.Fatalf("open: %v", err) + } + defer ix2.Close() + if !ix2.Lookup("legacy-1") { + t.Fatal("v1 WAL row not replayed") + } + if got := ix2.SearchFiltered(float64Vec(vec), 1, func(r Row) bool { + return r.ID == "legacy-1" && r.Root == "" && r.Source == "" + }); len(got) != 1 { + t.Fatal("v1 WAL row must replay with empty metadata") + } +} diff --git a/internal/brain/ann_filter_e2e_test.go b/internal/brain/ann_filter_e2e_test.go new file mode 100644 index 0000000..f74e835 --- /dev/null +++ b/internal/brain/ann_filter_e2e_test.go @@ -0,0 +1,336 @@ +//go:build cgo && system_ladybug + +// Filter-aware retrieval e2e gate (issue #3) on a synthetic fixture DB: a +// corpus of n info leafs that dominate BM25 and the vector neighborhood plus a +// single facts leaf. The old pipeline (fetch limit*3 candidates unfiltered, +// apply root/repo/as-of afterwards) returns zero facts on these queries — the +// measured live defect (43,133 leafs, 39 facts) — while the filtered retrieval +// finds the fact at n=5. All offline: deterministic embeddings, no model. +package brain + +import ( + "fmt" + "math/rand" + "path/filepath" + "strings" + "testing" + + "github.com/eSlider/2dph/internal/brain/ann" + "github.com/eSlider/2dph/internal/brain/rank" +) + +// rareRootQuery is the broad query the info leafs dominate. +const rareRootQuery = "matrix server docker host" + +// rareRootFact is the single fact leaf: short text (low BM25), a vector in the +// other cluster (far from a query aimed at the info cluster), a source that +// carries the repo filter, and a dated D24 interval. +const rareRootFact = "Матрикс: matrix — правило хоста docker, факт из двух источников." + +// rareRootLeafs returns n info leafs (dense query terms, embeddings in cluster +// A) plus the single facts leaf (cluster B). +func rareRootLeafs(n int) []LeafInput { + const dim = EmbedDim + info := make([]float32, dim) + info[0] = 1 + leafs := make([]LeafInput, 0, n+1) + rng := rand.New(rand.NewSource(3)) + for i := 0; i < n; i++ { + emb := make([]float64, dim) + for j := range emb { + emb[j] = float64(info[j]) + } + for j := 0; j < dim; j += 8 { + emb[j] += (rng.Float64() - 0.5) * 0.01 + } + leafs = append(leafs, LeafInput{ + Text: fmt.Sprintf("%s — info notes %d: %s %s %s %s.", + rareRootQuery, i, rareRootQuery, rareRootQuery, rareRootQuery, rareRootQuery), + Source: fmt.Sprintf("fixture-info-%d.md", i), + Root: "info", + Type: "reference", + How: "ann-filter-e2e", + Embedding: emb, + }) + } + fact := make([]float64, dim) + fact[1] = 1 + leafs = append(leafs, LeafInput{ + Text: rareRootFact, + Source: "repo-alpha/docs/AGENTS.md x 2dph/AGENTS.md", + Root: "facts", + Type: "fact", + Confidence: "confirmed", + ValidFrom: "2025-01-01", + ValidTo: "2025-12-31", + How: "ann-filter-e2e", + Embedding: fact, + }) + return leafs +} + +// queryNearInfoCluster returns a query embedding close to the info cluster, so +// the facts leaf is outside the unfiltered vector top-k. +func queryNearInfoCluster() []float64 { + q := make([]float64, EmbedDim) + q[0] = 1 + q[8] = 0.01 + return q +} + +// hasText reports whether any hit carries the fragment (case-insensitive). +func hasText(hits []Hit, fragment string) bool { + frag := strings.ToLower(fragment) + for _, h := range hits { + if strings.Contains(strings.ToLower(h.Text), frag) { + return true + } + } + return false +} + +// stubQueryEmbed points the query embedding at the info cluster (no model). +func stubQueryEmbed(t *testing.T, emb []float64) { + t.Helper() + prev := embedQueryFn + embedQueryFn = func(string) ([]float64, error) { return emb, nil } + t.Cleanup(func() { embedQueryFn = prev }) +} + +// TestFilteredRetrievalFindsRareRoot is the acceptance gate of issue #3: with +// n=5 the old pipeline loses the fact, the filtered one returns it. +func TestFilteredRetrievalFindsRareRoot(t *testing.T) { + const nInfo = 120 + leafs := rareRootLeafs(nInfo) + dbpath := annFixtureDBLeafs(t, leafs) + openFixtureRead(t, dbpath) + setAnnCfg(t, false, "") // vector path: exact scan, no index in this test + stubQueryEmbed(t, queryNearInfoCluster()) + + // Old behaviour, exactly as searchHits did before the fix: an unfiltered + // limit*3 window per retriever, root applied afterwards. + oldFTS, err := queryFTS(rareRootQuery, 5*3, rank.Filter{}) + if err != nil { + t.Fatalf("old fts: %v", err) + } + if got := rank.FilterHits(oldFTS, rank.Filter{Root: "facts"}); len(got) != 0 { + t.Fatalf("fixture must reproduce the defect: old FTS window already holds %d facts", len(got)) + } + oldVec, err := queryVector(queryNearInfoCluster(), 5*3, rank.Filter{}) + if err != nil { + t.Fatalf("old vec: %v", err) + } + if got := rank.FilterHits(oldVec, rank.Filter{Root: "facts"}); len(got) != 0 { + t.Fatalf("fixture must reproduce the defect: old vector window already holds %d facts", len(got)) + } + + // New behaviour: the filter is applied before the top-k cut. + hits, err := searchHits(rareRootQuery, "facts", "", 5, "", false, false) + if err != nil { + t.Fatalf("searchHits: %v", err) + } + if !hasText(hits, "matrix") { + t.Fatalf("root=facts returned no fact for %q: %v", rareRootQuery, hits) + } + for _, h := range hits { + if h.Root != "facts" { + t.Fatalf("filter leak: %s has root %q", h.ID, h.Root) + } + } +} + +// TestFilteredRetrievalRepoAndAsOf — repo (source substring) and as-of (D24) +// are eligible-aware too: the fact is found while in the interval and is +// honestly absent outside it, with no ineligible hit slipping through. +func TestFilteredRetrievalRepoAndAsOf(t *testing.T) { + dbpath := annFixtureDBLeafs(t, rareRootLeafs(80)) + openFixtureRead(t, dbpath) + setAnnCfg(t, false, "") + stubQueryEmbed(t, queryNearInfoCluster()) + + hits, err := searchHits(rareRootQuery, "", "repo-alpha", 5, "", false, false) + if err != nil { + t.Fatalf("repo search: %v", err) + } + if !hasText(hits, "matrix") { + t.Fatalf("repo=repo-alpha lost the fact: %v", hits) + } + for _, h := range hits { + if !strings.Contains(h.Source, "repo-alpha") { + t.Fatalf("repo filter leak: %s source %q", h.ID, h.Source) + } + } + + if hits, err = searchHits(rareRootQuery, "", "no-such-repo", 5, "", false, false); err != nil { + t.Fatalf("missing repo search: %v", err) + } else if len(hits) != 0 { + t.Fatalf("repo=no-such-repo returned %d hits, want 0", len(hits)) + } + + inWindow, err := searchHits(rareRootQuery, "facts", "", 5, "2025-06-01", false, false) + if err != nil { + t.Fatalf("as-of search: %v", err) + } + if !hasText(inWindow, "matrix") { + t.Fatalf("as-of inside the interval lost the fact: %v", inWindow) + } + outWindow, err := searchHits(rareRootQuery, "facts", "", 5, "2026-06-01", false, false) + if err != nil { + t.Fatalf("as-of search (outside): %v", err) + } + if hasText(outWindow, "matrix") { + t.Fatalf("as-of outside the interval still returned the fact: %v", outWindow) + } +} + +// TestFilteredScanRanksEligibleBeforeCut — the exact scan (no ANN index) must +// apply the filter before the top-k cut; the fact vector is the farthest from +// the query, so an unfiltered scan cannot see it. +func TestFilteredScanRanksEligibleBeforeCut(t *testing.T) { + dbpath := annFixtureDBLeafs(t, rareRootLeafs(60)) + openFixtureRead(t, dbpath) + setAnnCfg(t, false, "") + + unfiltered, err := queryVector(queryNearInfoCluster(), 5, rank.Filter{}) + if err != nil { + t.Fatalf("unfiltered scan: %v", err) + } + if hasText(unfiltered, "matrix — правило") { + t.Fatal("fixture sanity: the fact must not be in the unfiltered vector top-5") + } + filtered, err := queryVector(queryNearInfoCluster(), 5, rank.Filter{Root: "facts"}) + if err != nil { + t.Fatalf("filtered scan: %v", err) + } + if !hasText(filtered, "matrix — правило") { + t.Fatalf("filtered scan lost the far fact: %v", filtered) + } +} + +// TestFilteredANNIndexCarriesMetadata — the built index (extractRows → build) +// carries root/source/interval, so the ANN path filters without a graph lookup +// and survives a snapshot round-trip. +func TestFilteredANNIndexCarriesMetadata(t *testing.T) { + dbpath := annFixtureDBLeafs(t, rareRootLeafs(100)) + openFixtureRead(t, dbpath) + rows, err := extractRows(0) + if err != nil { + t.Fatalf("extractRows: %v", err) + } + facts := 0 + for _, r := range rows { + if r.Root == "facts" { + facts++ + if r.Source == "" || r.ValidFrom != "2025-01-01" || r.ValidTo != "2025-12-31" { + t.Fatalf("extractRows lost filter metadata: %+v", r) + } + } + } + if facts != 1 { + t.Fatalf("extracted %d facts rows, want 1", facts) + } + idxPath := filepath.Join(filepath.Dir(dbpath), "vector.ann") + idx := ann.New(ann.Params{RngSeed: 1}) + if err != nil { + t.Fatal(err) + } + if err := idx.Build(rows); err != nil { + t.Fatalf("build: %v", err) + } + if err := idx.SaveTo(idxPath); err != nil { + t.Fatalf("save: %v", err) + } + idx.Close() + setANNIndexPath(t, idxPath) + setAnnCfg(t, true, idxPath) + + hits, err := queryVector(queryNearInfoCluster(), 5, rank.Filter{Root: "facts"}) + if err != nil { + t.Fatalf("filtered ANN query: %v", err) + } + if len(hits) != 1 || hits[0].Root != "facts" { + t.Fatalf("filtered ANN query = %v, want the one fact", hits) + } +} + +// TestFilteredEvalGate — the gate of bin/brain/eval.go on the fixture: the +// filtered cases of the issue (root/repo/as_of before the cut) pass, a missing +// corpus slice is skipped instead of failing CI, and a wrong expectation still +// fails (the gate is not vacuous). +func TestFilteredEvalGate(t *testing.T) { + dbpath := annFixtureDBLeafs(t, rareRootLeafs(80)) + openFixtureRead(t, dbpath) + setAnnCfg(t, false, "") + stubQueryEmbed(t, queryNearInfoCluster()) + + prev := rank.EvalFilteredQuestions + t.Cleanup(func() { rank.EvalFilteredQuestions = prev }) + + // Facts slice exists and the queries hit it → gate passes. + rank.EvalFilteredQuestions = []rank.EvalQuestion{ + {Query: rareRootQuery, Fragment: "matrix", Root: "facts"}, + {Query: rareRootQuery, Fragment: "matrix", Repo: "repo-alpha"}, + {Query: rareRootQuery, Fragment: "matrix", Root: "facts", AsOf: "2025-06-01"}, + } + recalled, total, _, _ := evalFilteredGate() + if total != 3 || recalled != 3 { + t.Fatalf("filtered gate = %d/%d, want 3/3", recalled, total) + } + + // A slice the corpus lacks is skipped, not failed. + rank.EvalFilteredQuestions = []rank.EvalQuestion{ + {Query: rareRootQuery, Fragment: "matrix", Root: "hypothesis-only"}, + {Query: rareRootQuery, Fragment: "matrix", Repo: "no-such-repo"}, + } + if recalled, total, _, _ = evalFilteredGate(); total != 0 { + t.Fatalf("absent slices must be skipped, gate counted %d cases", total) + } + + // A wrong fragment fails: the gate measures the real retrieval. + rank.EvalFilteredQuestions = []rank.EvalQuestion{ + {Query: rareRootQuery, Fragment: "no-such-fragment", Root: "facts"}, + } + if recalled, total, _, _ = evalFilteredGate(); total != 1 || recalled != 0 { + t.Fatalf("wrong fragment must fail the case, gate = %d/%d", recalled, total) + } +} + +// TestFilteredANNStaleIndexFallsBackToScan — the index does not know the fact +// (built before it existed). An empty filtered ANN result must not become an +// empty answer: the exact scan proves existence. +func TestFilteredANNStaleIndexFallsBackToScan(t *testing.T) { + dbpath := annFixtureDBLeafs(t, rareRootLeafs(100)) + openFixtureRead(t, dbpath) + rows, err := extractRows(0) + if err != nil { + t.Fatalf("extractRows: %v", err) + } + fresh := rows[:0] + for _, r := range rows { + if r.Root != "facts" { + fresh = append(fresh, r) // simulate a stale index: fact not indexed + } + } + idxPath := filepath.Join(filepath.Dir(dbpath), "vector.ann") + idx := ann.New(ann.Params{RngSeed: 2}) + if err != nil { + t.Fatal(err) + } + if err := idx.Build(fresh); err != nil { + t.Fatalf("build: %v", err) + } + if err := idx.SaveTo(idxPath); err != nil { + t.Fatalf("save: %v", err) + } + idx.Close() + setANNIndexPath(t, idxPath) + setAnnCfg(t, true, idxPath) + + hits, err := queryVector(queryNearInfoCluster(), 5, rank.Filter{Root: "facts"}) + if err != nil { + t.Fatalf("filtered ANN query: %v", err) + } + if len(hits) != 1 || hits[0].Root != "facts" { + t.Fatalf("stale filtered index must fall back to the exact scan, got %v", hits) + } +} diff --git a/internal/brain/ann_integration_test.go b/internal/brain/ann_integration_test.go index 6f4ae26..52163fa 100644 --- a/internal/brain/ann_integration_test.go +++ b/internal/brain/ann_integration_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/eSlider/2dph/internal/brain/ann" + "github.com/eSlider/2dph/internal/brain/rank" ) func itoa(i int) string { return strconv.Itoa(i) } @@ -173,7 +174,7 @@ func TestANNQueryVectorUsesIndex(t *testing.T) { // A query near cluster 1 must return hits through the ANN path, with // metadata loaded from the graph (text/root populated). leafs := annFixtureLeafs(n, nClusters) - hits, err := annQueryVector(leafs[1].Embedding, 5) + hits, err := annQueryVector(leafs[1].Embedding, 5, rank.Filter{}) if err != nil { t.Fatalf("annQueryVector: %v", err) } @@ -195,14 +196,14 @@ func TestANNQueryVectorFallbackOnMissingIndex(t *testing.T) { // annQueryVector alone must yield nothing (empty index → fallback); // queryVector must still return hits via the brute-force scan. leafs := annFixtureLeafs(200, 8) - hits, err := annQueryVector(leafs[3].Embedding, 5) + hits, err := annQueryVector(leafs[3].Embedding, 5, rank.Filter{}) if err != nil { t.Fatalf("annQueryVector: %v", err) } if len(hits) != 0 { t.Fatal("annQueryVector without an index must return no hits") } - all, err := queryVector(leafs[3].Embedding, 5) + all, err := queryVector(leafs[3].Embedding, 5, rank.Filter{}) if err != nil { t.Fatalf("queryVector: %v", err) } @@ -340,7 +341,7 @@ func TestANNRecallParityWithScan(t *testing.T) { } truth := ann.ExactRank(toFloat32(q, EmbedDim), rows, 5) setAnnMode(annForceOn) - annHits, err := queryVector(q, 5) + annHits, err := queryVector(q, 5, rank.Filter{}) if err != nil { t.Fatal(err) } diff --git a/internal/brain/ann_rollout_test.go b/internal/brain/ann_rollout_test.go index 6de5613..fbf6903 100644 --- a/internal/brain/ann_rollout_test.go +++ b/internal/brain/ann_rollout_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/eSlider/2dph/internal/brain/ann" + "github.com/eSlider/2dph/internal/brain/rank" "github.com/eSlider/2dph/pkg/cli" ) @@ -495,7 +496,7 @@ func TestAnnWarmStartMissingFallsBack(t *testing.T) { t.Fatalf("warm start len = %d, want 0 (empty index)", st.Len) } leafs := annFixtureLeafs(150, 6) - hits, err := queryVector(leafs[2].Embedding, 5) + hits, err := queryVector(leafs[2].Embedding, 5, rank.Filter{}) if err != nil { t.Fatalf("queryVector fallback: %v", err) } @@ -525,7 +526,7 @@ func TestAnnWarmStartCorruptFallsBack(t *testing.T) { t.Fatal("corrupt warm start must record the error") } leafs := annFixtureLeafs(150, 6) - hits, err := queryVector(leafs[2].Embedding, 5) + hits, err := queryVector(leafs[2].Embedding, 5, rank.Filter{}) if err != nil { t.Fatalf("queryVector: %v", err) } diff --git a/internal/brain/annbridge.go b/internal/brain/annbridge.go index b902924..ec4db72 100644 --- a/internal/brain/annbridge.go +++ b/internal/brain/annbridge.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/eSlider/2dph/internal/brain/ann" + "github.com/eSlider/2dph/internal/brain/rank" "github.com/eSlider/2dph/internal/config" ) @@ -138,14 +139,25 @@ func WarmANN() { } // annQueryVector ranks the top-k vector hits through the ANN index and loads -// their metadata from the graph. Returns (nil, nil) when the index is -// unavailable or the query vector is degenerate — the caller falls back. -func annQueryVector(emb []float64, limit int) ([]Hit, error) { +// their metadata from the graph. The eligibility filter is applied inside the +// index scan (SearchFiltered), so ineligible rows never take a slot of the +// candidate window (issue #3: with facts at 0.09% of the corpus, filtering +// limit*3 candidates afterwards dropped them ~99% of the time). Returns +// (nil, nil) when the index is unavailable or the query vector is degenerate — +// the caller falls back. +func annQueryVector(emb []float64, limit int, f rank.Filter) ([]Hit, error) { idx, err := annIndex() if err != nil || idx == nil || idx.Len() == 0 { return nil, nil } - res := idx.Search64(emb, limit) + var res []ann.Result + if f.Empty() { + res = idx.Search64(emb, limit) + } else { + res = idx.SearchFiltered(emb, limit, func(r ann.Row) bool { + return f.Eligible(r.Root, r.Source, r.ValidFrom, r.ValidTo) + }) + } if len(res) == 0 { return nil, nil } @@ -156,6 +168,11 @@ func annQueryVector(emb []float64, limit int) ([]Hit, error) { // Leaf deleted between build and search: skip, not fatal. continue } + if !f.Eligible(h.Root, h.Source, h.ValidFrom, h.ValidTo) { + // The graph is the source of truth: a row whose metadata changed + // since the last index build must not leak through a filter. + continue + } out = append(out, h) } return out, nil diff --git a/internal/brain/anntool.go b/internal/brain/anntool.go index a2581a1..ccb1845 100644 --- a/internal/brain/anntool.go +++ b/internal/brain/anntool.go @@ -84,7 +84,8 @@ func extractRows(limit int) ([]ann.Row, error) { if conn == nil { return nil, fmt.Errorf("brain not open") } - stmt, err := conn.Prepare("MATCH (l:Leaf) RETURN l.id, l.embedding, l.text") + stmt, err := conn.Prepare("MATCH (l:Leaf) RETURN l.id, l.embedding, l.text, " + + "l.root, l.source, l.valid_from, l.valid_to") if err != nil { return nil, err } @@ -102,12 +103,19 @@ func extractRows(limit int) ([]ann.Row, error) { return nil, err } vals, err := row.GetAsSlice() - if err != nil || len(vals) < 3 { + if err != nil || len(vals) < 7 { continue } id := fmt.Sprint(vals[0]) + // Filter metadata (issue #3) travels with the vector so search can + // skip ineligible rows without a graph point lookup. + meta := ann.Row{ + ID: id, Root: nullStr(vals[3]), Source: nullStr(vals[4]), + ValidFrom: nullStr(vals[5]), ValidTo: nullStr(vals[6]), + } if e, ok := vals[1].([]any); ok && len(e) > 0 { - rows = append(rows, ann.Row{ID: id, Vec: anyToFloat32(e)}) + meta.Vec = anyToFloat32(e) + rows = append(rows, meta) } else { text := fmt.Sprint(vals[2]) if text == "" || text == "" { @@ -117,7 +125,8 @@ func extractRows(limit int) ([]ann.Row, error) { if err != nil { return nil, fmt.Errorf("embed %s: %w", id, err) } - rows = append(rows, ann.Row{ID: id, Vec: toFloat32(emb, dim)}) + meta.Vec = toFloat32(emb, dim) + rows = append(rows, meta) } if limit > 0 && len(rows) >= limit { break @@ -529,12 +538,13 @@ func runAnnStats(args []string) int { if f.jsonOut { _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ "index": path, "len": idx.Len(), "dirty_wal": idx.Dirty(), - "skipped": idx.Skipped(), + "skipped": idx.Skipped(), "format": ann.FormatVersion, "tombstones": idx.Tombstones(), "tombstone_ratio": idx.TombstoneRatio(), }) } else { - fmt.Printf("index %s: %d vectors, %d tombstones (%.1f%%), WAL %d, skipped %d\n", - path, idx.Len(), idx.Tombstones(), idx.TombstoneRatio()*100, idx.Dirty(), idx.Skipped()) + fmt.Printf("index %s: %d vectors, %d tombstones (%.1f%%), WAL %d, skipped %d, format v%d\n", + path, idx.Len(), idx.Tombstones(), idx.TombstoneRatio()*100, + idx.Dirty(), idx.Skipped(), ann.FormatVersion) } return 0 } diff --git a/internal/brain/bench_gate_test.go b/internal/brain/bench_gate_test.go index 3fe4b6f..cc1f496 100644 --- a/internal/brain/bench_gate_test.go +++ b/internal/brain/bench_gate_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/eSlider/2dph/internal/brain/bench" + "github.com/eSlider/2dph/internal/brain/rank" ) // benchFixtureLeafs returns synthetic leafs (Alice/Bob, no PII): each text @@ -111,7 +112,7 @@ func TestBenchGateRecallOffline(t *testing.T) { type ftsBenchSearcher struct{} func (ftsBenchSearcher) Search(_ context.Context, q string, limit int) ([]bench.Hit, error) { - hits, err := queryFTS(q, limit) + hits, err := queryFTS(q, limit, rank.Filter{}) if err != nil { return nil, err } diff --git a/internal/brain/ingest_race_test.go b/internal/brain/ingest_race_test.go index eabb302..4236de8 100644 --- a/internal/brain/ingest_race_test.go +++ b/internal/brain/ingest_race_test.go @@ -136,7 +136,7 @@ func TestIngestSwapRaceSafe(t *testing.T) { } emb := make([]float64, EmbedDim) emb[0] = 0.5 - if _, err := queryVector(emb, 5); err != nil { + if _, err := queryVector(emb, 5, rank.Filter{}); err != nil { t.Errorf("vector: %v", err) return } diff --git a/internal/brain/ingest_soak_test.go b/internal/brain/ingest_soak_test.go index 82d7f13..17b7a38 100644 --- a/internal/brain/ingest_soak_test.go +++ b/internal/brain/ingest_soak_test.go @@ -9,6 +9,7 @@ import ( "sync" "testing" + "github.com/eSlider/2dph/internal/brain/rank" "github.com/eSlider/2dph/internal/config" ) @@ -160,11 +161,11 @@ func TestIngestSoakWritePath(t *testing.T) { t.Errorf("get: %v", err) return } - if _, err := queryFTS("zebra", 5); err != nil { + if _, err := queryFTS("zebra", 5, rank.Filter{}); err != nil { t.Errorf("fts: %v", err) return } - if _, err := queryVector(vec, 5); err != nil { + if _, err := queryVector(vec, 5, rank.Filter{}); err != nil { t.Errorf("vector: %v", err) return } @@ -188,7 +189,7 @@ func TestIngestSoakWritePath(t *testing.T) { if _, err := h.Stats(ctx); err != nil { t.Fatalf("stats failed after soak: %v", err) } - hits, err := queryFTS("zebra", soakIngestIterations+10) + hits, err := queryFTS("zebra", soakIngestIterations+10, rank.Filter{}) if err != nil { t.Fatalf("fts failed after soak: %v", err) } diff --git a/internal/brain/rank/evalq.go b/internal/brain/rank/evalq.go index 68ff1d1..edc4d9e 100644 --- a/internal/brain/rank/evalq.go +++ b/internal/brain/rank/evalq.go @@ -7,10 +7,35 @@ const EvalRecallThreshold = 0.95 type EvalQuestion struct { Query string Fragment string + Root string // optional --root filter (facts|info) + Repo string // optional --repo filter (source substring) + AsOf string // optional --as-of filter (D24, YYYY-MM-DD) } +func (q EvalQuestion) Filter() Filter { + return Filter{Root: q.Root, Repo: q.Repo, AsOf: q.AsOf} +} + +// EvalQuestions is the unfiltered recall@5 gate. Do not add filters here: the +// unfiltered pipeline must stay measured on its own. var EvalQuestions = []EvalQuestion{ - {"hybrid search fts and vector", "BM25"}, - {"eslider devops engineer", "DevOps"}, - {"ladybugdb graph engine storage", "LadybugDB"}, + {Query: "hybrid search fts and vector", Fragment: "BM25"}, + {Query: "eslider devops engineer", Fragment: "DevOps"}, + {Query: "ladybugdb graph engine storage", Fragment: "LadybugDB"}, +} + +// EvalFilteredQuestions is the filter-aware gate (issue #3). Each case names a +// rare slice of the corpus (root=facts, a repo, an as-of day) and the fragment +// that must survive the top-5 cut; every returned hit must also be eligible, +// so a filter leak fails the gate too. The fragment of an unfiltered search +// sits far below the cut for these broad queries (measured live: "onlyoffice" +// fact at BM25 rank 43, "curaSoft" at 161), so the old +// "fetch limit*3, filter afterwards" behaviour returns zero facts here and +// fails the gate. +var EvalFilteredQuestions = []EvalQuestion{ + {Query: "onlyoffice", Fragment: "OnlyOffice", Root: "facts"}, + {Query: "curaSoft", Fragment: "CuraSoft", Root: "facts"}, + {Query: "dzh", Fragment: "DZH", Root: "facts"}, + {Query: "onlyoffice update file", Fragment: "OnlyOffice", Repo: "office-assistant"}, + {Query: "onlyoffice", Fragment: "OnlyOffice", Root: "facts", AsOf: "2026-01-01"}, } diff --git a/internal/brain/rank/filter.go b/internal/brain/rank/filter.go new file mode 100644 index 0000000..d909953 --- /dev/null +++ b/internal/brain/rank/filter.go @@ -0,0 +1,101 @@ +package rank + +import ( + "strings" + + "github.com/eSlider/2dph/internal/facts" +) + +// Filter is the eligibility predicate of a search: --root, --repo and --as-of. +// Retrieval applies it to candidates *before* the top-k cut (issue #3): with +// facts at 0.09% of the live corpus, filtering after the cut dropped the one +// eligible leaf from limit*3 candidates and answered a false "(not confirmed)". +// Keep the semantics here so the ANN index, the FTS query and the final ranking +// share one definition. +type Filter struct { + Root string // exact leaf root (facts|info); empty = any + Repo string // substring of the leaf source, mirroring FilterRepo; empty = any + AsOf string // YYYY-MM-DD, D24 validity interval; empty = ignore intervals +} + +// Empty reports whether no filter is active (the fast unfiltered path). +func (f Filter) Empty() bool { return f.Root == "" && f.Repo == "" && f.AsOf == "" } + +// Eligible reports whether one candidate row passes every active filter. +// root/source/validFrom/validTo are the row metadata as stored in the graph or +// in the ANN index. Empty intervals are legacy leafs and stay eligible (D24). +func (f Filter) Eligible(root, source, validFrom, validTo string) bool { + if f.Root != "" && root != f.Root { + return false + } + if f.Repo != "" && !strings.Contains(source, f.Repo) { + return false + } + if f.AsOf != "" && !facts.ActiveAt(validFrom, validTo, f.AsOf) { + return false + } + return true +} + +// FilterHits keeps the hits eligible for f, preserving order. An empty filter +// returns the input unchanged. +func FilterHits(hits []Hit, f Filter) []Hit { + if f.Empty() { + return hits + } + out := make([]Hit, 0, len(hits)) + for _, h := range hits { + if f.Eligible(h.Root, h.Source, h.ValidFrom, h.ValidTo) { + out = append(out, h) + } + } + return out +} + +// CandidateFactor is the first over-fetch step over the caller's limit; the +// same ×3 the unfiltered path always used. +const CandidateFactor = 3 + +// CandidateCap bounds the doubling ladder so one pathological filter cannot +// turn a query into an unbounded fetch. +const CandidateCap = 2048 + +// EscalateCandidates fetches candidates with an escalating budget — limit*3, +// then doubling up to CandidateCap — and stops as soon as fetch yields at +// least limit eligible rows or reports that it has no more matches (more == +// false: the retriever returned a short window, so a bigger budget would only +// repeat the same rows). fetch=nil or a non-positive limit is a no-op. +// +// Retrievers that push Filter down (ANN index, filtered FTS query) therefore +// still degrade gracefully: a short result can only mean the corpus truly has +// fewer eligible rows, and the caller may fall back to the exact scan instead +// of reporting an empty answer as fact. +func EscalateCandidates(limit int, fetch func(n int) (hits []Hit, more bool, err error)) ([]Hit, error) { + if limit <= 0 || fetch == nil { + return nil, nil + } + n := limit * CandidateFactor + if n <= 0 { + n = limit + } + var best []Hit + for { + got, more, err := fetch(n) + if err != nil { + return nil, err + } + if len(got) >= len(best) { + best = got + } + if len(best) >= limit { + return best, nil + } + if !more || n >= CandidateCap { + return best, nil + } + n *= 2 + if n > CandidateCap { + n = CandidateCap + } + } +} diff --git a/internal/brain/rank/filter_test.go b/internal/brain/rank/filter_test.go new file mode 100644 index 0000000..1defe2c --- /dev/null +++ b/internal/brain/rank/filter_test.go @@ -0,0 +1,248 @@ +package rank + +import ( + "errors" + "fmt" + "testing" +) + +// hit builds one candidate with the metadata the eligibility predicate reads. +func hit(id, root, source, from, to string) Hit { + return Hit{ID: id, Root: root, Source: source, ValidFrom: from, ValidTo: to} +} + +// TestFilterEligibleMatrix pins the eligibility semantics shared by the ANN +// index, the FTS query and the final ranking (issue #3): root is exact, repo +// is a source substring, as-of is the D24 interval, and an empty filter keeps +// everything. +func TestFilterEligibleMatrix(t *testing.T) { + row := func(root, source, from, to string) bool { + return Filter{}.Eligible(root, source, from, to) + } + if !row("facts", "docs/a.md x docs/b.md", "", "") { + t.Fatal("empty filter must keep every row") + } + + facts := Filter{Root: "facts"} + if !facts.Eligible("facts", "docs/a.md", "", "") { + t.Fatal("facts row must be eligible for root=facts") + } + if facts.Eligible("info", "docs/a.md", "", "") { + t.Fatal("info row must not be eligible for root=facts") + } + + repo := Filter{Repo: "office-assistant"} + if !repo.Eligible("info", "office-assistant/docs/a.md x office-assistant/AGENTS.md", "", "") { + t.Fatal("repo substring hit must be eligible") + } + if repo.Eligible("info", "docs/a.md x docs/b.md", "", "") { + t.Fatal("repo miss must not be eligible") + } + + asOf := Filter{AsOf: "2025-06-01"} + if !asOf.Eligible("info", "s", "2025-01-01", "2025-12-31") { + t.Fatal("interval covering as_of must be eligible") + } + if asOf.Eligible("info", "s", "2025-07-01", "") { + t.Fatal("interval starting after as_of must not be eligible") + } + if asOf.Eligible("info", "s", "", "2025-05-31") { + t.Fatal("interval ending before as_of must not be eligible") + } + if !asOf.Eligible("info", "s", "", "") { + t.Fatal("empty interval (legacy leaf) must stay eligible (D24)") + } + + all := Filter{Root: "facts", Repo: "office-assistant", AsOf: "2025-06-01"} + if !all.Eligible("facts", "office-assistant/x.md", "2025-01-01", "") { + t.Fatal("row passing every filter must be eligible") + } + if all.Eligible("info", "office-assistant/x.md", "2025-01-01", "") { + t.Fatal("one failing dimension must reject the row") + } +} + +func TestFilterEmpty(t *testing.T) { + if !(Filter{}).Empty() { + t.Fatal("zero Filter must be Empty") + } + for _, f := range []Filter{{Root: "facts"}, {Repo: "2dph"}, {AsOf: "2025-01-01"}} { + if f.Empty() { + t.Fatalf("%+v must not be Empty", f) + } + } +} + +// TestFilterHitsKeepsOrderAndLegacyRows — the post-fusion filter must behave +// exactly like the old per-dimension filters (same rows, same order). +func TestFilterHitsKeepsOrderAndLegacyRows(t *testing.T) { + hits := []Hit{ + hit("i1", "info", "docs/a.md", "", ""), + hit("f1", "facts", "2dph/AGENTS.md", "", ""), + hit("i2", "info", "docs/b.md", "2024-01-01", "2024-12-31"), + hit("f2", "facts", "2dph/AGENTS.md", "2025-01-01", ""), + } + got := FilterHits(hits, Filter{Root: "facts"}) + if len(got) != 2 || got[0].ID != "f1" || got[1].ID != "f2" { + t.Fatalf("root filter = %v, want [f1 f2]", ids(got)) + } + got = FilterHits(hits, Filter{AsOf: "2025-06-01"}) + if fmt.Sprint(ids(got)) != "[i1 f1 f2]" { + t.Fatalf("as-of filter = %v, want [i1 f1 f2] (legacy + open interval)", ids(got)) + } + if got := FilterHits(hits, Filter{}); len(got) != len(hits) { + t.Fatalf("empty filter dropped rows: %d of %d", len(got), len(hits)) + } +} + +// TestEscalateCandidatesSingleRung — a retriever that pre-filters answers at +// the first budget (limit*3) and is never asked twice. +func TestEscalateCandidatesSingleRung(t *testing.T) { + var calls []int + got, err := EscalateCandidates(5, func(n int) ([]Hit, bool, error) { + calls = append(calls, n) + out := make([]Hit, n) + for i := range out { + out[i] = hit(fmt.Sprintf("h%d", i), "facts", "s", "", "") + } + return out, false, nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 15 { + t.Fatalf("got %d hits, want the full limit*3 window (15)", len(got)) + } + if fmt.Sprint(calls) != "[15]" { + t.Fatalf("calls = %v, want one rung [15]", calls) + } +} + +// TestEscalateCandidatesDoublesWhenEligibleAreSparse — the as-of case: each +// full window yields only a fifth as eligible rows, so the ladder must double +// until limit eligible are collected. +func TestEscalateCandidatesDoublesWhenEligibleAreSparse(t *testing.T) { + var calls []int + got, err := EscalateCandidates(5, func(n int) ([]Hit, bool, error) { + calls = append(calls, n) + eligible := n / 5 + out := make([]Hit, eligible) + for i := range out { + out[i] = hit(fmt.Sprintf("h%d", i), "facts", "s", "", "") + } + return out, true, nil // a full window was available, more rows exist + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 6 { + t.Fatalf("got %d eligible, want 6 (n=30/5)", len(got)) + } + if fmt.Sprint(calls) != "[15 30]" { + t.Fatalf("calls = %v, want [15 30]", calls) + } +} + +// TestEscalateCandidatesStopsWhenRetrieverIsExhausted — more=false means the +// retriever returned a short window: no bigger budget can find new rows. +func TestEscalateCandidatesStopsWhenRetrieverIsExhausted(t *testing.T) { + var calls []int + got, err := EscalateCandidates(20, func(n int) ([]Hit, bool, error) { + calls = append(calls, n) + return []Hit{hit("f1", "facts", "s", "", "")}, false, nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ID != "f1" { + t.Fatalf("got %v, want the single eligible hit", ids(got)) + } + if len(calls) != 1 { + t.Fatalf("calls = %v, want one rung (exhausted)", calls) + } +} + +// TestEscalateCandidatesCap — a retriever that keeps claiming more rows is +// asked up to CandidateCap and then left alone (no unbounded fetch). +func TestEscalateCandidatesCap(t *testing.T) { + var calls []int + got, err := EscalateCandidates(1, func(n int) ([]Hit, bool, error) { + calls = append(calls, n) + return nil, true, nil // full window, nothing eligible + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("got %v, want none", ids(got)) + } + if last := calls[len(calls)-1]; last != CandidateCap { + t.Fatalf("last budget = %d, want the cap %d (calls %v)", last, CandidateCap, calls) + } + for i := 1; i < len(calls); i++ { + if calls[i] != calls[i-1]*2 && calls[i] != CandidateCap { + t.Fatalf("budget must double: %v", calls) + } + } +} + +func TestEscalateCandidatesErrorAndNoOp(t *testing.T) { + want := errors.New("fetch failed") + if _, err := EscalateCandidates(5, func(int) ([]Hit, bool, error) { + return nil, false, want + }); !errors.Is(err, want) { + t.Fatalf("err = %v, want %v", err, want) + } + called := false + got, err := EscalateCandidates(0, func(int) ([]Hit, bool, error) { + called = true + return nil, false, nil + }) + if err != nil || got != nil || called { + t.Fatalf("limit<=0 must be a no-op, got %v %v called=%v", got, err, called) + } +} + +// TestEvalFilteredQuestionsAreWellFormed pins the filtered gate table (issue +// #3): every case carries a filter, a fragment and a known root, and the +// unfiltered gate stays untouched at three cases. +func TestEvalFilteredQuestionsAreWellFormed(t *testing.T) { + if len(EvalQuestions) != 3 { + t.Fatalf("unfiltered gate changed: %d questions, want 3", len(EvalQuestions)) + } + for _, q := range EvalQuestions { + if q.Root != "" || q.Repo != "" || q.AsOf != "" { + t.Fatalf("unfiltered gate question %q must stay unfiltered", q.Query) + } + } + if len(EvalFilteredQuestions) == 0 { + t.Fatal("no filtered eval questions") + } + rooted, repoed, dated := 0, 0, 0 + for _, q := range EvalFilteredQuestions { + if q.Query == "" || q.Fragment == "" { + t.Fatalf("filtered case needs query and fragment: %+v", q) + } + if q.Filter().Empty() { + t.Fatalf("filtered case %q has no filter", q.Query) + } + if q.Root != "" && q.Root != "facts" && q.Root != "info" { + t.Fatalf("case %q: root %q is not facts|info", q.Query, q.Root) + } + if q.Root != "" { + rooted++ + } + if q.Repo != "" { + repoed++ + } + if q.AsOf != "" { + dated++ + if len(q.AsOf) != 10 || q.AsOf[4] != '-' || q.AsOf[7] != '-' { + t.Fatalf("case %q: as_of %q is not YYYY-MM-DD", q.Query, q.AsOf) + } + } + } + if rooted == 0 || repoed == 0 || dated == 0 { + t.Fatalf("gate must cover root=%d repo=%d as_of=%d", rooted, repoed, dated) + } +} diff --git a/internal/brain/rank/query.go b/internal/brain/rank/query.go index 57204ed..678ed5b 100644 --- a/internal/brain/rank/query.go +++ b/internal/brain/rank/query.go @@ -6,6 +6,19 @@ const FTSStmt = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) " + "RETURN node.id, node.text, node.root, node.source, score, node.confidence, " + "node.valid_from, node.valid_to ORDER BY score DESC LIMIT $n" +// FTSStmtFiltered is FTSStmt with the --root/--repo pre-filter pushed into the +// Cypher query (issue #3), so the top-n candidates are already eligible and a +// rare root (facts = 0.09% of the corpus) is not crowded out of the cut by +// info leafs. The predicate is static text with bound parameters ($root/$repo); +// an empty parameter disables that clause. --as-of stays a metadata filter in +// brain (it needs D24 interval semantics) and is served by over-fetch +// escalation (rank.EscalateCandidates). +const FTSStmtFiltered = "CALL QUERY_FTS_INDEX('Leaf', 'id', $q) WITH node, score " + + "WHERE ($root = '' OR node.root = $root) " + + "AND ($repo = '' OR contains(node.source, $repo)) " + + "RETURN node.id, node.text, node.root, node.source, score, node.confidence, " + + "node.valid_from, node.valid_to ORDER BY score DESC LIMIT $n" + // VecStmt was removed: vector ranking now runs in-process as a brute-force // cosine scan over l.embedding (see brain/queryVector), because liblbug's // HNSW QUERY_VECTOR_INDEX (Leaf_vec) SIGSEGVs on grown graphs (>= ~1300 diff --git a/internal/brain/rank/rank.go b/internal/brain/rank/rank.go index 3c34314..7a9287d 100644 --- a/internal/brain/rank/rank.go +++ b/internal/brain/rank/rank.go @@ -4,9 +4,6 @@ package rank import ( "sort" - "strings" - - "github.com/eSlider/2dph/internal/facts" ) type HopNode struct { @@ -50,16 +47,7 @@ func RankAndFilterAsOf(fts, vec []Hit, root, repo, asOf string, limit int) []Hit // sortDate is true the fused list is ordered by ValidFrom before the limit cut // so `--sort date` orders the full result set, not just the top N. func RankAndFilterSort(fts, vec []Hit, root, repo, asOf string, limit int, sortDate, desc bool) []Hit { - out := Hybrid(fts, vec, 0) - if root != "" { - out = FilterRoot(out, root) - } - if repo != "" { - out = FilterRepo(out, repo) - } - if asOf != "" { - out = FilterAsOf(out, asOf) - } + out := FilterHits(Hybrid(fts, vec, 0), Filter{Root: root, Repo: repo, AsOf: asOf}) if sortDate { out = SortByDate(out, desc) } @@ -96,16 +84,7 @@ func SortByDate(hits []Hit, desc bool) []Hit { // FilterAsOf keeps hits whose [valid_from, valid_to] covers asOf (D24). // Empty intervals stay (legacy leafs). Empty asOf keeps all. func FilterAsOf(hits []Hit, asOf string) []Hit { - if asOf == "" { - return hits - } - var out []Hit - for _, h := range hits { - if facts.ActiveAt(h.ValidFrom, h.ValidTo, asOf) { - out = append(out, h) - } - } - return out + return FilterHits(hits, Filter{AsOf: asOf}) } // Hybrid merges FTS and vector hits by reciprocal rank fusion. @@ -150,21 +129,9 @@ func Hybrid(fts, vec []Hit, limit int) []Hit { } func FilterRoot(hits []Hit, root string) []Hit { - var out []Hit - for _, h := range hits { - if h.Root == root { - out = append(out, h) - } - } - return out + return FilterHits(hits, Filter{Root: root}) } func FilterRepo(hits []Hit, repo string) []Hit { - var out []Hit - for _, h := range hits { - if strings.Contains(h.Source, repo) { - out = append(out, h) - } - } - return out + return FilterHits(hits, Filter{Repo: repo}) } diff --git a/internal/brain/read.go b/internal/brain/read.go index 96a853b..6318c6d 100644 --- a/internal/brain/read.go +++ b/internal/brain/read.go @@ -125,7 +125,7 @@ func MainEval(args []string) int { details := make([]any, 0, len(rank.EvalQuestions)) jsDetails := make([]map[string]any, 0, len(rank.EvalQuestions)) for _, q := range rank.EvalQuestions { - hits, err := queryFTS(q.Query, 5) + hits, err := queryFTS(q.Query, 5, rank.Filter{}) ok := false if err == nil { frag := strings.ToLower(q.Fragment) @@ -153,16 +153,28 @@ func MainEval(args []string) int { if n > 0 { recall = float64(recalled) / float64(n) } - passed := recall >= rank.EvalRecallThreshold + fRecalled, fTotal, fDetails, fJSDetails := evalFilteredGate() + fRecall := 0.0 + if fTotal > 0 { + fRecall = float64(fRecalled) / float64(fTotal) + } + // A corpus without the asked slice (e.g. no promoted facts yet) skips the + // filtered cases: that is not a retrieval regression. When cases do run, + // they are held to the same threshold as the unfiltered gate. + passed := recall >= rank.EvalRecallThreshold && + (fTotal == 0 || fRecall >= rank.EvalRecallThreshold) if jsonOut { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") enc.SetEscapeHTML(false) _ = enc.Encode(map[string]any{ - "recall@5": round3(recall), - "passed": passed, - "gate": n, - "details": jsDetails, + "recall@5": round3(recall), + "passed": passed, + "gate": n, + "details": jsDetails, + "filtered_recall@5": round3(fRecall), + "filtered_gate": fTotal, + "filtered_details": fJSDetails, }) } else { out := Dict{ @@ -170,6 +182,9 @@ func MainEval(args []string) int { {"passed", passed}, {"gate", n}, {"details", details}, + {"filtered_recall@5", round3(fRecall)}, + {"filtered_gate", fTotal}, + {"filtered_details", fDetails}, } fmt.Print(toYAML(out, 0)) } @@ -179,6 +194,119 @@ func MainEval(args []string) int { return 0 } +// evalFilteredGate runs the filter-aware cases of rank.EvalFilteredQuestions +// (issue #3) through the real retrieval path (root/repo/as-of applied before +// the top-k cut). A case passes when the fragment survives the top-5 cut and +// every returned hit is eligible — a filter leak or an empty answer from a +// rare root both fail. Cases whose corpus slice is absent are skipped: an +// empty corpus is not a retrieval bug (the same honesty rule as +// "throttled ≠ absence"). Returns recalled, total and the per-case reports. +func evalFilteredGate() (int, int, []any, []map[string]any) { + const k = 5 + recalled, total := 0, 0 + details := make([]any, 0, len(rank.EvalFilteredQuestions)) + jsDetails := make([]map[string]any, 0, len(rank.EvalFilteredQuestions)) + for _, q := range rank.EvalFilteredQuestions { + rep := map[string]any{ + "q": q.Query, "fragment": q.Fragment, "root": q.Root, + "repo": q.Repo, "as_of": q.AsOf, + } + if !corpusHasSlice(q) { + rep["skipped"] = "no matching leafs in corpus" + details = append(details, dictFromMap(rep)) + jsDetails = append(jsDetails, rep) + continue + } + emb, embErr := embedQueryFn(q.Query) + hits, err := searchHitsEmb(q.Query, emb, q.Filter(), k, false, false) + if embErr != nil { + rep["note"] = "fts-only (no embedding model)" + } + inTop, eligible := false, true + if err != nil { + rep["error"] = err.Error() + } else { + f := q.Filter() + for i, h := range hits { + if !f.Eligible(h.Root, h.Source, h.ValidFrom, h.ValidTo) { + eligible = false + } + if i < k && strings.Contains(strings.ToLower(h.Text), strings.ToLower(q.Fragment)) { + inTop = true + } + } + } + ok := err == nil && inTop && eligible + rep["in_top5"] = inTop + rep["eligible"] = eligible + rep["hits"] = len(hits) + total++ + if ok { + recalled++ + } + details = append(details, dictFromMap(rep)) + jsDetails = append(jsDetails, rep) + } + return recalled, total, details, jsDetails +} + +// corpusHasSlice reports whether the corpus holds at least one leaf the case +// could retrieve: the asked root, or a source containing the asked repo. +func corpusHasSlice(q rank.EvalQuestion) bool { + brainMu.RLock() + defer brainMu.RUnlock() + if conn == nil { + return false + } + stmtText := "MATCH (l:Leaf) RETURN count(*)" + params := map[string]any{} + switch { + case q.Root != "": + stmtText = "MATCH (l:Leaf {root:$root}) RETURN count(*)" + params["root"] = q.Root + case q.Repo != "": + stmtText = "MATCH (l:Leaf) WHERE contains(l.source, $repo) RETURN count(*)" + params["repo"] = q.Repo + default: + return true + } + stmt, err := conn.Prepare(stmtText) + if err != nil { + return false + } + defer stmt.Close() + res, err := conn.Execute(stmt, params) + if err != nil { + return false + } + defer res.Close() + if !res.HasNext() { + return false + } + row, err := res.Next() + if err != nil { + return false + } + vals, err := row.GetAsSlice() + if err != nil || len(vals) == 0 { + return false + } + return asInt(vals[0]) > 0 +} + +func dictFromMap(m map[string]any) Dict { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + out := make(Dict, 0, len(keys)) + for _, k := range keys { + out = append(out, KV{k, m[k]}) + } + return out +} + func lookupLeaf(id string) (map[string]string, string, error) { brainMu.RLock() defer brainMu.RUnlock() diff --git a/internal/brain/search.go b/internal/brain/search.go index a82a375..b461cd2 100644 --- a/internal/brain/search.go +++ b/internal/brain/search.go @@ -112,20 +112,55 @@ func runSearch(args []string) int { return 0 } +// searchHits is the deduction retrieval: FTS + vector candidates, both +// restricted to the rows eligible for root/repo/asOf *before* the top-k cut +// (issue #3). Filtering after the cut dropped a rare root (facts = 0.09% of +// the live corpus) out of the limit*3 window and answered a false +// "(not confirmed)" while the fact existed. func searchHits(query, root, repo string, limit int, asOf string, sortDate, sortDesc bool) ([]Hit, error) { emb, err := embedQueryFn(query) if err != nil { return nil, fmt.Errorf("embed: %w", err) } - fts, err := queryFTS(query, limit*3) + return searchHitsEmb(query, emb, rank.Filter{Root: root, Repo: repo, AsOf: asOf}, limit, sortDate, sortDesc) +} + +// searchHitsEmb is searchHits with the query vector supplied. A nil emb skips +// the vector candidates: the FTS side is then the whole deduction path (used by +// the filtered eval gate where no model is available). +func searchHitsEmb(query string, emb []float64, f rank.Filter, limit int, sortDate, sortDesc bool) ([]Hit, error) { + fts, err := escalatedFTS(query, limit, f) if err != nil { return nil, fmt.Errorf("fts: %w", err) } var vec []Hit - if vec, err = queryVector(emb, limit*3); err != nil { - fmt.Fprintf(os.Stderr, "vec: %v\n", err) + if len(emb) > 0 { + if vec, err = queryVector(emb, limit, f); err != nil { + fmt.Fprintf(os.Stderr, "vec: %v\n", err) + } } - return rank.RankAndFilterSort(fts, vec, root, repo, asOf, limit, sortDate, sortDesc), nil + return rank.RankAndFilterSort(fts, vec, f.Root, f.Repo, f.AsOf, limit, sortDate, sortDesc), nil +} + +// escalatedFTS fetches FTS candidates with the root/repo pre-filter pushed into +// the query and --as-of applied to the returned metadata, on the shared +// candidate ladder (limit*3, doubling up to rank.CandidateCap). Expired leafs +// therefore cannot occupy the top-k slots of an eligible one. +func escalatedFTS(text string, limit int, f rank.Filter) ([]Hit, error) { + return rank.EscalateCandidates(limit, func(n int) ([]Hit, bool, error) { + raw, err := queryFTS(text, n, f) + if err != nil { + return nil, false, err + } + got := rank.FilterHits(raw, f) + if f.AsOf != "" && len(got) < limit && n >= rank.CandidateCap { + // Honest, loud degradation: the as-of filter is metadata-only, so a + // cap hit cannot prove absence — say so instead of an empty answer. + fmt.Fprintf(os.Stderr, "fts: as-of candidate cap %d reached (%d eligible of %d)\n", + rank.CandidateCap, len(got), n) + } + return got, len(raw) == n, nil + }) } func attachHops(hits []Hit, n int) error { @@ -177,15 +212,25 @@ func b2i(err error) int { return 0 } -func queryFTS(text string, limit int) ([]Hit, error) { +// queryFTS returns up to limit FTS hits for text. A non-empty root/repo filter +// is pushed into the FTS query (rank.FTSStmtFiltered) so the n candidates are +// already eligible; as-of is applied by the caller (it needs D24 interval +// semantics). Unfiltered queries keep the plain statement byte for byte. +func queryFTS(text string, limit int, f rank.Filter) ([]Hit, error) { brainMu.RLock() defer brainMu.RUnlock() - stmt, err := conn.Prepare(rank.FTSStmt) + stmtText := rank.FTSStmt + params := map[string]any{"q": text, "n": limit} + if f.Root != "" || f.Repo != "" { + stmtText = rank.FTSStmtFiltered + params["root"], params["repo"] = f.Root, f.Repo + } + stmt, err := conn.Prepare(stmtText) if err != nil { return nil, err } defer stmt.Close() - res, err := conn.Execute(stmt, map[string]any{"q": text, "n": limit}) + res, err := conn.Execute(stmt, params) if err != nil { return nil, err } @@ -193,18 +238,40 @@ func queryFTS(text string, limit int) ([]Hit, error) { return rowsToHits(res) } -func queryVector(emb []float64, limit int) ([]Hit, error) { - // ANN path (issue #204): top-k from the incremental IVF index, then - // metadata via point lookups. Falls back to the brute-force scan when - // the index is absent/corrupt/disabled — search never fails because of - // ANN. - if annEnabled() { - if hits, err := annQueryVector(emb, limit); err != nil { - fmt.Fprintf(os.Stderr, "ann: %v\n", err) - } else if len(hits) > 0 { - return hits, nil +// queryVector returns eligible vector candidates for the ranker. The filter is +// applied by the retriever *before* its top-k cut, so ineligible rows never +// take a candidate slot (issue #3). The window starts at limit*3 (the breadth +// the unfiltered fusion always used) and escalates when a filtered window is +// short. +// +// The ANN index is the fast path (issue #204); when it is disabled, missing, +// empty or holds no eligible row — e.g. stale after a bulk import — the exact +// scan answers instead. A stale index must never fabricate an empty result for +// a leaf that exists in the graph. +func queryVector(emb []float64, limit int, f rank.Filter) ([]Hit, error) { + return rank.EscalateCandidates(limit, func(n int) ([]Hit, bool, error) { + if annEnabled() { + got, err := annQueryVector(emb, n, f) + if err != nil { + fmt.Fprintf(os.Stderr, "ann: %v\n", err) + } else if len(got) > 0 { + return got, len(got) == n, nil + } } - } + got, err := scanVector(emb, n, f) + if err != nil { + return nil, false, err + } + return got, len(got) == n, nil + }) +} + +// scanVector is the exact brute-force cosine ranking. Eligible rows are +// selected before the top-k cut, so a rare root is found here even when the +// ANN candidate window missed it (issue #3). Rows without an embedding column +// (a corpus rebuilt with ANN on, #248 B3) cannot be scanned — the ANN index is +// then the only vector source. +func scanVector(emb []float64, limit int, f rank.Filter) ([]Hit, error) { brainMu.RLock() defer brainMu.RUnlock() stmt, err := conn.Prepare(vecScanStmt) @@ -232,6 +299,11 @@ func queryVector(emb []float64, limit int) ([]Hit, error) { if err != nil || len(vals) < 8 { continue } + root, source := fmt.Sprint(vals[2]), fmt.Sprint(vals[3]) + vf, vt := nullStr(vals[5]), nullStr(vals[6]) + if !f.Eligible(root, source, vf, vt) { + continue + } e, ok := vals[7].([]any) if !ok || len(e) == 0 { continue @@ -239,9 +311,9 @@ func queryVector(emb []float64, limit int) ([]Hit, error) { sim := cosineToQuery(emb, e) all = append(all, scored{hit: Hit{ ID: fmt.Sprint(vals[0]), Text: fmt.Sprint(vals[1]), - Root: fmt.Sprint(vals[2]), Source: fmt.Sprint(vals[3]), - Confidence: fmt.Sprint(vals[4]), ValidFrom: nullStr(vals[5]), - ValidTo: nullStr(vals[6]), Score: sim, + Root: root, Source: source, + Confidence: fmt.Sprint(vals[4]), ValidFrom: vf, + ValidTo: vt, Score: sim, }, sim: sim}) } // Deterministic ranking: descending similarity, ties broken by id so the