From 1659d6253ef202f5d47922425c38c1ad10b3f2ef Mon Sep 17 00:00:00 2001 From: Henrique Franze <53534000+henrique-franze@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:27:10 -0300 Subject: [PATCH] MemoryRetriever: opt-in local embedding leg for per-turn retrieval (#2207) Off by default (LIFEOS_MEMORY_EMBEDDING=1), byte-identical when off. Cosine over a local Ollama embedding (loopback only), empty below a cutoff, previous 3 user messages embedded with the prompt. Query embed in the existing retrieval worker under the prompt budget; note vectors in the async MemoryIndexRefresh Stop hook, which also warms a cold model. Falls back to BM25 on any failure. INDEX_VERSION unchanged. Co-Authored-By: Claude Opus 5.5 --- .../DOCUMENTATION/Memory/CortexContract.md | 2 + .../DOCUMENTATION/Memory/MemorySystem.md | 2 +- .../LIFEOS/TOOLS/MemoryEmbedding.test.ts | 287 ++++++++++++++++++ .../install/LIFEOS/TOOLS/MemoryEmbedding.ts | 261 ++++++++++++++++ .../install/LIFEOS/TOOLS/MemoryRetriever.ts | 89 +++++- .../install/hooks/MemoryIndexRefresh.hook.ts | 12 +- LifeOS/install/hooks/MemoryTurnStart.hook.ts | 13 +- .../hooks/lib/recent-user-messages.test.ts | 41 +++ .../install/hooks/lib/recent-user-messages.ts | 62 ++++ LifeOS/install/hooks/lib/retrieval-worker.ts | 19 +- 10 files changed, 776 insertions(+), 12 deletions(-) create mode 100644 LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.test.ts create mode 100644 LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.ts create mode 100644 LifeOS/install/hooks/lib/recent-user-messages.test.ts create mode 100644 LifeOS/install/hooks/lib/recent-user-messages.ts diff --git a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/CortexContract.md b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/CortexContract.md index 4fe87e5c32..031a7382e3 100644 --- a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/CortexContract.md +++ b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/CortexContract.md @@ -186,6 +186,8 @@ A benchmark run produces a versioned report under `MEMORY/BENCHMARKS/` that both `vector_config` is currently `null`: there is no vector benchmark candidate and no vector index. A vector or hybrid index may be adopted only after a labeled report demonstrates retrieval-quality improvement over progressive BM25, and only if it is canonical-rebuildable and stays within separately documented disk and process bounds. Lower token use alone is not evidence for a vector index. +**Opt-in per-turn embedding leg.** Per-turn retrieval (`MemoryTurnStart` → `MemoryRetriever.getRelevantContextAsync`) ranks by embedding only when the operator sets `LIFEOS_MEMORY_EMBEDDING=1` (`MemoryEmbedding.ts`); off, which is the default, its output is identical to BM25. It is not a Cortex index: the Cortex commands, `CORTEX_INDEX_POLICY.json` and `vector_config` are unchanged. It meets this gate as follows. *Labeled evidence:* a held-out report against the shipped BM25 at the hook's exact call (public issue #2207). *Canonical-rebuildable:* vectors are a disposable cache derived from the canonical Markdown under `MEMORY/STATE/cache/memory-embeddings/`, keyed by path, mtime and model; deleting it only re-embeds. *Bounds:* about 4 KB per note on disk (1024-dimension float32, e.g. 2.0 MB for 486 notes); no LifeOS process, because embeddings come from the operator's own Ollama server and only a loopback URL is accepted; one query embed per prompt inside the 300 ms prompt budget, falling back to BM25 past it; note embeds only in the async Stop hook (`MemoryIndexRefresh`), at most 40 s per turn. + ## Truthful operational health `Cortex.ts status` reports contract availability and corpus shape. Operational health comes from: diff --git a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md index 5a3362d19b..9f7daf5c3f 100644 --- a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md +++ b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md @@ -69,7 +69,7 @@ Every command emits one `lifeos-cortex/v1` JSON envelope. The published schema i `LIFEOS/TOOLS/CortexAdapter.ts` ships the in-process adapter factory for `claude`, `hermes`, `codex`, and `subagent`. Adapter calls are read-only by default: every remember/propose call must individually opt in with `{allowWrite:true}`, corresponding to the CLI's explicit recognized `--adapter` plus `--allow-write`. Command/type binding is enforced before dispatch: `remember` accepts `memory|idea|knowledge`; `propose` accepts only `proposal`. Authorized items then delegate to `MemorySystem.add()` so the existing mutation tiers, proposal approval, target pinning, audit trail, snapshots, and shrink guard remain authoritative. `export` is read-only stdout serialization, not a filesystem writer. -Canonical Markdown/JSONL remains the source of truth. Cortex allows the intentional top-level `LIFEOS/MEMORY` symlink, resolves and pins its canonical target, and rejects nested symlinks or escapes. The shipped `LIFEOS/CORTEX_INDEX_POLICY.json` affirmatively declares `no-index-v1`: BM25 reads canonical files, `status` reports no indexes, and `rebuild --from-canonical` produces equal deterministic SHA-256 record-view digests without creating an index. A missing policy plus missing manifest warns rather than silently turning green. There is no CMEM, Chroma, SQLite FTS, vector index, cloud sync, external telemetry, MCP server, or Cortex daemon. +Canonical Markdown/JSONL remains the source of truth. Cortex allows the intentional top-level `LIFEOS/MEMORY` symlink, resolves and pins its canonical target, and rejects nested symlinks or escapes. The shipped `LIFEOS/CORTEX_INDEX_POLICY.json` affirmatively declares `no-index-v1`: BM25 reads canonical files, `status` reports no indexes, and `rebuild --from-canonical` produces equal deterministic SHA-256 record-view digests without creating an index. A missing policy plus missing manifest warns rather than silently turning green. There is no CMEM, Chroma, SQLite FTS, vector index, cloud sync, external telemetry, MCP server, or Cortex daemon. Per-turn retrieval (`MemoryTurnStart` → `MemoryRetriever`) has one opt-in exception, off by default: an embedding leg against a local Ollama server (`LIFEOS_MEMORY_EMBEDDING=1`, `MemoryEmbedding.ts`), whose evidence and bounds are in [`CortexContract.md`](./CortexContract.md) § Retrieval benchmark and vector evidence gate. `CaptureEnvelope.ts` provides the concrete `ingestCaptureEnvelope(input, consumer)` helper, which sanitizes through `createCaptureEnvelope` before invoking the consumer. It is source-neutral. Both legacy transcript capture writers are envelope-migrated (2026-08-03): `MemoryReviewer.ts` ingests transcript text via `transcriptCaptureEnvelope` and `SessionHarvester.ts` via its `ingestTranscriptText` boundary, each binding source/channel/captured_at/session provenance and making private-span stripping structural; harvester outputs (harvest-queue JSON, LEARNING files) carry that provenance. Probes (source tree only; tests are not in the public release payload): `test/tools/cortex/writers.test.ts`. The versioned benchmark imports the production validity/ranking/card functions, shares one ranking between full-fetch and card-first disclosure paths, and can write only an explicit versioned report under `MEMORY/BENCHMARKS/`. The verified live artifact is `LIFEOS/MEMORY/BENCHMARKS/cortex-benchmark-v1-20260802.json`; see the contract for its point-in-time metrics and non-claims. diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.test.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.test.ts new file mode 100644 index 0000000000..bbb50782fa --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.test.ts @@ -0,0 +1,287 @@ +/** + * MemoryEmbedding.test.ts — the opt-in embedding leg (public issue #2207). No network: unit tests + * inject the embedder, and the end-to-end tests run a fake Ollama on 127.0.0.1 over an isolated + * corpus (CORTEX_MEMORY_ROOT), so the live install is never read or written. + */ +import { describe, expect, test, beforeAll, afterAll } from "bun:test"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + embeddingConfig, + isLoopbackUrl, + loadCache, + saveCache, + rankByEmbedding, + refreshEmbeddings, + normalize, + ollamaEmbedder, + type Embedder, + type EmbeddingConfig, +} from "./MemoryEmbedding"; + +// A deterministic toy embedding: one axis per topic word, so cosine is predictable. +const AXES = ["deadlock", "fila", "heartbeat", "tennis", "dose"]; +const toyVec = (text: string) => { + const t = text.toLowerCase(); + const v = AXES.map((w) => (t.includes(w) ? 1 : 0)); + v.push(0.05); // never all-zero + return v; +}; +const toyEmbedder: Embedder = async (inputs) => inputs.map(toyVec); + +function tmpCfg(over: Partial = {}): EmbeddingConfig { + return { ...embeddingConfig({ LIFEOS_MEMORY_EMBEDDING: "1" }), cacheRoot: fs.mkdtempSync(path.join(os.tmpdir(), "emb-")), cutoff: 0.5, ...over }; +} +function note(dir: string, name: string, body: string, noteClass = "knowledge") { + const filePath = path.join(dir, name); + fs.writeFileSync(filePath, body); + return { filePath, title: name.replace(/\.md$/, ""), noteClass, body }; +} + +describe("config", () => { + test("off by default; on only with LIFEOS_MEMORY_EMBEDDING=1", () => { + expect(embeddingConfig({}).enabled).toBe(false); + expect(embeddingConfig({ LIFEOS_MEMORY_EMBEDDING: "true" }).enabled).toBe(false); + expect(embeddingConfig({ LIFEOS_MEMORY_EMBEDDING: "1" }).enabled).toBe(true); + }); + test("a non-loopback server leaves the leg off, so note text never leaves the machine", () => { + for (const url of ["http://192.0.2.10:11434", "http://ollama.example.com", "http://127.0.0.1.example.com:11434", "file:///etc/passwd"]) { + expect(isLoopbackUrl(url)).toBe(false); + expect(embeddingConfig({ LIFEOS_MEMORY_EMBEDDING: "1", LIFEOS_MEMORY_EMBEDDING_URL: url }).enabled).toBe(false); + } + for (const url of ["http://127.0.0.1:11434", "http://localhost:11434", "http://[::1]:11434"]) expect(isLoopbackUrl(url)).toBe(true); + }); + test("keep_alive is not sent unless configured", () => { + expect(embeddingConfig({ LIFEOS_MEMORY_EMBEDDING: "1" }).keepAlive).toBeUndefined(); + }); +}); + +describe("ollamaEmbedder", () => { + test("never follows a redirect, so a loopback server cannot forward note text elsewhere", async () => { + let leaked = 0; + const elsewhere = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { leaked++; return Response.json({ embeddings: [[1]] }); } }); + const redirector = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch() { return new Response(null, { status: 307, headers: { location: `http://127.0.0.1:${elsewhere.port}/api/embed` } }); } }); + try { + const cfg = tmpCfg({ url: `http://127.0.0.1:${redirector.port}` }); + await expect(ollamaEmbedder(cfg)(["secret note text"], 2000)).rejects.toThrow(); + expect(leaked).toBe(0); + } finally { elsewhere.stop(true); redirector.stop(true); } + }); +}); + +describe("cache", () => { + test("round-trips, keeps only the given paths, and reads another version as empty", () => { + const cfg = tmpCfg(); + const c = { dim: 3, entries: new Map([["a", { m: 1, v: normalize([1, 0, 0]) }], ["b", { m: 2, v: normalize([0, 1, 0]) }]]) }; + saveCache(cfg, c, new Set(["a"])); + const back = loadCache(cfg); + expect([...back.entries.keys()]).toEqual(["a"]); + expect(Array.from(back.entries.get("a")!.v)).toEqual([1, 0, 0]); + const idx = path.join(cfg.cacheRoot, "bge-m3", "index.json"); + fs.writeFileSync(idx, JSON.stringify({ ...JSON.parse(fs.readFileSync(idx, "utf8")), version: 999 })); + expect(loadCache(cfg).entries.size).toBe(0); + }); + test("a truncated vector file reads as empty, never as wrong vectors", () => { + const cfg = tmpCfg(); + saveCache(cfg, { dim: 3, entries: new Map([["a", { m: 1, v: normalize([1, 0, 0]) }]]) }, new Set(["a"])); + const dir = path.join(cfg.cacheRoot, "bge-m3"); + fs.writeFileSync(path.join(dir, JSON.parse(fs.readFileSync(path.join(dir, "index.json"), "utf8")).vectors), Buffer.alloc(4)); + expect(loadCache(cfg).entries.size).toBe(0); + }); + test("each save writes a new vector file the index points at, and keeps at most one previous file", () => { + const cfg = tmpCfg(); + const dir = path.join(cfg.cacheRoot, "bge-m3"); + const vecFiles = () => fs.readdirSync(dir).filter((n) => n.startsWith("vecs.")); + for (let i = 0; i < 4; i++) { + saveCache(cfg, { dim: 3, entries: new Map([["a", { m: i, v: normalize([1, i, 0]) }]]) }, new Set(["a"])); + Bun.sleepSync(2); + } + const current = JSON.parse(fs.readFileSync(path.join(dir, "index.json"), "utf8")).vectors; + expect(vecFiles()).toContain(current); + expect(vecFiles().length).toBeLessThanOrEqual(2); + expect(loadCache(cfg).entries.get("a")!.m).toBe(3); + }); +}); + +describe("refreshEmbeddings", () => { + test("embeds new and changed notes, skips fresh ones, prunes deleted ones, never embeds learning", async () => { + const cfg = tmpCfg(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "corpus-")); + const a = note(dir, "a.md", "deadlock na fila"); + const b = note(dir, "b.md", "heartbeat"); + const l = note(dir, "l.md", "deadlock", "learning"); + let calls = 0; + const counting: Embedder = async (inputs) => { calls += inputs.length; return inputs.map(toyVec); }; + expect(await refreshEmbeddings([a, b, l], cfg, { embed: counting })).toMatchObject({ embedded: 2, pending: 0, pruned: 0, warmed: true }); + expect(calls).toBe(2); + // nothing new: one tiny warm-up embed loads the model off the prompt path, and no note is re-embedded + expect(await refreshEmbeddings([a, b, l], cfg, { embed: counting })).toMatchObject({ embedded: 0, pending: 0, pruned: 0, warmed: true }); + expect(calls).toBe(3); + expect(await refreshEmbeddings([a, b, l], cfg, { embed: counting, warm: false })).toMatchObject({ warmed: false }); + expect(calls).toBe(3); + fs.utimesSync(a.filePath, new Date(), new Date(Date.now() + 5000)); // a changed + expect(await refreshEmbeddings([a], cfg, { embed: counting })).toMatchObject({ embedded: 1, pending: 0, pruned: 1 }); + expect([...loadCache(cfg).entries.keys()]).toEqual([a.filePath]); + }); + test("stops starting batches at the budget and keeps what it saved", async () => { + const cfg = tmpCfg(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "corpus-")); + const notes = Array.from({ length: 20 }, (_, i) => note(dir, `n${i}.md`, "fila")); + const r = await refreshEmbeddings(notes, cfg, { embed: toyEmbedder, budgetMs: 0 }); + expect(r).toMatchObject({ embedded: 0, pending: 20, pruned: 0, warmed: false }); + const r2 = await refreshEmbeddings(notes, cfg, { embed: toyEmbedder, budgetMs: 60_000 }); + expect(r2.embedded).toBe(20); + }); +}); + +describe("rankByEmbedding", () => { + async function setup() { + const cfg = tmpCfg(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "corpus-")); + const notes = [note(dir, "fila.md", "deadlock fila"), note(dir, "hb.md", "heartbeat"), note(dir, "learn.md", "deadlock fila", "learning")]; + await refreshEmbeddings(notes, cfg, { embed: toyEmbedder }); + return { cfg, notes }; + } + test("orders by cosine and never ranks learning notes", async () => { + const { cfg, notes } = await setup(); + const r = await rankByEmbedding("o deadlock da fila", notes, cfg, { embed: toyEmbedder }); + expect(r.order[0]).toBe(0); + expect(r.order).not.toContain(2); + expect(r.ranked).toBe(2); + }); + test("under the cutoff the turn injects nothing", async () => { + const { cfg, notes } = await setup(); + const r = await rankByEmbedding("tennis amanhã", notes, cfg, { embed: toyEmbedder }); + expect(r.order).toEqual([]); + expect(r.topCosine).toBeLessThan(cfg.cutoff); + }); + test("throws (so the caller falls back to keywords) with no vectors, a failing server, or a dimension change", async () => { + const cfg = tmpCfg(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "corpus-")); + const n = [note(dir, "x.md", "fila")]; + await expect(rankByEmbedding("fila", n, cfg, { embed: toyEmbedder })).rejects.toThrow(/no note vectors/); + await refreshEmbeddings(n, cfg, { embed: toyEmbedder }); + await expect(rankByEmbedding("fila", n, cfg, { embed: async () => { throw new Error("down"); } })).rejects.toThrow("down"); + await expect(rankByEmbedding("fila", n, cfg, { embed: async (i) => i.map(() => [1, 0]) })).rejects.toThrow(/dimension/); + }); +}); + +// ---------- end to end through MemoryRetriever, isolated corpus, fake Ollama on loopback ---------- + +describe("getRelevantContextAsync (end to end)", () => { + let server: ReturnType; + let root: string; + let hits = 0; + let down = false; + let slowMs = 0; + const env = { ...process.env }; + + beforeAll(() => { + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(req) { + hits++; + if (down) return new Response("down", { status: 500 }); + if (slowMs) await Bun.sleep(slowMs); + const body = (await req.json()) as { input: string[] }; + return Response.json({ embeddings: body.input.map(toyVec) }); + }, + }); + root = fs.mkdtempSync(path.join(os.tmpdir(), "mem-")); + fs.mkdirSync(path.join(root, "KNOWLEDGE", "Research"), { recursive: true }); + fs.writeFileSync(path.join(root, "KNOWLEDGE", "Research", "fila.md"), "---\ntitle: Fila de relatórios\ntype: knowledge\n---\nO deadlock 40001 na fila de relatórios acontece quando dois workers pegam o mesmo item.\n"); + fs.writeFileSync(path.join(root, "KNOWLEDGE", "Research", "hb.md"), "---\ntitle: Heartbeat\ntype: knowledge\n---\nO heartbeat chama /version a cada minuto.\n"); + process.env.CORTEX_MEMORY_ROOT = root; + // The two USER memory files resolve from LIFEOS_DIR, not CORTEX_MEMORY_ROOT: point it at the temp root + // too, so this test reads no file of the live install. + process.env.LIFEOS_DIR = path.join(root, "lifeos"); + // CLAUDE_CONFIG_DIR outranks LIFEOS_DIR in Paths.ts: set it too, so a dev shell that exports it + // (or LIFEOS_SKILLS_DIR) cannot point this test at the live install. + process.env.CLAUDE_CONFIG_DIR = root; + delete process.env.LIFEOS_SKILLS_DIR; + delete process.env.CLAUDE_PLUGIN_ROOT; + process.env.LIFEOS_MEMORY_EMBEDDING_URL = `http://127.0.0.1:${server.port}`; + process.env.LIFEOS_MEMORY_EMBEDDING_CACHE = path.join(root, "STATE", "cache", "memory-embeddings"); + process.env.LIFEOS_MEMORY_EMBEDDING_CUTOFF = "0.5"; + }); + afterAll(() => { server.stop(true); process.env = env; }); + + test("off: identical to getRelevantContext and no request is made", async () => { + delete process.env.LIFEOS_MEMORY_EMBEDDING; + const m = await import("./MemoryRetriever"); + for (const q of ["deadlock na fila de relatórios", "heartbeat version", "ok", "nada a ver"]) { + m.clearRelevantCache(); + const sync = m.getRelevantContext(q, { topK: 5, threshold: 0.2 }); + m.clearRelevantCache(); + const asyncR = await m.getRelevantContextAsync(q, { topK: 5, threshold: 0.2 }); + expect(asyncR).toEqual(sync); + } + expect(await m.refreshEmbeddingIndex()).toBeNull(); + expect(hits).toBe(0); + }); + + test("on: ranks by meaning, stays silent under the cutoff, and falls back to keywords when the server fails", async () => { + process.env.LIFEOS_MEMORY_EMBEDDING = "1"; + const m = await import("./MemoryRetriever"); + const idx = await m.refreshEmbeddingIndex(); + expect(idx?.embedded).toBe(2); + const before = hits; + expect((await m.refreshEmbeddingIndex())?.warmed).toBe(true); // the Stop hook warms the model when nothing is new + expect(hits).toBe(before + 1); + m.clearRelevantCache(); + const r = await m.getRelevantContextAsync("como tá o deadlock da fila?", { topK: 5, threshold: 0.2, timeoutMs: 2000 }); + expect(r.results[0]?.title).toBe("Fila de relatórios"); + expect(r.markdownBlock).toContain("## RELEVANT MEMORY"); + m.clearRelevantCache(); + expect((await m.getRelevantContextAsync("jogo de tennis amanhã", { timeoutMs: 2000 })).results).toEqual([]); + down = true; + m.clearRelevantCache(); + const fb = await m.getRelevantContextAsync("deadlock fila relatórios", { topK: 5, threshold: 0.2, timeoutMs: 2000 }); + m.clearRelevantCache(); + expect(fb).toEqual(m.getRelevantContext("deadlock fila relatórios", { topK: 5, threshold: 0.2 })); + down = false; + }); + + test("on: a slow server past the deadline still gets the keyword result in time", async () => { + process.env.LIFEOS_MEMORY_EMBEDDING = "1"; + const m = await import("./MemoryRetriever"); + await m.refreshEmbeddingIndex(); + slowMs = 1000; + m.clearRelevantCache(); + const t0 = Date.now(); + const r = await m.getRelevantContextAsync("deadlock fila relatórios", { topK: 5, threshold: 0.2, deadline: Date.now() + 150 }); + const took = Date.now() - t0; + slowMs = 0; + expect(took).toBeLessThan(400); + m.clearRelevantCache(); + expect(r).toEqual(m.getRelevantContext("deadlock fila relatórios", { topK: 5, threshold: 0.2 })); + }); + + test("on: the conversation context is embedded with the prompt but never decides whether to search", async () => { + process.env.LIFEOS_MEMORY_EMBEDDING = "1"; + const m = await import("./MemoryRetriever"); + await m.refreshEmbeddingIndex(); + m.clearRelevantCache(); + // "and how is it now?" alone matches no topic; with the previous message it is about the queue + expect((await m.getRelevantContextAsync("and how is it now?", { timeoutMs: 2000 })).results).toEqual([]); + m.clearRelevantCache(); + const withCtx = await m.getRelevantContextAsync("and how is it now?", { timeoutMs: 2000, context: "the deadlock in the report fila" }); + expect(withCtx.results[0]?.title).toBe("Fila de relatórios"); + // an approval-only prompt retrieves nothing even with a topical context (the keyword path's gate) + m.clearRelevantCache(); + expect((await m.getRelevantContextAsync("ok", { timeoutMs: 2000, context: "deadlock fila" })).results).toEqual([]); + }); + + test("worker: accepts the object message and, off, answers exactly as getRelevantContext", async () => { + delete process.env.LIFEOS_MEMORY_EMBEDDING; + const m = await import("./MemoryRetriever"); + // a Bun worker starts from the environment the process began with: hand it the isolated one + const w = new Worker(new URL("../../hooks/lib/retrieval-worker.ts", import.meta.url).href, { env: { ...process.env } } as any); + const got = await new Promise((res) => { w.onmessage = (e) => res(e.data); w.postMessage({ query: "deadlock fila relatórios", deadline: Date.now() + 2000, transcriptPath: "/does/not/exist.jsonl" }); }); + w.terminate(); + m.clearRelevantCache(); + expect(got.markdownBlock).toBe(m.getRelevantContext("deadlock fila relatórios", { topK: 5, threshold: 0.2, allowStaleIndex: true }).markdownBlock); + }); +}); diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.ts new file mode 100644 index 0000000000..9c54a83cee --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryEmbedding.ts @@ -0,0 +1,261 @@ +/** + * MemoryEmbedding.ts — the opt-in embedding leg of per-turn memory retrieval (public issue #2207). + * + * Off unless LIFEOS_MEMORY_EMBEDDING=1. Off, nothing here runs: no request, no file, no import side + * effect, and retrieval is the keyword (BM25) path exactly as before. + * + * On, the per-turn retriever ranks notes by cosine similarity between the query and each note's + * embedding, and injects nothing when the best cosine is under LIFEOS_MEMORY_EMBEDDING_CUTOFF. That + * cutoff is the "return nothing when nothing fits" signal the keyword score could not give (#2207). + * Measured on bge-m3: fusing the keyword score back in did not beat cosine alone (McNemar p = 0.77), + * so the leg ranks by cosine only. + * + * - Local only. Embeddings come from an Ollama server on this machine (`/api/embed`); a URL whose + * host is not loopback is refused, so note text never leaves the machine. + * - Off the prompt path except for one query embed. Note vectors are computed by the async Stop + * hook (MemoryIndexRefresh → refreshEmbeddings), never on the prompt, and a new note joins the + * ranking on a later turn. No daemon, no detached process, no new clock. + * - Vectors live in a disposable cache keyed by path + mtime + model under + * MEMORY/STATE/cache/memory-embeddings//. Deleting it only forces a re-embed. + * - MEMORY/LEARNING notes are not ranked by vector: they are raw transcript excerpts, read like + * conversation, and took 54% of top-5 slots on conversational queries while no labelled turn + * needed one (#2207). + * - Every failure (no server, no model, timeout, no vectors yet, dimension mismatch) throws; the + * caller (MemoryRetriever.getRelevantContextAsync) computes the keyword result first and serves it, + * with the query embed bounded by the hook's deadline. This module never takes down a turn. + * - Setup: a local Ollama with the model pulled (`ollama pull bge-m3`). A missing model is a 404, + * which reads as a failure: keyword retrieval every turn, and a line on the Stop hook's stderr. + */ +import * as fs from "fs"; +import * as path from "path"; +import { lifeosPath } from "./Paths"; + +export interface EmbeddingConfig { + enabled: boolean; + /** Ollama base URL, loopback only. */ + url: string; + model: string; + /** Minimum best cosine for a turn to inject anything. Model-specific; 0.56 measured on bge-m3. */ + cutoff: number; + /** Sent as Ollama `keep_alive` only when set; otherwise the server's own setting applies. */ + keepAlive?: string; + cacheRoot: string; +} + +const num = (v: string | undefined, d: number) => (v !== undefined && v.trim() !== "" && Number.isFinite(Number(v)) ? Number(v) : d); + +/** True for a URL whose host is this machine. */ +export function isLoopbackUrl(raw: string): boolean { + try { + const u = new URL(raw); + if (u.protocol !== "http:" && u.protocol !== "https:") return false; + const h = u.hostname.replace(/^\[|\]$/g, ""); + return h === "localhost" || h === "::1" || /^127(\.\d{1,3}){3}$/.test(h); + } catch { + return false; + } +} + +export function embeddingConfig(env: NodeJS.ProcessEnv = process.env): EmbeddingConfig { + const url = (env.LIFEOS_MEMORY_EMBEDDING_URL || "http://127.0.0.1:11434").replace(/\/+$/, ""); + return { + // A non-loopback URL leaves the leg off: retrieval stays keyword-only rather than send notes away. + enabled: env.LIFEOS_MEMORY_EMBEDDING === "1" && isLoopbackUrl(url), + url, + model: env.LIFEOS_MEMORY_EMBEDDING_MODEL || "bge-m3", + cutoff: num(env.LIFEOS_MEMORY_EMBEDDING_CUTOFF, 0.56), + keepAlive: env.LIFEOS_MEMORY_EMBEDDING_KEEP_ALIVE || undefined, + cacheRoot: env.LIFEOS_MEMORY_EMBEDDING_CACHE || lifeosPath("MEMORY", "STATE", "cache", "memory-embeddings"), + }; +} + +const MAX_DOC_CHARS = 6000; +const MAX_QUERY_CHARS = 4000; +const BATCH = 8; +const BATCH_TIMEOUT_MS = 15_000; + +// ---------- Ollama ---------- + +export type Embedder = (inputs: string[], timeoutMs: number) => Promise; + +/** The Ollama embedder for `cfg`. Throws on any failure. Injectable so tests need no network. */ +export function ollamaEmbedder(cfg: EmbeddingConfig): Embedder { + return async (inputs, timeoutMs) => { + if (!isLoopbackUrl(cfg.url)) throw new Error("embedding: refusing a non-loopback server"); + if (timeoutMs < 10) throw new Error("embedding: no time left"); + const keepAlive = cfg.keepAlive === undefined ? undefined : /^-?\d+$/.test(cfg.keepAlive) ? Number(cfg.keepAlive) : cfg.keepAlive; + const res = await fetch(`${cfg.url}/api/embed`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: cfg.model, input: inputs, truncate: true, ...(keepAlive === undefined ? {} : { keep_alive: keepAlive }) }), + signal: AbortSignal.timeout(Math.ceil(timeoutMs)), + redirect: "error", // a loopback server must not forward note text elsewhere + }); + if (!res.ok) throw new Error(`embedding: HTTP ${res.status}`); + const j = (await res.json()) as { embeddings?: number[][]; error?: string }; + if (!Array.isArray(j.embeddings) || j.embeddings.length !== inputs.length) throw new Error(`embedding: ${j.error ?? "response without embeddings"}`); + return j.embeddings; + }; +} + +export function normalize(v: ArrayLike): Float32Array { + let n = 0; + for (let i = 0; i < v.length; i++) n += v[i] * v[i]; + n = Math.sqrt(n) || 1; + const o = new Float32Array(v.length); + for (let i = 0; i < v.length; i++) o[i] = v[i] / n; + return o; +} + +// ---------- vector cache ---------- + +const CACHE_VERSION = 2; + +export interface VectorCache { dim: number; entries: Map } + +export const cacheDir = (cfg: EmbeddingConfig) => path.join(cfg.cacheRoot, cfg.model.replace(/[^A-Za-z0-9._-]/g, "_")); + +/** The cache on disk; empty when absent, from another version, or truncated. */ +export function loadCache(cfg: EmbeddingConfig): VectorCache { + const c: VectorCache = { dim: 0, entries: new Map() }; + try { + const dir = cacheDir(cfg); + const meta = JSON.parse(fs.readFileSync(path.join(dir, "index.json"), "utf8")) as { version: number; dim: number; vectors: string; entries: { p: string; m: number }[] }; + if (meta.version !== CACHE_VERSION || !meta.dim || !/^vecs\.[A-Za-z0-9.-]+\.bin$/.test(meta.vectors)) return c; + const buf = fs.readFileSync(path.join(dir, meta.vectors)); + const all = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); + if (all.length !== meta.entries.length * meta.dim) return c; + c.dim = meta.dim; + meta.entries.forEach((e, row) => c.entries.set(e.p, { m: e.m, v: all.subarray(row * meta.dim, (row + 1) * meta.dim) })); + } catch { /* absent = empty */ } + return c; +} + +/** + * Write the entries in `keep`. The vectors go to a new file named by generation, then index.json + * (renamed into place) points at it, so a reader always sees an index and the vectors written with + * it, even when two Stop hooks race. Older vector files are removed after the switch. + */ +export function saveCache(cfg: EmbeddingConfig, c: VectorCache, keep: Set): void { + const entries: { p: string; m: number }[] = []; + const rows: Float32Array[] = []; + for (const [p, e] of c.entries) if (keep.has(p)) { entries.push({ p, m: e.m }); rows.push(e.v); } + const bin = new Float32Array(rows.length * c.dim); + rows.forEach((r, i) => bin.set(r, i * c.dim)); + const dir = cacheDir(cfg); + fs.mkdirSync(dir, { recursive: true }); + const gen = `${Date.now()}-${process.pid}`; + const vectors = `vecs.${gen}.bin`; + fs.writeFileSync(path.join(dir, vectors), Buffer.from(bin.buffer)); + const tmp = path.join(dir, `index.json.${gen}.tmp`); + fs.writeFileSync(tmp, JSON.stringify({ version: CACHE_VERSION, model: cfg.model, dim: c.dim, vectors, entries })); + fs.renameSync(tmp, path.join(dir, "index.json")); + // Keep the newest previous file too: a reader that loaded the old index a moment ago can still read it. + const old = fs.readdirSync(dir).filter((n) => /^vecs\..+\.bin$/.test(n) && n !== vectors).sort().slice(0, -1); + for (const n of [...old, "vecs.bin"]) { try { fs.unlinkSync(path.join(dir, n)); } catch { /* gone or held open */ } } +} + +// ---------- corpus ---------- + +/** A note as this leg sees it. `body` is needed only to embed. */ +export interface EmbeddingNote { filePath: string; title: string; noteClass: string; body?: string } + +export const vectorRankable = (n: Pick) => n.noteClass !== "learning"; + +const mtimeOf = (p: string): number | null => { try { return fs.statSync(p).mtimeMs; } catch { return null; } }; + +export const noteText = (n: EmbeddingNote) => `${n.title}\n${(n.body ?? "").slice(0, MAX_DOC_CHARS)}`; + +// ---------- Stop hook: embed new and changed notes ---------- + +/** + * Embed every rankable note whose vector is missing or older than the file, and drop vectors of + * notes no longer in the corpus. Stops starting new batches at `budgetMs` and saves after each + * batch, so a large first index spreads over a few turns. With nothing to embed, sends one tiny + * embed so the model is loaded before the next prompt needs it (`warm: false` skips it). + */ +export async function refreshEmbeddings( + notes: EmbeddingNote[], + cfg: EmbeddingConfig = embeddingConfig(), + opts: { budgetMs?: number; embed?: Embedder; warm?: boolean } = {}, +): Promise<{ embedded: number; pending: number; pruned: number; warmed: boolean }> { + const t0 = Date.now(); + const budget = opts.budgetMs ?? 40_000; + const embed = opts.embed ?? ollamaEmbedder(cfg); + const c = loadCache(cfg); + const rankable = notes.filter(vectorRankable); + const keep = new Set(rankable.map((n) => n.filePath)); + const pruned = [...c.entries.keys()].filter((p) => !keep.has(p)).length; + const todo: { n: EmbeddingNote; m: number }[] = []; + for (const n of rankable) { + const m = mtimeOf(n.filePath); + if (m !== null && c.entries.get(n.filePath)?.m !== m) todo.push({ n, m }); + } + let embedded = 0; + // A batch starts only while a whole batch timeout still fits in the budget, so the hook never + // runs into its own timeout and a slow batch never throws away the ones already saved. + for (let i = 0; i < todo.length && Date.now() - t0 + BATCH_TIMEOUT_MS <= budget; i += BATCH) { + const batch = todo.slice(i, i + BATCH); + const vecs = await embed(batch.map((b) => noteText(b.n)), BATCH_TIMEOUT_MS); + for (let j = 0; j < batch.length; j++) { + const v = normalize(vecs[j]); + if (c.dim && v.length !== c.dim) c.entries.clear(); // a different model under the same name: start over + c.dim = v.length; + c.entries.set(batch[j].n.filePath, { m: batch[j].m, v }); + } + embedded += batch.length; + saveCache(cfg, c, keep); + } + if (!embedded && pruned && c.dim) saveCache(cfg, c, keep); + // Warm the model off the prompt path. Measured: with the model unloaded, the prompt's query embed + // outlives the 300 ms budget, the worker is terminated, the request is aborted, and Ollama cancels + // the load, so the model never loaded and every later turn fell back to keywords. One tiny embed + // here loads it (or refreshes its keep-alive) where nothing waits on it. Failure is harmless. + let warmed = embedded > 0; + if (!warmed && opts.warm !== false && Date.now() - t0 + BATCH_TIMEOUT_MS <= budget) { + try { await embed(["warm-up"], BATCH_TIMEOUT_MS); warmed = true; } catch { /* the next turn falls back to keywords */ } + } + return { embedded, pending: todo.length - embedded, pruned, warmed }; +} + +// ---------- prompt path: rank by cosine ---------- + +export interface EmbeddingRanking { + /** Indices into the candidates, best first; empty when the best cosine is under the cutoff. */ + order: number[]; + /** Cosine per entry of `order`. */ + cosines: number[]; + topCosine: number; + /** Candidates that had a vector and took part. */ + ranked: number; +} + +/** + * Rank `candidates` for `query`. A candidate without a vector yet is left out; one whose file + * changed keeps its previous vector until the Stop hook refreshes it. Throws when the leg cannot + * answer, so the caller serves the keyword result. + */ +export async function rankByEmbedding( + query: string, + candidates: Pick[], + cfg: EmbeddingConfig = embeddingConfig(), + opts: { timeoutMs?: number; embed?: Embedder; cache?: VectorCache } = {}, +): Promise { + const c = opts.cache ?? loadCache(cfg); + const pool: number[] = []; + candidates.forEach((cand, i) => { if (vectorRankable(cand) && c.entries.has(cand.filePath)) pool.push(i); }); + if (!pool.length) throw new Error("embedding: no note vectors yet"); + const [raw] = await (opts.embed ?? ollamaEmbedder(cfg))([query.slice(0, MAX_QUERY_CHARS)], opts.timeoutMs ?? 250); + const qv = normalize(raw); + if (qv.length !== c.dim) throw new Error("embedding: cache dimension differs from the model's"); + const cos = pool.map((i) => { + const v = c.entries.get(candidates[i].filePath)!.v; + let s = 0; + for (let k = 0; k < qv.length; k++) s += v[k] * qv[k]; + return s; + }); + const topCosine = cos.reduce((a, b) => Math.max(a, b), -Infinity); + if (topCosine < cfg.cutoff) return { order: [], cosines: [], topCosine, ranked: pool.length }; + const idx = pool.map((_, j) => j).sort((a, b) => cos[b] - cos[a]); + return { order: idx.map((j) => pool[j]), cosines: idx.map((j) => cos[j]), topCosine, ranked: pool.length }; +} diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts index c9ef91b878..599968b285 100755 --- a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts @@ -28,7 +28,8 @@ for (const __k of ["LIFEOS_DIR", "LIFEOS_CONFIG_DIR", "PROJECTS_DIR"]) { * * SEARCH: * BM25-style keyword matching + tag co-occurrence scoring. - * No embeddings, no vector DB — pure markdown + YAML frontmatter. + * No vector DB — pure markdown + YAML frontmatter. Per-turn retrieval has an opt-in embedding + * leg (LIFEOS_MEMORY_EMBEDDING=1, local Ollama only; MemoryEmbedding.ts), off by default. * * COMPRESSION: * Uses Inference.ts low level for optional LLM compression of matched content. @@ -63,6 +64,7 @@ import { homedir } from "node:os"; import { invariant, InvariantViolation } from "./Invariant"; import { getClaudeDir, getLifeosDir, lifeosPath } from "./Paths"; // public issue #1974 @sabo961, PR #1918 @PigLardLord import { ALL_DIRS } from "./KnowledgeSchema.ts"; // public PR #2149, @Bowlerjim; PR #2143, @joecoxtech +import { embeddingConfig, rankByEmbedding, refreshEmbeddings, type EmbeddingConfig } from "./MemoryEmbedding"; // opt-in leg, public issue #2207 // Normalize env path vars that Claude Code injects without shell expansion (LifeOS#1404) for (const k of ["LIFEOS_DIR", "LIFEOS_CONFIG_DIR", "PROJECTS_DIR"]) { @@ -1565,6 +1567,91 @@ export function getRelevantContext( return result; } +/** + * The per-turn entry for the retrieval worker (hooks/lib/retrieval-worker.ts). With the embedding + * leg off (the default) it is getRelevantContext, unchanged. With LIFEOS_MEMORY_EMBEDDING=1 it + * ranks the same corpus by embedding (MemoryEmbedding.ts) (public issue #2207). The keyword result is + * computed first and kept, and the query embed gets only the time left before `deadline` (epoch ms, + * from the hook): a failure or a slow embed returns that keyword result in time, so a stopped or + * cold server costs one keyword turn and never the turn. Without `deadline`, `timeoutMs` (default + * 250) bounds the embed. `context` (the previous user messages) is embedded after the query; it + * never decides whether to search: a prompt with no content words still retrieves nothing. + */ +export async function getRelevantContextAsync( + query: string, + options: RelevantContextOptions & { deadline?: number; timeoutMs?: number; context?: string } = {}, +): Promise { + const cfg = embeddingConfig(); + if (!cfg.enabled) return getRelevantContext(query, options); + const startedAt = Date.now(); + const keyword = computeRelevantContext(query, options); + let result = keyword; + try { + const timeoutMs = options.deadline !== undefined ? options.deadline - Date.now() - EMBED_RESULT_RESERVE_MS : options.timeoutMs ?? 250; + result = await computeEmbeddingContext(query, { ...options, timeoutMs }, cfg); + } catch (err) { + if (err instanceof InvariantViolation) throw err; + result = keyword; // fail open to the keyword result already in hand + } + logRetrieval(query, result.results.length, Date.now() - startedAt); + return result; +} + +/** Time kept after the query embed to build the hits and post them back before the hook's deadline. */ +const EMBED_RESULT_RESERVE_MS = 15; + +async function computeEmbeddingContext( + query: string, + options: RelevantContextOptions & { timeoutMs: number; context?: string }, + cfg: EmbeddingConfig, +): Promise { + const topK = options.topK ?? RELEVANT_DEFAULT_TOP_K; + const excerptChars = options.excerptChars ?? RELEVANT_EXCERPT_CHARS; + const empty: GetRelevantContextResult = { results: [], markdownBlock: "", totalSearched: 0, cached: false }; + // Same gate as the keyword path: a prompt with no content words ("ok", "go ahead") retrieves nothing. + const queryTerms = retrievalTerms(query); + if (queryTerms.length === 0) return empty; + const allNotes = corpusForQuery(queryTerms, options.allowStaleIndex === true); + if (allNotes.length === 0) return empty; + const embedded = options.context?.trim() ? `${query.trim()}\n${options.context.trim()}` : query; + const ranking = await rankByEmbedding(embedded, allNotes, cfg, { timeoutMs: options.timeoutMs }); + const results: RelevantResultItem[] = []; + for (let j = 0; j < ranking.order.length && results.length < topK; j++) { + const note = allNotes[ranking.order[j]]; + if (options.typeFilter && (note.frontmatter.type ?? "unknown") !== options.typeFilter) continue; + const body = note.noteClass === "memory" ? note.body : readBody(note.filePath); + if (body === null) continue; // a stale index entry whose file is gone gives way to the next one + const rawType = (note.frontmatter.type ?? "unknown") as string; + const knownType: RelevantResultItem["type"] = + rawType === "memory" || rawType === "idea" || rawType === "knowledge" ? rawType : "unknown"; + results.push({ + type: knownType, + noteClass: note.noteClass, + path: note.filePath, + title: (note.frontmatter.title as string) || path.basename(note.filePath, ".md"), + score: ranking.cosines[j], + excerpt: extractExcerpt({ ...note, body }, queryTerms).slice(0, excerptChars), + }); + } + return { results, markdownBlock: formatRelevantBlock(results), totalSearched: allNotes.length, cached: false }; +} + +/** + * Embed new and changed notes for the embedding leg; called by the async Stop hook after the + * keyword index refresh. Null when the leg is off. + */ +export async function refreshEmbeddingIndex(opts: { budgetMs?: number } = {}): Promise<{ embedded: number; pending: number; pruned: number; warmed: boolean } | null> { + const cfg = embeddingConfig(); + if (!cfg.enabled) return null; + const notes = [...indexedNotes(), ...loadMemoryFiles()].map((n) => ({ + filePath: n.filePath, + title: (n.frontmatter.title as string) || path.basename(n.filePath, ".md"), + noteClass: n.noteClass, + body: n.body, + })); + return refreshEmbeddings(notes, cfg, opts); +} + /** * Append one evidence row per retrieval to MEMORY/OBSERVABILITY/memory-retrievals.jsonl, * the log CortexHealth reads to answer "is recall actually running?". Without it diff --git a/LifeOS/install/hooks/MemoryIndexRefresh.hook.ts b/LifeOS/install/hooks/MemoryIndexRefresh.hook.ts index 60acd2ff3f..67e4a33e90 100755 --- a/LifeOS/install/hooks/MemoryIndexRefresh.hook.ts +++ b/LifeOS/install/hooks/MemoryIndexRefresh.hook.ts @@ -15,7 +15,7 @@ * it already would have; it is reported on stderr, never raised. */ -import { refreshIndex } from "../LIFEOS/TOOLS/MemoryRetriever"; +import { refreshIndex, refreshEmbeddingIndex } from "../LIFEOS/TOOLS/MemoryRetriever"; import { InvariantViolation } from "../LIFEOS/TOOLS/Invariant"; try { @@ -29,4 +29,14 @@ try { console.error(`${label}: ${err instanceof Error ? err.message : String(err)}`); } +// Opt-in embedding leg (public issue #2207): embed new and changed notes after the keyword index is +// fresh. Off by default (null, no work). Bounded under the hook's 60s timeout; a large first index +// finishes over a few turns. +try { + const emb = await refreshEmbeddingIndex({ budgetMs: 40_000 }); + if (emb && (emb.embedded || emb.pruned)) console.error(`memory embeddings: ${emb.embedded} embedded, ${emb.pending} pending, ${emb.pruned} pruned`); +} catch (err) { + console.error(`memory embedding refresh failed: ${err instanceof Error ? err.message : String(err)}`); +} + process.exit(0); diff --git a/LifeOS/install/hooks/MemoryTurnStart.hook.ts b/LifeOS/install/hooks/MemoryTurnStart.hook.ts index a40eb10fea..a91d8a3aff 100755 --- a/LifeOS/install/hooks/MemoryTurnStart.hook.ts +++ b/LifeOS/install/hooks/MemoryTurnStart.hook.ts @@ -249,10 +249,14 @@ export function acknowledgeEmission(sessionId: string, emissionId: string): void writeQueue(sessionId, updated); } -export type AsyncRetrieve = (prompt: string, waitMs: number) => Promise<{ markdownBlock: string } | null>; +/** `transcriptPath`: read by the worker only when the opt-in embedding leg is on (#2207). */ +export type AsyncRetrieve = (prompt: string, waitMs: number, transcriptPath?: string) => Promise<{ markdownBlock: string } | null>; /** The retriever in a worker; resolves null and terminates the worker once `waitMs` passes. */ -export const retrieveInWorker: AsyncRetrieve = (prompt, waitMs) => new Promise((resolve) => { +export const retrieveInWorker: AsyncRetrieve = (prompt, waitMs, transcriptPath) => new Promise((resolve) => { + // An absolute deadline, set before the worker boots: the embedding leg (opt-in, #2207) gives its query + // embed only what is left, keeping room to post the reply back. + const deadline = Date.now() + waitMs - 25; let w: Worker; try { w = new Worker(new URL("./lib/retrieval-worker.ts", import.meta.url).href); } catch (e) { process.stderr.write(`MemoryTurnStart ground error: worker did not start (${(e as Error)?.message ?? e})\n`); resolve({ markdownBlock: "" }); return; } @@ -268,7 +272,7 @@ export const retrieveInWorker: AsyncRetrieve = (prompt, waitMs) => new Promise(( const failed = (why: string) => { process.stderr.write(`MemoryTurnStart ground error: ${why}\n`); done({ markdownBlock: "" }); }; w.onmessage = (e: MessageEvent) => typeof e.data?.markdownBlock === "string" ? done({ markdownBlock: e.data.markdownBlock }) : failed(String(e.data?.error ?? "no block")); w.onerror = (e: ErrorEvent) => failed(e?.message || "worker error"); - w.postMessage(prompt); + w.postMessage({ query: prompt, deadline, transcriptPath }); }); /** One turn's stdout under the ceiling. Exported for tests. */ @@ -283,7 +287,8 @@ export async function compose(input: Record | null, opts: { ela const waitMs = Math.max(RETRIEVAL_MIN_WAIT_MS, SOFT_BUDGET_MS - elapsed()); const late = new Promise((r) => setTimeout(() => r(null), waitMs)); let g: { markdownBlock: string } | null = null; - try { g = await Promise.race([(opts.retrieve ?? retrieveInWorker)(m.groundQuery, waitMs), late]); } + const transcriptPath = typeof input?.transcript_path === "string" ? input.transcript_path : undefined; + try { g = await Promise.race([(opts.retrieve ?? retrieveInWorker)(m.groundQuery, waitMs, transcriptPath), late]); } catch (e) { process.stderr.write(`MemoryTurnStart ground error: ${(e as Error)?.message ?? e}\n`); g = { markdownBlock: "" }; } if (g === null) m.ground = SKIP_NOTICE; else if (g.markdownBlock) m.ground = `\n${g.markdownBlock}\n\n`; diff --git a/LifeOS/install/hooks/lib/recent-user-messages.test.ts b/LifeOS/install/hooks/lib/recent-user-messages.test.ts new file mode 100644 index 0000000000..50e69a727e --- /dev/null +++ b/LifeOS/install/hooks/lib/recent-user-messages.test.ts @@ -0,0 +1,41 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { recentUserMessages, conversationContext } from "./recent-user-messages"; + +const u = (content: unknown, extra: object = {}) => JSON.stringify({ type: "user", message: { role: "user", content }, ...extra }); +const a = (t: string) => JSON.stringify({ type: "assistant", message: { role: "assistant", content: [{ type: "text", text: t }] } }); + +function transcript(lines: string[]): string { + const p = join(mkdtempSync(join(tmpdir(), "tr-")), "t.jsonl"); + writeFileSync(p, lines.join("\n") + "\n"); + return p; +} + +test("query = current prompt + 3 previous text messages, newest first, skipping everything else", () => { + const p = transcript([ + u("old message zero"), + a("reply"), + u("first about the queue"), + u([{ type: "tool_result", tool_use_id: "x", content: "TOOL_RESULT" }]), + u("TASK_NOTIF"), + u("SYSTEM_REMINDER"), + u("hook output", { isMeta: true }), + u("HOOK_BLOCK"), + u([{ type: "text", text: "second with a text block" }, { type: "image", source: {} }]), + a("another reply"), + u("third about the worker"), + u("fourth and current"), // the transcript may already hold the current prompt + ]); + expect(recentUserMessages(p, 3, "fourth and current")).toEqual(["first about the queue", "second with a text block", "third about the worker"]); + expect(conversationContext("fourth and current", p)).toBe("third about the worker\nsecond with a text block\nfirst about the queue"); + const ctx = conversationContext("fourth and current", p); + for (const t of ["fourth and current", "old message zero", "TOOL_RESULT", "TASK_NOTIF", "SYSTEM_REMINDER", "HOOK_BLOCK", "hook output"]) expect(ctx).not.toContain(t); +}); + +test("a missing or invalid transcript never throws", () => { + expect(recentUserMessages(undefined)).toEqual([]); + expect(recentUserMessages("/does/not/exist.jsonl")).toEqual([]); + expect(conversationContext("hi", "/does/not/exist.jsonl")).toBe(""); +}); diff --git a/LifeOS/install/hooks/lib/recent-user-messages.ts b/LifeOS/install/hooks/lib/recent-user-messages.ts new file mode 100644 index 0000000000..eef3caded3 --- /dev/null +++ b/LifeOS/install/hooks/lib/recent-user-messages.ts @@ -0,0 +1,62 @@ +/** + * recent-user-messages.ts — the principal's last few text messages from a Claude Code JSONL transcript. + * + * Used by the opt-in embedding leg of per-turn memory retrieval (public issue #2207): the query is + * embedded together with the previous user messages, because a short turn ("and the worker?") only + * makes sense in its conversation. Measured on a held-out set of 70 real turns: prompt + 3 previous + * messages found the expected note in 31/45 turns vs 17/45 for the prompt alone (McNemar p = 0.0005). + * + * Skips tool results, task notifications, system reminders, hook output, slash-command echoes, and + * isMeta / sidechain entries. Reads only the transcript's tail. Never throws; any error returns []. + */ +import { openSync, readSync, fstatSync, closeSync } from "node:fs"; + +const TAIL_BYTES = 2_000_000; +const NOISE = /^\s*<(task-notification|system-reminder|command-name|command-message|command-args|local-command|lifeos-|user-prompt-submit-hook|bash-input|bash-stdout|bash-stderr)/i; + +function textOf(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter((b: any) => b && b.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("\n"); +} + +/** Up to `n` previous user text messages, oldest first, each capped at `maxChars`. */ +export function recentUserMessages(transcriptPath: string | undefined, n = 3, currentPrompt = "", maxChars = 600): string[] { + if (!transcriptPath) return []; + try { + const fd = openSync(transcriptPath, "r"); + let raw: string; + try { + const size = fstatSync(fd).size; + const start = Math.max(0, size - TAIL_BYTES); + const buf = Buffer.alloc(size - start); + readSync(fd, buf, 0, buf.length, start); + raw = buf.toString("utf8"); + } finally { closeSync(fd); } + const lines = raw.split("\n"); + if (raw.length >= TAIL_BYTES) lines.shift(); // the first line may be cut + const msgs: string[] = []; + for (const line of lines) { + if (!line.trim()) continue; + let o: any; + try { o = JSON.parse(line); } catch { continue; } + if (o?.type !== "user" || o.isMeta || o.isSidechain) continue; + const t = textOf(o.message?.content).trim(); + if (!t || NOISE.test(t)) continue; + msgs.push(t); + } + // the transcript may already hold the current prompt as its last entry + const cur = currentPrompt.trim(); + if (cur && msgs.length && msgs[msgs.length - 1] === cur) msgs.pop(); + return msgs.slice(-n).map((m) => m.slice(0, maxChars)); + } catch { return []; } +} + +/** The previous messages as one block, newest first: what the embedding leg embeds after the prompt. */ +export function conversationContext(prompt: string, transcriptPath: string | undefined): string { + return recentUserMessages(transcriptPath, 3, prompt).reverse().join("\n"); +} + diff --git a/LifeOS/install/hooks/lib/retrieval-worker.ts b/LifeOS/install/hooks/lib/retrieval-worker.ts index 3513d44067..56bd1dd2d6 100644 --- a/LifeOS/install/hooks/lib/retrieval-worker.ts +++ b/LifeOS/install/hooks/lib/retrieval-worker.ts @@ -1,15 +1,24 @@ /** - * retrieval-worker.ts — runs the BM25 retriever off the prompt hook's thread, so MemoryTurnStart can stop + * retrieval-worker.ts — runs the retriever off the prompt hook's thread, so MemoryTurnStart can stop * waiting at its budget and terminate this worker instead of holding the prompt (2026-10-16 native-unblock ISC-18). - * Message in: the query string. Message out: { markdownBlock } or { error }. + * Message in: the query string, or { query, deadline, transcriptPath } for the opt-in embedding leg (public + * issue #2207): deadline (epoch ms) is when the hook stops waiting, and with the leg on the previous user + * messages are read from transcriptPath here, off the hook's thread. Message out: { markdownBlock } or { error }. */ -import { getRelevantContext } from "../../LIFEOS/TOOLS/MemoryRetriever"; +import { getRelevantContextAsync } from "../../LIFEOS/TOOLS/MemoryRetriever"; +import { embeddingConfig } from "../../LIFEOS/TOOLS/MemoryEmbedding"; +import { conversationContext } from "./recent-user-messages"; declare const self: Worker; -self.onmessage = (e: MessageEvent) => { +self.onmessage = async (e: MessageEvent) => { try { - const r = getRelevantContext(String(e.data ?? ""), { topK: 5, threshold: 0.20, allowStaleIndex: true }); + const msg = e.data; + const query = typeof msg === "string" ? msg : String(msg?.query ?? ""); + const deadline = typeof msg?.deadline === "number" ? msg.deadline : undefined; + const transcriptPath = typeof msg?.transcriptPath === "string" ? msg.transcriptPath : undefined; + const context = transcriptPath && embeddingConfig().enabled ? conversationContext(query, transcriptPath) : undefined; + const r = await getRelevantContextAsync(query, { topK: 5, threshold: 0.20, allowStaleIndex: true, deadline, context }); postMessage({ markdownBlock: r.markdownBlock }); } catch (err) { postMessage({ error: err instanceof Error ? err.message : String(err) });