diff --git a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md index 5a3362d19b..8e13c62b42 100644 --- a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md +++ b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md @@ -149,7 +149,7 @@ Defined in `LIFEOS/TOOLS/MutationTier.ts` — code-only allowlist, default-deny - **`LIFEOS/TOOLS/MemoryTypes.ts`** — type registry. Resolvers for storage path per item. - **`LIFEOS/TOOLS/MemoryWriter.ts`** — set-overwrite primitive for Tier A memory files. Atomic rename, lock sidecar, prefix soft-convention, dedupe, cap enforcement. - **`LIFEOS/TOOLS/MemorySystem.ts`** — the single public API. `add(item)` routes by type; `find(query, options)` returns BM25 top-K. -- **`LIFEOS/TOOLS/MemoryRetriever.ts`** — BM25 retriever over the typed-item corpus: KNOWLEDGE notes, the `LEARNING/` tree, and the two hot-layer memory files. Pure-function, no LLM call. Every hit carries a `noteClass` (`knowledge` / `learning` / `memory`) and learning notes score at 0.7x so historical records never outrank curated knowledge on an equal match (public issue #1776, @umair-a11y). The CLI searches all of LEARNING; the per-turn hot path takes a recency-bounded slice (`HOT_PATH_LEARNING_LIMIT`, currently 500) because LEARNING grows monotonically and soon outnumbers the KNOWLEDGE corpus several times over, so a full scan on every turn costs noticeably more than a KNOWLEDGE-only search. +- **`LIFEOS/TOOLS/MemoryRetriever.ts`** — BM25 retriever over the typed-item corpus: KNOWLEDGE notes, the `LEARNING/` tree, and the two hot-layer memory files. Pure-function, no LLM call. Every hit carries a `noteClass` (`knowledge` / `learning` / `memory`) and learning notes score at 0.7x so historical records never outrank curated knowledge on an equal match (public issue #1776, @umair-a11y). The CLI searches all of LEARNING; the per-turn hot path takes a recency-bounded slice (`HOT_PATH_LEARNING_LIMIT`, currently 500) because LEARNING grows monotonically and soon outnumbers the KNOWLEDGE corpus several times over, so a full scan on every turn costs noticeably more than a KNOWLEDGE-only search. Query-side glue words are English (`QUERY_GLUE`); an install that writes in another language lists its own in `LIFEOS/USER/CONFIG/glue-words.txt`, one per line, `#` for comments. With no file nothing changes (public issue #2207). - **`LIFEOS/TOOLS/MemoryReviewer.ts`** — the autonomic centerpiece. Single-pass orchestrator: locate most-recent harness transcript, extract last N exchanges (every user event since the last reply joins one exchange, so typed words followed by a pasted image or skill body stay together), call `Inference.ts` at level `medium` on the configured inference vendor (`--tools ""`), parse `{items:[...]}` JSON, route each typed item through `MemorySystem.add()`. - **`LIFEOS/TOOLS/MutationTier.ts`** — tier classifier. Pure function, no config file (code-only allowlist). - **`LIFEOS/TOOLS/MemoryStatus.ts`** — read-only memory-status viewer for terminal use. diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.glue.test.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.glue.test.ts new file mode 100644 index 0000000000..8fc2c927da --- /dev/null +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.glue.test.ts @@ -0,0 +1,48 @@ +// Per-install glue words for the retriever's query side (public issue #2207). +import { afterAll, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { installGlueWords, retrievalTerms } from "./MemoryRetriever"; + +const dir = mkdtempSync(join(tmpdir(), "glue-words-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +const list = (name: string, text: string) => { const f = join(dir, name); writeFileSync(f, text); return f; }; + +test("no file: no extra words, and the query terms are what they were", () => { + const none = installGlueWords(join(dir, "missing.txt")); + expect(none.size).toBe(0); + expect(retrievalTerms("how do I adjust the lithium dose", none)).toEqual(["how", "adjust", "lithium", "dose"]); + expect(retrievalTerms("go ahead and check the Go build", none)).toEqual(["go", "check", "build"]); +}); + +test("the listed words leave the query; comments and blank lines are ignored", () => { + const words = installGlueWords(list("pt.txt", "# Portuguese glue words\nque\nde\n\npara o\ncomo # how\n")); + expect([...words].sort()).toEqual(["como", "de", "para", "que"]); + expect(retrievalTerms("como ajustar a dose de litio para o paciente que chegou", words)).toEqual(["ajustar", "dose", "litio", "paciente", "chegou"]); +}); + +test("each line goes through the same tokenizer as the query", () => { + const words = installGlueWords(list("tok.txt", "Você\nPOR FAVOR\n")); + const query = "você pode revisar por favor o laudo"; + expect(retrievalTerms(query, words)).toEqual(retrievalTerms(query, new Set()).filter(t => !words.has(t))); + expect(retrievalTerms(query, words)).not.toContain("favor"); +}); + +test("a listed word leaves the query even where QUERY_GLUE keeps it", () => { + expect(retrievalTerms("go ahead and check the Go build", installGlueWords(list("go.txt", "go\n")))).toEqual(["check", "build"]); +}); + +test("a list that cannot be read throws instead of reading as empty", () => { + mkdirSync(join(dir, "a-directory.txt")); + expect(() => installGlueWords(join(dir, "a-directory.txt"))).toThrow(); +}); + +test("an edit to the list is seen by the same process on its next read", () => { + const f = list("live.txt", "que\n"); + expect([...installGlueWords(f)]).toEqual(["que"]); + writeFileSync(f, "que\ncomo\nmais\n"); + expect([...installGlueWords(f)].sort()).toEqual(["como", "mais", "que"]); + rmSync(f); + expect(installGlueWords(f).size).toBe(0); +}); diff --git a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts index c9ef91b878..db13ebf34d 100755 --- a/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts +++ b/LifeOS/install/LIFEOS/TOOLS/MemoryRetriever.ts @@ -415,12 +415,41 @@ function computePrecomputed(note: KnowledgeNote): PrecomputedData { // Conversational glue is not evidence of a memory match. Keep short domain // terms (AI, UI, Go, etc.); an acknowledgement alone needs no new retrieval. const QUERY_GLUE = new Set("a an the and or but if then than to of in on at for from with by as is are was were be been being it its this that these those i me my we us our you your they them their he she his her have has had do does did can could would should will just please yes yeah okay ok thanks thank fix do go ahead continue proceed".split(" ")); -export function retrievalTerms(query: string): string[] { +/** + * Glue words of this install's own language, read from `LIFEOS/USER/CONFIG/glue-words.txt`: one word or phrase per + * line, `#` starts a comment. QUERY_GLUE is English; on an install that writes in another language, that language's + * articles, prepositions and pronouns otherwise outnumber the content words of every prompt and match almost every note. + * Each line goes through tokenize(), so the list always matches what the query side produces, whatever the tokenizer. + * No file means no extra words, and retrieval is exactly as before. Any other read error is thrown, never read as an + * empty list. The cache is keyed by the file's mtime and size, so a long-lived importer (Pulse, the retrieval worker) + * sees an edit on its next query. The index is untouched: the list filters the query only, so INDEX_VERSION does not change. + * public issue #2207 + */ +export const glueWordsPath = (): string => lifeosPath("USER", "CONFIG", "glue-words.txt"); +let installGlueCache: { key: string; words: Set } | null = null; +export function installGlueWords(file: string = glueWordsPath()): Set { + let key = `${file}|absent`; + let text = ""; + try { + const st = fs.statSync(file); + key = `${file}|${st.mtimeMs}|${st.size}`; + if (installGlueCache?.key === key) return installGlueCache.words; + text = fs.readFileSync(file, "utf-8"); + } catch (err) { + if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") throw err; + } + if (installGlueCache?.key === key) return installGlueCache.words; + const words = new Set(text.split("\n").map((l) => l.replace(/#.*/, "").trim()).filter(Boolean).flatMap(tokenize)); + installGlueCache = { key, words }; + return words; +} + +export function retrievalTerms(query: string, installGlue: Set = installGlueWords()): string[] { // Approval-only follow-ups refer to the current conversation, not the archive. // Match the whole utterance so an added topic still reaches retrieval. const approval = query.replace(/[’']/g, "").replace(/[,.!?]/g, " ").replace(/\s+/g, " ").trim(); if (/^(?:(?:well|okay|ok|yeah|yes|please) )*(?:do (?:it|that|whatever you can(?: now)?(?: to fix (?:it|that))?)|go ahead|fix (?:it|that)|continue|proceed)(?: now)?(?: and (?:then )?lets (?:just )?(?:keep observing|wait(?: (?:a little|a bit))?))?$/i.test(approval)) return []; - return [...new Set(tokenize(query))].filter(term => term === "go" || !QUERY_GLUE.has(term)); + return [...new Set(tokenize(query))].filter(term => !installGlue.has(term) && (term === "go" || !QUERY_GLUE.has(term))); } // One compiled matcher per term: scoring runs containsTerm for every term against every