diff --git a/docs/architecture/github-feed.md b/docs/architecture/github-feed.md
new file mode 100644
index 0000000000..bcf5026816
--- /dev/null
+++ b/docs/architecture/github-feed.md
@@ -0,0 +1,162 @@
+# GitHub tracker feed
+
+The GitHub feed maintains a team's tracker record in the team's ClickHouse database.
+Loom's serving processes read that record through the existing
+[read-only tracker source](../features/tracker/record-contract.md). The feed does not
+put issue bodies, comments or pull requests in Loom's deployment tables.
+
+## Ownership and credentials
+
+An operator runs the feed as a bounded CLI job, scheduled by the team's job runner.
+Before reading the registered endpoint or opening the record, the CLI checks its
+actual deployment's storage authority and whether its cell holds the feed organization.
+The external connection repeats that deployment gate; it never treats the record's
+metadata as deployment authority. The job closes both clients and the deployment's
+Postgres pool on exit.
+Initialization creates feed tables and contract views in an empty, dedicated database;
+it refuses existing objects. The operator then registers that database as a tracker
+source with a separate read-only credential. Configuration and sync require that
+registered endpoint and its repository allowlist. A separately supplied writer
+credential can write only the team's feed database; the reader credential is never
+upgraded or reused for writes.
+
+The source credential is a read-only fine-grained GitHub token or App installation
+token restricted to the configured repositories: Metadata, Issues and Pull requests,
+all read-only. Token permissions are operator-attested configuration: a successful
+GET, token prefix or endpoint-permission header cannot certify absence of write grants.
+An unidentified host token is not eligible for a read-only live probe. The token is
+sealed with the existing secret-root mechanism in the team's
+feed configuration table. No token appears in arguments, output or contract views.
+The CLI receives the database writer password through a named environment variable;
+it does not persist that password. GitHub requests use GET on the fixed
+`https://api.github.com` origin, refuse redirects and construct paths from validated
+repository names and numeric item ids. They never follow response URLs.
+
+The team owns the copied record, its backup retention and erasure. To erase copied
+content, the team stops its scheduled writer and removes the affected feed rows,
+comments and checkpoints before restarting with an exclusion or removing the source
+repository. Deleting a Loom namespace retires Loom's derived data and grants; it
+does not delete the team's record. This is the same boundary as any registered tracker
+database. The record-writer credential cannot address Loom's own database; copied
+content is never written there. Only the existing deployment-store lease is local.
+
+## Record and progress
+
+Raw items and comments are versioned in the team's database with the identity tuple,
+process actor and imported source class. Contract views expose the latest item version,
+contract UUIDs and effective visibility. Issue and pull-request comments are
+kept separately and included in the corresponding item's searchable body. Unsupported
+triage fields use the contract's typed defaults; edges are empty until evidence exists.
+Contract `updated_at` is the latest persisted publication timestamp across the item
+and its comments, so late imports and comment edits advance incremental tracker reads.
+Publication versions use whole milliseconds to match tracker cursors. Under the record's
+writer lease, each content batch receives the later of the record clock and one millisecond
+beyond every persisted content version for its organization, across all repositories.
+This ordering also covers namespaces that read multiple repositories through one cursor.
+Repository control writes likewise advance beyond the persisted version for their exact
+organization and repository, even when the receiving clock moves backward. Visibility
+revocation, credential rotation and cursor checkpoints therefore replace the prior control
+row rather than losing to its timestamp.
+Raw upstream timestamps remain separate and drive API cursors. No model or embedding
+call is required. Repository identities retain the registered allowlist's spelling,
+including when the API returns different capitalization.
+
+Owner, repository revision and publication watermark reads request
+`select_sequential_consistency = 1`; the writer lease alone does not make a replica
+current. Feed writers require SharedMergeTree storage or a single self-hosted server,
+following the [read-your-writes guarantees](clickhouse.md#read-your-writes).
+The setting alone does not cover acknowledged writes on self-hosted replicated tables.
+Writer validation accepts only `ReplacingMergeTree` and `SharedReplacingMergeTree`
+for the five feed tables in the connected record database. Replicated, distributed
+and unrecognized table engines are refused before the job can publish.
+
+Each repository has independent item, issue-comment and review-comment cursors. The
+[issues endpoint](https://docs.github.com/en/rest/issues/issues#list-repository-issues)
+lists issues and pull requests together; pull requests are hydrated through their
+numeric endpoint. The
+[issue comments endpoint](https://docs.github.com/en/rest/issues/comments#list-issue-comments-for-a-repository)
+and [review comments endpoint](https://docs.github.com/en/rest/pulls/comments#list-review-comments-in-a-repository)
+have separate updated-time scans. Each uses ascending `updated_at`, bounded pages,
+response bytes and request time. Cursor advancement follows acknowledged writes only,
+through a complete timestamp boundary. Requests lower the saved timestamp by one second
+because `since` is exclusive. A full page budget saves the earliest timestamp on its
+final page; the next turn replays that boundary. The budget is 2 to 100 pages: a preceding
+page permits progress while the final page retains an incomplete timestamp boundary.
+If one timestamp fills the entire configured page budget,
+the job reports a capacity error and keeps the checkpoint rather than skipping rows or
+claiming progress. The operator can raise the bounded page budget for that repository.
+Replayed rows replace the same identity. The API is a changing collection, not a snapshot;
+overlapping scans reduce pagination races without claiming transactional upstream reads.
+
+Every turn verifies repository identity and visibility before copying content. An
+unavailable, moved or mismatched repository first hides that repository's contract rows
+and then refuses the turn. Contract views also require a recent successful visibility
+observation, expiring after a bounded interval if the scheduler stops. Each item and
+comment publication stores the observation's privacy classification in the same row as
+its content. Views use the most restrictive classification from the repository, item
+and stored comments for both `is_private` and the contract UUID. A replica that learns
+new content before repository metadata therefore cannot label a private publication
+public. The standard tracker's existing predicate excludes those rows. Restricted
+namespaces retain their live direct-grant check. Visibility is a polling observation,
+not an instant permission mirror.
+A validated identity mismatch or private observation can hide the current record even
+when another turn changed its revision; this preserves current credentials and cursors.
+Generic failures from an older credential retain the revision check. Stale public
+observations cannot widen access.
+
+When a repository becomes public, configuration or sync resets its existing stream
+cursors before publishing public metadata. Bounded scans replay the items and comments
+and stamp each with the public observation. An item remains restricted until its own
+row and its stored comments have been replayed. Repeated scheduled turns continue a
+backlog under the same page and timestamp-boundary limits. Deleted upstream rows cannot
+be replayed; the team must erase those retained copies to clear their private
+classification.
+
+Incremental API lists do not report deleted issues or comments. The team handles those
+erasures explicitly; the feed does not claim deletion detection. Configuration, sealed
+credentials and cursors stay with the team's record, so CLI restarts do not depend on
+local files. Initialization binds the record to one Loom deployment's database UUID;
+configuration and sync refuse any other deployment. Concurrent jobs in that deployment
+use one publication lease keyed by the record database UUID, independent of source
+aliases and repository renames. Provider fetches happen outside that lease; initialization,
+configuration and publication share it so concurrent first-time identity observations
+cannot bypass serialization. An uncertain database write retains its claim through the bounded
+server send window. Record erasure requires the team to stop all writers first.
+
+## Validation and rollout
+
+Fakes cover pagination ties, partial responses, rate limits, retries, visibility changes,
+redirects and credential redaction. A private ClickHouse verifies the contract, current
+views, replay and interrupted cursor publication. A read-only token, when available,
+checks a bounded public-repository slice; a write-capable CLI token is never substituted.
+The feed adds no Loom wire operation or automatic worker. Existing deployments and
+registered trackers are unchanged until an operator initializes and schedules a feed.
+
+## Source entry points
+
+- [CLI](../../server/cli/tracker_feed.py): `register_tracker_feed` and `_run` select the
+ owned record and its separate writer credential; `_validate_writer` checks grants
+ and supported table engines.
+- [Feed application](../../server/tracker/app/github_feed.py): `initialize`, `configure`,
+ `sync` and `_publish` enforce ownership, stamp publication privacy and fence writes.
+- [Record store](../../server/tracker/store/github_feed.py): `statements` defines the
+ contract views; `repository`, `put_repository` and `put_items` read publication
+ state consistently.
+- [API client](../../server/clients/github_feed.py): `pages` bounds upstream scans;
+ [domain rules](../../server/tracker/domain/github_feed.py) validate identities and
+ choose replay boundaries through `next_cursor`.
+
+## Invariants
+
+- Copied content stays in the team's registered record, with team-owned erasure.
+- Repository identity, visibility and revision checks precede content publication;
+ control reads include acknowledged record writes on the supported storage topology.
+- Content and its privacy classification share one raw row version. Contract privacy
+ and UUIDs include every stored component's classification, even when tables lag
+ independently.
+- A private-to-public transition resets cursors before public publication, so retries
+ replay retained content without dropping its private classification prematurely.
+- The record-wide writer lease covers publication and checkpoint writes. An uncertain
+ send retains its claim until it settles; a cursor advances only after acknowledged writes.
+- Whole-millisecond content versions increase across every repository in the record,
+ so combined item/comment bodies remain visible to existing tracker cursors.
diff --git a/docs/features/tracker/github-feed.md b/docs/features/tracker/github-feed.md
new file mode 100644
index 0000000000..81fe53135d
--- /dev/null
+++ b/docs/features/tracker/github-feed.md
@@ -0,0 +1,118 @@
+# Maintain a GitHub tracker record
+
+`loom tracker-feed` copies repository issues, pull requests, issue comments and inline
+review comments into a team's dedicated ClickHouse database. Loom reads the resulting
+views through a [read-only tracker source](record-contract.md). The team owns the copied
+content, backups and erasure. The job makes no model or embedding calls.
+
+## Prepare the record
+
+Use an empty database with a stable database UUID, separate from the Loom deployment
+database. Give a dedicated writer permission to create tables and views and to read and
+insert rows in that database. Keep its credentials separate from the tracker reader.
+
+```sql
+CREATE DATABASE team_feed;
+CREATE USER feed_writer IDENTIFIED BY '…';
+GRANT CREATE TABLE, CREATE VIEW, SELECT, INSERT ON team_feed.* TO feed_writer;
+GRANT SELECT ON system.columns TO feed_writer;
+GRANT SELECT ON system.databases TO feed_writer;
+GRANT SELECT ON system.tables TO feed_writer;
+CREATE USER feed_reader IDENTIFIED BY '…' SETTINGS readonly = 2;
+GRANT SELECT ON system.columns TO feed_reader;
+```
+
+Supply the writer password as `LOOM_TRACKER_FEED_WRITER_PASSWORD` through the scheduler's
+secret facility. Run the CLI with the same Loom deployment configuration used by the
+serving process, including its ClickHouse connection and secret root.
+Set `LOOM_TRACKER_ORG` to the organization being initialized. The CLI verifies direct
+writer grants before use and refuses roles, grant options and authority outside this
+database, apart from those three system metadata tables.
+
+```bash
+loom tracker-feed init --org acme --host record.example --database team_feed \
+ --writer-user feed_writer
+```
+
+Initialization creates `github_issues`, `github_prs` and `edges` as definer views.
+Grant the reader access only to those views:
+
+```sql
+GRANT SELECT ON team_feed.github_issues TO feed_reader;
+GRANT SELECT ON team_feed.github_prs TO feed_reader;
+GRANT SELECT ON team_feed.edges TO feed_reader;
+```
+
+[Register the tracker source](record-contract.md#adding-a-tracker-source) in views mode
+using `feed_reader`. Include each repository in the source's namespace allowlist and
+use restricted namespaces for private records. Set the deployment's `LOOM_TRACKER_ORG`
+to the same organization. Source registration keeps its existing host, credential and
+contract checks. HTTP and private addresses require the development private-host
+setting; production connections use HTTPS.
+
+## Configure and schedule a repository
+
+Create a read-only fine-grained token or App installation token restricted to the
+repositories being copied. It needs Metadata, Issues and Pull requests read permissions.
+`--read-only-token` is the operator's permission attestation; the CLI cannot certify the
+absence of write grants from a successful request or token prefix.
+
+Supply that token as `LOOM_TRACKER_FEED_GITHUB_TOKEN` through the secret facility, then
+configure one allowlisted repository:
+
+```bash
+loom tracker-feed configure --org acme --source widgets --repo acme/widgets \
+ --writer-user feed_writer --read-only-token --max-pages 5
+loom tracker-feed sync --org acme --source widgets --repo acme/widgets \
+ --writer-user feed_writer
+```
+
+Configuration seals the token in the team's feed database. Sync needs the writer
+password and deployment secret root; it does not need the token environment variable.
+Copied repository identities keep the spelling in the registered source's allowlist,
+so namespace filters match even if API capitalization differs.
+App installation tokens [expire after one hour](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation).
+An external secret process must refresh the installation token and rerun `configure`
+before expiry. The feed does not issue or refresh installation tokens.
+Use `--writer-password-env` or `--token-env` to select another credential variable.
+During a root-secret rotation, successful metadata writes reseal credentials opened by
+`LOOM_JWT_SECRET_PREVIOUS`. Complete a successful turn before retiring the previous root.
+
+Schedule `sync` every five minutes or more frequently. Visibility observations expire
+after 15 minutes, hiding the repository if checks stop succeeding. Each job has a
+120-second limit and reads at most `--max-pages` pages per stream, with 100 items per
+page and a budget of 2 to 100 pages. The final page's timestamp boundary is replayed;
+the minimum leaves a preceding page from which to make progress.
+A capacity error leaves the cursor unchanged; raise
+the bounded page budget or shorten the repository's backlog before retrying. Counts
+on standard output report processed rows, including replayed rows.
+
+## Visibility and erasure
+
+Every stream verifies the repository's immutable identity and current visibility.
+Failed checks hide its views. Each item and comment keeps the observation's privacy
+classification with its content. The views use the most restrictive classification
+from the repository, item and stored comments, including when those tables reach a
+replica at different times. This is a polling observation with a bounded expiry,
+rather than an instant permission mirror. Normal tracker privacy and direct grant
+checks still apply.
+
+When a repository becomes public, configuration or sync resets its cursors and
+scheduled turns replay all streams within the same page budget. An item becomes public
+after its row and stored comments have been refreshed under public observations.
+Deleted upstream rows require erasure of the retained copies; replay cannot refresh them.
+
+Bodies include up to 32,768 item characters and the 64 most recently updated comments,
+with up to 2,048 characters per comment. Review requests and test evidence remain
+unknown when the source does not provide that evidence.
+The views' `updated_at` tracks item and comment publication in the record, so a comment
+imported after its item remains visible to incremental tracker reads.
+
+Incremental lists do not report deletions. To erase copied content, stop every scheduled
+writer, erase the affected raw rows, comments and repository configuration or cursors,
+and exclude the repository before restarting. Dropping a Loom namespace removes Loom's
+derived data; it does not erase the team's database or backups. Initialization binds a
+record to one deployment and organization. Use a separate record for another deployment.
+
+See the [feed architecture](../../architecture/github-feed.md) for cursor boundaries,
+concurrency and ownership.
diff --git a/docs/features/tracker/overview.md b/docs/features/tracker/overview.md
index 657b33b56c..c43c9c14bb 100644
--- a/docs/features/tracker/overview.md
+++ b/docs/features/tracker/overview.md
@@ -340,6 +340,7 @@ Thresholds, legs, judge, rerank, bisect scope and test-signal settings are in th
## Related
+- [GitHub feed](github-feed.md): maintain a team-owned record with a scheduled CLI job
- [Record contract](record-contract.md): tables, columns, aliases, mapping mode, sources
- [History and PRs](../code-index/history-and-prs.md): the code-plane ops that read a tracker
- [Adding repositories](../code-index/repositories.md)
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 72ae7ed997..ff52c7d0fe 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -334,6 +334,10 @@ python scripts/codegen/docs_reference.py
| `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` |
| `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` |
| `loom tracker-embed-reset` | Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. | `--namespace`, `--org`, `--kind`, `--apply`, `--dry-run` |
+| `loom tracker-feed` | Maintain a team's GitHub tracker record | |
+| `loom tracker-feed init` | | `--org`, `--writer-user`, `--writer-password-env`, `--host`, `--port`, `--database`, `--insecure` |
+| `loom tracker-feed configure` | | `--org`, `--writer-user`, `--writer-password-env`, `--source`, `--repo`, `--token-env`, `--read-only-token`, `--max-pages` |
+| `loom tracker-feed sync` | | `--org`, `--writer-user`, `--writer-password-env`, `--source`, `--repo` |
| `loom namespaces-adopt` | Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. | `--org`, `--owner`, `--prefix`, `--namespace`, `--kind`, `--apply`, `--dry-run`, `--show` |
| `loom sensitivity-sweep` | Label live rows the sensitivity detector would have labelled on write but a path skipped. | `--org`, `--namespace`, `--since-days`, `--limit`, `--show`, `--only-shape`, `--apply`, `--dry-run` |
| `loom integrity-repair` | Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search… | `--org`, `--namespace`, `--limit`, `--show`, `--apply`, `--dry-run` |
diff --git a/frontend/src/views/RecallLab.table.test.tsx b/frontend/src/views/RecallLab.table.test.tsx
index d7383be8fb..f2391d721e 100644
--- a/frontend/src/views/RecallLab.table.test.tsx
+++ b/frontend/src/views/RecallLab.table.test.tsx
@@ -106,6 +106,26 @@ describe("Recall Lab: the scores table", () => {
expect(screen.queryByRole("button", { name: "deploy_note" })).toBeNull();
});
+ it("keeps settled scores out of draft query edits and clears them with the query", async () => {
+ const result = hit("deploy_note", 0.9);
+ const readTerms = vi.fn(() => ({ semantic: 0.6, entity_ann_boost_effective: 0.06 }));
+ Object.defineProperty(result, "score_components", { get: readTerms });
+ searchMemory.mockResolvedValue({ results: [result], embedding_model: "m" });
+ render();
+ const table = await score();
+ expect(readTerms).toHaveBeenCalled();
+ readTerms.mockClear();
+
+ fireEvent.change(screen.getByLabelText("Recall query"), { target: { value: "deploy lane" } });
+ expect(screen.getByLabelText("Recall query")).toHaveValue("deploy lane");
+ expect(within(table).getByText("excerpt of deploy_note")).toBeInTheDocument();
+ expect(readTerms).not.toHaveBeenCalled();
+ expect(searchMemory).toHaveBeenCalledTimes(1);
+
+ fireEvent.change(screen.getByLabelText("Recall query"), { target: { value: "" } });
+ expect(screen.queryByRole("table", { name: "Recall scores" })).toBeNull();
+ });
+
it("draws the score as Click UI's table bar, relative to the top score", async () => {
searchMemory.mockResolvedValue({
results: [hit("deploy_note", 0.8), hit("rollback_runbook", 0.4)],
diff --git a/frontend/src/views/RecallLab.tsx b/frontend/src/views/RecallLab.tsx
index 3398a2b133..03a6f977eb 100644
--- a/frontend/src/views/RecallLab.tsx
+++ b/frontend/src/views/RecallLab.tsx
@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
import { Badge } from "@clickhouse/click-ui/Badge";
import { Button } from "@clickhouse/click-ui/Button";
import { Icon } from "@clickhouse/click-ui/Icon";
@@ -509,7 +509,21 @@ export function RecallLab() {
}, [query, identity, attempt]);
const showResults = !abstain || showWeak;
- const topScore = Math.max(0, ...(rows ?? []).map((row) => row.hit?.rank_score ?? 0));
+ const scoresTable = useMemo(() => {
+ if (rows == null || rows.length === 0 || !showResults) return null;
+ const topScore = Math.max(0, ...rows.map((row) => row.hit?.rank_score ?? 0));
+ return (
+
scoreRow(row, i, topScore, open, rowsQuery))}
+ />
+ );
+ }, [rows, rowsQuery, open, loading, showResults]);
const count =
rows != null && rows.length > 0 && showResults && error == null
? `${rows.length} row${rows.length === 1 ? "" : "s"}${model ? ` · ${model}` : ""}`
@@ -593,17 +607,7 @@ export function RecallLab() {
)}
)}
- {showResults && (
- scoreRow(row, i, topScore, open, rowsQuery))}
- />
- )}
+ {scoresTable}
>
)}
diff --git a/llms.txt b/llms.txt
index 1115e0a18a..ed0d423d8f 100644
--- a/llms.txt
+++ b/llms.txt
@@ -82,6 +82,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads
- [Cost and usage](docs/features/observability/cost-and-usage.md): Loom records every upstream dollar it spends on embedding and LLM calls against the operation that spent it, in the per-operation ledger `loom.agent_events`.
- [Telemetry](docs/features/observability/telemetry.md): Loom reports on itself in three layers:
- [Issue duplicate checks and regression candidates for agents](docs/features/tracker/overview.md): With `LOOM_TRACKER_ORG` set and a registered source, Loom lets agents check whether a bug is already reported and rank changes that may have introduced a regression, using tracker records and indexed PR history.
+- [Maintain a GitHub tracker record](docs/features/tracker/github-feed.md): `loom tracker-feed` copies repository issues, pull requests, issue comments and inline review comments into a team's dedicated ClickHouse database.
- [Tracker record contract](docs/features/tracker/record-contract.md): The `tracker.*` ops, and the tracker legs of `code.*` and `loom_search`, read a team's issues, pull requests and incidents live from the team's own ClickHouse with a read-only credential.
## Guides
@@ -153,6 +154,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads
- [Document sources](docs/architecture/document-sources.md): A document source pulls files from a document store (a shared-folder store, a page store, a project tracker, a meeting store) into a namespace's documents and keeps them in step, on any shared namespace or collection.
- [Documentation translation requests](docs/architecture/documentation-translation.md): The translation generator splits prepared English text into bounded requests, then checks the assembled translation before writing it.
- [Failure and recovery](docs/architecture/failure-and-recovery.md): The primary write and every access decision fail closed.
+- [GitHub tracker feed](docs/architecture/github-feed.md): The GitHub feed maintains a team's tracker record in the team's ClickHouse database.
- [Google ADK memory service](docs/architecture/google-adk-adapter.md): The Google ADK adapter implements the framework's memory service through Loom's Python and TypeScript SDKs.
- [Haystack document store and retriever](docs/architecture/haystack-adapter.md): The optional Haystack 3.5 adapter implements its document-store protocol and a native retrieval component through the synchronous Loom SDK.
- [Knowledge gaps](docs/architecture/knowledge-gaps.md): Loom records the searches that found nothing a caller could use and groups them, once a day, into knowledge gaps: the questions several people asked that the namespace's memory and documents do not answer.
diff --git a/server/cli/_ch.py b/server/cli/_ch.py
index 042d8d1de0..09b4e99bc7 100644
--- a/server/cli/_ch.py
+++ b/server/cli/_ch.py
@@ -27,7 +27,7 @@
from typing import TYPE_CHECKING, Literal
if TYPE_CHECKING:
- from server.core.ch import CHClient
+ from server.core.ch import CHClient, ExternalCHClient
class OrgNotHeld(SystemExit):
@@ -324,6 +324,17 @@ async def connect_checked(
await connect_declared(ch, orgs=orgs, org_free=org_free, check_authority=True)
+async def connect_external(
+ ch: ExternalCHClient, *, deployment: CHClient, orgs: Iterable[str | None]
+) -> None:
+ """Gate the deployment and its orgs before connecting a separate record database.
+
+ The external record does not carry the deployment's authority metadata.
+ """
+ await connect_checked(deployment, orgs=orgs)
+ await ch.connect()
+
+
async def disconnect(ch: CHClient) -> None:
"""Close the Postgres pool a command opened, then `ch`."""
from server.core import pg
diff --git a/server/cli/app.py b/server/cli/app.py
index 111da595fe..e82bab470f 100644
--- a/server/cli/app.py
+++ b/server/cli/app.py
@@ -49,6 +49,7 @@
register_tracker_embed_reset,
register_tracker_probe,
)
+from server.cli.tracker_feed import register_tracker_feed
from server.core.env_file import env_file_path
from server.wiring import wire
@@ -84,6 +85,7 @@ def _load_dotenv_like_the_container() -> None:
register_tracker_contract_check,
register_tracker_dupcheck_fit,
register_tracker_embed_reset,
+ register_tracker_feed,
)
diff --git a/server/cli/tracker_feed.py b/server/cli/tracker_feed.py
new file mode 100644
index 0000000000..5b6e393154
--- /dev/null
+++ b/server/cli/tracker_feed.py
@@ -0,0 +1,205 @@
+"""Operator-scheduled import into a team's dedicated tracker database."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from typing import Any
+
+from server.core.github_values import FeedInputError
+
+
+def _environment(name: str) -> str:
+ value = os.environ.get(name, "")
+ if not value:
+ raise FeedInputError(f"required credential environment variable is empty: {name}")
+ return value
+
+
+async def _registered(args: argparse.Namespace) -> tuple[Any, set[str], str]:
+ from server.settings import settings
+ from server.tracker.app import registry_sources
+ from server.tracker.domain import sources
+
+ if not settings.tracker_org or args.org != settings.tracker_org:
+ raise FeedInputError("feed organization must match the configured tracker organization")
+ # This is a new CLI process: an unsuccessful refresh cannot inherit a
+ # previously cached registered source.
+ await registry_sources.refresh_db_sources(force=True)
+ endpoint = sources.endpoint(args.source)
+ if endpoint is None or endpoint.mapping_json:
+ raise FeedInputError("feed requires a registered source exposing the contract views")
+ reader, _password = sources.credentials_for(args.source)
+ if not reader or args.writer_user == reader:
+ raise FeedInputError("feed writer and registered reader must be separate credentials")
+ repos = {
+ repo
+ for entry in sources.namespace_entries().values()
+ if entry.source == args.source
+ for repo in entry.repos
+ }
+ if not repos:
+ raise FeedInputError("registered source has no repository allowlist")
+ return endpoint, repos, reader
+
+
+async def _connection(args: argparse.Namespace, endpoint: Any, deployment: Any) -> Any:
+ from server.cli._ch import connect_external
+ from server.core.ch import ExternalCHClient
+ from server.core.hosts import vet_host
+ from server.settings import settings
+
+ why, address = await vet_host(
+ endpoint.host,
+ allow_private=bool(settings.tracker_self_service_allow_private_hosts),
+ )
+ if why:
+ raise FeedInputError("record database host refused")
+ if not endpoint.database or endpoint.database in {
+ "system",
+ "information_schema",
+ "INFORMATION_SCHEMA",
+ }:
+ raise FeedInputError("a dedicated record database is required")
+ if not endpoint.secure and not settings.tracker_self_service_allow_private_hosts:
+ raise FeedInputError("HTTP requires the development private-host setting")
+ client = ExternalCHClient(
+ host=endpoint.host,
+ address=address,
+ port=endpoint.port,
+ secure=endpoint.secure,
+ database=endpoint.database,
+ username=args.writer_user,
+ password=_environment(args.writer_password_env),
+ )
+ try:
+ await connect_external(client, deployment=deployment, orgs=(args.org,))
+ await _validate_writer(client, endpoint.database)
+ except BaseException:
+ await client.disconnect()
+ raise
+ return client
+
+
+async def _validate_writer(client: Any, database: str) -> None:
+ from server.tracker.domain.github_feed import writer_grants
+ from server.tracker.store import github_feed as store
+
+ for statement in ("SHOW GRANTS", "SHOW GRANTS FINAL"):
+ rows = await client.query_all(statement)
+ writer_grants([str(next(iter(row.values()))) for row in rows], database)
+ if any(
+ row["engine"] not in {"ReplacingMergeTree", "SharedReplacingMergeTree"}
+ for row in await store.table_engines(client)
+ ):
+ raise FeedInputError("unsupported feed record table engine; use a dedicated feed record")
+
+
+async def _run(args: argparse.Namespace) -> dict[str, Any]:
+ from types import SimpleNamespace
+
+ from server.cli._ch import connect_checked, disconnect
+ from server.core.ch import get_client, set_query_tag
+ from server.settings import settings
+ from server.tracker.app import github_feed as feed
+ from server.tracker.store import github_feed as store
+
+ if not settings.tracker_org or args.org != settings.tracker_org:
+ raise FeedInputError("feed organization must match the configured tracker organization")
+ set_query_tag(f"cli.tracker-feed.{args.feed_command}")
+ deployment = get_client()
+ record = None
+ try:
+ await connect_checked(deployment, orgs=(args.org,))
+ owner = await store.database_uuid(deployment)
+ allowed: set[str] = set()
+ if args.feed_command == "init":
+ endpoint = SimpleNamespace(
+ host=args.host,
+ port=args.port,
+ secure=not args.insecure,
+ database=args.database,
+ )
+ else:
+ endpoint, allowed, _reader = await _registered(args)
+ record = await _connection(args, endpoint, deployment)
+ if args.feed_command == "init":
+ await feed.initialize(record, args.org, owner)
+ return {"initialized": True}
+ if args.feed_command == "configure":
+ if not args.read_only_token:
+ raise FeedInputError("token permission attestation requires --read-only-token")
+ await feed.configure(
+ record,
+ org=args.org,
+ owner_uuid=owner,
+ repo=args.repo,
+ token=_environment(args.token_env),
+ allowed_repos=allowed,
+ max_pages=args.max_pages,
+ )
+ return {"configured": True}
+ return await feed.sync(
+ record,
+ org=args.org,
+ owner_uuid=owner,
+ repo=args.repo,
+ allowed_repos=allowed,
+ )
+ finally:
+ try:
+ if record is not None:
+ await record.disconnect()
+ finally:
+ await disconnect(deployment)
+
+
+def _command(args: argparse.Namespace) -> int:
+ try:
+ result = asyncio.run(_run(args))
+ except Exception as exc: # noqa: BLE001 — operator output must not quote provider or DB content
+ from server.tracker.domain.github_feed import FeedCapacityError, FeedInputError
+
+ detail = (
+ str(exc) if isinstance(exc, (FeedInputError, FeedCapacityError)) else type(exc).__name__
+ )
+ print(f"tracker feed refused: {detail}", file=sys.stderr)
+ return 1
+ print(json.dumps(result, sort_keys=True))
+ return 0
+
+
+def register_tracker_feed(sub: Any) -> None:
+ parser = sub.add_parser("tracker-feed", help="Maintain a team's GitHub tracker record")
+ commands = parser.add_subparsers(dest="feed_command", required=True)
+ for name in ("init", "configure", "sync"):
+ command = commands.add_parser(name)
+ command.add_argument("--org", required=True)
+ command.add_argument("--writer-user", required=True)
+ command.add_argument("--writer-password-env", default="LOOM_TRACKER_FEED_WRITER_PASSWORD")
+ command.set_defaults(func=_command)
+ if name == "init":
+ command.add_argument("--host", required=True)
+ command.add_argument("--port", type=int, default=8443)
+ command.add_argument("--database", required=True)
+ command.add_argument(
+ "--insecure", action="store_true", help="Use HTTP for a private development record"
+ )
+ else:
+ command.add_argument("--source", required=True)
+ command.add_argument(
+ "--repo", required=True, help="A repository in the source's allowlist"
+ )
+ if name == "configure":
+ command.add_argument("--token-env", default="LOOM_TRACKER_FEED_GITHUB_TOKEN")
+ command.add_argument(
+ "--read-only-token",
+ action="store_true",
+ help="Attest the token has only read permissions",
+ )
+ command.add_argument(
+ "--max-pages", type=int, default=5, help="Pages per stream, from 2 to 100"
+ )
diff --git a/server/clients/github_feed.py b/server/clients/github_feed.py
new file mode 100644
index 0000000000..0c1dbf6f91
--- /dev/null
+++ b/server/clients/github_feed.py
@@ -0,0 +1,93 @@
+"""Bounded read-only API calls for a team's tracker feed."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+import httpx
+
+from server.clients._http import get_client
+from server.clients._vendor import read_capped
+from server.clients.github import _headers
+from server.core.github_values import PAGE_SIZE, positive_id, repository, since_parameter
+
+API = "https://api.github.com"
+PAGE_MAX_BYTES = 4 * 1024 * 1024 # not a setting: fixed memory ceiling for one CLI response
+SCAN_MAX_BYTES = 24 * 1024 * 1024 # not a setting: fixed memory ceiling for one CLI scan
+STREAMS = {"items": "issues", "comments": "issues/comments", "review_comments": "pulls/comments"}
+
+
+class FeedProviderError(RuntimeError):
+ """A provider response failed without exposing its content."""
+
+
+@dataclass(frozen=True)
+class Batch:
+ items: list[dict[str, Any]]
+ complete: bool
+
+
+async def _get(path: str, token: str, params: dict[str, Any] | None = None) -> tuple[Any, int]:
+ try:
+ # not a setting: fixed per-request ceiling inside the operator CLI turn
+ async with asyncio.timeout(15):
+ async with get_client().stream(
+ "GET", API + path, headers=_headers(token), params=params, follow_redirects=False
+ ) as response:
+ if response.status_code != 200:
+ raise FeedProviderError(f"GitHub feed http_{response.status_code}")
+ body = await read_capped(response, PAGE_MAX_BYTES)
+ if body is None:
+ raise FeedProviderError("GitHub feed response exceeds the size bound")
+ return json.loads(body), len(body)
+ except (httpx.RequestError, TimeoutError, ValueError) as exc:
+ raise FeedProviderError(f"GitHub feed {type(exc).__name__}") from None
+
+
+async def metadata(repo: str, token: str) -> dict[str, Any]:
+ data, _ = await _get(f"/repos/{repository(repo)}", token)
+ if not isinstance(data, dict):
+ raise FeedProviderError("GitHub feed expected repository metadata")
+ return data
+
+
+async def pull_request(repo: str, token: str, number: int) -> dict[str, Any]:
+ number = positive_id(number)
+ data, _ = await _get(f"/repos/{repository(repo)}/pulls/{number}", token)
+ if not isinstance(data, dict) or data.get("number") != number:
+ raise FeedProviderError("GitHub feed mismatched pull request")
+ return data
+
+
+async def pages(
+ repo: str, token: str, stream: str, since: datetime | None, *, max_pages: int
+) -> Batch:
+ if stream not in STREAMS or not 2 <= max_pages <= 100:
+ raise FeedProviderError("GitHub feed requires a known stream and 2..100 max-pages")
+ path = f"/repos/{repository(repo)}/{STREAMS[stream]}"
+ params: dict[str, Any] = dict(sort="updated", direction="asc", per_page=PAGE_SIZE)
+ if stream == "items":
+ params["state"] = "all"
+ if since is not None:
+ params["since"] = since_parameter(since)
+ out: list[dict[str, Any]] = []
+ size = 0
+ for page in range(1, max_pages + 1):
+ data, count = await _get(path, token, {**params, "page": page})
+ size += count
+ if size > SCAN_MAX_BYTES:
+ raise FeedProviderError("GitHub feed scan exceeds the size bound")
+ if not isinstance(data, list) or any(not isinstance(item, dict) for item in data):
+ raise FeedProviderError("GitHub feed expected an object list")
+ if len(data) > PAGE_SIZE:
+ raise FeedProviderError("GitHub feed page exceeds its item bound")
+ out.extend(data)
+ if len(data) < PAGE_SIZE:
+ return Batch(out, complete=True)
+ if page < max_pages:
+ await asyncio.sleep(0.7)
+ return Batch(out, complete=False)
diff --git a/server/core/ch.py b/server/core/ch.py
index 7a696e221b..7c1e308460 100644
--- a/server/core/ch.py
+++ b/server/core/ch.py
@@ -1584,10 +1584,11 @@ async def _insert_rows(
settings = dedup_settings(token, settings)
from server.core import rmt_carry
+ send_clock = None
if rmt_carry.sent_under_lease() and not rmt_carry.carries(table):
- # Its start deadline is on the server's clock, which a carried insert's
- # admission measures: measured here, before the send holds a slot.
- await rmt_carry.measure_server_clock(self)
+ # An external record can have a different server clock from the
+ # deployment. Measure its receiver before the send holds a slot.
+ send_clock = await rmt_carry.measure_send_clock(self)
async def _q() -> None:
attempt = query_settings(settings)
@@ -1616,7 +1617,11 @@ async def _q() -> None:
fenced["query_id"] = rmt_carry.carry_query_id(
str(fenced.get("query_id") or uuid.uuid4().hex)
)
- deadline = rmt_carry.send_deadline()
+ deadline = (
+ rmt_carry.clock_send_deadline(send_clock)
+ if send_clock is not None
+ else rmt_carry.send_deadline()
+ )
rmt_carry.note_send(
str(fenced["query_id"]), deadline, float(fenced["max_execution_time"])
)
@@ -1789,6 +1794,47 @@ def _validate_required_insert_columns(
_PROJECTION_WARNED: set[tuple[str, str]] = set()
+class ExternalCHClient(CHClient):
+ """The statement wrapper for an operator's separately owned record database."""
+
+ def __init__(
+ self,
+ *,
+ host: str,
+ port: int,
+ secure: bool,
+ database: str,
+ username: str,
+ password: str,
+ address: str = "",
+ ) -> None:
+ super().__init__()
+ self._connection: dict[str, Any] = dict(
+ host=address or host,
+ server_host_name=host if address else None,
+ port=port,
+ secure=secure,
+ database=database,
+ username=username,
+ password=password,
+ )
+ self._slots = asyncio.Semaphore(2)
+
+ async def connect(self) -> None:
+ # A record database has no Loom schema or vector-capability probes.
+ client = await clickhouse_connect.get_async_client(
+ **self._connection,
+ compress=True,
+ tz_source="server",
+ send_receive_timeout=_driver_read_timeout_s(),
+ connector_limit=2,
+ connector_limit_per_host=2,
+ settings={"session_timezone": "UTC", "async_insert": 0, "wait_for_async_insert": 1},
+ )
+ client.server_tz = UTC
+ self._client = client
+
+
def describe_cache_clear() -> None:
"""Forget every cached column type. A test hook: nothing in the server
calls it, so a column added under a running process reaches that table's
diff --git a/server/core/github_values.py b/server/core/github_values.py
new file mode 100644
index 0000000000..ac37e6b6fc
--- /dev/null
+++ b/server/core/github_values.py
@@ -0,0 +1,28 @@
+"""Pure repository paths and timestamp bounds shared by GitHub readers."""
+
+import re
+from datetime import datetime, timedelta
+
+PAGE_SIZE = 100 # not a setting: the provider's per_page ceiling defines a full page
+
+
+class FeedInputError(ValueError):
+ """A source value cannot be represented without guessing its identity."""
+
+
+def positive_id(value: object) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or not 0 < value < 2**64:
+ raise FeedInputError("invalid upstream number or id")
+ return value
+
+
+def repository(value: str) -> str:
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,99}/[A-Za-z0-9_.-]{1,100}", value):
+ raise FeedInputError("repository must be owner/name")
+ if value.split("/")[1] in {".", ".."}:
+ raise FeedInputError("repository must be owner/name")
+ return value
+
+
+def since_parameter(value: datetime | None) -> str | None:
+ return (value - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ") if value else None
diff --git a/server/core/rmt_carry.py b/server/core/rmt_carry.py
index 31d0234f58..0e3fdd0ebf 100644
--- a/server/core/rmt_carry.py
+++ b/server/core/rmt_carry.py
@@ -409,6 +409,26 @@ def send_deadline() -> float:
now = server_now()
if now is None:
raise BackendUnavailableError("rmt carry: the server's clock is not measured yet; retry")
+ return _deadline_at(now)
+
+
+async def measure_send_clock(ch: Any) -> tuple[float, float]:
+ """A receiving client's clock, independent of the deployment's carry watcher."""
+ row = await ch.query_one("SELECT toUnixTimestamp64Milli(now64(3)) AS ms")
+ back = time.monotonic()
+ if row is None:
+ raise BackendUnavailableError("leased write: the target's clock cannot be read; retry")
+ # The sample precedes receipt; this anchor cannot overstate the receiver's clock.
+ return int(row["ms"]) / 1000, back
+
+
+def clock_send_deadline(anchor: tuple[float, float]) -> float:
+ """Advance an insert's receiving-server clock to this send's monotonic time."""
+ server_s, measured = anchor
+ return _deadline_at(server_s + time.monotonic() - measured)
+
+
+def _deadline_at(now: float) -> float:
lease = _lease_end.get()
start = now if lease is None else min(now, now + lease - time.monotonic())
return start + _TRANSIT_S
diff --git a/server/memory/domain/provenance.py b/server/memory/domain/provenance.py
index 4d89748d91..28197c585d 100644
--- a/server/memory/domain/provenance.py
+++ b/server/memory/domain/provenance.py
@@ -120,6 +120,7 @@ def cli_actor(command: str) -> str:
"process:session_brief",
"process:source-push",
"process:task_lease_sweep",
+ "process:tracker-github-feed",
}
)
diff --git a/server/tracker/app/github_feed.py b/server/tracker/app/github_feed.py
new file mode 100644
index 0000000000..56503e93dc
--- /dev/null
+++ b/server/tracker/app/github_feed.py
@@ -0,0 +1,310 @@
+"""An operator's bounded import into a separately owned tracker record."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any
+
+from server.clients import github_feed as provider
+from server.core.lease_writes import fenced
+from server.core.repo_base import _CHClient
+from server.core.secrets_at_rest import resealed, seal, unseal
+from server.memory.app.provenance import writer_actor
+from server.tracker.domain import github_feed as domain
+from server.tracker.store import github_feed as store
+
+VISIBILITY_TTL_S = 900 # not a setting: fixed freshness ceiling for stored access observations
+TURN_TIMEOUT_S = 120 # not a setting: execution ceiling for one operator CLI turn
+
+
+@dataclass(frozen=True)
+class _Observation:
+ repository_id: int
+ private: bool
+ observed: datetime
+
+
+def _restricts(current: dict[str, Any], proof: _Observation | None) -> bool:
+ return proof is not None and (
+ current["repository_id"] != proof.repository_id
+ or proof.private
+ and not current["is_private"]
+ )
+
+
+async def _invalidate(
+ ch: _CHClient, current: dict[str, Any], uncertain: Callable[[bool], None]
+) -> None:
+ uncertain(True)
+ await _put(ch, current | {"available": 0, "visible_until": domain.EPOCH})
+ uncertain(False)
+
+
+async def _observe(ch: _CHClient, repo: str, token: str) -> _Observation:
+ observed = await store.clock(ch)
+ data = await provider.metadata(repo, token)
+ ident, private = _metadata(repo, data)
+ return _Observation(ident, private, observed)
+
+
+def identity(org: str) -> dict[str, str]:
+ if not org or org == "-":
+ raise domain.FeedInputError("feed requires an explicit organization")
+ return dict(
+ org=org,
+ namespace="-",
+ agent="-",
+ user_id="-",
+ session_id="-",
+ actor=writer_actor("process:tracker-github-feed"),
+ source_class="imported",
+ )
+
+
+def _key(database_uuid: str) -> str:
+ return f"tracker-feed.writer/{database_uuid}"
+
+
+async def require_owner(ch: _CHClient, org: str, owner_uuid: str) -> None:
+ row = await store.owner(ch)
+ if (
+ not row
+ or row["org"] != org
+ or str(row["owner_uuid"]) != owner_uuid
+ or row["schema_version"] != 1
+ ):
+ raise domain.FeedInputError("record belongs to another deployment or schema")
+
+
+async def initialize(ch: _CHClient, org: str, owner_uuid: str) -> None:
+ target = await store.database_uuid(ch)
+ if target == owner_uuid:
+ raise domain.FeedInputError("record database must differ from the deployment database")
+ async with fenced(_key(target)) as uncertain:
+ found = {row["name"] for row in await store.objects(ch)}
+ if found:
+ if "feed_owner" not in found or found - set(store.TABLES + store.VIEWS):
+ raise domain.FeedInputError("feed initialization requires an empty database")
+ # The CREATE's comment claims ownership atomically across deployments;
+ # an interrupted first insert remains resumable by that owner alone.
+ tag = store.owner_tag(org, owner_uuid)
+ uncertain(True)
+ await store.claim_owner(ch, org, owner_uuid)
+ if await store.owner_marker(ch) != tag:
+ raise domain.FeedInputError("record belongs to another deployment or organization")
+ if await store.owner(ch) is None:
+ await ch.insert(
+ "feed_owner",
+ [
+ identity(org)
+ | dict(
+ owner_uuid=uuid.UUID(owner_uuid),
+ schema_version=1,
+ )
+ ],
+ )
+ await require_owner(ch, org, owner_uuid)
+ uncertain(False)
+ for statement in store.statements()[1:]:
+ uncertain(True)
+ await ch.execute(statement)
+ uncertain(False)
+
+
+def _metadata(repo: str, data: dict[str, Any]) -> tuple[int, bool]:
+ name = data.get("full_name")
+ ident = data.get("id")
+ private = data.get("private")
+ if not isinstance(name, str) or name.lower() != repo.lower() or not isinstance(private, bool):
+ raise domain.FeedInputError("repository identity or visibility is unverifiable")
+ return domain.positive_id(ident), private
+
+
+async def _put(ch: _CHClient, row: dict[str, Any]) -> None:
+ rotated = resealed(domain.SECRET_PURPOSE, row["token_enc"])
+ if rotated is not None:
+ row = row | {"token_enc": rotated}
+ await store.put_repository(ch, row | {"revision": uuid.uuid4()})
+
+
+async def configure(
+ ch: _CHClient,
+ *,
+ org: str,
+ owner_uuid: str,
+ repo: str,
+ token: str,
+ allowed_repos: set[str],
+ max_pages: int = 5,
+) -> None:
+ repo = domain.repository(repo)
+ registered = {name for name in allowed_repos if name.lower() == repo.lower()}
+ if not registered:
+ raise domain.FeedInputError("repository is outside the registered source allowlist")
+ if len(registered) > 1:
+ raise domain.FeedInputError("source allowlist must use one spelling per repository")
+ repo = registered.pop()
+ if not token or not 2 <= max_pages <= 100:
+ raise domain.FeedInputError("a read-only token and 2..100 max-pages are required")
+ await require_owner(ch, org, owner_uuid)
+ target = await store.database_uuid(ch)
+ snapshot = await store.repository(ch, org, repo)
+ try:
+ proof = await _observe(ch, repo, token)
+ except (provider.FeedProviderError, domain.FeedInputError):
+ if snapshot:
+ await _hide(ch, snapshot, target, owner_uuid)
+ raise
+ async with fenced(_key(target)) as uncertain:
+ await require_owner(ch, org, owner_uuid)
+ existing = await store.repository(ch, org, repo)
+ if existing and existing["repository_id"] != proof.repository_id:
+ await _invalidate(ch, existing, uncertain)
+ raise domain.FeedInputError("repository identity changed; use a separate record")
+ if (existing is None) != (snapshot is None) or (
+ existing and snapshot and str(existing["revision"]) != str(snapshot["revision"])
+ ):
+ if existing and _restricts(existing, proof):
+ await _invalidate(ch, existing, uncertain)
+ raise domain.FeedInputError("feed changed while configuring; retry the turn")
+ row = identity(org) | dict(
+ repository=repo,
+ repository_id=proof.repository_id,
+ owner_uuid=uuid.UUID(owner_uuid),
+ token_enc=seal(domain.SECRET_PURPOSE, token),
+ is_private=int(proof.private),
+ available=1,
+ visible_until=proof.observed + timedelta(seconds=VISIBILITY_TTL_S),
+ cursors=existing["cursors"]
+ if existing and (proof.private or not existing["is_private"])
+ else "{}",
+ max_pages=max_pages,
+ )
+ uncertain(True)
+ await _put(ch, row)
+ uncertain(False)
+
+
+async def _hide(
+ ch: _CHClient,
+ row: dict[str, Any],
+ target: str,
+ owner_uuid: str,
+ *,
+ proof: _Observation | None = None,
+) -> None:
+ async with fenced(_key(target)) as uncertain:
+ await require_owner(ch, row["org"], owner_uuid)
+ current = await store.repository(ch, row["org"], row["repository"])
+ if not current:
+ return
+ if str(current["revision"]) == str(row["revision"]) or _restricts(current, proof):
+ await _invalidate(ch, current, uncertain)
+
+
+async def _fetch(
+ row: dict[str, Any], stream: str, since: datetime | None, token: str
+) -> tuple[provider.Batch, dict[str, list[dict[str, Any]]]]:
+ repo = row["repository"]
+ batch = await provider.pages(repo, token, stream, since, max_pages=row["max_pages"])
+ records: dict[str, list[dict[str, Any]]] = {}
+ for item in batch.items:
+ if stream == "items":
+ is_pr = "pull_request" in item
+ data = (
+ await provider.pull_request(repo, token, domain.positive_id(item.get("number")))
+ if is_pr
+ else item
+ )
+ record = domain.item_row(repo, data, is_pr=is_pr)
+ table = "feed_prs" if is_pr else "feed_issues"
+ else:
+ record = domain.comment_row(repo, item, review=stream == "review_comments")
+ table = "feed_comments"
+ records.setdefault(table, []).append(record)
+ return batch, records
+
+
+async def _publish(
+ ch: _CHClient,
+ row: dict[str, Any],
+ owner_uuid: str,
+ target: str,
+ stream: str,
+ cursor: datetime | None,
+ records: dict[str, list[dict[str, Any]]],
+ proof: _Observation,
+) -> bool:
+ async with fenced(_key(target)) as uncertain:
+ await require_owner(ch, row["org"], owner_uuid)
+ current = await store.repository(ch, row["org"], row["repository"])
+ if not current:
+ return False
+ if str(current["revision"]) != str(row["revision"]):
+ if _restricts(current, proof):
+ await _invalidate(ch, current, uncertain)
+ return False
+ # Reset before the first public write so an interrupted replay resumes.
+ if current["is_private"] and not proof.private:
+ current["cursors"] = "{}"
+ current |= dict(
+ is_private=int(proof.private),
+ available=1,
+ visible_until=proof.observed + timedelta(seconds=VISIBILITY_TTL_S),
+ )
+ uncertain(True)
+ await _put(ch, current)
+ for table, items in records.items():
+ # Readers can learn this table's part before repository metadata.
+ stamped = [item | {"is_private": int(proof.private)} for item in items]
+ await store.put_items(ch, table, stamped, identity(row["org"]), row["repository"])
+ cursors = json.loads(current["cursors"])
+ if cursor is not None:
+ cursors[stream] = cursor.isoformat()
+ await _put(ch, current | {"cursors": json.dumps(cursors, sort_keys=True)})
+ uncertain(False)
+ return True
+
+
+async def sync(
+ ch: _CHClient, *, org: str, owner_uuid: str, repo: str, allowed_repos: set[str]
+) -> dict[str, int]:
+ repo = domain.repository(repo)
+ if repo.lower() not in {name.lower() for name in allowed_repos}:
+ raise domain.FeedInputError("repository is outside the registered source allowlist")
+ await require_owner(ch, org, owner_uuid)
+ target = await store.database_uuid(ch)
+ counts: dict[str, int] = {}
+ async with asyncio.timeout(TURN_TIMEOUT_S):
+ for stream in provider.STREAMS:
+ row = await store.repository(ch, org, repo)
+ if not row:
+ raise domain.FeedInputError("repository feed is not configured")
+ token = unseal(domain.SECRET_PURPOSE, row["token_enc"])
+ if not token:
+ await _hide(ch, row, target, owner_uuid)
+ raise domain.FeedInputError("feed credential cannot be opened")
+ cursors = json.loads(row["cursors"])
+ since = domain.timestamp(cursors[stream]) if stream in cursors else None
+ proof = None
+ try:
+ proof = await _observe(ch, repo, token)
+ if proof.repository_id != row["repository_id"]:
+ raise domain.FeedInputError("repository identity changed")
+ if row["is_private"] and not proof.private:
+ since = None
+ batch, records = await _fetch(row, stream, since, token)
+ cursor = domain.next_cursor(batch.items, complete=batch.complete, previous=since)
+ except (Exception, asyncio.CancelledError): # rejected scans invalidate visibility
+ await _hide(ch, row, target, owner_uuid, proof=proof)
+ raise
+ published = await _publish(ch, row, owner_uuid, target, stream, cursor, records, proof)
+ if not published:
+ raise domain.FeedInputError("feed changed while fetching; retry the turn")
+ counts[stream] = len(batch.items)
+ return counts
diff --git a/server/tracker/domain/github_feed.py b/server/tracker/domain/github_feed.py
new file mode 100644
index 0000000000..9a5ad63767
--- /dev/null
+++ b/server/tracker/domain/github_feed.py
@@ -0,0 +1,171 @@
+"""Bounded transformations for the team's GitHub record."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import urlsplit
+
+from server.core.github_values import PAGE_SIZE
+from server.core.github_values import FeedInputError as FeedInputError
+from server.core.github_values import positive_id as positive_id
+from server.core.github_values import repository as repository
+from server.core.github_values import since_parameter as since_parameter
+from server.tracker.domain.columns import COLUMNS_READ
+
+EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
+SECRET_PURPOSE = "tracker-github-feed"
+
+
+class FeedCapacityError(RuntimeError):
+ """The bounded scan cannot complete its current timestamp boundary."""
+
+
+def writer_grants(lines: list[str], database: str) -> None:
+ """Accept dedicated direct grants, refusing unknown or role-derived authority."""
+ quoted = "`" + database.replace("\\", "\\\\").replace("`", "\\`") + "`"
+ record_scopes = {database + ".*", quoted + ".*"}
+ metadata_scopes = {
+ f"{db}.{table}"
+ for db in ("system", "`system`")
+ for name in ("columns", "tables", "databases")
+ for table in (name, f"`{name}`")
+ }
+ allowed = {"SELECT", "INSERT", "CREATE TABLE", "CREATE VIEW", "SHOW TABLES", "SHOW COLUMNS"}
+ if not lines:
+ raise FeedInputError("writer grants could not be verified")
+ for line in lines:
+ types, on, rest = line.removeprefix("GRANT ").partition(" ON ")
+ scope, to, recipient = rest.rpartition(" TO ")
+ access = {name.strip().upper() for name in types.split(",")}
+ if (
+ not line.startswith("GRANT ")
+ or not on
+ or not to
+ or not recipient
+ or " WITH " in recipient.upper()
+ or not (
+ scope in record_scopes
+ and access <= allowed
+ or scope in metadata_scopes
+ and access == {"SELECT"}
+ )
+ ):
+ raise FeedInputError("writer grants must be direct and limited to the feed database")
+
+
+def timestamp(value: Any, *, optional: bool = False) -> datetime:
+ if not value and optional:
+ return EPOCH
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ raise ValueError("missing timezone")
+ return parsed.astimezone(UTC)
+ except (ValueError, OverflowError) as exc:
+ raise FeedInputError("invalid upstream timestamp") from exc
+
+
+def next_cursor(
+ items: list[dict[str, Any]], *, complete: bool, previous: datetime | None
+) -> datetime | None:
+ if not items:
+ return previous
+ stamps = [timestamp(item.get("updated_at")) for item in items]
+ # Provider ordering can carry high updated_at outliers. The beginning of
+ # the final page is a conservative replay boundary; its maximum is not.
+ boundary = max(stamps) if complete else min(stamps[-PAGE_SIZE:])
+ if not complete and previous is not None and boundary <= previous:
+ raise FeedCapacityError("timestamp boundary exceeds the page budget; increase max-pages")
+ return max(previous or EPOCH, boundary)
+
+
+def _default(family: str) -> Any:
+ if family.startswith("Array("):
+ return []
+ if family == "String":
+ return ""
+ if family == "DateTime":
+ return EPOCH
+ if family == "UUID":
+ return uuid.UUID(int=0)
+ return 0
+
+
+def _names(value: Any, key: str) -> list[str]:
+ if not isinstance(value, list):
+ return []
+ return [str(row.get(key) or "")[:256] for row in value if isinstance(row, dict)][:100]
+
+
+def item_row(repo: str, data: dict[str, Any], *, is_pr: bool) -> dict[str, Any]:
+ repo = repository(repo)
+ number = positive_id(data.get("number"))
+ kind = "github_pr" if is_pr else "github_issue"
+ row = {name: _default(family) for name, family in COLUMNS_READ[kind + "s"].items()}
+ source_id = f"{repo}#{number}"
+ user = data.get("user") or {}
+ row.update(
+ id=uuid.uuid5(uuid.NAMESPACE_DNS, f"{kind}:{source_id}"),
+ source_id=source_id,
+ number=number,
+ url=f"https://github.com/{repo}/{'pull' if is_pr else 'issues'}/{number}",
+ title=str(data.get("title") or "")[:1024],
+ body=str(data.get("body") or "")[:65536],
+ author=str(user.get("login") or "")[:256],
+ created_at=timestamp(data.get("created_at")),
+ updated_at=timestamp(data.get("updated_at")),
+ closed_at=timestamp(data.get("closed_at"), optional=True),
+ has_test=None,
+ )
+ if is_pr:
+ merged = timestamp(data.get("merged_at"), optional=True)
+ row.update(
+ pr_number=number,
+ state="merged" if merged != EPOCH else str(data.get("state") or ""),
+ is_merged=int(merged != EPOCH),
+ merged_at=merged,
+ review_requests_test=None,
+ base_branch=str((data.get("base") or {}).get("ref") or "")[:256],
+ additions=max(0, int(data.get("additions") or 0)),
+ deletions=max(0, int(data.get("deletions") or 0)),
+ )
+ else:
+ row.update(
+ labels=_names(data.get("labels"), "name"),
+ assignees=_names(data.get("assignees"), "login"),
+ state_reason=str(data.get("state_reason") or "")[:64],
+ )
+ return row
+
+
+def comment_row(repo: str, data: dict[str, Any], *, review: bool) -> dict[str, Any]:
+ repo = repository(repo)
+ url = str(data.get("pull_request_url" if review else "issue_url") or "")
+ parsed = urlsplit(url)
+ owner, _, path = parsed.path.removeprefix("/repos/").partition("/")
+ name, _, _ = path.partition("/")
+ prefix = f"/repos/{owner}/{name}/{'pulls' if review else 'issues'}/"
+ suffix = parsed.path.removeprefix(prefix)
+ if (
+ parsed.scheme != "https"
+ or parsed.netloc != "api.github.com"
+ or f"{owner}/{name}".lower() != repo.lower()
+ or not parsed.path.startswith(prefix)
+ or not suffix.isdecimal()
+ or int(suffix) <= 0
+ or parsed.query
+ or parsed.fragment
+ ):
+ raise FeedInputError("comment parent does not match the configured repository")
+ ident = positive_id(data.get("id"))
+ return dict(
+ comment_id=ident,
+ item_number=positive_id(int(suffix)),
+ review=int(review),
+ body=str(data.get("body") or "")[:65536],
+ author=str((data.get("user") or {}).get("login") or "")[:256],
+ created_at=timestamp(data.get("created_at")),
+ updated_at=timestamp(data.get("updated_at")),
+ )
diff --git a/server/tracker/store/github_feed.py b/server/tracker/store/github_feed.py
new file mode 100644
index 0000000000..ad1cd57607
--- /dev/null
+++ b/server/tracker/store/github_feed.py
@@ -0,0 +1,238 @@
+"""The team's record tables; no copied content targets the deployment database."""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from server.core.repo_base import _CHClient
+from server.tracker.domain.columns import COLUMNS_READ
+from server.tracker.domain.contract import concrete_type, uuid5_sql
+
+IDENTITY = {
+ "org": "String",
+ "namespace": "String",
+ "agent": "String",
+ "user_id": "String",
+ "session_id": "String",
+ "actor": "String",
+ "source_class": "String",
+}
+OWNER_COLUMNS = {**IDENTITY, "owner_uuid": "UUID", "schema_version": "UInt64"}
+REPOSITORY_COLUMNS = {
+ **IDENTITY,
+ "repository": "String",
+ "repository_id": "UInt64",
+ "owner_uuid": "UUID",
+ "token_enc": "String",
+ "is_private": "UInt8",
+ "available": "UInt8",
+ "visible_until": "DateTime64(3, 'UTC')",
+ "cursors": "String",
+ "max_pages": "UInt64",
+ "revision": "UUID",
+}
+COMMENT_COLUMNS = {
+ **IDENTITY,
+ "repository": "String",
+ "comment_id": "UInt64",
+ "item_number": "UInt64",
+ "review": "UInt8",
+ "is_private": "UInt8",
+ "body": "String",
+ "author": "String",
+ "created_at": "DateTime64(3, 'UTC')",
+ "updated_at": "DateTime64(3, 'UTC')",
+}
+TABLES = ("feed_owner", "feed_repositories", "feed_issues", "feed_prs", "feed_comments")
+VIEWS = ("github_issues", "github_prs", "edges")
+
+
+def owner_tag(org: str, owner_uuid: str) -> str:
+ return "loom-github-feed:" + hashlib.sha256(f"{org}\0{owner_uuid}".encode()).hexdigest()
+
+
+async def claim_owner(ch: _CHClient, org: str, owner_uuid: str) -> None:
+ # CREATE's COMMENT grammar requires a literal. The generated tag contains
+ # only a fixed prefix and hexadecimal digest, never caller-provided SQL.
+ tag = owner_tag(org, owner_uuid)
+ await ch.execute(statements()[0] + f" COMMENT '{tag}'")
+
+
+async def owner_marker(ch: _CHClient) -> str:
+ row = await ch.query_one(
+ "SELECT comment FROM system.tables WHERE database = currentDatabase() "
+ "AND name = 'feed_owner' LIMIT 1"
+ )
+ return str(row["comment"]) if row else ""
+
+
+def create_table(name: str, columns: dict[str, str], key: str) -> str:
+ if name not in TABLES:
+ raise ValueError("unknown feed table")
+ fields = ", ".join(f"`{column}` {kind}" for column, kind in columns.items())
+ return (
+ f"CREATE TABLE IF NOT EXISTS {name} ({fields}, "
+ "version DateTime64(3, 'UTC') DEFAULT now64(3)) "
+ f"ENGINE = ReplacingMergeTree(version) ORDER BY ({key})"
+ )
+
+
+def item_columns(table: str) -> dict[str, str]:
+ return {
+ **IDENTITY,
+ "repository": "String",
+ **{
+ name: "Nullable(UInt64)"
+ if name in {"has_test", "review_requests_test"}
+ else concrete_type(kind)
+ for name, kind in COLUMNS_READ[table].items()
+ },
+ }
+
+
+def item_view(table: str) -> str:
+ if table not in {"github_issues", "github_prs"}:
+ raise ValueError("unknown feed view")
+ raw = "feed_issues" if table == "github_issues" else "feed_prs"
+ kind = "github_issue" if table == "github_issues" else "github_pr"
+ private = "greatest(r.is_private, i.is_private, ifNull(c.is_private, 0))"
+ ident = uuid5_sql(f"concat('{kind}', if(({private}) != 0, '_private', ''), ':', i.source_id)")
+ columns = []
+ for column in COLUMNS_READ[table]:
+ expression = f"i.`{column}`"
+ if column == "id":
+ expression = ident
+ elif column == "is_private":
+ expression = private
+ elif column == "body":
+ expression = "concat(substringUTF8(i.body, 1, 32768), '\\n', ifNull(c.comments, ''))"
+ elif column == "updated_at":
+ expression = "greatest(i.version, ifNull(c.published_at, i.version))"
+ columns.append(f"{expression} AS `{column}`")
+ return f"""CREATE VIEW IF NOT EXISTS {table} DEFINER = CURRENT_USER SQL SECURITY DEFINER AS
+SELECT {", ".join(columns)}
+FROM {raw} AS i FINAL
+INNER JOIN feed_repositories AS r FINAL ON i.org = r.org AND i.repository = r.repository
+INNER JOIN feed_owner AS o FINAL ON i.org = o.org AND r.owner_uuid = o.owner_uuid
+LEFT JOIN (
+ SELECT org, repository, item_number, max(version) AS published_at,
+ max(is_private) AS is_private,
+ arrayStringConcat(arrayMap(x -> x.4, groupArraySorted(64)(
+ tuple(-toUnixTimestamp64Milli(updated_at), -toInt16(review),
+ -toInt128(comment_id), substringUTF8(body, 1, 2048))
+ )), '\\n') AS comments
+ FROM feed_comments FINAL GROUP BY org, repository, item_number
+) AS c ON i.org = c.org AND i.repository = c.repository AND i.number = c.item_number
+WHERE r.available = 1 AND r.visible_until > now64(3)"""
+
+
+def statements() -> list[str]:
+ return [
+ create_table("feed_owner", OWNER_COLUMNS, "org"),
+ create_table("feed_repositories", REPOSITORY_COLUMNS, "org, repository"),
+ create_table("feed_issues", item_columns("github_issues"), "org, repository, number"),
+ create_table("feed_prs", item_columns("github_prs"), "org, repository, number"),
+ create_table("feed_comments", COMMENT_COLUMNS, "org, repository, review, comment_id"),
+ item_view("github_issues"),
+ item_view("github_prs"),
+ """CREATE VIEW IF NOT EXISTS edges DEFINER = CURRENT_USER SQL SECURITY DEFINER AS
+SELECT toUUID('00000000-0000-0000-0000-000000000000') AS from_id,
+ '' AS from_type, from_id AS to_id, '' AS to_type, '' AS rel_type,
+ toDateTime(0) AS created_at WHERE 0""",
+ ]
+
+
+async def clock(ch: _CHClient) -> datetime:
+ row = await ch.query_one("SELECT toUInt64(toUnixTimestamp64Milli(now64(3))) AS ms")
+ if not row:
+ raise ValueError("record clock is unavailable")
+ return datetime.fromtimestamp(int(row["ms"]) / 1000, UTC)
+
+
+async def database_uuid(ch: _CHClient) -> str:
+ row = await ch.query_one(
+ "SELECT toString(uuid) AS id FROM system.databases WHERE name = currentDatabase() LIMIT 1"
+ )
+ if not row or not row["id"] or row["id"] == "00000000-0000-0000-0000-000000000000":
+ raise ValueError("feed requires a database with a stable UUID")
+ return str(row["id"])
+
+
+async def owner(ch: _CHClient) -> dict[str, Any] | None:
+ return await ch.query_one(
+ "SELECT * FROM feed_owner FINAL LIMIT 1 SETTINGS select_sequential_consistency = 1"
+ )
+
+
+async def objects(ch: _CHClient) -> list[dict[str, Any]]:
+ return await ch.query_all(
+ "SELECT name FROM system.tables WHERE database = currentDatabase() LIMIT 100"
+ )
+
+
+async def table_engines(ch: _CHClient) -> list[dict[str, Any]]:
+ return await ch.query_all(
+ "SELECT name, engine FROM system.tables WHERE database = currentDatabase() "
+ "AND name IN {tables:Array(String)} LIMIT 5 SETTINGS select_sequential_consistency = 1",
+ {"tables": list(TABLES)},
+ )
+
+
+async def put_repository(ch: _CHClient, row: dict[str, Any]) -> None:
+ """Replace control state under the record writer's lease, even after clock rollback."""
+ clock_row = await ch.query_one(
+ """SELECT toUnixTimestamp64Milli(now64(3)) AS now_ms,
+ toUnixTimestamp64Milli(max(version)) AS prior_ms
+FROM feed_repositories WHERE org = {org:String} AND repository = {repo:String}
+LIMIT 1 SETTINGS select_sequential_consistency = 1""",
+ {"org": row["org"], "repo": row["repository"]},
+ )
+ if not clock_row:
+ raise ValueError("repository publication clock is unavailable")
+ published_ms = max(int(clock_row["now_ms"]), int(clock_row["prior_ms"]) + 1)
+ version = datetime(1970, 1, 1, tzinfo=UTC) + timedelta(milliseconds=published_ms)
+ await ch.insert(
+ "feed_repositories", [{key: row[key] for key in REPOSITORY_COLUMNS} | {"version": version}]
+ )
+
+
+async def repository(ch: _CHClient, org: str, repo: str) -> dict[str, Any] | None:
+ return await ch.query_one(
+ "SELECT * FROM feed_repositories FINAL "
+ "WHERE org = {org:String} AND lower(repository) = lower({repo:String}) LIMIT 1 "
+ "SETTINGS select_sequential_consistency = 1",
+ {"org": org, "repo": repo},
+ )
+
+
+async def put_items(
+ ch: _CHClient, table: str, rows: list[dict[str, Any]], identity: dict[str, str], repo: str
+) -> None:
+ """Publish a batch under the record writer's lease."""
+ if table not in {"feed_issues", "feed_prs", "feed_comments"}:
+ raise ValueError("unknown feed record table")
+ if not rows:
+ return
+ clock_row = await ch.query_one(
+ """SELECT toUnixTimestamp64Milli(now64(3)) AS now_ms, max(published_ms) AS prior_ms
+FROM (
+ SELECT toUnixTimestamp64Milli(max(version)) AS published_ms
+ FROM feed_issues WHERE org = {org:String}
+ UNION ALL
+ SELECT toUnixTimestamp64Milli(max(version)) AS published_ms
+ FROM feed_prs WHERE org = {org:String}
+ UNION ALL
+ SELECT toUnixTimestamp64Milli(max(version)) AS published_ms
+ FROM feed_comments WHERE org = {org:String}
+) LIMIT 1 SETTINGS select_sequential_consistency = 1""",
+ {"org": identity["org"]},
+ )
+ if not clock_row:
+ raise ValueError("record publication clock is unavailable")
+ published_ms = max(int(clock_row["now_ms"]), int(clock_row["prior_ms"]) + 1)
+ version = datetime(1970, 1, 1, tzinfo=UTC) + timedelta(milliseconds=published_ms)
+ await ch.insert(
+ table, [row | identity | {"repository": repo, "version": version} for row in rows]
+ )
diff --git a/tests/analyzer/cases/tracker.py b/tests/analyzer/cases/tracker.py
index b217308b91..b277318a29 100644
--- a/tests/analyzer/cases/tracker.py
+++ b/tests/analyzer/cases/tracker.py
@@ -95,10 +95,24 @@ async def _embed(ch: RecordingCH, c: dict[str, Any]) -> None:
await repo.namespace_stats(ORG, NS)
+async def _feed_clock(ch: RecordingCH, c: dict[str, Any]) -> None:
+ from server.core.rmt_carry import measure_send_clock
+ from server.tracker.store.github_feed import clock
+
+ await clock(ch) # type: ignore[arg-type]
+ await measure_send_clock(ch)
+
+
def cases() -> list[Case]:
return [
Case("tracker_links", LINKS_MODULE, _links, {"limit": [10]}),
Case("tracker_embed", EMBED_MODULE, _embed, {"walled": (False, True)}),
Case("code_backport_gaps", GAPS_MODULE, _gaps, {"wanted": (False, True)}),
Case("tracker_daily_repo.readers", DAILY_MODULE, _readers, {"hours": (False,)}),
+ Case(
+ "tracker_feed.receiving_clock",
+ "server.tracker.store.github_feed",
+ _feed_clock,
+ rows=lambda sql, params: [{"ms": 1000}],
+ ),
]
diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt
index cff1fdce42..f1cc6a9984 100644
--- a/tests/analyzer/shapes.txt
+++ b/tests/analyzer/shapes.txt
@@ -1294,6 +1294,7 @@
96f6898359c7 SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER
971a9c534152 SELECT entity, substring(summary_text, ?, ?) AS content_excerpt, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- alias must NOT be `valid_from`: it would shadow the base column -
9761b739c189 SELECT m.org AS org, m.namespace AS namespace, m.user_id AS user_id, count() AS n FROM loom.memory AS m FINAL INNER JOIN loom.namespaces AS n FINAL ON m.org = n.org AND m.namespace = n.namespace WHERE
+976a3206306b SELECT toUInt64(toUnixTimestamp64Milli(now64(?))) AS ms
979ee17b2127 SELECT memory_key, toString(memory_id) AS id, version, memory_type, toString(status) AS status, source_class, agent, user_id, session_id, actor, updated_at, sensitivity, org, namespace, redacted_at FR
97b465efed16 SELECT source_uri, count() AS live, countIf(embedding_model = {model:String} AND embedding_dim > ?) AS searchable, countIf(embedding_dim = ?) AS unembedded, countIf(embedding_model != {model:String})
97f1fa79ed2d SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user
@@ -1778,6 +1779,7 @@ ca5008422344 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.
ca58f3affe58 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND thread_id = {tid:String} AND tombstoned_
ca924a95865f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g
ca9ec9e5facb SELECT count() AS c FROM ( SELECT evidence_id, argMax(tombstoned_at, created_at) AS tomb_latest FROM loom.memory_belief_sources WHERE org = {o:String} AND namespace = {n:String} AND memory_key = {k:St
+caa73a929cde SELECT toUnixTimestamp64Milli(now64(?)) AS ms
cabd0fc4905d ALTER TABLE loom.consolidation_runs DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ?
cacd77ea0602 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND user_id = {uid:String}
cad920c3af77 SELECT org, code_namespace, branch_namespace, pr_number, branch, verdict, epistemic, basis, reason, cherry_pick_pr, cherry_pick_sha, backport_pr, label_wanted, wanted_branch, wanted_source, settled, p
diff --git a/tests/integration/test_github_feed.py b/tests/integration/test_github_feed.py
new file mode 100644
index 0000000000..6a51c81b51
--- /dev/null
+++ b/tests/integration/test_github_feed.py
@@ -0,0 +1,295 @@
+"""The owned record's views enforce current visibility and bounded comments."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import sys
+import uuid
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from tests.integration._resources import unique_org
+
+pytestmark = pytest.mark.usefixtures("ch_target_required")
+
+
+@pytest.fixture
+async def feed_record(monkeypatch):
+ from server.cli.tracker_feed import _validate_writer
+ from server.core.ch import ExternalCHClient, get_client, set_query_tag
+ from server.settings import settings
+ from server.tracker.store import github_feed as store
+
+ set_query_tag("test.github_feed")
+ org = unique_org("github-feed")
+ suffix = uuid.uuid4().hex
+ database, writer, reader = "feed_" + suffix, "fw_" + suffix, "fr_" + suffix
+ password = uuid.uuid4().hex
+ password_env = "LOOM_FEED_TEST_PASSWORD_" + suffix
+ monkeypatch.setenv(password_env, password)
+ digest = hashlib.sha256(password.encode()).hexdigest()
+ deployment = get_client()
+ await deployment.connect()
+ params = dict(db=database, writer=writer, reader=reader)
+ record = None
+ readonly = None
+ try:
+ await deployment.execute("CREATE DATABASE {db:Identifier}", params)
+ for user in (writer, reader):
+ # Password syntax needs a literal; this locally generated digest is hex only.
+ await deployment.execute(
+ f"CREATE USER `{user}` IDENTIFIED WITH sha256_hash BY '{digest}'",
+ {"user": user},
+ )
+ await deployment.execute(
+ f"GRANT CREATE TABLE, CREATE VIEW, SELECT, INSERT ON `{database}`.* TO `{writer}`",
+ params,
+ )
+ for table in ("columns", "databases", "tables"):
+ await deployment.execute(f"GRANT SELECT ON system.{table} TO `{writer}`")
+ await deployment.execute(f"ALTER USER `{reader}` SETTINGS readonly = 2", params)
+ await deployment.execute(f"GRANT SELECT ON system.columns TO `{reader}`", params)
+ options = dict(
+ host=settings.ch_host,
+ port=settings.ch_port,
+ secure=bool(settings.ch_secure),
+ database=database,
+ password=password,
+ )
+ record = ExternalCHClient(**options, username=writer)
+ readonly = ExternalCHClient(**options, username=reader)
+ await record.connect()
+ await _validate_writer(record, database)
+ yield SimpleNamespace(
+ org=org,
+ database=database,
+ record=record,
+ readonly=readonly,
+ owner=await store.database_uuid(deployment),
+ deployment=deployment,
+ params=params,
+ password_env=password_env,
+ )
+ finally:
+ if readonly is not None:
+ await readonly.disconnect()
+ if record is not None:
+ await record.disconnect()
+ await deployment.execute("DROP DATABASE IF EXISTS {db:Identifier} SYNC", params)
+ await deployment.execute(f"DROP USER IF EXISTS `{writer}`, `{reader}`", params)
+ await deployment.disconnect()
+
+
+async def test_feed_record_roundtrip_visibility_and_recent_comments(feed_record, monkeypatch):
+ from server.tracker.app import github_feed as feed
+ from server.tracker.domain import github_feed as domain
+ from server.tracker.store import github_feed as store
+
+ org, record, owner = feed_record.org, feed_record.record, feed_record.owner
+ await feed.initialize(record, org, owner)
+ await feed.initialize(record, org, owner)
+ assert await store.owner_marker(record) == store.owner_tag(org, owner)
+ monkeypatch.setattr(
+ feed.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=7, full_name="widgets/repo", private=False)),
+ )
+ await feed.configure(
+ record,
+ org=org,
+ owner_uuid=owner,
+ repo="widgets/repo",
+ token="read-only-fixture",
+ allowed_repos={"widgets/repo"},
+ )
+ now = datetime.now(UTC).replace(microsecond=0)
+ item = domain.item_row(
+ "widgets/repo",
+ dict(
+ number=1,
+ title="Saved",
+ body="Issue body",
+ created_at=now.isoformat(),
+ updated_at=now.isoformat(),
+ ),
+ is_pr=False,
+ )
+ await store.put_items(record, "feed_issues", [item], feed.identity(org), "widgets/repo")
+ raw = await record.query_one("SELECT * EXCEPT version FROM feed_issues FINAL LIMIT 1")
+ raw = {
+ key: value.replace(tzinfo=UTC)
+ if isinstance(value, datetime) and value.tzinfo is None
+ else value
+ for key, value in raw.items()
+ }
+ expected = feed.identity(org) | {"repository": "widgets/repo"} | item
+ assert raw == expected
+ comments = [
+ domain.comment_row(
+ "widgets/repo",
+ dict(
+ id=n,
+ issue_url="https://api.github.com/repos/widgets/repo/issues/1",
+ body=f"comment-{n:03d}",
+ created_at=now.isoformat(),
+ updated_at=(now + timedelta(seconds=n)).isoformat(),
+ ),
+ review=False,
+ )
+ for n in range(1, 66)
+ ]
+ await store.put_items(record, "feed_comments", comments, feed.identity(org), "widgets/repo")
+ public = await record.query_one("SELECT id, body, is_private FROM github_issues LIMIT 1")
+ assert public["is_private"] == 0
+ assert "comment-065" in public["body"] and "comment-001" not in public["body"]
+ assert public["body"].count("comment-") == 64
+ row = await store.repository(record, org, "WIDGETS/REPO")
+ assert row and feed.unseal(domain.SECRET_PURPOSE, row["token_enc"]) == "read-only-fixture"
+ await feed._put(record, row | {"is_private": 1})
+ private = await record.query_one("SELECT id, is_private FROM github_issues LIMIT 1")
+ assert private["is_private"] == 1 and private["id"] != public["id"]
+ await feed._put(record, row | {"visible_until": now - timedelta(seconds=1)})
+ assert await record.query_one("SELECT count() AS n FROM github_issues") == {"n": 0}
+ with pytest.raises(domain.FeedInputError, match="another deployment or organization"):
+ await feed.initialize(record, org, str(uuid.uuid4()))
+
+
+async def test_feed_sync_replays_with_scoped_writer_and_readonly_views(feed_record, monkeypatch):
+ from clickhouse_connect.driver.exceptions import DatabaseError
+
+ from server.settings import settings
+ from server.tracker.app import github_feed as feed
+ from server.tracker.app.onboarding import probe_credential
+ from server.tracker.domain.contract import diff
+ from server.tracker.store import github_feed as store
+
+ f = feed_record
+ await feed.initialize(f.record, f.org, f.owner)
+ for view in store.VIEWS:
+ await f.deployment.execute(
+ f"GRANT SELECT ON `{f.database}`.`{view}` TO `{f.params['reader']}`"
+ )
+ await f.readonly.connect()
+ monkeypatch.setattr(settings, "tracker_self_service_allow_private_hosts", True)
+ probe = await probe_credential(
+ host=settings.ch_host,
+ port=settings.ch_port,
+ secure=bool(settings.ch_secure),
+ database=f.database,
+ user=f.params["reader"],
+ password=os.environ[f.password_env],
+ )
+ assert probe.grants.read_only and not probe.unreadable
+ report = diff("feed", f.database, probe.described)
+ assert report.ok, report
+ stamp = datetime.now(UTC).replace(microsecond=0).isoformat()
+ issue = dict(number=1, title="Issue", body="source issue", created_at=stamp, updated_at=stamp)
+ pr = dict(number=2, title="PR", body="source pr", created_at=stamp, updated_at=stamp)
+ comment = dict(
+ id=1,
+ issue_url="https://api.github.com/repos/widgets/repo/issues/1",
+ body="issue reply",
+ created_at=stamp,
+ updated_at=stamp,
+ )
+ review = dict(
+ id=2,
+ pull_request_url="https://api.github.com/repos/widgets/repo/pulls/2",
+ body="inline review",
+ created_at=stamp,
+ updated_at=stamp,
+ )
+ batches = {
+ "items": [issue, pr | {"pull_request": {}}],
+ "comments": [comment],
+ "review_comments": [review],
+ }
+
+ async def pages(repo, token, stream, since, **kwargs):
+ return feed.provider.Batch(batches[stream], True)
+
+ monkeypatch.setattr(
+ feed.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=7, full_name="widgets/repo", private=False)),
+ )
+ monkeypatch.setattr(feed.provider, "pages", pages)
+ monkeypatch.setattr(feed.provider, "pull_request", AsyncMock(return_value=pr))
+ options = dict(
+ org=f.org, owner_uuid=f.owner, repo="widgets/repo", allowed_repos={"widgets/repo"}
+ )
+ await feed.configure(f.record, **options, token="read-only-fixture")
+ assert await feed.sync(f.record, **options) == dict(items=2, comments=1, review_comments=1)
+ first = await store.repository(f.record, f.org, "widgets/repo")
+ assert first and set(json.loads(first["cursors"])) == set(batches)
+ assert await feed.sync(f.record, **options) == dict(items=2, comments=1, review_comments=1)
+ replay = await store.repository(f.record, f.org, "widgets/repo")
+ assert replay and replay["cursors"] == first["cursors"]
+ assert (await f.readonly.query_one("SELECT body FROM github_issues LIMIT 1"))[
+ "body"
+ ] == "source issue\nissue reply"
+ assert (
+ "inline review"
+ in (await f.readonly.query_one("SELECT body FROM github_prs LIMIT 1"))["body"]
+ )
+ assert await f.record.query_one("SELECT count() AS n FROM feed_comments FINAL") == {"n": 2}
+ with pytest.raises(DatabaseError, match="ACCESS_DENIED|Not enough privileges"):
+ await f.readonly.query_one("SELECT token_enc FROM feed_repositories LIMIT 1")
+ with pytest.raises(DatabaseError, match="ACCESS_DENIED|Not enough privileges"):
+ await f.record.query_one("SELECT count() FROM loom.memory")
+ monkeypatch.setattr(
+ feed.provider,
+ "metadata",
+ AsyncMock(side_effect=feed.provider.FeedProviderError("http_404")),
+ )
+ with pytest.raises(feed.provider.FeedProviderError):
+ await feed.sync(f.record, **options)
+ assert await f.readonly.query_one("SELECT count() AS n FROM github_issues") == {"n": 0}
+
+
+async def test_cli_initializes_the_owned_record_using_explicit_writer(feed_record):
+ from server.settings import settings
+ from server.tracker.store import github_feed as store
+
+ f = feed_record
+ process = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-m",
+ "server.cli",
+ "tracker-feed",
+ "init",
+ "--org",
+ f.org,
+ "--host",
+ settings.ch_host,
+ "--port",
+ str(settings.ch_port),
+ "--database",
+ f.database,
+ "--insecure",
+ "--writer-user",
+ f.params["writer"],
+ "--writer-password-env",
+ f.password_env,
+ env=dict(
+ os.environ, LOOM_TRACKER_ORG=f.org, LOOM_TRACKER_SELF_SERVICE_ALLOW_PRIVATE_HOSTS="true"
+ ),
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ try:
+ stdout, stderr = await asyncio.wait_for(process.communicate(), 30)
+ assert process.returncode == 0, stderr.decode()
+ assert json.loads(stdout) == {"initialized": True}
+ owner = await store.owner(f.record)
+ assert owner and owner["org"] == f.org and str(owner["owner_uuid"]) == f.owner
+ finally:
+ if process.returncode is None:
+ process.terminate()
+ await process.wait()
diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json
index f15450deab..c9b48a70c3 100644
--- a/tests/unit/snapshots/cli_help.json
+++ b/tests/unit/snapshots/cli_help.json
@@ -1,8 +1,8 @@
{
"loom": [
- "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...",
+ "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,tracker-feed,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...",
"Loom — vendor-neutral agent memory plane on ClickHouse.",
- "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).",
+ "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,tracker-feed,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. tracker-feed Maintain a team's GitHub tracker record namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).",
"options: -h, --help show this help message and exit --version show program's version number and exit"
],
"loom code": [
@@ -475,6 +475,23 @@
"usage: loom tracker-embed-reset [-h] --namespace NAMESPACE [--org ORG] [--kind KIND] [--apply | --dry-run]",
"options: -h, --help show this help message and exit --namespace NAMESPACE the tracker namespace --org ORG --kind KIND issue or pr (default: both) --apply clear the cursors (default: dry run) --dry-run report what --apply would write, write nothing (the default)"
],
+ "loom tracker-feed": [
+ "usage: loom tracker-feed [-h] {init,configure,sync} ...",
+ "positional arguments: {init,configure,sync}",
+ "options: -h, --help show this help message and exit"
+ ],
+ "loom tracker-feed init": [
+ "usage: loom tracker-feed init [-h] --org ORG --writer-user WRITER_USER [--writer-password-env WRITER_PASSWORD_ENV] --host HOST [--port PORT] --database DATABASE [--insecure]",
+ "options: -h, --help show this help message and exit --org ORG --writer-user WRITER_USER --writer-password-env WRITER_PASSWORD_ENV --host HOST --port PORT --database DATABASE --insecure Use HTTP for a private development record"
+ ],
+ "loom tracker-feed configure": [
+ "usage: loom tracker-feed configure [-h] --org ORG --writer-user WRITER_USER [--writer-password-env WRITER_PASSWORD_ENV] --source SOURCE --repo REPO [--token-env TOKEN_ENV] [--read-only-token] [--max-pages MAX_PAGES]",
+ "options: -h, --help show this help message and exit --org ORG --writer-user WRITER_USER --writer-password-env WRITER_PASSWORD_ENV --source SOURCE --repo REPO A repository in the source's allowlist --token-env TOKEN_ENV --read-only-token Attest the token has only read permissions --max-pages MAX_PAGES Pages per stream, from 2 to 100"
+ ],
+ "loom tracker-feed sync": [
+ "usage: loom tracker-feed sync [-h] --org ORG --writer-user WRITER_USER [--writer-password-env WRITER_PASSWORD_ENV] --source SOURCE --repo REPO",
+ "options: -h, --help show this help message and exit --org ORG --writer-user WRITER_USER --writer-password-env WRITER_PASSWORD_ENV --source SOURCE --repo REPO A repository in the source's allowlist"
+ ],
"loom namespaces-adopt": [
"usage: loom namespaces-adopt [-h] --org ORG --owner OWNER [--prefix PREFIX] [--namespace NAMESPACE] [--kind {bench,memory}] [--apply | --dry-run] [--show SHOW]",
"options: -h, --help show this help message and exit --org ORG Org whose namespaces to adopt --owner OWNER owner_email to write on every adopted row --prefix PREFIX Only namespaces under this prefix (repeatable); they are adopted as bench --namespace NAMESPACE An explicit namespace (repeatable) --kind {bench,memory} Force the kind on every adopted row --apply write the rows (default: dry run) --dry-run report what --apply would write, write nothing (the default) --show SHOW Namespaces to print (default 30)"
diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json
index 2020b283a4..e011a5fdc0 100644
--- a/tests/unit/snapshots/cli_tree.json
+++ b/tests/unit/snapshots/cli_tree.json
@@ -13365,6 +13365,527 @@
}
]
},
+ "tracker-feed": {
+ "prog": "loom tracker-feed",
+ "usage": null,
+ "description": null,
+ "epilog": null,
+ "formatter_class": "argparse:HelpFormatter",
+ "add_help": true,
+ "allow_abbrev": true,
+ "defaults": {},
+ "groups": [
+ {
+ "title": "positional arguments",
+ "description": null,
+ "dests": [
+ "feed_command"
+ ]
+ },
+ {
+ "title": "options",
+ "description": null,
+ "dests": [
+ "help"
+ ]
+ }
+ ],
+ "mutually_exclusive": [],
+ "arguments": [
+ {
+ "option_strings": [
+ "-h",
+ "--help"
+ ],
+ "dest": "help",
+ "action": "_HelpAction",
+ "default": "==SUPPRESS==",
+ "const": null,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "show this help message and exit",
+ "metavar": null
+ },
+ {
+ "option_strings": [],
+ "dest": "feed_command",
+ "action": "_SubParsersAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": "A...",
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null,
+ "subcommands": {
+ "init": {
+ "prog": "loom tracker-feed init",
+ "usage": null,
+ "description": null,
+ "epilog": null,
+ "formatter_class": "argparse:HelpFormatter",
+ "add_help": true,
+ "allow_abbrev": true,
+ "defaults": {
+ "func": "server.cli.tracker_feed:_command"
+ },
+ "groups": [
+ {
+ "title": "positional arguments",
+ "description": null,
+ "dests": []
+ },
+ {
+ "title": "options",
+ "description": null,
+ "dests": [
+ "help",
+ "org",
+ "writer_user",
+ "writer_password_env",
+ "host",
+ "port",
+ "database",
+ "insecure"
+ ]
+ }
+ ],
+ "mutually_exclusive": [],
+ "arguments": [
+ {
+ "option_strings": [
+ "-h",
+ "--help"
+ ],
+ "dest": "help",
+ "action": "_HelpAction",
+ "default": "==SUPPRESS==",
+ "const": null,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "show this help message and exit",
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--org"
+ ],
+ "dest": "org",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-user"
+ ],
+ "dest": "writer_user",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-password-env"
+ ],
+ "dest": "writer_password_env",
+ "action": "_StoreAction",
+ "default": "LOOM_TRACKER_FEED_WRITER_PASSWORD",
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": false,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--host"
+ ],
+ "dest": "host",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--port"
+ ],
+ "dest": "port",
+ "action": "_StoreAction",
+ "default": 8443,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": "builtins:int",
+ "required": false,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--database"
+ ],
+ "dest": "database",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--insecure"
+ ],
+ "dest": "insecure",
+ "action": "_StoreTrueAction",
+ "default": false,
+ "const": true,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "Use HTTP for a private development record",
+ "metavar": null
+ }
+ ]
+ },
+ "configure": {
+ "prog": "loom tracker-feed configure",
+ "usage": null,
+ "description": null,
+ "epilog": null,
+ "formatter_class": "argparse:HelpFormatter",
+ "add_help": true,
+ "allow_abbrev": true,
+ "defaults": {
+ "func": "server.cli.tracker_feed:_command"
+ },
+ "groups": [
+ {
+ "title": "positional arguments",
+ "description": null,
+ "dests": []
+ },
+ {
+ "title": "options",
+ "description": null,
+ "dests": [
+ "help",
+ "org",
+ "writer_user",
+ "writer_password_env",
+ "source",
+ "repo",
+ "token_env",
+ "read_only_token",
+ "max_pages"
+ ]
+ }
+ ],
+ "mutually_exclusive": [],
+ "arguments": [
+ {
+ "option_strings": [
+ "-h",
+ "--help"
+ ],
+ "dest": "help",
+ "action": "_HelpAction",
+ "default": "==SUPPRESS==",
+ "const": null,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "show this help message and exit",
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--org"
+ ],
+ "dest": "org",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-user"
+ ],
+ "dest": "writer_user",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-password-env"
+ ],
+ "dest": "writer_password_env",
+ "action": "_StoreAction",
+ "default": "LOOM_TRACKER_FEED_WRITER_PASSWORD",
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": false,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--source"
+ ],
+ "dest": "source",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--repo"
+ ],
+ "dest": "repo",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": "A repository in the source's allowlist",
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--token-env"
+ ],
+ "dest": "token_env",
+ "action": "_StoreAction",
+ "default": "LOOM_TRACKER_FEED_GITHUB_TOKEN",
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": false,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--read-only-token"
+ ],
+ "dest": "read_only_token",
+ "action": "_StoreTrueAction",
+ "default": false,
+ "const": true,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "Attest the token has only read permissions",
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--max-pages"
+ ],
+ "dest": "max_pages",
+ "action": "_StoreAction",
+ "default": 5,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": "builtins:int",
+ "required": false,
+ "help": "Pages per stream, from 2 to 100",
+ "metavar": null
+ }
+ ]
+ },
+ "sync": {
+ "prog": "loom tracker-feed sync",
+ "usage": null,
+ "description": null,
+ "epilog": null,
+ "formatter_class": "argparse:HelpFormatter",
+ "add_help": true,
+ "allow_abbrev": true,
+ "defaults": {
+ "func": "server.cli.tracker_feed:_command"
+ },
+ "groups": [
+ {
+ "title": "positional arguments",
+ "description": null,
+ "dests": []
+ },
+ {
+ "title": "options",
+ "description": null,
+ "dests": [
+ "help",
+ "org",
+ "writer_user",
+ "writer_password_env",
+ "source",
+ "repo"
+ ]
+ }
+ ],
+ "mutually_exclusive": [],
+ "arguments": [
+ {
+ "option_strings": [
+ "-h",
+ "--help"
+ ],
+ "dest": "help",
+ "action": "_HelpAction",
+ "default": "==SUPPRESS==",
+ "const": null,
+ "choices": null,
+ "nargs": 0,
+ "type": null,
+ "required": false,
+ "help": "show this help message and exit",
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--org"
+ ],
+ "dest": "org",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-user"
+ ],
+ "dest": "writer_user",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--writer-password-env"
+ ],
+ "dest": "writer_password_env",
+ "action": "_StoreAction",
+ "default": "LOOM_TRACKER_FEED_WRITER_PASSWORD",
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": false,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--source"
+ ],
+ "dest": "source",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": null,
+ "metavar": null
+ },
+ {
+ "option_strings": [
+ "--repo"
+ ],
+ "dest": "repo",
+ "action": "_StoreAction",
+ "default": null,
+ "const": null,
+ "choices": null,
+ "nargs": null,
+ "type": null,
+ "required": true,
+ "help": "A repository in the source's allowlist",
+ "metavar": null
+ }
+ ]
+ }
+ },
+ "aliases_of": {
+ "init": "loom tracker-feed init",
+ "configure": "loom tracker-feed configure",
+ "sync": "loom tracker-feed sync"
+ },
+ "choices_help": []
+ }
+ ]
+ },
"namespaces-adopt": {
"prog": "loom namespaces-adopt",
"usage": null,
@@ -15461,6 +15982,7 @@
"tracker-contract-check": "loom tracker-contract-check",
"tracker-dupcheck-fit": "loom tracker-dupcheck-fit",
"tracker-embed-reset": "loom tracker-embed-reset",
+ "tracker-feed": "loom tracker-feed",
"namespaces-adopt": "loom namespaces-adopt",
"sensitivity-sweep": "loom sensitivity-sweep",
"integrity-repair": "loom integrity-repair",
@@ -15726,6 +16248,11 @@
"help": "Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them.",
"metavar": "tracker-embed-reset"
},
+ {
+ "dest": "tracker-feed",
+ "help": "Maintain a team's GitHub tracker record",
+ "metavar": "tracker-feed"
+ },
{
"dest": "namespaces-adopt",
"help": "Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply.",
diff --git a/tests/unit/test_external_ch.py b/tests/unit/test_external_ch.py
new file mode 100644
index 0000000000..6ab7177d0c
--- /dev/null
+++ b/tests/unit/test_external_ch.py
@@ -0,0 +1,110 @@
+"""Record writers retain their explicit target across reconnects."""
+
+import time
+from datetime import UTC
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from server.core import ch
+
+
+@pytest.mark.asyncio
+async def test_external_client_uses_only_explicit_connection_and_utc(monkeypatch):
+ driver = SimpleNamespace(server_tz=None, close=AsyncMock())
+ connect = AsyncMock(return_value=driver)
+ monkeypatch.setattr(ch.clickhouse_connect, "get_async_client", connect)
+ client = ch.ExternalCHClient(
+ host="record.example",
+ port=8443,
+ secure=True,
+ database="team_record",
+ username="record_writer",
+ password="secret",
+ )
+ await client.connect()
+ assert connect.call_args.kwargs["host"] == "record.example"
+ assert connect.call_args.kwargs["database"] == "team_record"
+ assert connect.call_args.kwargs["username"] == "record_writer"
+ assert driver.server_tz is UTC
+ await client._reconnect(0)
+ assert connect.call_count == 2
+ assert connect.call_args.kwargs["database"] == "team_record"
+ await client.disconnect()
+
+
+@pytest.mark.asyncio
+async def test_external_failed_connect_keeps_existing_client(monkeypatch):
+ driver = SimpleNamespace(server_tz=None, close=AsyncMock())
+ connect = AsyncMock(side_effect=[driver, OSError("connection refused")])
+ monkeypatch.setattr(ch.clickhouse_connect, "get_async_client", connect)
+ client = ch.ExternalCHClient(
+ host="record.example",
+ port=8443,
+ secure=True,
+ database="team_record",
+ username="record_writer",
+ password="secret",
+ )
+ await client.connect()
+ with pytest.raises(OSError):
+ await client._reconnect(0)
+ assert client.client is driver and driver.close.await_count == 0
+ await client.disconnect()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deployment, external", [(1000.0, 1300.0), (1300.0, 1000.0)])
+async def test_leased_insert_measures_its_own_target_clock(monkeypatch, deployment, external):
+ from server.core import rmt_carry
+
+ monkeypatch.setattr(rmt_carry, "_server_anchor", (deployment, time.monotonic()))
+ monkeypatch.setattr(rmt_carry, "_skew_at", time.monotonic())
+ client = ch.CHClient()
+ client.query_one = AsyncMock(return_value={"ms": int(external * 1000)})
+ client._column_type_names = AsyncMock(return_value=None)
+ client._fenced_insert = AsyncMock()
+
+ async def run(name, call, **kwargs):
+ return await call()
+
+ monkeypatch.setattr(client, "_with_retry", run)
+ with rmt_carry.under_lease(time.monotonic() + 60):
+ await client._insert_rows("feed_issues", [{"number": 1}], None)
+ client.query_one.assert_awaited_once()
+ deadline = client._fenced_insert.call_args.args[-1]
+ assert external + 14 <= deadline <= external + 16
+ assert rmt_carry._server_anchor[0] == deployment
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "request_delay,response_delay,lease_left",
+ [(28.0, 0.0, -5.0), (0.0, 28.0, -5.0), (14.0, 14.0, -5.0), (28.0, 0.0, 0.25)],
+)
+async def test_receiving_clock_never_extends_a_lease_deadline(
+ monkeypatch, request_delay, response_delay, lease_left
+):
+ from server.core import rmt_carry
+ from tests.unit.clock_fake import ManualClock
+
+ clock = ManualClock(t=100.0)
+ monkeypatch.setattr(rmt_carry, "time", SimpleNamespace(monotonic=clock.monotonic))
+
+ async def query_one(sql):
+ clock.advance(request_delay)
+ sampled_server = 1000.0 + clock.t - 100.0
+ clock.advance(response_delay)
+ return {"ms": int(sampled_server * 1000)}
+
+ anchor = await rmt_carry.measure_send_clock(SimpleNamespace(query_one=query_one))
+ clock.advance(3.0)
+ actual_server = 1000.0 + clock.t - 100.0
+ lease_end = clock.t + lease_left
+ with rmt_carry.under_lease(lease_end):
+ deadline = rmt_carry.clock_send_deadline(anchor)
+ safe_deadline = actual_server + min(0.0, lease_left) + rmt_carry._TRANSIT_S
+ estimated_server = anchor[0] + clock.t - anchor[1]
+ assert estimated_server <= actual_server
+ assert deadline <= safe_deadline
diff --git a/tests/unit/test_github_feed.py b/tests/unit/test_github_feed.py
new file mode 100644
index 0000000000..08309dbeff
--- /dev/null
+++ b/tests/unit/test_github_feed.py
@@ -0,0 +1,165 @@
+"""A copied tracker record keeps upstream identities and complete cursor boundaries."""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from server.tracker.domain import github_feed as feed
+
+NOW = datetime(2025, 1, 2, tzinfo=UTC)
+
+
+def item(number: int, seconds: int = 0, **fields):
+ return {
+ "id": number + 100,
+ "number": number,
+ "title": "Retention decision",
+ "body": "Keep an explicit retention period.",
+ "user": {"login": "reader"},
+ "created_at": NOW.isoformat(),
+ "updated_at": (NOW + timedelta(seconds=seconds)).isoformat(),
+ **fields,
+ }
+
+
+def test_copied_item_matches_contract_identity_and_defaults():
+ row = feed.item_row("widgets/repo", item(7), is_pr=False)
+ assert row["source_id"] == "widgets/repo#7"
+ assert row["id"] == uuid.uuid5(uuid.NAMESPACE_DNS, "github_issue:widgets/repo#7")
+ assert row["author"] == "reader" and row["number"] == 7
+ assert row["closed_at"] == datetime(1970, 1, 1, tzinfo=UTC)
+ assert row["labels"] == [] and row["is_private"] == 0
+ assert row["triage_clean_on"] == [] and row["triage_component"] == ""
+ assert row["updated_at"] == NOW
+ assert row["has_test"] is None
+
+
+def test_pr_keeps_merge_metadata_and_leaves_review_evidence_unknown():
+ row = feed.item_row(
+ "widgets/repo",
+ item(
+ 7,
+ merged_at=NOW.isoformat(),
+ state="closed",
+ base={"ref": "main"},
+ requested_reviewers=[{"login": "maintainer"}],
+ additions=8,
+ deletions=2,
+ ),
+ is_pr=True,
+ )
+ assert row["state"] == "merged" and row["is_merged"] == 1
+ assert row["reviewers"] == [] and row["base_branch"] == "main"
+ assert row["review_requests_test"] is None and row["has_test"] is None
+ assert row["additions"] == 8 and row["deletions"] == 2
+
+
+def test_full_budget_replays_last_incomplete_timestamp():
+ batch = [item(n, 1 if n < 60 else 2) for n in range(100)]
+ boundary = NOW + timedelta(seconds=1)
+ assert feed.next_cursor(batch, complete=False, previous=NOW) == boundary
+ with pytest.raises(feed.FeedCapacityError):
+ feed.next_cursor(batch, complete=False, previous=boundary)
+ assert feed.since_parameter(NOW) == "2025-01-01T23:59:59Z"
+
+
+def test_dense_timestamp_refuses_silent_stall():
+ with pytest.raises(feed.FeedCapacityError, match="timestamp"):
+ feed.next_cursor([item(n) for n in range(100)], complete=False, previous=NOW)
+
+
+def test_complete_scan_advances_and_empty_scan_preserves_checkpoint():
+ assert feed.next_cursor([item(1, 2)], complete=True, previous=NOW) == NOW + timedelta(seconds=2)
+ assert feed.next_cursor([], complete=True, previous=NOW) == NOW
+
+
+def test_two_page_budget_progresses_while_replaying_the_final_page():
+ backlog = [item(n, n) for n in range(1, 301)]
+ first = feed.next_cursor(backlog[:200], complete=False, previous=NOW)
+ assert first == NOW + timedelta(seconds=101)
+ replay = [row for row in backlog if feed.timestamp(row["updated_at"]) >= first][:200]
+ assert feed.next_cursor(replay, complete=False, previous=first) == NOW + timedelta(seconds=201)
+
+
+@pytest.mark.parametrize("repo", ["../secret", "a/b/c", "a/%2f", "a/b?token=secret", "a/b\n"])
+def test_repository_path_cannot_change_origin_or_endpoint(repo):
+ with pytest.raises(feed.FeedInputError):
+ feed.repository(repo)
+
+
+def test_comment_parent_url_is_validated_without_following_it():
+ row = feed.comment_row(
+ "widgets/repo",
+ {
+ **item(1),
+ "issue_url": "https://api.github.com/repos/widgets/repo/issues/7",
+ },
+ review=False,
+ )
+ assert row["item_number"] == 7 and row["comment_id"] == 101
+ for url in (
+ "https://evil.example/issues/7",
+ "https://api.github.com/repos/widgets/other/issues/7",
+ "https://api.github.com/REPOS/widgets/repo/issues/7",
+ "https://api.github.com/repos/widgets/repo/ISSUES/7",
+ "https://api.github.com/repos/widgets/repo/issues/7?secret=1",
+ "https://api.github.com/repos/widgets/repo/issues/7#fragment",
+ "https://api.github.com/repos/widgets/repo/issues/0",
+ ):
+ with pytest.raises(feed.FeedInputError):
+ feed.comment_row("widgets/repo", {**item(1), "issue_url": url}, review=False)
+
+
+@pytest.mark.parametrize("review", [False, True])
+def test_comment_parent_accepts_registered_repository_case_difference(review):
+ parent = "pull_request_url" if review else "issue_url"
+ path = "pulls" if review else "issues"
+ row = feed.comment_row(
+ "Widgets/Repo",
+ {
+ **item(1),
+ parent: f"https://api.github.com/repos/WIDGETS/REPO/{path}/7",
+ },
+ review=review,
+ )
+ assert row["item_number"] == 7 and row["comment_id"] == 101
+
+
+def test_invalid_update_time_refuses_checkpoint_and_content():
+ with pytest.raises(feed.FeedInputError):
+ feed.item_row("widgets/repo", item(1, updated_at="bad-time"), is_pr=False)
+
+
+@pytest.mark.parametrize("scope", ["team_feed.*", "`team_feed`.*"])
+def test_writer_grants_accept_only_direct_record_and_metadata_authority(scope):
+ feed.writer_grants(
+ [
+ f"GRANT SELECT, INSERT, CREATE TABLE, CREATE VIEW ON {scope} TO writer",
+ "GRANT SELECT ON system.columns TO writer",
+ ],
+ "team_feed",
+ )
+
+
+@pytest.mark.parametrize(
+ "lines",
+ [
+ [],
+ ["GRANT ALL ON team_feed.* TO writer"],
+ ["GRANT SELECT ON system.* TO writer"],
+ ["GRANT SELECT ON other.* TO writer"],
+ ["GRANT SELECT ON team_feed.* TO writer WITH GRANT OPTION"],
+ ["GRANT reader_role TO writer"],
+ ["GRANT SELECT ON team_feed.* TO writer", "GRANT ALL ON *.* TO writer"],
+ ],
+)
+def test_writer_grants_refuse_unverifiable_or_excess_authority(lines):
+ with pytest.raises(feed.FeedInputError, match="writer grants"):
+ feed.writer_grants(lines, "team_feed")
+
+
+@pytest.mark.parametrize("number", [True, 1.5, "1", -1, 2**64])
+def test_item_identity_rejects_coercion_and_uint64_overflow(number):
+ with pytest.raises(feed.FeedInputError, match="number"):
+ feed.item_row("widgets/repo", item(1) | {"number": number}, is_pr=False)
diff --git a/tests/unit/test_github_feed_app.py b/tests/unit/test_github_feed_app.py
new file mode 100644
index 0000000000..654b2f0557
--- /dev/null
+++ b/tests/unit/test_github_feed_app.py
@@ -0,0 +1,527 @@
+"""Copied records publish only under the current feed configuration."""
+
+import asyncio
+import uuid
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from server.tracker.app import github_feed as app
+from server.tracker.domain.github_feed import FeedInputError
+
+OWNER = "00000000-0000-0000-0000-000000000001"
+TARGET = "00000000-0000-0000-0000-000000000002"
+NOW = datetime(2025, 1, 2, tzinfo=UTC)
+
+
+@pytest.fixture
+def env(monkeypatch):
+ state = SimpleNamespace(
+ row=app.identity("team")
+ | dict(
+ repository="widgets/repo",
+ repository_id=7,
+ owner_uuid=OWNER,
+ token_enc="sealed-secret",
+ is_private=0,
+ available=1,
+ visible_until=NOW + timedelta(seconds=900),
+ cursors="{}",
+ max_pages=5,
+ revision=uuid.uuid4(),
+ ),
+ writes=[],
+ marks=[],
+ records=[],
+ keys=[],
+ )
+
+ async def put(ch, row):
+ state.row = dict(row)
+ state.writes.append(dict(row))
+
+ async def records(ch, table, rows, identity, repo):
+ state.records.append((table, rows, identity, repo, state.row["is_private"]))
+
+ @asynccontextmanager
+ async def fence(name, **kwargs):
+ state.keys.append(name)
+ yield state.marks.append
+
+ monkeypatch.setattr(app, "fenced", fence)
+ monkeypatch.setattr(app.store, "clock", AsyncMock(return_value=NOW))
+ monkeypatch.setattr(app, "resealed", lambda purpose, token: None)
+ monkeypatch.setattr(app, "unseal", lambda purpose, token: "read-only-secret")
+ monkeypatch.setattr(
+ app.store,
+ "owner",
+ AsyncMock(
+ return_value=dict(
+ org="team",
+ owner_uuid=OWNER,
+ schema_version=1,
+ )
+ ),
+ )
+ monkeypatch.setattr(app.store, "database_uuid", AsyncMock(return_value=TARGET))
+ monkeypatch.setattr(
+ app.store, "repository", AsyncMock(side_effect=lambda *args: dict(state.row))
+ )
+ monkeypatch.setattr(app.store, "put_repository", put)
+ monkeypatch.setattr(app.store, "put_items", records)
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(
+ return_value=dict(
+ id=7,
+ full_name="widgets/repo",
+ private=False,
+ )
+ ),
+ )
+ monkeypatch.setattr(app.provider, "pages", AsyncMock(return_value=app.provider.Batch([], True)))
+ return state
+
+
+async def sync():
+ return await app.sync(
+ object(), org="team", owner_uuid=OWNER, repo="widgets/repo", allowed_repos={"widgets/repo"}
+ )
+
+
+@pytest.mark.asyncio
+async def test_empty_scan_checks_each_stream_without_advancing_cursor(env):
+ assert await sync() == dict(items=0, comments=0, review_comments=0)
+ assert env.row["cursors"] == "{}" and env.row["available"] == 1
+ assert env.keys == [f"tracker-feed.writer/{TARGET}"] * 3
+ assert not env.records and env.marks[-1] is False
+
+
+@pytest.mark.asyncio
+async def test_failed_visibility_check_hides_previously_public_content(env, monkeypatch):
+ monkeypatch.setattr(
+ app.provider, "metadata", AsyncMock(side_effect=app.provider.FeedProviderError("http_404"))
+ )
+ with pytest.raises(app.provider.FeedProviderError):
+ await sync()
+ assert env.row["available"] == 0 and env.row["visible_until"] == app.domain.EPOCH
+ assert not env.records and env.row["cursors"] == "{}"
+
+
+@pytest.mark.asyncio
+async def test_different_repository_id_hides_previous_copy(env, monkeypatch):
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(
+ return_value=dict(
+ id=8,
+ full_name="widgets/repo",
+ private=False,
+ )
+ ),
+ )
+ with pytest.raises(FeedInputError, match="identity changed"):
+ await sync()
+ assert env.row["available"] == 0 and not env.records
+
+
+@pytest.mark.asyncio
+async def test_credential_rotation_during_fetch_cannot_publish_stale_content(env, monkeypatch):
+ async def fetch(*args, **kwargs):
+ env.row["revision"] = uuid.uuid4()
+ env.row["token_enc"] = "replacement-secret"
+ return app.provider.Batch([], True)
+
+ monkeypatch.setattr(app.provider, "pages", fetch)
+ with pytest.raises(FeedInputError, match="changed while fetching"):
+ await sync()
+ assert env.row["token_enc"] == "replacement-secret" and not env.writes
+
+
+@pytest.mark.asyncio
+async def test_private_visibility_publishes_before_content(env, monkeypatch):
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(
+ return_value=dict(
+ id=7,
+ full_name="widgets/repo",
+ private=True,
+ )
+ ),
+ )
+ data = dict(number=1, title="private", created_at=NOW.isoformat(), updated_at=NOW.isoformat())
+ monkeypatch.setattr(
+ app.provider,
+ "pages",
+ AsyncMock(
+ side_effect=[
+ app.provider.Batch([data], True),
+ app.provider.Batch([], True),
+ app.provider.Batch([], True),
+ ]
+ ),
+ )
+ await sync()
+ assert env.records[0][-1] == 1
+ assert env.records[0][2]["actor"] == "process:tracker-github-feed"
+ assert env.records[0][2]["source_class"] == "imported"
+ assert '"items"' in env.row["cursors"]
+
+
+@pytest.mark.asyncio
+async def test_uncertain_content_write_keeps_send_claim_and_checkpoint(env, monkeypatch):
+ async def fail(*args):
+ raise TimeoutError("insert acknowledgement lost")
+
+ monkeypatch.setattr(app.store, "put_items", fail)
+ data = dict(number=1, created_at=NOW.isoformat(), updated_at=NOW.isoformat())
+ monkeypatch.setattr(
+ app.provider, "pages", AsyncMock(return_value=app.provider.Batch([data], True))
+ )
+ with pytest.raises(TimeoutError):
+ await sync()
+ assert env.marks[-1] is True and env.row["cursors"] == "{}"
+
+
+@pytest.mark.asyncio
+async def test_another_deployment_refuses_before_provider_read(env, monkeypatch):
+ monkeypatch.setattr(
+ app.store,
+ "owner",
+ AsyncMock(
+ return_value=dict(
+ org="team",
+ owner_uuid=TARGET,
+ schema_version=1,
+ )
+ ),
+ )
+ with pytest.raises(FeedInputError, match="another deployment"):
+ await sync()
+ app.provider.metadata.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_old_configure_observation_cannot_overwrite_newer_private_state(env, monkeypatch):
+ async def metadata(*args):
+ env.row["revision"] = uuid.uuid4()
+ env.row["is_private"] = 1
+ return dict(id=7, full_name="widgets/repo", private=False)
+
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ with pytest.raises(FeedInputError, match="changed while configuring"):
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ )
+ assert env.row["is_private"] == 1 and not env.writes
+
+
+@pytest.mark.asyncio
+async def test_private_observation_with_capacity_failure_hides_prior_public_record(
+ env, monkeypatch
+):
+ env.row["cursors"] = '{"items": "2025-01-02T00:00:00+00:00"}'
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(
+ return_value=dict(
+ id=7,
+ full_name="widgets/repo",
+ private=True,
+ )
+ ),
+ )
+ data = [
+ dict(number=n + 1, created_at=NOW.isoformat(), updated_at=NOW.isoformat())
+ for n in range(100)
+ ]
+ monkeypatch.setattr(
+ app.provider, "pages", AsyncMock(return_value=app.provider.Batch(data, False))
+ )
+ with pytest.raises(app.domain.FeedCapacityError):
+ await sync()
+ assert env.row["available"] == 0 and not env.records
+
+
+@pytest.mark.asyncio
+async def test_cancelled_scan_invalidates_visibility_without_advancing_cursor(env, monkeypatch):
+ monkeypatch.setattr(app.provider, "pages", AsyncMock(side_effect=asyncio.CancelledError))
+ with pytest.raises(asyncio.CancelledError):
+ await sync()
+ assert env.row["available"] == 0 and env.row["cursors"] == "{}"
+
+
+@pytest.mark.asyncio
+async def test_initializer_resumes_an_interrupted_owner_insert(env, monkeypatch):
+ ch = SimpleNamespace(execute=AsyncMock(), insert=AsyncMock())
+ monkeypatch.setattr(app.store, "objects", AsyncMock(return_value=[{"name": "feed_owner"}]))
+ monkeypatch.setattr(
+ app.store, "owner_marker", AsyncMock(return_value=app.store.owner_tag("team", OWNER))
+ )
+ monkeypatch.setattr(
+ app.store,
+ "owner",
+ AsyncMock(side_effect=[None, dict(org="team", owner_uuid=OWNER, schema_version=1)]),
+ )
+ await app.initialize(ch, "team", OWNER)
+ assert ch.insert.await_count == 1
+ assert ch.insert.call_args.args[0] == "feed_owner"
+ assert ch.execute.await_count == len(app.store.statements())
+ assert env.marks[-1] is False
+
+
+@pytest.mark.asyncio
+async def test_initializer_refuses_another_atomic_owner_claim(env, monkeypatch):
+ ch = SimpleNamespace(execute=AsyncMock(), insert=AsyncMock())
+ monkeypatch.setattr(app.store, "objects", AsyncMock(return_value=[]))
+ monkeypatch.setattr(
+ app.store, "owner_marker", AsyncMock(return_value=app.store.owner_tag("other", TARGET))
+ )
+ with pytest.raises(FeedInputError, match="another deployment or organization"):
+ await app.initialize(ch, "team", OWNER)
+ ch.insert.assert_not_awaited()
+ assert ch.execute.await_count == 1
+ assert env.marks[-1] is True
+
+
+@pytest.mark.asyncio
+async def test_successful_publish_reseals_previous_root_credentials(env, monkeypatch):
+ monkeypatch.setattr(
+ app, "resealed", lambda purpose, token: "current-root" if token == "sealed-secret" else None
+ )
+ await sync()
+ assert env.writes and all(row["token_enc"] == "current-root" for row in env.writes)
+
+
+@pytest.mark.asyncio
+async def test_visibility_expiry_uses_receiving_clock_before_provider_wait(env, monkeypatch):
+ monkeypatch.setattr(app.store, "clock", AsyncMock(return_value=NOW), raising=False)
+ await sync()
+ assert env.row["visible_until"] == NOW + timedelta(seconds=app.VISIBILITY_TTL_S)
+
+
+@pytest.mark.asyncio
+async def test_configure_reused_repository_name_hides_previous_identity(env, monkeypatch):
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=8, full_name="widgets/repo", private=False)),
+ )
+ with pytest.raises(FeedInputError, match="identity changed"):
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ )
+ assert env.row["available"] == 0
+ assert env.row["visible_until"] == app.domain.EPOCH
+
+
+@pytest.mark.asyncio
+async def test_name_reuse_hides_current_row_even_if_snapshot_revision_changed(env, monkeypatch):
+ async def metadata(*args):
+ env.row["revision"] = uuid.uuid4()
+ env.row["token_enc"] = "rotated-secret"
+ return dict(id=8, full_name="widgets/repo", private=False)
+
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ with pytest.raises(FeedInputError, match="identity changed"):
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ )
+ assert env.row["available"] == 0 and env.row["token_enc"] == "rotated-secret"
+
+
+@pytest.mark.asyncio
+async def test_sync_name_reuse_hides_current_row_after_concurrent_publication(env, monkeypatch):
+ async def metadata(*args):
+ env.row["revision"] = uuid.uuid4()
+ env.row["token_enc"] = "rotated-secret"
+ return dict(id=8, full_name="widgets/repo", private=False)
+
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ with pytest.raises(FeedInputError, match="identity changed"):
+ await sync()
+ assert env.row["available"] == 0 and env.row["token_enc"] == "rotated-secret"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ["fetch_failure", "publication", "configure"])
+async def test_private_observation_tightens_access_after_revision_change(env, monkeypatch, path):
+ async def metadata(*args):
+ env.row["revision"] = uuid.uuid4()
+ env.row["token_enc"] = "rotated-secret"
+ return dict(id=7, full_name="widgets/repo", private=True)
+
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ if path == "fetch_failure":
+ monkeypatch.setattr(
+ app.provider, "pages", AsyncMock(side_effect=app.provider.FeedProviderError("http_503"))
+ )
+ with pytest.raises((FeedInputError, app.provider.FeedProviderError)):
+ if path == "configure":
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ )
+ else:
+ await sync()
+ assert env.row["available"] == 0 and env.row["token_enc"] == "rotated-secret"
+
+
+@pytest.mark.asyncio
+async def test_failed_old_credential_cannot_hide_rotated_configuration(env, monkeypatch):
+ async def metadata(*args):
+ env.row["revision"] = uuid.uuid4()
+ env.row["token_enc"] = "rotated-secret"
+ raise app.provider.FeedProviderError("http_401")
+
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ with pytest.raises(app.provider.FeedProviderError):
+ await sync()
+ assert env.row["available"] == 1 and env.row["token_enc"] == "rotated-secret"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("registered", ["widgets/repo", "Widgets/Repo"])
+async def test_configure_preserves_registered_repository_spelling(env, monkeypatch, registered):
+ env.row["repository"] = registered
+ monkeypatch.setattr(app, "seal", lambda *args: "sealed-secret")
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=7, full_name="WIDGETS/REPO", private=False)),
+ )
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/REPO",
+ token="read-only-secret",
+ allowed_repos={registered},
+ )
+ assert env.row["repository"] == registered
+ data = dict(number=1, created_at=NOW.isoformat(), updated_at=NOW.isoformat())
+ monkeypatch.setattr(
+ app.provider,
+ "pages",
+ AsyncMock(
+ side_effect=[
+ app.provider.Batch([data], True),
+ app.provider.Batch([], True),
+ app.provider.Batch([], True),
+ ]
+ ),
+ )
+ await sync()
+ assert env.records[0][3] == registered
+ assert env.records[0][1][0]["source_id"] == registered + "#1"
+
+
+@pytest.mark.asyncio
+async def test_configure_refuses_one_page_before_any_provider_request(env, monkeypatch):
+ monkeypatch.setattr(app, "seal", lambda *args: "sealed-secret")
+ with pytest.raises(FeedInputError, match="2..100 max-pages"):
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ max_pages=1,
+ )
+ app.provider.metadata.assert_not_awaited()
+ assert not env.writes
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "before_private,after_private", [(True, False), (False, False), (True, True)]
+)
+async def test_visibility_transition_replays_streams_only_when_repository_becomes_public(
+ env, monkeypatch, before_private, after_private
+):
+ cursors = {stream: NOW.isoformat() for stream in app.provider.STREAMS}
+ env.row.update(is_private=int(before_private), cursors=app.json.dumps(cursors))
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=7, full_name="widgets/repo", private=after_private)),
+ )
+ await sync()
+ replay = before_private and not after_private
+ assert [call.args[3] for call in app.provider.pages.await_args_list] == [
+ None if replay else NOW
+ ] * len(app.provider.STREAMS)
+ assert app.json.loads(env.row["cursors"]) == ({} if replay else cursors)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "before_private,after_private", [(True, False), (False, False), (True, True)]
+)
+async def test_configure_replays_content_only_when_repository_becomes_public(
+ env, monkeypatch, before_private, after_private
+):
+ cursors = {stream: NOW.isoformat() for stream in app.provider.STREAMS}
+ env.row.update(is_private=int(before_private), cursors=app.json.dumps(cursors))
+ monkeypatch.setattr(app, "seal", lambda *args: "sealed-secret")
+ monkeypatch.setattr(
+ app.provider,
+ "metadata",
+ AsyncMock(return_value=dict(id=7, full_name="widgets/repo", private=after_private)),
+ )
+ await app.configure(
+ object(),
+ org="team",
+ owner_uuid=OWNER,
+ repo="widgets/repo",
+ token="read-only-secret",
+ allowed_repos={"widgets/repo"},
+ )
+ replay = before_private and not after_private
+ assert app.json.loads(env.row["cursors"]) == ({} if replay else cursors)
+
+
+@pytest.mark.asyncio
+async def test_interrupted_public_replay_retains_reset_cursors(env, monkeypatch):
+ env.row.update(
+ is_private=1,
+ cursors=app.json.dumps({stream: NOW.isoformat() for stream in app.provider.STREAMS}),
+ )
+ data = dict(number=1, created_at=NOW.isoformat(), updated_at=NOW.isoformat())
+ monkeypatch.setattr(
+ app.provider, "pages", AsyncMock(return_value=app.provider.Batch([data], True))
+ )
+ monkeypatch.setattr(app.store, "put_items", AsyncMock(side_effect=TimeoutError("uncertain")))
+ with pytest.raises(TimeoutError):
+ await sync()
+ assert env.row["is_private"] == 0
+ assert app.json.loads(env.row["cursors"]) == {}
+ assert app.provider.pages.await_args.args[3] is None
diff --git a/tests/unit/test_github_feed_client.py b/tests/unit/test_github_feed_client.py
new file mode 100644
index 0000000000..fbb7e9ddfc
--- /dev/null
+++ b/tests/unit/test_github_feed_client.py
@@ -0,0 +1,112 @@
+"""Feed requests stay on the API origin and never accept partial provider pages."""
+
+import json
+from datetime import UTC, datetime
+
+import httpx
+import pytest
+
+from server.clients import github_feed as client
+
+
+@pytest.fixture
+def transport(monkeypatch):
+ calls = []
+ replies = []
+
+ def handle(request):
+ calls.append(request)
+ return replies.pop(0)
+
+ http = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+ monkeypatch.setattr(client, "get_client", lambda: http)
+ return calls, replies, http
+
+
+@pytest.mark.asyncio
+async def test_pages_use_separate_streams_and_exclusive_since_overlap(transport):
+ calls, replies, http = transport
+ replies.extend([httpx.Response(200, json=[{"id": 1}]), httpx.Response(200, json=[])])
+ try:
+ batch = await client.pages(
+ "widgets/repo",
+ "secret",
+ "review_comments",
+ datetime(2025, 1, 2, tzinfo=UTC),
+ max_pages=2,
+ )
+ assert batch.complete and batch.items == [{"id": 1}]
+ assert str(calls[0].url).startswith(
+ "https://api.github.com/repos/widgets/repo/pulls/comments?"
+ )
+ assert calls[0].url.params["since"] == "2025-01-01T23:59:59Z"
+ assert calls[0].headers["authorization"] == "Bearer secret"
+ finally:
+ await http.aclose()
+
+
+@pytest.mark.asyncio
+async def test_full_page_reports_incomplete_boundary(transport):
+ calls, replies, http = transport
+ replies.extend(
+ httpx.Response(200, json=[{"id": n} for n in range(start, start + 100)])
+ for start in (0, 100)
+ )
+ try:
+ batch = await client.pages("widgets/repo", "secret", "items", None, max_pages=2)
+ assert not batch.complete and len(batch.items) == 200
+ assert calls[0].url.params["state"] == "all"
+ finally:
+ await http.aclose()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [301, 401, 403, 404, 429, 500])
+async def test_refusal_never_logs_body_or_follows_redirect(transport, status):
+ calls, replies, http = transport
+ replies.append(
+ httpx.Response(
+ status, text="private-canary-secret", headers={"location": "https://evil.example"}
+ )
+ )
+ try:
+ with pytest.raises(client.FeedProviderError) as error:
+ await client.metadata("widgets/repo", "secret")
+ assert str(status) in str(error.value)
+ assert "private-canary" not in str(error.value) and len(calls) == 1
+ finally:
+ await http.aclose()
+
+
+@pytest.mark.asyncio
+async def test_bad_list_entry_refuses_whole_page(transport):
+ _, replies, http = transport
+ replies.append(httpx.Response(200, json=[{"id": 1}, "bad"]))
+ try:
+ with pytest.raises(client.FeedProviderError):
+ await client.pages("widgets/repo", "secret", "items", None, max_pages=2)
+ finally:
+ await http.aclose()
+
+
+@pytest.mark.asyncio
+async def test_size_bound_is_not_bypassed_by_json(transport, monkeypatch):
+ _, replies, http = transport
+ monkeypatch.setattr(client, "PAGE_MAX_BYTES", 32)
+ replies.append(httpx.Response(200, content=json.dumps({"data": "x" * 64}).encode()))
+ try:
+ with pytest.raises(client.FeedProviderError):
+ await client.metadata("widgets/repo", "secret")
+ finally:
+ await http.aclose()
+
+
+@pytest.mark.asyncio
+async def test_one_page_budget_refuses_before_request(transport):
+ calls, _, http = transport
+ try:
+ with pytest.raises(client.FeedProviderError, match="2..100 max-pages"):
+ await client.pages("widgets/repo", "secret", "items", None, max_pages=1)
+ assert not calls
+ finally:
+ await http.aclose()
diff --git a/tests/unit/test_github_feed_consistency.py b/tests/unit/test_github_feed_consistency.py
new file mode 100644
index 0000000000..b6353e307b
--- /dev/null
+++ b/tests/unit/test_github_feed_consistency.py
@@ -0,0 +1,141 @@
+"""Record publications read acknowledged state before changing visibility or progress."""
+
+import uuid
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from server.core import ch
+from server.tracker.app import github_feed as app
+from server.tracker.store import github_feed as store
+
+NOW = datetime(2025, 1, 2, microsecond=123000, tzinfo=UTC)
+NOW_MS = int(NOW.timestamp()) * 1000 + NOW.microsecond // 1000
+OWNER = "00000000-0000-0000-0000-000000000001"
+TARGET = "00000000-0000-0000-0000-000000000002"
+
+
+@pytest.fixture
+async def replicas(monkeypatch):
+ state = SimpleNamespace(session={}, lag=False, latest=None, stale=None, writes=[])
+
+ async def query(sql, parameters=None, settings=None, **kwargs):
+ effective = state.session | (settings or {})
+ consistent = effective.get("select_sequential_consistency") == 1 or (
+ "select_sequential_consistency=1" in "".join(sql.split())
+ )
+ fresh = not state.lag or consistent
+ if "FROM feed_owner" in sql:
+ row = dict(org="team", owner_uuid=OWNER, schema_version=1)
+ elif "FROM feed_repositories" in sql:
+ row = dict(state.latest if fresh else state.stale)
+ elif "AS now_ms" in sql:
+ row = dict(now_ms=NOW_MS, prior_ms=NOW_MS + (7 if fresh else 2))
+ else:
+ raise AssertionError("unrouted record query: " + sql)
+ return SimpleNamespace(column_names=list(row), result_rows=[list(row.values())], summary={})
+
+ async def connect(**kwargs):
+ state.session = dict(kwargs["settings"])
+ return SimpleNamespace(server_tz=None, query=query, close=AsyncMock())
+
+ async def direct(name, call, **kwargs):
+ return await call()
+
+ async def insert(table, rows, settings=None):
+ state.writes.extend((table, dict(row)) for row in rows)
+ if table == "feed_repositories":
+ state.latest = dict(rows[-1])
+
+ @asynccontextmanager
+ async def fence(name, **kwargs):
+ yield lambda value: None
+
+ monkeypatch.setattr(ch.clickhouse_connect, "get_async_client", connect)
+ monkeypatch.setattr(ch, "guard_pending_schema", AsyncMock())
+ monkeypatch.setattr(app, "fenced", fence)
+ monkeypatch.setattr(app, "resealed", lambda *args: None)
+ client = ch.ExternalCHClient(
+ host="record.example",
+ port=8443,
+ secure=True,
+ database="team_record",
+ username="record_writer",
+ password="fixture-secret",
+ )
+ monkeypatch.setattr(client, "_with_retry", direct)
+ monkeypatch.setattr(client, "insert", insert)
+ await client.connect()
+ state.client = client
+ try:
+ yield state
+ finally:
+ await client.disconnect()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("lag", [False, True])
+async def test_old_public_observation_cannot_replace_acknowledged_private_revision(replicas, lag):
+ state = replicas
+ state.lag = lag
+ state.stale = app.identity("team") | dict(
+ repository="widgets/repo",
+ repository_id=7,
+ owner_uuid=OWNER,
+ token_enc="sealed-secret",
+ is_private=0,
+ available=1,
+ visible_until=NOW + timedelta(seconds=900),
+ cursors="{}",
+ max_pages=5,
+ revision=uuid.uuid4(),
+ )
+ state.latest = state.stale | {"is_private": 1, "revision": uuid.uuid4()}
+ private_revision = state.latest["revision"]
+ published = await app._publish(
+ state.client,
+ dict(state.stale),
+ OWNER,
+ TARGET,
+ "items",
+ None,
+ {},
+ app._Observation(7, False, NOW),
+ )
+ assert published is False
+ assert state.latest["is_private"] == 1 and state.latest["revision"] == private_revision
+ assert not state.writes
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("lag", [False, True])
+async def test_new_repository_publication_exceeds_acknowledged_watermark(replicas, lag):
+ replicas.lag = lag
+ await store.put_items(
+ replicas.client,
+ "feed_comments",
+ [{"comment_id": 1}],
+ app.identity("team"),
+ "widgets/second",
+ )
+ assert replicas.writes[0][1]["version"] == NOW + timedelta(milliseconds=8)
+
+
+@pytest.mark.asyncio
+async def test_reconnected_writer_preserves_acknowledged_publication_floor(replicas):
+ previous = replicas.client.client
+ await replicas.client._reconnect(replicas.client.generation)
+ assert replicas.client.client is not previous
+ previous.close.assert_awaited_once()
+ replicas.lag = True
+ await store.put_items(
+ replicas.client,
+ "feed_comments",
+ [{"comment_id": 1}],
+ app.identity("team"),
+ "widgets/second",
+ )
+ assert replicas.writes[0][1]["version"] == NOW + timedelta(milliseconds=8)
diff --git a/tests/unit/test_github_feed_sql.py b/tests/unit/test_github_feed_sql.py
new file mode 100644
index 0000000000..67d13b11d5
--- /dev/null
+++ b/tests/unit/test_github_feed_sql.py
@@ -0,0 +1,294 @@
+"""Feed publication timestamps keep late comments in incremental tracker reads."""
+
+import json
+import uuid
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+
+import pytest
+
+from server.tracker.app import github_feed as app
+from server.tracker.domain import github_feed as domain
+from server.tracker.store import github_feed as store
+from server.tracker.store.source.base import SourceBase
+from server.tracker.store.source.embed import EmbedReads
+from tests.unit.ch_fake import RecordingCH
+from tests.unit.clickhouse_local import CLICKHOUSE, run
+
+pytestmark = pytest.mark.skipif(CLICKHOUSE is None, reason="clickhouse binary not installed")
+
+
+def _record():
+ return [
+ "USE loom",
+ *store.statements(),
+ "INSERT INTO feed_owner (org, owner_uuid) "
+ "VALUES ('team', '00000000-0000-0000-0000-000000000001')",
+ "INSERT INTO feed_repositories "
+ "(org, repository, owner_uuid, available, visible_until) "
+ "VALUES ('team', 'widgets/repo', '00000000-0000-0000-0000-000000000001', "
+ "1, '2100-01-01 00:00:00')",
+ ]
+
+
+def _read(setup, sql, params=None):
+ rows = run(setup, sql, params)
+ for row in rows:
+ for column in ("created_at", "updated_at", "closed_at"):
+ if isinstance(row.get(column), str):
+ row[column] = datetime.fromisoformat(row[column])
+ return rows
+
+
+@pytest.mark.parametrize("view,raw", [("github_issues", "feed_issues"), ("github_prs", "feed_prs")])
+@pytest.mark.parametrize("edit", [False, True])
+@pytest.mark.asyncio
+async def test_late_comment_publication_advances_the_saved_tracker_cursor(view, raw, edit):
+ setup = [
+ *_record(),
+ f"INSERT INTO {raw} "
+ "(org, repository, number, source_id, body, updated_at, version) "
+ "VALUES ('team', 'widgets/repo', 1, 'widgets/repo#1', 'Item body', "
+ "'2000-01-01 00:00:00', '2025-01-02 00:00:00.001')",
+ f"CREATE TABLE saved_cursor ENGINE = Memory AS SELECT updated_at, source_id FROM {view}",
+ "INSERT INTO feed_comments "
+ "(org, repository, comment_id, item_number, body, updated_at, version) "
+ "VALUES ('team', 'widgets/repo', 1, 1, 'Late reply', "
+ "'1999-01-01 00:00:00', '2025-01-02 00:00:00.002')",
+ ]
+ if edit:
+ setup += [
+ "TRUNCATE TABLE saved_cursor",
+ f"INSERT INTO saved_cursor SELECT updated_at, source_id FROM {view}",
+ "INSERT INTO feed_comments "
+ "(org, repository, comment_id, item_number, body, updated_at, version) "
+ "VALUES ('team', 'widgets/repo', 1, 1, 'Edited reply', "
+ "'1999-01-01 00:00:01', '2025-01-02 00:00:00.003')",
+ ]
+ rows = run(
+ setup,
+ f"SELECT body, toString(updated_at) AS published FROM {view} "
+ "WHERE (updated_at, source_id) > (SELECT updated_at, source_id FROM saved_cursor)",
+ )
+ assert rows == [
+ {
+ "body": "Item body\n" + ("Edited reply" if edit else "Late reply"),
+ "published": "2025-01-02 00:00:00.00" + ("3" if edit else "2"),
+ }
+ ]
+
+ class Source(EmbedReads):
+ def __init__(self):
+ self.db = "loom"
+
+ def repos(self):
+ return ["widgets/repo"]
+
+ def _privacy(self):
+ return "is_private = 0 AND "
+
+ async def _rows(self, name, sql, params, **kwargs):
+ return _read(setup, sql, params)
+
+ source = Source()
+ cursor = datetime(2025, 1, 2, microsecond=2000 if edit else 1000, tzinfo=UTC)
+ published = cursor.replace(microsecond=3000 if edit else 2000)
+ kind = "issue" if view == "github_issues" else "pr"
+ for since_id in (None, "widgets/repo#1"):
+ options = dict(kind=kind, closed_days=365, body_chars=600, since_id=since_id)
+ changed = await source.embed_items("team", "feed", since=cursor, **options)
+ assert len(changed) == 1 and changed[0]["body"] == rows[0]["body"]
+ assert changed[0]["updated_at"] == published.replace(tzinfo=None)
+ assert await source.embed_items("team", "feed", since=published, **options) == []
+
+
+def test_millisecond_reader_resumes_after_two_thousand_rows_from_one_publication():
+ setup = [
+ *_record(),
+ "INSERT INTO feed_issues "
+ "(org, repository, number, source_id, updated_at, version) "
+ "SELECT 'team', 'widgets/repo', number + 1, "
+ "concat('widgets/repo#', toString(number + 1)), "
+ "toDateTime64('2000-01-01 00:00:00', 3), "
+ "toDateTime64('2025-01-02 00:00:00.123456', 6) FROM numbers(2001)",
+ ]
+ select = "SELECT source_id, updated_at FROM github_issues "
+ order = " ORDER BY updated_at, source_id LIMIT 2000"
+ first = _read(setup, select + order)
+ assert len(first) == 2000
+ cursor = first[-1]
+ second = _read(
+ setup,
+ select + "WHERE (updated_at, source_id) > ({since:DateTime64(3)}, {id:String})" + order,
+ {"since": cursor["updated_at"], "id": cursor["source_id"]},
+ )
+ assert len(second) == 1
+ assert second[0]["source_id"] not in {row["source_id"] for row in first}
+ assert cursor["updated_at"] == datetime(2025, 1, 2, microsecond=123000)
+
+
+def _insert_recorded(table, row):
+ def value(item):
+ if isinstance(item, datetime):
+ return item.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S.%f")
+ return str(item)
+
+ columns = ", ".join(row)
+ data = json.dumps(row, default=value)
+ return f"INSERT INTO {table} ({columns}) SELECT {columns} FROM format(JSONEachRow, $${data}$$)"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("view,raw", [("github_issues", "feed_issues"), ("github_prs", "feed_prs")])
+@pytest.mark.parametrize("publication", ["item", "comment"])
+@pytest.mark.parametrize("metadata_private,published_private", [(0, 1), (0, 0), (1, 0), (1, 1)])
+async def test_contract_reader_keeps_publication_privacy_when_repository_metadata_lags(
+ monkeypatch, view, raw, publication, metadata_private, published_private
+):
+ owner = "00000000-0000-0000-0000-000000000001"
+ now = datetime(2025, 1, 2, tzinfo=UTC)
+ current = app.identity("team") | dict(
+ repository="widgets/repo",
+ repository_id=7,
+ owner_uuid=uuid.UUID(owner),
+ token_enc="",
+ is_private=metadata_private,
+ available=1,
+ visible_until=datetime(2100, 1, 1, tzinfo=UTC),
+ cursors="{}",
+ max_pages=2,
+ revision=uuid.UUID(int=3),
+ )
+ ch = (
+ RecordingCH()
+ .on("FROM feed_owner", [{"org": "team", "owner_uuid": owner, "schema_version": 1}])
+ .on("AS now_ms", [{"now_ms": int(now.timestamp()) * 1000, "prior_ms": 0}])
+ .on("FROM feed_repositories", [current])
+ )
+
+ @asynccontextmanager
+ async def fence(*args, **kwargs):
+ yield lambda uncertain: None
+
+ monkeypatch.setattr(app, "fenced", fence)
+ monkeypatch.setattr(app, "resealed", lambda *args: None)
+ item = domain.item_row(
+ "widgets/repo",
+ dict(number=1, body="Item body", created_at=now.isoformat(), updated_at=now.isoformat()),
+ is_pr=view == "github_prs",
+ )
+ comment = domain.comment_row(
+ "widgets/repo",
+ dict(
+ id=1,
+ issue_url="https://api.github.com/repos/widgets/repo/issues/1",
+ pull_request_url="https://api.github.com/repos/widgets/repo/pulls/1",
+ body="New reply",
+ created_at=now.isoformat(),
+ updated_at=now.isoformat(),
+ ),
+ review=view == "github_prs",
+ )
+ table = raw if publication == "item" else "feed_comments"
+ stream = (
+ "items"
+ if publication == "item"
+ else ("review_comments" if view == "github_prs" else "comments")
+ )
+ assert await app._publish(
+ ch,
+ current,
+ owner,
+ "00000000-0000-0000-0000-000000000002",
+ stream,
+ now,
+ {table: [item if publication == "item" else comment]},
+ app._Observation(7, bool(published_private), now),
+ )
+ # One replica has learned the raw publication but still holds the previous
+ # unexpired repository part. Keep actual writer rows and omit its metadata writes.
+ setup = [*_record()[:-1], _insert_recorded("feed_repositories", current)]
+ if publication == "comment":
+ setup.append(
+ _insert_recorded(raw, item | app.identity("team") | {"repository": "widgets/repo"})
+ )
+ setup.extend(_insert_recorded(table, row) for row in ch.inserted(table))
+ ordinary = object.__new__(SourceBase)
+ ordinary.cfg = SimpleNamespace(restricted=False)
+ rows = run(
+ setup,
+ f"SELECT 'ordinary' AS reach, body, is_private, toString(id) AS id FROM {view} FINAL "
+ f"WHERE {ordinary._privacy()}1 UNION ALL "
+ f"SELECT 'restricted' AS reach, body, is_private, toString(id) AS id FROM {view} FINAL",
+ )
+ body = "Item body\n" + ("New reply" if publication == "comment" else "")
+ effective_private = int(bool(metadata_private or published_private))
+ kind = "github_issue" if view == "github_issues" else "github_pr"
+ identity = f"{kind}{'_private' if effective_private else ''}:widgets/repo#1"
+ expected = {
+ "body": body,
+ "is_private": effective_private,
+ "id": str(uuid.uuid5(uuid.NAMESPACE_DNS, identity)),
+ }
+ assert [row for row in rows if row["reach"] == "ordinary"] == (
+ [] if effective_private else [{"reach": "ordinary", **expected}]
+ )
+ assert [row for row in rows if row["reach"] == "restricted"] == [
+ {"reach": "restricted", **expected}
+ ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("change", ["hide", "configure", "cursor"])
+async def test_repository_controls_replace_future_versions_in_the_record(change):
+ clock_row = run([], "SELECT toUnixTimestamp64Milli(now64(3)) AS ms")[0]
+ prior = domain.EPOCH + timedelta(days=1, milliseconds=int(clock_row["ms"]))
+ current = app.identity("team") | dict(
+ repository="widgets/repo",
+ repository_id=7,
+ owner_uuid=uuid.UUID("00000000-0000-0000-0000-000000000001"),
+ token_enc="old-sealed",
+ is_private=0,
+ available=1,
+ visible_until=prior,
+ cursors="{}",
+ max_pages=2,
+ revision=uuid.UUID(int=3),
+ version=prior,
+ )
+ setup = [
+ *_record(),
+ _insert_recorded("feed_repositories", current),
+ "INSERT INTO feed_issues (org, repository, number, source_id, body) "
+ "VALUES ('team', 'widgets/repo', 1, 'widgets/repo#1', 'Public item')",
+ ]
+ changes = {
+ "hide": {"available": 0, "visible_until": domain.EPOCH},
+ "configure": {"token_enc": "rotated-sealed", "max_pages": 7},
+ "cursor": {"cursors": json.dumps({"items": "2025-01-02T00:00:00+00:00"})},
+ }[change]
+ ch = RecordingCH().on("AS now_ms", lambda sql, params: run(setup, sql, params))
+ await store.put_repository(ch, current | changes | {"revision": uuid.UUID(int=4)})
+ written = ch.inserted("feed_repositories")[0]
+ assert written["version"] == prior + timedelta(milliseconds=1)
+ rows = run(
+ [*setup, _insert_recorded("feed_repositories", written)],
+ "SELECT available, token_enc, cursors, max_pages, "
+ "toUnixTimestamp64Milli(visible_until) AS visible_ms, "
+ "toUnixTimestamp64Milli(version) AS version_ms, "
+ "(SELECT count() FROM github_issues) AS visible_items "
+ "FROM feed_repositories FINAL WHERE org = 'team' AND repository = 'widgets/repo'",
+ )
+ assert len(rows) == 1
+ for column in ("available", "token_enc", "cursors"):
+ assert rows[0][column] == written[column]
+ assert int(rows[0]["max_pages"]) == written["max_pages"]
+ expected_visible_ms = (
+ int(written["visible_until"].timestamp()) * 1000
+ + written["visible_until"].microsecond // 1000
+ )
+ prior_ms = int(prior.timestamp()) * 1000 + prior.microsecond // 1000
+ assert int(rows[0]["visible_ms"]) == expected_visible_ms
+ assert int(rows[0]["version_ms"]) == prior_ms + 1
+ assert int(rows[0]["visible_items"]) == (0 if change == "hide" else 1)
diff --git a/tests/unit/test_github_feed_store.py b/tests/unit/test_github_feed_store.py
new file mode 100644
index 0000000000..76818c851b
--- /dev/null
+++ b/tests/unit/test_github_feed_store.py
@@ -0,0 +1,192 @@
+"""Feed content publications advance beyond every persisted record version."""
+
+import json
+import uuid
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from unittest.mock import AsyncMock
+
+import pytest
+
+from server.tracker.app import github_feed as app
+from server.tracker.store import github_feed as store
+from tests.unit.ch_fake import RecordingCH
+
+NOW = datetime(2025, 1, 2, microsecond=123000, tzinfo=UTC)
+NOW_MS = int(NOW.timestamp()) * 1000 + NOW.microsecond // 1000
+IDENTITY = {name: "team" if name == "org" else "-" for name in store.IDENTITY}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("item_table", ["feed_issues", "feed_prs"])
+async def test_item_then_comment_publication_advances_with_an_unchanged_record_clock(item_table):
+ ch = RecordingCH()
+
+ def publication_clock(sql, params):
+ versions = [
+ int(row["version"].timestamp()) * 1000 + row["version"].microsecond // 1000
+ for call in ch.calls
+ if call.kind == "insert"
+ for row in call.rows
+ ]
+ return [{"now_ms": NOW_MS, "prior_ms": max(versions, default=0)}]
+
+ ch.on("AS now_ms", publication_clock)
+ await store.put_items(ch, item_table, [{"number": 1}], IDENTITY, "widgets/repo")
+ await store.put_items(ch, "feed_comments", [{"comment_id": 1}], IDENTITY, "widgets/repo")
+ item = ch.inserted(item_table)[0]["version"]
+ comment = ch.inserted("feed_comments")[0]["version"]
+ assert item == NOW
+ assert comment == NOW + timedelta(milliseconds=1)
+ assert item.microsecond % 1000 == comment.microsecond % 1000 == 0
+ assert all(call.params == {"org": "team"} for call in ch.calls if call.kind == "query_one")
+
+
+@pytest.mark.asyncio
+async def test_publication_advances_beyond_persisted_versions_ahead_of_the_record_clock():
+ ch = RecordingCH().on("AS now_ms", [{"now_ms": NOW_MS, "prior_ms": NOW_MS + 7}])
+ await store.put_items(ch, "feed_comments", [{"comment_id": 1}], IDENTITY, "widgets/repo")
+ assert ch.inserted("feed_comments")[0]["version"] == NOW + timedelta(milliseconds=8)
+
+
+@pytest.mark.asyncio
+async def test_other_repository_publication_exceeds_the_records_logical_watermark():
+ ch = RecordingCH()
+ await ch.insert(
+ "feed_issues",
+ [IDENTITY | {"repository": "widgets/first", "version": NOW + timedelta(milliseconds=7)}],
+ )
+
+ def publication_clock(sql, params):
+ versions = [
+ int(row["version"].timestamp()) * 1000 + row["version"].microsecond // 1000
+ for call in ch.calls
+ if call.kind == "insert"
+ for row in call.rows
+ if row["org"] == params["org"]
+ and ("repo" not in params or row["repository"] == params["repo"])
+ ]
+ return [{"now_ms": NOW_MS, "prior_ms": max(versions, default=0)}]
+
+ ch.on("AS now_ms", publication_clock)
+ await store.put_items(ch, "feed_prs", [{"number": 1}], IDENTITY, "widgets/second")
+ assert ch.inserted("feed_prs")[0]["version"] == NOW + timedelta(milliseconds=8)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("prior_offset_ms", [10_000, 0, -10_000])
+@pytest.mark.parametrize("change", ["hide", "configure", "cursor"])
+async def test_repository_controls_replace_persisted_rows_despite_clock_rollback(
+ monkeypatch, prior_offset_ms, change
+):
+ owner = "00000000-0000-0000-0000-000000000001"
+ target = "00000000-0000-0000-0000-000000000002"
+ prior = NOW + timedelta(milliseconds=prior_offset_ms)
+ previous_cursor = (NOW - timedelta(days=1)).isoformat()
+ original = app.identity("team") | dict(
+ repository="widgets/repo",
+ repository_id=7,
+ owner_uuid=owner,
+ token_enc="sealed-secret",
+ is_private=0,
+ available=1,
+ visible_until=NOW + timedelta(seconds=900),
+ cursors=json.dumps({"items": previous_cursor}),
+ max_pages=2,
+ revision=uuid.uuid4(),
+ version=prior,
+ )
+ ch = RecordingCH()
+ await ch.insert(
+ "feed_repositories",
+ [
+ original,
+ original | {"repository": "widgets/other", "version": NOW + timedelta(hours=1)},
+ original | {"org": "other-team", "version": NOW + timedelta(hours=2)},
+ ],
+ )
+
+ def persisted(params):
+ return [
+ row
+ for row in ch.inserted("feed_repositories")
+ if row["org"] == params["org"] and row["repository"] == params["repo"]
+ ]
+
+ def publication_clock(sql, params):
+ assert "WHERE org = {org:String} AND repository = {repo:String}" in sql
+ assert "LIMIT 1 SETTINGS select_sequential_consistency = 1" in sql
+ latest = max(row.get("version", NOW) for row in persisted(params))
+ prior_ms = int(latest.timestamp()) * 1000 + latest.microsecond // 1000
+ return [{"now_ms": NOW_MS, "prior_ms": prior_ms}]
+
+ def final_repository(sql, params):
+ # Model ReplacingMergeTree's maximum-version winner, including the
+ # server-clock default when the writer omits an explicit version.
+ return [max(persisted(params), key=lambda row: row.get("version", NOW))]
+
+ ch.on("AS now_ms", publication_clock)
+ ch.on("FROM feed_repositories FINAL", final_repository)
+ ch.on("AS ms", [{"ms": NOW_MS}])
+ ch.on("FROM feed_owner", [{"org": "team", "owner_uuid": owner, "schema_version": 1}])
+ ch.on("FROM system.databases", [{"id": target}])
+ keys = []
+
+ @asynccontextmanager
+ async def fence(key, **kwargs):
+ keys.append(key)
+ yield lambda uncertain: None
+
+ monkeypatch.setattr(app, "fenced", fence)
+ monkeypatch.setattr(app, "resealed", lambda *args: None)
+ monkeypatch.setattr(app, "unseal", lambda *args: "read-only-secret")
+ monkeypatch.setattr(app, "seal", lambda *args: "rotated-sealed")
+ metadata = AsyncMock(return_value={"id": 7, "full_name": "widgets/repo", "private": False})
+ monkeypatch.setattr(app.provider, "metadata", metadata)
+ if change == "hide":
+ metadata.side_effect = app.provider.FeedProviderError("http_404")
+ with pytest.raises(app.provider.FeedProviderError):
+ await app.sync(
+ ch,
+ org="team",
+ owner_uuid=owner,
+ repo="widgets/repo",
+ allowed_repos={"widgets/repo"},
+ )
+ elif change == "configure":
+ await app.configure(
+ ch,
+ org="team",
+ owner_uuid=owner,
+ repo="widgets/repo",
+ token="rotated-read-only-secret",
+ allowed_repos={"widgets/repo"},
+ max_pages=7,
+ )
+ else:
+ assert await app._publish(
+ ch, original, owner, target, "items", NOW, {}, app._Observation(7, True, NOW)
+ )
+ written = ch.inserted("feed_repositories")[3:]
+ expected_version = max(NOW, prior + timedelta(milliseconds=1))
+ assert written[0]["version"] == expected_version
+ assert all(row["version"] > prior and row["version"].microsecond % 1000 == 0 for row in written)
+ assert all(
+ before["version"] < after["version"]
+ for before, after in zip(written, written[1:], strict=False)
+ )
+ current = await store.repository(ch, "team", "widgets/repo")
+ assert current == written[-1]
+ assert current["revision"] != original["revision"]
+ assert keys == [f"tracker-feed.writer/{target}"]
+ if change == "hide":
+ assert current["available"] == 0 and current["visible_until"] == app.domain.EPOCH
+ assert current["token_enc"] == original["token_enc"]
+ assert current["cursors"] == original["cursors"]
+ elif change == "configure":
+ assert current["token_enc"] == "rotated-sealed" and current["max_pages"] == 7
+ assert current["cursors"] == original["cursors"]
+ else:
+ assert current["is_private"] == 1 and current["available"] == 1
+ assert json.loads(current["cursors"])["items"] == NOW.isoformat()
+ assert current["token_enc"] == original["token_enc"]
diff --git a/tests/unit/test_rmt_carry.py b/tests/unit/test_rmt_carry.py
index da7ac22d5f..ceec90eb4c 100644
--- a/tests/unit/test_rmt_carry.py
+++ b/tests/unit/test_rmt_carry.py
@@ -477,6 +477,10 @@ async def _types(self: Any, table: str, columns: list[str]) -> Any:
monkeypatch.setattr(type(ch_mod.CHClient()), "client", property(lambda self: driver))
async def _stray(self: Any, sql: str, params: Any = None) -> Any:
+ if "toUnixTimestamp64Milli(now64(3))" in sql:
+ now = rmt_carry.server_now()
+ assert now is not None
+ return {"ms": int(now * 1000)}
assert "system.columns" in sql
return {"n": driver.stray}
diff --git a/tests/unit/test_sdk_set_coalescer.py b/tests/unit/test_sdk_set_coalescer.py
index 55bc8d8609..303cd0f0e4 100644
--- a/tests/unit/test_sdk_set_coalescer.py
+++ b/tests/unit/test_sdk_set_coalescer.py
@@ -136,11 +136,21 @@ async def test_two_sets_go_out_as_two_point_writes(agent: Agent) -> None:
assert r1["memory_id"] == "id-a" and "coalesced" not in r2
-async def test_third_set_inside_the_window_starts_a_bulk(agent: Agent) -> None:
+async def test_third_set_inside_the_window_starts_a_bulk(
+ agent: Agent, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from tests.unit.clock_fake import ManualClock
+
+ loop = asyncio.get_running_loop()
+ clock = ManualClock(t=loop.time())
+ monkeypatch.setattr(loop, "time", clock.monotonic)
t: Any = agent.transport
t.row_verdicts = {"d": "merge", "e": "conflict"}
t.bulk_extra = {"merged": ["d"], "conflicted": ["e"]}
- results = await asyncio.gather(*[agent.memory.set(k, {"v": k}) for k in "abcde"])
+ pending = asyncio.gather(*[agent.memory.set(k, {"v": k}) for k in "abcde"])
+ await asyncio.sleep(0)
+ await agent.memory.flush_sets()
+ results = await pending
assert sorted(t.paths()) == ["/v1/memory.bulk_set", "/v1/memory.set", "/v1/memory.set"]
bulk = [p for path, p in t.calls if path == "/v1/memory.bulk_set"][0]
assert [e["memory_key"] for e in bulk["entries"]] == ["c", "d", "e"]
diff --git a/tests/unit/test_tracker_feed_cli.py b/tests/unit/test_tracker_feed_cli.py
new file mode 100644
index 0000000000..251fa45116
--- /dev/null
+++ b/tests/unit/test_tracker_feed_cli.py
@@ -0,0 +1,305 @@
+"""Operator errors stay actionable without exposing database credentials."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from server.cli import tracker_feed as cli
+from server.core.github_values import FeedInputError
+
+
+def test_missing_credential_names_only_the_environment_variable(monkeypatch):
+ monkeypatch.delenv("FEED_TEST_MISSING_SECRET", raising=False)
+ with pytest.raises(FeedInputError, match="FEED_TEST_MISSING_SECRET"):
+ cli._environment("FEED_TEST_MISSING_SECRET")
+
+
+@pytest.mark.asyncio
+async def test_deployment_connect_failure_still_disconnects(monkeypatch):
+ from server.core import ch
+
+ deployment = SimpleNamespace(
+ connected=False, connect=AsyncMock(side_effect=OSError("failed")), disconnect=AsyncMock()
+ )
+ monkeypatch.setattr(ch, "get_client", lambda: deployment)
+ from server.settings import settings
+
+ monkeypatch.setattr(settings, "tracker_org", "team")
+ with pytest.raises(OSError):
+ await cli._run(SimpleNamespace(feed_command="init", org="team"))
+ deployment.disconnect.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_record_connect_failure_still_disconnects(monkeypatch):
+ from server.cli import _ch
+ from server.core import ch, hosts
+
+ client = SimpleNamespace(
+ connect=AsyncMock(side_effect=OSError("failed")), disconnect=AsyncMock()
+ )
+ monkeypatch.setattr(ch, "ExternalCHClient", lambda **kwargs: client)
+ monkeypatch.setattr(hosts, "vet_host", AsyncMock(return_value=("", "203.0.113.1")))
+ monkeypatch.setenv("FEED_TEST_WRITER_PASSWORD", "secret")
+ endpoint = SimpleNamespace(host="record.example", database="record", port=8443, secure=True)
+ args = SimpleNamespace(
+ org="team", writer_user="writer", writer_password_env="FEED_TEST_WRITER_PASSWORD"
+ )
+ deployment = SimpleNamespace()
+ monkeypatch.setattr(_ch, "connect_checked", AsyncMock())
+ with pytest.raises(OSError):
+ await cli._connection(args, endpoint, deployment)
+ client.disconnect.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "grant",
+ [
+ "GRANT ALL ON *.* TO writer",
+ "GRANT INSERT ON loom.* TO writer",
+ "GRANT broad_role TO writer",
+ ],
+)
+async def test_record_connection_refuses_broad_or_role_writer_grants(monkeypatch, grant):
+ from server.cli import _ch
+ from server.core import ch, hosts
+
+ client = SimpleNamespace(
+ connect=AsyncMock(),
+ disconnect=AsyncMock(),
+ query_all=AsyncMock(return_value=[{"grant": grant}]),
+ )
+ monkeypatch.setattr(ch, "ExternalCHClient", lambda **kwargs: client)
+ monkeypatch.setattr(hosts, "vet_host", AsyncMock(return_value=("", "203.0.113.1")))
+ monkeypatch.setenv("FEED_TEST_WRITER_PASSWORD", "secret")
+ endpoint = SimpleNamespace(host="record.example", database="record", port=8443, secure=True)
+ args = SimpleNamespace(
+ org="team", writer_user="writer", writer_password_env="FEED_TEST_WRITER_PASSWORD"
+ )
+ deployment = SimpleNamespace()
+ monkeypatch.setattr(_ch, "connect_checked", AsyncMock())
+ with pytest.raises(FeedInputError, match="writer grants"):
+ await cli._connection(args, endpoint, deployment)
+ client.disconnect.assert_awaited_once()
+
+
+def test_unexpected_failure_does_not_print_secret(monkeypatch, capsys):
+ monkeypatch.setattr(cli, "_run", AsyncMock(side_effect=OSError("password=secret")))
+ assert cli._command(SimpleNamespace()) == 1
+ assert capsys.readouterr().err == "tracker feed refused: OSError\n"
+
+
+@pytest.mark.asyncio
+async def test_init_refuses_unusable_org_before_connecting(monkeypatch):
+ from server.core import ch
+ from server.settings import settings
+
+ connect = AsyncMock()
+ monkeypatch.setattr(settings, "tracker_org", "team")
+ monkeypatch.setattr(ch, "get_client", connect)
+ with pytest.raises(FeedInputError, match="configured tracker organization"):
+ await cli._run(SimpleNamespace(feed_command="init", org="other"))
+ connect.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "engine",
+ [
+ "",
+ "ReplacingMergeTree",
+ "SharedReplacingMergeTree",
+ "ReplicatedReplacingMergeTree",
+ "Distributed",
+ "UnknownEngine",
+ ],
+)
+async def test_writer_refuses_record_engines_without_acknowledged_read_consistency(engine):
+ from server.tracker.store import github_feed as store
+
+ async def query_all(sql, params=None):
+ if sql.startswith("SHOW GRANTS"):
+ return [
+ {
+ "grant": "GRANT SELECT, INSERT, CREATE TABLE, CREATE VIEW ON team_record.* TO writer"
+ }
+ ]
+ assert params == {"tables": list(store.TABLES)}
+ assert "database = currentDatabase()" in sql
+ return [{"name": "feed_repositories", "engine": engine}] if engine else []
+
+ client = SimpleNamespace(query_all=query_all)
+ if engine in {"", "ReplacingMergeTree", "SharedReplacingMergeTree"}:
+ await cli._validate_writer(client, "team_record")
+ else:
+ with pytest.raises(FeedInputError, match="unsupported feed record table engine"):
+ await cli._validate_writer(client, "team_record")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("refusal", ["authority", "cell"])
+async def test_init_refuses_deployment_admission_before_record_or_write(monkeypatch, refusal):
+ from server.cli import _ch
+ from server.core import ch, pg, pg_profile
+ from server.settings import settings
+ from server.tracker.app import github_feed as feed
+ from server.tracker.store import github_feed as store
+
+ deployment = SimpleNamespace(connected=False, connect=AsyncMock(), disconnect=AsyncMock())
+ record = SimpleNamespace(disconnect=AsyncMock())
+ admission_error = (
+ OSError("wrong deployment") if refusal == "authority" else _ch.OrgNotHeld("not held")
+ )
+ authority = AsyncMock(side_effect=admission_error if refusal == "authority" else None)
+ held = AsyncMock(side_effect=admission_error if refusal == "cell" else None)
+ monkeypatch.setattr(settings, "tracker_org", "team")
+ monkeypatch.setattr(ch, "get_client", lambda: deployment)
+ monkeypatch.setattr(pg_profile, "check_configured", authority)
+ monkeypatch.setattr(_ch, "check_orgs", held)
+ monkeypatch.setattr(pg, "close_pool", AsyncMock())
+ owner = AsyncMock(return_value="owner")
+ monkeypatch.setattr(store, "database_uuid", owner)
+ external = AsyncMock(return_value=record)
+ monkeypatch.setattr(cli, "_connection", external)
+ initialize = AsyncMock()
+ monkeypatch.setattr(feed, "initialize", initialize)
+ args = SimpleNamespace(
+ feed_command="init",
+ org="team",
+ host="record.example",
+ port=8443,
+ insecure=False,
+ database="record",
+ )
+
+ with pytest.raises(type(admission_error), match=str(admission_error)):
+ await cli._run(args)
+
+ authority.assert_awaited_once_with(deployment)
+ if refusal == "cell":
+ held.assert_awaited_once_with(("team",))
+ else:
+ held.assert_not_awaited()
+ owner.assert_not_awaited()
+ external.assert_not_awaited()
+ initialize.assert_not_awaited()
+ deployment.disconnect.assert_awaited()
+
+
+@pytest.mark.asyncio
+async def test_init_checks_actual_deployment_before_external_record_and_closes_both(monkeypatch):
+ from server.cli import _ch
+ from server.core import ch, hosts, pg, pg_profile
+ from server.settings import settings
+ from server.tracker.app import github_feed as feed
+ from server.tracker.store import github_feed as store
+
+ events = []
+ deployment = SimpleNamespace(connected=False)
+
+ async def deployment_connect():
+ events.append("deployment connect")
+ deployment.connected = True
+
+ async def deployment_disconnect():
+ events.append("deployment disconnect")
+
+ async def authority(client):
+ assert client is deployment
+ events.append("authority")
+
+ async def held(orgs):
+ assert orgs == ("team",)
+ events.append("held")
+
+ async def record_connect():
+ events.append("record connect")
+
+ async def record_disconnect():
+ events.append("record disconnect")
+
+ async def owner(client):
+ assert client is deployment
+ events.append("owner")
+ return "owner"
+
+ async def vet(*args, **kwargs):
+ events.append("vet")
+ return "", "203.0.113.1"
+
+ deployment.connect = AsyncMock(side_effect=deployment_connect)
+ deployment.disconnect = AsyncMock(side_effect=deployment_disconnect)
+ record = SimpleNamespace(
+ connect=AsyncMock(side_effect=record_connect),
+ disconnect=AsyncMock(side_effect=record_disconnect),
+ )
+ checked = AsyncMock(side_effect=authority)
+ monkeypatch.setattr(settings, "tracker_org", "team")
+ monkeypatch.setattr(ch, "get_client", lambda: deployment)
+ monkeypatch.setattr(ch, "ExternalCHClient", lambda **kwargs: record)
+ monkeypatch.setattr(pg_profile, "check_configured", checked)
+ monkeypatch.setattr(_ch, "check_orgs", AsyncMock(side_effect=held))
+ close_pool = AsyncMock()
+ monkeypatch.setattr(pg, "close_pool", close_pool)
+ monkeypatch.setattr(hosts, "vet_host", AsyncMock(side_effect=vet))
+ monkeypatch.setattr(store, "database_uuid", AsyncMock(side_effect=owner))
+ validated = AsyncMock()
+ monkeypatch.setattr(cli, "_validate_writer", validated)
+ initialize = AsyncMock()
+ monkeypatch.setattr(feed, "initialize", initialize)
+ monkeypatch.setenv("FEED_TEST_WRITER_PASSWORD", "secret")
+ args = SimpleNamespace(
+ feed_command="init",
+ org="team",
+ host="record.example",
+ port=8443,
+ insecure=False,
+ database="record",
+ writer_user="writer",
+ writer_password_env="FEED_TEST_WRITER_PASSWORD",
+ )
+
+ assert await cli._run(args) == {"initialized": True}
+
+ assert checked.await_count >= 1
+ assert all(call.args == (deployment,) for call in checked.await_args_list)
+ assert events.index("authority") < events.index("held") < events.index("owner")
+ assert events.index("owner") < events.index("vet") < events.index("record connect")
+ validated.assert_awaited_once_with(record, "record")
+ initialize.assert_awaited_once_with(record, "team", "owner")
+ record.connect.assert_awaited_once()
+ record.disconnect.assert_awaited_once()
+ deployment.disconnect.assert_awaited_once()
+ close_pool.assert_awaited_once()
+ assert events[-2:] == ["record disconnect", "deployment disconnect"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("refusal", ["authority", "cell"])
+async def test_external_record_connect_refuses_unadmitted_deployment(monkeypatch, refusal):
+ from server.cli import _ch
+ from server.core import pg, pg_profile
+
+ deployment = SimpleNamespace(connected=True, disconnect=AsyncMock())
+ record = SimpleNamespace(connect=AsyncMock())
+ error = OSError("wrong deployment") if refusal == "authority" else _ch.OrgNotHeld("not held")
+ authority = AsyncMock(side_effect=error if refusal == "authority" else None)
+ held = AsyncMock(side_effect=error if refusal == "cell" else None)
+ close_pool = AsyncMock()
+ monkeypatch.setattr(pg_profile, "check_configured", authority)
+ monkeypatch.setattr(_ch, "check_orgs", held)
+ monkeypatch.setattr(pg, "close_pool", close_pool)
+
+ with pytest.raises(type(error), match=str(error)):
+ await _ch.connect_external(record, deployment=deployment, orgs=("team",))
+
+ authority.assert_awaited_once_with(deployment)
+ if refusal == "cell":
+ held.assert_awaited_once_with(("team",))
+ else:
+ held.assert_not_awaited()
+ record.connect.assert_not_awaited()
+ deployment.disconnect.assert_awaited_once()
+ close_pool.assert_awaited_once()