Deploy the library server: book highlights (registry API §11)

CONTRIBUTING.md

@@ -51,6 +51,7 @@ npx tsx tools/reader-library-e2e.mjs [url] [shots]   # libraries: gallery, uploa
 node tools/reader-stream-e2e.mjs [url] [shots]       # a streamed 60-page book: what is fetched, and when
 npx tsx tools/reader-library-book-e2e.ts <library> <token> [url] [--read-token t]   # a book published into a (test) library, opened from it
 npx tsx tools/reader-library-auth-e2e.ts [url] [--port 4923]   # signing in to a private library (a fake one in-process), refresh, sign-out
+npx tsx tools/reader-media-recover-e2e.ts <library> <token> <disk-dir>   # media adopted at two paths, one deleted while the book is open: the reader recovers (a disk test library, ADOPT_PREFIXES=books/, built showcase)
 npx tsx tools/question-current-e2e.ts <library> <token> [--read-token t]   # a corrected question reaches readers with no new links item (test library: writes)
 node tools/reader-insert-menu-e2e.mjs [url]          # author mode's right-click menu: what is added lands where you clicked
 node tools/reader-save-e2e.mjs [url]                 # author mode's Save: the filled button, Ctrl+S, onDirtyChange, the ref's save()

docs/guides/run-a-library-server.md

@@ -167,6 +167,10 @@ the asset with that path as its location. Nothing is copied.
   cache). If every copy is gone, readers get a 404 (they verify what they get, so other bytes at a path are
   refused). Adopting a path again HEADs it and reads nothing. The schema step that allows this (step 4: `locations`
   keyed by key, store and path, with `checked_at` and `missing`) runs by itself on the first start.
+  A book already open in the reader recovers too: a picture, video, audio or model whose URL fails to load is asked
+  for again once (the reader drops the index shard it held and fetches it past the browser's cache) and drawn from
+  the copy the library names now; the same goes for a signed link (`/v1/assets/{key}/link`) that expired while a
+  video played, when the host's resolver hands out links.
 - A key the library already has (uploaded, or adopted elsewhere) stays where it is; adopting it again only updates
   its title, tags and the other descriptive fields, and reads nothing.
 

packages/library-server/README.md

@@ -23,6 +23,11 @@ DATABASE_URL=pglite:./.data/db npx tsx src/cli.ts key add ci --roles author
 
 (PGlite allows one process at a time: stop `serve` first, or use the admin API.) Docker, from the repository root: `docker build -f packages/library-server/Dockerfile -t chalk-library-server .`
 
+It also keeps the **highlights** of books (registry API §11): the lines of a book's text and the questions linked to
+them, as records an editor and a matcher change one by one (`v1/books/<slug>/lines`, `…/links`), so reading apps get
+them from the library. They are migration step 5 of its database (two tables, `book_lines` and `book_links`), applied
+when the server starts.
+
 Every setting, Postgres and S3 setup, bucket CORS, deploying and security:
 [docs/guides/run-a-library-server.md](../../docs/guides/run-a-library-server.md). Using libraries in the
 reader and the editor: [docs/guides/media-libraries.md](../../docs/guides/media-libraries.md).

packages/library-server/src/adopt.test.ts

@@ -356,7 +356,7 @@ describe('adopting: off by default, and a private library', () => {
     mkdirSync(join(dir, 'chalk-books', 'race'), { recursive: true }); writeFileSync(join(dir, ...path.split('/')), bytes);
     // after every statement the app runs, look at what a reader would see
     const states: string[] = [];
-    const watched: Db = { close: () => db.close(), async query<T>(sql: string, params?: unknown[]) {
+    const watched: Db = { close: () => db.close(), begin: (fn) => db.begin(fn), async query<T>(sql: string, params?: unknown[]) {
       const r = await db.query<T>(sql, params);
       const [x] = await db.query<{ status: string; path: string | null }>("SELECT a.status, l.path FROM assets a LEFT JOIN locations l ON l.key = a.key AND l.store = 'library' WHERE a.key = $1", [k.key]);
       if (x) states.push(`${x.status}:${x.path ?? '-'}`);

packages/library-server/src/app.test.ts

@@ -180,7 +180,7 @@ describe('the reference library (disk + PGlite)', () => {
 
   it('POST /v1/assets/current runs one SQL statement however many tags, and uses the tags index', async () => {
     const seen: string[] = [];
-    const counting: Db = { query: (sql, params) => { seen.push(sql); return db.query(sql, params); }, close: async () => {} };
+    const counting: Db = { query: (sql, params) => { seen.push(sql); return db.query(sql, params); }, begin: (fn) => db.begin(fn), close: async () => {} };
     const app2 = createLibraryApp({ db: counting, storage: diskStorage(dir, { uploadBase: BASE }), publicUrl: `${BASE}/files/`, callers: tokenCallers('teacher:t-1') });
     seen.length = 0;
     const res = await app2.fetch(new Request(`${BASE}/v1/assets/current`, { method: 'POST', headers: { Authorization: 'Bearer t-1', 'Content-Type': 'application/json' }, body: JSON.stringify({ tags: Array.from({ length: 300 }, (_, i) => `question:n-${i}`) }) }));

packages/library-server/src/app.ts

@@ -13,6 +13,7 @@ import { createAuth, type AuthOptions } from './auth/service.js';
 import type { Db } from './db.js';
 import { adoptedStore, adoptPrefixes, assetAt, registerAdopt } from './adopt.js';
 import { baseOf, registerDelivery, type DeliveryOptions } from './delivery.js';
+import { registerHighlights } from './highlights.js';
 import { servedType, type DiskStorage, type Storage } from './storage.js';
 
 /**
@@ -100,7 +101,7 @@ export function createLibraryApp(o: LibraryOptions) {
   // what this address is, for someone who opens it in a browser
   app.get('/', (c) => c.json({
     name: o.name ?? 'Chalk library', api: 'Chalk registry API v1 (spec/1.2/registry-api.md)',
-    endpoints: ['GET /v1/stores', 'GET /v1/search?q=', 'GET /v1/assets/{key}', 'POST /v1/assets/check', 'POST /v1/assets/current', 'POST /v1/uploads', 'POST /v1/assets', 'GET /v1/files/{key}', 'GET /v1/assets/{key}/link', 'POST /v1/auth/token',
+    endpoints: ['GET /v1/stores', 'GET /v1/search?q=', 'GET /v1/assets/{key}', 'POST /v1/assets/check', 'POST /v1/assets/current', 'POST /v1/uploads', 'POST /v1/assets', 'GET /v1/files/{key}', 'GET /v1/assets/{key}/link', 'GET /v1/books/{slug}/lines?pages=', 'GET /v1/books/{slug}/links?line=', 'POST /v1/auth/token',
       'GET /.well-known/openid-configuration', 'GET /healthz'],
   }));
 
@@ -185,6 +186,7 @@ export function createLibraryApp(o: LibraryOptions) {
   const adopting = { db, storage, prefixes: adoptPrefixes(o.adoptPrefixes), publicUrl: o.publicUrl, publicRead: o.publicRead !== false, maxBytes, types, fail, ensureStore, describe,
     recordOf: async (key: string) => record((await rowOf(key))!, true), sha256Of, base: (c: Context) => baseOf(c, (o as DeliveryOptions).serverUrl) };
   registerAdopt(app, adopting);
+  registerHighlights(app, { db, fail }); // §11: the lines of books and the questions linked to them
 
   // ---- §4 reading ----
   app.get('/v1/assets/:key', async (c) => {

packages/library-server/src/db.ts

@@ -7,6 +7,8 @@ import { mkdirSync, readFileSync } from 'node:fs';
  */
 export interface Db {
   query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
+  /** Runs `fn` in one transaction: its queries all happen or none does (an error rolls back). */
+  begin<T>(fn: (query: Db['query']) => Promise<T>): Promise<T>;
   close(): Promise<void>;
 }
 
@@ -33,6 +35,7 @@ export async function openDb(url: string, o: { schema?: string } = {}): Promise<
     const pg = new PGlite({ ...(dir ? { dataDir: dir } : {}), extensions: { pg_trgm } });
     db = {
       async query<T>(sql: string, params: unknown[] = []) { return (await pg.query<T>(sql, params)).rows; },
+      begin: (fn) => pg.transaction((tx) => fn(async <T>(sql: string, params: unknown[] = []) => (await tx.query<T>(sql, params)).rows)),
       async close() { await pg.close(); },
     };
     await migrate(db, (sql) => pg.transaction((tx) => tx.exec(sql)).then(() => undefined));
@@ -47,6 +50,7 @@ export async function openDb(url: string, o: { schema?: string } = {}): Promise<
     const sql = postgres(url, { max: 10, onnotice: () => {}, connection: { search_path: `${schema}, public` } });
     db = {
       async query<T>(text: string, params: unknown[] = []) { return (await sql.unsafe(text, params as never[])) as unknown as T[]; },
+      begin: <T>(fn: (query: Db['query']) => Promise<T>) => sql.begin((tx) => fn(async <R>(text: string, params: unknown[] = []) => (await tx.unsafe(text, params as never[])) as unknown as R[])) as Promise<T>,
       async close() { await sql.end(); },
     };
     await migrate(db, (text) => sql.begin((tx) => tx.unsafe(text).simple()).then(() => undefined));
@@ -74,8 +78,56 @@ ALTER TABLE locations ADD PRIMARY KEY (key, store, path);
 ALTER TABLE locations ADD COLUMN IF NOT EXISTS checked_at timestamptz;
 ALTER TABLE locations ADD COLUMN IF NOT EXISTS missing boolean NOT NULL DEFAULT false;
 INSERT INTO chalk_library_schema (version) VALUES (4);`);
+  // step 5: the highlights of books (registry-api §11), editable records: the lines of a book and the questions linked to them
+  if (!done.has(5)) await exec(`${HIGHLIGHTS_SQL}\nINSERT INTO chalk_library_schema (version) VALUES (5);`);
 }
 
+/** Step 5: book highlights (spec/1.2/registry-api.md §11). Questions are named by their lasting id only (no foreign key: they are manifests). */
+const HIGHLIGHTS_SQL = `
+CREATE TABLE IF NOT EXISTS book_lines (
+  id             uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  book           text NOT NULL,
+  key            text NOT NULL,                   -- the line's lasting name in the book: "<page id>/<hash>/<n>"
+  page_id        text NOT NULL,
+  page_no        integer NOT NULL,
+  idx            integer NOT NULL,                -- place on the page, from 0
+  block          integer NOT NULL,                -- the paragraph
+  kind           text NOT NULL,                   -- body, heading, formula, caption
+  text           text NOT NULL,
+  text_hash      text NOT NULL,
+  chapter_no     integer,
+  chapter_title  text,
+  section        text,
+  rects          jsonb NOT NULL DEFAULT '[]',     -- [{x, y, w, fs}] in page units, one per printed line
+  retired_at     timestamptz,                     -- the text layer no longer has it: kept, so its links are kept
+  created_at     timestamptz NOT NULL DEFAULT now(),
+  updated_at     timestamptz NOT NULL DEFAULT now(),
+  UNIQUE (book, key)
+);
+CREATE INDEX IF NOT EXISTS book_lines_page ON book_lines (book, page_no, idx);
+CREATE INDEX IF NOT EXISTS book_lines_chapter ON book_lines (book, chapter_no);
+CREATE TABLE IF NOT EXISTS book_links (
+  id             uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  book           text NOT NULL,
+  question_id    uuid NOT NULL,                   -- the question manifest's lasting id (spec 1.7)
+  line_id        uuid NOT NULL REFERENCES book_lines (id) ON DELETE CASCADE,
+  role           text NOT NULL DEFAULT 'answers_from',
+  scope          text NOT NULL DEFAULT 'sentence',
+  method         text NOT NULL DEFAULT 'auto',    -- auto (a matcher's) or human
+  status         text NOT NULL DEFAULT 'suggested', -- suggested, confirmed, rejected (a removal remembered)
+  score          real,
+  rationale      text,
+  question_type  text,
+  reviewed_by    text,
+  reviewed_at    timestamptz,
+  created_at     timestamptz NOT NULL DEFAULT now(),
+  UNIQUE (question_id, line_id, role)
+);
+CREATE INDEX IF NOT EXISTS book_links_line ON book_links (line_id);
+CREATE INDEX IF NOT EXISTS book_links_book ON book_links (book, status);
+CREATE INDEX IF NOT EXISTS book_links_question ON book_links (question_id);
+`;
+
 /**
  * Step 2: the library as its own identity provider (spec/1.2/registry-api.md §10). Not part of the reference
  * registry schema: another registry may issue tokens however it likes. Secrets are kept only as hashes.