diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 23a5be3..b5559b0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,6 +51,7 @@ npx tsx tools/reader-library-e2e.mjs [url] [shots] # libraries: gallery, uploa node tools/reader-stream-e2e.mjs [url] [shots] # a streamed 60-page book: what is fetched, and when npx tsx tools/reader-library-book-e2e.ts [url] [--read-token t] # a book published into a (test) library, opened from it npx tsx tools/reader-library-auth-e2e.ts [url] [--port 4923] # signing in to a private library (a fake one in-process), refresh, sign-out +npx tsx tools/reader-media-recover-e2e.ts # media adopted at two paths, one deleted while the book is open: the reader recovers (a disk test library, ADOPT_PREFIXES=books/, built showcase) npx tsx tools/question-current-e2e.ts [--read-token t] # a corrected question reaches readers with no new links item (test library: writes) node tools/reader-insert-menu-e2e.mjs [url] # author mode's right-click menu: what is added lands where you clicked node tools/reader-save-e2e.mjs [url] # author mode's Save: the filled button, Ctrl+S, onDirtyChange, the ref's save() diff --git a/docs/guides/run-a-library-server.md b/docs/guides/run-a-library-server.md index 0a70227..1d415c4 100644 --- a/docs/guides/run-a-library-server.md +++ b/docs/guides/run-a-library-server.md @@ -167,6 +167,10 @@ the asset with that path as its location. Nothing is copied. cache). If every copy is gone, readers get a 404 (they verify what they get, so other bytes at a path are refused). Adopting a path again HEADs it and reads nothing. The schema step that allows this (step 4: `locations` keyed by key, store and path, with `checked_at` and `missing`) runs by itself on the first start. + A book already open in the reader recovers too: a picture, video, audio or model whose URL fails to load is asked + for again once (the reader drops the index shard it held and fetches it past the browser's cache) and drawn from + the copy the library names now; the same goes for a signed link (`/v1/assets/{key}/link`) that expired while a + video played, when the host's resolver hands out links. - A key the library already has (uploaded, or adopted elsewhere) stays where it is; adopting it again only updates its title, tags and the other descriptive fields, and reads nothing. diff --git a/packages/library-server/README.md b/packages/library-server/README.md index ff61081..75eef8d 100644 --- a/packages/library-server/README.md +++ b/packages/library-server/README.md @@ -23,6 +23,11 @@ DATABASE_URL=pglite:./.data/db npx tsx src/cli.ts key add ci --roles author (PGlite allows one process at a time: stop `serve` first, or use the admin API.) Docker, from the repository root: `docker build -f packages/library-server/Dockerfile -t chalk-library-server .` +It also keeps the **highlights** of books (registry API §11): the lines of a book's text and the questions linked to +them, as records an editor and a matcher change one by one (`v1/books//lines`, `…/links`), so reading apps get +them from the library. They are migration step 5 of its database (two tables, `book_lines` and `book_links`), applied +when the server starts. + Every setting, Postgres and S3 setup, bucket CORS, deploying and security: [docs/guides/run-a-library-server.md](../../docs/guides/run-a-library-server.md). Using libraries in the reader and the editor: [docs/guides/media-libraries.md](../../docs/guides/media-libraries.md). diff --git a/packages/library-server/src/adopt.test.ts b/packages/library-server/src/adopt.test.ts index bcc3684..71c4c10 100644 --- a/packages/library-server/src/adopt.test.ts +++ b/packages/library-server/src/adopt.test.ts @@ -356,7 +356,7 @@ describe('adopting: off by default, and a private library', () => { mkdirSync(join(dir, 'chalk-books', 'race'), { recursive: true }); writeFileSync(join(dir, ...path.split('/')), bytes); // after every statement the app runs, look at what a reader would see const states: string[] = []; - const watched: Db = { close: () => db.close(), async query(sql: string, params?: unknown[]) { + const watched: Db = { close: () => db.close(), begin: (fn) => db.begin(fn), async query(sql: string, params?: unknown[]) { const r = await db.query(sql, params); const [x] = await db.query<{ status: string; path: string | null }>("SELECT a.status, l.path FROM assets a LEFT JOIN locations l ON l.key = a.key AND l.store = 'library' WHERE a.key = $1", [k.key]); if (x) states.push(`${x.status}:${x.path ?? '-'}`); diff --git a/packages/library-server/src/app.test.ts b/packages/library-server/src/app.test.ts index 3081e43..cefe7d2 100644 --- a/packages/library-server/src/app.test.ts +++ b/packages/library-server/src/app.test.ts @@ -180,7 +180,7 @@ describe('the reference library (disk + PGlite)', () => { it('POST /v1/assets/current runs one SQL statement however many tags, and uses the tags index', async () => { const seen: string[] = []; - const counting: Db = { query: (sql, params) => { seen.push(sql); return db.query(sql, params); }, close: async () => {} }; + const counting: Db = { query: (sql, params) => { seen.push(sql); return db.query(sql, params); }, begin: (fn) => db.begin(fn), close: async () => {} }; const app2 = createLibraryApp({ db: counting, storage: diskStorage(dir, { uploadBase: BASE }), publicUrl: `${BASE}/files/`, callers: tokenCallers('teacher:t-1') }); seen.length = 0; const res = await app2.fetch(new Request(`${BASE}/v1/assets/current`, { method: 'POST', headers: { Authorization: 'Bearer t-1', 'Content-Type': 'application/json' }, body: JSON.stringify({ tags: Array.from({ length: 300 }, (_, i) => `question:n-${i}`) }) })); diff --git a/packages/library-server/src/app.ts b/packages/library-server/src/app.ts index 9e38c52..e0ef0ab 100644 --- a/packages/library-server/src/app.ts +++ b/packages/library-server/src/app.ts @@ -13,6 +13,7 @@ import { createAuth, type AuthOptions } from './auth/service.js'; import type { Db } from './db.js'; import { adoptedStore, adoptPrefixes, assetAt, registerAdopt } from './adopt.js'; import { baseOf, registerDelivery, type DeliveryOptions } from './delivery.js'; +import { registerHighlights } from './highlights.js'; import { servedType, type DiskStorage, type Storage } from './storage.js'; /** @@ -100,7 +101,7 @@ export function createLibraryApp(o: LibraryOptions) { // what this address is, for someone who opens it in a browser app.get('/', (c) => c.json({ name: o.name ?? 'Chalk library', api: 'Chalk registry API v1 (spec/1.2/registry-api.md)', - endpoints: ['GET /v1/stores', 'GET /v1/search?q=', 'GET /v1/assets/{key}', 'POST /v1/assets/check', 'POST /v1/assets/current', 'POST /v1/uploads', 'POST /v1/assets', 'GET /v1/files/{key}', 'GET /v1/assets/{key}/link', 'POST /v1/auth/token', + endpoints: ['GET /v1/stores', 'GET /v1/search?q=', 'GET /v1/assets/{key}', 'POST /v1/assets/check', 'POST /v1/assets/current', 'POST /v1/uploads', 'POST /v1/assets', 'GET /v1/files/{key}', 'GET /v1/assets/{key}/link', 'GET /v1/books/{slug}/lines?pages=', 'GET /v1/books/{slug}/links?line=', 'POST /v1/auth/token', 'GET /.well-known/openid-configuration', 'GET /healthz'], })); @@ -185,6 +186,7 @@ export function createLibraryApp(o: LibraryOptions) { const adopting = { db, storage, prefixes: adoptPrefixes(o.adoptPrefixes), publicUrl: o.publicUrl, publicRead: o.publicRead !== false, maxBytes, types, fail, ensureStore, describe, recordOf: async (key: string) => record((await rowOf(key))!, true), sha256Of, base: (c: Context) => baseOf(c, (o as DeliveryOptions).serverUrl) }; registerAdopt(app, adopting); + registerHighlights(app, { db, fail }); // §11: the lines of books and the questions linked to them // ---- §4 reading ---- app.get('/v1/assets/:key', async (c) => { diff --git a/packages/library-server/src/db.ts b/packages/library-server/src/db.ts index cbaa99b..ddad91c 100644 --- a/packages/library-server/src/db.ts +++ b/packages/library-server/src/db.ts @@ -7,6 +7,8 @@ import { mkdirSync, readFileSync } from 'node:fs'; */ export interface Db { query>(sql: string, params?: unknown[]): Promise; + /** Runs `fn` in one transaction: its queries all happen or none does (an error rolls back). */ + begin(fn: (query: Db['query']) => Promise): Promise; close(): Promise; } @@ -33,6 +35,7 @@ export async function openDb(url: string, o: { schema?: string } = {}): Promise< const pg = new PGlite({ ...(dir ? { dataDir: dir } : {}), extensions: { pg_trgm } }); db = { async query(sql: string, params: unknown[] = []) { return (await pg.query(sql, params)).rows; }, + begin: (fn) => pg.transaction((tx) => fn(async (sql: string, params: unknown[] = []) => (await tx.query(sql, params)).rows)), async close() { await pg.close(); }, }; await migrate(db, (sql) => pg.transaction((tx) => tx.exec(sql)).then(() => undefined)); @@ -47,6 +50,7 @@ export async function openDb(url: string, o: { schema?: string } = {}): Promise< const sql = postgres(url, { max: 10, onnotice: () => {}, connection: { search_path: `${schema}, public` } }); db = { async query(text: string, params: unknown[] = []) { return (await sql.unsafe(text, params as never[])) as unknown as T[]; }, + begin: (fn: (query: Db['query']) => Promise) => sql.begin((tx) => fn(async (text: string, params: unknown[] = []) => (await tx.unsafe(text, params as never[])) as unknown as R[])) as Promise, async close() { await sql.end(); }, }; await migrate(db, (text) => sql.begin((tx) => tx.unsafe(text).simple()).then(() => undefined)); @@ -74,8 +78,56 @@ ALTER TABLE locations ADD PRIMARY KEY (key, store, path); ALTER TABLE locations ADD COLUMN IF NOT EXISTS checked_at timestamptz; ALTER TABLE locations ADD COLUMN IF NOT EXISTS missing boolean NOT NULL DEFAULT false; INSERT INTO chalk_library_schema (version) VALUES (4);`); + // step 5: the highlights of books (registry-api §11), editable records: the lines of a book and the questions linked to them + if (!done.has(5)) await exec(`${HIGHLIGHTS_SQL}\nINSERT INTO chalk_library_schema (version) VALUES (5);`); } +/** Step 5: book highlights (spec/1.2/registry-api.md §11). Questions are named by their lasting id only (no foreign key: they are manifests). */ +const HIGHLIGHTS_SQL = ` +CREATE TABLE IF NOT EXISTS book_lines ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + book text NOT NULL, + key text NOT NULL, -- the line's lasting name in the book: "//" + page_id text NOT NULL, + page_no integer NOT NULL, + idx integer NOT NULL, -- place on the page, from 0 + block integer NOT NULL, -- the paragraph + kind text NOT NULL, -- body, heading, formula, caption + text text NOT NULL, + text_hash text NOT NULL, + chapter_no integer, + chapter_title text, + section text, + rects jsonb NOT NULL DEFAULT '[]', -- [{x, y, w, fs}] in page units, one per printed line + retired_at timestamptz, -- the text layer no longer has it: kept, so its links are kept + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (book, key) +); +CREATE INDEX IF NOT EXISTS book_lines_page ON book_lines (book, page_no, idx); +CREATE INDEX IF NOT EXISTS book_lines_chapter ON book_lines (book, chapter_no); +CREATE TABLE IF NOT EXISTS book_links ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + book text NOT NULL, + question_id uuid NOT NULL, -- the question manifest's lasting id (spec 1.7) + line_id uuid NOT NULL REFERENCES book_lines (id) ON DELETE CASCADE, + role text NOT NULL DEFAULT 'answers_from', + scope text NOT NULL DEFAULT 'sentence', + method text NOT NULL DEFAULT 'auto', -- auto (a matcher's) or human + status text NOT NULL DEFAULT 'suggested', -- suggested, confirmed, rejected (a removal remembered) + score real, + rationale text, + question_type text, + reviewed_by text, + reviewed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (question_id, line_id, role) +); +CREATE INDEX IF NOT EXISTS book_links_line ON book_links (line_id); +CREATE INDEX IF NOT EXISTS book_links_book ON book_links (book, status); +CREATE INDEX IF NOT EXISTS book_links_question ON book_links (question_id); +`; + /** * Step 2: the library as its own identity provider (spec/1.2/registry-api.md §10). Not part of the reference * registry schema: another registry may issue tokens however it likes. Secrets are kept only as hashes. diff --git a/packages/library-server/src/highlights.test.ts b/packages/library-server/src/highlights.test.ts new file mode 100644 index 0000000..d4d5be0 --- /dev/null +++ b/packages/library-server/src/highlights.test.ts @@ -0,0 +1,217 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createLibraryApp, diskStorage, openDb, planLines, tokenCallers, type Db } from './index.js'; + +const BASE = 'http://library.test'; +const BOOK = 'physics-9-10-bn'; +const Q1 = '11111111-1111-4111-8111-111111111111'; +const Q2 = '22222222-2222-4222-8222-222222222222'; + +type Line = { key: string; pageId: string; pageNo: number; idx: number; block: number; kind: string; text: string; textHash: string; chapterNo: number; chapterTitle: string; section: null; rects: { x: number; y: number; w: number; fs: number }[] }; +const line = (pageNo: number, idx: number, block = 0, text = `line ${pageNo}.${idx}`): Line => ({ + key: `p${pageNo}/${idx}/0`, pageId: `p${pageNo}`, pageNo, idx, block, kind: 'body', text, textHash: `h${pageNo}.${idx}`, chapterNo: 1, chapterTitle: 'Physical quantities', section: null, + rects: [{ x: 100, y: 100 + idx * 30, w: 500, fs: 18 }], +}); + +describe('book highlights (registry-api §11)', () => { + let db: Db, app: ReturnType; + const call = async (method: string, path: string, body?: unknown, token = 'author-t') => { + const res = await app.fetch(new Request(`${BASE}${path}`, { method, headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}) }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) })); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- answers of many shapes, checked by the assertions + const answer = (await res.json()) as any; + return { status: res.status, body: answer }; + }; + const b = (p: string) => `/v1/books/${BOOK}${p}`; + const ids = async () => Object.fromEntries(((await call('GET', b('/lines/all?retired=1'))).body.items as { id: string; key: string }[]).map((l) => [l.key, l.id])); + + beforeAll(async () => { db = await openDb('pglite:'); }, 60_000); + afterAll(async () => { await db.close(); }); + beforeEach(async () => { + await db.query('DELETE FROM book_links'); await db.query('DELETE FROM book_lines'); + app = createLibraryApp({ db, storage: diskStorage(mkdtempSync(join(tmpdir(), 'chalk-hl-')), { uploadBase: BASE }), publicUrl: `${BASE}/files/`, callers: tokenCallers('author:author-t'), publicRead: false }); + await call('POST', b('/lines/sync'), { lines: [line(6, 0), line(6, 1), line(6, 2, 1), line(7, 0)] }); + }); + + it('reads need a token on a private library, writes need author', async () => { + expect((await call('GET', b('/lines?pages=6'), undefined, '')).status).toBe(401); + expect((await call('POST', b('/lines/sync'), { lines: [] }, '')).status).toBe(401); + expect((await call('GET', '/v1/books/-bad/lines?pages=6')).status).toBe(400); + }); + + it('sync writes a line\'s new words under the same key', async () => { + await call('POST', b('/lines/sync'), { lines: [{ ...line(6, 0), text: 'corrected words', textHash: 'h-new' }, line(6, 1), line(6, 2, 1), line(7, 0)] }); + const l = ((await call('GET', b('/lines/all'))).body.items as { key: string; text: string; textHash: string }[]).find((x) => x.key === 'p6/0/0')!; + expect(l).toMatchObject({ text: 'corrected words', textHash: 'h-new' }); + }); + + it('a question\'s lock is the same whatever the case of its id: an upper-case id is serialised with the lower-case one', async () => { + const L = await ids(); + const add = await call('POST', b('/links'), { questionId: Q1.toUpperCase(), lineId: L['p6/0/0'], by: 'rafi' }); + expect(add.body.questionId).toBe(Q1); + expect((await call('DELETE', b(`/links/${add.body.id}`), { by: 'rafi' })).body).toEqual({ removed: 1 }); + expect((await call('DELETE', b('/links/33333333-3333-4333-8333-333333333333'), { by: 'rafi' })).status).toBe(404); + }); + + it('ids are compared in lower case wherever they come in: a line moved onto itself, a retired line, a placement, a question\'s state', async () => { + const L = await ids(); + const up = (id: string) => id.toUpperCase(); + await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], by: 'rafi' }); + expect((await call('POST', b(`/lines/${L['p6/0/0']}/links/move`), { to: up(L['p6/0/0']!), by: 'rafi' })).body).toEqual({ moved: 0 }); + expect((await call('GET', b(`/links?line=${up(L['p6/0/0']!)}`))).body.items).toHaveLength(1); + expect((await call('POST', '/v1/highlights/questions', { questionIds: [up(Q1)] })).body.items).toEqual({ [Q1]: { live: 1, human: true, any: true } }); + expect((await call('POST', b('/links/place'), { questionId: up(Q2), proposals: [{ lineId: up(L['p6/1/0']!), role: 'answers_from' }] })).body).toEqual({ placed: 1 }); + await call('POST', b('/lines/sync'), { lines: [line(6, 0), line(6, 1), line(6, 2, 1)] }); + expect((await call('POST', b('/links'), { questionId: Q1, lineId: up(L['p7/0/0']!), by: 'rafi' })).status).toBe(409); + }); + + it('a question kind is a plain name: __proto__ is refused, and kinds are counted as own keys', async () => { + const L = await ids(); + expect((await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], questionType: '__proto__', by: 'rafi' })).status).toBe(400); + expect((await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], questionType: 'constructor', by: 'rafi' })).status).toBe(201); + const page = (await call('GET', b('/lines?pages=6'))).body.pages['6']; + expect(page[0].typeCounts).toEqual({ constructor: { confirmed: 1, suggested: 0 } }); + expect(({} as Record).confirmed).toBeUndefined(); + }); + + it('sync adds, changes, retires and revives lines by key; a re-run changes nothing', async () => { + const again = await call('POST', b('/lines/sync'), { lines: [line(6, 0), line(6, 1), line(6, 2, 1), line(7, 0)] }); + expect(again.body).toEqual({ added: 0, changed: 0, retired: 0, revived: 0, linksRetired: 0 }); + const moved = { ...line(6, 1), idx: 5 }; + const r = await call('POST', b('/lines/sync'), { lines: [line(6, 0), moved, line(8, 0)] }); + expect(r.body).toMatchObject({ added: 1, changed: 1, retired: 2, revived: 0 }); + const back = await call('POST', b('/lines/sync'), { lines: [line(6, 0), moved, line(8, 0), line(7, 0)] }); + expect(back.body).toMatchObject({ added: 0, revived: 1, retired: 0 }); + }); + + it('planLines matches the dashboard\'s plan: no change, a moved line, a missing one', () => { + const stored = [{ id: 'a', key: 'p6/0/0', pageId: 'p6', text: 'line 6.0', textHash: 'h6.0', retiredAt: null, pageNo: 6, idx: 0, block: 0, kind: 'body', chapterNo: 1, chapterTitle: 'Physical quantities', section: null, rects: line(6, 0).rects }]; + expect(planLines(stored, [line(6, 0)])).toMatchObject({ write: [], retire: [] }); + expect(planLines(stored, [])).toMatchObject({ retire: ['p6/0/0'] }); + expect(planLines(stored, [{ ...line(6, 0), text: 'corrected words' }])).toMatchObject({ changed: 1 }); + }); + + it('a person tags a line: confirmed and theirs; the page shows counts by kind; a reader may ask for linked lines only', async () => { + const L = await ids(); + const add = await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], questionType: 'mcq', by: 'rafi' }); + expect(add.status).toBe(201); + expect(add.body).toMatchObject({ questionId: Q1, method: 'human', status: 'confirmed', reviewedBy: 'rafi', questionType: 'mcq', line: { key: 'p6/0/0', pageNo: 6 } }); + const page = (await call('GET', b('/lines?pages=6,7'))).body.pages; + expect(page['6']).toHaveLength(3); + expect(page['6'][0]).toMatchObject({ id: L['p6/0/0'], links: 1, suggested: 0, typeCounts: { mcq: { confirmed: 1, suggested: 0 } } }); + expect(page['6'][0].textHash).toBeUndefined(); + expect((await call('GET', b('/lines?pages=6&linked=1'))).body.pages['6']).toHaveLength(1); + expect((await call('GET', b('/pages/counts'))).body.counts).toEqual({ '6': 1 }); + expect((await call('GET', b('/chapters'))).body.items).toEqual([{ no: 1, title: 'Physical quantities' }]); + }); + + it('taking a link off keeps it as a rejected row the readers never see', async () => { + const L = await ids(); + const add = await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], by: 'rafi' }); + expect((await call('DELETE', b(`/links/${add.body.id}`), { by: 'rafi' })).body).toEqual({ removed: 1 }); + expect((await call('GET', b(`/links?line=${L['p6/0/0']}`))).body.items).toEqual([]); + expect((await call('GET', b('/lines?pages=6'))).body.pages['6'][0].links).toBe(0); + const all = (await call('GET', b('/links/all?rejected=1'))).body.items; + expect(all).toHaveLength(1); + expect(all[0]).toMatchObject({ status: 'rejected', method: 'human' }); + // the matcher may not place a question a person took a tag off + expect((await call('POST', b('/links/place'), { questionId: Q1, proposals: [{ lineId: L['p6/1/0'], role: 'answers_from' }] })).body).toEqual({ placed: 0 }); + }); + + it('the matcher places a question only while it is in the pool; one row per line; retired lines skipped', async () => { + const L = await ids(); + const place = await call('POST', b('/links/place'), { questionId: Q2, questionType: 'cq', proposals: [ + { lineId: L['p6/0/0'], role: 'answers_from', score: 0.9, rationale: 'same words' }, { lineId: L['p6/0/0'], role: 'tests_concept' }, { lineId: L['p6/1/0'], role: 'answers_from', score: 0.5 }] }); + expect(place.body).toEqual({ placed: 2 }); + expect((await call('POST', b('/links/place'), { questionId: Q2, proposals: [{ lineId: L['p7/0/0'], role: 'answers_from' }] })).body).toEqual({ placed: 0 }); + const q = (await call('POST', '/v1/highlights/questions', { questionIds: [Q1, Q2] })).body.items; + expect(q).toEqual({ [Q1]: { live: 0, human: false, any: false }, [Q2]: { live: 2, human: false, any: true } }); + expect((await call('GET', '/v1/highlights/tagged')).body.questionIds).toEqual([Q2]); + // confirmed first, then by score + const on = (await call('GET', b(`/links?line=${L['p6/0/0']}¶graph=1`))).body.items; + expect(on.map((l: { line: { key: string } }) => l.line.key)).toEqual(['p6/0/0', 'p6/1/0'].slice(0, on.length)); + }); + + it('a re-index retires the matcher\'s links on lines that went, and keeps a person\'s (listed as orphaned)', async () => { + const L = await ids(); + await call('POST', b('/links/place'), { questionId: Q2, proposals: [{ lineId: L['p7/0/0'], role: 'answers_from' }] }); + await call('POST', b('/links'), { questionId: Q1, lineId: L['p7/0/0'], by: 'rafi' }); + const r = await call('POST', b('/lines/sync'), { lines: [line(6, 0), line(6, 1), line(6, 2, 1)] }); + expect(r.body).toMatchObject({ retired: 1, linksRetired: 1 }); + const orphaned = (await call('GET', b('/links/orphaned'))).body.items; + expect(orphaned).toHaveLength(1); + expect(orphaned[0]).toMatchObject({ questionId: Q1, line: { key: 'p7/0/0' } }); + expect(orphaned[0].line.retiredAt).not.toBeNull(); + expect((await call('POST', b('/links'), { questionId: Q1, lineId: L['p7/0/0'], by: 'rafi' })).status).toBe(409); + }); + + it('moving a link onto a pair the question already has merges into it', async () => { + const L = await ids(); + const a = await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], by: 'rafi' }); + const other = await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/1/0'], by: 'rafi' }); + const moved = await call('PATCH', b(`/links/${a.body.id}`), { lineId: L['p6/1/0'], by: 'mim' }); + expect(moved.body).toMatchObject({ id: other.body.id, status: 'confirmed', reviewedBy: 'mim' }); + expect((await call('GET', b('/links/all?rejected=1'))).body.items).toHaveLength(1); + const role = await call('PATCH', b(`/links/${other.body.id}`), { role: 'formula_source', by: 'mim' }); + expect(role.body).toMatchObject({ role: 'formula_source', method: 'human' }); + expect((await call('PATCH', b('/links/33333333-3333-4333-8333-333333333333'), { by: 'mim' })).status).toBe(404); + }); + + it('moves and takes off every link of a line (or its paragraph)', async () => { + const L = await ids(); + await call('POST', b('/links/place'), { questionId: Q2, proposals: [{ lineId: L['p6/0/0'], role: 'answers_from' }] }); + await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/0/0'], by: 'rafi' }); + await call('POST', b('/links'), { questionId: Q1, lineId: L['p6/2/0'], by: 'rafi' }); + expect((await call('POST', b(`/lines/${L['p6/0/0']}/links/move`), { to: L['p6/2/0'], by: 'mim' })).body).toEqual({ moved: 1 }); + const at = (await call('GET', b(`/links?line=${L['p6/2/0']}`))).body.items; + expect(at.map((l: { questionId: string }) => l.questionId).sort()).toEqual([Q1, Q2]); + // p6/0 and p6/1 are one paragraph (block 0); p6/2 is another + expect((await call('POST', b(`/lines/${L['p6/1/0']}/links/remove`), { paragraph: true, by: 'mim' })).body).toEqual({ removed: 0 }); + expect((await call('POST', b(`/lines/${L['p6/2/0']}/links/remove`), { by: 'mim' })).body).toEqual({ removed: 2 }); + }); + + it('a question\'s highlight grows, shrinks and moves; a line taken off is remembered', async () => { + const L = await ids(); + await call('POST', b('/links/place'), { questionId: Q1, proposals: [{ lineId: L['p6/0/0'], role: 'answers_from' }, { lineId: L['p6/1/0'], role: 'answers_from' }] }); + const r = await call('PUT', b(`/questions/${Q1}/lines`), { role: 'answers_from', add: [L['p6/2/0']], remove: [L['p6/0/0'], L['p7/0/0']], questionType: 'mcq', by: 'rafi' }); + expect(r.status).toBe(200); + expect(r.body.items.map((l: { line: { key: string } }) => l.line.key)).toEqual(['p6/1/0', 'p6/2/0']); + expect(r.body.items.every((l: { method: string; status: string }) => l.method === 'human' && l.status === 'confirmed')).toBe(true); + const rejected = (await call('GET', b('/links/all?rejected=1'))).body.items.filter((l: { status: string }) => l.status === 'rejected'); + expect(rejected.map((l: { line: { key: string } }) => l.line.key).sort()).toEqual(['p6/0/0', 'p7/0/0']); + expect((await call('GET', b(`/links?question=${Q1}`))).body.items).toHaveLength(2); + expect((await call('PUT', b(`/questions/${Q1}/lines`), { role: 'answers_from', add: [], remove: [], by: 'rafi' })).status).toBe(400); + expect((await call('PUT', b(`/questions/${Q1}/lines`), { role: 'answers_from', add: [L['p6/0/0']], remove: [L['p6/0/0']], by: 'rafi' })).status).toBe(400); + }); + + it('review lists links in book order with a total', async () => { + const L = await ids(); + await call('POST', b('/links/place'), { questionId: Q2, proposals: [{ lineId: L['p7/0/0'], role: 'answers_from' }, { lineId: L['p6/0/0'], role: 'answers_from' }] }); + const r = (await call('GET', b('/links?status=suggested&limit=1'))).body; + expect(r.total).toBe(2); + expect(r.items).toHaveLength(1); + expect(r.items[0].line.key).toBe('p6/0/0'); + // chapter 0 is a chapter like any other (the lines here are in chapter 1) + expect((await call('GET', b('/links?chapter=0'))).body.total).toBe(0); + expect((await call('GET', b('/links?chapter=1'))).body.total).toBe(2); + }); + + it('import copies lines and links with their ids', async () => { + const lineId = '44444444-4444-4444-8444-444444444444', linkId = '55555555-5555-4555-8555-555555555555'; + const r = await call('POST', '/v1/books/other-book/import', { + lines: [{ ...line(3, 0), id: lineId, retiredAt: null }], + links: [{ id: linkId, questionId: Q1, lineId, role: 'answers_from', scope: 'sentence', method: 'auto', status: 'suggested', score: 0.7, rationale: 'x', questionType: 'mcq', reviewedBy: null, reviewedAt: null, createdAt: '2026-10-30T00:00:00Z' }], + }); + expect(r.body).toEqual({ lines: 1, links: 1 }); + const got = (await call('GET', `/v1/books/other-book/links?line=${lineId}`)).body.items; + expect(got[0]).toMatchObject({ id: linkId, score: 0.7, questionType: 'mcq' }); + expect((await call('POST', '/v1/books/other-book/import', { lines: [], links: [{ id: linkId, questionId: Q1, lineId: '66666666-6666-4666-8666-666666666666', role: 'answers_from', scope: 'sentence', method: 'auto', status: 'suggested' }] })).status).toBe(404); + // the same ids sent for another book are refused, not reported as copied + const again = await call('POST', '/v1/books/third-book/import', { lines: [{ ...line(3, 0), id: lineId, retiredAt: null }], links: [] }); + expect(again.status).toBe(409); + expect(again.body.error.message).toMatch(/another book/); + // moving a line onto itself still checks the line + expect((await call('POST', b('/lines/77777777-7777-4777-8777-777777777777/links/move'), { to: '77777777-7777-4777-8777-777777777777', by: 'x' })).status).toBe(404); + }); +}); diff --git a/packages/library-server/src/highlights.ts b/packages/library-server/src/highlights.ts new file mode 100644 index 0000000..3ac9706 --- /dev/null +++ b/packages/library-server/src/highlights.ts @@ -0,0 +1,508 @@ +import type { Context, Hono } from 'hono'; +import { z } from 'zod'; +import { requireRole, type AuthVariables } from './auth/guard.js'; +import type { Db } from './db.js'; + +/** + * The highlights of books (spec/1.2/registry-api.md §11): the lines of a book's text and the questions linked to them, + * as records that change one by one. An editor adds, moves and takes off links; a matcher suggests them; reading apps + * ask for the lines of the pages they show. Reads need `reader`, writes `author`. + * + * A rejected link is kept: it is the memory that a pair was taken off, so a matcher never puts it back. A link is a + * person's decision when its method is `human`, it was reviewed, or it is confirmed; a question with one is never + * placed by a matcher (`links/place`). + */ + +type Query = Db['query']; +type Fail = (c: Context, status: number, code: string, message: string) => Response; + +const SLUG = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const ROLES = ['answers_from', 'tests_concept', 'uses_figure', 'formula_source'] as const; +const STATUSES = ['suggested', 'confirmed', 'rejected'] as const; +const SCOPES = ['sentence', 'paragraph', 'section'] as const; +/** The most lines one change of a question's highlight may name: a highlight is a few lines, not a chapter. */ +export const MAX_LINE_CHANGES = 200; + +/** A uuid, in its canonical form (lower case, as Postgres prints one): every id is compared as the database stores it. */ +const uuid = z.string().uuid().transform((s) => s.toLowerCase()); +/** An id from the URL (a path part or a query parameter) in the same canonical form; anything else is left as it is and fails its check. */ +const idOf = (v: string | undefined) => v?.toLowerCase(); +const by = z.string().min(1).max(200); +const role = z.enum(ROLES); +const scope = z.enum(SCOPES); +/** A question's kind as the writer names it (`mcq`, `cq`, …): a plain lower-case name, so it is safe as a key of the counts it is grouped by. */ +const TYPE_NAME = /^[a-z][a-z0-9_]{0,39}$/; +const questionType = z.string().regex(TYPE_NAME).optional(); +const rect = z.object({ x: z.number(), y: z.number(), w: z.number(), fs: z.number() }); +const lineIn = z.object({ + key: z.string().min(1).max(300), pageId: z.string().min(1).max(200), pageNo: z.number().int().min(0), idx: z.number().int().min(0), block: z.number().int().min(0), + kind: z.string().min(1).max(40), text: z.string().max(20_000), textHash: z.string().min(1).max(200), + chapterNo: z.number().int().nullable().optional(), chapterTitle: z.string().max(500).nullable().optional(), section: z.string().max(2000).nullable().optional(), + rects: z.array(rect).max(200), +}); +const syncSchema = z.object({ lines: z.array(lineIn).max(200_000) }); +const addSchema = z.object({ questionId: uuid, lineId: uuid, role: role.optional(), scope: scope.optional(), questionType, by }); +const patchSchema = z.object({ status: z.enum(STATUSES).optional(), role: role.optional(), scope: scope.optional(), lineId: uuid.optional(), by }); +const bySchema = z.object({ by }); +const removeSchema = z.object({ paragraph: z.boolean().optional(), by }); +const moveSchema = z.object({ to: uuid, by }); +const linesSchema = z.object({ role, add: z.array(uuid), remove: z.array(uuid), scope: scope.optional(), questionType, by }); +const placeSchema = z.object({ + questionId: uuid, questionType, + proposals: z.array(z.object({ lineId: uuid, role, scope: scope.optional(), score: z.number().nullable().optional(), rationale: z.string().max(2000).nullable().optional() })).max(500), +}); +const importSchema = z.object({ + lines: z.array(lineIn.extend({ id: uuid, retiredAt: z.string().nullable().optional() })).max(200_000), + links: z.array(z.object({ + id: uuid, questionId: uuid, lineId: uuid, role: z.string().min(1).max(40), scope: z.string().min(1).max(40), method: z.string().min(1).max(20), status: z.enum(STATUSES), + score: z.number().nullable().optional(), rationale: z.string().nullable().optional(), questionType: z.string().regex(TYPE_NAME).nullable().optional(), + reviewedBy: z.string().nullable().optional(), reviewedAt: z.string().nullable().optional(), createdAt: z.string().optional(), + })).max(500_000), +}); +const questionsSchema = z.object({ questionIds: z.array(uuid).max(5000) }); + +/** A person's decision, on the links table as `k`. */ +const HUMAN = `(k.method = 'human' OR k.reviewed_by IS NOT NULL OR k.status = 'confirmed')`; + +const LINE_COLS = `l.id, l.key, l.page_id AS "pageId", l.page_no AS "pageNo", l.idx, l.block, l.kind, l.text, l.text_hash AS "textHash", + l.chapter_no AS "chapterNo", l.chapter_title AS "chapterTitle", l.section, l.rects, l.retired_at AS "retiredAt"`; +const LINK_COLS = `k.id, k.question_id AS "questionId", k.line_id AS "lineId", k.role, k.scope, k.method, k.status, k.score, k.rationale, + k.question_type AS "questionType", k.reviewed_by AS "reviewedBy", k.reviewed_at AS "reviewedAt", k.created_at AS "createdAt", + jsonb_build_object('id', l.id, 'key', l.key, 'pageId', l.page_id, 'pageNo', l.page_no, 'idx', l.idx, 'block', l.block, 'text', l.text, + 'chapterNo', l.chapter_no, 'retiredAt', l.retired_at) AS line`; +const LINKS_FROM = `FROM book_links k JOIN book_lines l ON l.id = k.line_id`; + +type LineRow = { id: string; key: string; pageId: string; text: string; textHash: string; retiredAt: string | Date | null; pageNo: number; idx: number; block: number; kind: string; chapterNo: number | null; chapterTitle: string | null; section: string | null; rects: unknown }; + +class HighlightError extends Error { + constructor(readonly status: 400 | 404 | 409, readonly code: string, message: string) { super(message); } +} +const notFound = (what: string) => new HighlightError(404, 'not_found', what); +const retired = () => new HighlightError(409, 'conflict', 'that line is no longer in the book'); + +/** + * Takes the lock of a question for the rest of the transaction: writes to one question happen one after another. The id is taken in its + * canonical form (lower case, as Postgres prints a uuid), so two spellings of one id share the lock. + */ +const lockQuestion = (q: Query, id: string) => q(`SELECT pg_advisory_xact_lock(hashtext($1))`, [`chalk-highlights:q:${id.toLowerCase()}`]); + +/** + * Locks every question with a link on these lines, in id order (so two bulk changes never wait on each other crosswise), and gives their ids: + * a bulk change then touches only those questions' links. + */ +async function lockQuestionsOf(q: Query, lineIds: string[]): Promise { + const ids = (await q<{ id: string }>(`SELECT DISTINCT question_id::text AS id FROM book_links WHERE line_id = ANY($1::uuid[]) ORDER BY 1`, [lineIds])).map((r) => r.id); + for (const id of ids) await lockQuestion(q, id); + return ids; +} + +async function linkView(q: Query, id: string) { + return (await q(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE k.id = $1`, [id]))[0]!; +} + +/** The line, if the book has it; `live` also refuses a retired one. */ +async function lineOf(q: Query, book: string, id: string, live: boolean) { + const [l] = await q<{ id: string; retired: boolean }>(`SELECT id, retired_at IS NOT NULL AS retired FROM book_lines WHERE id = $1 AND book = $2`, [id, book]); + if (!l) throw notFound('that line is not in this book'); + if (live && l.retired) throw retired(); + return l; +} + +/** The lines a change on one line covers: the line, or (`paragraph`) every live line of its paragraph. */ +async function covered(q: Query, book: string, lineId: string, paragraph: boolean): Promise { + const rows = paragraph + ? await q<{ id: string }>(`SELECT x.id FROM book_lines x JOIN book_lines l ON l.id = $2 AND l.book = $1 + WHERE x.book = $1 AND x.page_no = l.page_no AND x.block = l.block AND x.retired_at IS NULL`, [book, lineId]) + : await q<{ id: string }>(`SELECT id FROM book_lines WHERE book = $1 AND id = $2`, [book, lineId]); + return rows.map((r) => r.id); +} + +/** Boxes compared by their numbers: jsonb keeps an object's keys in its own order, so the JSON text of the same box differs. */ +const boxes = (v: unknown) => JSON.stringify(Array.isArray(v) ? v.map((r: { x?: unknown; y?: unknown; w?: unknown; fs?: unknown }) => [r.x, r.y, r.w, r.fs]) : v); +const sameRects = (a: unknown, b: unknown) => boxes(a) === boxes(b); + +/** What a sync would do: lines matched by key; one not sent any more is retired, not deleted. */ +export function planLines(stored: LineRow[], lines: z.infer[]) { + const have = new Map(stored.map((r) => [r.key, r])); + const want = new Set(lines.map((r) => r.key)); + const write: z.infer[] = []; + let added = 0, revived = 0; + for (const r of lines) { + const h = have.get(r.key); + if (!h) { write.push(r); added++; continue; } + if (h.retiredAt || h.pageId !== r.pageId || h.text !== r.text || h.textHash !== r.textHash || h.pageNo !== r.pageNo || h.idx !== r.idx || h.block !== r.block || h.kind !== r.kind || h.chapterNo !== (r.chapterNo ?? null) + || h.chapterTitle !== (r.chapterTitle ?? null) || h.section !== (r.section ?? null) || !sameRects(h.rects, r.rects)) { + write.push(r); + if (h.retiredAt) revived++; + } + } + const retire = stored.filter((s) => !want.has(s.key) && !s.retiredAt).map((s) => s.key); + return { write, added, changed: write.length - added - revived, revived, retire }; +} + +const CHUNK = 1000; +const chunks = (list: T[]) => Array.from({ length: Math.ceil(list.length / CHUNK) }, (_, i) => list.slice(i * CHUNK, (i + 1) * CHUNK)); +const LINE_RECORDSET = `x(id uuid, key text, "pageId" text, "pageNo" int, idx int, block int, kind text, text text, "textHash" text, + "chapterNo" int, "chapterTitle" text, section text, rects jsonb, "retiredAt" timestamptz)`; + +export function registerHighlights(app: Hono<{ Variables: AuthVariables }>, o: { db: Db; fail: Fail }) { + const { db, fail } = o; + const run = async (c: Context, role: 'reader' | 'author', fn: (book: string) => Promise) => { + const denied = requireRole(c, role); if (denied) return denied; + const book = c.req.param('slug') ?? ''; + if (!SLUG.test(book)) return fail(c, 400, 'invalid', 'not a book slug'); + try { return await fn(book); } catch (err) { + if (err instanceof HighlightError) return fail(c, err.status, err.code, err.message); + throw err; + } + }; + const parse = async (c: Context, schema: z.ZodType): Promise => { + const r = schema.safeParse(await c.req.json().catch(() => null)); + if (!r.success) throw new HighlightError(400, 'invalid', r.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')); + return r.data; + }; + const int = (v: string | undefined) => (v !== undefined && /^\d{1,9}$/.test(v) ? Number(v) : undefined); + const statusFilter = (c: Context) => (c.req.query('status') === 'confirmed' ? `k.status = 'confirmed'` : `k.status <> 'rejected'`); + + // ---- §11.2 reading ---- + + app.get('/v1/books/:slug/lines', (c) => run(c, 'reader', async (book) => { + const pages = [...new Set((c.req.query('pages') ?? '').split(',').map((s) => int(s.trim())).filter((n): n is number => n !== undefined))]; + if (pages.length > 50) throw new HighlightError(400, 'invalid', 'at most 50 pages at once'); + const out: Record = Object.fromEntries(pages.map((p) => [String(p), []])); + if (!pages.length) return c.json({ pages: out }); + const lines = await db.query & { id: string; pageNo: number }>( + `SELECT ${LINE_COLS} FROM book_lines l WHERE l.book = $1 AND l.page_no = ANY($2::int[]) AND l.retired_at IS NULL ORDER BY l.page_no, l.idx`, [book, pages]); + const links = await db.query<{ lineId: string; status: string; type: string | null }>( + `SELECT k.line_id AS "lineId", k.status, k.question_type AS type ${LINKS_FROM} WHERE k.book = $1 AND l.page_no = ANY($2::int[]) AND l.retired_at IS NULL AND ${statusFilter(c)}`, [book, pages]); + // the kinds are counted in a Map and only then made an object (Object.fromEntries defines own keys), so no kind can reach a prototype + const byLine = new Map }>(); + for (const k of links) { + const s = byLine.get(k.lineId) ?? { links: 0, suggested: 0, types: new Map() }; + s.links++; + if (k.status === 'suggested') s.suggested++; + if (k.type) { + const t = s.types.get(k.type) ?? { confirmed: 0, suggested: 0 }; + if (k.status === 'confirmed') t.confirmed++; else t.suggested++; + s.types.set(k.type, t); + } + byLine.set(k.lineId, s); + } + const linked = c.req.query('linked') === '1'; + for (const l of lines) { + const counted = byLine.get(l.id); + const s = { links: counted?.links ?? 0, suggested: counted?.suggested ?? 0, typeCounts: Object.fromEntries(counted?.types ?? []) }; + if (linked && !s.links) continue; + const rest: Record = { ...l }; + delete rest.textHash; + delete rest.retiredAt; + out[String(l.pageNo)]!.push({ ...rest, ...s }); + } + return c.json({ pages: out }); + })); + + app.get('/v1/books/:slug/lines/all', (c) => run(c, 'reader', async (book) => { + const chapter = int(c.req.query('chapter')); + const items = await db.query( + `SELECT ${LINE_COLS} FROM book_lines l WHERE l.book = $1 ${c.req.query('retired') === '1' ? '' : 'AND l.retired_at IS NULL'} ${chapter === undefined ? '' : 'AND l.chapter_no = $2'} + ORDER BY l.page_no, l.idx, l.key`, chapter === undefined ? [book] : [book, chapter]); + return c.json({ items }); + })); + + app.get('/v1/books/:slug/chapters', (c) => run(c, 'reader', async (book) => { + const items = await db.query(`SELECT DISTINCT chapter_no AS no, chapter_title AS title FROM book_lines WHERE book = $1 AND chapter_no IS NOT NULL ORDER BY chapter_no`, [book]); + return c.json({ items }); + })); + + app.get('/v1/books/:slug/pages/counts', (c) => run(c, 'reader', async (book) => { + const rows = await db.query<{ pageNo: number; n: number }>( + `SELECT l.page_no AS "pageNo", count(DISTINCT k.question_id)::int AS n ${LINKS_FROM} WHERE k.book = $1 AND l.retired_at IS NULL AND k.status <> 'rejected' GROUP BY l.page_no`, [book]); + return c.json({ counts: Object.fromEntries(rows.map((r) => [String(r.pageNo), r.n])) }); + })); + + app.get('/v1/books/:slug/links', (c) => run(c, 'reader', async (book) => { + const line = idOf(c.req.query('line')), question = idOf(c.req.query('question')); + if (line !== undefined) { + if (!uuid.safeParse(line).success) throw new HighlightError(400, 'invalid', 'line: not a uuid'); + const where = c.req.query('paragraph') === '1' + ? `l.id IN (SELECT x.id FROM book_lines x JOIN book_lines y ON y.id = $2 AND y.book = $1 WHERE x.book = $1 AND x.page_no = y.page_no AND x.block = y.block AND x.retired_at IS NULL)` + : `l.id = $2`; + const items = await db.query(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE k.book = $1 AND ${where} AND ${statusFilter(c)} + ORDER BY (k.status = 'confirmed') DESC, k.score DESC, k.created_at DESC`, [book, line]); + return c.json({ items }); + } + if (question !== undefined) { + if (!uuid.safeParse(question).success) throw new HighlightError(400, 'invalid', 'question: not a uuid'); + return c.json({ items: await linksOfQuestion(db.query, book, question) }); + } + // review: by chapter, status and method, in book order + const where = ['k.book = $1'], params: unknown[] = [book]; + const chapter = int(c.req.query('chapter')); + if (chapter !== undefined) { params.push(chapter); where.push(`l.chapter_no = $${params.length}`); } + const status = c.req.query('status'); + if (status && (STATUSES as readonly string[]).includes(status)) { params.push(status); where.push(`k.status = $${params.length}`); } else where.push(`k.status <> 'rejected'`); + const method = c.req.query('method'); + if (method) { params.push(method); where.push(`k.method = $${params.length}`); } + const cond = where.join(' AND '); + const limit = Math.min(int(c.req.query('limit')) ?? 50, 200), offset = int(c.req.query('offset')) ?? 0; + const [counted, items] = await Promise.all([ + db.query<{ total: number }>(`SELECT count(*)::int AS total ${LINKS_FROM} WHERE ${cond}`, params), + db.query(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE ${cond} ORDER BY l.page_no, l.idx, k.created_at DESC LIMIT ${limit} OFFSET ${offset}`, params), + ]); + return c.json({ items, total: counted[0]?.total ?? 0 }); + })); + + app.get('/v1/books/:slug/links/all', (c) => run(c, 'reader', async (book) => { + const items = await db.query(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE k.book = $1 ${c.req.query('rejected') === '1' ? '' : `AND k.status <> 'rejected'`} ORDER BY k.question_id, l.key, k.role`, [book]); + return c.json({ items }); + })); + + app.get('/v1/books/:slug/links/orphaned', (c) => run(c, 'reader', async (book) => { + const items = await db.query(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE k.book = $1 AND ${HUMAN} AND k.status <> 'rejected' AND l.retired_at IS NOT NULL ORDER BY l.page_no, l.idx`, [book]); + return c.json({ items }); + })); + + app.post('/v1/highlights/questions', async (c) => { + const denied = requireRole(c, 'reader'); if (denied) return denied; + const r = questionsSchema.safeParse(await c.req.json().catch(() => null)); + if (!r.success) return fail(c, 400, 'invalid', r.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')); + const rows = await db.query<{ id: string; live: number; human: boolean }>( + `SELECT k.question_id AS id, (count(*) FILTER (WHERE k.status <> 'rejected'))::int AS live, bool_or(${HUMAN}) AS human + FROM book_links k WHERE k.question_id = ANY($1::uuid[]) GROUP BY k.question_id`, [r.data.questionIds]); + const found = new Map(rows.map((x) => [x.id, x])); + return c.json({ items: Object.fromEntries(r.data.questionIds.map((id) => [id, { live: found.get(id)?.live ?? 0, human: found.get(id)?.human ?? false, any: found.has(id) }])) }); + }); + + app.get('/v1/highlights/tagged', async (c) => { + const denied = requireRole(c, 'reader'); if (denied) return denied; + const rows = await db.query<{ id: string }>(`SELECT DISTINCT question_id AS id FROM book_links WHERE status <> 'rejected' ORDER BY 1`); + return c.json({ questionIds: rows.map((r) => r.id) }); + }); + + // ---- §11.3 writing ---- + + app.post('/v1/books/:slug/lines/sync', (c) => run(c, 'author', async (book) => { + const { lines } = await parse(c, syncSchema); + const result = await db.begin(async (q) => { + // one sync of a book at a time + await q(`SELECT pg_advisory_xact_lock(hashtext($1))`, [`chalk-highlights:sync:${book}`]); + const stored = await q(`SELECT ${LINE_COLS} FROM book_lines l WHERE l.book = $1`, [book]); + const plan = planLines(stored, lines); + for (const part of chunks(plan.write)) { + await q(`INSERT INTO book_lines (book, key, page_id, page_no, idx, block, kind, text, text_hash, chapter_no, chapter_title, section, rects) + SELECT $1, x.key, x."pageId", x."pageNo", x.idx, x.block, x.kind, x.text, x."textHash", x."chapterNo", x."chapterTitle", x.section, coalesce(x.rects, '[]') + FROM jsonb_to_recordset($2::jsonb) AS ${LINE_RECORDSET} + ON CONFLICT (book, key) DO UPDATE SET page_id = EXCLUDED.page_id, page_no = EXCLUDED.page_no, idx = EXCLUDED.idx, block = EXCLUDED.block, kind = EXCLUDED.kind, + text = EXCLUDED.text, text_hash = EXCLUDED.text_hash, chapter_no = EXCLUDED.chapter_no, chapter_title = EXCLUDED.chapter_title, section = EXCLUDED.section, rects = EXCLUDED.rects, retired_at = NULL, updated_at = now()`, + [book, JSON.stringify(part)]); + } + for (const part of chunks(plan.retire)) await q(`UPDATE book_lines SET retired_at = now(), updated_at = now() WHERE book = $1 AND retired_at IS NULL AND key = ANY($2::text[])`, [book, part]); + // a person's link on a line that is gone is kept (links/orphaned lists it); a matcher's alone is turned down, and its question can be placed again + const gone = await q(`UPDATE book_links k SET status = 'rejected', rationale = 'retired: the line is no longer in the book after re-indexing' + WHERE k.book = $1 AND NOT ${HUMAN} AND k.status <> 'rejected' AND k.line_id IN (SELECT id FROM book_lines WHERE book = $1 AND retired_at IS NOT NULL) RETURNING k.id`, [book]); + return { added: plan.added, changed: plan.changed, retired: plan.retire.length, revived: plan.revived, linksRetired: gone.length }; + }); + return c.json(result); + })); + + app.post('/v1/books/:slug/links', (c) => run(c, 'author', async (book) => { + const b = await parse(c, addSchema); + const id = await db.begin(async (q) => { + await lockQuestion(q, b.questionId); + await lineOf(q, book, b.lineId, true); + const [row] = await q<{ id: string }>(`INSERT INTO book_links (book, question_id, line_id, role, scope, method, status, question_type, reviewed_by, reviewed_at) + VALUES ($1, $2, $3, $4, $5, 'human', 'confirmed', $6, $7, now()) + ON CONFLICT (question_id, line_id, role) DO UPDATE SET status = 'confirmed', scope = EXCLUDED.scope, method = 'human', + question_type = coalesce(EXCLUDED.question_type, book_links.question_type), reviewed_by = EXCLUDED.reviewed_by, reviewed_at = now() + RETURNING id`, [book, b.questionId, b.lineId, b.role ?? 'answers_from', b.scope ?? 'sentence', b.questionType ?? null, b.by]); + return row!.id; + }); + return c.json(await linkView(db.query, id), 201); + })); + + app.patch('/v1/books/:slug/links/:id', (c) => run(c, 'author', async (book) => { + const p = await parse(c, patchSchema); + const linkId = idOf(c.req.param('id'))!; + if (!uuid.safeParse(linkId).success) throw notFound('link not found'); + const kept = await db.begin(async (q) => { + const [cur0] = await q<{ questionId: string }>(`SELECT question_id AS "questionId" FROM book_links WHERE id = $1 AND book = $2`, [linkId, book]); + if (!cur0) throw notFound('link not found'); + await lockQuestion(q, cur0.questionId); + const [cur] = await q<{ lineId: string; role: string }>(`SELECT line_id AS "lineId", role FROM book_links WHERE id = $1`, [linkId]); + if (!cur) throw notFound('link not found'); + const set: Record = { reviewed_by: p.by }; + if (p.status) set.status = p.status; + if (p.role) set.role = p.role; + if (p.scope) set.scope = p.scope; + // a person who changes what it says (role, scope or line) makes it theirs; confirming a suggestion leaves it the matcher's; turning one down is a person's decision + if (p.role || p.scope || p.lineId || p.status === 'rejected') set.method = 'human'; + if (p.lineId && p.lineId !== cur.lineId) { + await lineOf(q, book, p.lineId, true); + set.line_id = p.lineId; + if (p.status === undefined) set.status = 'confirmed'; + } + const target = (set.line_id as string | undefined) ?? cur.lineId, targetRole = (set.role as string | undefined) ?? cur.role; + const [other] = await q<{ id: string }>(`SELECT id FROM book_links WHERE question_id = $1 AND line_id = $2 AND role = $3 AND id <> $4`, [cur0.questionId, target, targetRole, linkId]); + if (other) { + // the question already has that role on that line: that row is kept, confirmed and the person's, and this one goes + await q(`UPDATE book_links SET status = 'confirmed', method = 'human', reviewed_by = $2, reviewed_at = now() WHERE id = $1`, [other.id, p.by]); + await q(`DELETE FROM book_links WHERE id = $1`, [linkId]); + return other.id; + } + const cols = Object.keys(set); + await q(`UPDATE book_links SET ${cols.map((k, i) => `${k} = $${i + 2}`).join(', ')}, reviewed_at = now() WHERE id = $1`, [linkId, ...cols.map((k) => set[k])]); + return linkId; + }); + return c.json(await linkView(db.query, kept)); + })); + + app.delete('/v1/books/:slug/links/:id', (c) => run(c, 'author', async (book) => { + const { by: who } = await parse(c, bySchema); + const linkId = idOf(c.req.param('id'))!; + if (!uuid.safeParse(linkId).success) throw notFound('link not found'); + // under the question's lock: taking a link off is a person's decision, so a matcher placing the question at the same moment must see it + await db.begin(async (q) => { + const [cur] = await q<{ questionId: string }>(`SELECT question_id AS "questionId" FROM book_links WHERE id = $1 AND book = $2`, [linkId, book]); + if (!cur) throw notFound('link not found'); + await lockQuestion(q, cur.questionId); + await q(`UPDATE book_links SET status = 'rejected', method = 'human', reviewed_by = $2, reviewed_at = now() WHERE id = $1`, [linkId, who]); + }); + return c.json({ removed: 1 }); + })); + + app.post('/v1/books/:slug/lines/:id/links/remove', (c) => run(c, 'author', async (book) => { + const b = await parse(c, removeSchema); + const lineId = idOf(c.req.param('id'))!; + if (!uuid.safeParse(lineId).success) throw notFound('that line is not in this book'); + const removed = await db.begin(async (q) => { + const ids = await covered(q, book, lineId, Boolean(b.paragraph)); + if (!ids.length) throw notFound('that line is not in this book'); + const locked = await lockQuestionsOf(q, ids); + const gone = await q(`UPDATE book_links SET status = 'rejected', method = 'human', reviewed_by = $2, reviewed_at = now() + WHERE line_id = ANY($1::uuid[]) AND status <> 'rejected' AND question_id = ANY($3::uuid[]) RETURNING id`, [ids, b.by, locked]); + return gone.length; + }); + return c.json({ removed }); + })); + + app.post('/v1/books/:slug/lines/:id/links/move', (c) => run(c, 'author', async (book) => { + const b = await parse(c, moveSchema); + const from = idOf(c.req.param('id'))!; + if (!uuid.safeParse(from).success) throw notFound('that line is not in this book'); + const moved = await db.begin(async (q) => { + await lineOf(q, book, b.to, true); + await lineOf(q, book, from, false); + // checked after both lines are known: moving a line onto itself still answers 404 for a line the book lacks and 409 for a retired one + if (from === b.to) return 0; + // the questions involved, locked in one order; only their links move (one that arrives meanwhile, of another question, stays where it is) + const locked = await lockQuestionsOf(q, [from]); + // the other line already has that tag: it is kept, becomes the person's, and is confirmed if it had been turned down; rejected rows of the old line stay there + await q(`UPDATE book_links x SET status = CASE WHEN x.status = 'rejected' THEN 'confirmed' ELSE x.status END, method = 'human', reviewed_by = $3, reviewed_at = now() + FROM book_links l WHERE l.line_id = $1 AND l.status <> 'rejected' AND l.question_id = ANY($4::uuid[]) AND x.line_id = $2 AND x.question_id = l.question_id AND x.role = l.role`, [from, b.to, b.by, locked]); + await q(`DELETE FROM book_links l WHERE l.line_id = $1 AND l.status <> 'rejected' AND l.question_id = ANY($3::uuid[]) + AND EXISTS (SELECT 1 FROM book_links x WHERE x.line_id = $2 AND x.question_id = l.question_id AND x.role = l.role)`, [from, b.to, locked]); + const rows = await q(`UPDATE book_links SET line_id = $2, method = 'human', reviewed_by = $3, reviewed_at = now() + WHERE line_id = $1 AND status <> 'rejected' AND question_id = ANY($4::uuid[]) RETURNING id`, [from, b.to, b.by, locked]); + return rows.length; + }); + return c.json({ moved }); + })); + + app.put('/v1/books/:slug/questions/:question/lines', (c) => run(c, 'author', async (book) => { + const b = await parse(c, linesSchema); + const questionId = idOf(c.req.param('question'))!; + if (!uuid.safeParse(questionId).success) throw new HighlightError(400, 'invalid', 'question: not a uuid'); + const add = [...new Set(b.add)], remove = [...new Set(b.remove)]; + if (!add.length && !remove.length) throw new HighlightError(400, 'invalid', 'name a line to add or take off'); + if (add.length + remove.length > MAX_LINE_CHANGES) throw new HighlightError(400, 'invalid', `at most ${MAX_LINE_CHANGES} lines at once`); + if (add.some((s) => remove.includes(s))) throw new HighlightError(400, 'invalid', 'a line cannot be added and taken off at once'); + await db.begin(async (q) => { + await lockQuestion(q, questionId); + const named = [...add, ...remove]; + const lines = await q<{ id: string; retired: boolean }>(`SELECT id, retired_at IS NOT NULL AS retired FROM book_lines WHERE book = $1 AND id = ANY($2::uuid[])`, [book, named]); + if (lines.length !== named.length) throw notFound('that line is not in this book'); + if (lines.some((l) => l.retired && add.includes(l.id))) throw retired(); + // the lines the highlight keeps become the person's + await q(`UPDATE book_links SET method = 'human', status = 'confirmed', reviewed_by = $4, reviewed_at = now() + WHERE book = $1 AND question_id = $2 AND role = $3 AND status <> 'rejected' AND NOT (line_id = ANY($5::uuid[]))`, [book, questionId, b.role, b.by, remove]); + // a line taken off is the person's removal whatever its row was meanwhile (or none), so a matcher never places the question again + for (const lineId of remove) await q(`INSERT INTO book_links (book, question_id, line_id, role, scope, method, status, question_type, reviewed_by, reviewed_at) + VALUES ($1, $2, $3, $4, $5, 'human', 'rejected', $6, $7, now()) + ON CONFLICT (question_id, line_id, role) DO UPDATE SET status = 'rejected', method = 'human', reviewed_by = EXCLUDED.reviewed_by, reviewed_at = now()`, + [book, questionId, lineId, b.role, b.scope ?? 'sentence', b.questionType ?? null, b.by]); + for (const lineId of add) await q(`INSERT INTO book_links (book, question_id, line_id, role, scope, method, status, question_type, reviewed_by, reviewed_at) + VALUES ($1, $2, $3, $4, $5, 'human', 'confirmed', $6, $7, now()) + ON CONFLICT (question_id, line_id, role) DO UPDATE SET status = 'confirmed', method = 'human', reviewed_by = EXCLUDED.reviewed_by, reviewed_at = now(), + question_type = coalesce(EXCLUDED.question_type, book_links.question_type)${b.scope ? ', scope = EXCLUDED.scope' : ''}`, + [book, questionId, lineId, b.role, b.scope ?? 'sentence', b.questionType ?? null, b.by]); + }); + return c.json({ items: await linksOfQuestion(db.query, book, questionId) }); + })); + + app.post('/v1/books/:slug/links/place', (c) => run(c, 'author', async (book) => { + const b = await parse(c, placeSchema); + if (!b.proposals.length) return c.json({ placed: 0 }); + const placed = await db.begin(async (q) => { + await lockQuestion(q, b.questionId); + // still in the pool: no live link and no person's decision, in any book + const [busy] = await q(`SELECT 1 FROM book_links k WHERE k.question_id = $1 AND (k.status <> 'rejected' OR ${HUMAN}) LIMIT 1`, [b.questionId]); + if (busy) return 0; + const have = new Set((await q<{ id: string }>(`SELECT line_id AS id FROM book_links WHERE question_id = $1`, [b.questionId])).map((r) => r.id)); + // FOR SHARE: a sync that retires one of these lines waits for this transaction (and then turns the new suggestions down), and one + // that already retired it is waited for, so no suggestion lands on a line that is gone + const live = new Set((await q<{ id: string }>(`SELECT id FROM book_lines WHERE book = $1 AND retired_at IS NULL AND id = ANY($2::uuid[]) FOR SHARE`, [book, b.proposals.map((p) => p.lineId)])).map((r) => r.id)); + const once = new Set(); // one row per line, whatever the roles asked for + let n = 0; + for (const p of b.proposals) { + if (!live.has(p.lineId) || have.has(p.lineId) || once.has(p.lineId)) continue; + once.add(p.lineId); + n += (await q(`INSERT INTO book_links (book, question_id, line_id, role, scope, score, method, status, rationale, question_type) + VALUES ($1, $2, $3, $4, $5, $6, 'auto', 'suggested', $7, $8) ON CONFLICT DO NOTHING RETURNING id`, + [book, b.questionId, p.lineId, p.role, p.scope ?? 'sentence', p.score ?? null, p.rationale ?? null, b.questionType ?? null])).length; + } + return n; + }); + return c.json({ placed }); + })); + + app.post('/v1/books/:slug/import', (c) => run(c, 'author', async (book) => { + const b = await parse(c, importSchema); + await db.begin(async (q) => { + // an id that is another book's is refused, not skipped: the answer must not count records it did not write + const elsewhere = async (table: 'book_lines' | 'book_links', ids: string[]) => + ids.length ? (await q<{ id: string }>(`SELECT id FROM ${table} WHERE id = ANY($1::uuid[]) AND book <> $2 LIMIT 1`, [ids, book]))[0]?.id : undefined; + const lineClash = await elsewhere('book_lines', b.lines.map((l) => l.id)); + if (lineClash) throw new HighlightError(409, 'conflict', `line ${lineClash} belongs to another book`); + const linkClash = await elsewhere('book_links', b.links.map((l) => l.id)); + if (linkClash) throw new HighlightError(409, 'conflict', `link ${linkClash} belongs to another book`); + for (const part of chunks(b.lines)) { + await q(`INSERT INTO book_lines (id, book, key, page_id, page_no, idx, block, kind, text, text_hash, chapter_no, chapter_title, section, rects, retired_at) + SELECT x.id, $1, x.key, x."pageId", x."pageNo", x.idx, x.block, x.kind, x.text, x."textHash", x."chapterNo", x."chapterTitle", x.section, coalesce(x.rects, '[]'), x."retiredAt" + FROM jsonb_to_recordset($2::jsonb) AS ${LINE_RECORDSET} + ON CONFLICT (id) DO UPDATE SET key = EXCLUDED.key, page_id = EXCLUDED.page_id, page_no = EXCLUDED.page_no, idx = EXCLUDED.idx, block = EXCLUDED.block, kind = EXCLUDED.kind, + text = EXCLUDED.text, text_hash = EXCLUDED.text_hash, chapter_no = EXCLUDED.chapter_no, chapter_title = EXCLUDED.chapter_title, section = EXCLUDED.section, + rects = EXCLUDED.rects, retired_at = EXCLUDED.retired_at, updated_at = now() + WHERE book_lines.book = $1`, [book, JSON.stringify(part)]); + } + for (const part of chunks(b.links)) { + const bad = await q<{ id: string }>(`SELECT x."lineId" AS id FROM jsonb_to_recordset($2::jsonb) AS x("lineId" uuid) + WHERE NOT EXISTS (SELECT 1 FROM book_lines l WHERE l.id = x."lineId" AND l.book = $1) LIMIT 1`, [book, JSON.stringify(part)]); + if (bad.length) throw notFound(`line ${bad[0]!.id} is not in this book`); + await q(`INSERT INTO book_links (id, book, question_id, line_id, role, scope, method, status, score, rationale, question_type, reviewed_by, reviewed_at, created_at) + SELECT x.id, $1, x."questionId", x."lineId", x.role, x.scope, x.method, x.status, x.score, x.rationale, x."questionType", x."reviewedBy", x."reviewedAt", coalesce(x."createdAt", now()) + FROM jsonb_to_recordset($2::jsonb) AS x(id uuid, "questionId" uuid, "lineId" uuid, role text, scope text, method text, status text, score real, rationale text, + "questionType" text, "reviewedBy" text, "reviewedAt" timestamptz, "createdAt" timestamptz) + ON CONFLICT (id) DO UPDATE SET question_id = EXCLUDED.question_id, line_id = EXCLUDED.line_id, role = EXCLUDED.role, scope = EXCLUDED.scope, method = EXCLUDED.method, + status = EXCLUDED.status, score = EXCLUDED.score, rationale = EXCLUDED.rationale, question_type = EXCLUDED.question_type, reviewed_by = EXCLUDED.reviewed_by, + reviewed_at = EXCLUDED.reviewed_at, created_at = EXCLUDED.created_at + WHERE book_links.book = $1`, [book, JSON.stringify(part)]); + } + }); + return c.json({ lines: b.lines.length, links: b.links.length }); + })); +} + +/** A question's live links on live lines of the book, in book order (what `GET links?question=` and `PUT questions/…/lines` answer). */ +function linksOfQuestion(q: Query, book: string, questionId: string) { + return q(`SELECT ${LINK_COLS} ${LINKS_FROM} WHERE k.book = $1 AND k.question_id = $2 AND k.status <> 'rejected' AND l.retired_at IS NULL ORDER BY l.page_no, l.idx, k.role`, [book, questionId]); +} diff --git a/packages/library-server/src/index.ts b/packages/library-server/src/index.ts index af9a239..f0c841b 100644 --- a/packages/library-server/src/index.ts +++ b/packages/library-server/src/index.ts @@ -4,6 +4,7 @@ export { s3Storage, diskStorage, servedType, type Storage, type S3Options, type export { registerDelivery, FILE_SECONDS, LINK_SECONDS, type DeliveryOptions } from './delivery.js'; export { scanStorage } from './scan.js'; export { adoptPrefixes, locate } from './adopt.js'; +export { registerHighlights, planLines, MAX_LINE_CHANGES } from './highlights.js'; export { libraryFromEnv, tokenCallers } from './config.js'; export { checkLibrary, type Check } from './conformance.js'; export { createAuth, checkKeySetup, AuthError, type Auth, type AuthOptions, type TokenResponse, type UserRecord, type ApiKeyRecord } from './auth/service.js'; diff --git a/packages/reader/src/ChalkReader.tsx b/packages/reader/src/ChalkReader.tsx index ee224cb..26afb27 100644 --- a/packages/reader/src/ChalkReader.tsx +++ b/packages/reader/src/ChalkReader.tsx @@ -12,7 +12,7 @@ import { AudioEmbed, LabFrame, ModelViewer, VideoEmbed, localLabStorage, type La import { Modal } from './render/Modal.js'; import { FloatingWindow } from './render/FloatingWindow.js'; import { aspectFor, type Box as WindowBox } from './render/windowGeometry.js'; -import { useAssetUrl } from './render/assets.js'; +import { useAssetSrc, useResume } from './render/assets.js'; import { packOverlay, unpackOverlay, type BookPatch, type MergeReport, type PatchResult } from '@chalk-format/container'; import type { Overlay } from '@chalk-format/model'; import { useAnnotations } from './annotate/state.js'; @@ -887,10 +887,10 @@ function tooSmall(el: HTMLElement, lab: boolean): boolean { /** A full-size image, drawn as HTML so that it can be tapped when its author made it open (spec 1.3). */ function OpenableImage({ book, o, label, onActivate }: { book: Book; o: Extract; label: string; onActivate?: ((el: HTMLElement) => void) | undefined }) { - const url = useAssetUrl(book, o.assetId); + const { url, onError, onLoad } = useAssetSrc(book, o.assetId); return ( - {url ? : } + {url ? : } ); } @@ -908,10 +908,11 @@ function SnipView({ book, pageId, renderHtmlObject }: { book: Book; pageId: stri /** Icon-mode media opened in the dialog, at a comfortable size. */ function MediaInModal({ book, o, locale, theme, storage, onClose, fill }: { book: Book; o: LessonObject; locale: Locale; theme: 'light' | 'dark'; storage: LabStorage | null; onClose: () => void; fill?: boolean }) { - const url = useAssetUrl(book, o.type === 'image' ? o.assetId : o.type === 'embed' ? o.assetId : undefined); + const { url, onError, onLoad } = useAssetSrc(book, o.type === 'image' ? o.assetId : o.type === 'embed' ? o.assetId : undefined); + const resume = useResume(onError, onLoad); const pr = usePresenter(); const mediaClass = fill ? 'chalk-window-media' : 'chalk-modal-media'; - if (o.type === 'image') return url ? {pr.present(o).label : null; + if (o.type === 'image') return url ? {pr.present(o).label : null; if (o.type !== 'embed') return null; const big = fill ? { position: 'relative' as const, width: '100%', height: '100%' } : { position: 'relative' as const, width: '100%', aspectRatio: o.embed === 'model3d' ? '4 / 3' : '16 / 9' }; if (isLabObject(o)) return ; @@ -919,8 +920,8 @@ function MediaInModal({ book, o, locale, theme, storage, onClose, fill }: { book const p = pr.present(o); // opened by a tap: the student asked, so it plays (with sound unless the author muted it) const R = pr.renderers.video; - if (o.embed === 'video' && R) return
{R({ object: o, src: url ?? null, poster: null, presentation: p, where: 'popup', ...pr.env })}
; - if (o.embed === 'video') return url ?