From ef12bf7febec274baefccfc902b4aa112d0e72fe Mon Sep 17 00:00:00 2001 From: David Hamilton Date: Tue, 27 Oct 2026 12:03:28 -0400 Subject: [PATCH] Export complete reference candidates for reviewed integrations Signed-off-by: David Hamilton --- README.md | 1 + docs/integrations.md | 2 + tests/test_corpus.py | 89 +++++++++++++++++++++++++++++++++++++++++++- tools/README.md | 23 ++++++++++++ tools/corpus.py | 37 +++++++++++++++--- 5 files changed, 145 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 22b27d2..0db13d7 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Keep the small setup until you have a specific need: | See a completed small setup | [Fictional Meridian example](examples/meridian/) | | Several audiences or content types need different guidance | [Growth guide](docs/growing-a-writing-system.md) and [fictional Harbor example](examples/harbor/) | | Check metadata or find references locally | [Local tools](tools/README.md) — Python 3, no packages, accounts, or API keys | +| Give an optional integration complete reference text | [Candidate export](tools/README.md#export-complete-reference-candidates) — preserves source bytes; approval and source verification remain separate | | Understand the source and generated views | [Architecture](docs/architecture.md) | | Connect an editor, agent, search service, or docs site | [Integration contract](docs/integrations.md) | | Propose a reference change from observed work | [Learning loop](LEARNING.md) | diff --git a/docs/integrations.md b/docs/integrations.md index c513b8c..6da8f00 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -35,6 +35,8 @@ Use the same corpus interpretation across interfaces where practical. Keep adapt The optional `tools/corpus.py` utilities support corpus `check`, `index`, and `search`; consult `python3 tools/corpus.py --help`. Their outputs are inputs to this workflow, not approval decisions. This document defines no network API or required response schema. +When an integration needs complete references rather than index metadata or search excerpts, the optional [candidate export](../tools/README.md#export-complete-reference-candidates) preserves each file's original text and byte hash. Its reported revision is explicitly unverified; exporting does not select policy, approve content, sanitize directives, or satisfy the cross-repository contract below. A tool such as Ghost can consume reviewed language data under an explicitly authored scope without becoming a second place to maintain the rules. + ## Crossing a repository boundary Follow [PINNING.md](../PINNING.md), rather than asking a drafting agent to fetch a source repository. A human reviewer or trusted integration validates the canonical source and exact revision, rejects operational directives, and projects approved declarative language data into a reviewed repository-local snapshot. diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 8c046e9..742598f 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -20,6 +20,7 @@ SPEC.loader.exec_module(corpus) BASE = ('---\ntitle: "Sample"\nbrand: "fictional"\ndomain: "shared"\n' 'enforcement: "should"\n---\n\n# Sample\nA payment failed.\n') +REVISION = "a1" * 20 class CorpusTests(unittest.TestCase): @@ -71,17 +72,103 @@ def test_nested_hashes_and_determinism(self): first.write_bytes(first.read_bytes() + b"\n") self.assertNotEqual(before.stdout, self.cli("index").stdout) + def test_export_preserves_complete_original_bytes_and_leaves_index_unchanged(self): + text = BASE.replace('title: "Sample"', '# Source comment\ntitle: "Café"') + text += ('\nNon-negotiable (**must**).\n\n## Exact wording\n' + '> Funds arrive in 1–2 days. \n\n' + 'Scope: fictional US-English email; owner review required.\n\t\n') + raw = text.replace("\n", "\r\n").encode("utf-8") + path = self.put() + path.write_bytes(raw) + before = self.cli("index").stdout + search_before = self.cli("search", "funds").stdout + result = self.cli("export", "--reported-revision", REVISION.upper()) + self.assertEqual(result.returncode, 0, result.stderr) + exported = json.loads(result.stdout) + self.assertEqual(exported["kind"], "chalkline-reference-export") + self.assertEqual(exported["source"], { + "reported_revision": REVISION, "verification": "unverified"}) + record = exported["records"][0] + self.assertEqual(record["text"].encode("utf-8"), raw) + self.assertEqual(record["sha256"], hashlib.sha256(raw).hexdigest()) + self.assertEqual(record["bytes"], len(raw)) + self.assertEqual(record["metadata"]["enforcement"], "should") + self.assertIn("(**must**)", record["text"]) + self.assertNotIn(str(self.root), result.stdout) + self.assertEqual(result.stdout, + self.cli("export", "--reported-revision", REVISION).stdout) + self.assertEqual(before, self.cli("index").stdout) + self.assertEqual(search_before, self.cli("search", "funds").stdout) + self.assertEqual(json.loads(before)["records"], [ + {key: value for key, value in record.items() if key != "text"}]) + + def test_export_scope_excludes_calibration_and_preserves_vocabulary_as_data(self): + self.put("z/voice.md") + self.put("vocabulary.md", BASE.replace('"should"', '"may"')) + self.put("README.md", "placeholder") + (self.root / "calibration").mkdir() + (self.root / "calibration" / "001-held-out.md").write_text("REVIEWER ONLY") + (self.root / "AGENTS.md").write_text("OPERATIONAL INSTRUCTIONS") + result = self.cli("export", "--reported-revision", "b" * 64) + self.assertEqual(result.returncode, 0, result.stderr) + exported = json.loads(result.stdout) + self.assertEqual(exported["source"]["reported_revision"], "b" * 64) + self.assertEqual([r["path"] for r in exported["records"]], [ + "references/vocabulary.md", "references/z/voice.md"]) + self.assertNotIn("REVIEWER ONLY", result.stdout) + self.assertNotIn("OPERATIONAL INSTRUCTIONS", result.stdout) + + def test_export_empty_template_is_not_coverage(self): + result = self.cli("export", "--reported-revision", REVISION) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["records"], []) + + def test_export_requires_full_reported_revision_before_creating_output(self): + self.put() + output = self.root / "candidate.json" + for revision in (None, "main", "a" * 39, "a" * 41, "g" * 40, "a" * 63): + with self.subTest(revision=revision): + args = ["export", "--output", str(output)] + if revision is not None: + args += ["--reported-revision", revision] + result = self.cli(*args) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertEqual(result.stdout, "") + self.assertFalse(output.exists()) + + def test_export_rejects_bad_corpus_without_partial_output_or_overwrite(self): + self.put("a.md") + self.put("z.md", "invalid frontmatter") + output = self.root / "candidate.json" + result = self.cli("export", "--reported-revision", REVISION, + "--output", str(output)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, "") + self.assertFalse(output.exists()) + self.put("z.md") + result = self.cli("export", "--reported-revision", REVISION, + "--output", str(output)) + self.assertEqual(result.returncode, 0, result.stderr) + first = output.read_bytes() + self.put("a.md", BASE + "Changed.\n") + result = self.cli("export", "--reported-revision", REVISION, + "--output", str(output)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(output.read_bytes(), first) + def assert_limit_failure(self, constant, limit, message): with mock.patch.object(corpus, constant, limit): with self.assertRaisesRegex(corpus.CorpusError, message): corpus.load_corpus(self.root) - for command in ("check", "index", "search"): + for command in ("check", "index", "search", "export"): for to_file in (False, True): with self.subTest(command=command, to_file=to_file): output = self.root / "result.json" args = [command, "--root", str(self.root)] if command == "search": args += ["payment", "--brand", "other"] + if command == "export": + args += ["--reported-revision", REVISION] if to_file: args += ["--output", str(output)] with mock.patch("sys.stdout", new_callable=io.StringIO) as stdout, \ diff --git a/tools/README.md b/tools/README.md index 59c11a6..94545bf 100644 --- a/tools/README.md +++ b/tools/README.md @@ -69,6 +69,29 @@ Exit status `0` means the operation completed, including an empty corpus or no s Hashes identify bytes; they do not prove approval or trustworthy provenance. These tools do not evaluate prose, resolve conflicts, interpret inline enforcement, verify source-record attestations, or authorize publication. They do not implement the cross-repository snapshot validator described in [PINNING.md](../PINNING.md). +## Export complete reference candidates + +An index or search excerpt is not enough to apply a writing reference: it can omit inline `must` rules, exceptions, exact wording, or the approval scope. The optional `export` command returns complete source text for a reader or integration to review. It does not change the drafting protocol or create an approved consumer snapshot. + +```sh +python3 tools/corpus.py export --root examples/meridian \ + --reported-revision FULL_GIT_OBJECT_ID --output /tmp/chalkline-candidates.json +``` + +Replace `FULL_GIT_OBJECT_ID` with the full 40- or 64-character revision you are reporting. This is **unverified attribution supplied by the caller**. The tool does not read Git, compare the working tree with that revision, establish source identity, or attest approval. Local files may differ from the reported revision. A receiving owner must independently verify the source and bytes before deployment under [PINNING.md](../PINNING.md). + +The deterministic JSON has `schema_version: 1`, `kind: "chalkline-reference-export"`, a `source` object containing `reported_revision` and `verification: "unverified"`, and path-sorted `records`. Each record retains the index fields (`path`, `metadata`, `sha256`, `bytes`) and adds `text`: the entire original UTF-8 file, including frontmatter, comments, line endings, provenance, and trailing whitespace. Encoding the decoded `text` as UTF-8 recovers the bytes identified by its hash. The JSON's own indentation and escape sequences are transport, not new language rules. + +Export uses the same bounded corpus loader and new-file-only output behavior as the other commands. It validates the whole corpus before producing output. Empty corpora return an empty `records` list; that is a coverage gap, not approval. Existing `check`, `index`, and `search` output shapes are unchanged. + +All reference files are candidates. Export does not choose applicable rules or classify arbitrary prose as safe declarative data. In particular: + +- `vocabulary.md` remains conversational context, not terminology authority for generated copy. Its presence or `teaching` setting cannot establish the recipient's consent. +- Calibration pairs, setup protocols, root instruction files, and scripts are outside `references/` and are excluded. This command provides no calibration coverage; keep held-out evaluation targets out of any consumer context. +- Operational directives inside a structurally valid reference are preserved. A human reviewer or trusted integration must reject them before a drafting agent sees a cross-repository snapshot. Hashing and JSON encoding do not sanitize prose. + +Keep candidate exports outside `references/` and distinct from authored sources. For a consumer such as Ghost, explicitly review the selected references, their applicability and authority, and the receiving workflow's conflict and missing-source policy. Preserve complete conditions, inline enforcement, exact-wording scope, and required review. The export supplies neither a Ghost package nor a PINNING attestation, and does not require a service, network, or Ghost installation. + ## Traceability receipt checker Drafting and review now include a [traceability and confidence receipt](../docs/traceability.md) by default. Plain text is sufficient. For an integration that needs structured JSON, the optional checker validates the receipt's fields, internal evidence references, and score arithmetic: diff --git a/tools/corpus.py b/tools/corpus.py index 634db24..f1e19ac 100644 --- a/tools/corpus.py +++ b/tools/corpus.py @@ -105,7 +105,7 @@ def parse(raw, path): return metadata, lines[end + 1:], end + 2 -def load_corpus(root): +def load_corpus(root, include_text=False): root = Path(root).absolute() # Refuse symlinks in the supplied root, including its parent components. for component in (root,) + tuple(root.parents): @@ -166,9 +166,12 @@ def walk(directory, depth=0): metadata, body, first_line = parse(raw, path) except CorpusError as exc: raise CorpusError(path + ": " + str(exc)) from exc - records.append({"path": path, "metadata": metadata, - "sha256": hashlib.sha256(raw).hexdigest(), - "bytes": len(raw), "body": body, "first_line": first_line}) + record = {"path": path, "metadata": metadata, + "sha256": hashlib.sha256(raw).hexdigest(), + "bytes": len(raw), "body": body, "first_line": first_line} + if include_text: + record["text"] = raw.decode("utf-8") + records.append(record) walk(references) return sorted(records, key=lambda record: record["path"]) @@ -177,6 +180,12 @@ def public_record(record): return {key: record[key] for key in ("path", "metadata", "sha256", "bytes")} +def reported_revision(value): + if not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", value): + raise argparse.ArgumentTypeError("expected a full 40- or 64-character Git object ID") + return value.lower() + + def search(records, query, filters): terms = query.casefold().split() results = [] @@ -232,6 +241,12 @@ def search(records, query, filters): matching, stemming, ranking, implicit shared-domain fallback, or rule evaluation. Matches list body lines containing any term; title-only matches have no lines. Empty query lists all records satisfying filters. Empty corpus is valid. + Export adds each reference's complete original UTF-8 text, including + frontmatter and line endings. UTF-8 encoding of text recovers the hashed bytes. + --reported-revision is a required caller-reported full Git object ID (40 or + 64 hex characters); it is NOT verified against Git or the local files. + Export is unreviewed source data, not an approved snapshot or trust validation. + It includes vocabulary (conversation only); calibration remains excluded. No network, model, content execution, link following, or default disk writes. --output explicitly creates a NEW file (never overwrites; parents must exist). Check validates structure only, not writing quality or authority of the rules. @@ -242,6 +257,7 @@ def search(records, query, filters): python3 tools/corpus.py index --root examples/meridian python3 tools/corpus.py search payment --root examples/meridian --domain shared python3 tools/corpus.py index --root . --output /tmp/chalkline-index.json + python3 tools/corpus.py export --root . --reported-revision FULL_GIT_OBJECT_ID """ @@ -249,11 +265,14 @@ def parser(): cli = argparse.ArgumentParser(description=__doc__, epilog=HELP, formatter_class=argparse.RawDescriptionHelpFormatter) commands = cli.add_subparsers(dest="command", required=True) - for name in ("check", "index", "search"): + for name in ("check", "index", "search", "export"): sub = commands.add_parser(name, epilog=HELP, formatter_class=argparse.RawDescriptionHelpFormatter) sub.add_argument("--root", default=".", help="directory containing references/ (default: .)") sub.add_argument("--output", metavar="NEW_FILE", help="create JSON file instead of stdout; never overwrite") + if name == "export": + sub.add_argument("--reported-revision", required=True, type=reported_revision, + metavar="FULL_GIT_OBJECT_ID", help="caller-reported revision; not verified") if name == "search": sub.add_argument("query", nargs="?", default="", help="literal terms; AND matching; empty lists all") sub.add_argument("--brand", help="exact brand slug filter") @@ -265,12 +284,18 @@ def parser(): def main(argv=None): args = parser().parse_args(argv) try: - records = load_corpus(args.root) + records = load_corpus(args.root, include_text=args.command == "export") result = {"schema_version": 1} if args.command == "check": result.update({"valid": True, "count": len(records)}) elif args.command == "index": result["records"] = [public_record(record) for record in records] + elif args.command == "export": + result.update({"kind": "chalkline-reference-export", + "source": {"reported_revision": args.reported_revision, + "verification": "unverified"}, + "records": [dict(public_record(record), text=record["text"]) + for record in records]}) else: filters = {key: getattr(args, key) for key in ("brand", "domain", "enforcement") if getattr(args, key) is not None}