Export complete reference candidates for reviewed integrations

README.md

@@ -74,6 +74,7 @@ Keep the small setup until you have a specific need:
 | See a completed small setup | [Fictional Meridian example](examples/meridian/) |
 | Several audiences or content types need different guidance | [Growth guide](docs/growing-a-writing-system.md) and [fictional Harbor example](examples/harbor/) |
 | Check metadata or find references locally | [Local tools](tools/README.md) — Python 3, no packages, accounts, or API keys |
+| Give an optional integration complete reference text | [Candidate export](tools/README.md#export-complete-reference-candidates) — preserves source bytes; approval and source verification remain separate |
 | Understand the source and generated views | [Architecture](docs/architecture.md) |
 | Connect an editor, agent, search service, or docs site | [Integration contract](docs/integrations.md) |
 | Propose a reference change from observed work | [Learning loop](LEARNING.md) |

docs/integrations.md

@@ -35,6 +35,8 @@ Use the same corpus interpretation across interfaces where practical. Keep adapt
 
 The optional `tools/corpus.py` utilities support corpus `check`, `index`, and `search`; consult `python3 tools/corpus.py --help`. Their outputs are inputs to this workflow, not approval decisions. This document defines no network API or required response schema.
 
+When an integration needs complete references rather than index metadata or search excerpts, the optional [candidate export](../tools/README.md#export-complete-reference-candidates) preserves each file's original text and byte hash. Its reported revision is explicitly unverified; exporting does not select policy, approve content, sanitize directives, or satisfy the cross-repository contract below. A tool such as Ghost can consume reviewed language data under an explicitly authored scope without becoming a second place to maintain the rules.
+
 ## Crossing a repository boundary
 
 Follow [PINNING.md](../PINNING.md), rather than asking a drafting agent to fetch a source repository. A human reviewer or trusted integration validates the canonical source and exact revision, rejects operational directives, and projects approved declarative language data into a reviewed repository-local snapshot.

tests/test_corpus.py

@@ -20,6 +20,7 @@
 SPEC.loader.exec_module(corpus)
 BASE = ('---\ntitle: "Sample"\nbrand: "fictional"\ndomain: "shared"\n'
         'enforcement: "should"\n---\n\n# Sample\nA payment failed.\n')
+REVISION = "a1" * 20
 
 
 class CorpusTests(unittest.TestCase):
@@ -71,17 +72,103 @@ def test_nested_hashes_and_determinism(self):
         first.write_bytes(first.read_bytes() + b"\n")
         self.assertNotEqual(before.stdout, self.cli("index").stdout)
 
+    def test_export_preserves_complete_original_bytes_and_leaves_index_unchanged(self):
+        text = BASE.replace('title: "Sample"', '# Source comment\ntitle: "Café"')
+        text += ('\nNon-negotiable (**must**).\n\n## Exact wording\n'
+                 '> Funds arrive in 1–2 days.  \n\n'
+                 'Scope: fictional US-English email; owner review required.\n\t\n')
+        raw = text.replace("\n", "\r\n").encode("utf-8")
+        path = self.put()
+        path.write_bytes(raw)
+        before = self.cli("index").stdout
+        search_before = self.cli("search", "funds").stdout
+        result = self.cli("export", "--reported-revision", REVISION.upper())
+        self.assertEqual(result.returncode, 0, result.stderr)
+        exported = json.loads(result.stdout)
+        self.assertEqual(exported["kind"], "chalkline-reference-export")
+        self.assertEqual(exported["source"], {
+            "reported_revision": REVISION, "verification": "unverified"})
+        record = exported["records"][0]
+        self.assertEqual(record["text"].encode("utf-8"), raw)
+        self.assertEqual(record["sha256"], hashlib.sha256(raw).hexdigest())
+        self.assertEqual(record["bytes"], len(raw))
+        self.assertEqual(record["metadata"]["enforcement"], "should")
+        self.assertIn("(**must**)", record["text"])
+        self.assertNotIn(str(self.root), result.stdout)
+        self.assertEqual(result.stdout,
+                         self.cli("export", "--reported-revision", REVISION).stdout)
+        self.assertEqual(before, self.cli("index").stdout)
+        self.assertEqual(search_before, self.cli("search", "funds").stdout)
+        self.assertEqual(json.loads(before)["records"], [
+            {key: value for key, value in record.items() if key != "text"}])
+
+    def test_export_scope_excludes_calibration_and_preserves_vocabulary_as_data(self):
+        self.put("z/voice.md")
+        self.put("vocabulary.md", BASE.replace('"should"', '"may"'))
+        self.put("README.md", "placeholder")
+        (self.root / "calibration").mkdir()
+        (self.root / "calibration" / "001-held-out.md").write_text("REVIEWER ONLY")
+        (self.root / "AGENTS.md").write_text("OPERATIONAL INSTRUCTIONS")
+        result = self.cli("export", "--reported-revision", "b" * 64)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        exported = json.loads(result.stdout)
+        self.assertEqual(exported["source"]["reported_revision"], "b" * 64)
+        self.assertEqual([r["path"] for r in exported["records"]], [
+            "references/vocabulary.md", "references/z/voice.md"])
+        self.assertNotIn("REVIEWER ONLY", result.stdout)
+        self.assertNotIn("OPERATIONAL INSTRUCTIONS", result.stdout)
+
+    def test_export_empty_template_is_not_coverage(self):
+        result = self.cli("export", "--reported-revision", REVISION)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(result.stdout)["records"], [])
+
+    def test_export_requires_full_reported_revision_before_creating_output(self):
+        self.put()
+        output = self.root / "candidate.json"
+        for revision in (None, "main", "a" * 39, "a" * 41, "g" * 40, "a" * 63):
+            with self.subTest(revision=revision):
+                args = ["export", "--output", str(output)]
+                if revision is not None:
+                    args += ["--reported-revision", revision]
+                result = self.cli(*args)
+                self.assertEqual(result.returncode, 2, result.stderr)
+                self.assertEqual(result.stdout, "")
+                self.assertFalse(output.exists())
+
+    def test_export_rejects_bad_corpus_without_partial_output_or_overwrite(self):
+        self.put("a.md")
+        self.put("z.md", "invalid frontmatter")
+        output = self.root / "candidate.json"
+        result = self.cli("export", "--reported-revision", REVISION,
+                          "--output", str(output))
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertEqual(result.stdout, "")
+        self.assertFalse(output.exists())
+        self.put("z.md")
+        result = self.cli("export", "--reported-revision", REVISION,
+                          "--output", str(output))
+        self.assertEqual(result.returncode, 0, result.stderr)
+        first = output.read_bytes()
+        self.put("a.md", BASE + "Changed.\n")
+        result = self.cli("export", "--reported-revision", REVISION,
+                          "--output", str(output))
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertEqual(output.read_bytes(), first)
+
     def assert_limit_failure(self, constant, limit, message):
         with mock.patch.object(corpus, constant, limit):
             with self.assertRaisesRegex(corpus.CorpusError, message):
                 corpus.load_corpus(self.root)
-            for command in ("check", "index", "search"):
+            for command in ("check", "index", "search", "export"):
                 for to_file in (False, True):
                     with self.subTest(command=command, to_file=to_file):
                         output = self.root / "result.json"
                         args = [command, "--root", str(self.root)]
                         if command == "search":
                             args += ["payment", "--brand", "other"]
+                        if command == "export":
+                            args += ["--reported-revision", REVISION]
                         if to_file:
                             args += ["--output", str(output)]
                         with mock.patch("sys.stdout", new_callable=io.StringIO) as stdout, \

tools/README.md

@@ -69,6 +69,29 @@ Exit status `0` means the operation completed, including an empty corpus or no s
 
 Hashes identify bytes; they do not prove approval or trustworthy provenance. These tools do not evaluate prose, resolve conflicts, interpret inline enforcement, verify source-record attestations, or authorize publication. They do not implement the cross-repository snapshot validator described in [PINNING.md](../PINNING.md).
 
+## Export complete reference candidates
+
+An index or search excerpt is not enough to apply a writing reference: it can omit inline `must` rules, exceptions, exact wording, or the approval scope. The optional `export` command returns complete source text for a reader or integration to review. It does not change the drafting protocol or create an approved consumer snapshot.
+
+```sh
+python3 tools/corpus.py export --root examples/meridian \
+  --reported-revision FULL_GIT_OBJECT_ID --output /tmp/chalkline-candidates.json
+```
+
+Replace `FULL_GIT_OBJECT_ID` with the full 40- or 64-character revision you are reporting. This is **unverified attribution supplied by the caller**. The tool does not read Git, compare the working tree with that revision, establish source identity, or attest approval. Local files may differ from the reported revision. A receiving owner must independently verify the source and bytes before deployment under [PINNING.md](../PINNING.md).
+
+The deterministic JSON has `schema_version: 1`, `kind: "chalkline-reference-export"`, a `source` object containing `reported_revision` and `verification: "unverified"`, and path-sorted `records`. Each record retains the index fields (`path`, `metadata`, `sha256`, `bytes`) and adds `text`: the entire original UTF-8 file, including frontmatter, comments, line endings, provenance, and trailing whitespace. Encoding the decoded `text` as UTF-8 recovers the bytes identified by its hash. The JSON's own indentation and escape sequences are transport, not new language rules.
+
+Export uses the same bounded corpus loader and new-file-only output behavior as the other commands. It validates the whole corpus before producing output. Empty corpora return an empty `records` list; that is a coverage gap, not approval. Existing `check`, `index`, and `search` output shapes are unchanged.
+
+All reference files are candidates. Export does not choose applicable rules or classify arbitrary prose as safe declarative data. In particular:
+
+- `vocabulary.md` remains conversational context, not terminology authority for generated copy. Its presence or `teaching` setting cannot establish the recipient's consent.
+- Calibration pairs, setup protocols, root instruction files, and scripts are outside `references/` and are excluded. This command provides no calibration coverage; keep held-out evaluation targets out of any consumer context.
+- Operational directives inside a structurally valid reference are preserved. A human reviewer or trusted integration must reject them before a drafting agent sees a cross-repository snapshot. Hashing and JSON encoding do not sanitize prose.
+
+Keep candidate exports outside `references/` and distinct from authored sources. For a consumer such as Ghost, explicitly review the selected references, their applicability and authority, and the receiving workflow's conflict and missing-source policy. Preserve complete conditions, inline enforcement, exact-wording scope, and required review. The export supplies neither a Ghost package nor a PINNING attestation, and does not require a service, network, or Ghost installation.
+
 ## Traceability receipt checker
 
 Drafting and review now include a [traceability and confidence receipt](../docs/traceability.md) by default. Plain text is sufficient. For an integration that needs structured JSON, the optional checker validates the receipt's fields, internal evidence references, and score arithmetic:

tools/corpus.py

@@ -105,7 +105,7 @@ def parse(raw, path):
     return metadata, lines[end + 1:], end + 2
 
 
-def load_corpus(root):
+def load_corpus(root, include_text=False):
     root = Path(root).absolute()
     # Refuse symlinks in the supplied root, including its parent components.
     for component in (root,) + tuple(root.parents):
@@ -166,9 +166,12 @@ def walk(directory, depth=0):
                     metadata, body, first_line = parse(raw, path)
                 except CorpusError as exc:
                     raise CorpusError(path + ": " + str(exc)) from exc
-                records.append({"path": path, "metadata": metadata,
-                                "sha256": hashlib.sha256(raw).hexdigest(),
-                                "bytes": len(raw), "body": body, "first_line": first_line})
+                record = {"path": path, "metadata": metadata,
+                          "sha256": hashlib.sha256(raw).hexdigest(),
+                          "bytes": len(raw), "body": body, "first_line": first_line}
+                if include_text:
+                    record["text"] = raw.decode("utf-8")
+                records.append(record)
     walk(references)
     return sorted(records, key=lambda record: record["path"])
 
@@ -177,6 +180,12 @@ def public_record(record):
     return {key: record[key] for key in ("path", "metadata", "sha256", "bytes")}
 
 
+def reported_revision(value):
+    if not re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", value):
+        raise argparse.ArgumentTypeError("expected a full 40- or 64-character Git object ID")
+    return value.lower()
+
+
 def search(records, query, filters):
     terms = query.casefold().split()
     results = []
@@ -232,6 +241,12 @@ def search(records, query, filters):
   matching, stemming, ranking, implicit shared-domain fallback, or rule evaluation.
   Matches list body lines containing any term; title-only matches have no lines.
   Empty query lists all records satisfying filters. Empty corpus is valid.
+  Export adds each reference's complete original UTF-8 text, including
+  frontmatter and line endings. UTF-8 encoding of text recovers the hashed bytes.
+  --reported-revision is a required caller-reported full Git object ID (40 or
+  64 hex characters); it is NOT verified against Git or the local files.
+  Export is unreviewed source data, not an approved snapshot or trust validation.
+  It includes vocabulary (conversation only); calibration remains excluded.
   No network, model, content execution, link following, or default disk writes.
   --output explicitly creates a NEW file (never overwrites; parents must exist).
   Check validates structure only, not writing quality or authority of the rules.
@@ -242,18 +257,22 @@ def search(records, query, filters):
   python3 tools/corpus.py index --root examples/meridian
   python3 tools/corpus.py search payment --root examples/meridian --domain shared
   python3 tools/corpus.py index --root . --output /tmp/chalkline-index.json
+  python3 tools/corpus.py export --root . --reported-revision FULL_GIT_OBJECT_ID
 """
 
 
 def parser():
     cli = argparse.ArgumentParser(description=__doc__, epilog=HELP,
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
     commands = cli.add_subparsers(dest="command", required=True)
-    for name in ("check", "index", "search"):
+    for name in ("check", "index", "search", "export"):
         sub = commands.add_parser(name, epilog=HELP,
                                   formatter_class=argparse.RawDescriptionHelpFormatter)
         sub.add_argument("--root", default=".", help="directory containing references/ (default: .)")
         sub.add_argument("--output", metavar="NEW_FILE", help="create JSON file instead of stdout; never overwrite")
+        if name == "export":
+            sub.add_argument("--reported-revision", required=True, type=reported_revision,
+                             metavar="FULL_GIT_OBJECT_ID", help="caller-reported revision; not verified")
         if name == "search":
             sub.add_argument("query", nargs="?", default="", help="literal terms; AND matching; empty lists all")
             sub.add_argument("--brand", help="exact brand slug filter")
@@ -265,12 +284,18 @@ def parser():
 def main(argv=None):
     args = parser().parse_args(argv)
     try:
-        records = load_corpus(args.root)
+        records = load_corpus(args.root, include_text=args.command == "export")
         result = {"schema_version": 1}
         if args.command == "check":
             result.update({"valid": True, "count": len(records)})
         elif args.command == "index":
             result["records"] = [public_record(record) for record in records]
+        elif args.command == "export":
+            result.update({"kind": "chalkline-reference-export",
+                           "source": {"reported_revision": args.reported_revision,
+                                      "verification": "unverified"},
+                           "records": [dict(public_record(record), text=record["text"])
+                                       for record in records]})
         else:
             filters = {key: getattr(args, key) for key in ("brand", "domain", "enforcement")
                        if getattr(args, key) is not None}