From afedff517624f560359528398d57b6c1c42556fe Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 20:09:48 +0200 Subject: [PATCH 01/25] doc-links(md): link Markdown sections to the code files and definitions they name A Markdown document is documentation in itself: its sections now link to the code they reference explicitly, as MENTIONS edges (via "markdown") from the Section node (the File node for text before the first heading, and for a heading whose title repeats in the file, whose Section node it does not own). An agent can go from a document to the code and back through the graph: trace_path inbound with edge_types ["MENTIONS"] lists the sections that name a definition. Families (the "syntax" property; each with its ship gate): - link: the destination of an inline link or a link reference definition (no URL, no in-page anchor, no image); - path: a bare path in prose (a slash and a file extension); - code_path: a code span or HTML span holding a path, a file name or a directory, with an optional line range (#L3-L9, :3-9) or member (path::Name); - code_name: a code span holding a qualified name (a.b, a::b, A#b, A\B, mod:attr). Resolution is guess-free, with the rules the documentation field tests measured: - a path is read relative to the document and, when it has a directory part, to the repository root; a file name alone only next to the document; - a line range binds the innermost code definition holding the whole range, else the file with "target_lines" on the edge; path::member the one definition of that name in the file; - a qualified name binds the one code definition whose qualifier chain ends with the written qualifier, else the one module or package so named; a match as written outranks one that needs case folding; a field binds only through its type, never through an instance; module:attr only in Python; product documentation never binds test code; - a bare name is never an edge. Unresolved references are rows in doc_link_unresolved: missing (the directory or the repository's own root is there, the name is not), ambiguous, test_only_target. A URL, an image, a URL route (/docs), a file of a type the indexer does not read, and code of other projects are no references to the repository and leave no row. Extraction scans the document's bytes once (doclink_md.c); every scan of a line is linear, or n log n where it sorts, using position lists that grow with the bytes they track, not with the line. Front matter, fenced code, HTML comments and MDX import/export lines are not scanned. The incremental route's node proxies now carry their line numbers (pipeline_delta.c), so a re-resolved document binds a line range exactly as a full build does; the Markdown resolver reads definitions from its own by-file list of the run's nodes, never from DEFINES edges, which an incremental run holds for re-extracted files only. Measured on four repositories against the main branch (which links file to file only, REFERENCES_FILE): fastapi 2,899 MENTIONS edges (2,225 REFERENCES_FILE before), cosmos-sdk 1,286 with 205 distinct definitions reached and 188 edges from 41 ADRs into code, dapr 184, curl 52. Link pairs agree with the field-test harvester (cosmos-sdk 408 of 409; the difference is a link of a document to itself, skipped on purpose). A hand check of 45 sampled qualified-name links found one wrong, fixed by the exact-case rule above. Tests: doc_links_md (extraction, repeated headings, the span classifier table, resolution with every reason, back navigation, incremental == full for a code edit, a document edit and an added name). Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/doclink.c | 8 + internal/cbm/doclink.h | 53 +- internal/cbm/doclink_md.c | 1788 +++++++++++++++++++++++++++++++++ src/pipeline/doc_links.c | 20 +- src/pipeline/doc_links.h | 9 + src/pipeline/doc_links_md.c | 1122 +++++++++++++++++++++ src/pipeline/pipeline_delta.c | 12 +- tests/test_doc_links_md.c | 419 ++++++++ tests/test_main.c | 2 + 10 files changed, 3428 insertions(+), 9 deletions(-) create mode 100644 internal/cbm/doclink_md.c create mode 100644 src/pipeline/doc_links_md.c create mode 100644 tests/test_doc_links_md.c diff --git a/Makefile.cbm b/Makefile.cbm index 120746a795..4e365971be 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -375,6 +375,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_dbt.c \ $(CBM_DIR)/doclink.c \ $(CBM_DIR)/doclink_cs.c \ + $(CBM_DIR)/doclink_md.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -497,6 +498,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_doclinks.c \ src/pipeline/doc_links.c \ src/pipeline/doc_links_cs.c \ + src/pipeline/doc_links_md.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -759,7 +761,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index d7d9b8b419..bf7f695f66 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -143,6 +143,9 @@ typedef struct { * the same doc): only the first label carries the doc's references. */ const char *twin_label; const char *twin_of; + /* A document language: the whole file is the documentation, so one scan + * of its bytes replaces parse_doc, the file-level doc and the scope. */ + void (*scan_file)(CBMExtractCtx *ctx); } doclink_lang_t; static const doclink_lang_t DOCLINK_LANGS[] = { @@ -159,6 +162,7 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_cs_project_scan_scope, .scope_tag = CBM_DOCLINK_CS_SCOPE_TAG, .portable_scope = cbm_doclink_cs_portable_scope}, + {.lang = CBM_LANG_MARKDOWN, .scan_file = cbm_doclink_md_scan_file}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { @@ -388,6 +392,10 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) { if (!L) { return; } + if (L->scan_file) { + L->scan_file(ctx); + return; + } int twin_count = 0; doclink_twin_key_t *twins = L->twin_label ? twin_keys(ctx, L->twin_of, &twin_count) : NULL; CBMDefArray *defs = &ctx->result->defs; diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 7b85ad45a3..6e5bb035fa 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -22,6 +22,10 @@ * identical on full and incremental runs. Optional. * scope_tag, the blob's tag line and its persisted form (line * portable_scope numbers dropped). Optional. + * scan_file instead of parse_doc, for a DOCUMENT language whose + * text is the documentation (Markdown): one pass over + * the file's bytes pushes every reference, with the + * section it is written in as its source. Optional. * State a language's hooks need between their calls for one file lives in * ctx->doclink_state (cbm.h), never in a static or thread-local. * A language without a row produces no tokens and no scope. The resolving @@ -61,6 +65,11 @@ X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true) \ X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true) \ X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true) \ + /* markdown (doclink_md.c): what a document's text names explicitly */ \ + X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true) \ + X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true) \ + X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true) \ + X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true) \ /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false) @@ -79,7 +88,8 @@ enum { * edge source is the file's File node, which the resolving half looks up * from the file it is resolving; `source_qn` is not the source then. Set * by the driver (cbm_doclinks_extract) on what a parser pushes for the - * file-level doc -- a parser never sets it. */ + * file-level doc -- a parser never sets it. A scan_file hook sets it on a + * reference written before the document's first section. */ CBM_DOCLINK_FLAG_FILE = 1, }; @@ -221,4 +231,45 @@ size_t cbm_doclink_cs_norm_type(const char *in, size_t len, char *out, size_t ca /* The C# scope blob starts with this tag line. */ #define CBM_DOCLINK_CS_SCOPE_TAG "cs1" +/* ── Markdown (doclink_md.c) ───────────────────────────────────────── */ + +/* The scan_file hook: every explicit reference of a Markdown or MDX file (a + * link destination, a bare path, a path or a qualified name in a code span), + * each with the Section definition it is written under as its source. */ +void cbm_doclink_md_scan_file(CBMExtractCtx *ctx); + +/* What a code span's text names. */ +typedef enum { + CBM_DOCLINK_MD_NONE = 0, /* nothing linkable (a bare name, a command, a snippet, ...) */ + CBM_DOCLINK_MD_FILE, /* a path with a directory part, or a dot file */ + CBM_DOCLINK_MD_FILENAME, /* a file name alone: it names a file of the document's + * own directory, never one found elsewhere */ + CBM_DOCLINK_MD_DIR, /* a directory */ + CBM_DOCLINK_MD_QUALIFIED, /* a qualified name of two or more identifiers + * (`a.b`, `a::b`, `A#b`, `A\B`, `mod:attr`) */ +} CBMDocLinkMdShape; + +typedef struct { + CBMDocLinkMdShape shape; + const char *path; /* NUL-terminated, in the caller's buffer; for QUALIFIED the + * identifiers joined by '.' */ + const char *member; /* the text after a `path::`, NUL-terminated in the same + * buffer: the member the path's file declares; NULL: none */ + uint32_t first_line; /* a line range written with the path (`#L3-L9`, `:3-9`, `:3`); */ + uint32_t last_line; /* 0 and 0 when there is none */ + bool instance; /* QUALIFIED: written through an instance (`self.x`, `this.x`): + * the qualifier is no type, so it never names a field */ + bool colon; /* QUALIFIED: written `module:attr` (Python's entry-point form) */ +} CBMDocLinkMdPath; + +/* Classify the text of a code span. Returns false, with shape NONE, when it + * names no path and no qualified name, or when `buf` (cap bytes) cannot hold + * it: a name is never cut. Pure: the same text always gives the same answer. */ +bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_t cap, + CBMDocLinkMdPath *out); + +/* Parse a `#L3`, `#L3-L9` or `#L3C2-L9C5` fragment (without the `#`): true + * and the lines, or false. */ +bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last); + #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_md.c b/internal/cbm/doclink_md.c new file mode 100644 index 0000000000..0cd2d350fd --- /dev/null +++ b/internal/cbm/doclink_md.c @@ -0,0 +1,1788 @@ +/* + * doclink_md.c — Markdown references to code: the extraction half. + * + * In a Markdown document the text IS the documentation, so its references are + * not taken from a definition's doc comment: one pass over the file's bytes + * finds the explicit ones and gives each to the section it is written in -- + * the Section definition (a heading) at or above its line, or the file itself + * for text before the first heading (CBM_DOCLINK_FLAG_FILE). Families + * (doclink.h): + * + * link the destination of an inline link `[text](dest)` or of a link + * reference definition `[label]: dest`, when it is no URL, no + * in-page anchor and no image + * path a bare path in prose: a slash and a file extension + * code_path a code span, or an HTML span, whose text is a path, a + * file name or a directory, with an optional line range + * (`#L3-L9`, `:3-9`) or member (`path::Name`) + * + * Never scanned: YAML front matter, fenced code blocks, HTML comments, the + * import/export lines of MDX. Never a reference here: a URL, an in-page + * anchor, an image, a name without a path (a bare name is at most a + * suggestion, never an edge). + * + * Cost. Each scan of a line is linear in the line, or n log n where it sorts. + * The searches that could otherwise rescan a stretch of the line for every + * candidate (the end of a link destination, the closing parenthesis, the + * closing backticks of a code span) are answered from position lists of the + * few bytes they look for, by binary search. The lists grow with the number + * of those bytes, not with the line, and are released at the end of the + * file. A list that cannot grow marks the file's doc links failed, so the + * resolving half fails the layer instead of publishing a graph that silently + * misses references. + * + * The rules are those the documentation field tests measured (H0): explicit + * paths and links bind EXACTLY, at 99 % precision on 53 repositories. + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" + +#include +#include +#include +#include + +#define MD_SLEN(s) (sizeof(s) - SKIP_ONE) + +enum { + MD_MASK = 0x01, /* a byte the later scans of the line must not see */ + MD_FENCE_MIN = 3, /* ``` or ~~~ */ + MD_INDENT_MAX = 3, /* a link reference definition may be indented this far */ + MD_SPAN_MAX = 150, /* longer code-span text is a snippet, not a name */ + MD_EXT_MAX = 12, /* a file extension: a letter and at most 11 more */ + MD_TAG_MAX = 200, /* an HTML tag inside a span is at most this long */ + MD_CALL_ARGS = 60, /* `name(args)`: the argument text of a call form */ + MD_CALL_ARGS_WIDE = 80, /* the same with spaces or commas in the arguments */ + MD_TINY_DOTFILE = 3, /* `.a` is no file name, `.ab` is */ + MD_LIST_INIT = 16, /* first capacity of a position list */ + MD_NUM_DIGITS = 9, /* a line number has at most this many digits */ + MD_PAIR = 2, /* ints per pair entry */ + MD_URL_SCHEMES = 3, /* http, https, ftp */ +}; + +/* UTF-8 well-formedness (RFC 3629, table 3-7 of the Unicode standard). */ +enum { + UTF8_CTRL_END = 0x20, + UTF8_DEL = 0x7f, + UTF8_ASCII_END = 0x80, + UTF8_CONT_MIN = 0x80, + UTF8_CONT_MAX = 0xBF, + UTF8_LEAD2_MIN = 0xC2, + UTF8_LEAD2_MAX = 0xDF, + UTF8_LEAD3_MIN = 0xE0, + UTF8_LEAD3_MAX = 0xEF, + UTF8_E0_SECOND_MIN = 0xA0, + UTF8_SURROGATE_LEAD = 0xED, + UTF8_ED_SECOND_MAX = 0x9F, + UTF8_LEAD4_MIN = 0xF0, + UTF8_LEAD4_MAX = 0xF4, + UTF8_F0_SECOND_MIN = 0x90, + UTF8_F4_SECOND_MAX = 0x8F, +}; + +/* ── Small character classes (ASCII; bytes >= 0x80 are word bytes) ── */ + +static bool md_blank(unsigned char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v'; +} + +static bool md_sp_tab(unsigned char c) { + return c == ' ' || c == '\t'; +} + +static bool md_alpha(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool md_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static bool md_alnum(unsigned char c) { + return md_alpha(c) || md_digit(c); +} + +/* Python's \w on text: letters, digits, '_' and any non-ASCII byte. */ +static bool md_word(unsigned char c) { + return md_alnum(c) || c == '_' || c >= 0x80; +} + +static unsigned char md_lower(unsigned char c) { + return (c >= 'A' && c <= 'Z') ? (unsigned char)(c - 'A' + 'a') : c; +} + +static bool md_has(const char *s, size_t n, char c) { + return n > 0 && memchr(s, c, n) != NULL; +} + +static bool md_eq_ci(const char *s, size_t n, const char *word) { + size_t w = strlen(word); + if (n != w) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (md_lower((unsigned char)s[i]) != (unsigned char)word[i]) { + return false; + } + } + return true; +} + +static bool md_in_list(const char *s, size_t n, const char *const *list, bool ci) { + for (size_t i = 0; list[i]; i++) { + if (ci ? md_eq_ci(s, n, list[i]) : (strlen(list[i]) == n && memcmp(s, list[i], n) == 0)) { + return true; + } + } + return false; +} + +/* File extensions that make a dotted token a FILE NAME. Strict: none of them + * is a common member name (`.log`, `.get` and `.map` are not here). */ +static const char *const MD_FILE_EXT[] = { + "py", "pyi", "js", "jsx", "mjs", "cjs", "ts", + "tsx", "mts", "cts", "go", "rs", "c", "h", + "cc", "cpp", "cxx", "hpp", "hh", "java", "kt", + "kts", "scala", "groovy", "cs", "php", "rb", "sh", + "bash", "zsh", "ps1", "md", "mdx", "rst", "adoc", + "txt", "json", "jsonc", "yaml", "yml", "toml", "ini", + "cfg", "xml", "html", "htm", "css", "scss", "sass", + "less", "vue", "svelte", "proto", "sql", "lock", "gradle", + "csproj", "sln", "fsproj", "props", "targets", "cmake", "mk", + "tf", "tfvars", "hcl", "png", "jpg", "jpeg", "gif", + "svg", "pdf", "ipynb", "dockerfile", "swift", "dart", "lua", + "pl", "pm", "ex", "exs", "erl", "clj", "hs", + "ml", "zig", "sol", "graphql", "gql", "bzl", "nix", + "plist", "env", "conf", "properties", "editorconfig", "gitignore", "npmrc", + "nvmrc", "j2", "jinja", "tmpl", "tpl", "erb", "hbs", + "mustache", "patch", "diff", "whl", "jar", "zip", "gz", + "tgz", "so", "dll", "dylib", "wasm", "service", "socket", + "timer", "mod", "sum", NULL}; + +/* File names that are files without an extension. */ +static const char *const MD_SPECIAL_FILES[] = { + "Makefile", "Dockerfile", "Containerfile", "Jenkinsfile", "Vagrantfile", "Gemfile", "Rakefile", + "Procfile", "Kconfig", "CODEOWNERS", "MAINTAINERS", "LICENSE", "NOTICE", "go.mod", + "go.sum", "Cargo.lock", "Justfile", "BUILD", "WORKSPACE", NULL}; + +/* Host names written as code: a site, not a file. */ +static const char *const MD_DOMAINS[] = {"docs.rs", "crates.io", "pkg.go.dev", + "npmjs.com", "pypi.org", "github.com", + "gitlab.com", "readthedocs.io", "rubygems.org", + "nuget.org", "go.dev", "golang.org", + "godoc.org", "packagist.org", "hex.pm", + "mvnrepository.com", "docs.python.org", NULL}; + +/* `.ext`, a letter and up to 11 of [A-Za-z0-9_+-], at the end of `s`. */ +static bool md_ext_ok(const char *s, size_t n) { + const char *dot = NULL; + for (size_t i = n; i > 0; i--) { + if (s[i - SKIP_ONE] == '.') { + dot = s + i - SKIP_ONE; + break; + } + } + if (!dot) { + return false; + } + size_t ext_len = n - (size_t)(dot - s) - SKIP_ONE; + if (ext_len == 0 || ext_len > MD_EXT_MAX || !md_alpha((unsigned char)dot[SKIP_ONE])) { + return false; + } + for (size_t i = 2; i <= ext_len; i++) { + unsigned char c = (unsigned char)dot[i]; + if (!md_alnum(c) && c != '_' && c != '+' && c != '-') { + return false; + } + } + return true; +} + +/* `^[A-Za-z][A-Za-z0-9+.-]*://`, `mailto:` or `www.`. */ +static bool md_is_url(const char *s, size_t n) { + if (n >= MD_SLEN("mailto:") && md_eq_ci(s, MD_SLEN("mailto:"), "mailto:")) { + return true; + } + if (n >= MD_SLEN("www.") && md_eq_ci(s, MD_SLEN("www."), "www.")) { + return true; + } + if (n == 0 || !md_alpha((unsigned char)s[0])) { + return false; + } + size_t i = SKIP_ONE; + while (i < n && (md_alnum((unsigned char)s[i]) || s[i] == '+' || s[i] == '.' || s[i] == '-')) { + i++; + } + return i + MD_SLEN("://") <= n && memcmp(s + i, "://", MD_SLEN("://")) == 0; +} + +/* `^[A-Za-z_$][\w$]*$` */ +static bool md_ident(const char *s, size_t n) { + if (n == 0) { + return false; + } + unsigned char c0 = (unsigned char)s[0]; + if (!(md_alpha(c0) || c0 == '_' || c0 == '$' || c0 >= 0x80)) { + return false; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!md_word((unsigned char)s[i]) && s[i] != '$') { + return false; + } + } + return true; +} + +/* Every byte of s[0..n) is a word byte or one of `extra`. */ +static bool md_all_of(const char *s, size_t n, const char *extra) { + if (n == 0) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!md_word((unsigned char)s[i]) && !strchr(extra, s[i])) { + return false; + } + } + return true; +} + +/* ── Line numbers written with a path ────────────────────────────── */ + +/* The decimal number s[0..n): false when it is empty, not all digits or too + * long to be a line. */ +static bool md_number(const char *s, size_t n, uint32_t *out) { + if (n == 0 || n > MD_NUM_DIGITS) { + return false; + } + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + if (!md_digit((unsigned char)s[i])) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (uint32_t)(s[i] - '0'); + } + *out = v; + return v > 0; +} + +/* Digits ending at s[end): their start. */ +static size_t md_digits_back(const char *s, size_t end) { + size_t i = end; + while (i > 0 && md_digit((unsigned char)s[i - SKIP_ONE])) { + i--; + } + return i; +} + +bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last) { + /* L[C][-[L][C]] */ + if (!frag || frag[0] != 'L') { + return false; + } + const char *p = frag + SKIP_ONE; + const char *q = p; + while (md_digit((unsigned char)*q)) { + q++; + } + uint32_t a = 0; + if (!md_number(p, (size_t)(q - p), &a)) { + return false; + } + if (*q == 'C') { + q++; + while (md_digit((unsigned char)*q)) { + q++; + } + } + uint32_t b = a; + if (*q == '-') { + q++; + if (*q == 'L') { + q++; + } + p = q; + while (md_digit((unsigned char)*q)) { + q++; + } + if (!md_number(p, (size_t)(q - p), &b)) { + return false; + } + if (*q == 'C') { + q++; + while (md_digit((unsigned char)*q)) { + q++; + } + } + } + if (*q != '\0') { + return false; + } + *first = a; + *last = b < a ? a : b; + return true; +} + +/* A line range at the end of p[0..*n): `#L[-[L]]`, `:`, `:-` + * or `::`. On a match the range is cut off (*n shrinks). */ +static bool md_line_suffix(const char *p, size_t *n, uint32_t *first, uint32_t *last) { + size_t len = *n; + /* `#L` form: the last '#' */ + for (size_t i = len; i > 0; i--) { + if (p[i - SKIP_ONE] != '#') { + continue; + } + size_t h = i - SKIP_ONE; + if (h + SKIP_ONE < len && p[h + SKIP_ONE] == 'L') { + size_t a0 = h + PAIR_LEN; + size_t a1 = a0; + while (a1 < len && md_digit((unsigned char)p[a1])) { + a1++; + } + uint32_t a = 0; + if (md_number(p + a0, a1 - a0, &a)) { + uint32_t b = a; + size_t k = a1; + bool ok = k == len; + if (!ok && p[k] == '-') { + k++; + if (k < len && p[k] == 'L') { + k++; + } + ok = md_number(p + k, len - k, &b); + } + if (ok) { + *first = a; + *last = b < a ? a : b; + *n = h; + return true; + } + } + } + break; + } + /* `:` form: digits at the end */ + size_t d2 = md_digits_back(p, len); + uint32_t b = 0; + if (d2 == len || d2 == 0 || !md_number(p + d2, len - d2, &b)) { + return false; + } + char sep = p[d2 - SKIP_ONE]; + if (sep == '-' || sep == ':') { + size_t d1 = md_digits_back(p, d2 - SKIP_ONE); + uint32_t a = 0; + if (d1 > 0 && d1 < d2 - SKIP_ONE && p[d1 - SKIP_ONE] == ':' && + md_number(p + d1, d2 - SKIP_ONE - d1, &a)) { + *first = a; + *last = b < a ? a : b; + *n = d1 - SKIP_ONE; + return true; + } + } + if (sep == ':') { + *first = b; + *last = b; + *n = d2 - SKIP_ONE; + return true; + } + return false; +} + +/* ── Code-span classification ────────────────────────────────────── */ + +/* Drop one trailing balanced `<...>` or `[...]` (type arguments). */ +static size_t md_strip_generics(const char *s, size_t n) { + static const char pairs[][PAIR_LEN] = {{'<', '>'}, {'[', ']'}}; + for (size_t p = 0; p < sizeof(pairs) / sizeof(pairs[0]); p++) { + char o = pairs[p][0]; + char c = pairs[p][SKIP_ONE]; + if (n == 0 || s[n - SKIP_ONE] != c || !md_has(s, n, o)) { + continue; + } + int depth = 0; + for (size_t k = n; k > 0; k--) { + char ch = s[k - SKIP_ONE]; + if (ch == c) { + depth++; + } else if (ch == o) { + depth--; + if (depth == 0) { + if (k - SKIP_ONE > 0) { + return k - SKIP_ONE; + } + break; + } + } + } + return n; + } + return n; +} + +/* `name(args)` with a short argument list that holds no parenthesis: the + * callee's length, or 0. `wide` is the first form of the field test (args + * with a space or comma, up to 80 bytes, callee [\w$.:#\\]); otherwise the + * second (args up to 60 bytes, callee [\w$.:#\\<>\[\]!~&*-]). */ +static size_t md_callee(const char *s, size_t n, bool wide) { + if (n < PAIR_LEN || s[n - SKIP_ONE] != ')') { + return 0; + } + size_t open = n; + for (size_t i = n - SKIP_ONE; i > 0; i--) { + char c = s[i - SKIP_ONE]; + if (c == ')') { + return 0; + } + if (c == '(') { + open = i - SKIP_ONE; + break; + } + } + if (open == n || open == 0) { + return 0; + } + size_t args = n - open - PAIR_LEN; + if (args > (wide ? (size_t)MD_CALL_ARGS_WIDE : (size_t)MD_CALL_ARGS)) { + return 0; + } + if (wide && !md_has(s + open + SKIP_ONE, args, ' ') && + !md_has(s + open + SKIP_ONE, args, ',')) { + return 0; + } + return md_all_of(s, open, wide ? "$.:#\\" : "$.:#\\<>[]!~&*-") ? open : 0; +} + +static void md_shape(CBMDocLinkMdPath *out, CBMDocLinkMdShape shape, const char *path, + const char *member, uint32_t first, uint32_t last) { + out->shape = shape; + out->path = path; + out->member = member; + out->first_line = first; + out->last_line = last; + out->instance = false; + out->colon = false; +} + +/* The path branch: `s` has a slash (or a Windows path with an extension). */ +static bool md_classify_path(char *s, size_t n, CBMDocLinkMdPath *out) { + for (size_t i = 0; i < n; i++) { + if (s[i] == '\\') { + s[i] = '/'; + } + } + if (n >= PAIR_LEN && s[0] == '/' && s[SKIP_ONE] == '/') { + return false; + } + uint32_t first = 0; + uint32_t last = 0; + (void)md_line_suffix(s, &n, &first, &last); + s[n] = '\0'; + const char *member = NULL; + char *sep = strstr(s, "::"); + if (sep) { + *sep = '\0'; + member = sep + PAIR_LEN; + n = (size_t)(sep - s); + if (!member[0]) { + member = NULL; + } + } + if (n == 0) { + return false; + } + /* an import path (`github.com/x/y`, `k8s.io/client-go`): a host-like first + * segment -- dot-separated word groups -- and a last segment (as written, + * a trailing slash leaves it empty) without a file extension */ + size_t host = 0; + while (host < n && s[host] != '/') { + host++; + } + bool hostlike = host > 0 && host < n && md_has(s, host, '.') && md_all_of(s, host, ".-") && + s[0] != '.' && s[host - SKIP_ONE] != '.'; + for (size_t i = 0; hostlike && i + SKIP_ONE < host; i++) { + hostlike = !(s[i] == '.' && s[i + SKIP_ONE] == '.'); + } + size_t tail = n; + while (tail > 0 && s[tail - SKIP_ONE] != '/') { + tail--; + } + if (hostlike && !md_ext_ok(s + tail, n - tail)) { + return false; + } + /* the base name, after any trailing slashes */ + size_t end = n; + while (end > 0 && s[end - SKIP_ONE] == '/') { + end--; + } + size_t base = end; + while (base > 0 && s[base - SKIP_ONE] != '/') { + base--; + } + const char *last_seg = s + base; + size_t last_len = end - base; + if (!md_all_of(s, n, "./@+~-")) { + return false; + } + if (s[n - SKIP_ONE] == '/') { + md_shape(out, CBM_DOCLINK_MD_DIR, s, NULL, 0, 0); + return true; + } + const char *dot = last_len > 0 ? memchr(last_seg, '.', last_len) : NULL; + if (dot && last_seg[0] != '.') { + const char *ext = strrchr(last_seg, '.') + SKIP_ONE; + size_t ext_len = (size_t)(last_seg + last_len - ext); + if (ext_len > 0 && ext[0] >= 'A' && ext[0] <= 'Z' && md_ident(ext, ext_len) && + !md_in_list(ext, ext_len, MD_FILE_EXT, true)) { + return false; /* `pkg/Type.Member`: a name, not a file (a later rung) */ + } + if (md_ext_ok(last_seg, last_len)) { + md_shape(out, CBM_DOCLINK_MD_FILE, s, member, first, last); + return true; + } + } + if (last_len > SKIP_ONE && last_seg[0] == '.') { + md_shape(out, CBM_DOCLINK_MD_FILE, s, NULL, first, last); + return true; + } + md_shape(out, CBM_DOCLINK_MD_DIR, s, NULL, 0, 0); + return true; +} + +static const char *const MD_RUST_ROOTS[] = {"crate", "self", "super", "Self", NULL}; +static const char *const MD_INSTANCE_ROOTS[] = {"self", "this", "cls", "$this", NULL}; + +/* Split s[0..n) at every `sep`, check that each piece is an identifier (type + * arguments dropped), and join the pieces with '.' in place. A first piece in + * `skip_first` (Rust's `crate`, an instance's `self`) is dropped and *skipped + * says so. False when a piece is no identifier or fewer than two remain. */ +static bool md_join(char *s, size_t n, const char *sep, const char *const *skip_first, + bool *skipped, CBMDocLinkMdPath *out) { + size_t sl = strlen(sep); + size_t pieces = 0; + size_t w = 0; + size_t i = 0; + *skipped = false; + for (;;) { + size_t e = i; + while (e + sl <= n && memcmp(s + e, sep, sl) != 0) { + e++; + } + bool last_piece = e + sl > n; + if (last_piece) { + e = n; + } + size_t len = md_strip_generics(s + i, e - i); + if (pieces == 0 && !*skipped && skip_first && !last_piece && + md_in_list(s + i, len, skip_first, false)) { + *skipped = true; + } else { + if (!md_ident(s + i, len)) { + return false; + } + if (w > 0) { + s[w++] = '.'; + } + memmove(s + w, s + i, len); + w += len; + pieces++; + } + if (last_piece) { + break; + } + i = e + sl; + } + s[w] = '\0'; + if (pieces < PAIR_LEN) { + return false; + } + md_shape(out, CBM_DOCLINK_MD_QUALIFIED, s, NULL, 0, 0); + return true; +} + +/* `main.rs::run`: a file name with a member (an identifier). */ +static bool md_filename_member(char *s, char *sep, uint32_t first, uint32_t last, + CBMDocLinkMdPath *out) { + const char *head = s; + size_t head_len = (size_t)(sep - s); + const char *tail = sep + PAIR_LEN; + const char *tail_last = tail; + for (const char *t = strstr(tail, "::"); t; t = strstr(t + PAIR_LEN, "::")) { + tail_last = t + PAIR_LEN; + } + const char *hd = head_len ? memchr(head, '.', head_len) : NULL; + if (!hd || !md_ident(tail_last, strlen(tail_last)) || !md_all_of(head, head_len, "./\\-")) { + return false; + } + const char *ext = head; + for (size_t i = head_len; i > 0; i--) { + if (head[i - SKIP_ONE] == '.') { + ext = head + i; + break; + } + } + size_t ext_len = head_len - (size_t)(ext - head); + for (size_t i = 0; i < ext_len; i++) { + if (ext[i] < 'a' || ext[i] > 'z') { + return false; + } + } + if (ext_len == 0) { + return false; + } + *sep = '\0'; + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, tail[0] ? tail : NULL, first, last); + return true; +} + +/* One `mark` between a head `[A-Za-z_][\w.]*` and a tail `[A-Za-z_]\w*` + * (`dotted_tail`: `[A-Za-z_][\w.]*`): the shapes of `Type#member` and of + * `module:attr`. */ +static bool md_head_mark_tail(const char *s, size_t n, char mark, bool dotted_tail) { + const char *m = memchr(s, mark, n); + if (!m || m == s || memchr(m + SKIP_ONE, mark, n - (size_t)(m - s) - SKIP_ONE)) { + return false; + } + unsigned char c0 = (unsigned char)s[0]; + size_t head = (size_t)(m - s); + size_t tail = n - head - SKIP_ONE; + if (tail == 0) { + return false; + } + unsigned char t0 = (unsigned char)m[SKIP_ONE]; + return (md_alpha(c0) || c0 == '_') && md_all_of(s, head, ".") && (md_alpha(t0) || t0 == '_') && + md_all_of(m + SKIP_ONE, tail, dotted_tail ? "." : ""); +} + +/* No slash: a file name, a special file, `file.ext::member`, or a qualified + * name. A bare name is neither: at most a suggestion, never a link. */ +static bool md_classify_name(char *s, size_t n, CBMDocLinkMdPath *out) { + if (md_in_list(s, n, MD_SPECIAL_FILES, false)) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, 0, 0); + return true; + } + uint32_t first = 0; + uint32_t last = 0; + size_t cut = n; + if (md_line_suffix(s, &cut, &first, &last)) { + const char *dot = NULL; + for (size_t i = cut; i > 0; i--) { + if (s[i - SKIP_ONE] == '.') { + dot = s + i - SKIP_ONE; + break; + } + } + size_t ext_len = dot ? cut - (size_t)(dot - s) - SKIP_ONE : 0; + if (dot && md_in_list(dot + SKIP_ONE, ext_len, MD_FILE_EXT, true)) { + n = cut; + s[n] = '\0'; + } else { + first = 0; + last = 0; + } + } + bool skipped = false; + if (strstr(s, "::")) { + /* identifiers only: a qualified name; otherwise perhaps `file.ext::member` */ + char copy[MD_SPAN_MAX + SKIP_ONE]; + memcpy(copy, s, n + SKIP_ONE); + if (md_join(s, n, "::", MD_RUST_ROOTS, &skipped, out)) { + return true; + } + memcpy(s, copy, n + SKIP_ONE); + return md_filename_member(s, strstr(s, "::"), first, last, out); + } + if (md_has(s, n, '#')) { + if (!md_head_mark_tail(s, n, '#', false)) { + return false; + } + s[strcspn(s, "#")] = '.'; + return md_join(s, n, ".", NULL, &skipped, out); + } + if (md_has(s, n, '\\')) { + while (n > 0 && s[n - SKIP_ONE] == '\\') { + n--; + } + size_t lead = 0; + while (lead < n && s[lead] == '\\') { + lead++; + } + return md_join(s + lead, n - lead, "\\", NULL, &skipped, out); + } + if (md_has(s, n, ':')) { + if (!md_head_mark_tail(s, n, ':', true)) { + return false; + } + s[strcspn(s, ":")] = '.'; + if (!md_join(s, n, ".", NULL, &skipped, out)) { + return false; + } + out->colon = true; + return true; + } + if (!md_has(s, n, '.')) { + return false; + } + if (s[0] == '.') { + const char *rest = s + SKIP_ONE; + size_t rest_len = n - SKIP_ONE; + if (md_ident(rest, rest_len) && !md_in_list(rest, rest_len, MD_FILE_EXT, true)) { + return false; /* `.member` */ + } + if (md_all_of(rest, rest_len, ".-")) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, 0, 0); + return true; + } + return false; + } + const char *dot = strrchr(s, '.'); + const char *ext = dot + SKIP_ONE; + size_t ext_len = n - (size_t)(ext - s); + if (md_in_list(ext, ext_len, MD_FILE_EXT, true) && md_all_of(s, n, ".+-")) { + size_t stem_len = (size_t)(dot - s); + bool one_dot = memchr(s, '.', stem_len) == NULL; + if (one_dot && s[0] >= 'A' && s[0] <= 'Z' && md_eq_ci(ext, ext_len, "js")) { + return false; /* `Node.js`: a product */ + } + bool mod_sum = md_eq_ci(ext, ext_len, "mod") || md_eq_ci(ext, ext_len, "sum"); + if (!mod_sum || (stem_len == PAIR_LEN && memcmp(s, "go", PAIR_LEN) == 0)) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, first, last); + return true; + } + } + if (!md_join(s, n, ".", MD_INSTANCE_ROOTS, &skipped, out)) { + return false; + } + out->instance = skipped; /* `self.x`: the qualifier is an instance, not a type */ + return true; +} + +bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_t cap, + CBMDocLinkMdPath *out) { + md_shape(out, CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0); + if (!text || !buf) { + return false; + } + while (len > 0 && md_blank((unsigned char)text[0])) { + text++; + len--; + } + while (len > 0 && md_blank((unsigned char)text[len - SKIP_ONE])) { + len--; + } + if (len == 0 || len > MD_SPAN_MAX || len + SKIP_ONE > cap || md_has(text, len, '\n')) { + return false; + } + memcpy(buf, text, len); + buf[len] = '\0'; + char *s = buf; + size_t n = len; + if (md_in_list(s, n, MD_DOMAINS, true) || md_is_url(s, n)) { + return false; + } + bool only_slashes = true; + for (size_t i = 0; i < n; i++) { + only_slashes = only_slashes && s[i] == '/'; + } + if (only_slashes) { + return false; + } + if (n > SKIP_ONE && strchr("$%>#", s[0]) && s[SKIP_ONE] == ' ') { + return false; /* a shell prompt */ + } + if (n > SKIP_ONE && s[0] == '-' && + (md_alpha((unsigned char)s[SKIP_ONE]) || s[SKIP_ONE] == '-')) { + return false; /* an option */ + } + if (s[0] == '$' || + (n >= PAIR_LEN && (s[0] == '"' || s[0] == '\'') && s[n - SKIP_ONE] == s[0]) || + (s[0] == '<' && s[n - SKIP_ONE] == '>')) { + return false; /* a variable, a string, markup */ + } + size_t eq = 0; + while (eq < n && (md_alnum((unsigned char)s[eq]) || s[eq] == '_')) { + eq++; + } + if (eq > 0 && eq < n && s[eq] == '=' && !md_digit((unsigned char)s[0]) && !md_has(s, n, ' ')) { + return false; /* NAME=value */ + } + size_t callee = md_callee(s, n, true); + if (callee) { + n = callee; + s[n] = '\0'; + } + if (md_has(s, n, ' ') || md_has(s, n, '\t')) { + return false; /* a declaration, a command line, a snippet */ + } + if (s[0] == '@') { + size_t slash = 0; + while (slash < n && s[slash] != '/') { + slash++; + } + if (slash < n && slash > SKIP_ONE) { + return false; /* `@scope/package`: an import path (a later rung) */ + } + s++; + n--; + if (n == 0) { + return false; + } + } + callee = md_callee(s, n, false); + if (callee) { + n = callee; + s[n] = '\0'; + } + while (n > 0 && strchr(";,:", s[n - SKIP_ONE])) { + n--; + } + if (n > SKIP_ONE && s[n - SKIP_ONE] == '!') { + n--; + } + while (n > 0 && s[n - SKIP_ONE] == '?') { + n--; + } + n = md_strip_generics(s, n); + while (n > 0 && strchr("&*~", s[0])) { + s++; + n--; + } + if (n >= PAIR_LEN && s[0] == ':' && s[SKIP_ONE] == ':') { + s += PAIR_LEN; + n -= PAIR_LEN; + } + if (n == 0) { + return false; + } + s[n] = '\0'; + size_t first_slash = 0; + while (first_slash < n && s[first_slash] != '/') { + first_slash++; + } + bool slash_path = first_slash < n && !strstr(s, "::"); + if (first_slash < n && !slash_path) { + /* `a::b/c`: a "::" after the first slash still makes a path */ + char *dc = strstr(s, "::"); + slash_path = dc && (size_t)(dc - s) > first_slash; + } + bool windows_path = false; + if (!slash_path && md_has(s, n, '\\')) { + const char *bs = strrchr(s, '\\'); + size_t tail = n - (size_t)(bs - s) - SKIP_ONE; + windows_path = tail > 0 && md_all_of(bs + SKIP_ONE, tail, ".-") && md_ext_ok(bs, tail + 1); + } + if (slash_path || windows_path) { + return md_classify_path(s, n, out); + } + return md_classify_name(s, n, out); +} + +/* ── Position lists ──────────────────────────────────────────────── */ + +typedef struct { + int32_t *v; + int n; + int cap; +} md_list_t; + +static bool md_push_int(md_list_t *l, int32_t x) { + if (l->n == l->cap) { + if (l->cap > INT32_MAX / PAIR_LEN) { + return false; + } + int ncap = l->cap ? l->cap * PAIR_LEN : MD_LIST_INIT; + int32_t *grown = + (int32_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*grown)); + if (!grown) { + return false; + } + l->v = grown; + l->cap = ncap; + } + l->v[l->n++] = x; + return true; +} + +static bool md_push_pair(md_list_t *l, int32_t a, int32_t b) { + return md_push_int(l, a) && md_push_int(l, b); +} + +static void md_list_free(md_list_t *l) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + l->v = NULL; + l->n = 0; + l->cap = 0; +} + +/* Index of the first value >= x in an ascending list. */ +static int md_lower_bound(const md_list_t *l, int32_t x) { + int lo = 0; + int hi = l->n; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (l->v[mid] < x) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* The first value >= x, or -1. */ +static int32_t md_next_at(const md_list_t *l, int32_t x) { + int i = md_lower_bound(l, x); + return i < l->n ? l->v[i] : CBM_NOT_FOUND; +} + +/* ── The scan ────────────────────────────────────────────────────── */ + +typedef struct { + uint32_t line; + const char *qn; + /* Another heading of the file has the same qualified name: the graph keeps + * one node for both, which owns only one of the headings, so this + * section's references belong to the file. */ + bool shared_qn; +} md_section_t; + +typedef struct { + bool in_front; + bool in_fence; + bool in_comment; + char fence_char; + int fence_len; +} md_state_t; + +typedef struct { + CBMExtractCtx *ctx; + md_section_t *secs; + int nsecs; + char *buf; /* the current line, masked in place */ + int len; + uint32_t line; + char *span_buf; /* classification scratch, MD_SPAN_MAX + 1 bytes */ + md_list_t runs; /* backtick runs: start, length */ + md_list_t by_len; /* the runs as (length, start), sorted */ + md_list_t ranges; /* ranges to mask: start, end */ + md_list_t stack; /* open '[' while matching */ + md_list_t cands; /* link candidates: '[', ']' */ + md_list_t opens; /* '(' */ + md_list_t closes; /* ')' */ + md_list_t bal; /* each ')' as (depth before it, position), sorted */ + md_list_t gts; /* '>' */ + md_list_t blanks; /* runs of spaces and tabs: start, end */ + bool failed; +} md_scan_t; + +static int md_section_cmp(const void *a, const void *b) { + const md_section_t *x = (const md_section_t *)a; + const md_section_t *y = (const md_section_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + return strcmp(x->qn, y->qn); +} + +static int md_section_ptr_qn_cmp(const void *a, const void *b) { + const md_section_t *x = *(const md_section_t *const *)a; + const md_section_t *y = *(const md_section_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static int md_pair_cmp(const void *a, const void *b) { + const int32_t *x = (const int32_t *)a; + const int32_t *y = (const int32_t *)b; + if (x[0] != y[0]) { + return x[0] < y[0] ? -1 : 1; + } + return (x[SKIP_ONE] > y[SKIP_ONE]) - (x[SKIP_ONE] < y[SKIP_ONE]); +} + +static void md_fail(md_scan_t *s) { + s->failed = true; + s->ctx->result->doc_links.failed = true; +} + +/* Text that may be a reference: well-formed UTF-8 (no overlong form, no + * surrogate, nothing above U+10FFFF) without control bytes. */ +static bool md_clean_text(const char *t, int n) { + int i = 0; + while (i < n) { + unsigned char c = (unsigned char)t[i]; + if (c < UTF8_CTRL_END || c == UTF8_DEL) { + return false; + } + int need; + unsigned char lo = UTF8_CONT_MIN; + unsigned char hi = UTF8_CONT_MAX; + if (c < UTF8_ASCII_END) { + need = 0; + } else if (c >= UTF8_LEAD2_MIN && c <= UTF8_LEAD2_MAX) { + need = SKIP_ONE; + } else if (c >= UTF8_LEAD3_MIN && c <= UTF8_LEAD3_MAX) { + need = PAIR_LEN; + lo = c == UTF8_LEAD3_MIN ? UTF8_E0_SECOND_MIN : lo; /* no overlong form */ + hi = c == UTF8_SURROGATE_LEAD ? UTF8_ED_SECOND_MAX : hi; /* no surrogate */ + } else if (c >= UTF8_LEAD4_MIN && c <= UTF8_LEAD4_MAX) { + need = PAIR_LEN + SKIP_ONE; + lo = c == UTF8_LEAD4_MIN ? UTF8_F0_SECOND_MIN : lo; + hi = c == UTF8_LEAD4_MAX ? UTF8_F4_SECOND_MAX : hi; + } else { + return false; + } + if (i + need >= n + (need == 0)) { + return false; + } + for (int k = SKIP_ONE; k <= need; k++) { + unsigned char cc = (unsigned char)t[i + k]; + unsigned char kl = k == SKIP_ONE ? lo : UTF8_CONT_MIN; + unsigned char kh = k == SKIP_ONE ? hi : UTF8_CONT_MAX; + if (cc < kl || cc > kh) { + return false; + } + } + i += need + SKIP_ONE; + } + return true; +} + +static void md_emit(md_scan_t *s, int syntax, const char *text, int n) { + while (n > 0 && md_blank((unsigned char)text[0])) { + text++; + n--; + } + while (n > 0 && md_blank((unsigned char)text[n - SKIP_ONE])) { + n--; + } + if (n <= 0 || memchr(text, MD_MASK, (size_t)n) || !md_clean_text(text, n)) { + return; + } + int lo = 0; + int hi = s->nsecs; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (s->secs[mid].line <= s->line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + const md_section_t *sec = lo > 0 ? &s->secs[lo - SKIP_ONE] : NULL; + if (sec && sec->shared_qn) { + sec = NULL; /* the file, not the node another heading owns */ + } + char *raw = cbm_arena_strndup(s->ctx->arena, text, (size_t)n); + if (!raw) { + md_fail(s); + return; + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (s->ctx->module_qn ? s->ctx->module_qn : ""), + .raw = raw, + .line = s->line, + .def_line = sec ? sec->line : SKIP_ONE, + .syntax = (uint16_t)syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&s->ctx->result->doc_links, s->ctx->arena, link); +} + +/* A code span's text: a reference when it names a path or a qualified name. */ +static void md_emit_span(md_scan_t *s, const char *text, int n) { + CBMDocLinkMdPath p; + if (n > 0 && + cbm_doclink_md_classify_span(text, (size_t)n, s->span_buf, MD_SPAN_MAX + SKIP_ONE, &p)) { + md_emit(s, + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_MD_CODE_NAME + : CBM_DOCLINK_MD_CODE_PATH, + text, n); + } +} + +static void md_mask(md_scan_t *s, int a, int b) { + if (a < 0) { + a = 0; + } + if (b > s->len) { + b = s->len; + } + if (a < b) { + memset(s->buf + a, MD_MASK, (size_t)(b - a)); + } +} + +static int md_find(const char *buf, int len, int from, const char *needle, bool ci) { + int k = (int)strlen(needle); + for (int i = from; i + k <= len; i++) { + int j = 0; + while (j < k && (ci ? md_lower((unsigned char)buf[i + j]) : (unsigned char)buf[i + j]) == + (unsigned char)needle[j]) { + j++; + } + if (j == k) { + return i; + } + } + return CBM_NOT_FOUND; +} + +/* HTML spans (tables in READMEs write code that way): their text, + * without inner tags and with the common entities decoded. */ +static void md_html_code(md_scan_t *s) { + int i = 0; + while (i < s->len) { + int open = md_find(s->buf, s->len, i, "len && s->buf[after] == '>') { + body = after + SKIP_ONE; + } else if (after < s->len && md_blank((unsigned char)s->buf[after])) { + const char *gt = memchr(s->buf + after, '>', (size_t)(s->len - after)); + if (!gt) { + return; + } + body = (int)(gt - s->buf) + SKIP_ONE; + } else { + i = open + SKIP_ONE; + continue; + } + int close = md_find(s->buf, s->len, body, "", true); + if (close < 0) { + return; + } + /* text without tags, entities decoded; never longer than the body */ + char *text = s->span_buf + MD_SPAN_MAX + SKIP_ONE; /* second half of the scratch */ + int w = 0; + int k = body; + while (k < close && w < MD_SPAN_MAX + SKIP_ONE) { + char c = s->buf[k]; + if (c == '<') { + int lim = k + MD_TAG_MAX + PAIR_LEN; /* `<`, up to 200 bytes, `>` */ + const char *gt = memchr(s->buf + k + SKIP_ONE, '>', + (size_t)((lim < close ? lim : close) - k - SKIP_ONE)); + if (gt) { + k = (int)(gt - s->buf) + SKIP_ONE; + continue; + } + } else if (c == '&') { + static const struct { + const char *ent; + char ch; + } ents[] = {{"<", '<'}, {">", '>'}, {"&", '&'}, {""", '"'}, + {"'", '\''}, {"'", '\''}, {"'", '\''}, {" ", ' '}}; + bool hit = false; + for (size_t e = 0; e < sizeof(ents) / sizeof(ents[0]); e++) { + int el = (int)strlen(ents[e].ent); + if (k + el <= close && memcmp(s->buf + k, ents[e].ent, (size_t)el) == 0) { + text[w++] = ents[e].ch; + k += el; + hit = true; + break; + } + } + if (hit) { + continue; + } + } + text[w++] = c; + k++; + } + if (k >= close) { + md_emit_span(s, text, w); /* a text longer than a span's limit is no name */ + } + md_mask(s, open, close + (int)MD_SLEN("")); + i = close + (int)MD_SLEN(""); + } +} + +/* Code spans: a run of N backticks up to the next run of exactly N. An + * escaped first backtick (an odd run of backslashes before it) does not open + * a span. Closers come from the runs sorted by (length, start). */ +static void md_code_spans(md_scan_t *s) { + s->runs.n = 0; + s->by_len.n = 0; + s->ranges.n = 0; + for (int i = 0; i < s->len;) { + if (s->buf[i] != '`') { + i++; + continue; + } + int j = i; + while (j < s->len && s->buf[j] == '`') { + j++; + } + if (!md_push_pair(&s->runs, i, j - i) || !md_push_pair(&s->by_len, j - i, i)) { + md_fail(s); + return; + } + i = j; + } + int nruns = s->runs.n / PAIR_LEN; + if (nruns < PAIR_LEN) { + return; + } + qsort(s->by_len.v, (size_t)nruns, sizeof(int32_t) * PAIR_LEN, md_pair_cmp); + int pos = 0; + for (int r = 0; r < nruns; r++) { + int start = s->runs.v[r * PAIR_LEN]; + int len = s->runs.v[(r * PAIR_LEN) + SKIP_ONE]; + if (start < pos) { + continue; + } + int bs = 0; + while (start - bs > 0 && s->buf[start - bs - SKIP_ONE] == '\\') { + bs++; + } + if (bs % PAIR_LEN) { + start++; + len--; + if (len == 0) { + continue; + } + } + /* the first run of exactly `len` starting at or after start + len */ + int lo = 0; + int hi = nruns; + int32_t key[PAIR_LEN] = {len, start + len}; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (md_pair_cmp(&s->by_len.v[mid * PAIR_LEN], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < nruns && s->by_len.v[lo * PAIR_LEN] == len) { + int close = s->by_len.v[(lo * PAIR_LEN) + SKIP_ONE]; + int a = start + len; + int b = close; + if (b - a >= PAIR_LEN && s->buf[a] == ' ' && s->buf[b - SKIP_ONE] == ' ') { + bool all_space = true; + for (int k = a; k < b && all_space; k++) { + all_space = s->buf[k] == ' '; + } + if (!all_space) { + a++; + b--; + } + } + md_emit_span(s, s->buf + a, b - a); + if (!md_push_pair(&s->ranges, start, close + len)) { + md_fail(s); + return; + } + pos = close + len; + } else { + pos = start + len; + } + } + for (int i = 0; i < s->ranges.n; i += PAIR_LEN) { + md_mask(s, s->ranges.v[i], s->ranges.v[i + SKIP_ONE]); + } +} + +/* `blanks` holds the space/tab runs as start, end, start, end, ... -- one + * strictly ascending list. The first value above i: an END when i lies in a + * run (its index is odd), a START (or nothing) when it does not. */ +static int md_blank_after(const md_scan_t *s, int i) { + return md_lower_bound(&s->blanks, i + SKIP_ONE); +} + +/* The first space or tab at or after i (the line's end when there is none). */ +static int md_next_blank(const md_scan_t *s, int i) { + int k = md_blank_after(s, i); + if (k % PAIR_LEN == SKIP_ONE) { + return i; + } + return k < s->blanks.n ? s->blanks.v[k] : s->len; +} + +/* The first byte at or after i that is no space or tab. */ +static int md_skip_blank(const md_scan_t *s, int i) { + int k = md_blank_after(s, i); + return k % PAIR_LEN == SKIP_ONE ? s->blanks.v[k] : i; +} + +/* Where a link destination that starts at t ends: at the first space or tab, + * or at the first ')' at the destination's own parenthesis depth 0. */ +static int md_dest_end(const md_scan_t *s, int t) { + int depth = md_lower_bound(&s->opens, t) - md_lower_bound(&s->closes, t); + int end = md_next_blank(s, t); + /* the first ')' at or after t with the depth t had */ + int nb = s->bal.n / PAIR_LEN; + int lo = 0; + int hi = nb; + int32_t key[PAIR_LEN] = {depth, t}; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (md_pair_cmp(&s->bal.v[mid * PAIR_LEN], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < nb && s->bal.v[lo * PAIR_LEN] == depth) { + int p = s->bal.v[(lo * PAIR_LEN) + SKIP_ONE]; + if (p < end) { + end = p; + } + } + return end; +} + +static bool md_line_lists(md_scan_t *s) { + s->stack.n = 0; + s->cands.n = 0; + s->opens.n = 0; + s->closes.n = 0; + s->bal.n = 0; + s->gts.n = 0; + s->blanks.n = 0; + int depth = 0; + for (int i = 0; i < s->len; i++) { + char c = s->buf[i]; + bool ok = true; + switch (c) { + case '[': + ok = md_push_int(&s->stack, i); + break; + case ']': + if (s->stack.n > 0) { + int o = s->stack.v[--s->stack.n]; + if (i + SKIP_ONE < s->len && s->buf[i + SKIP_ONE] == '(') { + ok = md_push_pair(&s->cands, o, i); + } + } + break; + case '(': + ok = md_push_int(&s->opens, i); + depth++; + break; + case ')': + ok = md_push_int(&s->closes, i) && md_push_pair(&s->bal, depth, i); + depth--; + break; + case '>': + ok = md_push_int(&s->gts, i); + break; + case ' ': + case '\t': { + int j = i; + while (j < s->len && md_sp_tab((unsigned char)s->buf[j])) { + j++; + } + ok = md_push_pair(&s->blanks, i, j); + i = j - SKIP_ONE; + break; + } + default: + break; + } + if (!ok) { + return false; + } + } + if (s->bal.n > PAIR_LEN) { + qsort(s->bal.v, (size_t)(s->bal.n / PAIR_LEN), sizeof(int32_t) * PAIR_LEN, md_pair_cmp); + } + return true; +} + +static void md_link_target(md_scan_t *s, int a, int b, bool image) { + while (a < b && md_blank((unsigned char)s->buf[a])) { + a++; + } + while (b > a && md_blank((unsigned char)s->buf[b - SKIP_ONE])) { + b--; + } + if (a >= b || image || s->buf[a] == '#' || memchr(s->buf + a, MD_MASK, (size_t)(b - a)) || + md_is_url(s->buf + a, (size_t)(b - a))) { + return; /* empty, an image, an in-page anchor, masked text, a URL */ + } + md_emit(s, CBM_DOCLINK_MD_LINK, s->buf + a, b - a); +} + +/* Inline links and images `[text](dest "title")`. */ +static void md_links(md_scan_t *s) { + if (!md_line_lists(s)) { + md_fail(s); + return; + } + s->ranges.n = 0; + int pos = 0; + int floor = 0; + for (int c = 0; c < s->cands.n; c += PAIR_LEN) { + int o = s->cands.v[c]; + int k = s->cands.v[c + SKIP_ONE]; + if (k < pos || o < floor) { + continue; + } + int t = k + PAIR_LEN; + while (t < s->len && s->buf[t] == ' ') { + t++; + } + int ta; + int tb; + int after; + if (t < s->len && s->buf[t] == '<') { + int e = md_next_at(&s->gts, t); + if (e < 0) { + pos = k + PAIR_LEN; + continue; + } + ta = t + SKIP_ONE; + tb = e; + after = e + SKIP_ONE; + } else { + int e = md_dest_end(s, t); + ta = t; + tb = e; + after = e; + } + int close = md_next_at(&s->closes, after); + if (close < 0) { + pos = k + PAIR_LEN; + continue; + } + int q = md_skip_blank(s, after); + if (q < close && !strchr("\"'(", s->buf[q])) { + pos = k + PAIR_LEN; + continue; + } + bool image = o > 0 && s->buf[o - SKIP_ONE] == '!'; + md_link_target(s, ta, tb, image); + if (!md_push_pair(&s->ranges, o - (image ? SKIP_ONE : 0), close + SKIP_ONE)) { + md_fail(s); + return; + } + pos = close + SKIP_ONE; + floor = pos; + } + for (int i = 0; i < s->ranges.n; i += PAIR_LEN) { + md_mask(s, s->ranges.v[i], s->ranges.v[i + SKIP_ONE]); + } +} + +/* A link reference definition `[label]: dest` (the whole line). */ +static void md_refdef(md_scan_t *s) { + int i = 0; + while (i < s->len && i < MD_INDENT_MAX && md_blank((unsigned char)s->buf[i])) { + i++; + } + if (i >= s->len || s->buf[i] != '[') { + return; + } + const char *rb = memchr(s->buf + i + SKIP_ONE, ']', (size_t)(s->len - i - SKIP_ONE)); + if (!rb) { + return; + } + int j = (int)(rb - s->buf); + if (j == i + SKIP_ONE || j + SKIP_ONE >= s->len || s->buf[j + SKIP_ONE] != ':') { + return; + } + int k = j + PAIR_LEN; + while (k < s->len && md_blank((unsigned char)s->buf[k])) { + k++; + } + if (k < s->len && s->buf[k] == '<') { + k++; + } + int e = k; + while (e < s->len && !md_blank((unsigned char)s->buf[e]) && s->buf[e] != '>') { + e++; + } + if (e > k) { + md_link_target(s, k, e, false); + } + md_mask(s, 0, s->len); +} + +static bool md_url_stop(unsigned char c) { + return md_blank(c) || c == '<' || c == '>' || c == '"' || c == '\'' || c == '`' || + c == MD_MASK || c == ')' || c == ']' || c == '|'; +} + +static bool md_path_stop(unsigned char c) { + return md_blank(c) || strchr("()[]{}<>\"',;:`*|", (int)c) != NULL || c == MD_MASK; +} + +/* Bare URLs are masked (they are no repository paths); then every token with + * a slash and a file extension is a bare path. */ +static void md_urls_and_paths(md_scan_t *s) { + static const char *const schemes[MD_URL_SCHEMES] = {"http://", "https://", "ftp://"}; + for (int i = 0; i < s->len; i++) { + char c = s->buf[i]; + if (c != 'h' && c != 'f') { + continue; + } + for (int k = 0; k < MD_URL_SCHEMES; k++) { + int sl = (int)strlen(schemes[k]); + if (i + sl < s->len && memcmp(s->buf + i, schemes[k], (size_t)sl) == 0 && + !md_url_stop((unsigned char)s->buf[i + sl])) { + int j = i + sl; + while (j < s->len && !md_url_stop((unsigned char)s->buf[j])) { + j++; + } + int e = j; + while (e > i && strchr(".,;:!?", s->buf[e - SKIP_ONE])) { + e--; + } + md_mask(s, i, e); + i = j - SKIP_ONE; + break; + } + } + } + for (int i = 0; i < s->len;) { + if (md_path_stop((unsigned char)s->buf[i])) { + i++; + continue; + } + int j = i; + while (j < s->len && !md_path_stop((unsigned char)s->buf[j])) { + j++; + } + int e = j; + while (e > i && s->buf[e - SKIP_ONE] == '.') { + e--; + } + if (e == i) { + i = j; /* only dots */ + continue; + } + const char *tok = s->buf + i; + size_t n = (size_t)(e - i); + bool path = md_has(tok, n, '/') && !(n >= PAIR_LEN && tok[0] == '/' && tok[1] == '/'); + for (size_t k = 0; path && k < n; k++) { + unsigned char c = (unsigned char)tok[k]; + path = md_alnum(c) || strchr("_-+@~./", (int)c) != NULL; + } + if (path) { + size_t end = n; + while (end > 0 && tok[end - SKIP_ONE] == '/') { + end--; + } + size_t base = end; + while (base > 0 && tok[base - SKIP_ONE] != '/') { + base--; + } + const char *last = tok + base; + size_t last_len = end - base; + size_t dots = 0; + for (size_t k = 0; k < last_len; k++) { + dots += last[k] == '.'; + } + bool tiny_dotfile = last_len > 0 && last[0] == '.' && dots == SKIP_ONE && + last_len < (size_t)MD_TINY_DOTFILE; + if (md_ext_ok(last, last_len) && !tiny_dotfile) { + md_emit(s, CBM_DOCLINK_MD_PATH, tok, (int)n); + } + } + i = j; + } +} + +static void md_inline(md_scan_t *s) { + md_html_code(s); + if (!s->failed) { + md_code_spans(s); + } + if (!s->failed) { + md_links(s); + } + if (!s->failed) { + md_refdef(s); + md_urls_and_paths(s); + } +} + +/* Leading blanks skipped: the line's content. */ +static const char *md_lstrip(const char *p, int len, int *rest) { + int i = 0; + while (i < len && md_blank((unsigned char)p[i])) { + i++; + } + *rest = len - i; + return p + i; +} + +static bool md_trim_eq(const char *p, int len, const char *word) { + int n = 0; + const char *t = md_lstrip(p, len, &n); + while (n > 0 && md_blank((unsigned char)t[n - SKIP_ONE])) { + n--; + } + return (size_t)n == strlen(word) && memcmp(t, word, (size_t)n) == 0; +} + +/* One line of the document. `raw` is not NUL-terminated. */ +static void md_line(md_scan_t *s, md_state_t *st, const char *raw, int len, bool mdx) { + if (s->line == SKIP_ONE && md_trim_eq(raw, len, "---")) { + st->in_front = true; + return; + } + if (st->in_front) { + if (md_trim_eq(raw, len, "---") || md_trim_eq(raw, len, "...")) { + st->in_front = false; + } + return; + } + int rest = 0; + const char *t = md_lstrip(raw, len, &rest); + if (st->in_fence) { + int k = 0; + while (k < rest && t[k] == st->fence_char) { + k++; + } + int tail = k; + while (tail < rest && md_blank((unsigned char)t[tail])) { + tail++; + } + if (k >= st->fence_len && tail == rest) { + st->in_fence = false; + } + return; + } + if (rest >= MD_FENCE_MIN && (t[0] == '`' || t[0] == '~')) { + int k = 0; + while (k < rest && t[k] == t[0]) { + k++; + } + if (k >= MD_FENCE_MIN && !md_has(t + k, (size_t)(rest - k), '`')) { + st->in_fence = true; + st->fence_char = t[0]; + st->fence_len = k; + return; + } + } + memcpy(s->buf, raw, (size_t)len); + s->buf[len] = '\0'; + s->len = len; + int from = 0; + if (st->in_comment) { + int end = md_find(s->buf, s->len, 0, "-->", false); + if (end < 0) { + return; + } + st->in_comment = false; + memset(s->buf, ' ', (size_t)end + MD_SLEN("-->")); + from = end + (int)MD_SLEN("-->"); + } + for (;;) { + int a = md_find(s->buf, s->len, from, "", false); + if (b < 0) { + s->len = a; + s->buf[a] = '\0'; + st->in_comment = true; + break; + } + memset(s->buf + a, ' ', (size_t)(b + (int)MD_SLEN("-->") - a)); + from = b + (int)MD_SLEN("-->"); + } + if (mdx && + (strncmp(s->buf, "import", MD_SLEN("import")) == 0 || + strncmp(s->buf, "export", MD_SLEN("export")) == 0) && + s->len > (int)MD_SLEN("import") && md_blank((unsigned char)s->buf[MD_SLEN("import")])) { + return; + } + md_inline(s); +} + +/* The file's sections: its Section definitions by start line. False only when + * memory ran out (a file without headings has none, and that is no failure). */ +static bool md_sections(CBMExtractCtx *ctx, CBMArena *a, md_section_t **out, int *count) { + const CBMDefArray *defs = &ctx->result->defs; + int n = 0; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + n += d->label && d->qualified_name && strcmp(d->label, "Section") == 0; + } + *out = NULL; + *count = 0; + if (n == 0) { + return true; + } + md_section_t *secs = (md_section_t *)cbm_arena_alloc(a, (size_t)n * sizeof(*secs)); + if (!secs) { + return false; + } + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->label && d->qualified_name && strcmp(d->label, "Section") == 0) { + secs[(*count)++] = + (md_section_t){.line = d->start_line, .qn = d->qualified_name, .shared_qn = false}; + } + } + /* repeated qualified names, found through a copy sorted by name */ + const md_section_t **by_qn = + (const md_section_t **)cbm_arena_alloc(a, (size_t)n * sizeof(*by_qn)); + if (!by_qn) { + return false; + } + for (int i = 0; i < *count; i++) { + by_qn[i] = &secs[i]; + } + qsort(by_qn, (size_t)*count, sizeof(*by_qn), md_section_ptr_qn_cmp); + for (int i = SKIP_ONE; i < *count; i++) { + if (strcmp(by_qn[i]->qn, by_qn[i - SKIP_ONE]->qn) == 0) { + ((md_section_t *)by_qn[i])->shared_qn = true; + ((md_section_t *)by_qn[i - SKIP_ONE])->shared_qn = true; + } + } + qsort(secs, (size_t)*count, sizeof(*secs), md_section_cmp); + *out = secs; + return true; +} + +void cbm_doclink_md_scan_file(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena; + md_scan_t s = {.ctx = ctx}; + bool have_secs = md_sections(ctx, scratch, &s.secs, &s.nsecs); + s.buf = (char *)cbm_arena_alloc(scratch, (size_t)ctx->source_len + SKIP_ONE); + s.span_buf = (char *)cbm_arena_alloc(scratch, (size_t)(MD_SPAN_MAX + SKIP_ONE) * PAIR_LEN); + if (!s.buf || !s.span_buf || !have_secs) { + ctx->result->doc_links.failed = true; + return; + } + const char *rel = ctx->rel_path ? ctx->rel_path : ""; + size_t rel_len = strlen(rel); + bool mdx = rel_len >= MD_SLEN(".mdx") && + md_eq_ci(rel + rel_len - MD_SLEN(".mdx"), MD_SLEN(".mdx"), ".mdx"); + md_state_t st = {0}; + const char *src = ctx->source; + int n = ctx->source_len; + int pos = 0; + while (pos < n && !s.failed) { + const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); + int end = nl ? (int)(nl - src) : n; + int len = end - pos; + if (len > 0 && src[pos + len - SKIP_ONE] == '\r') { + len--; + } + s.line++; + md_line(&s, &st, src + pos, len, mdx); + pos = end + SKIP_ONE; + } + md_list_free(&s.runs); + md_list_free(&s.by_len); + md_list_free(&s.ranges); + md_list_free(&s.stack); + md_list_free(&s.cands); + md_list_free(&s.opens); + md_list_free(&s.closes); + md_list_free(&s.bal); + md_list_free(&s.gts); + md_list_free(&s.blanks); +} diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 8ac85fcb57..bdad5249ad 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -46,6 +46,7 @@ const char *cbm_doclink_reason_name(int reason) { * to this file. */ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, + &cbm_doclink_md_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; @@ -328,6 +329,9 @@ typedef struct { int64_t src; int64_t tgt; uint32_t line; + uint32_t target_first; /* the target lines the reference names; 0: none */ + uint32_t target_last; + const char *via; uint16_t syntax; bool exact; } doclink_mention_t; @@ -398,11 +402,16 @@ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t j++; } char props[CBM_SZ_256]; + char lines[CBM_SZ_64] = ""; + if (m[i].target_first > 0) { + snprintf(lines, sizeof(lines), ",\"target_lines\":[%u,%u]", m[i].target_first, + m[i].target_last); + } snprintf(props, sizeof(props), - "{\"via\":\"doc_comment\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u," - "\"count\":%d}", - cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", m[i].line, - j - i); + "{\"via\":\"%s\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u," + "\"count\":%d%s}", + m[i].via, cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", + m[i].line, j - i, lines); if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, props) == 0) { mark_failed(dl, rel, "alloc"); /* an edge that is not there is not counted */ } else { @@ -488,6 +497,9 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe mentions[nm++] = (doclink_mention_t){.src = src->id, .tgt = out.target->id, .line = link->line, + .target_first = out.target_first, + .target_last = out.target_last, + .via = R->via ? R->via : "doc_comment", .syntax = link->syntax, .exact = out.exact}; continue; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 20f9fbfaa5..a51ada8c2c 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -67,6 +67,10 @@ typedef struct { const cbm_gbuf_node_t *target; bool exact; /* tier: "exact" (qualified/doc ID/alias) vs "unique" (scope lookup) */ int reason; + /* Lines of the target that the reference names (a document's `file#L3-L9`): + * the edge's "target_lines". 0 and 0: the reference names no lines. */ + uint32_t target_first; + uint32_t target_last; } cbm_doclink_outcome_t; /* A file's stored doc-link scope (incremental runs: files not re-extracted). */ @@ -222,9 +226,14 @@ typedef struct { * the run. */ int (*scope_accepted)(const char *scope); char *(*rejected_scope)(const char *scope); + /* The `via` of its MENTIONS edges: what kind of text the reference is + * written in. NULL: "doc_comment". */ + const char *via; } cbm_doclink_resolver_t; extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver; +/* Markdown documents (doc_links_md.c): via "markdown". */ +extern const cbm_doclink_resolver_t cbm_doclink_md_resolver; #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS /* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c new file mode 100644 index 0000000000..6251cb3364 --- /dev/null +++ b/src/pipeline/doc_links_md.c @@ -0,0 +1,1122 @@ +/* + * doc_links_md.c — Markdown references -> MENTIONS edges: the resolving half. + * + * A Markdown reference (doclink_md.c) names a repository PATH, so it is looked + * up in the graph as written: the File node, or the Folder node, at that path, + * read relative to the document's own directory and -- for a path with a + * directory part -- relative to the repository root as well (documents are + * written both ways). A file name alone is read relative to the document + * only: a file of that name elsewhere in the repository is not what the + * document says (the field tests' rule for bare file names). + * + * Into the file: a line range picks the innermost code definition that holds + * the WHOLE range; a member (`path::Name`, `path::Owner::name`) picks the one + * definition of that name in the file. Otherwise the edge goes to the file, + * and the range stays on it ("target_lines"). Every edge is exact: the + * document wrote the path. + * + * Unresolved: + * missing the path is not in the graph, but its directory is: the + * document names something that is not there (doc rot) + * ambiguous a file name alone that is not next to the document but exists + * elsewhere in the repository: the document does not say which + * Not a reference to this repository at all (LOCAL: no edge, no row): a link + * back into the same document; a path that is neither indexed nor of a type + * the indexer reads (an image, an archive, an extensionless page); a path + * whose directory is not in the repository either (a file a tutorial asks + * the reader to create, a system path, a path above the root); and, in a + * code span, a rooted single segment without a file extension, which is a + * URL route (`/docs`, `/health`), not the directory of that name. + */ +#include "pipeline/doc_links.h" + +#include "discover/discover.h" /* cbm_language_for_filename */ +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include + +enum { + MDR_PATH_CAP = 1024, /* the longest repository path looked up */ + MDR_TRIES = 2, /* relative to the document, relative to the root */ + MDR_BASENAME_INIT = 1024, /* first capacity of the file-name table */ + MDR_HEX_BASE = 16, + MDR_PCT_LEN = 3, /* `%XX` */ +}; + +/* Labels whose nodes are code a range or a member can name. */ +/* Labels whose nodes are code a range, a member or a name can bind. Every one + * of them is proxied on the incremental route (pipeline_delta.c preseed), so + * a re-resolved document binds what a full build binds; Macro nodes are not + * proxied, and are no target here. */ +static const char *const MDR_CODE_LABELS[] = { + "Function", "Method", "Class", "Struct", "Interface", "Trait", "Enum", "Type", + "Variable", "Constant", "Field", "Table", "View", "Trigger", NULL}; + +static unsigned char mdr_norm_c(unsigned char c); +static bool mdr_in(const char *s, size_t n, const char *const *list); +static void mdr_destroy(void *index); + +/* A module or package a qualified name can name: a code file without its + * extension (`pkg/routing.py` is `pkg.routing`), a package directory through + * its index file (`pkg/__init__.py`, `mod.rs`, `index.ts`, `lib.rs`), or a + * directory. */ +typedef struct mdr_pkg { + const char *chain; /* the pieces before its name, '.'-joined */ + const cbm_gbuf_node_t *node; + const char *file; /* its code file (NULL: a directory) */ + struct mdr_pkg *next; /* the next one of the same name */ +} mdr_pkg_t; + +/* A code definition and its file: the by-file list ranges and members are + * read from. */ +typedef struct { + const char *file; + const cbm_gbuf_node_t *node; +} mdr_def_t; + +typedef struct { + const char *project; + size_t project_len; + const char **run_paths; /* run_file -> rel_path (NULL: not a Markdown file of this run) */ + int run_count; + /* base name -> number of File nodes with it (a file name alone that is + * not next to the document: ambiguous when the name exists anywhere) */ + CBMHashTable *basenames; + CBMHashTable *pkgs; /* name -> mdr_pkg_t list */ + CBMHashTable *top; /* normalized top-level directory and code-file names */ + CBMArena arena; /* the strings and records of pkgs and top */ + mdr_def_t *defs; /* every code definition, sorted by (file, start line, id) */ + int ndefs; + CBMHashTable *folders; /* directory path -> its Folder node */ +} mdr_index_t; + +static const char *mdr_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +static bool mdr_code_file(const char *path) { + CBMLanguage lang = cbm_language_for_filename(mdr_base(path)); + return lang != CBM_LANG_COUNT && lang != CBM_LANG_MARKDOWN && lang != CBM_LANG_RST && + lang != CBM_LANG_HTML && lang != CBM_LANG_CSS && lang != CBM_LANG_SCSS && + !cbm_has_config_extension(path); +} + +/* The stems of a package directory's index file. */ +static const char *const MDR_INDEX_STEMS[] = {"__init__", "mod", "index", "lib", NULL}; + +/* Lower-cased copy, '-' read as '_'. */ +static char *mdr_norm_dup(CBMArena *a, const char *s, size_t n) { + char *out = cbm_arena_strndup(a, s, n); + for (size_t i = 0; out && i < n; i++) { + out[i] = (char)mdr_norm_c((unsigned char)out[i]); + } + return out; +} + +/* Add the module `key` (a path without extension, '/'-separated, whose last + * piece may hold dots) unless one is there. False when memory ran out. */ +static bool mdr_add_pkg(mdr_index_t *x, CBMHashTable *seen, const char *key, size_t klen, + const cbm_gbuf_node_t *node, const char *file) { + char *seen_key = cbm_arena_strndup(&x->arena, key, klen); /* borrowed by `seen` */ + char *k = seen_key ? cbm_arena_strndup(&x->arena, key, klen) : NULL; + if (!k) { + return false; + } + if (cbm_ht_get(seen, seen_key)) { + return true; /* the first file of a key wins (files come sorted by path) */ + } + cbm_ht_set(seen, seen_key, (void *)node); + /* pieces: '/'- and '.'-separated; the name is the last one */ + for (char *c = k; *c; c++) { + if (*c == '/') { + *c = '.'; + } + } + char *last = strrchr(k, '.'); + const char *name = last ? last + SKIP_ONE : k; + if (!name[0]) { + return true; + } + mdr_pkg_t *p = (mdr_pkg_t *)cbm_arena_alloc(&x->arena, sizeof(*p)); + if (!p) { + return false; + } + if (last) { + *last = '\0'; + } + p->chain = last ? k : ""; + p->node = node; + p->file = file; + p->next = (mdr_pkg_t *)cbm_ht_get(x->pkgs, name); + cbm_ht_set(x->pkgs, name, p); + return true; +} + +static bool mdr_add_top(mdr_index_t *x, const char *s, size_t n) { + char *k = mdr_norm_dup(&x->arena, s, n); + if (!k) { + return false; + } + cbm_ht_set(x->top, k, (void *)k); + return true; +} + +static int mdr_def_cmp(const void *a, const void *b) { + const mdr_def_t *x = (const mdr_def_t *)a; + const mdr_def_t *y = (const mdr_def_t *)b; + int c = strcmp(x->file, y->file); + if (c != 0) { + return c; + } + if (x->node->start_line != y->node->start_line) { + return x->node->start_line < y->node->start_line ? -1 : 1; + } + return (x->node->id > y->node->id) - (x->node->id < y->node->id); +} + +/* Every code definition of the graph, sorted by file. False when memory ran + * out. */ +static bool mdr_build_defs(mdr_index_t *x, const cbm_gbuf_t *graph) { + int total = 0; + for (size_t l = 0; MDR_CODE_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, MDR_CODE_LABELS[l], &nodes, &count) == 0) { + total += count; + } + } + if (total == 0) { + return true; + } + x->defs = (mdr_def_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)total * sizeof(*x->defs)); + if (!x->defs) { + return false; + } + for (size_t l = 0; MDR_CODE_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, MDR_CODE_LABELS[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count && x->ndefs < total; i++) { + if (nodes[i]->file_path && nodes[i]->file_path[0]) { + x->defs[x->ndefs++] = (mdr_def_t){.file = nodes[i]->file_path, .node = nodes[i]}; + } + } + } + qsort(x->defs, (size_t)x->ndefs, sizeof(*x->defs), mdr_def_cmp); + return true; +} + +/* The module index over the graph's code files and directories. */ +static bool mdr_build_pkgs(mdr_index_t *x, const cbm_gbuf_t *graph) { + CBMHashTable *seen = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + if (!seen) { + return false; + } + bool ok = true; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, "File", &nodes, &count) == 0) { + for (int i = 0; ok && i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !fp[0] || !mdr_code_file(fp)) { + continue; + } + const char *base = mdr_base(fp); + const char *dot = strrchr(base, '.'); + size_t stem_len = dot && dot != base ? (size_t)(dot - base) : strlen(base); + size_t dir_len = base > fp ? (size_t)(base - fp - SKIP_ONE) : 0; + ok = mdr_add_pkg(x, seen, fp, (size_t)(base - fp) + stem_len, nodes[i], fp); + if (ok && dir_len > 0 && mdr_in(base, stem_len, MDR_INDEX_STEMS)) { + ok = mdr_add_pkg(x, seen, fp, dir_len, nodes[i], fp); + } + if (ok && dir_len == 0) { + ok = mdr_add_top(x, base, stem_len); + } + } + } + if (ok && cbm_gbuf_find_by_label(graph, "Folder", &nodes, &count) == 0) { + for (int i = 0; ok && i < count; i++) { + const char *dp = nodes[i]->file_path; + if (!dp || !dp[0]) { + continue; + } + ok = mdr_add_pkg(x, seen, dp, strlen(dp), nodes[i], NULL); + if (ok && !strchr(dp, '/')) { + ok = mdr_add_top(x, dp, strlen(dp)); + } + } + } + cbm_ht_free(seen); + return ok; +} + +static void *mdr_build(const cbm_doclink_build_in_t *in) { + mdr_index_t *x = (mdr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->project_len = x->project ? strlen(x->project) : 0; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->basenames = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->pkgs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->top = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->folders = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + if ((in->run_file_count > 0 && !x->run_paths) || !x->basenames || !x->pkgs || !x->top || + !x->folders || !x->project || !mdr_build_pkgs(x, in->graph) || + !mdr_build_defs(x, in->graph)) { + mdr_destroy(x); + return NULL; + } + for (int i = 0; i < in->file_count; i++) { + int rf = in->files[i].run_file; + if (rf >= 0 && rf < x->run_count) { + x->run_paths[rf] = in->files[i].rel_path; /* valid until destroy */ + } + } + const cbm_gbuf_node_t **folders = NULL; + int nfolders = 0; + if (cbm_gbuf_find_by_label(in->graph, "Folder", &folders, &nfolders) == 0) { + for (int i = 0; i < nfolders; i++) { + if (folders[i]->file_path && folders[i]->file_path[0]) { + /* the graph outlives the index: its strings may be keys */ + cbm_ht_set(x->folders, folders[i]->file_path, (void *)folders[i]); + } + } + } + const cbm_gbuf_node_t **files = NULL; + int nfiles = 0; + if (cbm_gbuf_find_by_label(in->graph, "File", &files, &nfiles) == 0) { + for (int i = 0; i < nfiles; i++) { + if (!files[i]->file_path || !files[i]->file_path[0]) { + continue; + } + const char *base = mdr_base(files[i]->file_path); /* the graph outlives the index */ + uintptr_t n = (uintptr_t)cbm_ht_get(x->basenames, base); + cbm_ht_set(x->basenames, base, (void *)(n + SKIP_ONE)); + } + } + return x; +} + +static void mdr_destroy(void *index) { + mdr_index_t *x = (mdr_index_t *)index; + if (!x) { + return; + } + cbm_ht_free(x->basenames); + cbm_ht_free(x->pkgs); + cbm_ht_free(x->top); + cbm_ht_free(x->folders); + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->defs); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* ── Paths ───────────────────────────────────────────────────────── */ + +static int mdr_hex(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + CBM_DECIMAL_BASE; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + CBM_DECIMAL_BASE; + } + return CBM_NOT_FOUND; +} + +/* `base` + "/" + `p`, normalized: "." dropped, ".." pops a segment. False + * when the path climbs above the root or does not fit. */ +static bool mdr_join(const char *base, const char *p, char *out, size_t cap) { + size_t w = 0; + const char *parts[PAIR_LEN] = {base, p}; + for (int k = 0; k < PAIR_LEN; k++) { + const char *s = parts[k]; + while (s && *s) { + const char *e = strchr(s, '/'); + size_t n = e ? (size_t)(e - s) : strlen(s); + if (n == 0 || (n == SKIP_ONE && s[0] == '.')) { + /* empty or "." */ + } else if (n == PAIR_LEN && s[0] == '.' && s[SKIP_ONE] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + if (w > 0) { + w--; /* the slash before the popped segment */ + } + } else { + if (w + n + PAIR_LEN > cap) { + return false; + } + if (w > 0) { + out[w++] = '/'; + } + memcpy(out + w, s, n); + w += n; + } + s = e ? e + SKIP_ONE : NULL; + } + } + out[w] = '\0'; + return true; +} + +static const cbm_gbuf_node_t *mdr_folder(const mdr_index_t *x, const cbm_gbuf_t *graph, + const char *path) { + (void)graph; + return path[0] ? (const cbm_gbuf_node_t *)cbm_ht_get(x->folders, path) : NULL; +} + +static bool mdr_code_label(const char *label) { + for (size_t i = 0; label && MDR_CODE_LABELS[i]; i++) { + if (strcmp(label, MDR_CODE_LABELS[i]) == 0) { + return true; + } + } + return false; +} + +/* The member text's last segment (after "::" or "."), and the one before it. */ +static void mdr_member_names(const char *member, const char **name, size_t *name_len, + const char **owner, size_t *owner_len) { + const char *segs[PAIR_LEN] = {NULL, NULL}; + size_t lens[PAIR_LEN] = {0, 0}; + const char *s = member; + while (*s) { + size_t n = strcspn(s, ":."); + if (n > 0) { + segs[0] = segs[SKIP_ONE]; + lens[0] = lens[SKIP_ONE]; + segs[SKIP_ONE] = s; + lens[SKIP_ONE] = n; + } + s += n; + while (*s == ':' || *s == '.') { + s++; + } + } + *name = segs[SKIP_ONE]; + *name_len = lens[SKIP_ONE]; + *owner = segs[0]; + *owner_len = lens[0]; +} + +/* The QN segment before the node's own name ends with `owner`. */ +static bool mdr_owned_by(const cbm_gbuf_node_t *n, const char *owner, size_t owner_len) { + const char *qn = n->qualified_name; + size_t ql = qn ? strlen(qn) : 0; + size_t nl = n->name ? strlen(n->name) : 0; + if (!owner || ql < nl + owner_len + PAIR_LEN) { + return false; + } + size_t end = ql - nl - SKIP_ONE; /* the '.' before the name */ + if (qn[end] != '.') { + return false; + } + size_t start = end - owner_len; + return memcmp(qn + start, owner, owner_len) == 0 && (start == 0 || qn[start - SKIP_ONE] == '.'); +} + +/* The code definitions of `path`: [*lo, *hi) of the index's sorted list. */ +static void mdr_file_defs(const mdr_index_t *x, const char *path, int *lo, int *hi) { + int a = 0; + int b = x->ndefs; + while (a < b) { + int mid = a + ((b - a) / PAIR_LEN); + if (strcmp(x->defs[mid].file, path) < 0) { + a = mid + SKIP_ONE; + } else { + b = mid; + } + } + int e = a; + while (e < x->ndefs && strcmp(x->defs[e].file, path) == 0) { + e++; + } + *lo = a; + *hi = e; +} + +/* The code definition of file `path` that the range or the member names, or + * NULL. Read from the index's definitions by file, never from DEFINES edges: + * an incremental run's graph holds the edges of the files it re-extracts + * only, while every node is there. */ +static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path, uint32_t first, + uint32_t last, const char *member) { + int lo = 0; + int hi = 0; + mdr_file_defs(x, path, &lo, &hi); + const cbm_gbuf_node_t *best = NULL; + if (first > 0) { + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (n->start_line <= 0 || (uint32_t)n->start_line > first || + (uint32_t)n->end_line < last) { + continue; + } + int span = n->end_line - n->start_line; + int best_span = best ? best->end_line - best->start_line : 0; + if (!best || span < best_span || + (span == best_span && (n->start_line > best->start_line || + (n->start_line == best->start_line && n->id < best->id)))) { + best = n; + } + } + if (best) { + return best; + } + } + if (!member) { + return NULL; + } + const char *name; + const char *owner; + size_t name_len; + size_t owner_len; + mdr_member_names(member, &name, &name_len, &owner, &owner_len); + if (!name) { + return NULL; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *owned = NULL; + int owned_hits = 0; + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (!n->name || strlen(n->name) != name_len || memcmp(n->name, name, name_len) != 0) { + continue; + } + hit = n; + hits++; + if (owner && mdr_owned_by(n, owner, owner_len)) { + owned = n; + owned_hits++; + } + } + if (owned_hits == SKIP_ONE) { + return owned; + } + return owned_hits == 0 && hits == SKIP_ONE ? hit : NULL; +} + +/* Is `path` of a file type the indexer reads? */ +static bool mdr_indexable(const char *path) { + return cbm_language_for_filename(mdr_base(path)) != CBM_LANG_COUNT; +} + +/* ── Qualified names ─────────────────────────────────────────────── */ + +/* Path pieces a written qualifier may leave out (`pkg.Foo` for + * `pkg/src/Foo`, `pkg.mod.Foo` for `pkg/mod.rs`'s Foo). */ +static const char *const MDR_TRANSPARENT[] = { + "src", "lib", "mod", "__init__", "index", "main", "java", "kotlin", "scala", "groovy", NULL}; + +/* Directory names and file names of test code. */ +static const char *const MDR_TEST_DIRS[] = {"test", + "tests", + "testing", + "__tests__", + "spec", + "specs", + "testdata", + "test_data", + "fixtures", + "e2e", + "integration_tests", + "testutil", + "testutils", + "benchmark", + "benchmarks", + "bench", + "mocks", + "mock", + "fakes", + "fake", + "testlib", + NULL}; + +static unsigned char mdr_norm_c(unsigned char c) { + if (c >= 'A' && c <= 'Z') { + return (unsigned char)(c - 'A' + 'a'); + } + return c == '-' ? '_' : c; +} + +/* Equal after lower-casing and reading '-' as '_'. */ +static bool mdr_norm_eq(const char *a, size_t al, const char *b, size_t bl) { + if (al != bl) { + return false; + } + for (size_t i = 0; i < al; i++) { + if (mdr_norm_c((unsigned char)a[i]) != mdr_norm_c((unsigned char)b[i])) { + return false; + } + } + return true; +} + +/* s[0..n) ends with `suffix`. */ +static bool mdr_ends(const char *s, size_t n, const char *suffix) { + size_t k = strlen(suffix); + return n >= k && memcmp(s + n - k, suffix, k) == 0; +} + +static bool mdr_in(const char *s, size_t n, const char *const *list) { + for (size_t i = 0; list[i]; i++) { + if (mdr_norm_eq(s, n, list[i], strlen(list[i]))) { + return true; + } + } + return false; +} + +/* A test file or a file in a test directory. */ +static bool mdr_test_path(const char *path) { + const char *s = path; + for (const char *slash = strchr(s, '/'); slash; slash = strchr(s, '/')) { + if (mdr_in(s, (size_t)(slash - s), MDR_TEST_DIRS)) { + return true; + } + s = slash + SKIP_ONE; + } + const char *base = s; + if (strncmp(base, "test_", strlen("test_")) == 0 || strcmp(base, "conftest.py") == 0) { + return true; + } + const char *dot = strrchr(base, '.'); + if (!dot) { + return false; + } + size_t stem = (size_t)(dot - base); + const char *ext = dot + SKIP_ONE; + bool lower_ext = ext[0] != '\0'; + for (const char *e = ext; *e; e++) { + lower_ext = lower_ext && *e >= 'a' && *e <= 'z'; + } + static const char *const jvm_like[] = {"java", "kt", "cs", "scala", "groovy", "php", NULL}; + return (lower_ext && (mdr_ends(base, stem, "_test") || mdr_ends(base, stem, ".test") || + mdr_ends(base, stem, ".spec"))) || + (strcmp(ext, "rb") == 0 && mdr_ends(base, stem, "_spec")) || + (mdr_in(ext, strlen(ext), jvm_like) && + (mdr_ends(base, stem, "Test") || mdr_ends(base, stem, "Tests"))); +} + +typedef struct { + const char *path; /* as written (no fragment, decoded) */ + uint32_t first; + uint32_t last; + const char *member; + bool name_only; /* a file name alone: the document's directory only */ + bool dir; /* written as a directory */ + bool link; /* a link: relative to the document by definition */ +} mdr_ref_t; + +static void mdr_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *target, uint32_t first, + uint32_t last) { + out->kind = CBM_DOCLINK_EDGE; + out->target = target; + out->exact = true; + out->target_first = first; + out->target_last = last; +} + +enum { MDR_MAX_PIECES = 80 }; /* a code span is at most 150 bytes */ + +typedef struct { + const char *s[MDR_MAX_PIECES]; + size_t n[MDR_MAX_PIECES]; + int count; +} mdr_pieces_t; + +/* The '.'-separated pieces of s. False when there are too many. */ +static bool mdr_split(const char *s, mdr_pieces_t *p) { + p->count = 0; + for (;;) { + if (p->count == MDR_MAX_PIECES) { + return false; + } + size_t n = strcspn(s, "."); + p->s[p->count] = s; + p->n[p->count] = n; + p->count++; + if (!s[n]) { + return true; + } + s += n + SKIP_ONE; + } +} + +/* Is the written qualifier q[0..nq) a piece-aligned suffix of `chain` (a + * node's '.'-joined qualifier pieces), when chain pieces in MDR_TRANSPARENT + * may be skipped? `exact`: every piece as written; otherwise lower-cased and + * '-' read as '_' (`rootMulti.Store` names package rootmulti). */ +static bool mdr_chain_suffix(const char *chain, size_t clen, const mdr_pieces_t *q, int nq, + bool exact) { + size_t ce = clen; + bool more = clen > 0; + int j = nq - SKIP_ONE; + while (j >= 0) { + if (!more) { + return false; + } + size_t cs = ce; + while (cs > 0 && chain[cs - SKIP_ONE] != '.') { + cs--; + } + const char *piece = chain + cs; + size_t pl = ce - cs; + bool same = exact ? (pl == q->n[j] && memcmp(piece, q->s[j], pl) == 0) + : mdr_norm_eq(piece, pl, q->s[j], q->n[j]); + if (same) { + j--; + } else if (!mdr_in(piece, pl, MDR_TRANSPARENT)) { + return false; + } + more = cs > 0; + ce = more ? cs - SKIP_ONE : 0; + } + return true; +} + +/* The node's qualifier chain (its QN without the project and its own name); + * false when the QN does not have that shape. */ +static bool mdr_node_chain(const mdr_index_t *x, const cbm_gbuf_node_t *n, const char **chain, + size_t *clen) { + const char *qn = n->qualified_name; + size_t name_len = n->name ? strlen(n->name) : 0; + if (!qn || name_len == 0 || strncmp(qn, x->project, x->project_len) != 0 || + qn[x->project_len] != '.') { + return false; + } + const char *rest = qn + x->project_len + SKIP_ONE; + size_t rl = strlen(rest); + if (rl < name_len || strcmp(rest + rl - name_len, n->name) != 0) { + return false; + } + if (rl == name_len) { + *chain = rest; + *clen = 0; + return true; + } + if (rest[rl - name_len - SKIP_ONE] != '.') { + return false; + } + *chain = rest; + *clen = rl - name_len - SKIP_ONE; + return true; +} + +static bool mdr_python_file(const char *path) { + return mdr_ends(path, strlen(path), ".py") || mdr_ends(path, strlen(path), ".pyi"); +} + +static bool mdr_classlike(const char *label) { + static const char *const labels[] = {"Class", "Struct", "Interface", "Trait", + "Enum", "Type", NULL}; + for (size_t i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* Does the repository itself define the written root (`fastapi` of + * `fastapi.routing.X`): a top-level directory or code file, or a type? */ +static bool mdr_root_known(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *root, + size_t len) { + char key[MDR_PATH_CAP]; + if (len >= sizeof(key)) { + return false; + } + for (size_t i = 0; i < len; i++) { + key[i] = (char)mdr_norm_c((unsigned char)root[i]); + } + key[len] = '\0'; + if (cbm_ht_get(x->top, key)) { + return true; + } + memcpy(key, root, len); + key[len] = '\0'; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, key, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + if (mdr_classlike(nodes[i]->label) && nodes[i]->file_path && + mdr_code_file(nodes[i]->file_path)) { + return true; + } + } + } + return false; +} + +/* The one non-member definition named `name` in the module file `file`. */ +static const cbm_gbuf_node_t *mdr_module_item(const mdr_index_t *x, const char *file, + const char *name) { + int lo = 0; + int hi = 0; + mdr_file_defs(x, file, &lo, &hi); + const cbm_gbuf_node_t *hit = NULL; + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (!n->name || strcmp(n->name, name) != 0 || strcmp(n->label, "Method") == 0 || + strcmp(n->label, "Field") == 0) { + continue; + } + if (hit) { + return NULL; + } + hit = n; + } + return hit; +} + +/* Candidates of one class: the first one, and how many (counting stops at + * two: one is an edge, two are ambiguous). */ +typedef struct { + const cbm_gbuf_node_t *node; + const mdr_pkg_t *pkg; + int hits; +} mdr_tally_t; + +static void mdr_tally(mdr_tally_t *t, const cbm_gbuf_node_t *node, const mdr_pkg_t *pkg) { + if (t->hits == 0) { + t->node = node; + t->pkg = pkg; + } + if (t->hits < PAIR_LEN) { + t->hits++; + } +} + +/* A qualified name written in a code span: EXACT only. The one code + * definition named like its last piece whose qualifier chain ends with the + * written qualifier, or the one module or package so named. A match with + * every piece as written outranks one that needs case folding, so + * `fastapi.middleware` names the package fastapi/middleware, not the method + * FastAPI.middleware. Order: definitions as written, modules as written, + * definitions folded, modules folded; the first class with a candidate + * decides. */ +static void mdr_resolve_name(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const CBMDocLinkMdPath *p, cbm_doclink_outcome_t *out) { + mdr_pieces_t q; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + if (!mdr_split(p->path, &q) || q.count < PAIR_LEN) { + return; + } + int nq = q.count - SKIP_ONE; /* the qualifier pieces */ + char name[MDR_PATH_CAP]; + size_t name_len = q.n[nq]; + if (name_len >= sizeof(name)) { + return; + } + memcpy(name, q.s[nq], name_len); + name[name_len] = '\0'; + bool doc_test = mdr_test_path(doc); + enum { DEF_EXACT, MOD_EXACT, DEF_FOLDED, MOD_FOLDED, TALLIES }; + mdr_tally_t tally[TALLIES] = {{0}}; + int test_hits = 0; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + count = 0; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *chain = NULL; + size_t clen = 0; + if (!mdr_code_label(n->label) || !n->file_path || !mdr_code_file(n->file_path) || + !mdr_node_chain(x, n, &chain, &clen)) { + continue; + } + if (p->colon && !mdr_python_file(n->file_path)) { + continue; /* `module:attr` is Python's form */ + } + if (strcmp(n->label, "Field") == 0) { + /* a field only through its type, written as the type is named: + * never through an instance (`self.x`, `app.state`) */ + const char *owner = chain + clen; + while (owner > chain && owner[-SKIP_ONE] != '.') { + owner--; + } + size_t ol = (size_t)(chain + clen - owner); + if (p->instance || ol != q.n[nq - SKIP_ONE] || + memcmp(owner, q.s[nq - SKIP_ONE], ol) != 0) { + continue; + } + } + bool exact = mdr_chain_suffix(chain, clen, &q, nq, true); + if (!exact && !mdr_chain_suffix(chain, clen, &q, nq, false)) { + continue; + } + if (!doc_test && mdr_test_path(n->file_path)) { + test_hits++; + continue; + } + mdr_tally(&tally[exact ? DEF_EXACT : DEF_FOLDED], n, NULL); + } + /* modules and packages so named */ + for (const mdr_pkg_t *m = (const mdr_pkg_t *)cbm_ht_get(x->pkgs, name); m; m = m->next) { + size_t cl = strlen(m->chain); + bool exact = mdr_chain_suffix(m->chain, cl, &q, nq, true); + if (!exact && !mdr_chain_suffix(m->chain, cl, &q, nq, false)) { + continue; + } + if (!doc_test && mdr_test_path(m->file ? m->file : m->node->file_path)) { + test_hits++; + continue; + } + mdr_tally(&tally[exact ? MOD_EXACT : MOD_FOLDED], m->node, m); + } + for (int k = 0; k < TALLIES; k++) { + if (tally[k].hits == 0) { + continue; + } + if (tally[k].hits > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + const mdr_pkg_t *mod = tally[k].pkg; + const cbm_gbuf_node_t *item = mod && mod->file ? mdr_module_item(x, mod->file, name) : NULL; + mdr_edge(out, item ? item : tally[k].node, 0, 0); + return; + } + if (test_hits > 0) { + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + if (mdr_root_known(x, graph, q.s[0], q.n[0])) { + out->reason = CBM_DOCLINK_REASON_MISSING; /* the repository's name, but nothing so named */ + return; + } + out->kind = CBM_DOCLINK_LOCAL; /* code of another project (a library, the standard one) */ +} + +static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const mdr_ref_t *r, cbm_doclink_outcome_t *out) { + const char *p = r->path; + bool rooted = p[0] == '/'; + bool relative = strncmp(p, "./", PAIR_LEN) == 0 || strncmp(p, "../", PAIR_LEN + 1) == 0; + char doc_dir[MDR_PATH_CAP]; + const char *slash = strrchr(doc, '/'); + size_t dl = slash ? (size_t)(slash - doc) : 0; + if (dl >= sizeof(doc_dir)) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + memcpy(doc_dir, doc, dl); + doc_dir[dl] = '\0'; + char tries[MDR_TRIES][MDR_PATH_CAP]; + int nt = 0; + int attempted = 0; + if (!rooted) { + attempted++; + nt += mdr_join(doc_dir, p, tries[nt], MDR_PATH_CAP); + } + if (!relative && !r->name_only) { + attempted++; + const char *q = p; + while (*q == '/') { + q++; + } + if (mdr_join("", q, tries[nt], MDR_PATH_CAP) && (nt == 0 || strcmp(tries[0], tries[nt]))) { + nt++; + } + } + for (int i = 0; i < nt; i++) { + const char *t = tries[i]; + if (!t[0]) { + continue; /* the repository root itself */ + } + const cbm_gbuf_node_t *file = cbm_pipeline_file_node(graph, x->project, t); + if (file) { + if (strcmp(t, doc) == 0) { + out->kind = CBM_DOCLINK_LOCAL; /* a link into the same document */ + return; + } + const cbm_gbuf_node_t *seg = (r->first > 0 || r->member) + ? mdr_segment(x, t, r->first, r->last, r->member) + : NULL; + mdr_edge(out, seg ? seg : file, r->first, r->last); + return; + } + const cbm_gbuf_node_t *folder = mdr_folder(x, graph, t); + if (folder) { + mdr_edge(out, folder, 0, 0); + return; + } + } + out->kind = CBM_DOCLINK_UNRESOLVED; + if ((attempted > 0 && nt == 0) || (!r->dir && !mdr_indexable(p))) { + /* above the repository root; a file or a page the indexer does not read */ + out->kind = CBM_DOCLINK_LOCAL; + return; + } + if (r->name_only && !r->link && cbm_ht_get(x->basenames, mdr_base(p))) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + /* doc rot when the reference points into the repository: its directory is + * there (a bare name in a link: the document's own directory) */ + for (int i = 0; i < nt; i++) { + char parent[MDR_PATH_CAP]; + const char *ps = strrchr(tries[i], '/'); + size_t pl = ps ? (size_t)(ps - tries[i]) : 0; + memcpy(parent, tries[i], pl); + parent[pl] = '\0'; + if (pl > 0 && mdr_folder(x, graph, parent) && (strchr(p, '/') || r->link)) { + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + if (pl == 0 && r->link && !rooted) { + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + } + out->kind = CBM_DOCLINK_LOCAL; /* not in this repository, nor its directory */ +} + +/* A link destination: `<...>` dropped, %XX decoded, `#L` fragment read, query + * dropped. False when nothing of a path is left (an anchor or a query). */ +static bool mdr_link_path(const char *raw, char *buf, size_t cap, mdr_ref_t *r) { + size_t n = strlen(raw); + while (n > 0 && (raw[0] == '<' || raw[0] == '>')) { + raw++; + n--; + } + while (n > 0 && (raw[n - SKIP_ONE] == '<' || raw[n - SKIP_ONE] == '>')) { + n--; + } + size_t w = 0; + for (size_t i = 0; i < n; i++) { + if (w + SKIP_ONE >= cap) { + return false; + } + int h1 = i + PAIR_LEN < n && raw[i] == '%' ? mdr_hex(raw[i + SKIP_ONE]) : CBM_NOT_FOUND; + int h2 = h1 >= 0 ? mdr_hex(raw[i + PAIR_LEN]) : CBM_NOT_FOUND; + if (h1 >= 0 && h2 >= 0 && (h1 * MDR_HEX_BASE) + h2 != 0) { + buf[w++] = (char)((h1 * MDR_HEX_BASE) + h2); + i += MDR_PCT_LEN - SKIP_ONE; + } else { + buf[w++] = raw[i]; + } + } + buf[w] = '\0'; + char *hash = strchr(buf, '#'); + if (hash) { + *hash = '\0'; + (void)cbm_doclink_md_line_fragment(hash + SKIP_ONE, &r->first, &r->last); + } + char *query = strchr(buf, '?'); + if (query) { + *query = '\0'; + } + r->path = buf; + r->dir = buf[0] && buf[strlen(buf) - SKIP_ONE] == '/'; + r->name_only = !strchr(buf, '/'); + return buf[0] != '\0'; +} + +static void mdr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const mdr_index_t *x = (const mdr_index_t *)index; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + char buf[MDR_PATH_CAP]; + mdr_ref_t r = {0}; + switch (link->syntax) { + case CBM_DOCLINK_MD_LINK: + r.link = true; + if (!mdr_link_path(link->raw, buf, sizeof(buf), &r)) { + out->kind = CBM_DOCLINK_LOCAL; /* an anchor, a query: nothing of a path */ + return; + } + break; + case CBM_DOCLINK_MD_PATH: { + size_t n = strlen(link->raw); + if (n >= sizeof(buf)) { + break; /* longer than any path looked up: unparseable */ + } + memcpy(buf, link->raw, n + SKIP_ONE); + r.path = buf; + break; + } + case CBM_DOCLINK_MD_CODE_NAME: { + CBMDocLinkMdPath p; + if (cbm_doclink_md_classify_span(link->raw, strlen(link->raw), buf, sizeof(buf), &p) && + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + mdr_resolve_name(x, graph, doc, &p, out); + return; + } + break; + } + case CBM_DOCLINK_MD_CODE_PATH: { + CBMDocLinkMdPath p; + if (!cbm_doclink_md_classify_span(link->raw, strlen(link->raw), buf, sizeof(buf), &p) || + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + break; + } + r.path = p.path; + r.member = p.member; + r.first = p.first_line; + r.last = p.last_line; + r.name_only = p.shape == CBM_DOCLINK_MD_FILENAME; + r.dir = p.shape == CBM_DOCLINK_MD_DIR; + if (r.dir && r.path[0] == '/' && !strchr(r.path + SKIP_ONE, '/')) { + /* `/docs`, `/health`: one rooted segment is a URL route, not the + * directory of that name (`/x/bank` stays a path) */ + out->kind = CBM_DOCLINK_LOCAL; + return; + } + break; + } + default: + break; + } + if (!r.path || !r.path[0]) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + mdr_resolve_path(x, graph, doc, &r, out); +} + +const cbm_doclink_resolver_t cbm_doclink_md_resolver = { + .langs = {CBM_LANG_MARKDOWN}, + .lang_count = 1, + .scope_tag = NULL, + .build = mdr_build, + .destroy = mdr_destroy, + .resolve = mdr_resolve, + .via = "markdown", +}; diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 8634a633e6..a237881ec8 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -300,7 +300,8 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g " CASE WHEN label IN ('Module','Variable')" " AND instr(properties, '\"http_client\"') > 0" " THEN json_object('http_client', json_extract(properties, '$.http_client')," - " 'http_base_url', json_extract(properties, '$.http_base_url')) END" + " 'http_base_url', json_extract(properties, '$.http_base_url')) END," + " start_line, end_line" " FROM nodes" " WHERE project = ?1 AND label NOT IN" " ('Macro','Comment','Section','Branch','Commit','Tag')" @@ -318,10 +319,15 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g const char *qn = (const char *)sqlite3_column_text(stmt, 3); const char *fp = (const char *)sqlite3_column_text(stmt, 4); const char *client_props = (const char *)sqlite3_column_text(stmt, 5); + /* Lines too: a document reference that names a line range binds the + * definition holding it (doc_links_md.c), in a re-resolved document as + * in a full build. */ + int start_line = sqlite3_column_int(stmt, 6); + int end_line = sqlite3_column_int(stmt, 7); /* Pin the gbuf id to the database id: proxies ARE their rows. */ cbm_gbuf_set_next_id(gbuf, id); - int64_t got = cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", 0, 0, - client_props ? client_props : "{}"); + int64_t got = cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", start_line, + end_line, client_props ? client_props : "{}"); if (got != id) { /* A QN collision inside the preseed set would silently split * identity between RAM and disk; the run cannot be trusted. */ diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c new file mode 100644 index 0000000000..5f33416def --- /dev/null +++ b/tests/test_doc_links_md.c @@ -0,0 +1,419 @@ +/* + * test_doc_links_md.c — Markdown documents -> MENTIONS edges. + * + * Extraction (doclink_md.c: which text is a reference, its family, line and + * section), the span classifier, the resolver (doc_links_md.c: paths, ranges, + * members, qualified names, the unresolved reasons, what is no reference at + * all), back navigation, and incremental == full. Pipeline tests index a real + * fixture through cbm_pipeline_run (helpers: test_doc_mentions_helpers.h). + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include + +/* ── the fixture ─────────────────────────────────────────────────── */ + +/* docs/guide.md, line by line (the line numbers are asserted below). */ +static const char + MD_GUIDE[] = "Intro names `pkg/config.go` before any heading.\n" /* 1 */ + "\n" /* 2 */ + "# Guide\n" /* 3 */ + "\n" /* 4 */ + "See [the config](../pkg/config.go) and [routing][r].\n" /* 5 */ + "\n" /* 6 */ + "[r]: ../app/routing.py\n" /* 7 */ + "\n" /* 8 */ + "## Paths\n" /* 9 */ + "\n" /* 10 */ + "Range `app/routing.py#L4-L5`, span `app/routing.py:1-9`, bare " + "app/applications.py.\n" /* 11 */ + "Dir `app/middleware/`, route `/docs`, ![x](img/a.png) and " + "https://example.org/a/b.py.\n" /* 12 */ + "\n" /* 13 */ + "## Names\n" /* 14 */ + "\n" /* 15 */ + "Use `app.routing.APIRouter.add`, `app.middleware`, `pkg.Config.Name`, " + "`self.Name`.\n" /* 16 */ + "Also `os.path.join`, `app.Nope`, `test_routing.only_in_tests`, `main.py`, " + "`../app/gone.py`.\n" /* 17 */ + "\n" /* 18 */ + "```\n" /* 19 */ + "`app/fenced.py` inside a fence\n" /* 20 */ + "```\n" /* 21 */ + "\n" /* 22 */ + "app/html.py in HTML, `cfg.Name` through an instance.\n"; /* 23 */ + +static void md_write_fixture(const char *tmp) { + th_write_file(TH_PATH(tmp, "docs/guide.md"), MD_GUIDE); + th_write_file(TH_PATH(tmp, "app/__init__.py"), ""); + th_write_file(TH_PATH(tmp, "app/routing.py"), "class APIRouter:\n" /* 1 */ + " prefix = ''\n" /* 2 */ + "\n" /* 3 */ + " def add(self, p):\n" /* 4 */ + " return p\n" /* 5 */ + "\n" /* 6 */ + "\n" /* 7 */ + "def helper():\n" /* 8 */ + " return 1\n"); /* 9 */ + th_write_file(TH_PATH(tmp, "app/applications.py"), "class App:\n" + " def middleware(self):\n" + " return None\n"); + th_write_file(TH_PATH(tmp, "app/middleware/__init__.py"), "VALUE = 1\n"); + th_write_file(TH_PATH(tmp, "tests/test_routing.py"), "def only_in_tests():\n" + " return 0\n"); + th_write_file(TH_PATH(tmp, "lib/main.py"), "def run():\n" + " return None\n"); + th_write_file(TH_PATH(tmp, "pkg/config.go"), "package pkg\n" + "\n" + "type Config struct {\n" + "\tName string\n" + "}\n" + "\n" + "func Util() {}\n"); +} + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const CBMDocLink *md_token(const CBMFileResult *r, const char *raw) { + return dm_find_token(r, raw); +} + +static bool md_ends_with(const char *s, const char *suffix) { + size_t a = s ? strlen(s) : 0; + size_t b = strlen(suffix); + return a >= b && strcmp(s + a - b, suffix) == 0; +} + +TEST(doc_links_md_extract_tokens) { + CBMFileResult *r = dm_extract(MD_GUIDE, CBM_LANG_MARKDOWN, "docs/guide.md"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->doc_links.failed); + /* before the first heading: the file is the source */ + const CBMDocLink *t = md_token(r, "pkg/config.go"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 1); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH); + ASSERT_TRUE((t->flags & CBM_DOCLINK_FLAG_FILE) != 0); + /* an inline link and a link reference definition, under "Guide" */ + t = md_token(r, "../pkg/config.go"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 5); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK); + ASSERT_TRUE(md_ends_with(t->source_qn, "Guide")); + ASSERT_EQ(t->def_line, 3); + ASSERT_EQ(t->flags & CBM_DOCLINK_FLAG_FILE, 0); + t = md_token(r, "../app/routing.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 7); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK); + /* paths under "Paths" */ + t = md_token(r, "app/routing.py#L4-L5"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 11); + ASSERT_TRUE(md_ends_with(t->source_qn, "Paths")); + ASSERT_NOT_NULL(md_token(r, "app/routing.py:1-9")); + t = md_token(r, "app/applications.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_PATH); + ASSERT_NOT_NULL(md_token(r, "app/middleware/")); + ASSERT_NOT_NULL(md_token(r, "/docs")); /* a token; the resolver calls it a route */ + /* an image, a URL: no tokens */ + ASSERT_NULL(md_token(r, "img/a.png")); + ASSERT_NULL(md_token(r, "https://example.org/a/b.py")); + ASSERT_NULL(md_token(r, "a/b.py")); + /* qualified names under "Names" */ + t = md_token(r, "app.routing.APIRouter.add"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_NAME); + ASSERT_EQ(t->line, 16); + ASSERT_TRUE(md_ends_with(t->source_qn, "Names")); + ASSERT_NOT_NULL(md_token(r, "pkg.Config.Name")); + ASSERT_NULL(md_token(r, "self.Name")); /* one identifier after `self.`: a bare name */ + ASSERT_NOT_NULL(md_token(r, "os.path.join")); + ASSERT_NOT_NULL(md_token(r, "main.py")); + /* fenced code and HTML comments are not scanned; HTML is */ + ASSERT_NULL(md_token(r, "app/fenced.py")); + ASSERT_NULL(md_token(r, "app/commented.py")); + t = md_token(r, "app/html.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 23); + cbm_free_result(r); + PASS(); +} + +/* Two headings with one title share one Section node, which owns only one of + * them: their references belong to the file, never to the other heading. */ +TEST(doc_links_md_repeated_headings) { + const char *src = "# Changes\n" /* 1 */ + "\n" /* 2 */ + "## Fixes\n" /* 3 */ + "\n" /* 4 */ + "See `app/one.py`.\n" /* 5 */ + "\n" /* 6 */ + "## Fixes\n" /* 7 */ + "\n" /* 8 */ + "See `app/two.py`.\n" /* 9 */ + "\n" /* 10 */ + "## Unique\n" /* 11 */ + "\n" /* 12 */ + "See `app/three.py`.\n"; /* 13 */ + CBMFileResult *r = dm_extract(src, CBM_LANG_MARKDOWN, "CHANGES.md"); + ASSERT_NOT_NULL(r); + const CBMDocLink *one = md_token(r, "app/one.py"); + const CBMDocLink *two = md_token(r, "app/two.py"); + const CBMDocLink *three = md_token(r, "app/three.py"); + ASSERT_NOT_NULL(one); + ASSERT_NOT_NULL(two); + ASSERT_NOT_NULL(three); + ASSERT_TRUE((one->flags & CBM_DOCLINK_FLAG_FILE) != 0); + ASSERT_TRUE((two->flags & CBM_DOCLINK_FLAG_FILE) != 0); + ASSERT_EQ(three->flags & CBM_DOCLINK_FLAG_FILE, 0); + ASSERT_TRUE(md_ends_with(three->source_qn, "Unique")); + cbm_free_result(r); + PASS(); +} + +typedef struct { + const char *text; + CBMDocLinkMdShape shape; + const char *path; + const char *member; + uint32_t first; + uint32_t last; + bool instance; + bool colon; +} md_case_t; + +TEST(doc_links_md_classify_span) { + static const md_case_t cases[] = { + {"src/app/main.go", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 0, 0, false, false}, + {"src/app/", CBM_DOCLINK_MD_DIR, "src/app/", NULL, 0, 0, false, false}, + {"src/app/main.go#L3-L9", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 3, 9, false, false}, + {"src/app/main.go:12", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 12, 12, false, false}, + {"src/lib.rs::Type::run", CBM_DOCLINK_MD_FILE, "src/lib.rs", "Type::run", 0, 0, false, + false}, + {"src\\win\\file.cs", CBM_DOCLINK_MD_FILE, "src/win/file.cs", NULL, 0, 0, false, false}, + {"main.go:7-8", CBM_DOCLINK_MD_FILENAME, "main.go", NULL, 7, 8, false, false}, + {"README.md", CBM_DOCLINK_MD_FILENAME, "README.md", NULL, 0, 0, false, false}, + {"Makefile", CBM_DOCLINK_MD_FILENAME, "Makefile", NULL, 0, 0, false, false}, + {"main.rs::run", CBM_DOCLINK_MD_FILENAME, "main.rs", "run", 0, 0, false, false}, + {"fastapi.routing.APIRouter", CBM_DOCLINK_MD_QUALIFIED, "fastapi.routing.APIRouter", NULL, + 0, 0, false, false}, + {"crate::extract::Json", CBM_DOCLINK_MD_QUALIFIED, "extract.Json", NULL, 0, 0, false, + false}, + {"Vec::new", CBM_DOCLINK_MD_QUALIFIED, "Vec.new", NULL, 0, 0, false, false}, + {"self.app.state", CBM_DOCLINK_MD_QUALIFIED, "app.state", NULL, 0, 0, true, false}, + {"pkg.module:attr", CBM_DOCLINK_MD_QUALIFIED, "pkg.module.attr", NULL, 0, 0, false, true}, + {"Foo#bar", CBM_DOCLINK_MD_QUALIFIED, "Foo.bar", NULL, 0, 0, false, false}, + {"App\\Http\\Kernel", CBM_DOCLINK_MD_QUALIFIED, "App.Http.Kernel", NULL, 0, 0, false, + false}, + {"server.New()", CBM_DOCLINK_MD_QUALIFIED, "server.New", NULL, 0, 0, false, false}, + /* no path and no qualified name */ + {"foo", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"self.x", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"Node.js", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"github.com/x/y", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"https://x.org/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"$HOME/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"PATH=/usr/bin", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"--verbose", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"go run main.go", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"\"a/b.py\"", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"pkg/Type.Member", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"@scope/pkg", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + const md_case_t *c = &cases[i]; + char buf[256]; + CBMDocLinkMdPath p; + bool ok = cbm_doclink_md_classify_span(c->text, strlen(c->text), buf, sizeof(buf), &p); + if (ok != (c->shape != CBM_DOCLINK_MD_NONE) || p.shape != c->shape) { + printf(" case %s: shape %d, expected %d\n", c->text, (int)p.shape, (int)c->shape); + FAIL("classification differs"); + } + if (!ok) { + continue; + } + if (strcmp(p.path, c->path) != 0 || + (c->member ? !p.member || strcmp(p.member, c->member) != 0 : p.member != NULL) || + p.first_line != c->first || p.last_line != c->last || p.instance != c->instance || + p.colon != c->colon) { + printf(" case %s: path %s member %s lines %u-%u instance %d colon %d\n", c->text, + p.path, p.member ? p.member : "(none)", p.first_line, p.last_line, p.instance, + p.colon); + FAIL("classification differs"); + } + } + /* a name never fits a buffer by being cut */ + char tiny[8]; + CBMDocLinkMdPath p; + ASSERT_FALSE(cbm_doclink_md_classify_span("src/app/main.go", 15, tiny, sizeof(tiny), &p)); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +/* Properties of the MENTIONS edge from the section or file whose qualified + * name ends with `src` to the node whose qualified name ends with `tgt`. */ +static int md_edge(const char *db, const char *src, const char *tgt, char *props, size_t cap) { + int n = 0; + dm_edge(db, src, tgt, props, cap, &n); + return n; +} + +TEST(doc_links_md_resolve) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + md_write_fixture(tmp); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + char reason[64]; + char syntax[64]; + + /* a link and a link reference definition: their files */ + ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"via\":\"markdown\"")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"link\"")); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":5")); + ASSERT_EQ(md_edge(db, "Guide", "routing.py.__file__", props, sizeof(props)), 1); + /* the reference before the first heading: from the File node */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.label = 'File' AND s.file_path = 'docs/guide.md' " + "AND t.file_path = 'pkg/config.go' AND t.label = 'File'"), + 1); + /* a range held by one definition: that definition, the range on the edge */ + ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"target_lines\":[4,5]")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_path\"")); + /* a range across definitions: the file, with the range */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Paths' AND t.label = 'File' " + "AND t.file_path = 'app/routing.py' " + "AND e.properties LIKE '%\"target_lines\":[1,9]%'"), + 1); + /* a bare path in prose; a directory */ + ASSERT_EQ(md_edge(db, "Paths", "applications.py.__file__", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"path\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Paths' AND t.label = 'Folder' " + "AND t.file_path = 'app/middleware'"), + 1); + /* a route, an image, a URL: neither edges nor rows */ + dm_row(db, "docs/guide.md", "/docs", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'MENTIONS' AND t.label = 'Folder' " + "AND t.file_path = 'docs'"), + 0); + dm_row(db, "docs/guide.md", "img/a.png", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + + /* qualified names: a method, a field through its type */ + ASSERT_EQ(md_edge(db, "Names", "routing.APIRouter.add", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_name\"")); + ASSERT_EQ(md_edge(db, "Names", "Config.Name", props, sizeof(props)), 1); + /* `app.middleware`: the package as written, not the method App.middleware */ + ASSERT_EQ(md_edge(db, "Names", "App.middleware", props, sizeof(props)), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Names' " + "AND t.file_path = 'app/middleware/__init__.py'"), + 1); + /* a field never through an instance */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'MENTIONS' AND t.label = 'Field' " + "AND e.properties LIKE '%\"line\":23%'"), + 0); + + /* unresolved, by reason */ + dm_row(db, "docs/guide.md", "app.Nope", reason, sizeof(reason), syntax, sizeof(syntax)); + ASSERT_STR_EQ(reason, "missing"); + ASSERT_STR_EQ(syntax, "code_name"); + dm_row(db, "docs/guide.md", "test_routing.only_in_tests", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "test_only_target"); + dm_row(db, "docs/guide.md", "main.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "ambiguous"); /* lib/main.py, but not next to the document */ + dm_row(db, "docs/guide.md", "../app/gone.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + dm_row(db, "docs/guide.md", "app/html.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + /* another project's code is no reference to this repository */ + dm_row(db, "docs/guide.md", "os.path.join", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + + /* back navigation: the sections that mention a method, one hop away */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND t.qualified_name LIKE '%.APIRouter.add' AND s.label = 'Section'"), + 2); + + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full ─────────────────────────────────────────── */ + +TEST(doc_links_md_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + md_write_fixture(repo); + char db[512]; + char full_db[512]; + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + + /* a body edit of a linked code file: its linking document re-resolves */ + th_write_file(TH_PATH(repo, "app/routing.py"), "class APIRouter:\n" + " prefix = ''\n" + "\n" + " def add(self, p):\n" + " return p + p\n" + "\n" + "\n" + "def helper():\n" + " return 2\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "code body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + /* the document itself changes: a new link, a reference gone */ + char guide[sizeof(MD_GUIDE) + 64]; + snprintf(guide, sizeof(guide), "%s\nSee also [helpers](../app/routing.py#L8-L9).\n", MD_GUIDE); + th_write_file(TH_PATH(repo, "docs/guide.md"), guide); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* the name a row calls missing is added: the run goes FULL and the row + * becomes an edge, exactly as a full index has it */ + th_write_file(TH_PATH(repo, "app/nope.py"), "def Nope():\n" + " return None\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "missing name added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_md) { + RUN_TEST(doc_links_md_extract_tokens); + RUN_TEST(doc_links_md_repeated_headings); + RUN_TEST(doc_links_md_classify_span); + RUN_TEST(doc_links_md_resolve); + RUN_TEST(doc_links_md_incremental); +} diff --git a/tests/test_main.c b/tests/test_main.c index dcbc1252c4..a4fa4a154f 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1821,6 +1821,7 @@ extern void suite_doclinks(void); extern void suite_doc_mentions(void); extern void suite_doc_mentions_cost(void); extern void suite_doc_mentions_msbuild(void); +extern void suite_doc_links_md(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2288,6 +2289,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions); RUN_SELECTED_SUITE(doc_mentions_cost); RUN_SELECTED_SUITE(doc_mentions_msbuild); + RUN_SELECTED_SUITE(doc_links_md); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 6b0761160dc932237d02fb1b1da44f2703bbf811 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 20:48:44 +0200 Subject: [PATCH 02/25] doc-links(adr): architecture decision records as ADR nodes with their facts and SUPERSEDES A Markdown file that is an architecture decision record now also defines an ADR node (File DEFINES ADR): its name is the record's canonical id ("ADR-12"; another prefix as written, "DEC-5"; a dated record keeps its file stem), its qualified name ".__adr__", and its properties carry the record's facts: adr_id, title, status (normalized: proposed, accepted, implemented, superseded, deprecated, rejected, abandoned, draft, unknown), status_text, date, deciders, superseded_by, and the decision itself as the docstring, so search finds a record by what it decided. Its sections link to code like any document's (MENTIONS, previous commit). Detection is per file, with the field tests' rules: a file in a conventional ADR directory (adr, adrs, decisions, decision-records, ...) whose name carries an id or whose text has ADR structure; any adr-NNN file; a numbered or dated file with ADR structure. Never README, index or template files without an id, never test fixtures. Facts come from front matter, then header fields (bullets, bold, tables), then the Status section; the date also from a "Last updated" field, the change log's first date and the file name. An ADR's own statement that it supersedes another ("Supersedes [ADR-3](...)", "Supersedes ADR-3"; "Replaces ADR-3" where it opens a line, never "replaces" as a verb of prose) becomes a SUPERSEDES edge between the two ADR nodes, by link or by id. "Superseded by" is written in the old record about another file; an edge from the other node would be owned by a file whose text does not hold it, so it stays a fact of the old record (status, superseded_by). The link layer gains a per-family edge type (doclink.h: the family list's "edge" column), so a family can produce SUPERSEDES instead of MENTIONS; edges are grouped by (type, source, target). Definitions gain a tail field for a document's facts (extra_props: key, value pairs), carried through result compaction and written by both property builders. Measured against the field-test harvester on five repositories: detection 182 of 182 records, the same files; title 182/182; status 179/182 (cosmos ADR-050 and its annexes say "Status: ARCHIVED", kept as status_text with status unknown); date 174/182 (semantic-kernel's "date: {2024-05-02}" is read as a date where the harvester dropped it). SUPERSEDES: semantic-kernel 0038 -> 0015, log4brains 20201016 -> 20200926, each checked against its sentence. Markdown MENTIONS unchanged on fastapi, curl, dapr, cosmos-sdk. Not covered: ADRs in reStructuredText, and ADR-log configuration files (.adr-dir, log4brains) as a detection source. Tests: doc_links_md_adr (detection and exclusions, facts from front matter and from a Status section, SUPERSEDES by link and by id, no edge from prose, the record's MENTIONS, incremental == full for a status edit and for an added record named by id). Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + internal/cbm/cbm.h | 6 + internal/cbm/doc_adr.c | 1352 +++++++++++++++++++++++++++++++ internal/cbm/doclink.c | 8 +- internal/cbm/doclink.h | 55 +- internal/cbm/doclink_md.c | 6 + internal/cbm/result_compact.c | 1 + src/pipeline/doc_links.c | 17 +- src/pipeline/doc_links_md.c | 93 ++- src/pipeline/pass_definitions.c | 4 + src/pipeline/pass_parallel.c | 4 + src/pipeline/pipeline.c | 3 +- tests/test_doc_links_md.c | 151 ++++ 13 files changed, 1675 insertions(+), 26 deletions(-) create mode 100644 internal/cbm/doc_adr.c diff --git a/Makefile.cbm b/Makefile.cbm index 4e365971be..3204c1566b 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -376,6 +376,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink.c \ $(CBM_DIR)/doclink_cs.c \ $(CBM_DIR)/doclink_md.c \ + $(CBM_DIR)/doc_adr.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 63265cebcd..561862b2ca 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -288,6 +288,12 @@ typedef struct { * are bound to the owning result's source identity before cross-file use. */ uint32_t test_name_start_byte, test_name_end_byte; uint32_t test_body_start_byte, test_body_end_byte; + /* Documents only: facts a document definition adds to its node's + * properties, as a NULL-terminated key, value, key, value, ... list (an + * ADR's id, status, date, deciders, title). Keys are fixed literals; the + * values are escaped where the properties are written. Tail field: + * zero-init stays valid. */ + const char **extra_props; } CBMDefinition; /* Argument captured from a call expression */ diff --git a/internal/cbm/doc_adr.c b/internal/cbm/doc_adr.c new file mode 100644 index 0000000000..6913e340dd --- /dev/null +++ b/internal/cbm/doc_adr.c @@ -0,0 +1,1352 @@ +/* + * doc_adr.c — architecture decision records: detection, facts, supersedes. + * + * A Markdown file that is an ADR gets one more definition: label "ADR", name + * its canonical id ("ADR-12"; another prefix as written, "DEC-5"; a dated + * record without a number keeps its file stem), qualified name + * ".__adr__", and the record's facts as node properties (adr_id, + * title, status, date, deciders, superseded_by). Its File DEFINES it like any + * definition, its sections link to code like any document's (doclink_md.c), + * and its own "supersedes X" / "replaces X" statements become tokens of the + * `supersedes` family, whose source is the ADR node: the resolver turns them + * into SUPERSEDES edges between ADR nodes. + * + * "Superseded by Y" is written in the OLD record, about another file: an edge + * from Y's node would be owned by a file whose text does not hold it, so it + * stays a fact of this record (status superseded, superseded_by) and the link + * itself is a MENTIONS edge of the status section. + * + * Detection (the field tests' rules, 99.2 % on 379 files): a file in a + * conventional ADR directory (adr, adrs, decisions, decision-records, ...) + * whose name carries an id or whose text has ADR structure; any `adr-NNN-*` + * file; a numbered or dated file with ADR structure. Never README, index or + * template files without an id, never test fixtures. Structure: a status, and + * a context, decision, consequences, decision outcome or considered options + * section. ADR-log configuration files (.adr-dir, log4brains) are not read: + * per-file detection, a pure function of the path and the text. + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "helpers.h" /* cbm_memmem */ + +#include +#include +#include +#include +#include + +enum { + ADR_ID_DIGITS = 5, /* an ADR number has at most this many digits */ + ADR_PREFIX_MIN = 2, /* `DEC-5`: a prefix of 2 ... */ + ADR_PREFIX_MAX = 12, /* ... to 12 letters */ + ADR_DOC_MAX = 500, /* the node's docstring: the decision, collapsed */ + ADR_FIELD_MAX = 256, /* a fact value is cut at a character boundary */ + ADR_FM_SCAN = 400, /* front matter ends within this many lines */ + ADR_HEAD_SCAN = 40, /* header fields within this many lines when there is no heading */ + ADR_NAME_MAX = 64, /* canonical id buffer */ + ADR_YEAR_MIN = 1900, + ADR_YEAR_MAX = 2099, + ADR_MONTHS = 12, + ADR_DAYS = 31, + ADR_YEAR_DIGITS = 4, + ADR_MD_DIGITS = 2, + ADR_DATE8 = 8, + ADR_ISO_LEN = 10, /* yyyy-mm-dd */ + ADR_MONTH_ABBR = 3, + ADR_BOM_LEN = 3, /* EF BB BF */ + ADR_RELATION_SPAN = 400, /* a relation's targets follow it in this many bytes */ +}; + +/* ── Text helpers ────────────────────────────────────────────────── */ + +static bool adr_blank(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +static bool adr_digit(char c) { + return c >= '0' && c <= '9'; +} + +static bool adr_alpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static char adr_lower(char c) { + return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; +} + +/* s[0..n) starts with `word`, ignoring case. */ +static bool adr_starts_ci(const char *s, size_t n, const char *word) { + size_t w = strlen(word); + if (n < w) { + return false; + } + for (size_t i = 0; i < w; i++) { + if (adr_lower(s[i]) != word[i]) { + return false; + } + } + return true; +} + +static bool adr_eq_ci(const char *s, size_t n, const char *word) { + return n == strlen(word) && adr_starts_ci(s, n, word); +} + +/* Trim blanks and Markdown emphasis/code marks around a value. */ +static void adr_trim(const char **s, size_t *n) { + const char *p = *s; + size_t len = *n; + while (len > 0 && (adr_blank(*p) || *p == '*' || *p == '_' || *p == '`' || *p == '"')) { + p++; + len--; + } + while (len > 0 && + (adr_blank(p[len - SKIP_ONE]) || p[len - SKIP_ONE] == '*' || p[len - SKIP_ONE] == '_' || + p[len - SKIP_ONE] == '`' || p[len - SKIP_ONE] == '"')) { + len--; + } + *s = p; + *n = len; +} + +/* A copy of s[0..n) with Markdown links reduced to their text, `**`, `__` + * and backticks dropped, blanks collapsed, cut at `cap` bytes on a UTF-8 + * boundary. NULL when nothing is left. */ +static char *adr_plain(CBMArena *a, const char *s, size_t n, size_t cap) { + char *out = (char *)cbm_arena_alloc(a, (n < cap ? n : cap) + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + bool space = false; + for (size_t i = 0; i < n && w < cap; i++) { + char c = s[i]; + if (c == '[') { + continue; /* [text](target): keep the text */ + } + if (c == ']' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '(') { + const char *close = memchr(s + i, ')', n - i); + if (close) { + i = (size_t)(close - s); + continue; + } + } + if (c == '*' || c == '`' || (c == '_' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '_')) { + continue; + } + if (adr_blank(c) || c == '\n') { + space = w > 0; + continue; + } + if (space && w < cap) { + out[w++] = ' '; + } + space = false; + if (w < cap) { + out[w++] = c; + } + } + while (w > 0 && ((unsigned char)out[w - SKIP_ONE] & 0xC0) == 0x80) { + w--; /* never end inside a character */ + } + if (w > 0 && ((unsigned char)out[w - SKIP_ONE] & 0xC0) == 0xC0) { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +typedef struct { + const char *s; /* the line, not NUL-terminated */ + int n; + bool text; /* not front matter, not fenced code, not an HTML comment */ +} adr_line_t; + +typedef struct { + adr_line_t *v; + int count; + int fm_end; /* the first line after the front matter (0: none) */ +} adr_lines_t; + +static bool adr_trim_eq(const char *s, int n, const char *word) { + const char *p = s; + size_t len = (size_t)n; + while (len > 0 && adr_blank(*p)) { + p++; + len--; + } + while (len > 0 && adr_blank(p[len - SKIP_ONE])) { + len--; + } + return len == strlen(word) && memcmp(p, word, len) == 0; +} + +static bool adr_split_lines(CBMArena *a, const char *src, int len, adr_lines_t *out) { + int count = 0; + for (int i = 0; i < len; i++) { + count += src[i] == '\n'; + } + count++; + out->v = (adr_line_t *)cbm_arena_alloc(a, (size_t)count * sizeof(*out->v)); + if (!out->v) { + return false; + } + out->count = 0; + out->fm_end = 0; + int pos = 0; + if (len >= ADR_BOM_LEN && memcmp(src, "\xEF\xBB\xBF", ADR_BOM_LEN) == 0) { + pos = ADR_BOM_LEN; /* a byte-order mark is no part of the first line */ + } + while (pos <= len && out->count < count) { + const char *nl = pos < len ? memchr(src + pos, '\n', (size_t)(len - pos)) : NULL; + int end = nl ? (int)(nl - src) : len; + int n = end - pos; + if (n > 0 && src[pos + n - SKIP_ONE] == '\r') { + n--; + } + out->v[out->count++] = (adr_line_t){.s = src + pos, .n = n, .text = true}; + pos = end + SKIP_ONE; + } + /* front matter */ + if (out->count > 0 && adr_trim_eq(out->v[0].s, out->v[0].n, "---")) { + for (int k = 1; k < out->count && k < ADR_FM_SCAN; k++) { + if (adr_trim_eq(out->v[k].s, out->v[k].n, "---") || + adr_trim_eq(out->v[k].s, out->v[k].n, "...")) { + out->fm_end = k + SKIP_ONE; + break; + } + } + } + for (int k = 0; k < out->fm_end; k++) { + out->v[k].text = false; + } + /* fences and HTML comments */ + char fence = 0; + int fence_len = 0; + bool comment = false; + for (int k = out->fm_end; k < out->count; k++) { + adr_line_t *l = &out->v[k]; + int i = 0; + while (i < l->n && adr_blank(l->s[i])) { + i++; + } + int run = 0; + while (i + run < l->n && (l->s[i + run] == '`' || l->s[i + run] == '~') && + l->s[i + run] == l->s[i]) { + run++; + } + if (fence) { + l->text = false; + if (run >= fence_len && l->s[i] == fence) { + fence = 0; + } + continue; + } + if (run >= PAIR_LEN + SKIP_ONE) { + fence = l->s[i]; + fence_len = run; + l->text = false; + continue; + } + if (comment) { + l->text = false; + if (cbm_memmem(l->s, (size_t)l->n, "-->", PAIR_LEN + SKIP_ONE)) { + comment = false; + } + continue; + } + if (l->n - i >= 4 && memcmp(l->s + i, "", PAIR_LEN + SKIP_ONE)) { + comment = true; + l->text = false; + } + } + return true; +} + +/* ── Front matter (top-level scalars and lists) ──────────────────── */ + +/* The value of front-matter key `key` (a list joined with ", "), or NULL. */ +static char *adr_fm_value(CBMArena *a, const adr_lines_t *L, const char *key) { + size_t kl = strlen(key); + for (int k = SKIP_ONE; k < L->fm_end - SKIP_ONE; k++) { + const adr_line_t *l = &L->v[k]; + if ((size_t)l->n <= kl || l->s[0] == ' ' || l->s[0] == '\t' || + !adr_starts_ci(l->s, (size_t)l->n, key) || l->s[kl] != ':') { + continue; + } + const char *v = l->s + kl + SKIP_ONE; + size_t vn = (size_t)l->n - kl - SKIP_ONE; + adr_trim(&v, &vn); + if (vn > 0 && v[0] == '[' && v[vn - SKIP_ONE] == ']') { + v++; + vn -= PAIR_LEN; + } + if (vn > 0) { + return adr_plain(a, v, vn, ADR_FIELD_MAX); + } + /* a block list on the following lines */ + char buf[ADR_FIELD_MAX + SKIP_ONE]; + size_t w = 0; + for (int j = k + SKIP_ONE; j < L->fm_end - SKIP_ONE; j++) { + const adr_line_t *it = &L->v[j]; + int i = 0; + while (i < it->n && adr_blank(it->s[i])) { + i++; + } + if (i == 0 || i >= it->n || it->s[i] != '-') { + break; + } + const char *iv = it->s + i + SKIP_ONE; + size_t ivn = (size_t)(it->n - i - SKIP_ONE); + adr_trim(&iv, &ivn); + if (ivn > 0 && w + ivn + PAIR_LEN < sizeof(buf)) { + if (w > 0) { + buf[w++] = ','; + buf[w++] = ' '; + } + memcpy(buf + w, iv, ivn); + w += ivn; + } + } + return w > 0 ? adr_plain(a, buf, w, ADR_FIELD_MAX) : NULL; + } + return NULL; +} + +/* ── Headings (the file's Section definitions) ───────────────────── */ + +typedef struct { + int line; /* 0-based index into the lines */ + int level; /* 1..6 */ + const char *title; + size_t title_n; +} adr_head_t; + +static int adr_head_cmp(const void *a, const void *b) { + const adr_head_t *x = (const adr_head_t *)a; + const adr_head_t *y = (const adr_head_t *)b; + return (x->line > y->line) - (x->line < y->line); +} + +/* The level of the heading on line k: `#` count, or 1/2 for a setext + * underline of `=`/`-` on the next line; 0 when the line is no heading. */ +static int adr_heading_level(const adr_lines_t *L, int k, const char **title, size_t *title_n) { + const adr_line_t *l = &L->v[k]; + int i = 0; + while (i < l->n && i < (PAIR_LEN + SKIP_ONE) && l->s[i] == ' ') { + i++; + } + int hashes = 0; + while (i + hashes < l->n && l->s[i + hashes] == '#') { + hashes++; + } + if (hashes > 0 && hashes <= (PAIR_LEN * PAIR_LEN + PAIR_LEN) && + (i + hashes == l->n || adr_blank(l->s[i + hashes]))) { + const char *t = l->s + i + hashes; + size_t tn = (size_t)(l->n - i - hashes); + while (tn > 0 && (t[tn - SKIP_ONE] == '#' || adr_blank(t[tn - SKIP_ONE]))) { + tn--; + } + adr_trim(&t, &tn); + *title = t; + *title_n = tn; + return hashes; + } + if (k + SKIP_ONE < L->count) { + const adr_line_t *u = &L->v[k + SKIP_ONE]; + int j = 0; + while (j < u->n && u->s[j] == ' ') { + j++; + } + char c = j < u->n ? u->s[j] : 0; + int run = 0; + while (j + run < u->n && u->s[j + run] == c) { + run++; + } + bool rest_blank = true; + for (int r = j + run; r < u->n; r++) { + rest_blank = rest_blank && adr_blank(u->s[r]); + } + if ((c == '=' || c == '-') && run >= SKIP_ONE && rest_blank) { + const char *t = l->s; + size_t tn = (size_t)l->n; + adr_trim(&t, &tn); + *title = t; + *title_n = tn; + return c == '=' ? SKIP_ONE : PAIR_LEN; + } + } + return 0; +} + +/* Section kinds the facts and the detection read. */ +typedef enum { + ADR_SEC_OTHER = 0, + ADR_SEC_STATUS, + ADR_SEC_CONTEXT, + ADR_SEC_DECISION, + ADR_SEC_OUTCOME, + ADR_SEC_OPTIONS, + ADR_SEC_CONSEQUENCES, +} adr_sec_t; + +static adr_sec_t adr_section_kind(const char *t, size_t n) { + /* a leading "1." / "2.3 " numbering is no part of the title */ + size_t i = 0; + while (i < n && (adr_digit(t[i]) || t[i] == '.')) { + i++; + } + if (i > 0 && i < n && t[i] == ' ') { + t += i + SKIP_ONE; + n -= i + SKIP_ONE; + } + while (n > 0 && (t[n - SKIP_ONE] == ':' || adr_blank(t[n - SKIP_ONE]))) { + n--; + } + if (adr_eq_ci(t, n, "status") || adr_starts_ci(t, n, "status:")) { + return ADR_SEC_STATUS; + } + if (adr_eq_ci(t, n, "context") || adr_starts_ci(t, n, "context and problem") || + adr_eq_ci(t, n, "problem") || adr_eq_ci(t, n, "problem statement") || + adr_eq_ci(t, n, "background") || adr_eq_ci(t, n, "motivation")) { + return ADR_SEC_CONTEXT; + } + if (adr_eq_ci(t, n, "decision outcome")) { + return ADR_SEC_OUTCOME; + } + if (adr_eq_ci(t, n, "decision") || adr_eq_ci(t, n, "decisions") || + adr_eq_ci(t, n, "the decision")) { + return ADR_SEC_DECISION; + } + if (adr_eq_ci(t, n, "considered options") || adr_eq_ci(t, n, "options considered") || + adr_eq_ci(t, n, "alternatives") || adr_eq_ci(t, n, "alternatives considered")) { + return ADR_SEC_OPTIONS; + } + if (adr_eq_ci(t, n, "consequences") || adr_starts_ci(t, n, "positive consequences") || + adr_starts_ci(t, n, "negative consequences")) { + return ADR_SEC_CONSEQUENCES; + } + return ADR_SEC_OTHER; +} + +/* ── File names ──────────────────────────────────────────────────── */ + +typedef enum { + ADR_ID_NONE = 0, + ADR_ID_ADR, /* adr-012-title.md */ + ADR_ID_NUMBER, /* 0012-title.md */ + ADR_ID_PREFIX, /* DEC-005-title.md */ + ADR_ID_DATE, /* 2023-04-05-title.md, 20230405-title.md */ +} adr_id_kind_t; + +typedef struct { + adr_id_kind_t kind; + unsigned number; + char prefix[ADR_PREFIX_MAX + SKIP_ONE]; + char date[ADR_ISO_LEN + SKIP_ONE]; +} adr_id_t; + +/* Digits at s, at most ADR_ID_DIGITS, followed by a separator or the end. */ +static bool adr_number_at(const char *s, size_t n, size_t *used, unsigned *value) { + size_t i = 0; + unsigned v = 0; + while (i < n && adr_digit(s[i])) { + if (i >= ADR_ID_DIGITS + ADR_ID_DIGITS) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (unsigned)(s[i] - '0'); + i++; + } + size_t significant = i; + for (size_t z = 0; z < i && s[z] == '0' && significant > SKIP_ONE; z++) { + significant--; + } + if (i == 0 || significant > ADR_ID_DIGITS) { + return false; + } + if (i < n && s[i] != '-' && s[i] != '_' && s[i] != ' ' && s[i] != '.') { + return false; + } + *used = i; + *value = v; + return true; +} + +static bool adr_date_prefix(const char *s, size_t n, char *iso) { + /* yyyy-mm-dd or yyyymmdd, then '-' or '_' */ + if (n > ADR_ISO_LEN && adr_digit(s[0]) && s[4] == '-' && s[7] == '-' && + (s[ADR_ISO_LEN] == '-' || s[ADR_ISO_LEN] == '_')) { + for (int i = 0; i < ADR_ISO_LEN; i++) { + if (i != 4 && i != 7 && !adr_digit(s[i])) { + return false; + } + } + memcpy(iso, s, ADR_ISO_LEN); + iso[ADR_ISO_LEN] = '\0'; + return (s[0] == '1' && s[1] == '9') || (s[0] == '2' && s[1] == '0'); + } + if (n > ADR_DATE8 && (s[ADR_DATE8] == '-' || s[ADR_DATE8] == '_')) { + for (int i = 0; i < ADR_DATE8; i++) { + if (!adr_digit(s[i])) { + return false; + } + } + snprintf(iso, ADR_ISO_LEN + SKIP_ONE, "%.4s-%.2s-%.2s", s, s + 4, s + 6); + return (s[0] == '1' && s[1] == '9') || (s[0] == '2' && s[1] == '0'); + } + return false; +} + +static void adr_filename_id(const char *base, adr_id_t *id) { + memset(id, 0, sizeof(*id)); + size_t n = strlen(base); + size_t used = 0; + unsigned v = 0; + if (adr_starts_ci(base, n, "adr")) { + size_t i = PAIR_LEN + SKIP_ONE; + if (i < n && (base[i] == '-' || base[i] == '_' || base[i] == ' ')) { + i++; + } + if (adr_number_at(base + i, n - i, &used, &v)) { + id->kind = ADR_ID_ADR; + id->number = v; + return; + } + } + if (adr_date_prefix(base, n, id->date)) { + id->kind = ADR_ID_DATE; + return; + } + if (adr_number_at(base, n, &used, &v) && used < n) { + id->kind = ADR_ID_NUMBER; + id->number = v; + return; + } + size_t p = 0; + while (p < n && adr_alpha(base[p])) { + p++; + } + if (p >= ADR_PREFIX_MIN && p <= ADR_PREFIX_MAX && p < n && (base[p] == '-' || base[p] == '_') && + adr_number_at(base + p + SKIP_ONE, n - p - SKIP_ONE, &used, &v) && + p + SKIP_ONE + used < n) { + id->kind = ADR_ID_PREFIX; + id->number = v; + for (size_t i = 0; i < p; i++) { + id->prefix[i] = + (base[i] >= 'a' && base[i] <= 'z') ? (char)(base[i] - 'a' + 'A') : base[i]; + } + id->prefix[p] = '\0'; + } +} + +static const char *const ADR_CONV_DIRS[] = {"adr", + "adrs", + "decisions", + "decision-records", + "decision_records", + "architecture-decisions", + "architecture_decisions", + "architectural-decisions", + "architecture-decision-records", + "decision-log", + NULL}; + +static const char *const ADR_FIXTURE_DIRS[] = { + "test", "tests", "__tests__", "integration-tests", "integration_tests", + "e2e-tests", "e2e_tests", "testdata", "test-data", "test_data", + "fixture", "fixtures", "__fixtures__", "__mocks__", "mock", + "mocks", NULL}; + +static const char *const ADR_NON_ADR_STEMS[] = {"readme", "index", "_index", "toc", + "summary", "contributing", "process", "changelog", + "glossary", "license", NULL}; + +static bool adr_in(const char *s, size_t n, const char *const *list) { + for (size_t i = 0; list[i]; i++) { + if (adr_eq_ci(s, n, list[i])) { + return true; + } + } + return false; +} + +/* Does a directory of `rel` match the list? */ +static bool adr_dir_in(const char *rel, const char *const *list) { + const char *s = rel; + for (const char *slash = strchr(s, '/'); slash; slash = strchr(s, '/')) { + if (adr_in(s, (size_t)(slash - s), list)) { + return true; + } + s = slash + SKIP_ONE; + } + return false; +} + +/* README, index, template ... (only for a name without an id). */ +static bool adr_non_adr_name(const char *base) { + size_t n = strlen(base); + const char *dot = strchr(base, '.'); + size_t stem = dot ? (size_t)(dot - base) : n; + if (adr_in(base, stem, ADR_NON_ADR_STEMS)) { + return true; + } + for (size_t i = 0; i + strlen("template") <= n; i++) { + bool start = i == 0 || base[i - SKIP_ONE] == '-' || base[i - SKIP_ONE] == '_' || + base[i - SKIP_ONE] == '.'; + size_t e = i + strlen("template"); + bool end = e == n || base[e] == '-' || base[e] == '_' || base[e] == '.'; + if (start && end && adr_starts_ci(base + i, n - i, "template")) { + return true; + } + } + return false; +} + +/* ── Status, dates, people ───────────────────────────────────────── */ + +static const struct { + const char *status; + const char *words[8]; +} ADR_STATUS_WORDS[] = { + {"superseded", {"supersed", "replaced by", NULL}}, + {"deprecated", {"deprecat", "obsolete", NULL}}, + {"rejected", {"reject", "declined", NULL}}, + {"abandoned", {"abandon", "withdrawn", "cancelled", "canceled", NULL}}, + {"implemented", {"implemented", "done", "completed", "complete", NULL}}, + {"accepted", {"accept", "approved", "adopted", "agreed", "decided", "active", "final", NULL}}, + {"proposed", + {"propos", "in review", "under review", "pending", "rfc", "under consideration", NULL}}, + {"draft", {"draft", "wip", "work in progress", "in progress", NULL}}, +}; + +/* The normalized status of a status text: the first status word (at a word + * start, not after "not ", "non-" or "un"), "unknown" when none. */ +static const char *adr_norm_status(const char *s) { + size_t n = strlen(s); + for (size_t i = 0; i < n; i++) { + if (i > 0 && (adr_alpha(s[i - SKIP_ONE]) || adr_digit(s[i - SKIP_ONE]))) { + continue; + } + bool negated = (i >= 4 && adr_starts_ci(s + i - 4, 4, "not ")) || + (i >= 4 && adr_starts_ci(s + i - 4, 4, "non-")); + for (size_t k = 0; k < sizeof(ADR_STATUS_WORDS) / sizeof(ADR_STATUS_WORDS[0]); k++) { + for (size_t w = 0; ADR_STATUS_WORDS[k].words[w]; w++) { + /* a whole word, or a stem ("accept" in "accepted") */ + if (adr_starts_ci(s + i, n - i, ADR_STATUS_WORDS[k].words[w]) && !negated && + !(i >= PAIR_LEN && adr_starts_ci(s + i - PAIR_LEN, PAIR_LEN, "un"))) { + return ADR_STATUS_WORDS[k].status; + } + } + } + } + return "unknown"; +} + +static const char *const ADR_MONTHS_ABBR[ADR_MONTHS] = {"jan", "feb", "mar", "apr", "may", "jun", + "jul", "aug", "sep", "oct", "nov", "dec"}; + +static int adr_month(const char *s, size_t n) { + if (n < ADR_MONTH_ABBR) { + return 0; + } + for (int m = 0; m < ADR_MONTHS; m++) { + if (adr_starts_ci(s, n, ADR_MONTHS_ABBR[m])) { + return m + SKIP_ONE; + } + } + return 0; +} + +static int adr_num(const char *s, size_t n, size_t *used) { + size_t i = 0; + int v = 0; + while (i < n && adr_digit(s[i]) && i < ADR_YEAR_DIGITS + SKIP_ONE) { + v = (v * CBM_DECIMAL_BASE) + (s[i] - '0'); + i++; + } + *used = i; + return v; +} + +/* An explicit date in s: yyyy-mm-dd (or / .), yyyymmdd, "d Month yyyy", + * "Month d, yyyy", or "Month yyyy" (month precision). Ambiguous numeric + * day/month orders give nothing. */ +static bool adr_parse_date(const char *s, char *out, size_t cap) { + size_t n = strlen(s); + for (size_t i = 0; i < n; i++) { + if (i > 0 && (adr_digit(s[i - SKIP_ONE]) || adr_alpha(s[i - SKIP_ONE]))) { + continue; + } + size_t u = 0; + int y = adr_num(s + i, n - i, &u); + if (u == ADR_YEAR_DIGITS && y >= ADR_YEAR_MIN && y <= ADR_YEAR_MAX && i + u < n && + (s[i + u] == '-' || s[i + u] == '/' || s[i + u] == '.')) { + size_t u2 = 0; + int mo = adr_num(s + i + u + SKIP_ONE, n - i - u - SKIP_ONE, &u2); + size_t at = i + u + SKIP_ONE + u2; + if (u2 >= SKIP_ONE && u2 <= ADR_MD_DIGITS && at < n && s[at] == s[i + u]) { + size_t u3 = 0; + int d = adr_num(s + at + SKIP_ONE, n - at - SKIP_ONE, &u3); + if (u3 >= SKIP_ONE && u3 <= ADR_MD_DIGITS && mo >= 1 && mo <= ADR_MONTHS && + d >= 1 && d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", y, mo, d); + return true; + } + } + } + /* yyyy Month d: "2019 Jul 31" */ + if (u == ADR_YEAR_DIGITS && y >= ADR_YEAR_MIN && y <= ADR_YEAR_MAX && i + u < n && + (s[i + u] == ' ' || s[i + u] == ',')) { + const char *p = s + i + u; + while (p < s + n && (*p == ' ' || *p == ',')) { + p++; + } + int mo = adr_month(p, (size_t)(s + n - p)); + if (mo) { + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ' ' || *p == ',')) { + p++; + } + size_t ud = 0; + int d = adr_num(p, (size_t)(s + n - p), &ud); + if (ud >= SKIP_ONE && ud <= ADR_MD_DIGITS && d >= 1 && d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", y, mo, d); + return true; + } + } + } + /* a/b/yyyy, a.b.yyyy: only when one of a, b is above 12 */ + if (u >= SKIP_ONE && u <= ADR_MD_DIGITS && i + u < n && + (s[i + u] == '/' || s[i + u] == '.')) { + char sep = s[i + u]; + size_t u2 = 0; + int b = adr_num(s + i + u + SKIP_ONE, n - i - u - SKIP_ONE, &u2); + size_t at = i + u + SKIP_ONE + u2; + size_t u3 = 0; + int yy = (u2 >= SKIP_ONE && u2 <= ADR_MD_DIGITS && at < n && s[at] == sep) + ? adr_num(s + at + SKIP_ONE, n - at - SKIP_ONE, &u3) + : 0; + if (u3 == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX) { + int mo = 0; + int d = 0; + if (y > ADR_MONTHS && b >= 1 && b <= ADR_MONTHS && y <= ADR_DAYS) { + d = y; + mo = b; + } else if (b > ADR_MONTHS && y >= 1 && y <= ADR_MONTHS && b <= ADR_DAYS) { + mo = y; + d = b; + } + if (mo) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } + } + if (u == ADR_DATE8) { + int yy = y / 10000; + int mo = (y / 100) % 100; + int d = y % 100; + if (yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && mo >= 1 && mo <= ADR_MONTHS && d >= 1 && + d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } + /* d Month yyyy */ + if (u >= SKIP_ONE && u <= ADR_MD_DIGITS && i + u < n && s[i + u] == ' ') { + const char *m = s + i + u + SKIP_ONE; + int mo = adr_month(m, n - (size_t)(m - s)); + if (mo) { + const char *p = m; + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ',' || *p == ' ')) { + p++; + } + size_t uy = 0; + int yy = adr_num(p, (size_t)(s + n - p), &uy); + if (uy == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && y >= 1 && + y <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, y); + return true; + } + } + } + /* Month d, yyyy / Month yyyy */ + int mo = u == 0 ? adr_month(s + i, n - i) : 0; + if (mo) { + const char *p = s + i; + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ' ')) { + p++; + } + size_t ud = 0; + int d = adr_num(p, (size_t)(s + n - p), &ud); + if (ud >= SKIP_ONE && ud <= ADR_MD_DIGITS) { + const char *q = p + ud; + while (q < s + n && (adr_alpha(*q) || *q == ',' || *q == ' ')) { + q++; /* "5th, " */ + } + size_t uy = 0; + int yy = adr_num(q, (size_t)(s + n - q), &uy); + if (uy == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && d >= 1 && + d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } else if (ud == ADR_YEAR_DIGITS && d >= ADR_YEAR_MIN && d <= ADR_YEAR_MAX) { + snprintf(out, cap, "%04d-%02d", d, mo); + return true; + } + } + } + return false; +} + +/* ── Header fields ───────────────────────────────────────────────── */ + +/* `Status: Accepted`, `* Status: Accepted`, `**Status:** Accepted`, + * `| Status | Accepted |`: the value of `key` on line l, or NULL. */ +static char *adr_field(CBMArena *a, const adr_line_t *l, const char *key) { + char buf[ADR_FIELD_MAX * PAIR_LEN]; + size_t w = 0; + for (int i = 0; i < l->n && w + SKIP_ONE < sizeof(buf); i++) { + if (l->s[i] == '*' || (l->s[i] == '_' && i + SKIP_ONE < l->n && l->s[i + 1] == '_')) { + continue; /* emphasis around the key */ + } + buf[w++] = l->s[i]; + } + buf[w] = '\0'; + const char *p = buf; + while (*p && adr_blank(*p)) { + p++; + } + bool table = *p == '|'; + if (table) { + p++; + while (*p && adr_blank(*p)) { + p++; + } + } else if ((*p == '-' || *p == '+') && adr_blank(p[SKIP_ONE])) { + p += PAIR_LEN; + while (*p && adr_blank(*p)) { + p++; + } + } + size_t kl = strlen(key); + if (!adr_starts_ci(p, strlen(p), key)) { + return NULL; + } + p += kl; + while (*p && adr_blank(*p)) { + p++; + } + if (*p != (table ? '|' : ':')) { + return NULL; + } + p++; + const char *end = table ? strchr(p, '|') : NULL; + size_t vn = end ? (size_t)(end - p) : strlen(p); + adr_trim(&p, &vn); + return vn > 0 ? adr_plain(a, p, vn, ADR_FIELD_MAX) : NULL; +} + +/* ── The record ──────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + adr_lines_t L; + adr_head_t *heads; + int nheads; + int title_head; /* index into heads, or -1 */ +} adr_doc_t; + +/* The paragraph after heading h (its first non-blank text lines). */ +static char *adr_first_paragraph(CBMArena *a, const adr_doc_t *d, int h, size_t cap) { + int start = d->heads[h].line + SKIP_ONE; + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + int k = start; + while (k < end && + (!d->L.v[k].text || d->L.v[k].n == 0 || adr_trim_eq(d->L.v[k].s, d->L.v[k].n, "") || + d->L.v[k].s[0] == '=' || d->L.v[k].s[0] == '-')) { + k++; + } + if (k >= end) { + return NULL; + } + const char *s = d->L.v[k].s; + int last = k; + while (last + SKIP_ONE < end && d->L.v[last + SKIP_ONE].text && + !adr_trim_eq(d->L.v[last + SKIP_ONE].s, d->L.v[last + SKIP_ONE].n, "")) { + last++; + } + const char *e = d->L.v[last].s + d->L.v[last].n; + return adr_plain(a, s, (size_t)(e - s), cap); +} + +/* The text of section h, as one collapsed string. */ +static char *adr_section_text(CBMArena *a, const adr_doc_t *d, int h, size_t cap) { + int start = d->heads[h].line + SKIP_ONE; + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + if (start >= end) { + return NULL; + } + const char *s = d->L.v[start].s; + const char *e = d->L.v[end - SKIP_ONE].s + d->L.v[end - SKIP_ONE].n; + return e > s ? adr_plain(a, s, (size_t)(e - s), cap) : NULL; +} + +static int adr_find_section(const adr_doc_t *d, adr_sec_t kind) { + for (int h = 0; h < d->nheads; h++) { + if (adr_section_kind(d->heads[h].title, d->heads[h].title_n) == kind) { + return h; + } + } + return CBM_NOT_FOUND; +} + +/* The value of header field `key` between the title and the first body + * heading (the first lines when there is no heading). */ +static char *adr_header_field(CBMArena *a, const adr_doc_t *d, const char *key) { + int from = d->title_head >= 0 ? d->heads[d->title_head].line + SKIP_ONE : d->L.fm_end; + int to = d->L.count; + for (int h = 0; h < d->nheads; h++) { + if (h != d->title_head && d->heads[h].line >= from) { + to = d->heads[h].line; + break; + } + } + if (to == d->L.count && to - from > ADR_HEAD_SCAN) { + to = from + ADR_HEAD_SCAN; + } + for (int k = from; k < to; k++) { + if (!d->L.v[k].text) { + continue; + } + char *v = adr_field(a, &d->L.v[k], key); + if (v) { + return v; + } + } + return NULL; +} + +/* ── Relations ───────────────────────────────────────────────────── */ + +/* `ADR-12`, `ADR 012`, `adr_7`: the number, and the reference's length. */ +static bool adr_id_ref(const char *s, size_t n, unsigned *number, size_t *len) { + if (n < PAIR_LEN * PAIR_LEN || !adr_starts_ci(s, n, "adr") || + (s[3] != '-' && s[3] != '_' && s[3] != ' ')) { + return false; + } + size_t used = 0; + unsigned v = 0; + size_t i = PAIR_LEN * PAIR_LEN; + size_t d = 0; + while (i + d < n && adr_digit(s[i + d])) { + d++; + } + if (d == 0 || (i + d < n && (adr_alpha(s[i + d]) || s[i + d] == '_'))) { + return false; + } + if (!adr_number_at(s + i, d, &used, &v)) { + return false; + } + *number = v; + *len = i + d; + return true; +} + +static void adr_push_relation(adr_doc_t *d, const char *adr_qn, const char *raw, size_t n, + uint32_t line) { + CBMArena *a = d->ctx->arena; + char *text = cbm_arena_strndup(a, raw, n); + if (!text) { + d->ctx->result->doc_links.failed = true; + return; + } + CBMDocLink link = {.source_qn = adr_qn, + .raw = text, + .line = line, + .def_line = SKIP_ONE, + .syntax = (uint16_t)CBM_DOCLINK_MD_SUPERSEDES, + .flags = 0}; + cbm_doclinks_push(&d->ctx->result->doc_links, a, link); +} + +/* The targets after a relation phrase, up to the end of its sentence: link + * destinations and ADR ids. Forward relations become tokens; for a backward + * one ("superseded by") the targets are joined into `*joined`. */ +static void adr_targets(adr_doc_t *d, const char *adr_qn, const char *s, size_t n, uint32_t line, + bool forward, char *joined, size_t jcap) { + size_t end = n < ADR_RELATION_SPAN ? n : ADR_RELATION_SPAN; + for (size_t i = 0; i + SKIP_ONE < end; i++) { + if ((s[i] == '.' && (i + SKIP_ONE == end || adr_blank(s[i + SKIP_ONE])) && + !(i > 0 && adr_digit(s[i - SKIP_ONE]))) || + s[i] == ';') { + end = i; + break; + } + } + for (size_t i = 0; i < end; i++) { + const char *target = NULL; + size_t tn = 0; + const char *rb = s[i] == '[' ? memchr(s + i, ']', end - i) : NULL; + if (rb && rb + SKIP_ONE < s + end && rb[SKIP_ONE] == '(') { + /* [text](destination): one reference, the destination; ids in + * the link text name the same record */ + const char *dest = rb + PAIR_LEN; + const char *close = memchr(dest, ')', (size_t)(s + end - dest)); + if (close) { + target = dest; + tn = (size_t)(close - dest); + i = (size_t)(close - s); + } + } else { + unsigned num = 0; + size_t len = 0; + if ((i == 0 || !adr_alpha(s[i - SKIP_ONE])) && adr_id_ref(s + i, end - i, &num, &len)) { + target = s + i; + tn = len; + i += len - SKIP_ONE; + } + } + if (!target || tn == 0) { + continue; + } + if (forward) { + adr_push_relation(d, adr_qn, target, tn, line); + } else if (joined) { + size_t w = strlen(joined); + bool seen = false; + for (const char *p = joined; *p && !seen; p++) { + seen = strncmp(p, target, tn) == 0 && (p[tn] == '\0' || p[tn] == ',') && + (p == joined || p[-SKIP_ONE] == ' '); + } + if (!seen && w + tn + PAIR_LEN < jcap) { + if (w > 0) { + joined[w++] = ','; + joined[w++] = ' '; + } + memcpy(joined + w, target, tn); + joined[w + tn] = '\0'; + } + } + } +} + +/* Only list markers, quote marks, table bars and emphasis before position i + * of the line: the phrase opens the line's statement. */ +static bool adr_line_head(const adr_line_t *l, int i) { + for (int k = 0; k < i; k++) { + char c = l->s[k]; + if (!adr_blank(c) && c != '-' && c != '*' && c != '+' && c != '>' && c != '_' && c != '|') { + return false; + } + } + return true; +} + +/* "supersedes" and "superseded by" are ADR words wherever they stand; + * "replaces" and "replaced by" are ordinary verbs of technical prose ("this + * middleware replaces the recovery described in ADR-22") and state a relation + * only where they open a line or list item ("Replaces ADR-3"). */ +static void adr_relations(adr_doc_t *d, const char *adr_qn, char *superseded_by, size_t cap) { + static const struct { + const char *phrase; + bool forward; + bool line_head; + } phrases[] = {{"superseded by", false, false}, + {"replaced by", false, true}, + {"supersedes", true, false}, + {"replaces", true, true}}; + for (int k = d->L.fm_end; k < d->L.count; k++) { + const adr_line_t *l = &d->L.v[k]; + if (!l->text) { + continue; + } + for (int i = 0; i < l->n; i++) { + if (i > 0 && adr_alpha(l->s[i - SKIP_ONE])) { + continue; + } + for (size_t p = 0; p < sizeof(phrases) / sizeof(phrases[0]); p++) { + size_t pl = strlen(phrases[p].phrase); + if (adr_starts_ci(l->s + i, (size_t)(l->n - i), phrases[p].phrase) && + (i + (int)pl == l->n || !adr_alpha(l->s[i + (int)pl])) && + (!phrases[p].line_head || adr_line_head(l, i))) { + adr_targets(d, adr_qn, l->s + i + pl, (size_t)(l->n - i - (int)pl), + (uint32_t)(k + SKIP_ONE), phrases[p].forward, superseded_by, cap); + i += (int)pl - SKIP_ONE; + break; + } + } + } + } +} + +/* A template's choice list, `{proposed | accepted}` or `[a | b]`: the field + * was never filled in. A filled-in value that kept its braces (`{proposed}`) + * is a value. */ +static bool adr_placeholder(const char *s) { + for (const char *p = s; *p; p++) { + if (*p != '{' && *p != '[') { + continue; + } + char close = *p == '{' ? '}' : ']'; + for (const char *q = p + SKIP_ONE; *q && *q != close; q++) { + if (*q == '|') { + return true; + } + } + } + return false; +} + +/* The first date in the record's change-log section ("Changelog", "Change + * log", "History", "Revision history"): `* 2019-12-12: Initial version`. */ +static bool adr_changelog_date(const adr_doc_t *d, char *out, size_t cap) { + for (int h = 0; h < d->nheads; h++) { + const char *t = d->heads[h].title; + size_t n = d->heads[h].title_n; + while (n > 0 && (t[n - SKIP_ONE] == ':' || adr_blank(t[n - SKIP_ONE]))) { + n--; + } + if (!adr_eq_ci(t, n, "changelog") && !adr_eq_ci(t, n, "change log") && + !adr_eq_ci(t, n, "history") && !adr_eq_ci(t, n, "revision history")) { + continue; + } + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + for (int k = d->heads[h].line + SKIP_ONE; k < end; k++) { + char line[ADR_FIELD_MAX]; + const adr_line_t *l = &d->L.v[k]; + if (!l->text || l->n == 0) { + continue; + } + snprintf(line, sizeof(line), "%.*s", l->n, l->s); + if (adr_parse_date(line, out, cap)) { + return true; + } + } + return false; + } + return false; +} + +/* ── The driver ──────────────────────────────────────────────────── */ + +static bool adr_ends_ci(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n > k && adr_eq_ci(s + n - k, k, suffix); +} + +static bool adr_markdown_path(const char *rel) { + return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown"); +} + +void cbm_adr_extract(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0 || !ctx->rel_path || + !adr_markdown_path(ctx->rel_path)) { + return; + } + const char *rel = ctx->rel_path; + const char *base = strrchr(rel, '/'); + base = base ? base + SKIP_ONE : rel; + adr_id_t id; + adr_filename_id(base, &id); + bool conv = adr_dir_in(rel, ADR_CONV_DIRS); + if ((id.kind == ADR_ID_NONE && adr_non_adr_name(base)) || adr_dir_in(rel, ADR_FIXTURE_DIRS)) { + return; + } + if (!conv && id.kind == ADR_ID_NONE) { + return; /* structure alone never makes a record */ + } + CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena; + CBMArena *a = ctx->arena; + adr_doc_t d = {.ctx = ctx, .title_head = CBM_NOT_FOUND}; + if (!adr_split_lines(scratch, ctx->source, ctx->source_len, &d.L)) { + ctx->result->doc_links.failed = true; + return; + } + /* headings: the Section definitions, read back at their lines */ + int nsec = 0; + for (int i = 0; i < ctx->result->defs.count; i++) { + const CBMDefinition *def = &ctx->result->defs.items[i]; + nsec += def->label && strcmp(def->label, "Section") == 0; + } + d.heads = + nsec > 0 ? (adr_head_t *)cbm_arena_alloc(scratch, (size_t)nsec * sizeof(*d.heads)) : NULL; + if (nsec > 0 && !d.heads) { + ctx->result->doc_links.failed = true; + return; + } + for (int i = 0; i < ctx->result->defs.count; i++) { + const CBMDefinition *def = &ctx->result->defs.items[i]; + if (!def->label || strcmp(def->label, "Section") != 0 || def->start_line < SKIP_ONE || + (int)def->start_line > d.L.count) { + continue; + } + int k = (int)def->start_line - SKIP_ONE; + const char *t = NULL; + size_t tn = 0; + int level = adr_heading_level(&d.L, k, &t, &tn); + if (level > 0) { + d.heads[d.nheads++] = + (adr_head_t){.line = k, .level = level, .title = t, .title_n = tn}; + } + } + qsort(d.heads, (size_t)d.nheads, sizeof(*d.heads), adr_head_cmp); + for (int h = 0; h < d.nheads; h++) { + if (d.heads[h].level == SKIP_ONE) { + d.title_head = h; + break; + } + } + /* status: front matter, header field, the Status section */ + char *status_raw = adr_fm_value(a, &d.L, "status"); + if (!status_raw) { + status_raw = adr_header_field(a, &d, "status"); + } + int status_sec = adr_find_section(&d, ADR_SEC_STATUS); + if (!status_raw && status_sec >= 0) { + const adr_head_t *sh = &d.heads[status_sec]; + const char *colon = memchr(sh->title, ':', sh->title_n); + if (colon && colon + SKIP_ONE < sh->title + sh->title_n) { + status_raw = + adr_plain(a, colon + SKIP_ONE, (size_t)(sh->title + sh->title_n - colon - SKIP_ONE), + ADR_FIELD_MAX); + } else { + status_raw = adr_first_paragraph(a, &d, status_sec, ADR_FIELD_MAX); + } + } + bool placeholder = status_raw && adr_placeholder(status_raw); + bool structure = status_raw && (adr_find_section(&d, ADR_SEC_CONTEXT) >= 0 || + adr_find_section(&d, ADR_SEC_DECISION) >= 0 || + adr_find_section(&d, ADR_SEC_CONSEQUENCES) >= 0 || + adr_find_section(&d, ADR_SEC_OUTCOME) >= 0 || + adr_find_section(&d, ADR_SEC_OPTIONS) >= 0); + bool is_adr = (conv && (id.kind != ADR_ID_NONE || structure)) || id.kind == ADR_ID_ADR || + ((id.kind == ADR_ID_NUMBER || id.kind == ADR_ID_DATE) && structure); + if (!is_adr) { + return; + } + /* the record's facts */ + char name[ADR_NAME_MAX]; + if (id.kind == ADR_ID_ADR || id.kind == ADR_ID_NUMBER) { + snprintf(name, sizeof(name), "ADR-%u", id.number); + } else if (id.kind == ADR_ID_PREFIX) { + snprintf(name, sizeof(name), "%s-%u", id.prefix, id.number); + } else { + const char *dot = strrchr(base, '.'); + size_t stem = dot ? (size_t)(dot - base) : strlen(base); + snprintf(name, sizeof(name), "%.*s", + (int)(stem < sizeof(name) - SKIP_ONE ? stem : sizeof(name) - SKIP_ONE), base); + } + char *title = NULL; + if (d.title_head >= 0) { + title = + adr_plain(a, d.heads[d.title_head].title, d.heads[d.title_head].title_n, ADR_FIELD_MAX); + } + if (!title) { + title = adr_fm_value(a, &d.L, "title"); + } + char date[ADR_ISO_LEN + SKIP_ONE] = ""; + /* date: front matter, header field, "last updated", the change log's + * first date, the file name */ + char *date_raw = adr_fm_value(a, &d.L, "date"); + if (!date_raw) { + date_raw = adr_header_field(a, &d, "date"); + } + if (!date_raw) { + date_raw = adr_header_field(a, &d, "last updated"); + } + if (!(date_raw && adr_parse_date(date_raw, date, sizeof(date))) && + !(!date_raw && adr_changelog_date(&d, date, sizeof(date))) && id.date[0]) { + memcpy(date, id.date, sizeof(id.date)); + } + static const char *const people_keys[] = {"deciders", "decision-makers", "decision_makers", + "authors", "author", NULL}; + char *deciders = NULL; + for (size_t k = 0; !deciders && people_keys[k]; k++) { + deciders = adr_fm_value(a, &d.L, people_keys[k]); + } + for (size_t k = 0; !deciders && people_keys[k]; k++) { + deciders = adr_header_field(a, &d, people_keys[k]); + } + if (deciders && adr_placeholder(deciders)) { + deciders = NULL; /* a template placeholder */ + } + int decision = adr_find_section(&d, ADR_SEC_DECISION); + if (decision < 0) { + decision = adr_find_section(&d, ADR_SEC_OUTCOME); + } + if (decision < 0) { + decision = adr_find_section(&d, ADR_SEC_CONTEXT); + } + char *doc = decision >= 0 ? adr_section_text(a, &d, decision, ADR_DOC_MAX) : NULL; + char *qn = cbm_fqn_compute(a, ctx->project, rel, CBM_ADR_QN_NAME); + char *adr_name = cbm_arena_strdup(a, name); + char *adr_date = date[0] ? cbm_arena_strdup(a, date) : NULL; + char superseded_by[ADR_FIELD_MAX] = ""; + char *fm_by = adr_fm_value(a, &d.L, "superseded-by"); + if (!fm_by) { + fm_by = adr_fm_value(a, &d.L, "superseded_by"); + } + if (fm_by) { + snprintf(superseded_by, sizeof(superseded_by), "%s", fm_by); + } + if (!qn || !adr_name) { + ctx->result->doc_links.failed = true; + return; + } + adr_relations(&d, qn, superseded_by, sizeof(superseded_by)); + const char *status = placeholder || !status_raw ? NULL : adr_norm_status(status_raw); + const char *facts[][PAIR_LEN] = { + {"adr_id", adr_name}, + {"title", title}, + {"status", status}, + {"status_text", placeholder ? NULL : status_raw}, + {"date", adr_date}, + {"deciders", deciders}, + {"superseded_by", superseded_by[0] ? cbm_arena_strdup(a, superseded_by) : NULL}, + }; + /* sized by the facts themselves: room for every pair and the terminator */ + size_t nfacts = sizeof(facts) / sizeof(facts[0]); + const char **kv = + (const char **)cbm_arena_alloc(a, ((nfacts * PAIR_LEN) + SKIP_ONE) * sizeof(char *)); + if (!kv) { + ctx->result->doc_links.failed = true; + return; + } + size_t w = 0; + for (size_t f = 0; f < nfacts; f++) { + if (facts[f][SKIP_ONE] && facts[f][SKIP_ONE][0]) { + kv[w++] = facts[f][0]; + kv[w++] = facts[f][SKIP_ONE]; + } + } + kv[w] = NULL; + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = adr_name; + def.qualified_name = qn; + def.label = "ADR"; + def.file_path = rel; + def.start_line = SKIP_ONE; + def.end_line = (uint32_t)d.L.count; + def.is_exported = true; + def.docstring = doc; + def.extra_props = kv; + cbm_defs_push(&ctx->result->defs, a, def); +} diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index bf7f695f66..a67364e34e 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -69,7 +69,8 @@ void cbm_doclink_test_note_doc_work(uint64_t copied, uint64_t parse_input, uint6 * doclink.h (language, name, external, and the ship gate). */ static const CBMDocLinkFamily DOCLINK_FAMILIES[CBM_DOCLINK_SYNTAX_COUNT] = { -#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships) [id] = {lang, name, external, ships}, +#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships, edge) \ + [id] = {lang, name, external, ships, edge}, CBM_DOCLINK_FAMILY_LIST(DOCLINK_FAMILY_ROW) #undef DOCLINK_FAMILY_ROW }; @@ -87,6 +88,11 @@ const char *cbm_doclink_syntax_name(int syntax) { return f ? f->name : ""; } +const char *cbm_doclink_syntax_edge(int syntax) { + const CBMDocLinkFamily *f = cbm_doclink_family(syntax); + return f && f->edge ? f->edge : "MENTIONS"; +} + bool cbm_doclink_syntax_is_external(int syntax) { const CBMDocLinkFamily *f = cbm_doclink_family(syntax); return f && f->external; diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 6e5bb035fa..99995202e2 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -43,9 +43,9 @@ /* Every link family, across languages: one line per (language, reference * form), and the ONE place a language leg adds its families -- * - * X(enum value, language, name, external, ships) + * X(enum value, language, name, external, ships, edge) * - * name the "syntax" MENTIONS-edge property and doc_link_unresolved + * name the "syntax" edge property and doc_link_unresolved * column, so it is public surface; with the language it is the * tier a release audit judges. Names repeat across languages * (`see` is C#'s, Java's, Kotlin's ...): the value, not the name, @@ -56,26 +56,33 @@ * still resolve, but a resolved one stays a doc_link_unresolved * row with reason below_bar_tier (an unresolved one keeps its own * reason). Nothing else about the family changes. + * edge the type of the edge a resolved reference becomes: MENTIONS + * (documentation names code), SUPERSEDES (an ADR states that it + * supersedes another). The edge's source is always the + * definition the reference is written in, so a file owns the + * edges its text produces, whatever their type. * * The enum below and the family table (doclink.c) are both generated from this * list, so a value cannot exist without its name and its gate. */ -#define CBM_DOCLINK_FAMILY_LIST(X) \ - /* csharp */ \ - X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true) \ - X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true) \ - X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true) \ - X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true) \ - /* markdown (doclink_md.c): what a document's text names explicitly */ \ - X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true) \ - X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true) \ - X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true) \ - X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true) \ - /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ - X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false) +#define CBM_DOCLINK_FAMILY_LIST(X) \ + /* csharp */ \ + X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ + /* markdown (doclink_md.c): what a document's text names explicitly */ \ + X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ + /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ + X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ + /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ + X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") typedef enum { CBM_DOCLINK_NONE = 0, -#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships) id, +#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships, edge) id, CBM_DOCLINK_FAMILY_LIST(CBM_DOCLINK_FAMILY_VALUE) #undef CBM_DOCLINK_FAMILY_VALUE CBM_DOCLINK_SYNTAX_COUNT @@ -99,8 +106,12 @@ typedef struct { const char *name; bool external; bool ships; + const char *edge; /* the type of the edge a resolved reference becomes */ } CBMDocLinkFamily; +/* "MENTIONS" or "SUPERSEDES"; "MENTIONS" for an unknown value. */ +const char *cbm_doclink_syntax_edge(int syntax); + /* The family of a syntax value; NULL for a value that names none. */ const CBMDocLinkFamily *cbm_doclink_family(int syntax); @@ -272,4 +283,16 @@ bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_ * and the lines, or false. */ bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last); +/* ── ADRs (doc_adr.c) ──────────────────────────────────────────────── */ + +/* When the Markdown file being extracted is an architecture decision record: + * push its ADR definition (name its canonical id, qualified name + * ".__adr__", facts as extra properties) and a `supersedes` token for + * each of its own supersedes / replaces statements. Called by the Markdown + * scan, after the Section definitions exist. */ +void cbm_adr_extract(CBMExtractCtx *ctx); + +/* The qualified-name tail of an ADR node: "" + this. */ +#define CBM_ADR_QN_NAME "__adr__" + #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_md.c b/internal/cbm/doclink_md.c index 0cd2d350fd..5a0f70bdba 100644 --- a/internal/cbm/doclink_md.c +++ b/internal/cbm/doclink_md.c @@ -61,6 +61,7 @@ enum { MD_NUM_DIGITS = 9, /* a line number has at most this many digits */ MD_PAIR = 2, /* ints per pair entry */ MD_URL_SCHEMES = 3, /* http, https, ftp */ + MD_BOM_LEN = 3, /* EF BB BF */ }; /* UTF-8 well-formedness (RFC 3629, table 3-7 of the Unicode standard). */ @@ -1764,6 +1765,9 @@ void cbm_doclink_md_scan_file(CBMExtractCtx *ctx) { const char *src = ctx->source; int n = ctx->source_len; int pos = 0; + if (n >= MD_BOM_LEN && memcmp(src, "\xEF\xBB\xBF", MD_BOM_LEN) == 0) { + pos = MD_BOM_LEN; /* a byte-order mark is no part of the first line */ + } while (pos < n && !s.failed) { const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); int end = nl ? (int)(nl - src) : n; @@ -1785,4 +1789,6 @@ void cbm_doclink_md_scan_file(CBMExtractCtx *ctx) { md_list_free(&s.bal); md_list_free(&s.gts); md_list_free(&s.blanks); + /* an architecture decision record: its node, facts and supersedes */ + cbm_adr_extract(ctx); } diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 9691b6c961..438a5d5354 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -276,6 +276,7 @@ static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) { cr_str(c, &d->http_client); cr_str(c, &d->http_base_url); cr_str(c, &d->variants); + cr_list(c, &d->extra_props); } static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index bdad5249ad..df012626f8 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -339,6 +339,10 @@ typedef struct { static int mention_cmp(const void *a, const void *b) { const doclink_mention_t *x = (const doclink_mention_t *)a; const doclink_mention_t *y = (const doclink_mention_t *)b; + int edge = strcmp(cbm_doclink_syntax_edge(x->syntax), cbm_doclink_syntax_edge(y->syntax)); + if (edge != 0) { + return edge; + } if (x->src != y->src) { return x->src < y->src ? -1 : 1; } @@ -372,7 +376,8 @@ void cbm_doclinks_test_fail_edge_insert_after(int nth) { #endif /* Insert one MENTIONS edge; 0 when it could not be stored. */ -static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *props) { +static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *type, + const char *props) { #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS int n = atomic_load(&doclinks_test_edge_fail_after); while (n > 0) { @@ -384,10 +389,10 @@ static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, co } } #endif - return cbm_gbuf_insert_edge(gb, src, tgt, "MENTIONS", props); + return cbm_gbuf_insert_edge(gb, src, tgt, type, props); } -/* Emit one MENTIONS edge per (source, target): first line, its syntax, the +/* Emit one edge per (edge type, source, target): first line, its syntax, the * mention count, and tier exact when any mention bound exactly. A failed * insert fails the layer: the edge is not there. */ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t *m, int n, @@ -397,7 +402,9 @@ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t while (i < n) { int j = i; bool exact = false; - while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt) { + const char *type = cbm_doclink_syntax_edge(m[i].syntax); + while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt && + strcmp(cbm_doclink_syntax_edge(m[j].syntax), type) == 0) { exact = exact || m[j].exact; j++; } @@ -412,7 +419,7 @@ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t "\"count\":%d%s}", m[i].via, cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", m[i].line, j - i, lines); - if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, props) == 0) { + if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, type, props) == 0) { mark_failed(dl, rel, "alloc"); /* an edge that is not there is not counted */ } else { atomic_fetch_add_explicit(&dl->edges, 1, memory_order_relaxed); diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 6251cb3364..07db1d4f3a 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -39,6 +39,7 @@ #include "graph_buffer/graph_buffer.h" #include +#include #include #include @@ -47,7 +48,8 @@ enum { MDR_TRIES = 2, /* relative to the document, relative to the root */ MDR_BASENAME_INIT = 1024, /* first capacity of the file-name table */ MDR_HEX_BASE = 16, - MDR_PCT_LEN = 3, /* `%XX` */ + MDR_PCT_LEN = 3, /* `%XX` */ + MDR_ADR_DIGITS = 5, /* an ADR number has at most this many digits */ }; /* Labels whose nodes are code a range or a member can name. */ @@ -94,7 +96,8 @@ typedef struct { CBMArena arena; /* the strings and records of pkgs and top */ mdr_def_t *defs; /* every code definition, sorted by (file, start line, id) */ int ndefs; - CBMHashTable *folders; /* directory path -> its Folder node */ + CBMHashTable *adr_by_file; /* file path -> its ADR node (doc_adr.c) */ + CBMHashTable *folders; /* directory path -> its Folder node */ } mdr_index_t; static const char *mdr_base(const char *path) { @@ -213,6 +216,15 @@ static bool mdr_build_defs(mdr_index_t *x, const cbm_gbuf_t *graph) { } } qsort(x->defs, (size_t)x->ndefs, sizeof(*x->defs), mdr_def_cmp); + const cbm_gbuf_node_t **adrs = NULL; + int nadr = 0; + if (cbm_gbuf_find_by_label(graph, "ADR", &adrs, &nadr) == 0) { + for (int i = 0; i < nadr; i++) { + if (adrs[i]->file_path && adrs[i]->file_path[0]) { + cbm_ht_set(x->adr_by_file, adrs[i]->file_path, (void *)adrs[i]); + } + } + } return true; } @@ -276,9 +288,10 @@ static void *mdr_build(const cbm_doclink_build_in_t *in) { x->basenames = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); x->pkgs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); x->top = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->adr_by_file = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); x->folders = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); if ((in->run_file_count > 0 && !x->run_paths) || !x->basenames || !x->pkgs || !x->top || - !x->folders || !x->project || !mdr_build_pkgs(x, in->graph) || + !x->adr_by_file || !x->folders || !x->project || !mdr_build_pkgs(x, in->graph) || !mdr_build_defs(x, in->graph)) { mdr_destroy(x); return NULL; @@ -322,6 +335,7 @@ static void mdr_destroy(void *index) { cbm_ht_free(x->basenames); cbm_ht_free(x->pkgs); cbm_ht_free(x->top); + cbm_ht_free(x->adr_by_file); cbm_ht_free(x->folders); cbm_arena_destroy(&x->arena); cbm_free(CBM_MEM_CLASS_OTHER, x->defs); @@ -1042,6 +1056,76 @@ static bool mdr_link_path(const char *raw, char *buf, size_t cap, mdr_ref_t *r) return buf[0] != '\0'; } +/* `ADR-12`, `ADR 012`, `adr_7` (the whole text): the canonical node name. */ +static bool mdr_adr_id(const char *raw, char *name, size_t cap) { + size_t n = strlen(raw); + if (n < PAIR_LEN * PAIR_LEN || mdr_norm_c((unsigned char)raw[0]) != 'a' || + mdr_norm_c((unsigned char)raw[SKIP_ONE]) != 'd' || + mdr_norm_c((unsigned char)raw[PAIR_LEN]) != 'r' || + (raw[PAIR_LEN + SKIP_ONE] != '-' && raw[PAIR_LEN + SKIP_ONE] != '_' && + raw[PAIR_LEN + SKIP_ONE] != ' ')) { + return false; + } + unsigned v = 0; + size_t digits = 0; + for (size_t i = PAIR_LEN * PAIR_LEN; i < n; i++) { + if (raw[i] < '0' || raw[i] > '9' || digits >= MDR_ADR_DIGITS) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (unsigned)(raw[i] - '0'); + digits++; + } + return digits > 0 && snprintf(name, cap, "ADR-%u", v) > 0; +} + +/* An ADR's own "supersedes X": X by its id, or the file a link names, to its + * ADR node. */ +static void mdr_resolve_adr(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const char *raw, cbm_doclink_outcome_t *out) { + char name[MDR_PATH_CAP]; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + if (mdr_adr_id(raw, name, sizeof(name))) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + if (nodes[i]->label && strcmp(nodes[i]->label, "ADR") == 0) { + hit = nodes[i]; + hits++; + } + } + } + if (hits == SKIP_ONE) { + mdr_edge(out, hit, 0, 0); + } else if (hits > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; /* two ADR logs number alike */ + } + return; + } + char buf[MDR_PATH_CAP]; + mdr_ref_t r = {.link = true}; + if (!mdr_link_path(raw, buf, sizeof(buf), &r)) { + return; + } + mdr_resolve_path(x, graph, doc, &r, out); + if (out->kind != CBM_DOCLINK_EDGE) { + return; /* missing, ambiguous, or no file of this repository */ + } + const cbm_gbuf_node_t *adr = + out->target->file_path + ? (const cbm_gbuf_node_t *)cbm_ht_get(x->adr_by_file, out->target->file_path) + : NULL; + if (!adr) { + out->kind = CBM_DOCLINK_UNRESOLVED; /* the file is there, but it is no record */ + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + mdr_edge(out, adr, 0, 0); +} + static void mdr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { (void)state; @@ -1055,6 +1139,9 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD char buf[MDR_PATH_CAP]; mdr_ref_t r = {0}; switch (link->syntax) { + case CBM_DOCLINK_MD_SUPERSEDES: + mdr_resolve_adr(x, graph, doc, link->raw, out); + return; case CBM_DOCLINK_MD_LINK: r.link = true; if (!mdr_link_path(link->raw, buf, sizeof(buf), &r)) { diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 74fd06d969..60e1a6bccf 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -323,6 +323,10 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) append_json_string(buf, bufsize, &pos, "route_method", def->route_method); append_json_string(buf, bufsize, &pos, "http_client", def->http_client); append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url); + /* A document definition's facts (an ADR's id, status, date, ...): key, value pairs. */ + for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) { + append_json_string(buf, bufsize, &pos, kv[0], kv[SKIP_ONE]); + } /* MinHash fingerprint — append if present and buffer has room. */ if (def->fingerprint && def->fingerprint_k > 0 && diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 4b35fae6f2..940bef7a72 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -551,6 +551,10 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) append_json_string(buf, bufsize, &pos, "route_method", def->route_method); append_json_string(buf, bufsize, &pos, "http_client", def->http_client); append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url); + /* A document definition's facts (an ADR's id, status, date, ...): key, value pairs. */ + for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) { + append_json_string(buf, bufsize, &pos, kv[0], kv[SKIP_ONE]); + } /* MinHash fingerprint — append if present and buffer has room. * Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index dea2d7a9db..38e1cb4b2e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1895,7 +1895,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c } str_def_fp += def->fingerprint ? (size_t)def->fingerprint_k * sizeof(uint32_t) : 0; str_def_misc += census_len(def->route_path) + census_len(def->route_method) + - census_len(def->impl_trait) + census_len(def->http_base_url); + census_len(def->impl_trait) + census_len(def->http_base_url) + + census_list(def->extra_props); } for (int c = 0; c < r->calls.count; c++) { const CBMCall *call = &r->calls.items[c]; diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 5f33416def..4ec817f8a4 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -410,10 +410,161 @@ TEST(doc_links_md_incremental) { PASS(); } +/* ── ADRs ────────────────────────────────────────────────────────── */ + +static void md_write_adrs(const char *tmp, const char *first_status) { + char first[1024]; + snprintf(first, sizeof(first), + "---\n" + "status: %s\n" + "date: 2024-03-05\n" + "deciders: Ana, Bo\n" + "---\n" + "# Use X for storage\n" + "\n" + "## Context and Problem Statement\n" + "\n" + "We need storage.\n" + "\n" + "## Decision Outcome\n" + "\n" + "We use X, see `app/routing.py`.\n" + "\n" + "Superseded by [ADR-0002](0002-use-y.md).\n", + first_status); + th_write_file(TH_PATH(tmp, "docs/adr/0001-use-x.md"), first); + th_write_file(TH_PATH(tmp, "docs/adr/0002-use-y.md"), + "# 2. Use Y instead\n" + "\n" + "Date: 12 May 2024\n" + "\n" + "## Status\n" + "\n" + "Accepted\n" + "\n" + "Supersedes [ADR-0001](0001-use-x.md).\n" + "\n" + "## Context\n" + "\n" + "X was slow.\n" + "\n" + "## Decision\n" + "\n" + "We use Y.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0003-cache.md"), + "# Cache reads\n" + "\n" + "## Status\n" + "\n" + "Proposed\n" + "\n" + "- Replaces ADR-2\n" + "\n" + "## Decision\n" + "\n" + "The cache replaces the reader described in [ADR-1](0001-use-x.md).\n"); + th_write_file(TH_PATH(tmp, "docs/adr/README.md"), "# Decisions\n\n## Status\n\nAccepted\n"); + th_write_file(TH_PATH(tmp, "docs/adr/template.md"), + "# Title\n\n## Status\n\n{proposed | accepted}\n\n## Decision\n\nTBD\n"); +} + +/* Properties of the ADR node of `file`; "" when there is none. */ +static void md_adr_props(const char *db, const char *file, char *out, size_t cap) { + out[0] = '\0'; + sqlite3 *h = NULL; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(h, + "SELECT name || ' ' || properties FROM nodes WHERE label = 'ADR' " + "AND file_path = ?1", + -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, file, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW) { + snprintf(out, cap, "%s", (const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + sqlite3_close(h); +} + +TEST(doc_links_md_adr) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_adr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + md_write_fixture(tmp); + md_write_adrs(tmp, "superseded"); + char db[512]; + snprintf(db, sizeof(db), "%s/adr.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[2048]; + + /* three records; README and template are none */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'ADR'"), 3); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'DEFINES' AND t.label = 'ADR'"), + 3); + /* facts: front matter */ + md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props)); + ASSERT_TRUE(strncmp(props, "ADR-1 ", strlen("ADR-1 ")) == 0); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"superseded\"")); + ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-03-05\"")); + ASSERT_NOT_NULL(strstr(props, "\"deciders\":\"Ana, Bo\"")); + ASSERT_NOT_NULL(strstr(props, "\"title\":\"Use X for storage\"")); + ASSERT_NOT_NULL(strstr(props, "\"superseded_by\":\"0002-use-y.md\"")); + ASSERT_NOT_NULL(strstr(props, "We use X")); /* the decision, searchable */ + /* facts: header field date, Status section */ + md_adr_props(db, "docs/adr/0002-use-y.md", props, sizeof(props)); + ASSERT_TRUE(strncmp(props, "ADR-2 ", strlen("ADR-2 ")) == 0); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"accepted\"")); + ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-05-12\"")); + md_adr_props(db, "docs/adr/README.md", props, sizeof(props)); + ASSERT_STR_EQ(props, ""); + + /* SUPERSEDES: by link, and by id at a line head; never from prose that + * uses "replaces" as a verb, never from the superseded record */ + const char *sup = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' " + "AND s.file_path = '%s' AND t.file_path = '%s'"; + char q[512]; + snprintf(q, sizeof(q), sup, "docs/adr/0002-use-y.md", "docs/adr/0001-use-x.md"); + ASSERT_EQ(dm_count(db, q), 1); + snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0002-use-y.md"); + ASSERT_EQ(dm_count(db, q), 1); + snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0001-use-x.md"); + ASSERT_EQ(dm_count(db, q), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'SUPERSEDES'"), 2); + /* the record's sections still link to code */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.file_path = 'docs/adr/0001-use-x.md' " + "AND t.file_path = 'app/routing.py'"), + 1); + + /* incremental == full: a status edit, then a new record that a statement + * names by id */ + char full_db[512]; + snprintf(full_db, sizeof(full_db), "%s/adr-full.db", tmp); + md_write_adrs(tmp, "deprecated"); + ASSERT_EQ(dm_step(tmp, db, full_db, "ADR status edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props)); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"deprecated\"")); + th_write_file(TH_PATH(tmp, "docs/adr/0004-later.md"), "# Later\n\n## Status\n\nAccepted\n\n" + "Supersedes ADR-3.\n\n## Decision\n\n" + "Later.\n"); + ASSERT_EQ(dm_step(tmp, db, full_db, "ADR added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + snprintf(q, sizeof(q), sup, "docs/adr/0004-later.md", "docs/adr/0003-cache.md"); + ASSERT_EQ(dm_count(db, q), 1); + th_cleanup(tmp); + PASS(); +} + SUITE(doc_links_md) { RUN_TEST(doc_links_md_extract_tokens); RUN_TEST(doc_links_md_repeated_headings); RUN_TEST(doc_links_md_classify_span); RUN_TEST(doc_links_md_resolve); RUN_TEST(doc_links_md_incremental); + RUN_TEST(doc_links_md_adr); } From 033ca372a8f2378ce1566756819cccd8665b0e28 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 22:59:48 +0200 Subject: [PATCH 03/25] doc-links(pdf): PDF pages as Sections, linked to the code their text names A PDF is now indexed: its text layer is read by a dependency-free extractor (internal/cbm/pdf, zlib only) and every page becomes a Section node ("page N", the page's text as its docstring, property page), so search finds what a page says and get_code_snippet shows a page's text, never the file's bytes. The code a page names structurally becomes MENTIONS edges from that page, via "pdf": pdf_path a repository path ("pkg/server/handler.go", "./x/y") pdf_file a file name with a code extension ("models.py", "Pool.sol") pdf_qn a qualified name ("app.models.User.save", "a::b", "x->y") Bare identifiers never become edges (field test E6c: a type-target rule failed its pre-registered bar), and only EXACT or UNIQUE resolutions link: the field test's tiers (path-exact, component-aligned path-suffix, qn-exact, qn-suffix, the latter also through a node's owner -- a Go method's QN has no receiver -- and with the same one-entity collapses). A mention split across a line break is proposed as one reference and replaces its fragments only when it resolves. Hygiene as measured: no links from PDFs under test or fixture directories (R1), none to data files (R2), none for keyword arguments (R3); a test, mock or fixture target is a test_only_target row (R4, with the test directories the field test found missing). A path or name whose first component is not of this repository is no row at all. The extractor is a port of the field-tested prototype: xref tables and streams, /Prev chains, repair near a wrong offset and reconstruction from the object headers, object streams, Flate (with partial recovery), LZW, ASCII85, ASCIIHex, RunLength and PNG/TIFF predictors, simple fonts (base encodings, /Differences, built-in Type1 encodings, standard-14 widths, Type3), ToUnicode and encoding CMaps, Type0/CID fonts with the embedded TrueType cmap read backwards, Form XObjects, inline images skipped, word and line breaks from glyph geometry, right-to-left runs reversed. Its geometry is bit-exact on every platform (no FMA contraction; CPython's hypot). Untrusted input: every read is bounds-checked, recursion is depth-limited, integers from the file saturate, a decoded stream is cut at 512 MiB, and every lookup that could rescan the file per object or per glyph is answered from an index built once. Not read: encrypted documents (reported as such); scanned pages have no text layer. The mention scanner is the field test's (idents.py) with CPython's semantics: NFKC per code point plus canonical composition and the str predicates, from generated Unicode 16.0 tables (pdf_unicode.c), and re's leftmost-first matching. Pipeline: CBM_LANG_PDF (no grammar; the registry ledger gets a DOCUMENT capability, the call-node manifest exempts it like the other grammar-free classification), extraction before any grammar lookup, a resolver hook file_prepare (a file's state from all its references before any resolves). Incremental: a proxy node carries parent_class where owner + name differs from its QN (Go methods, class-body variables), so a re-resolved document binds a Go method as a full build does. Proof against the field-test prototype on its corpus (749 PDFs from 21 repositories, restored at the recorded commits, sha256-verified): text byte-identical on all 1,492 pages of the 720 comparable PDFs (0.73 s vs 14.7 s); mention tokens identical on all 960 page-text sets of E6, E6b and E6c (26,435 tokens); links identical to the frozen Python pipeline run over the same graph on semantic-kernel 5/5, cosmos-sdk 30/30, Polly 11/11, mediastore3 3/3, dask 4/4. 44,070 seeded mutants (truncations, flips, nesting and keyword bombs, on the originals and on uncompressed copies) under ASan and UBSan: 0 findings after the first campaign's integer overflows were fixed as a class. Tests: suite doc_links_pdf (layout, fonts and CMaps, object and xref streams, xref repair and reconstruction, Form XObjects incl. a self-referencing one, inline images, filters, hostile input, the scanner against the field test's output, the pipeline's tiers, hygiene and rows, back navigation, the snippet, incremental == full). Signed-off-by: Martin Vogel --- Makefile.cbm | 9 +- internal/cbm/cbm.c | 20 + internal/cbm/cbm.h | 1 + internal/cbm/doc_pdf.c | 1306 ++++++++++ internal/cbm/doclink.h | 21 + internal/cbm/lang_specs.c | 5 + internal/cbm/pdf/pdf.h | 60 + internal/cbm/pdf/pdf_font.c | 1978 +++++++++++++++ internal/cbm/pdf/pdf_internal.h | 321 +++ internal/cbm/pdf/pdf_obj.c | 2594 ++++++++++++++++++++ internal/cbm/pdf/pdf_tables.c | 1619 ++++++++++++ internal/cbm/pdf/pdf_text.c | 1040 ++++++++ internal/cbm/pdf/pdf_unicode.c | 3124 ++++++++++++++++++++++++ internal/cbm/pdf/pdf_unicode.h | 103 + src/discover/language.c | 2 + src/discover/userconfig.c | 1 + src/mcp/mcp.c | 40 +- src/pipeline/doc_links.c | 8 +- src/pipeline/doc_links.h | 8 + src/pipeline/doc_links_pdf.c | 807 ++++++ src/pipeline/pipeline_delta.c | 67 +- tests/repro/repro_call_node_manifest.c | 8 +- tests/repro/repro_language_registry.c | 23 +- tests/test_doc_links_pdf.c | 793 ++++++ tests/test_main.c | 2 + 25 files changed, 13942 insertions(+), 18 deletions(-) create mode 100644 internal/cbm/doc_pdf.c create mode 100644 internal/cbm/pdf/pdf.h create mode 100644 internal/cbm/pdf/pdf_font.c create mode 100644 internal/cbm/pdf/pdf_internal.h create mode 100644 internal/cbm/pdf/pdf_obj.c create mode 100644 internal/cbm/pdf/pdf_tables.c create mode 100644 internal/cbm/pdf/pdf_text.c create mode 100644 internal/cbm/pdf/pdf_unicode.c create mode 100644 internal/cbm/pdf/pdf_unicode.h create mode 100644 src/pipeline/doc_links_pdf.c create mode 100644 tests/test_doc_links_pdf.c diff --git a/Makefile.cbm b/Makefile.cbm index 3204c1566b..0c9eca047a 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -377,6 +377,12 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_cs.c \ $(CBM_DIR)/doclink_md.c \ $(CBM_DIR)/doc_adr.c \ + $(CBM_DIR)/doc_pdf.c \ + $(CBM_DIR)/pdf/pdf_obj.c \ + $(CBM_DIR)/pdf/pdf_font.c \ + $(CBM_DIR)/pdf/pdf_text.c \ + $(CBM_DIR)/pdf/pdf_tables.c \ + $(CBM_DIR)/pdf/pdf_unicode.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ @@ -500,6 +506,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links.c \ src/pipeline/doc_links_cs.c \ src/pipeline/doc_links_md.c \ + src/pipeline/doc_links_pdf.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -762,7 +769,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index d8ac6c8e72..c740953f60 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2802,6 +2802,26 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_test_fault_inject(rel_path); #endif + /* A PDF has no grammar: its text layer is read directly, a Section per + * page plus the pages' code mentions (doc_pdf.c). */ + if (language == CBM_LANG_PDF) { + result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language); + CBMExtractCtx pdf_ctx = { + .arena = a, + .scratch = scratch, + .result = result, + .source = source, + .source_len = source_len, + .language = language, + .project = project, + .rel_path = rel_path, + .module_qn = result->module_qn, + }; + cbm_pdf_extract_document(&pdf_ctx); + cbm_index_mark_done(rel_path); + return result; + } + // Get language spec const CBMLangSpec *spec = cbm_lang_spec(language); if (!spec) { diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 561862b2ca..0cb9c323cf 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -209,6 +209,7 @@ typedef enum { CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI) CBM_LANG_PLSQL, // Oracle PL/SQL CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language) + CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c) CBM_LANG_COUNT } CBMLanguage; diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.c new file mode 100644 index 0000000000..b7a59a7a53 --- /dev/null +++ b/internal/cbm/doc_pdf.c @@ -0,0 +1,1306 @@ +/* + * doc_pdf.c — PDF documents in the graph. + * + * A PDF's text layer (pdf/) becomes one Section per page: name "page N", the + * page's text as its docstring, so search finds what a page says. The code a + * page names becomes doc-link tokens for the resolver (doc_links_pdf.c): + * STRUCTURAL mentions only -- paths ("src/auth/handler.go", "./x/y", + * "Pool.sol") and qualified names ("pkg.Type.method", "a::b", "x->y"). Bare + * identifiers never become tokens (field test E6c: bare names stay + * suggestions). + * + * The scanner is the field-tested one (E6 idents.py), ported with CPython's + * semantics: NFKC per code point plus canonical composition, the re module's + * leftmost-first matching, str predicates from the same Unicode tables + * (pdf/pdf_unicode.c). A mention split across two lines ("src/auth/" / + * "handler.go", "Handler::" / "run") is proposed as a JOIN token that lists + * the line-local tokens it would replace; the resolver keeps the join only + * when it resolves EXACT or UNIQUE, and the fragments stand otherwise. + * + * Hygiene from the field test, applied here where the text decides it: R1 a + * PDF under a test or fixture directory gives no tokens; R3 a mention written + * as a keyword argument ("(x, foo.bar=1)") is no reference. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "foundation/mem_core.h" +#include "helpers.h" +#include "pdf/pdf.h" +#include "pdf/pdf_unicode.h" + +#include +#include +#include + +enum { + PDF_R3_WINDOW = 400, /* the field test looked this far back for "(" or "," */ + PDF_PAGE_NAME = 32, + PDF_LIST_MIN = 64, + PDF_U8_MAX = 4, +}; + +#define PDF_JOIN_SEP '\x1f' /* between a join's forms */ +#define PDF_JOIN_FRAG '\x1e' /* before a join's fragment token indices */ + +/* ── Word lists (the field test's, verbatim) ─────────────────────── */ + +static const char *const CODE_EXT[] = { + "py", "pyi", "pyx", "pxd", "c", "h", "cc", "cpp", "cxx", "hpp", + "hh", "hxx", "cs", "csx", "java", "kt", "kts", "scala", "go", "rs", + "rb", "php", "js", "jsx", "mjs", "cjs", "ts", "tsx", "swift", "m", + "mm", "sol", "sh", "bash", "zsh", "ps1", "psm1", "lua", "r", "jl", + "dart", "ex", "exs", "erl", "hs", "ml", "fs", "fsx", "vb", "sql", + "proto", "graphql", "gql", "thrift", "yaml", "yml", "json", "toml", "ini", "cfg", + "conf", "xml", "html", "htm", "css", "scss", "less", "md", "rst", "txt", + "adoc", "ipynb", "gradle", "csproj", "sln", "props", "targets", "cmake", "mk", "tf", + "hcl", "bzl", "lock", "env", "pdf", "vue", "svelte", "tex", "bib", "jsonl", + "csv", "tsv", "pom", "jar", "war", "dll", "so", "wasm", "zig", "nim", + "v", NULL}; + +static const char *const TLDS[] = {"com", "org", "net", "io", "dev", "edu", "gov", "de", "uk", + "ai", "co", "info", "app", "eu", "fr", "ch", "us", "ca", + "au", "jp", "cn", "ru", "nl", "se", "no", "fi", "it", + "es", "at", "be", "biz", "me", "tv", "ly", "gg", "sh", + "xyz", "cloud", "tech", "blog", NULL}; + +static const char *const SLASH_STOP[] = { + "and/or", "or/and", "input/output", "i/o", "read/write", "yes/no", "on/off", + "true/false", "he/she", "his/her", "s/he", "w/o", "n/a", "km/h", + "m/s", "tcp/ip", "client/server", "him/her", "either/or", NULL}; + +/* the field test's FIX: directories whose PDFs are fixtures (R1) */ +static const char *const FIXTURE_DIRS[] = {"test", "tests", "cypress", "baseline_images", + "resources", "samples", "e2e", "fixtures", + "testdata", "test_data", "data", "examples", + NULL}; + +/* ── Code-point buffers ──────────────────────────────────────────── */ + +typedef struct { + uint32_t *v; + int n; + int cap; + bool fail; +} u32v_t; + +static bool u32_push(u32v_t *b, uint32_t c) { + if (b->fail) { + return false; + } + if (b->n == b->cap) { + int ncap = b->cap ? b->cap * 2 : PDF_LIST_MIN; + uint32_t *g = + (uint32_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->v, (size_t)ncap * sizeof(*g)); + if (!g) { + b->fail = true; + return false; + } + b->v = g; + b->cap = ncap; + } + b->v[b->n++] = c; + return true; +} + +static void u32_free(u32v_t *b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b->v); + memset(b, 0, sizeof(*b)); +} + +/* ── normalise(): NFKC, zero-width characters dropped, hyphens unified ── */ + +static const pdf_nfkc_t *nfkc_find(uint32_t cp) { + int lo = 0; + int hi = PDF_NFKC_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (PDF_NFKC[mid].cp < cp) { + lo = mid + 1; + } else if (PDF_NFKC[mid].cp > cp) { + hi = mid - 1; + } else { + return &PDF_NFKC[mid]; + } + } + return NULL; +} + +static bool ucomp(uint32_t a, uint32_t b, uint32_t *out) { + /* Hangul, algorithmically */ + enum { + SBASE = 0xAC00, + LBASE = 0x1100, + VBASE = 0x1161, + TBASE = 0x11A7, + LCOUNT = 19, + VCOUNT = 21, + TCOUNT = 28, + NCOUNT = VCOUNT * TCOUNT, + SCOUNT = LCOUNT * NCOUNT + }; + if (a >= LBASE && a < LBASE + LCOUNT && b >= VBASE && b < VBASE + VCOUNT) { + *out = SBASE + ((a - LBASE) * VCOUNT + (b - VBASE)) * TCOUNT; + return true; + } + if (a >= SBASE && a < SBASE + SCOUNT && (a - SBASE) % TCOUNT == 0 && b > TBASE && + b < TBASE + TCOUNT) { + *out = a + (b - TBASE); + return true; + } + int lo = 0; + int hi = PDF_UCOMP_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + const pdf_ucomp_t *e = &PDF_UCOMP[mid]; + if (e->first < a || (e->first == a && e->second < b)) { + lo = mid + 1; + } else if (e->first > a || e->second > b) { + hi = mid - 1; + } else { + *out = e->composite; + return true; + } + } + return false; +} + +static bool normalise(const char *s, size_t n, u32v_t *out) { + size_t i = 0; + while (i < n) { + unsigned char c = (unsigned char)s[i]; + uint32_t cp = c; + size_t k = 1; + if (c >= 0xF0 && i + 3 < n) { + cp = ((uint32_t)(c & 0x07) << 18) | ((uint32_t)(s[i + 1] & 0x3F) << 12) | + ((uint32_t)(s[i + 2] & 0x3F) << 6) | (uint32_t)(s[i + 3] & 0x3F); + k = 4; + } else if (c >= 0xE0 && i + 2 < n) { + cp = ((uint32_t)(c & 0x0F) << 12) | ((uint32_t)(s[i + 1] & 0x3F) << 6) | + (uint32_t)(s[i + 2] & 0x3F); + k = 3; + } else if (c >= 0xC0 && i + 1 < n) { + cp = ((uint32_t)(c & 0x1F) << 6) | (uint32_t)(s[i + 1] & 0x3F); + k = 2; + } + i += k; + const pdf_nfkc_t *m = nfkc_find(cp); + int cnt = m ? m->len : 1; + for (int j = 0; j < cnt; j++) { + uint32_t x = m ? PDF_NFKC_SEQ[m->off + j] : cp; + uint32_t comp; + if (out->n > 0 && ucomp(out->v[out->n - 1], x, &comp)) { + out->v[out->n - 1] = comp; + } else if (!u32_push(out, x)) { + return false; + } + } + } + /* zero-width characters out; U+2010/U+2011 -> '-' */ + int k = 0; + for (int j = 0; j < out->n; j++) { + uint32_t x = out->v[j]; + if (x == 0x200B || x == 0x200C || x == 0x200D || x == 0x2060 || x == 0xFEFF) { + continue; + } + if (x == 0x2010 || x == 0x2011) { + x = '-'; + } + out->v[k++] = x; + } + out->n = k; + return true; +} + +/* ── Character classes of the field test's regexes ───────────────── */ + +static bool is_alnum_ascii(uint32_t c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +static bool is_alpha_ascii(uint32_t c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +/* [A-Za-z0-9_.$@\-] */ +static bool path_seg(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '.' || c == '$' || c == '@' || c == '-'; +} + +/* [A-Za-z0-9_.\-] */ +static bool path_tail(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '.' || c == '-'; +} + +/* [A-Za-z0-9_] */ +static bool path_end(uint32_t c) { + return is_alnum_ascii(c) || c == '_'; +} + +/* [A-Za-z_$] */ +static bool ident_start(uint32_t c) { + return is_alpha_ascii(c) || c == '_' || c == '$'; +} + +/* [\w$] */ +static bool ident_cont(uint32_t c) { + return pdf_u_word(c) || c == '$'; +} + +/* [A-Za-z0-9_$] (IDC) */ +static bool idc(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '$'; +} + +/* ── Candidates ──────────────────────────────────────────────────── */ + +typedef enum { + MK_PATH = 0, + MK_PATH1, + MK_FILE, + MK_QUAL, + MK_SNAKE, + MK_CAMEL, + MK_CALL, +} mention_kind_t; + +typedef struct { + int start; + int end; + mention_kind_t kind; + int norm_start; /* norm = a rewrite of the surface, kept in a pool */ + int norm_len; +} cand_t; + +typedef struct { + cand_t *v; + int n; + int cap; + bool fail; + u32v_t pool; /* norms */ +} candv_t; + +static bool cand_push(candv_t *cv, cand_t c) { + if (cv->fail) { + return false; + } + if (cv->n == cv->cap) { + int ncap = cv->cap ? cv->cap * 2 : PDF_LIST_MIN; + cand_t *g = (cand_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, cv->v, (size_t)ncap * sizeof(*g)); + if (!g) { + cv->fail = true; + return false; + } + cv->v = g; + cv->cap = ncap; + } + cv->v[cv->n++] = c; + return true; +} + +static void candv_free(candv_t *cv) { + cbm_free(CBM_MEM_CLASS_EXTRACT, cv->v); + u32_free(&cv->pool); + memset(cv, 0, sizeof(*cv)); +} + +static bool span_free(const uint8_t *taken, int a, int b) { + for (int i = a; i < b; i++) { + if (taken[i]) { + return false; + } + } + return true; +} + +static void span_take(uint8_t *taken, int a, int b) { + memset(taken + a, 1, (size_t)(b - a)); +} + +/* URL_RE: (?:https?|ftp|file)://\S+ | www\.\S+ | \S+@\S+\.[A-Za-z]{2,} */ +typedef struct { + int run_start; /* the non-space run the email alternative last looked at */ + int run_end; + int at; /* its last '@' with a domain after it, or -1 */ + int at_end; /* where that match ends */ +} url_cache_t; + +static bool lit_at(const uint32_t *t, int n, int p, const char *lit) { + for (int i = 0; lit[i]; i++) { + if (p + i >= n || t[p + i] != (uint32_t)(unsigned char)lit[i]) { + return false; + } + } + return true; +} + +static int run_end_from(const uint32_t *t, int n, int p) { + while (p < n && !pdf_u_space(t[p])) { + p++; + } + return p; +} + +static void url_cache_run(const uint32_t *t, int n, int p, url_cache_t *uc) { + if (p >= uc->run_start && p < uc->run_end) { + return; /* the run is known: validity of an '@' does not depend on p */ + } + /* the scan enters a run at its start (every earlier position of a run + * filled this cache); scanning back is only for a caller that did not */ + int rs = p; + while (rs > 0 && !pdf_u_space(t[rs - 1])) { + rs--; + } + uc->run_start = rs; + uc->run_end = run_end_from(t, n, p); + uc->at = -1; + uc->at_end = -1; + int re = uc->run_end; + /* the largest '.' followed by two ASCII letters, scanning from the right; + * for each '@' (right to left) the domain is the largest such '.' >= at+2 */ + int best_dot = -1; + for (int a = re - 1; a > rs; a--) { + if (a + 2 < re && t[a] == '.' && is_alpha_ascii(t[a + 1]) && is_alpha_ascii(t[a + 2])) { + if (best_dot < 0) { + best_dot = a; + } + } + if (t[a] == '@' && best_dot >= a + 2) { + int e = best_dot + 1; + while (e < re && is_alpha_ascii(t[e])) { + e++; + } + uc->at = a; + uc->at_end = e; + return; + } + } +} + +static int url_at(const uint32_t *t, int n, int p, url_cache_t *uc) { + if (lit_at(t, n, p, "https://") || lit_at(t, n, p, "http://") || lit_at(t, n, p, "ftp://") || + lit_at(t, n, p, "file://")) { + int q = p; + while (t[q] != '/') { + q++; + } + q += 2; + if (q < n && !pdf_u_space(t[q])) { + return run_end_from(t, n, q); + } + } + if (lit_at(t, n, p, "www.") && p + 4 < n && !pdf_u_space(t[p + 4])) { + return run_end_from(t, n, p + 4); + } + if (pdf_u_space(t[p])) { + return -1; + } + url_cache_run(t, n, p, uc); + if (uc->at > p) { + return uc->at_end; + } + return -1; +} + +/* PATH_RE's body after the optional prefix: (?:[seg]+/)+[tail]*[end] */ +static int path_body(const uint32_t *t, int n, int q) { + int p = q; + int last_ts = -1; + for (;;) { + int r = p; + while (r < n && path_seg(t[r])) { + r++; + } + if (r > p && r < n && t[r] == '/') { + last_ts = r + 1; + p = r + 1; + continue; + } + break; + } + if (last_ts < 0) { + return -1; + } + /* tail starts, from the last segment back to the first */ + int ts = last_ts; + for (;;) { + int r = ts; + int last_e = -1; + while (r < n && path_tail(t[r])) { + if (path_end(t[r])) { + last_e = r; + } + r++; + } + if (last_e >= 0) { + return last_e + 1; + } + /* give back one segment: the previous '/' before ts-1 */ + int s = ts - 2; + while (s >= q && t[s] != '/') { + s--; + } + if (s < q) { + return -1; + } + ts = s + 1; + } +} + +static int path_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '/' || b == '.' || b == '-' || b == '~' || b == '$' || b == '@') { + return -1; + } + } + /* (?:\.{1,2}/|~/|/)? tried in order, then without */ + int e; + if (p + 2 < n && t[p] == '.' && t[p + 1] == '.' && t[p + 2] == '/' && + (e = path_body(t, n, p + 3)) >= 0) { + return e; + } + if (p + 1 < n && t[p] == '.' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) { + return e; + } + if (p + 1 < n && t[p] == '~' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) { + return e; + } + if (t[p] == '/' && (e = path_body(t, n, p + 1)) >= 0) { + return e; + } + return path_body(t, n, p); +} + +/* QUAL_RE: [A-Za-z_$][\w$]*(?:(?:\.|::|->|#)[A-Za-z_$][\w$]*)+(?:\(\))? */ +static int qual_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') { + return -1; + } + } + if (!ident_start(t[p])) { + return -1; + } + int q = p + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + int groups = 0; + for (;;) { + int sep = 0; + if (q < n && t[q] == '.') { + sep = 1; + } else if (q + 1 < n && t[q] == ':' && t[q + 1] == ':') { + sep = 2; + } else if (q + 1 < n && t[q] == '-' && t[q + 1] == '>') { + sep = 2; + } else if (q < n && t[q] == '#') { + sep = 1; + } + if (!sep || q + sep >= n || !ident_start(t[q + sep])) { + break; + } + q += sep + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + groups++; + } + if (!groups) { + return -1; + } + if (q + 1 < n && t[q] == '(' && t[q + 1] == ')') { + q += 2; + } + return q; +} + +/* TOK_RE: [A-Za-z_$][\w$]* (whole identifier) */ +static int tok_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') { + return -1; + } + } + if (!ident_start(t[p])) { + return -1; + } + int q = p + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + return q; +} + +static uint32_t ascii_lower(uint32_t c) { + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static bool in_list_lower(const uint32_t *s, int n, const char *const *list) { + for (int i = 0; list[i]; i++) { + const char *w = list[i]; + int k = 0; + while (k < n && w[k] && ascii_lower(s[k]) == (uint32_t)(unsigned char)w[k]) { + k++; + } + if (k == n && !w[k]) { + return true; + } + } + return false; +} + +/* str.islower / str.isupper of a whole string */ +static bool str_islower(const uint32_t *s, int n) { + bool cased = false; + for (int i = 0; i < n; i++) { + if (pdf_u_upper(s[i]) || pdf_u_title(s[i])) { + return false; + } + cased = cased || pdf_u_lower(s[i]); + } + return cased; +} + +static bool str_isupper(const uint32_t *s, int n) { + bool cased = false; + for (int i = 0; i < n; i++) { + if (pdf_u_lower(s[i]) || pdf_u_title(s[i])) { + return false; + } + cased = cased || pdf_u_upper(s[i]); + } + return cased; +} + +static bool is_camel(const uint32_t *t, int n) { + if (n < 3 || str_isupper(t, n) || str_islower(t, n)) { + return false; + } + /* [A-Z]{2,}s: PDFs, APIs, IDs */ + if (t[n - 1] == 's') { + bool caps = true; + for (int i = 0; i < n - 1; i++) { + caps = caps && t[i] >= 'A' && t[i] <= 'Z'; + } + if (caps) { + return false; + } + } + for (int i = 0; i + 1 < n; i++) { + uint32_t a = t[i]; + uint32_t b = t[i + 1]; + if (pdf_u_lower(a) && pdf_u_upper(b)) { + return true; + } + if (i >= 1 && pdf_u_upper(t[i - 1]) && pdf_u_upper(a) && pdf_u_lower(b)) { + return true; + } + } + return false; +} + +/* The extension after the last '.' of a segment, when it is a code extension. */ +static bool has_code_ext(const uint32_t *s, int n) { + int dot = -1; + for (int i = 0; i < n; i++) { + if (s[i] == '.') { + dot = i; + } + } + return dot >= 0 && in_list_lower(s + dot + 1, n - dot - 1, CODE_EXT); +} + +static void path_candidate(const uint32_t *t, int a, int b, candv_t *cv) { + /* s.strip("./~"), split on '/', drop empties */ + int sa = a; + int sb = b; + while (sa < sb && (t[sa] == '.' || t[sa] == '/' || t[sa] == '~')) { + sa++; + } + while (sb > sa && (t[sb - 1] == '.' || t[sb - 1] == '/' || t[sb - 1] == '~')) { + sb--; + } + bool any_seg = false; + bool all_numeric = true; + int last_s = -1; + int last_e = -1; + for (int i = sa; i <= sb;) { + int j = i; + while (j < sb && t[j] != '/') { + j++; + } + if (j > i) { + any_seg = true; + for (int k = i; k < j; k++) { + if (!((t[k] >= '0' && t[k] <= '9') || t[k] == '.' || t[k] == '-')) { + all_numeric = false; + } + } + last_s = i; + last_e = j; + } + i = j + 1; + } + if (!any_seg || all_numeric) { + return; /* numeric-path */ + } + if (in_list_lower(t + a, b - a, SLASH_STOP)) { + return; + } + int slashes = 0; + for (int i = a; i < b; i++) { + slashes += t[i] == '/'; + } + bool has_ext = has_code_ext(t + last_s, last_e - last_s); + bool prefixed = + lit_at(t, b, a, "./") || lit_at(t, b, a, "../") || lit_at(t, b, a, "~/") || t[a] == '/'; + cand_t c = {a, b, (slashes >= 2 || has_ext || prefixed) ? MK_PATH : MK_PATH1, 0, 0}; + /* norm: leading ../ ./ ~/ / removed (repeatedly), trailing '/' removed */ + int na = a; + for (;;) { + if (lit_at(t, b, na, "../")) { + na += 3; + } else if (lit_at(t, b, na, "./") || lit_at(t, b, na, "~/")) { + na += 2; + } else if (na < b && t[na] == '/') { + na += 1; + } else { + break; + } + } + int nb = b; + while (nb > na && t[nb - 1] == '/') { + nb--; + } + c.norm_start = cv->pool.n; + for (int i = na; i < nb; i++) { + u32_push(&cv->pool, t[i]); + } + c.norm_len = nb - na; + cand_push(cv, c); +} + +static void qual_candidate(const uint32_t *t, int a, int b, candv_t *cv) { + int ce = b; + if (b - a >= 2 && t[b - 2] == '(' && t[b - 1] == ')') { + ce = b - 2; + } + bool colons = false; + bool arrow = false; + int ns = cv->pool.n; + for (int i = a; i < ce; i++) { + if (i + 1 < ce && t[i] == ':' && t[i + 1] == ':') { + colons = true; + u32_push(&cv->pool, '.'); + i++; + } else if (i + 1 < ce && t[i] == '-' && t[i + 1] == '>') { + arrow = true; + u32_push(&cv->pool, '.'); + i++; + } else if (t[i] == '#') { + u32_push(&cv->pool, '.'); + } else { + u32_push(&cv->pool, t[i]); + } + } + if (cv->pool.fail) { + cv->fail = true; + return; + } + const uint32_t *norm = cv->pool.v + ns; + int nl = cv->pool.n - ns; + bool short_segs = true; + bool has_us = false; + int seg_s = 0; + int last_s = 0; + for (int i = 0; i <= nl; i++) { + if (i == nl || norm[i] == '.') { + if (i - seg_s > 2) { + short_segs = false; + } + last_s = seg_s; + seg_s = i + 1; + } else if (norm[i] == '_') { + has_us = true; + } + } + bool lower = str_islower(norm, nl); + if ((short_segs && lower) || + (in_list_lower(norm + last_s, nl - last_s, TLDS) && lower && !has_us)) { + cv->pool.n = ns; /* abbreviation or domain: taken, no candidate */ + return; + } + bool file = in_list_lower(norm + last_s, nl - last_s, CODE_EXT) && !colons && !arrow; + cand_t c = {a, b, file ? MK_FILE : MK_QUAL, ns, nl}; + cand_push(cv, c); +} + +static void tok_candidate(const uint32_t *t, int n, int a, int b, uint8_t *taken, candv_t *cv) { + int len = b - a; + /* "_" in t.strip("_") */ + int sa = a; + int sb = b; + while (sa < sb && t[sa] == '_') { + sa++; + } + while (sb > sa && t[sb - 1] == '_') { + sb--; + } + bool inner_us = false; + for (int i = sa; i < sb; i++) { + inner_us = inner_us || t[i] == '_'; + } + bool dunder = len > 4 && t[a] == '_' && t[a + 1] == '_' && t[b - 1] == '_' && t[b - 2] == '_'; + mention_kind_t kind; + if (inner_us || dunder) { + kind = MK_SNAKE; + } else if (is_camel(t + a, len)) { + kind = MK_CAMEL; + } else if (b < n && t[b] == '(' && len >= 2 && !str_isupper(t + a, len)) { + kind = MK_CALL; + } else { + return; + } + span_take(taken, a, b); + cand_t c = {a, b, kind, cv->pool.n, len}; + for (int i = a; i < b; i++) { + u32_push(&cv->pool, t[i]); + } + cand_push(cv, c); +} + +static int cand_cmp(const void *x, const void *y) { + const cand_t *a = (const cand_t *)x; + const cand_t *b = (const cand_t *)y; + return a->start < b->start ? -1 : a->start > b->start ? 1 : 0; +} + +/* candidates(): line-local candidates of text, sorted by start. */ +static bool candidates(const uint32_t *t, int n, bool with_tok, candv_t *cv) { + uint8_t *taken = (uint8_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1)); + if (!taken) { + return false; + } + url_cache_t uc = {-1, -1, -1, -1}; + for (int p = 0; p < n;) { + int e = url_at(t, n, p, &uc); + if (e > p) { + span_take(taken, p, e); + p = e; + } else { + p++; + } + } + for (int p = 0; p < n;) { + int e = path_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + span_take(taken, p, e); + path_candidate(t, p, e, cv); + } + p = e; + } + for (int p = 0; p < n;) { + int e = qual_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + span_take(taken, p, e); + qual_candidate(t, p, e, cv); + } + p = e; + } + if (with_tok) { + for (int p = 0; p < n;) { + int e = tok_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + tok_candidate(t, n, p, e, taken, cv); + } + p = e; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, taken); + if (cv->n > 1) { + qsort(cv->v, (size_t)cv->n, sizeof(cand_t), cand_cmp); + } + return !cv->fail && !cv->pool.fail; +} + +/* ── Joins across a line break ───────────────────────────────────── */ + +typedef enum { JK_NONE = 0, JK_SOFT, JK_HYPHEN, JK_CODE } join_kind_t; + +static join_kind_t join_kind(const uint32_t *prev, int pn, const uint32_t *nxt, int nn) { + if (!pn || !nn) { + return JK_NONE; + } + if (prev[pn - 1] == 0x00AD && pdf_u_alpha(nxt[0])) { + return JK_SOFT; + } + if (prev[pn - 1] == '-' && pn >= 2 && pdf_u_alpha(prev[pn - 2]) && pdf_u_lower(nxt[0])) { + return JK_HYPHEN; + } + static const char *const sufs[] = {"::", "->", "_", "/"}; + for (int k = 0; k < 4; k++) { + int sl = (int)strlen(sufs[k]); + if (pn > sl && lit_at(prev, pn, pn - sl, sufs[k]) && idc(prev[pn - sl - 1]) && + idc(nxt[0])) { + return JK_CODE; + } + } + if (idc(prev[pn - 1])) { + /* (?:\.|::|->|_)[A-Za-z_] | \(\) at the start of nxt */ + int s = 0; + if (nxt[0] == '.' || nxt[0] == '_') { + s = 1; + } else if (nn >= 2 && + ((nxt[0] == ':' && nxt[1] == ':') || (nxt[0] == '-' && nxt[1] == '>'))) { + s = 2; + } + if (s && s < nn && (is_alpha_ascii(nxt[s]) || nxt[s] == '_')) { + return JK_CODE; + } + if (nn >= 2 && nxt[0] == '(' && nxt[1] == ')') { + return JK_CODE; + } + } + return JK_NONE; +} + +/* [A-Za-z0-9_$@.:/\-#>­] and [A-Za-z0-9_$@.:/\-#>()] */ +static bool left_run_char(uint32_t c) { + return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' || + c == '>' || c == 0x00AD; +} + +static bool right_run_char(uint32_t c) { + return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' || + c == '>' || c == '(' || c == ')'; +} + +/* ── Emission ────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const char *page_qn; + uint32_t page; + const uint32_t *flat; + int n; + const int *line_starts; + int nlines; + bool failed; +} pdf_emit_t; + +static uint32_t line_of(const pdf_emit_t *e, int off) { + int lo = 0; + int hi = e->nlines - 1; + while (lo < hi) { + int mid = lo + (hi - lo + 1) / 2; + if (e->line_starts[mid] <= off) { + lo = mid; + } else { + hi = mid - 1; + } + } + return (uint32_t)lo + 1; +} + +/* R3: "foo.bar=" right after "(" or "," -- a keyword argument, not a reference */ +static bool kwarg_at(const pdf_emit_t *e, int start, int end) { + if (end >= e->n || e->flat[end] != '=' || (end + 1 < e->n && e->flat[end + 1] == '=')) { + return false; + } + int lim = start - PDF_R3_WINDOW < 0 ? 0 : start - PDF_R3_WINDOW; + for (int i = start - 1; i >= lim; i--) { + if (!pdf_u_space(e->flat[i])) { + return e->flat[i] == '(' || e->flat[i] == ','; + } + } + return false; +} + +static int u8_put(uint32_t cp, char *o) { + if (cp < 0x80) { + o[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + o[0] = (char)(0xC0 | (cp >> 6)); + o[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + if (cp < 0x10000) { + o[0] = (char)(0xE0 | (cp >> 12)); + o[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + o[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + o[0] = (char)(0xF0 | (cp >> 18)); + o[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + o[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + o[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +static int kind_syntax(mention_kind_t k) { + switch (k) { + case MK_PATH: + case MK_PATH1: + return CBM_DOCLINK_PDF_PATH; + case MK_FILE: + return CBM_DOCLINK_PDF_FILE; + case MK_QUAL: + return CBM_DOCLINK_PDF_QN; + default: + return CBM_DOCLINK_PDF_NAME; + } +} + +static bool structural(mention_kind_t k) { + return k == MK_PATH || k == MK_PATH1 || k == MK_FILE || k == MK_QUAL; +} + +/* Push a token; returns its index in the file's token array, or -1. */ +static int emit(pdf_emit_t *e, int syntax, const char *raw, uint32_t line, uint16_t flags) { + CBMArena *a = e->ctx->arena; + char *r = cbm_arena_strdup(a, raw); + if (!r) { + e->failed = true; + return -1; + } + CBMDocLink link = { + .source_qn = e->page_qn, + .raw = r, + .line = line, + .def_line = e->page, + .syntax = (uint16_t)syntax, + .flags = flags, + }; + int idx = e->ctx->result->doc_links.count; + cbm_doclinks_push(&e->ctx->result->doc_links, a, link); + if (e->ctx->result->doc_links.count != idx + 1) { + e->failed = true; + return -1; + } + return idx; +} + +typedef struct { + char *p; + size_t n; + size_t cap; + bool fail; +} sbuf_t; + +static void sb_put(sbuf_t *b, const char *s, size_t n) { + if (b->fail) { + return; + } + if (b->n + n + 1 > b->cap) { + size_t ncap = b->cap ? b->cap * 2 : 256; + while (ncap < b->n + n + 1) { + ncap *= 2; + } + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->p, ncap); + if (!g) { + b->fail = true; + return; + } + b->p = g; + b->cap = ncap; + } + memcpy(b->p + b->n, s, n); + b->n += n; + b->p[b->n] = '\0'; +} + +static void sb_cps(sbuf_t *b, const uint32_t *s, int n) { + char u[PDF_U8_MAX]; + for (int i = 0; i < n; i++) { + int k = u8_put(s[i], u); + sb_put(b, u, (size_t)k); + } +} + +/* One page: its line-local structural tokens, then its join proposals. */ +static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn) { + u32v_t norm = {0}; + if (!normalise(text, len, &norm)) { + u32_free(&norm); + ctx->result->doc_links.failed = true; + return; + } + /* lines, each stripped; flat = lines joined by '\n' */ + u32v_t flat = {0}; + int *starts = NULL; + int nlines = 0; + int cap = 0; + int *lens = NULL; + for (int i = 0; i <= norm.n;) { + int j = i; + while (j < norm.n && norm.v[j] != '\n') { + j++; + } + int a = i; + int b = j; + while (a < b && pdf_u_space(norm.v[a])) { + a++; + } + while (b > a && pdf_u_space(norm.v[b - 1])) { + b--; + } + if (nlines == cap) { + cap = cap ? cap * 2 : PDF_LIST_MIN; + int *g1 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, starts, (size_t)cap * sizeof(int)); + if (g1) { + starts = g1; + } + int *g2 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, lens, (size_t)cap * sizeof(int)); + if (g2) { + lens = g2; + } + if (!g1 || !g2) { + flat.fail = true; + break; + } + } + if (nlines) { + u32_push(&flat, '\n'); + } + starts[nlines] = flat.n; + lens[nlines] = b - a; + nlines++; + for (int k = a; k < b; k++) { + u32_push(&flat, norm.v[k]); + } + if (j >= norm.n) { + break; + } + i = j + 1; + } + u32_free(&norm); + candv_t cv = {0}; + if (flat.fail || !candidates(flat.v, flat.n, false, &cv)) { + ctx->result->doc_links.failed = true; + goto done; + } + pdf_emit_t e = {ctx, page_qn, page, flat.v, flat.n, starts, nlines, false}; + /* the token index of every structural candidate (-1: not emitted) */ + int *tok = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(cv.n ? cv.n : 1) * sizeof(int)); + if (!tok) { + ctx->result->doc_links.failed = true; + goto done; + } + sbuf_t sb = {0}; + for (int i = 0; i < cv.n; i++) { + const cand_t *c = &cv.v[i]; + tok[i] = -1; + if (!structural(c->kind) || kwarg_at(&e, c->start, c->end)) { + continue; + } + sb.n = 0; + sb_cps(&sb, cv.pool.v + c->norm_start, c->norm_len); + if (sb.fail) { + break; + } + tok[i] = emit(&e, kind_syntax(c->kind), sb.p ? sb.p : "", line_of(&e, c->start), 0); + } + /* joins */ + for (int li = 0; li + 1 < nlines && !sb.fail && !e.failed; li++) { + const uint32_t *prev = flat.v + starts[li]; + const uint32_t *nxt = flat.v + starts[li + 1]; + int pn = lens[li]; + int nn = lens[li + 1]; + join_kind_t jk = join_kind(prev, pn, nxt, nn); + if (jk == JK_NONE) { + continue; + } + int ls = pn; + while (ls > 0 && left_run_char(prev[ls - 1])) { + ls--; + } + int re = 0; + while (re < nn && right_run_char(nxt[re])) { + re++; + } + if (ls == pn || re == 0) { + continue; + } + int left_len = pn - ls; + int dropped = (jk == JK_HYPHEN || jk == JK_SOFT) ? 1 : 0; + int junction = left_len - dropped; + u32v_t joined = {0}; + for (int k = ls; k < pn - dropped; k++) { + u32_push(&joined, prev[k]); + } + for (int k = 0; k < re; k++) { + u32_push(&joined, nxt[k]); + } + candv_t jc = {0}; + if (joined.fail || !candidates(joined.v, joined.n, true, &jc)) { + u32_free(&joined); + candv_free(&jc); + ctx->result->doc_links.failed = true; + break; + } + int a0 = starts[li] + ls; + int b0 = starts[li + 1]; + for (int k = 0; k < jc.n; k++) { + const cand_t *c = &jc.v[k]; + if (!(c->start < junction && junction < c->end)) { + continue; + } + int s_flat = a0 + c->start; + int e_flat = b0 + (c->end - junction); + if (structural(c->kind) && kwarg_at(&e, s_flat, e_flat)) { + continue; + } + sb.n = 0; + const uint32_t *nm = jc.pool.v + c->norm_start; + sb_cps(&sb, nm, c->norm_len); + if (dropped) { + /* alts: the norm with the hyphen kept, at the junction */ + /* Python slices clamp: past the end, the hyphen goes last */ + int at = junction - c->start; + at = at > c->norm_len ? c->norm_len : at; + char sep = PDF_JOIN_SEP; + sb_put(&sb, &sep, 1); + sb_cps(&sb, nm, at); + sb_put(&sb, "-", 1); + sb_cps(&sb, nm + at, c->norm_len - at); + } + char fs = PDF_JOIN_FRAG; + sb_put(&sb, &fs, 1); + bool first = true; + for (int f = 0; f < cv.n; f++) { + const cand_t *x = &cv.v[f]; + if (x->start < e_flat && x->end > s_flat && tok[f] >= 0) { + char num[16]; + int nlen = snprintf(num, sizeof(num), first ? "%d" : ",%d", tok[f]); + sb_put(&sb, num, (size_t)nlen); + first = false; + } + } + if (sb.fail) { + break; + } + emit(&e, kind_syntax(c->kind), sb.p, line_of(&e, s_flat), CBM_DOCLINK_FLAG_JOIN); + } + u32_free(&joined); + candv_free(&jc); + } + if (sb.fail || e.failed) { + ctx->result->doc_links.failed = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, sb.p); + cbm_free(CBM_MEM_CLASS_EXTRACT, tok); +done: + candv_free(&cv); + u32_free(&flat); + cbm_free(CBM_MEM_CLASS_EXTRACT, starts); + cbm_free(CBM_MEM_CLASS_EXTRACT, lens); +} + +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn) { + scan_page(ctx, text, len, page, page_qn); +} +#endif + +/* ── The document ────────────────────────────────────────────────── */ + +/* R1: the field test's category(): a PDF under a test or fixture directory. */ +static bool fixture_pdf(const char *rel_path) { + const char *s = rel_path; + for (;;) { + const char *slash = strchr(s, '/'); + if (!slash) { + return false; + } + size_t n = (size_t)(slash - s); + char comp[256]; + if (n < sizeof(comp)) { + for (size_t i = 0; i < n; i++) { + comp[i] = (char)(s[i] >= 'A' && s[i] <= 'Z' ? s[i] + ('a' - 'A') : s[i]); + } + comp[n] = '\0'; + for (int i = 0; FIXTURE_DIRS[i]; i++) { + if (strcmp(comp, FIXTURE_DIRS[i]) == 0) { + return true; + } + } + if (strstr(comp, "test") && strcmp(comp, "testing") != 0) { + return true; + } + } + s = slash + 1; + } +} + +void cbm_pdf_extract_document(CBMExtractCtx *ctx) { + CBMFileResult *result = ctx->result; + CBMArena *a = ctx->arena; + cbm_pdf_result_t pr; + if (!cbm_pdf_extract((const unsigned char *)ctx->source, (size_t)ctx->source_len, a, &pr)) { + result->has_error = true; + result->error_msg = cbm_arena_strdup(a, "pdf: out of memory"); + return; + } + if (pr.status != CBM_PDF_OK) { + static const char *const why[] = {"ok", "pdf: not a PDF", "pdf: no pages", "pdf: encrypted", + "pdf: out of memory"}; + result->has_error = pr.status != CBM_PDF_ENCRYPTED; + result->error_msg = cbm_arena_strdup(a, why[pr.status]); + return; + } + bool fixture = fixture_pdf(ctx->rel_path ? ctx->rel_path : ""); + for (int i = 0; i < pr.npages; i++) { + uint32_t page = (uint32_t)i + 1; + char name[PDF_PAGE_NAME]; + char seg[PDF_PAGE_NAME]; + snprintf(name, sizeof(name), "page %u", page); + snprintf(seg, sizeof(seg), "page_%u", page); + char *nm = cbm_arena_strdup(a, name); + char *qn = cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg); + char *pv = cbm_arena_sprintf(a, "%u", page); + const char **kv = (const char **)cbm_arena_alloc(a, 3 * sizeof(char *)); + if (!nm || !qn || !pv || !kv) { + result->has_error = true; + return; + } + kv[0] = "page"; + kv[1] = pv; + kv[2] = NULL; + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = nm; + def.qualified_name = qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = page; + def.end_line = page; + def.is_exported = true; + def.docstring = pr.pages[i].len ? pr.pages[i].text : NULL; + def.extra_props = kv; + cbm_defs_push(&result->defs, a, def); + if (!fixture && pr.pages[i].len) { + scan_page(ctx, pr.pages[i].text, pr.pages[i].len, page, qn); + } + } +} diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 99995202e2..ecdae767cd 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -77,6 +77,12 @@ X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ + /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ + /* names) only decides which line fragments a cross-line join replaces */ \ + X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") @@ -98,6 +104,10 @@ enum { * file-level doc -- a parser never sets it. A scan_file hook sets it on a * reference written before the document's first section. */ CBM_DOCLINK_FLAG_FILE = 1, + /* A PDF mention that runs across a line break, proposed as one reference. + * `raw` holds its forms (separated by 0x1F) and, after 0x1E, the indices + * of the line-local tokens it replaces when it resolves (doc_pdf.c). */ + CBM_DOCLINK_FLAG_JOIN = 2, }; /* One link family: a line of CBM_DOCLINK_FAMILY_LIST. */ @@ -295,4 +305,15 @@ void cbm_adr_extract(CBMExtractCtx *ctx); /* The qualified-name tail of an ADR node: "" + this. */ #define CBM_ADR_QN_NAME "__adr__" +/* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's + * text as docstring, property page) and the pages' structural code mentions + * as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */ +void cbm_pdf_extract_document(CBMExtractCtx *ctx); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +/* Test seam: scan one page's text as cbm_pdf_extract_document does (tokens + * into ctx->result->doc_links), without a PDF around it. */ +void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn); +#endif + #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 2c385cf33d..a37c3e52ce 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -2713,6 +2713,11 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { empty_types, NULL, NULL, tree_sitter_objectscript_routine, NULL}, + // CBM_LANG_PDF — PDF documents. No grammar row: extraction reads the text + // layer itself (cbm_pdf_extract_document, doc_pdf.c) before any grammar is + // looked up, so this language never reaches cbm_lang_spec()/cbm_ts_language(). + // Left as a zero spec. + // CBM_LANG_OBJECTSCRIPT_EXPORT — Studio Export XML. No grammar row: the // pipeline transcodes Export XML to UDL (iris_export_xml.c) and re-extracts // each class as CBM_LANG_OBJECTSCRIPT_UDL, so this language never reaches diff --git a/internal/cbm/pdf/pdf.h b/internal/cbm/pdf/pdf.h new file mode 100644 index 0000000000..508b2158bd --- /dev/null +++ b/internal/cbm/pdf/pdf.h @@ -0,0 +1,60 @@ +/* + * pdf.h — the text layer of a PDF, page by page. + * + * Reads what a PDF says in text (content-stream text operators decoded through + * the document's fonts) and nothing else: no rendering, no images, no OCR, no + * JavaScript, actions or URIs, nothing fetched, nothing executed. Input is + * untrusted: every read is bounds-checked, every recursion is depth-limited, + * every decoder is guarded against decompression bombs. + * + * Not read: encrypted documents (the result says so). Scanned pages have no + * text layer and come back empty. + */ +#ifndef CBM_PDF_H +#define CBM_PDF_H + +#include "../arena.h" + +#include +#include + +typedef enum { + CBM_PDF_OK = 0, + CBM_PDF_NOT_PDF, /* no "%PDF-" within the first 1024 bytes */ + CBM_PDF_PARSE_FAILURE, /* no page could be found */ + CBM_PDF_ENCRYPTED, /* an Encrypt dictionary: the text is not read */ + CBM_PDF_NOMEM, +} cbm_pdf_status_t; + +typedef struct { + const char *text; /* UTF-8, NUL-terminated, no NUL inside */ + size_t len; + int glyphs; /* glyphs shown on the page */ + int unmapped; /* glyphs without a Unicode value (U+FFFD in the text) */ +} cbm_pdf_page_t; + +/* How the document was read: counts of the repairs and limits that applied. */ +typedef struct { + int xref_reconstructed; /* cross-reference rebuilt from a scan of the file */ + int xref_offset_fixed; /* an xref section found near a wrong offset */ + int stream_length_recovered; /* a stream's /Length was wrong */ + int flate_recovered; /* a damaged Flate stream decoded in part */ + int stream_guard_hits; /* a decoded stream cut at 512 MiB */ + int filters_unsupported; /* a stream with a filter this reader does not decode */ + int op_errors; /* content operators that could not be applied */ + int page_errors; /* pages whose content stopped early */ +} cbm_pdf_counts_t; + +typedef struct { + cbm_pdf_status_t status; + cbm_pdf_page_t *pages; + int npages; + cbm_pdf_counts_t counts; +} cbm_pdf_result_t; + +/* Read the text layer of data[0..len). Everything the result points to is + * allocated from `out` and lives as long as it. Returns false only when memory + * ran out (status CBM_PDF_NOMEM). */ +bool cbm_pdf_extract(const unsigned char *data, size_t len, CBMArena *out, cbm_pdf_result_t *res); + +#endif /* CBM_PDF_H */ diff --git a/internal/cbm/pdf/pdf_font.c b/internal/cbm/pdf/pdf_font.c new file mode 100644 index 0000000000..b1d531e297 --- /dev/null +++ b/internal/cbm/pdf/pdf_font.c @@ -0,0 +1,1978 @@ +/* + * pdf_font.c — fonts of the PDF text-layer extractor: what a shown byte + * string says in Unicode, and how far each glyph advances. + * + * Simple fonts map one byte per glyph: ToUnicode first, then the encoding's + * glyph name (a base encoding, /Differences, a Type1 program's built-in + * table), then -- for fonts whose encoding cannot be read -- the byte itself + * when it is printable ASCII. Type0 fonts split bytes by their CMap's code + * space (or two bytes per code), map through ToUnicode, a UCS-2 CMap, or the + * embedded TrueType program's cmap read backwards. + * + * Ranges (CMap bfrange/cidrange, CID widths) are painted once into disjoint + * segments that keep the prototype's precedence (the first range written wins + * a code; the last /W entry wins a CID), so a lookup is a binary search. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" + +#include +#include +#include + +enum { + PDF_CODE_BYTES = 4, /* CMap codes longer than this are ignored */ + PDF_BIG_RANGE = 4096, /* bfranges up to this size are written out */ + PDF_GID_SPACE = 65536, + PDF_UTF8_MAX = 4, + PDF_INLINE_MAX = 15, /* pdf_glyph_out_t.inl capacity, minus the NUL */ + PDF_STD_FIRST = 0x20, + PDF_STD_LAST = 0x7E, + PDF_DEFAULT_WIDTH = 500, + PDF_COURIER_WIDTH = 600, + PDF_TYPE1_CLEAR = 65536, /* the prototype's look into a Type1 program without Length1 */ + PDF_FLAG_SYMBOLIC = 4, + PDF_LIST_MIN = 16, +}; + +#define PDF_SURR_LO 0xD800U +#define PDF_SURR_HI 0xDFFFU +#define PDF_MAX_CP 0x10FFFFU +#define PDF_REPL 0xFFFDU + +/* ── UTF-8 ───────────────────────────────────────────────────────── */ + +/* One code point as UTF-8. U+0000 is written C0 80 so a glyph that maps to it + * still counts as text in the layout (the prototype emitted it); the page + * text drops it at the end. Surrogates become U+FFFD. */ +static int utf8_put(uint32_t cp, char *out) { + if (cp == 0) { + out[0] = (char)0xC0; + out[1] = (char)0x80; + return 2; + } + if ((cp >= PDF_SURR_LO && cp <= PDF_SURR_HI) || cp > PDF_MAX_CP) { + cp = PDF_REPL; + } + if (cp < 0x80) { + out[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + if (cp < 0x10000) { + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +static char *persist(pdf_doc_t *d, const char *s, size_t n) { + char *o = (char *)cbm_arena_alloc(d->a, n + 1); + if (!o) { + d->nomem = true; + return NULL; + } + memcpy(o, s, n); + o[n] = '\0'; + return o; +} + +/* UTF-16BE bytes as UTF-8 (the prototype's _u16be): one byte is a Latin-1 + * character; an odd tail byte is dropped; a lone surrogate is U+FFFD. Writes + * to buf when it fits, else to the arena. */ +static const char *u16be(pdf_doc_t *d, const unsigned char *b, size_t n, char *buf, size_t cap) { + char tmp[64]; + char *out = tmp; + size_t need = n * 2 + 4; + if (need > sizeof(tmp)) { + out = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, need); + if (!out) { + d->nomem = true; + return NULL; + } + } + size_t k = 0; + if (n == 1) { + k = (size_t)utf8_put(b[0], out); + } else { + n &= ~(size_t)1; + for (size_t i = 0; i < n; i += 2) { + uint32_t u = ((uint32_t)b[i] << 8) | b[i + 1]; + if (u >= PDF_SURR_LO && u <= 0xDBFFU && i + 3 < n) { + uint32_t lo = ((uint32_t)b[i + 2] << 8) | b[i + 3]; + if (lo >= 0xDC00U && lo <= PDF_SURR_HI) { + uint32_t cp = 0x10000U + ((u - PDF_SURR_LO) << 10) + (lo - 0xDC00U); + k += (size_t)utf8_put(cp, out + k); + i += 2; + continue; + } + } + k += (size_t)utf8_put(u, out + k); /* a lone surrogate becomes U+FFFD */ + } + } + const char *res; + if (buf && k < cap) { + memcpy(buf, out, k); + buf[k] = '\0'; + res = buf; + } else { + res = persist(d, out, k); + } + if (out != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, out); + } + return res; +} + +/* ── Glyph names ─────────────────────────────────────────────────── */ + +static const char *glyph_lookup(const unsigned char *name, size_t n) { + int lo = 0; + int hi = PDF_GLYPH_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + const char *g = PDF_GLYPHS[mid].name; + size_t gn = strlen(g); + size_t m = gn < n ? gn : n; + int c = m ? memcmp(g, name, m) : 0; + if (!c) { + c = gn < n ? -1 : gn > n ? 1 : 0; + } + if (!c) { + return PDF_GLYPHS[mid].utf8; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return NULL; +} + +static bool all_hex(const unsigned char *s, size_t n) { + for (size_t i = 0; i < n; i++) { + unsigned char c = s[i]; + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + return true; +} + +static uint32_t hex_num(const unsigned char *s, size_t n) { + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = s[i]; + uint32_t x = c <= '9' ? (uint32_t)(c - '0') : (uint32_t)((c | 0x20) - 'a' + 10); + v = v * 16 + x; + } + return v; +} + +static const char *g2u(pdf_doc_t *d, const unsigned char *name, size_t n, int depth) { + const char *r = glyph_lookup(name, n); + if (r) { + return r; + } + const unsigned char *base = name; + size_t bn = n; + const unsigned char *dot = n > 1 ? (const unsigned char *)memchr(name + 1, '.', n - 1) : NULL; + if (dot) { + const unsigned char *first = (const unsigned char *)memchr(name, '.', n); + bn = (size_t)(first - name); + r = glyph_lookup(base, bn); + if (r) { + return r; + } + if (!bn) { + return NULL; + } + } + if (memchr(base, '_', bn)) { + if (depth > 0) { + return NULL; + } + char tmp[256]; + size_t k = 0; + char *big = NULL; + size_t cap = sizeof(tmp); + char *out = tmp; + size_t i = 0; + for (;;) { + const unsigned char *us = (const unsigned char *)memchr(base + i, '_', bn - i); + size_t pe = us ? (size_t)(us - base) : bn; + const char *part = g2u(d, base + i, pe - i, depth + 1); + if (!part) { + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + return NULL; + } + size_t pl = strlen(part); + if (k + pl + 1 > cap) { + size_t ncap = (k + pl + 1) * 2; + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, big, ncap); + if (!g) { + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + d->nomem = true; + return NULL; + } + if (!big) { + memcpy(g, tmp, k); + } + big = g; + out = big; + cap = ncap; + } + memcpy(out + k, part, pl); + k += pl; + if (!us) { + break; + } + i = pe + 1; + } + const char *res = persist(d, out, k); + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + return res; + } + if (bn >= 7 && memcmp(base, "uni", 3) == 0 && (bn - 3) % 4 == 0 && all_hex(base + 3, bn - 3)) { + size_t units = (bn - 3) / 4; + char *o = (char *)cbm_arena_alloc(d->a, units * PDF_UTF8_MAX + 1); + if (!o) { + d->nomem = true; + return NULL; + } + size_t k = 0; + for (size_t i = 3; i < bn; i += 4) { + uint32_t v = hex_num(base + i, 4); + if (v >= PDF_SURR_LO && v <= PDF_SURR_HI) { + return NULL; + } + k += (size_t)utf8_put(v, o + k); + } + o[k] = '\0'; + return o; + } + if (bn >= 5 && bn <= 7 && base[0] == 'u' && all_hex(base + 1, bn - 1)) { + uint32_t v = hex_num(base + 1, bn - 1); + if (v <= PDF_MAX_CP && !(v >= PDF_SURR_LO && v <= PDF_SURR_HI)) { + char b[PDF_UTF8_MAX]; + int k = utf8_put(v, b); + return persist(d, b, (size_t)k); + } + } + if (bn == 1) { + char b[PDF_UTF8_MAX]; + int k = utf8_put(base[0], b); + return persist(d, b, (size_t)k); + } + return NULL; +} + +const char *pdf_glyph_to_utf8(pdf_doc_t *d, const unsigned char *name, size_t n) { + return name ? g2u(d, name, n, 0) : NULL; +} + +/* ── Interval painting ───────────────────────────────────────────── */ + +typedef struct { + uint64_t lo; + uint64_t hi; + int idx; /* written order */ +} pdf_iv_t; + +typedef struct { + uint64_t lo; + uint64_t hi; + int idx; +} pdf_seg_t; + +typedef struct { + pdf_iv_t *v; + int n; + int cap; +} iv_list_t; + +static bool ivl_push(pdf_doc_t *d, iv_list_t *l, uint64_t lo, uint64_t hi, int idx) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_LIST_MIN; + pdf_iv_t *g = + (pdf_iv_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].lo = lo; + l->v[l->n].hi = hi; + l->v[l->n].idx = idx; + l->n++; + return true; +} + +static int iv_lo_cmp(const void *x, const void *y) { + const pdf_iv_t *a = (const pdf_iv_t *)x; + const pdf_iv_t *b = (const pdf_iv_t *)y; + if (a->lo != b->lo) { + return a->lo < b->lo ? -1 : 1; + } + return a->idx < b->idx ? -1 : a->idx > b->idx ? 1 : 0; +} + +static int u64_cmp(const void *x, const void *y) { + uint64_t a = *(const uint64_t *)x; + uint64_t b = *(const uint64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +/* A binary heap of interval positions, ordered by written order (smallest + * first for first-wins, largest first for last-wins). */ +typedef struct { + int *h; + int n; + const pdf_iv_t *iv; + bool first_wins; +} iv_heap_t; + +static bool heap_before(const iv_heap_t *hp, int a, int b) { + return hp->first_wins ? hp->iv[a].idx < hp->iv[b].idx : hp->iv[a].idx > hp->iv[b].idx; +} + +static void heap_push(iv_heap_t *hp, int x) { + int i = hp->n++; + hp->h[i] = x; + while (i > 0) { + int p = (i - 1) / 2; + if (!heap_before(hp, hp->h[i], hp->h[p])) { + break; + } + int t = hp->h[i]; + hp->h[i] = hp->h[p]; + hp->h[p] = t; + i = p; + } +} + +static void heap_pop(iv_heap_t *hp) { + hp->h[0] = hp->h[--hp->n]; + int i = 0; + for (;;) { + int l = 2 * i + 1; + int r = l + 1; + int m = i; + if (l < hp->n && heap_before(hp, hp->h[l], hp->h[m])) { + m = l; + } + if (r < hp->n && heap_before(hp, hp->h[r], hp->h[m])) { + m = r; + } + if (m == i) { + break; + } + int t = hp->h[i]; + hp->h[i] = hp->h[m]; + hp->h[m] = t; + i = m; + } +} + +/* Disjoint segments, ascending, each owned by the winning interval. The list + * is consumed (sorted). Intervals must have hi < UINT64_MAX. */ +static pdf_seg_t *paint(pdf_doc_t *d, iv_list_t *l, bool first_wins, int *nout) { + *nout = 0; + if (!l->n) { + return NULL; + } + int n = l->n; + qsort(l->v, (size_t)n, sizeof(*l->v), iv_lo_cmp); + uint64_t *pts = (uint64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)n * 2 * sizeof(uint64_t)); + int *heap = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)n * sizeof(int)); + pdf_seg_t *seg = (pdf_seg_t *)pdf_big(d, (size_t)n * 2 * sizeof(pdf_seg_t)); + if (!pts || !heap || !seg) { + cbm_free(CBM_MEM_CLASS_EXTRACT, pts); + cbm_free(CBM_MEM_CLASS_EXTRACT, heap); + d->nomem = true; + return NULL; + } + int np = 0; + for (int i = 0; i < n; i++) { + pts[np++] = l->v[i].lo; + pts[np++] = l->v[i].hi + 1; + } + qsort(pts, (size_t)np, sizeof(uint64_t), u64_cmp); + iv_heap_t hp = {heap, 0, l->v, first_wins}; + int next = 0; + int ns = 0; + for (int k = 0; k < np; k++) { + if (k > 0 && pts[k] == pts[k - 1]) { + continue; + } + uint64_t x = pts[k]; + while (next < n && l->v[next].lo <= x) { + heap_push(&hp, next++); + } + while (hp.n && l->v[hp.h[0]].hi < x) { + heap_pop(&hp); + } + if (!hp.n) { + continue; + } + int k2 = k + 1; + while (k2 < np && pts[k2] == x) { + k2++; + } + if (k2 >= np) { + break; + } + uint64_t end = pts[k2] - 1; + int w = l->v[hp.h[0]].idx; + if (ns && seg[ns - 1].idx == w && seg[ns - 1].hi + 1 == x) { + seg[ns - 1].hi = end; + } else { + seg[ns].lo = x; + seg[ns].hi = end; + seg[ns].idx = w; + ns++; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, pts); + cbm_free(CBM_MEM_CLASS_EXTRACT, heap); + *nout = ns; + return seg; +} + +static int seg_find(const pdf_seg_t *s, int n, uint64_t key) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (s[mid].hi < key) { + lo = mid + 1; + } else if (s[mid].lo > key) { + hi = mid - 1; + } else { + return s[mid].idx; + } + } + return -1; +} + +/* ── CMaps ───────────────────────────────────────────────────────── */ + +typedef struct { + uint64_t key; /* bytes << 32 | code */ + const char *utf8; + int64_t cid; + int seq; +} cm_ent_t; + +typedef struct { + uint64_t lo; + uint64_t hi; + const unsigned char *prefix; + size_t plen; + uint32_t last; +} cm_urange_t; + +typedef struct { + uint64_t lo; + int64_t cid0; +} cm_crange_t; + +typedef struct { + cm_ent_t *uni; + int nuni; + cm_urange_t *ur; /* big bfranges in written order */ + pdf_seg_t *urseg; + int nurseg; + cm_ent_t *cid; + int ncid; + cm_crange_t *cr; /* cidranges in written order */ + pdf_seg_t *crseg; + int ncrseg; + pdf_seg_t *space[PDF_CODE_BYTES + 1]; /* code space per byte count (merged) */ + int nspace[PDF_CODE_BYTES + 1]; + bool has_space; +} pdf_cmap_t; + +typedef struct { + cm_ent_t *v; + int n; + int cap; +} ent_list_t; + +static bool el_push(pdf_doc_t *d, ent_list_t *l, cm_ent_t e) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_LIST_MIN; + cm_ent_t *g = + (cm_ent_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + e.seq = l->n; + l->v[l->n++] = e; + return true; +} + +static int ent_cmp(const void *x, const void *y) { + const cm_ent_t *a = (const cm_ent_t *)x; + const cm_ent_t *b = (const cm_ent_t *)y; + if (a->key != b->key) { + return a->key < b->key ? -1 : 1; + } + return a->seq < b->seq ? -1 : a->seq > b->seq ? 1 : 0; +} + +/* Sorted, one entry per key: the last written. */ +static cm_ent_t *el_finish(pdf_doc_t *d, ent_list_t *l, int *nout) { + *nout = 0; + if (!l->n) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return NULL; + } + qsort(l->v, (size_t)l->n, sizeof(*l->v), ent_cmp); + cm_ent_t *o = (cm_ent_t *)pdf_big(d, (size_t)l->n * sizeof(*o)); + int k = 0; + if (o) { + for (int i = 0; i < l->n; i++) { + if (i + 1 < l->n && l->v[i + 1].key == l->v[i].key) { + continue; + } + o[k++] = l->v[i]; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + *nout = k; + return o; +} + +static const cm_ent_t *ent_find(const cm_ent_t *v, int n, uint64_t key) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (v[mid].key < key) { + lo = mid + 1; + } else if (v[mid].key > key) { + hi = mid - 1; + } else { + return &v[mid]; + } + } + return NULL; +} + +static uint32_t be_code(const unsigned char *s, size_t n) { + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + v = (v << 8) | s[i]; + } + return v; +} + +static uint64_t cm_key(size_t n, uint32_t code) { + return ((uint64_t)n << 32) | code; +} + +typedef struct { + ent_list_t uni; + ent_list_t cid; + cm_urange_t *ur; + int nur; + int cur_cap; + iv_list_t ur_iv; + cm_crange_t *cr; + int ncr; + int ccap; + iv_list_t cr_iv; + iv_list_t space[PDF_CODE_BYTES + 1]; + bool any_space; /* the prototype tested its code-space list for being non-empty */ +} cm_build_t; + +static bool is_code_str(const pdf_val_t *v) { + return v && v->kind == PV_STR && v->n >= 1 && v->n <= PDF_CODE_BYTES; +} + +static bool is_int_val(const pdf_val_t *v) { + return v && v->kind == PV_NUM && v->is_int; +} + +static void cm_flush(pdf_doc_t *d, cm_build_t *b, const char *state, pdf_val_t **buf, int n) { + if (strcmp(state, "codespacerange") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + if (lo && lo->kind == PV_STR && lo->n && hi && hi->kind == PV_STR) { + b->any_space = true; + } + if (is_code_str(lo) && hi && hi->kind == PV_STR) { + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (z >= a) { + ivl_push(d, &b->space[m], a, z, b->space[m].n); + } + } + } + } else if (strcmp(state, "bfchar") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *s = buf[i]; + const pdf_val_t *dst = buf[i + 1]; + if (!is_code_str(s)) { + continue; + } + cm_ent_t e = {cm_key(s->n, be_code(s->s, s->n)), NULL, 0, 0}; + if (dst && dst->kind == PV_STR) { + e.utf8 = u16be(d, dst->s, dst->n, NULL, 0); + } else if (dst && dst->kind == PV_NAME) { + e.utf8 = g2u(d, dst->s, dst->n, 0); + if (!e.utf8) { + continue; + } + } else if (dst && dst->kind == PV_NUM && dst->is_int) { + char tmp[PDF_UTF8_MAX]; + uint32_t cp = (dst->inum >= 0 && dst->inum <= (int64_t)PDF_MAX_CP) + ? (uint32_t)dst->inum + : PDF_REPL; + int k = utf8_put(cp, tmp); + e.utf8 = persist(d, tmp, (size_t)k); + } else { + continue; + } + if (e.utf8) { + el_push(d, &b->uni, e); + } + } + } else if (strcmp(state, "bfrange") == 0) { + for (int i = 0; i + 2 < n; i += 3) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + const pdf_val_t *dst = buf[i + 2]; + if (!is_code_str(lo) || !hi || hi->kind != PV_STR) { + continue; + } + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (z < a) { + continue; + } + if (dst && dst->kind == PV_ARR) { + uint64_t cnt = z - a + 1; + for (int j = 0; j < dst->count && (uint64_t)j < cnt; j++) { + const pdf_val_t *x = dst->items[j]; + if (x && x->kind == PV_STR) { + cm_ent_t e = {cm_key(m, (uint32_t)(a + (uint64_t)j)), NULL, 0, 0}; + e.utf8 = u16be(d, x->s, x->n, NULL, 0); + if (e.utf8) { + el_push(d, &b->uni, e); + } + } + } + } else if (dst && dst->kind == PV_STR && dst->n) { + unsigned char two[2] = {0, dst->s[0]}; + const unsigned char *db = dst->s; + size_t dn = dst->n; + if (dn == 1) { + db = two; + dn = 2; + } + uint32_t last = ((uint32_t)db[dn - 2] << 8) | db[dn - 1]; + size_t plen = dn - 2; + if (z - a <= PDF_BIG_RANGE) { + unsigned char tmp[64]; + unsigned char *w = + plen + 2 <= sizeof(tmp) + ? tmp + : (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, plen + 2); + if (!w) { + d->nomem = true; + return; + } + memcpy(w, db, plen); + for (uint64_t j = 0; j <= z - a; j++) { + uint32_t u = (last + (uint32_t)j) & 0xFFFFU; + w[plen] = (unsigned char)(u >> 8); + w[plen + 1] = (unsigned char)u; + cm_ent_t e = {cm_key(m, (uint32_t)(a + j)), NULL, 0, 0}; + e.utf8 = u16be(d, w, plen + 2, NULL, 0); + if (!e.utf8 || !el_push(d, &b->uni, e)) { + break; + } + } + if (w != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, w); + } + } else { + if (b->nur == b->cur_cap) { + int ncap = b->cur_cap ? b->cur_cap * 2 : PDF_LIST_MIN; + cm_urange_t *g = (cm_urange_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->ur, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + b->ur = g; + b->cur_cap = ncap; + } + cm_urange_t *r = &b->ur[b->nur]; + r->lo = cm_key(m, (uint32_t)a); + r->hi = cm_key(m, (uint32_t)z); + /* the token lives in the parse arena: keep a copy */ + r->prefix = (const unsigned char *)persist(d, (const char *)db, plen); + if (!r->prefix) { + return; + } + r->plen = plen; + r->last = last; + ivl_push(d, &b->ur_iv, r->lo, r->hi, b->nur); + b->nur++; + } + } + } + } else if (strcmp(state, "cidrange") == 0 || strcmp(state, "notdefrange") == 0) { + for (int i = 0; i + 2 < n; i += 3) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + const pdf_val_t *c = buf[i + 2]; + if (!is_code_str(lo) || !hi || hi->kind != PV_STR || !is_int_val(c)) { + continue; + } + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (b->ncr == b->ccap) { + int ncap = b->ccap ? b->ccap * 2 : PDF_LIST_MIN; + cm_crange_t *g = (cm_crange_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->cr, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + b->cr = g; + b->ccap = ncap; + } + b->cr[b->ncr].lo = cm_key(m, (uint32_t)a); + b->cr[b->ncr].cid0 = c->inum; + if (z >= a) { + ivl_push(d, &b->cr_iv, cm_key(m, (uint32_t)a), cm_key(m, (uint32_t)z), b->ncr); + } + b->ncr++; + } + } else if (strcmp(state, "cidchar") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *s = buf[i]; + const pdf_val_t *c = buf[i + 1]; + if (is_code_str(s) && is_int_val(c)) { + cm_ent_t e = {cm_key(s->n, be_code(s->s, s->n)), NULL, c->inum, 0}; + el_push(d, &b->cid, e); + } + } + } +} + +static const char *const CM_BEGIN[] = {"begincodespacerange", "beginbfchar", "beginbfrange", + "begincidrange", "begincidchar", "beginnotdefrange"}; + +static pdf_cmap_t *cmap_parse(pdf_doc_t *d, const unsigned char *data, size_t len) { + if (!d->scratch_arena_live) { + cbm_arena_init_lazy(&d->scratch_arena, CBM_ARENA_DEFAULT_BLOCK_SIZE); + d->scratch_arena_live = true; + } + CBMArena *tmp = &d->scratch_arena; + CBMArena *saved = d->cur; + d->cur = tmp; + cm_build_t b; + memset(&b, 0, sizeof(b)); + pdf_val_t **buf = NULL; + int nbuf = 0; + int cap = 0; + char state[32] = ""; + bool in_state = false; + bool failed = false; + size_t pos = 0; + for (;;) { + pdf_tok_t t; + pdf_lex(d, data, len, &pos, &t); + if (t.kind == PT_EOF || d->nomem) { + break; + } + if (t.kind == PT_KW) { + bool begun = false; + for (size_t i = 0; i < sizeof(CM_BEGIN) / sizeof(CM_BEGIN[0]); i++) { + if (pdf_kw_is(&t, CM_BEGIN[i])) { + snprintf(state, sizeof(state), "%s", CM_BEGIN[i] + 5); + in_state = true; + nbuf = 0; + begun = true; + break; + } + } + if (!begun && in_state && t.n >= 3 && memcmp(t.s, "end", 3) == 0) { + d->cur = saved; + cm_flush(d, &b, state, buf, nbuf); + d->cur = tmp; + in_state = false; + nbuf = 0; + cbm_arena_rewind(tmp); + } + continue; + } + if (!in_state) { + continue; + } + pdf_val_t *v = NULL; + if (t.kind == PT_STR || t.kind == PT_NUM || t.kind == PT_NAME) { + v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = t.kind == PT_STR ? PV_STR : t.kind == PT_NUM ? PV_NUM : PV_NAME; + v->is_int = t.is_int; + v->num = t.num; + v->inum = t.inum; + v->s = t.s; + v->n = t.n; + } + } else if (t.kind == PT_AS) { + v = pdf_parse_content_array(d, data, len, &pos, 0); + if (!v) { + failed = true; + break; + } + } else { + continue; + } + if (!v) { + break; + } + if (nbuf == cap) { + int ncap = cap ? cap * 2 : PDF_LIST_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)buf, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + break; + } + buf = g; + cap = ncap; + } + buf[nbuf++] = v; + } + d->cur = saved; + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)buf); + cbm_arena_rewind(tmp); + pdf_cmap_t *cm = NULL; + if (!failed && !d->nomem) { + cm = (pdf_cmap_t *)cbm_arena_calloc(d->a, sizeof(*cm)); + } + if (cm) { + cm->uni = el_finish(d, &b.uni, &cm->nuni); + cm->cid = el_finish(d, &b.cid, &cm->ncid); + b.uni.v = NULL; + b.cid.v = NULL; + if (b.nur) { + cm->ur = (cm_urange_t *)pdf_big(d, (size_t)b.nur * sizeof(*cm->ur)); + if (cm->ur) { + memcpy(cm->ur, b.ur, (size_t)b.nur * sizeof(*cm->ur)); + } + cm->urseg = paint(d, &b.ur_iv, true, &cm->nurseg); + } + if (b.ncr) { + cm->cr = (cm_crange_t *)pdf_big(d, (size_t)b.ncr * sizeof(*cm->cr)); + if (cm->cr) { + memcpy(cm->cr, b.cr, (size_t)b.ncr * sizeof(*cm->cr)); + } + cm->crseg = paint(d, &b.cr_iv, true, &cm->ncrseg); + } + for (int m = 1; m <= PDF_CODE_BYTES; m++) { + if (b.space[m].n) { + cm->space[m] = paint(d, &b.space[m], true, &cm->nspace[m]); + } + } + cm->has_space = b.any_space; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, b.uni.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cid.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.ur); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.ur_iv.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cr); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cr_iv.v); + for (int m = 0; m <= PDF_CODE_BYTES; m++) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b.space[m].v); + } + return d->nomem ? NULL : cm; +} + +/* ToUnicode: an exact entry, else the first big range holding the code. */ +static const char *cmap_lookup(pdf_doc_t *d, const pdf_cmap_t *cm, size_t n, uint32_t code, + char *buf, size_t cap) { + uint64_t key = cm_key(n, code); + const cm_ent_t *e = ent_find(cm->uni, cm->nuni, key); + if (e) { + return e->utf8; + } + int r = seg_find(cm->urseg, cm->nurseg, key); + if (r < 0) { + return NULL; + } + const cm_urange_t *u = &cm->ur[r]; + uint32_t v = (u->last + (uint32_t)(key - u->lo)) & 0xFFFFU; + unsigned char tmp[64]; + unsigned char *w = u->plen + 2 <= sizeof(tmp) + ? tmp + : (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, u->plen + 2); + if (!w) { + d->nomem = true; + return NULL; + } + memcpy(w, u->prefix, u->plen); + w[u->plen] = (unsigned char)(v >> 8); + w[u->plen + 1] = (unsigned char)v; + const char *res = u16be(d, w, u->plen + 2, buf, cap); + if (w != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, w); + } + return res; +} + +static bool cmap_to_cid(const pdf_cmap_t *cm, size_t n, uint32_t code, int64_t *cid) { + uint64_t key = cm_key(n, code); + const cm_ent_t *e = ent_find(cm->cid, cm->ncid, key); + if (e) { + *cid = e->cid; + return true; + } + int r = seg_find(cm->crseg, cm->ncrseg, key); + if (r < 0) { + return false; + } + *cid = pdf_sat_add(cm->cr[r].cid0, (int64_t)(key - cm->cr[r].lo)); + return true; +} + +/* ── TrueType cmap, read backwards ───────────────────────────────── */ + +static bool rd16(const unsigned char *p, size_t len, size_t at, uint32_t *v) { + if (at > len || len - at < 2) { + return false; + } + *v = ((uint32_t)p[at] << 8) | p[at + 1]; + return true; +} + +static bool rd32(const unsigned char *p, size_t len, size_t at, uint32_t *v) { + if (at > len || len - at < 4) { + return false; + } + *v = ((uint32_t)p[at] << 24) | ((uint32_t)p[at + 1] << 16) | ((uint32_t)p[at + 2] << 8) | + p[at + 3]; + return true; +} + +/* "next gid not yet assigned, from x" with path halving; nxt has 65537 slots */ +static uint32_t dsu_next(uint32_t *nxt, uint32_t x) { + while (nxt[x] != x) { + nxt[x] = nxt[nxt[x]]; + x = nxt[x]; + } + return x; +} + +static void dsu_take(uint32_t *nxt, uint32_t x) { + nxt[x] = x + 1; +} + +typedef struct { + uint32_t pe; + uint64_t off; + int rank; + int order; +} ttf_sub_t; + +static int ttf_sub_cmp(const void *x, const void *y) { + const ttf_sub_t *a = (const ttf_sub_t *)x; + const ttf_sub_t *b = (const ttf_sub_t *)y; + if (a->rank != b->rank) { + return a->rank < b->rank ? -1 : 1; + } + return a->order < b->order ? -1 : 1; +} + +/* Fill rev[gid] = code point + 1 from one subtable; false: a read past the end + * (the prototype then had no map at all). */ +static bool ttf_fmt4(const unsigned char *p, size_t len, size_t so, uint32_t *rev, uint32_t *nxt, + bool *any) { + uint32_t segx2; + if (!rd16(p, len, so + 6, &segx2)) { + return false; + } + uint32_t seg = segx2 / 2; + size_t ends = so + 14; + size_t starts = so + 16 + segx2; + size_t deltas = so + 16 + 2 * (size_t)segx2; + size_t ro_pos = so + 16 + 3 * (size_t)segx2; + /* the prototype unpacked each array from a slice of segx2 bytes: an odd + * count or a short slice failed the whole map */ + if (seg && ((segx2 & 1U) || ro_pos + segx2 > len)) { + return false; + } + for (uint32_t s = 0; s < seg; s++) { + uint32_t end; + uint32_t start; + uint32_t delta; + uint32_t ro; + rd16(p, len, ends + 2 * (size_t)s, &end); + rd16(p, len, starts + 2 * (size_t)s, &start); + rd16(p, len, deltas + 2 * (size_t)s, &delta); + rd16(p, len, ro_pos + 2 * (size_t)s, &ro); + uint32_t last = end < 0xFFFEU ? end : 0xFFFEU; + if (start > last) { + continue; + } + if (ro == 0) { + /* gid = (c + delta) & 0xFFFF walks gids upward; assign only those + * still free (the first assignment wins) */ + uint32_t g0 = (start + delta) & 0xFFFFU; + uint32_t span = last - start + 1; + uint32_t done = 0; + while (done < span) { + uint32_t g = (g0 + done) & 0xFFFFU; + uint32_t piece = + (PDF_GID_SPACE - g) < (span - done) ? (PDF_GID_SPACE - g) : (span - done); + uint32_t x = dsu_next(nxt, g); + while (x < g + piece) { + if (x != 0) { + rev[x] = start + done + (x - g) + 1; + *any = true; + } + dsu_take(nxt, x); + x = dsu_next(nxt, x); + } + done += piece; + } + } else { + for (uint32_t c = start; c <= last; c++) { + size_t a = ro_pos + 2 * (size_t)s + ro + 2 * (size_t)(c - start); + uint32_t g = 0; + if (a + 2 <= len) { + g = ((uint32_t)p[a] << 8) | p[a + 1]; + } + if (g) { + g = (g + delta) & 0xFFFFU; + } + if (g && !rev[g] && nxt[g] == g) { + rev[g] = c + 1; + dsu_take(nxt, g); + *any = true; + } + } + } + } + return true; +} + +static bool ttf_fmt12(const unsigned char *p, size_t len, size_t so, uint32_t *rev, uint32_t *nxt, + bool *any) { + uint32_t ng; + if (!rd32(p, len, so + 12, &ng)) { + return false; + } + if ((uint64_t)so + 16 + (uint64_t)ng * 12 > len) { + return false; + } + for (uint32_t i = 0; i < ng; i++) { + uint32_t a; + uint32_t b; + uint32_t g0; + size_t at = so + 16 + 12 * (size_t)i; + rd32(p, len, at, &a); + rd32(p, len, at + 4, &b); + rd32(p, len, at + 8, &g0); + uint64_t cmax = (uint64_t)a + PDF_GID_SPACE; + if ((uint64_t)b < cmax) { + cmax = b; + } + if (cmax > PDF_MAX_CP) { + cmax = PDF_MAX_CP; + } + if ((uint64_t)a > cmax || g0 >= PDF_GID_SPACE) { + continue; + } + uint64_t glast = (uint64_t)g0 + (cmax - a); + if (glast >= PDF_GID_SPACE) { + glast = PDF_GID_SPACE - 1; + } + uint32_t x = dsu_next(nxt, g0); + while (x <= glast) { + rev[x] = a + (x - g0) + 1; + *any = true; + dsu_take(nxt, x); + x = dsu_next(nxt, x); + } + } + return true; +} + +/* The prototype's ttf_reverse_cmap: gid -> code point + 1 (0: none), or NULL. */ +static uint32_t *ttf_reverse(pdf_doc_t *d, const unsigned char *p, size_t len) { + uint32_t num; + if (!rd16(p, len, 4, &num)) { + return NULL; + } + size_t off = 0; + bool found = false; + for (uint32_t i = 0; i < num; i++) { + size_t r = 12 + 16 * (size_t)i; + if (r + 4 <= len && memcmp(p + r, "cmap", 4) == 0) { + uint32_t o; + if (!rd32(p, len, r + 8, &o)) { + return NULL; + } + off = o; + found = true; + break; + } + if (r >= len) { + break; + } + } + if (!found) { + return NULL; + } + uint32_t n; + if (!rd16(p, len, off + 2, &n)) { + return NULL; + } + ttf_sub_t *subs = + (ttf_sub_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1) * sizeof(*subs)); + uint32_t *rev = (uint32_t *)pdf_big(d, PDF_GID_SPACE * sizeof(uint32_t)); + uint32_t *nxt = + (uint32_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (PDF_GID_SPACE + 1) * sizeof(uint32_t)); + if (!subs || !rev || !nxt) { + cbm_free(CBM_MEM_CLASS_EXTRACT, subs); + cbm_free(CBM_MEM_CLASS_EXTRACT, nxt); + d->nomem = true; + return NULL; + } + bool bad = false; + for (uint32_t i = 0; i < n; i++) { + uint32_t pl; + uint32_t en; + uint32_t o; + size_t at = off + 4 + 8 * (size_t)i; + if (!rd16(p, len, at, &pl) || !rd16(p, len, at + 2, &en) || !rd32(p, len, at + 4, &o)) { + bad = true; + break; + } + subs[i].pe = (pl << 16) | en; + subs[i].off = (uint64_t)off + o; + subs[i].order = (int)i; + uint32_t pe = subs[i].pe; + subs[i].rank = pe == 0x0003000AU ? 0 + : pe == 0x00030001U ? 1 + : pe == 0x00000004U ? 2 + : pe == 0x00000003U ? 3 + : 9; + } + uint32_t *res = NULL; + if (!bad) { + qsort(subs, n, sizeof(*subs), ttf_sub_cmp); + for (uint32_t i = 0; i < n && !res; i++) { + uint32_t pe = subs[i].pe; + if (!(pe == 0x0003000AU || pe == 0x00030001U || pe == 0x00000004U || + pe == 0x00000003U || pe == 0x00000006U || pe == 0x00000000U || + pe == 0x00000001U)) { + continue; + } + if (subs[i].off > len) { + break; + } + size_t so = (size_t)subs[i].off; + uint32_t fmt; + if (!rd16(p, len, so, &fmt)) { + break; + } + memset(rev, 0, PDF_GID_SPACE * sizeof(uint32_t)); + for (uint32_t g = 0; g <= PDF_GID_SPACE; g++) { + nxt[g] = g; + } + bool any = false; + bool ok = true; + if (fmt == 4) { + ok = ttf_fmt4(p, len, so, rev, nxt, &any); + } else if (fmt == 12) { + ok = ttf_fmt12(p, len, so, rev, nxt, &any); + } + if (!ok) { + break; + } + if (any) { + res = rev; + } + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, subs); + cbm_free(CBM_MEM_CLASS_EXTRACT, nxt); + return res; +} + +/* ── Fonts ───────────────────────────────────────────────────────── */ + +typedef enum { + BASE_NONE = 0, + BASE_TABLE, /* a base encoding, code -> UTF-8 */ + BASE_NAMES, /* a Type1 program's built-in table, code -> glyph name */ +} pdf_base_kind_t; + +struct pdf_font { + bool multibyte; + bool type3; + double em; + double wscale; + pdf_cmap_t *tounicode; + /* simple fonts */ + const char *map1[256]; + double widths[256]; + bool has_width[256]; + double missing; + bool has_missing; + bool std14; + const unsigned short *std14_tab; /* NULL with std14: Courier */ + /* Type0 */ + pdf_cmap_t *enc; + bool ucs2; + bool cid_type2; + double dw; + pdf_seg_t *wseg; + int nwseg; + double *wval; + pdf_val_t *fdesc; + pdf_val_t *c2g_val; + bool rev_done; + uint32_t *rev; + const unsigned char *c2g; + size_t c2g_len; +}; + +static const char ASCII1[128][2] = { +#define A(c) {(char)(c), 0} + A(0), A(1), A(2), A(3), A(4), A(5), A(6), A(7), A(8), A(9), A(10), A(11), + A(12), A(13), A(14), A(15), A(16), A(17), A(18), A(19), A(20), A(21), A(22), A(23), + A(24), A(25), A(26), A(27), A(28), A(29), A(30), A(31), A(32), A(33), A(34), A(35), + A(36), A(37), A(38), A(39), A(40), A(41), A(42), A(43), A(44), A(45), A(46), A(47), + A(48), A(49), A(50), A(51), A(52), A(53), A(54), A(55), A(56), A(57), A(58), A(59), + A(60), A(61), A(62), A(63), A(64), A(65), A(66), A(67), A(68), A(69), A(70), A(71), + A(72), A(73), A(74), A(75), A(76), A(77), A(78), A(79), A(80), A(81), A(82), A(83), + A(84), A(85), A(86), A(87), A(88), A(89), A(90), A(91), A(92), A(93), A(94), A(95), + A(96), A(97), A(98), A(99), A(100), A(101), A(102), A(103), A(104), A(105), A(106), A(107), + A(108), A(109), A(110), A(111), A(112), A(113), A(114), A(115), A(116), A(117), A(118), A(119), + A(120), A(121), A(122), A(123), A(124), A(125), A(126), A(127), +#undef A +}; + +static bool name_eq(const pdf_val_t *v, const char *s) { + return pdf_is_name(v, s); +} + +static bool bytes_starts(const unsigned char *s, size_t n, const char *pre) { + size_t m = strlen(pre); + return n >= m && memcmp(s, pre, m) == 0; +} + +static bool std14_name(const unsigned char *s, size_t n) { + for (int i = 0; PDF_STD14[i]; i++) { + size_t m = strlen(PDF_STD14[i]); + if (m == n && memcmp(PDF_STD14[i], s, n) == 0) { + return true; + } + } + return false; +} + +static bool py_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool py_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +/* A name character of the prototype's "dup N /name put" pattern. */ +static bool t1_name_char(unsigned char c) { + return !py_space(c) && c != '/' && c != '[' && c != ']' && c != '{' && c != '}' && c != '(' && + c != ')' && c != '<' && c != '>'; +} + +/* The built-in encoding of a Type1 program: names per code (NULL: none), or + * STANDARD (*standard), or neither (false). */ +static bool type1_builtin(pdf_doc_t *d, pdf_val_t *ff, const unsigned char **names, size_t *nlen, + bool *standard) { + *standard = false; + if (!ff || ff->kind != PV_STREAM) { + return false; + } + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, ff->stream, &len); + if (!data || !len) { + return false; + } + pdf_val_t *l1 = pdf_resolve(d, pdf_dget_raw(ff->stream->dict, "Length1")); + size_t clear = len < PDF_TYPE1_CLEAR ? len : PDF_TYPE1_CLEAR; + if (l1 && l1->kind == PV_NUM && l1->is_int && l1->inum > 0 && (uint64_t)l1->inum <= len) { + clear = (size_t)l1->inum; + } + /* /Encoding\s+StandardEncoding\s+def */ + for (size_t i = 0; i + 9 <= clear; i++) { + if (data[i] != '/' || memcmp(data + i, "/Encoding", 9) != 0) { + continue; + } + size_t p = i + 9; + size_t w = p; + while (p < clear && py_space(data[p])) { + p++; + } + if (p == w || clear - p < 16 || memcmp(data + p, "StandardEncoding", 16) != 0) { + continue; + } + p += 16; + w = p; + while (p < clear && py_space(data[p])) { + p++; + } + if (p == w || clear - p < 3 || memcmp(data + p, "def", 3) != 0) { + continue; + } + *standard = true; + return true; + } + /* dup\s+(\d+)\s*\/([^\s/\[\]{}()<>]+)\s+put */ + bool hit = false; + size_t i = 0; + while (i + 3 <= clear) { + if (data[i] != 'd' || memcmp(data + i, "dup", 3) != 0) { + i++; + continue; + } + size_t p = i + 3; + size_t w = p; + while (p < clear && py_space(data[p])) { + p++; + } + size_t ds = p; + while (p < clear && py_digit(data[p])) { + p++; + } + bool ok = ds > w && p > ds; + size_t de = p; + while (ok && p < clear && py_space(data[p])) { + p++; + } + ok = ok && p < clear && data[p] == '/'; + size_t ns = p + 1; + if (ok) { + p = ns; + while (p < clear && t1_name_char(data[p])) { + p++; + } + ok = p > ns; + } + size_t ne = p; + if (ok) { + size_t w2 = p; + while (p < clear && py_space(data[p])) { + p++; + } + ok = p > w2 && clear - p >= 3 && memcmp(data + p, "put", 3) == 0; + } + if (!ok) { + i++; + continue; + } + uint64_t c = 0; + for (size_t k = ds; k < de && c < 256; k++) { + c = c * 10 + (uint64_t)(data[k] - '0'); + } + if (c < 256) { + names[c] = data + ns; + nlen[c] = ne - ns; + hit = true; + } + i = p + 3; + } + return hit; +} + +static const char *const *base_table(const pdf_val_t *name) { + if (name_eq(name, "StandardEncoding")) { + return PDF_ENC_STANDARD; + } + if (name_eq(name, "WinAnsiEncoding")) { + return PDF_ENC_WINANSI; + } + if (name_eq(name, "MacRomanEncoding")) { + return PDF_ENC_MACROMAN; + } + if (name_eq(name, "PDFDocEncoding")) { + return PDF_ENC_PDFDOC; + } + return NULL; +} + +static int dbl_cmp(const void *x, const void *y) { + double a = *(const double *)x; + double b = *(const double *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +static pdf_val_t *font_desc(pdf_doc_t *d, pdf_val_t *fd) { + pdf_val_t *v = pdf_dget(d, fd, "FontDescriptor"); + return (v && v->kind == PV_DICT) ? v : NULL; +} + +/* Which of FontFile, FontFile2, FontFile3 is present (the last listed wins). */ +static const char *font_prog(const pdf_val_t *fdesc) { + const char *prog = NULL; + static const char *const keys[] = {"FontFile", "FontFile2", "FontFile3"}; + for (int i = 0; i < 3; i++) { + if (pdf_dget_raw(fdesc, keys[i])) { + prog = keys[i]; + } + } + return prog; +} + +static void font_init_simple(pdf_doc_t *d, pdf_font_t *f, pdf_val_t *fd, const pdf_val_t *subtype, + const unsigned char *base, size_t base_n) { + pdf_val_t *fdesc = font_desc(d, fd); + pdf_val_t *flags = pdf_dget(d, fdesc, "Flags"); + int64_t fl = pdf_is_num(flags) ? (flags->is_int ? flags->inum : pdf_d2i(flags->num)) : 0; + bool symbolic = (fl & PDF_FLAG_SYMBOLIC) != 0; + pdf_val_t *fcv = pdf_dget(d, fd, "FirstChar"); + int64_t fc = pdf_is_num(fcv) ? (fcv->is_int ? fcv->inum : pdf_d2i(fcv->num)) : 0; + pdf_val_t *ws = pdf_dget(d, fd, "Widths"); + bool any_width = false; + double *all = NULL; + int nall = 0; + if (ws && ws->kind == PV_ARR) { + all = (double *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, + (size_t)(ws->count ? ws->count : 1) * sizeof(double)); + for (int i = 0; i < ws->count; i++) { + pdf_val_t *w = pdf_resolve(d, ws->items[i]); + if (!pdf_is_num(w)) { + continue; + } + any_width = true; + int64_t code = pdf_sat_add(fc, i); + if (code >= 0 && code < 256) { + f->widths[code] = w->num; + f->has_width[code] = true; + } + if (all) { + all[nall++] = w->num; + } + } + } + pdf_val_t *mw = pdf_dget(d, fdesc, "MissingWidth"); + if (pdf_is_num(mw) && mw->num > 0) { + f->missing = mw->num; + f->has_missing = true; + } + const unsigned char *bare = base; + size_t bare_n = base_n; + const unsigned char *plus = base_n ? (const unsigned char *)memchr(base, '+', base_n) : NULL; + if (plus) { + bare = plus + 1; + bare_n = base_n - (size_t)(bare - base); + } + f->std14 = std14_name(bare, bare_n) && !any_width; + if (f->std14) { + /* std14_width reads the name after the LAST '+' */ + const unsigned char *b = base; + size_t bn = base_n; + for (size_t i = 0; i < base_n; i++) { + if (base[i] == '+') { + b = base + i + 1; + bn = base_n - i - 1; + } + } + f->std14_tab = bytes_starts(b, bn, "Courier") ? NULL + : bytes_starts(b, bn, "Times") ? PDF_WIDTH_TIMES + : PDF_WIDTH_HELVETICA; + } + const char *prog = font_prog(fdesc); + bool is_t1 = name_eq(subtype, "Type1") || name_eq(subtype, "MMType1"); + bool is_tt = name_eq(subtype, "TrueType"); + if (f->type3) { + pdf_val_t *fm = pdf_dget(d, fd, "FontMatrix"); + double m0 = 0.001; + if (fm && fm->kind == PV_ARR && fm->count == 6) { + pdf_val_t *x = pdf_resolve(d, fm->items[0]); + m0 = pdf_is_num(x) ? x->num : 0.0; + } + f->wscale = m0 != 0.0 ? m0 : 0.001; + double sc = f->wscale < 0 ? -f->wscale : f->wscale; + int npos = 0; + for (int i = 0; i < nall; i++) { + if (all[i] > 0) { + all[npos++] = all[i] * sc; + } + } + qsort(all, (size_t)npos, sizeof(double), dbl_cmp); + double med = npos ? all[npos / 2] : 0.5; + if (med >= 0.2 && med <= 1.5) { + f->em = 1.0; + } else { + double e = 2.0 * med; + f->em = e < 0.05 ? 0.05 : e > 20.0 ? 20.0 : e; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, all); + /* the encoding */ + pdf_val_t *enc = pdf_dget(d, fd, "Encoding"); + const char *const *table = NULL; + pdf_val_t *diffs = NULL; + if (enc && enc->kind == PV_NAME) { + table = base_table(enc); + } else if (enc && enc->kind == PV_DICT) { + pdf_val_t *be = pdf_dget(d, enc, "BaseEncoding"); + if (be) { + table = base_table(be); + } + diffs = pdf_dget(d, enc, "Differences"); + } + pdf_base_kind_t kind = table ? BASE_TABLE : BASE_NONE; + bool fallback_identity = false; + const unsigned char *names[256]; + size_t nlen[256]; + memset((void *)names, 0, sizeof(names)); + memset(nlen, 0, sizeof(nlen)); + if (kind == BASE_NONE) { + bool got = false; + if (prog && strcmp(prog, "FontFile") == 0 && is_t1) { + bool standard; + if (type1_builtin(d, pdf_dget(d, fdesc, "FontFile"), names, nlen, &standard)) { + got = true; + if (standard) { + table = PDF_ENC_STANDARD; + kind = BASE_TABLE; + } else { + kind = BASE_NAMES; + } + } + } + if (!got && bytes_starts(bare, bare_n, "Symbol") && !prog) { + table = PDF_ENC_SYMBOL; + kind = BASE_TABLE; + got = true; + } + if (!got) { + if ((is_t1 || is_tt) && !symbolic) { + table = PDF_ENC_STANDARD; + kind = BASE_TABLE; + } else if (!f->type3) { + fallback_identity = true; + } + } + } + bool named[256]; + memset(named, 0, sizeof(named)); + if (kind == BASE_NAMES) { + for (int c = 0; c < 256; c++) { + named[c] = names[c] != NULL; + } + } + if (diffs && diffs->kind == PV_ARR) { + int64_t code = 0; + for (int i = 0; i < diffs->count; i++) { + pdf_val_t *x = pdf_resolve(d, diffs->items[i]); + if (pdf_is_num(x)) { + code = x->is_int ? x->inum : pdf_d2i(x->num); + } else if (x && x->kind == PV_NAME) { + if (code >= 0 && code < 256) { + names[code] = x->s; + nlen[code] = x->n; + named[code] = true; + } + code = pdf_sat_add(code, 1); + } + } + } + for (int c = 0; c < 256; c++) { + const char *u = NULL; + if (f->tounicode) { + u = cmap_lookup(d, f->tounicode, 1, (uint32_t)c, NULL, 0); + if (!u) { + u = cmap_lookup(d, f->tounicode, 2, (uint32_t)c, NULL, 0); + } + } + if (!u) { + if (named[c]) { + u = g2u(d, names[c], nlen[c], 0); + } else if (kind == BASE_TABLE) { + u = table[c]; + } + } + bool entry_none = !named[c] && kind == BASE_NONE; + if (!u && (fallback_identity || entry_none) && c >= PDF_STD_FIRST && c <= PDF_STD_LAST && + !f->type3) { + u = ASCII1[c]; + } + f->map1[c] = u; + } +} + +static void font_init_type0(pdf_doc_t *d, pdf_font_t *f, pdf_val_t *fd) { + f->multibyte = true; + pdf_val_t *enc = pdf_dget(d, fd, "Encoding"); + if (enc && enc->kind == PV_NAME) { + if (!name_eq(enc, "Identity-H") && !name_eq(enc, "Identity-V")) { + for (size_t i = 0; i + 4 <= enc->n; i++) { + if (memcmp(enc->s + i, "UCS2", 4) == 0) { + f->ucs2 = true; + } + } + for (size_t i = 0; i + 5 <= enc->n; i++) { + if (memcmp(enc->s + i, "UTF16", 5) == 0) { + f->ucs2 = true; + } + } + } + } else if (enc && enc->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, enc->stream, &len); + f->enc = cmap_parse(d, data ? data : (const unsigned char *)"", data ? len : 0); + } + pdf_val_t *desc = pdf_dget(d, fd, "DescendantFonts"); + pdf_val_t *d0 = NULL; + if (desc && desc->kind == PV_ARR && desc->count) { + d0 = pdf_resolve(d, desc->items[0]); + } + if (!d0 || d0->kind != PV_DICT) { + d0 = NULL; + } + f->cid_type2 = name_eq(pdf_dget(d, d0, "Subtype"), "CIDFontType2"); + pdf_val_t *dw = pdf_dget(d, d0, "DW"); + f->dw = pdf_is_num(dw) ? dw->num : 1000.0; + pdf_val_t *W = pdf_dget(d, d0, "W"); + if (W && W->kind == PV_ARR) { + iv_list_t iv = {0}; + double *vals = NULL; + int nv = 0; + int vcap = 0; + int i = 0; + while (i < W->count && !d->nomem) { + pdf_val_t *first = pdf_resolve(d, W->items[i]); + pdf_val_t *nxt = i + 1 < W->count ? pdf_resolve(d, W->items[i + 1]) : NULL; + if (pdf_is_num(first) && nxt && nxt->kind == PV_ARR) { + int64_t c0 = first->is_int ? first->inum : pdf_d2i(first->num); + for (int j = 0; j < nxt->count; j++) { + pdf_val_t *w = pdf_resolve(d, nxt->items[j]); + int64_t c = pdf_sat_add(c0, j); + if (!pdf_is_num(w) || c < 0 || c > (int64_t)UINT32_MAX) { + continue; + } + if (nv == vcap) { + vcap = vcap ? vcap * 2 : PDF_LIST_MIN; + double *g = (double *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, vals, + (size_t)vcap * sizeof(double)); + if (!g) { + d->nomem = true; + break; + } + vals = g; + } + vals[nv] = w->num; + ivl_push(d, &iv, (uint64_t)c, (uint64_t)c, nv); + nv++; + } + i += 2; + } else if (pdf_is_num(first) && pdf_is_num(nxt) && i + 2 < W->count) { + pdf_val_t *w = pdf_resolve(d, W->items[i + 2]); + int64_t a = first->is_int ? first->inum : pdf_d2i(first->num); + double span = nxt->num - first->num; + if (pdf_is_num(w) && span >= 0 && span <= 65535) { + int64_t z = nxt->is_int ? nxt->inum : pdf_d2i(nxt->num); + if (z >= 0 && a <= (int64_t)UINT32_MAX) { + uint64_t lo = a < 0 ? 0 : (uint64_t)a; + if (nv == vcap) { + vcap = vcap ? vcap * 2 : PDF_LIST_MIN; + double *g = (double *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, vals, + (size_t)vcap * sizeof(double)); + if (!g) { + d->nomem = true; + break; + } + vals = g; + } + vals[nv] = w->num; + if ((uint64_t)z >= lo) { + ivl_push(d, &iv, lo, (uint64_t)z, nv); + } + nv++; + } + } + i += 3; + } else { + i += 1; + } + } + f->wseg = paint(d, &iv, false, &f->nwseg); + cbm_free(CBM_MEM_CLASS_EXTRACT, iv.v); + if (nv) { + f->wval = (double *)pdf_big(d, (size_t)nv * sizeof(double)); + if (f->wval) { + memcpy(f->wval, vals, (size_t)nv * sizeof(double)); + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, vals); + } + f->fdesc = font_desc(d, d0); + f->c2g_val = pdf_dget(d, d0, "CIDToGIDMap"); +} + +static pdf_font_t *font_new(pdf_doc_t *d, pdf_val_t *fd) { + pdf_font_t *f = (pdf_font_t *)cbm_arena_calloc(d->a, sizeof(*f)); + if (!f) { + d->nomem = true; + return NULL; + } + f->em = 1.0; + f->wscale = 0.001; + pdf_val_t *subtype = pdf_dget(d, fd, "Subtype"); + pdf_val_t *basev = pdf_dget(d, fd, "BaseFont"); + const unsigned char *base = + (basev && basev->kind == PV_NAME) ? basev->s : (const unsigned char *)""; + size_t base_n = (basev && basev->kind == PV_NAME) ? basev->n : 0; + f->type3 = name_eq(subtype, "Type3"); + pdf_val_t *tu = pdf_dget(d, fd, "ToUnicode"); + if (tu && tu->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, tu->stream, &len); + if (data && len) { + f->tounicode = cmap_parse(d, data, len); + } + } + if (name_eq(subtype, "Type0")) { + font_init_type0(d, f, fd); + } else { + font_init_simple(d, f, fd, subtype, base, base_n); + } + return d->nomem ? NULL : f; +} + +pdf_font_t *pdf_font_for(pdf_doc_t *d, pdf_val_t *resources, const pdf_val_t *name) { + if (!resources || resources->kind != PV_DICT || !name || name->kind != PV_NAME) { + return NULL; + } + pdf_val_t *fdict = pdf_dget(d, resources, "Font"); + if (!fdict || fdict->kind != PV_DICT) { + return NULL; + } + pdf_val_t *ref = NULL; + for (int lo = 0, hi = fdict->nkv - 1; lo <= hi;) { + int mid = lo + (hi - lo) / 2; + size_t kn = fdict->kv[mid].klen; + size_t m = kn < name->n ? kn : name->n; + int c = m ? memcmp(fdict->kv[mid].key, name->s, m) : 0; + if (!c) { + c = kn < name->n ? -1 : kn > name->n ? 1 : 0; + } + if (!c) { + ref = fdict->kv[mid].val; + break; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + if (ref && ref->kind == PV_NULL) { + ref = NULL; + } + uint64_t key = 0; + bool keyed = false; + if (ref && ref->kind == PV_REF) { + key = (1ULL << 63) | (((uint64_t)ref->ref_num & 0xFFFFFFFFFFULL) << 22) | + ((uint64_t)ref->ref_gen & 0x3FFFFFULL); + keyed = true; + int64_t i = pdf_map_get(&d->fonts, key); + if (i >= 0) { + return d->font_list[i]; + } + } + pdf_val_t *fd = pdf_resolve(d, ref); + if (!fd || fd->kind != PV_DICT) { + return NULL; + } + if (!keyed) { + key = (uint64_t)(uintptr_t)fd & ~(1ULL << 63); + int64_t i = pdf_map_get(&d->fonts, key); + if (i >= 0) { + return d->font_list[i]; + } + } + pdf_font_t *f = font_new(d, fd); + if (d->nfonts == d->font_cap) { + int ncap = d->font_cap ? d->font_cap * 2 : PDF_LIST_MIN; + pdf_font_t **g = (pdf_font_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)d->font_list, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return f; + } + d->font_list = g; + d->font_cap = ncap; + } + d->font_list[d->nfonts] = f; + if (!pdf_map_put(&d->fonts, key, d->nfonts)) { + d->nomem = true; + } + d->nfonts++; + return f; +} + +double pdf_font_em(const pdf_font_t *f) { + return f->em; +} + +static const uint32_t *font_rev(pdf_doc_t *d, pdf_font_t *f) { + if (f->rev_done) { + return f->rev; + } + f->rev_done = true; + pdf_val_t *ff = pdf_dget(d, f->fdesc, "FontFile2"); + if (ff && ff->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, ff->stream, &len); + if (data && len) { + f->rev = ttf_reverse(d, data, len); + } + } + if (f->c2g_val && f->c2g_val->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *m = pdf_stream_decode(d, f->c2g_val->stream, &len); + f->c2g = m; + f->c2g_len = m ? len : 0; + } + return f->rev; +} + +void pdf_font_decode(pdf_doc_t *d, pdf_font_t *f, const unsigned char *s, size_t len, + pdf_glyph_out_t *out, size_t *n) { + size_t k = 0; + if (!f->multibyte) { + for (size_t i = 0; i < len; i++) { + unsigned c = s[i]; + double w; + if (f->has_width[c]) { + w = f->widths[c]; + } else if (f->std14) { + if (!f->std14_tab) { + w = PDF_COURIER_WIDTH; + } else { + w = (c >= PDF_STD_FIRST && c <= PDF_STD_LAST) ? f->std14_tab[c - PDF_STD_FIRST] + : PDF_DEFAULT_WIDTH; + } + } else if (f->has_missing) { + w = f->missing; + } else { + w = f->type3 ? 0.0 : PDF_DEFAULT_WIDTH; + } + out[k].utf8 = f->map1[c]; + out[k].width = w * f->wscale; + out[k].space = c == ' '; + k++; + } + *n = k; + return; + } + size_t i = 0; + while (i < len) { + size_t nb = 0; + uint32_t code = 0; + if (f->enc && f->enc->has_space) { + for (int m = 1; m <= PDF_CODE_BYTES && !nb; m++) { + if (!f->enc->nspace[m] || i + (size_t)m > len) { + continue; + } + uint32_t c = be_code(s + i, (size_t)m); + if (seg_find(f->enc->space[m], f->enc->nspace[m], c) >= 0) { + nb = (size_t)m; + code = c; + } + } + if (!nb) { + nb = 1; + code = s[i]; + } + } else if (i + 1 < len) { + nb = 2; + code = ((uint32_t)s[i] << 8) | s[i + 1]; + } else { + nb = 1; + code = s[i]; + } + i += nb; + pdf_glyph_out_t *g = &out[k++]; + g->utf8 = NULL; + g->inl[0] = '\0'; + if (f->tounicode) { + g->utf8 = cmap_lookup(d, f->tounicode, nb, code, g->inl, sizeof(g->inl)); + } + int64_t cid = code; + if (f->enc) { + int64_t c2; + if (cmap_to_cid(f->enc, nb, code, &c2)) { + cid = c2; + } + } + if (!g->utf8 && f->ucs2 && !(code >= PDF_SURR_LO && code <= PDF_SURR_HI)) { + int m = utf8_put(code, g->inl); + g->inl[m] = '\0'; + g->utf8 = g->inl; + } + if (!g->utf8 && f->cid_type2) { + const uint32_t *rev = font_rev(d, f); + if (rev) { + int64_t gid = cid; + if (f->c2g && f->c2g_len) { + gid = 0; + if (cid >= 0 && (uint64_t)cid * 2 + 2 <= f->c2g_len) { + gid = ((int64_t)f->c2g[2 * cid] << 8) | f->c2g[2 * cid + 1]; + } + } + if (gid >= 0 && gid < PDF_GID_SPACE && rev[gid]) { + int m = utf8_put(rev[gid] - 1, g->inl); + g->inl[m] = '\0'; + g->utf8 = g->inl; + } + } + } + double w = f->dw; + if (cid >= 0 && f->nwseg) { + int r = seg_find(f->wseg, f->nwseg, (uint64_t)cid); + if (r >= 0 && f->wval) { + w = f->wval[r]; + } + } + g->width = w * 0.001; + g->space = nb == 1 && code == ' '; + } + *n = k; +} diff --git a/internal/cbm/pdf/pdf_internal.h b/internal/cbm/pdf/pdf_internal.h new file mode 100644 index 0000000000..d97881e501 --- /dev/null +++ b/internal/cbm/pdf/pdf_internal.h @@ -0,0 +1,321 @@ +/* + * pdf_internal.h — the PDF text-layer extractor's internals (pdf_*.c). + * + * Stages, each in its own file: + * pdf_obj.c lexer, values, the document (xref tables and streams, /Prev + * chains, reconstruction when offsets are broken), indirect + * objects, object streams, stream filters + * pdf_font.c glyph names, ToUnicode and encoding CMaps, simple fonts + * (base encodings, /Differences, built-in Type1 encodings, + * standard-14 widths, Type3), Type0/CID fonts, the TrueType + * cmap fallback + * pdf_text.c content streams (graphics and text state, text operators, + * Form XObjects, inline images skipped), layout (word and line + * breaks from glyph geometry), pages, the driver + * pdf_tables.c generated static tables + * + * The reader is a port of the field-tested prototype (field test E6): the + * same tokens, repairs, font rules and layout thresholds, so its text is the + * text that was measured. Where the prototype's cost could grow faster than + * the input (a search per object, a table walk per glyph), this port answers + * from an index built once, with the same result. + */ +#ifndef CBM_PDF_INTERNAL_H +#define CBM_PDF_INTERNAL_H + +#include "pdf.h" + +#include +#include +#include + +/* Layout geometry must come out bit for bit the same on every platform (and + * the same as the measured prototype): no fused multiply-add. GCC in ISO C + * mode never contracts; clang does unless told. */ +#if defined(__clang__) +#pragma STDC FP_CONTRACT OFF +#endif + +/* ── Tables (pdf_tables.c) ───────────────────────────────────────── */ + +typedef struct { + const char *name; + const char *utf8; +} pdf_glyph_t; + +enum { PDF_STD_WIDTHS = 95 }; /* codes 0x20..0x7E */ + +extern const pdf_glyph_t PDF_GLYPHS[]; +extern const int PDF_GLYPH_COUNT; +extern const char *const PDF_ENC_STANDARD[256]; +extern const char *const PDF_ENC_WINANSI[256]; +extern const char *const PDF_ENC_MACROMAN[256]; +extern const char *const PDF_ENC_PDFDOC[256]; +extern const char *const PDF_ENC_SYMBOL[256]; +extern const unsigned short PDF_WIDTH_HELVETICA[PDF_STD_WIDTHS]; +extern const unsigned short PDF_WIDTH_TIMES[PDF_STD_WIDTHS]; +extern const char *const PDF_STD14[]; + +/* ── Limits: structure, never content ────────────────────────────── */ + +enum { + PDF_MAX_NESTING = 256, /* arrays/dicts inside one value */ + PDF_MAX_RESOLVE = 32, /* reference chains */ + PDF_MAX_FORM_DEPTH = 16, /* Form XObjects inside Form XObjects */ + PDF_MAX_PAGE_DEPTH = 64, /* page-tree depth */ + PDF_MAX_PREV = 4096, /* xref sections along /Prev */ + PDF_HEADER_WINDOW = 1024, /* "%PDF-" must start within this many bytes */ +}; + +/* A decoded stream above this size is cut and counted: a decompression bomb is + * no text layer, and real documents stay far below it. */ +#define PDF_MAX_STREAM_OUT ((size_t)512 << 20) + +/* ── Integers from the file: saturated, never undefined ──────────── */ + +static inline int64_t pdf_sat_add(int64_t a, int64_t b) { + int64_t r; + if (__builtin_add_overflow(a, b, &r)) { + return b > 0 ? INT64_MAX : INT64_MIN; + } + return r; +} + +/* Python's int() of a float: truncated toward zero; saturated here, NaN 0. */ +static inline int64_t pdf_d2i(double x) { + if (x != x) { + return 0; + } + if (x >= 9223372036854775807.0) { + return INT64_MAX; + } + if (x <= -9223372036854775808.0) { + return INT64_MIN; + } + return (int64_t)x; +} + +/* ── Values ──────────────────────────────────────────────────────── */ + +typedef enum { + PV_NULL = 0, + PV_BOOL, + PV_NUM, + PV_NAME, + PV_STR, + PV_ARR, + PV_DICT, + PV_REF, + PV_STREAM, + PV_KW, +} pdf_kind_t; + +typedef struct pdf_val pdf_val_t; +typedef struct pdf_stream pdf_stream_t; + +typedef struct { + const unsigned char *key; /* a name's bytes, NUL-terminated */ + size_t klen; + pdf_val_t *val; +} pdf_kv_t; + +struct pdf_val { + pdf_kind_t kind; + bool b; + bool is_int; + double num; + int64_t inum; + const unsigned char *s; /* PV_NAME, PV_STR, PV_KW (NUL-terminated copies) */ + size_t n; + pdf_val_t **items; /* PV_ARR */ + int count; + pdf_kv_t *kv; /* PV_DICT: sorted by key, one entry per key (the last written) */ + int nkv; + int64_t ref_num; /* PV_REF */ + int64_t ref_gen; + pdf_stream_t *stream; /* PV_STREAM */ +}; + +struct pdf_stream { + pdf_val_t *dict; + const unsigned char *raw; + size_t raw_len; + bool done; + const unsigned char *decoded; /* NULL: nothing decodable (image, unsupported) */ + size_t dec_len; +}; + +/* ── Lexer ───────────────────────────────────────────────────────── */ + +typedef enum { + PT_EOF = 0, + PT_NUM, + PT_NAME, + PT_STR, + PT_DS, /* << */ + PT_DE, /* >> */ + PT_AS, /* [ */ + PT_AE, /* ] */ + PT_BS, /* { */ + PT_BE, /* } */ + PT_KW, +} pdf_tok_kind_t; + +typedef struct { + pdf_tok_kind_t kind; + bool is_int; + double num; + int64_t inum; + const unsigned char *s; /* NAME, STR: arena copies; KW: points into the data */ + size_t n; +} pdf_tok_t; + +typedef struct pdf_doc pdf_doc_t; + +/* The next token of data[*pos..len); *pos moves past it. */ +void pdf_lex(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, pdf_tok_t *t); +/* The value that starts with token t. NULL: out of memory or nested too deep. */ +pdf_val_t *pdf_parse_value(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + const pdf_tok_t *t, int depth); +/* An array operand of a content stream: no references, an operator ends it. */ +pdf_val_t *pdf_parse_content_array(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + int depth); +bool pdf_kw_is(const pdf_tok_t *t, const char *kw); + +/* ── Integer maps (open addressing, owned by the document) ───────── */ + +typedef struct { + uint64_t *keys; + int64_t *vals; + uint32_t cap; /* a power of two, or 0 */ + uint32_t n; +} pdf_map_t; + +/* -1 when absent. */ +int64_t pdf_map_get(const pdf_map_t *m, uint64_t key); +bool pdf_map_put(pdf_map_t *m, uint64_t key, int64_t val); +void pdf_map_free(pdf_map_t *m); + +/* ── Document ────────────────────────────────────────────────────── */ + +typedef struct pdf_objstm pdf_objstm_t; + +typedef struct { + int64_t num; + uint8_t type; /* 0 free, 1 at an offset, 2 inside an object stream */ + int64_t a; /* type 1: offset; type 2: the object stream's number */ + int64_t b; /* type 1: generation; type 2: index in the stream */ + pdf_val_t *obj; + bool cached; + bool loading; + pdf_objstm_t *stm; /* this object as a decoded object stream */ +} pdf_xent_t; + +typedef struct { + pdf_xent_t *v; + int n; + int cap; + pdf_map_t by_num; /* object number -> index in v */ +} pdf_xref_t; + +/* One "N G obj" header found by scanning the whole file. */ +typedef struct { + int64_t num; + int64_t gen; + size_t off; /* where N starts */ + uint8_t num_digits; + uint8_t gen_digits; + bool canonical; /* both written without leading zeros */ + bool reg_after; /* "obj" runs on into a regular character */ +} pdf_hdr_t; + +typedef struct { + int64_t num; + int idx; +} pdf_numidx_t; + +typedef struct pdf_font pdf_font_t; + +struct pdf_doc { + const unsigned char *data; + size_t len; + size_t hdr; /* offset of "%PDF-" */ + CBMArena *a; /* the document's values, fonts, CMaps */ + CBMArena *cur; /* where the lexer and the value parser allocate (a, or a + * content stream's operand arena) */ + bool peek; /* lex without copying names or strings */ + bool nomem; /* an allocation failed: the result is CBM_PDF_NOMEM */ + pdf_xref_t xref; + pdf_val_t *trailer; /* a dict */ + bool reconstructed; + /* every "N G obj" in file order, built on first need */ + pdf_hdr_t *hdrs; + int nhdrs; + bool hdrs_built; + pdf_numidx_t *hdr_sorted; /* (num, index) sorted, for the per-object fallback */ + /* blocks from the memory core, freed with the document */ + void **owned; + int nowned; + int owned_cap; + /* an arena for token values that end with one CMap's parse */ + CBMArena scratch_arena; + bool scratch_arena_live; + /* a scratch buffer for string and stream assembly */ + unsigned char *scratch; + size_t scratch_cap; + /* fonts by reference (num << 20 | gen) or by inline dict address */ + pdf_map_t fonts; + pdf_font_t **font_list; + int nfonts; + int font_cap; + cbm_pdf_counts_t *counts; +}; + +pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num); +pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v); +/* dict[key] as written (a reference stays a reference); NULL when absent, null, + * or the value is no dict. */ +pdf_val_t *pdf_dget_raw(const pdf_val_t *dict, const char *key); +/* Whether dict has key at all (a null value counts). */ +bool pdf_dhas(const pdf_val_t *dict, const char *key); +/* dict[key] as written, a null value included; NULL only when absent. */ +pdf_val_t *pdf_dfind(const pdf_val_t *dict, const char *key); +pdf_val_t *pdf_dfind_n(const pdf_val_t *dict, const unsigned char *key, size_t n); +/* dict[key] resolved; NULL when absent or null. */ +pdf_val_t *pdf_dget(pdf_doc_t *d, const pdf_val_t *dict, const char *key); +bool pdf_is_name(const pdf_val_t *v, const char *name); +bool pdf_is_num(const pdf_val_t *v); +/* A number's value; 0 for anything else. */ +double pdf_num(const pdf_val_t *v); +/* The decoded data of a stream; NULL when it has none. */ +const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len); +bool pdf_doc_open(pdf_doc_t *d); +void pdf_doc_close(pdf_doc_t *d); + +/* Memory: arena values, and big blocks the document owns. */ +void *pdf_alloc(pdf_doc_t *d, size_t n); +void *pdf_calloc(pdf_doc_t *d, size_t n); +void *pdf_big(pdf_doc_t *d, size_t n); /* owned, freed with the document */ +bool pdf_own(pdf_doc_t *d, void *block); /* hand a memory-core block to the document */ +bool pdf_scratch(pdf_doc_t *d, size_t need); /* d->scratch holds at least `need` bytes */ + +/* ── Fonts (pdf_font.c) ──────────────────────────────────────────── */ + +typedef struct { + const char *utf8; /* NULL: unmapped; "" maps to nothing; may point into inl */ + double width; /* advance in text-space units per unit of font size */ + bool space; /* the single-byte code 32 */ + char inl[16]; /* a short mapping made for this glyph */ +} pdf_glyph_out_t; + +/* The font resources[/Font][name] names; NULL when there is none. Cached. */ +pdf_font_t *pdf_font_for(pdf_doc_t *d, pdf_val_t *resources, const pdf_val_t *name); +/* Decode shown bytes into glyphs, out[0..*n) (out holds at least len entries). */ +void pdf_font_decode(pdf_doc_t *d, pdf_font_t *f, const unsigned char *s, size_t len, + pdf_glyph_out_t *out, size_t *n); +/* The text-space size of one em per unit of font size. */ +double pdf_font_em(const pdf_font_t *f); +/* A glyph name's Unicode value as UTF-8 (arena or static); NULL when unknown. */ +const char *pdf_glyph_to_utf8(pdf_doc_t *d, const unsigned char *name, size_t n); + +#endif /* CBM_PDF_INTERNAL_H */ diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.c new file mode 100644 index 0000000000..c60c2bb8d8 --- /dev/null +++ b/internal/cbm/pdf/pdf_obj.c @@ -0,0 +1,2594 @@ +/* + * pdf_obj.c — lexer, values, the cross-reference, indirect objects, object + * streams and stream filters of the PDF text-layer extractor. + * + * Repairs follow the field-tested prototype: an xref section is looked for + * near a wrong offset; a stream whose /Length does not end at "endstream" is + * cut at the next "endstream"; an object missing at its offset is looked up + * by its header; when the cross-reference cannot be read at all it is rebuilt + * from every "N G obj" header in the file. Every header the repairs need comes + * from one scan of the file, built on first need. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" +#include "helpers.h" + +#include +#include +#include +#include +#include + +enum { + PDF_NUM_BUF = 64, /* a number's text for strtod */ + PDF_XREF_NEAR = 1024, /* how far around a wrong xref offset to look */ + PDF_TAIL_WINDOW = 32, /* where "endstream" must follow /Length */ + PDF_INFLATE_CHUNK = 4096, /* the prototype's recovery chunk */ + PDF_INFLATE_GROW = 65536, /* output growth step */ + PDF_ZLIB_WBITS = 15, + PDF_MAX_FIELD = 8, /* bytes of one xref-stream field */ + PDF_A85_GROUP = 5, + PDF_A85_BASE = 85, + PDF_LZW_CLEAR = 256, + PDF_LZW_EOD = 257, + PDF_LZW_FIRST = 258, + PDF_LZW_MAX_BITS = 12, + PDF_LZW_MIN_BITS = 9, + PDF_LZW_TABLE = 4096, + PDF_RLD_EOD = 128, + PDF_PNG_PRED = 10, + PDF_TIFF_PRED = 2, + PDF_BITS_PER_BYTE = 8, + PDF_MAP_MIN = 16, + PDF_OWN_MIN = 16, + PDF_SCRATCH_MIN = 256, + PDF_HEX_BASE = 16, + PDF_OCT_DIGITS = 3, +}; + +#define PDF_SAT_LIMIT ((INT64_MAX - 9) / 10) + +static pdf_val_t PDF_NULL_VALUE = {.kind = PV_NULL}; + +/* ── Memory ──────────────────────────────────────────────────────── */ + +void *pdf_alloc(pdf_doc_t *d, size_t n) { + void *p = cbm_arena_alloc(d->cur, n ? n : 1); + if (!p) { + d->nomem = true; + } + return p; +} + +void *pdf_calloc(pdf_doc_t *d, size_t n) { + void *p = cbm_arena_calloc(d->cur, n ? n : 1); + if (!p) { + d->nomem = true; + } + return p; +} + +bool pdf_own(pdf_doc_t *d, void *block) { + if (!block) { + return false; + } + if (d->nowned == d->owned_cap) { + int ncap = d->owned_cap ? d->owned_cap * 2 : PDF_OWN_MIN; + void **g = (void **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)d->owned, + (size_t)ncap * sizeof(*g)); + if (!g) { + cbm_free(CBM_MEM_CLASS_EXTRACT, block); + d->nomem = true; + return false; + } + d->owned = g; + d->owned_cap = ncap; + } + d->owned[d->nowned++] = block; + return true; +} + +void *pdf_big(pdf_doc_t *d, size_t n) { + void *p = cbm_alloc(CBM_MEM_CLASS_EXTRACT, n ? n : 1); + if (!p) { + d->nomem = true; + return NULL; + } + return pdf_own(d, p) ? p : NULL; +} + +bool pdf_scratch(pdf_doc_t *d, size_t need) { + if (need <= d->scratch_cap) { + return true; + } + size_t ncap = d->scratch_cap ? d->scratch_cap : PDF_SCRATCH_MIN; + while (ncap < need) { + if (ncap > SIZE_MAX / 2) { + ncap = need; + break; + } + ncap *= 2; + } + unsigned char *g = (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, d->scratch, ncap); + if (!g) { + d->nomem = true; + return false; + } + d->scratch = g; + d->scratch_cap = ncap; + return true; +} + +/* A growable byte buffer from the memory core (decoder output). */ +typedef struct { + unsigned char *p; + size_t n; + size_t cap; + bool fail; +} pdf_buf_t; + +static bool buf_reserve(pdf_buf_t *b, size_t add) { + if (b->fail) { + return false; + } + if (add > SIZE_MAX - b->n) { + b->fail = true; + return false; + } + size_t need = b->n + add; + if (need <= b->cap) { + return true; + } + size_t ncap = b->cap ? b->cap : PDF_INFLATE_GROW; + while (ncap < need) { + if (ncap > SIZE_MAX / 2) { + ncap = need; + break; + } + ncap *= 2; + } + unsigned char *g = (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->p, ncap); + if (!g) { + b->fail = true; + return false; + } + b->p = g; + b->cap = ncap; + return true; +} + +static bool buf_put(pdf_buf_t *b, const unsigned char *src, size_t n) { + if (!n) { + return !b->fail; + } + if (!buf_reserve(b, n)) { + return false; + } + memcpy(b->p + b->n, src, n); + b->n += n; + return true; +} + +static bool buf_byte(pdf_buf_t *b, unsigned char c) { + if (!buf_reserve(b, 1)) { + return false; + } + b->p[b->n++] = c; + return true; +} + +static void buf_free(pdf_buf_t *b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b->p); + memset(b, 0, sizeof(*b)); +} + +/* ── Integer maps ────────────────────────────────────────────────── */ + +static uint64_t map_mix(uint64_t x) { + /* splitmix64's finalizer; the seed keeps crafted object numbers from + * clustering (it changes speed only, never a result). */ + static const uint64_t seed = 0x9e3779b97f4a7c15ULL; + x += seed; + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9ULL; + x ^= x >> 27; + x *= 0x94d049bb133111ebULL; + x ^= x >> 31; + return x; +} + +int64_t pdf_map_get(const pdf_map_t *m, uint64_t key) { + if (!m->cap) { + return -1; + } + uint32_t mask = m->cap - 1; + uint32_t i = (uint32_t)map_mix(key) & mask; + while (m->vals[i] >= 0) { + if (m->keys[i] == key) { + return m->vals[i]; + } + i = (i + 1) & mask; + } + return -1; +} + +static bool map_grow(pdf_map_t *m) { + uint32_t ncap = m->cap ? m->cap * 2 : PDF_MAP_MIN; + if (ncap < m->cap) { + return false; + } + uint64_t *keys = (uint64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)ncap * sizeof(*keys)); + int64_t *vals = (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)ncap * sizeof(*vals)); + if (!keys || !vals) { + cbm_free(CBM_MEM_CLASS_EXTRACT, keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, vals); + return false; + } + for (uint32_t i = 0; i < ncap; i++) { + vals[i] = -1; + } + uint32_t mask = ncap - 1; + for (uint32_t i = 0; i < m->cap; i++) { + if (m->vals[i] < 0) { + continue; + } + uint32_t j = (uint32_t)map_mix(m->keys[i]) & mask; + while (vals[j] >= 0) { + j = (j + 1) & mask; + } + keys[j] = m->keys[i]; + vals[j] = m->vals[i]; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, m->keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, m->vals); + m->keys = keys; + m->vals = vals; + m->cap = ncap; + return true; +} + +bool pdf_map_put(pdf_map_t *m, uint64_t key, int64_t val) { + if ((m->n + 1) * 2 > m->cap && !map_grow(m)) { + return false; + } + uint32_t mask = m->cap - 1; + uint32_t i = (uint32_t)map_mix(key) & mask; + while (m->vals[i] >= 0) { + if (m->keys[i] == key) { + m->vals[i] = val; + return true; + } + i = (i + 1) & mask; + } + m->keys[i] = key; + m->vals[i] = val; + m->n++; + return true; +} + +void pdf_map_free(pdf_map_t *m) { + cbm_free(CBM_MEM_CLASS_EXTRACT, m->keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, m->vals); + memset(m, 0, sizeof(*m)); +} + +/* ── Characters ──────────────────────────────────────────────────── */ + +static inline bool is_ws(unsigned char c) { + return c == 0 || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' '; +} + +static inline bool is_delim(unsigned char c) { + return c == '(' || c == ')' || c == '<' || c == '>' || c == '[' || c == ']' || c == '{' || + c == '}' || c == '/' || c == '%'; +} + +static inline bool is_reg(unsigned char c) { + return !is_ws(c) && !is_delim(c); +} + +static inline bool is_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static int hex_val(unsigned char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +/* A digit run as a saturated integer. */ +static int64_t sat_digits(const unsigned char *s, size_t n) { + int64_t v = 0; + for (size_t i = 0; i < n; i++) { + if (v > PDF_SAT_LIMIT) { + return INT64_MAX; + } + v = v * 10 + (s[i] - '0'); + } + return v; +} + +/* ── Lexer ───────────────────────────────────────────────────────── */ + +bool pdf_kw_is(const pdf_tok_t *t, const char *kw) { + size_t n = strlen(kw); + return t->kind == PT_KW && t->n == n && memcmp(t->s, kw, n) == 0; +} + +/* [+-]?(\d+\.?\d*|\.\d+) not followed by a regular character. */ +static bool lex_num(const unsigned char *data, size_t len, size_t p, size_t *end, pdf_tok_t *t) { + size_t q = p; + bool neg = false; + if (data[q] == '+' || data[q] == '-') { + neg = data[q] == '-'; + q++; + } + size_t ds = q; + bool dot = false; + if (q < len && is_digit(data[q])) { + while (q < len && is_digit(data[q])) { + q++; + } + if (q < len && data[q] == '.') { + dot = true; + q++; + while (q < len && is_digit(data[q])) { + q++; + } + } + } else if (q + 1 < len && data[q] == '.' && is_digit(data[q + 1])) { + dot = true; + q++; + while (q < len && is_digit(data[q])) { + q++; + } + } else { + return false; + } + if (q < len && is_reg(data[q])) { + return false; + } + t->kind = PT_NUM; + t->is_int = !dot; + if (!dot) { + int64_t v = sat_digits(data + ds, q - ds); + t->inum = neg ? -v : v; + t->num = (double)t->inum; + } else { + char buf[PDF_NUM_BUF]; + size_t n = q - p; + if (n < sizeof(buf)) { + memcpy(buf, data + p, n); + buf[n] = '\0'; + t->num = strtod(buf, NULL); + } else { + /* a number this long is noise; read its leading digits */ + memcpy(buf, data + p, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + t->num = strtod(buf, NULL); + } + t->inum = (int64_t)0; + } + *end = q; + return true; +} + +static void lex_name(pdf_doc_t *d, const unsigned char *s, size_t n, pdf_tok_t *t) { + t->kind = PT_NAME; + if (d->peek) { + t->s = s; + t->n = n; + return; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, n + 1); + if (!o) { + t->s = (const unsigned char *)""; + t->n = 0; + return; + } + size_t k = 0; + for (size_t i = 0; i < n;) { + if (s[i] == '#' && i + 2 < n && hex_val(s[i + 1]) >= 0 && hex_val(s[i + 2]) >= 0) { + o[k++] = (unsigned char)(hex_val(s[i + 1]) * PDF_HEX_BASE + hex_val(s[i + 2])); + i += 3; + } else { + o[k++] = s[i++]; + } + } + o[k] = '\0'; + t->s = o; + t->n = k; +} + +static bool lstr_special(unsigned char c) { + return c == '(' || c == ')' || c == '\\' || c == '\r'; +} + +/* A literal string from data[i..] (after its "("); returns the position after it. */ +static size_t lex_lstr(pdf_doc_t *d, const unsigned char *data, size_t len, size_t i, + pdf_tok_t *t) { + int depth = 1; + size_t k = 0; + bool keep = !d->peek; + size_t end = len; + t->kind = PT_STR; + for (;;) { + size_t j = i; + while (j < len && !lstr_special(data[j])) { + j++; + } + if (keep && j > i) { + if (!pdf_scratch(d, k + (j - i) + PDF_OCT_DIGITS + 1)) { + keep = false; + } else { + memcpy(d->scratch + k, data + i, j - i); + k += j - i; + } + } + if (j >= len) { + end = len; + break; + } + if (keep && !pdf_scratch(d, k + 2)) { + keep = false; + } + unsigned char c = data[j]; + int out = -1; + if (c == '\\') { + if (j + 1 >= len) { + end = len; + break; + } + unsigned char nx = data[j + 1]; + static const char esc_from[] = "nrtbf()\\"; + static const char esc_to[] = "\n\r\t\b\f()\\"; + const char *e = (nx != 0) ? strchr(esc_from, nx) : NULL; + if (e) { + out = (unsigned char)esc_to[e - esc_from]; + i = j + 2; + } else if (nx >= '0' && nx <= '7') { + size_t q = j + 1; + unsigned v = 0; + while (q < len && q < j + 1 + PDF_OCT_DIGITS && data[q] >= '0' && data[q] <= '7') { + v = v * PDF_BITS_PER_BYTE + (unsigned)(data[q] - '0'); + q++; + } + out = (int)(v & 0xFFU); + i = q; + } else if (nx == '\r') { + i = (j + 2 < len && data[j + 2] == '\n') ? j + 3 : j + 2; + } else if (nx == '\n') { + i = j + 2; + } else { + out = nx; + i = j + 2; + } + } else if (c == '\r') { + out = '\n'; + i = (j + 1 < len && data[j + 1] == '\n') ? j + 2 : j + 1; + } else if (c == '(') { + depth++; + out = c; + i = j + 1; + } else { + depth--; + if (depth == 0) { + end = j + 1; + break; + } + out = c; + i = j + 1; + } + if (keep && out >= 0) { + d->scratch[k++] = (unsigned char)out; + } + } + if (d->peek) { + t->s = NULL; + t->n = 0; + return end; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, k + 1); + if (o) { + if (k) { + memcpy(o, d->scratch, k); + } + o[k] = '\0'; + } + t->s = o ? o : (const unsigned char *)""; + t->n = o ? k : 0; + return end; +} + +/* A hex string from data[i..] (after its "<"). */ +static size_t lex_hstr(pdf_doc_t *d, const unsigned char *data, size_t len, size_t i, + pdf_tok_t *t) { + const unsigned char *gt = (const unsigned char *)memchr(data + i, '>', len - i); + size_t e = gt ? (size_t)(gt - data) : len; + t->kind = PT_STR; + size_t end = e < len ? e + 1 : len; + if (d->peek) { + t->s = NULL; + t->n = 0; + return end; + } + size_t nhex = 0; + for (size_t q = i; q < e; q++) { + nhex += hex_val(data[q]) >= 0; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, (nhex + 1) / 2 + 1); + if (!o) { + t->s = (const unsigned char *)""; + t->n = 0; + return end; + } + size_t k = 0; + int hi = -1; + for (size_t q = i; q < e; q++) { + int v = hex_val(data[q]); + if (v < 0) { + continue; + } + if (hi < 0) { + hi = v; + } else { + o[k++] = (unsigned char)(hi * PDF_HEX_BASE + v); + hi = -1; + } + } + if (hi >= 0) { + o[k++] = (unsigned char)(hi * PDF_HEX_BASE); + } + o[k] = '\0'; + t->s = o; + t->n = k; + return end; +} + +void pdf_lex(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, pdf_tok_t *t) { + size_t p = *pos; + memset(t, 0, sizeof(*t)); + while (p < len) { + unsigned char c = data[p]; + if (is_ws(c)) { + p++; + continue; + } + if (c == '%') { + while (p < len && data[p] != '\r' && data[p] != '\n') { + p++; + } + continue; + } + if (is_digit(c) || c == '+' || c == '-' || c == '.') { + size_t e; + if (lex_num(data, len, p, &e, t)) { + *pos = e; + return; + } + } + switch (c) { + case '/': { + size_t q = p + 1; + while (q < len && is_reg(data[q])) { + q++; + } + lex_name(d, data + p + 1, q - p - 1, t); + *pos = q; + return; + } + case '(': + *pos = lex_lstr(d, data, len, p + 1, t); + return; + case '<': + if (p + 1 < len && data[p + 1] == '<') { + t->kind = PT_DS; + *pos = p + 2; + return; + } + *pos = lex_hstr(d, data, len, p + 1, t); + return; + case '>': + if (p + 1 < len && data[p + 1] == '>') { + t->kind = PT_DE; + *pos = p + 2; + return; + } + p++; + continue; + case ')': + p++; + continue; + case '[': + t->kind = PT_AS; + *pos = p + 1; + return; + case ']': + t->kind = PT_AE; + *pos = p + 1; + return; + case '{': + t->kind = PT_BS; + *pos = p + 1; + return; + case '}': + t->kind = PT_BE; + *pos = p + 1; + return; + default: { + size_t q = p; + while (q < len && is_reg(data[q])) { + q++; + } + t->kind = PT_KW; + t->s = data + p; + t->n = q - p; + *pos = q; + return; + } + } + } + t->kind = PT_EOF; + *pos = len; +} + +/* The next token without copying names or strings. */ +static void lex_peek(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + pdf_tok_t *t) { + bool was = d->peek; + d->peek = true; + pdf_lex(d, data, len, pos, t); + d->peek = was; +} + +/* ── Values ──────────────────────────────────────────────────────── */ + +static pdf_val_t *new_val(pdf_doc_t *d, pdf_kind_t kind) { + pdf_val_t *v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = kind; + } + return v; +} + +static pdf_val_t *val_from_tok(pdf_doc_t *d, const pdf_tok_t *t) { + pdf_val_t *v = new_val(d, t->kind == PT_NUM ? PV_NUM : t->kind == PT_NAME ? PV_NAME : PV_STR); + if (!v) { + return NULL; + } + v->is_int = t->is_int; + v->num = t->num; + v->inum = t->inum; + v->s = t->s; + v->n = t->n; + return v; +} + +typedef struct { + pdf_val_t **v; + int n; + int cap; + bool fail; +} val_list_t; + +static void vl_push(val_list_t *l, pdf_val_t *x) { + if (l->fail) { + return; + } + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_MAP_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)l->v, + (size_t)ncap * sizeof(*g)); + if (!g) { + l->fail = true; + return; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n++] = x; +} + +static pdf_val_t *vl_finish_array(pdf_doc_t *d, val_list_t *l) { + pdf_val_t *arr = NULL; + if (!l->fail) { + arr = new_val(d, PV_ARR); + if (arr && l->n) { + arr->items = (pdf_val_t **)pdf_alloc(d, (size_t)l->n * sizeof(*arr->items)); + if (arr->items) { + memcpy((void *)arr->items, (void *)l->v, (size_t)l->n * sizeof(*arr->items)); + arr->count = l->n; + } else { + arr = NULL; + } + } + } else { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l->v); + return arr; +} + +typedef struct { + pdf_kv_t kv; + int seq; +} kv_seq_t; + +static int kv_key_cmp(const unsigned char *a, size_t an, const unsigned char *b, size_t bn) { + size_t m = an < bn ? an : bn; + int c = m ? memcmp(a, b, m) : 0; + if (c) { + return c; + } + return an < bn ? -1 : an > bn ? 1 : 0; +} + +static int kv_seq_cmp(const void *x, const void *y) { + const kv_seq_t *a = (const kv_seq_t *)x; + const kv_seq_t *b = (const kv_seq_t *)y; + int c = kv_key_cmp(a->kv.key, a->kv.klen, b->kv.key, b->kv.klen); + if (c) { + return c; + } + return a->seq < b->seq ? -1 : a->seq > b->seq ? 1 : 0; +} + +typedef struct { + kv_seq_t *v; + int n; + int cap; + bool fail; +} kv_list_t; + +static void kl_push(kv_list_t *l, const unsigned char *key, size_t klen, pdf_val_t *val) { + if (l->fail) { + return; + } + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_MAP_MIN; + kv_seq_t *g = + (kv_seq_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + l->fail = true; + return; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].kv.key = key; + l->v[l->n].kv.klen = klen; + l->v[l->n].kv.val = val; + l->v[l->n].seq = l->n; + l->n++; +} + +/* A dict from key/value pairs in written order: one entry per key, the last + * written (keep_last) or the first. */ +static pdf_val_t *kl_finish_dict(pdf_doc_t *d, kv_list_t *l, bool keep_last) { + pdf_val_t *dict = NULL; + if (l->fail) { + d->nomem = true; + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return NULL; + } + if (l->n > 1) { + qsort(l->v, (size_t)l->n, sizeof(*l->v), kv_seq_cmp); + } + dict = new_val(d, PV_DICT); + if (dict && l->n) { + dict->kv = (pdf_kv_t *)pdf_alloc(d, (size_t)l->n * sizeof(*dict->kv)); + if (!dict->kv) { + dict = NULL; + } else { + int k = 0; + for (int i = 0; i < l->n;) { + int j = i; + while (j + 1 < l->n && kv_key_cmp(l->v[j + 1].kv.key, l->v[j + 1].kv.klen, + l->v[i].kv.key, l->v[i].kv.klen) == 0) { + j++; + } + dict->kv[k++] = keep_last ? l->v[j].kv : l->v[i].kv; + i = j + 1; + } + dict->nkv = k; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return dict; +} + +static bool kw_ends_value(const pdf_tok_t *t) { + return pdf_kw_is(t, "endobj") || pdf_kw_is(t, "stream") || pdf_kw_is(t, "endstream"); +} + +pdf_val_t *pdf_parse_value(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + const pdf_tok_t *t, int depth) { + if (depth > PDF_MAX_NESTING || d->nomem) { + return NULL; + } + switch (t->kind) { + case PT_NUM: + if (t->is_int) { + size_t p2 = *pos; + pdf_tok_t t2; + lex_peek(d, data, len, &p2, &t2); + if (t2.kind == PT_NUM && t2.is_int) { + pdf_tok_t t3; + lex_peek(d, data, len, &p2, &t3); + if (pdf_kw_is(&t3, "R")) { + pdf_val_t *r = new_val(d, PV_REF); + if (r) { + r->ref_num = t->inum; + r->ref_gen = t2.inum; + *pos = p2; + } + return r; + } + } + } + return val_from_tok(d, t); + case PT_NAME: + case PT_STR: + return val_from_tok(d, t); + case PT_AS: { + val_list_t l = {0}; + for (;;) { + pdf_tok_t a; + pdf_lex(d, data, len, pos, &a); + if (a.kind == PT_AE || a.kind == PT_EOF) { + break; + } + if (a.kind == PT_DE || a.kind == PT_BE || a.kind == PT_BS) { + continue; + } + if (a.kind == PT_KW && kw_ends_value(&a)) { + break; + } + pdf_val_t *x = pdf_parse_value(d, data, len, pos, &a, depth + 1); + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l.v); + return NULL; + } + vl_push(&l, x); + } + return vl_finish_array(d, &l); + } + case PT_DS: { + kv_list_t l = {0}; + for (;;) { + size_t save = *pos; + pdf_tok_t k; + pdf_lex(d, data, len, pos, &k); + if (k.kind == PT_DE || k.kind == PT_EOF) { + break; + } + if (k.kind == PT_KW && (kw_ends_value(&k) || pdf_kw_is(&k, "obj"))) { + *pos = save; + break; + } + if (k.kind != PT_NAME) { + continue; + } + pdf_tok_t k2; + pdf_lex(d, data, len, pos, &k2); + if (k2.kind == PT_DE) { + kl_push(&l, k.s, k.n, &PDF_NULL_VALUE); + break; + } + pdf_val_t *x = pdf_parse_value(d, data, len, pos, &k2, depth + 1); + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l.v); + return NULL; + } + kl_push(&l, k.s, k.n, x); + } + return kl_finish_dict(d, &l, true); + } + case PT_KW: + if (pdf_kw_is(t, "true") || pdf_kw_is(t, "false")) { + pdf_val_t *b = new_val(d, PV_BOOL); + if (b) { + b->b = t->s[0] == 't'; + } + return b; + } + if (pdf_kw_is(t, "null")) { + return &PDF_NULL_VALUE; + } + { + pdf_val_t *k = new_val(d, PV_KW); + if (k) { + k->s = t->s; + k->n = t->n; + } + return k; + } + default: + return &PDF_NULL_VALUE; + } +} + +pdf_val_t *pdf_parse_content_array(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + int depth) { + if (depth > PDF_MAX_NESTING) { + return NULL; + } + val_list_t l = {0}; + for (;;) { + pdf_tok_t a; + pdf_lex(d, data, len, pos, &a); + if (a.kind == PT_AE || a.kind == PT_EOF) { + break; + } + pdf_val_t *x = NULL; + if (a.kind == PT_NUM || a.kind == PT_STR || a.kind == PT_NAME) { + x = val_from_tok(d, &a); + } else if (a.kind == PT_AS) { + x = pdf_parse_content_array(d, data, len, pos, depth + 1); + } else if (a.kind == PT_DS) { + x = pdf_parse_value(d, data, len, pos, &a, depth + 1); + } else if (a.kind == PT_KW) { + *pos = (size_t)(a.s - data); /* the operator is the caller's */ + break; + } else { + continue; + } + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l.v); + return NULL; + } + vl_push(&l, x); + } + return vl_finish_array(d, &l); +} + +/* ── Dict access ─────────────────────────────────────────────────── */ + +pdf_val_t *pdf_dfind_n(const pdf_val_t *dict, const unsigned char *key, size_t kn) { + if (!dict || dict->kind != PV_DICT) { + return NULL; + } + int lo = 0; + int hi = dict->nkv - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int c = kv_key_cmp(dict->kv[mid].key, dict->kv[mid].klen, key, kn); + if (c == 0) { + return dict->kv[mid].val; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return NULL; +} + +static pdf_val_t *dict_find(const pdf_val_t *dict, const char *key) { + return pdf_dfind_n(dict, (const unsigned char *)key, strlen(key)); +} + +pdf_val_t *pdf_dfind(const pdf_val_t *dict, const char *key) { + return dict_find(dict, key); +} + +bool pdf_dhas(const pdf_val_t *dict, const char *key) { + return dict_find(dict, key) != NULL; +} + +pdf_val_t *pdf_dget_raw(const pdf_val_t *dict, const char *key) { + pdf_val_t *v = dict_find(dict, key); + return (v && v->kind != PV_NULL) ? v : NULL; +} + +pdf_val_t *pdf_dget(pdf_doc_t *d, const pdf_val_t *dict, const char *key) { + pdf_val_t *v = pdf_resolve(d, pdf_dget_raw(dict, key)); + return (v && v->kind != PV_NULL) ? v : NULL; +} + +bool pdf_is_name(const pdf_val_t *v, const char *name) { + size_t n = strlen(name); + return v && v->kind == PV_NAME && v->n == n && memcmp(v->s, name, n) == 0; +} + +bool pdf_is_num(const pdf_val_t *v) { + return v && v->kind == PV_NUM; +} + +double pdf_num(const pdf_val_t *v) { + return pdf_is_num(v) ? v->num : 0.0; +} + +/* ── Objects ─────────────────────────────────────────────────────── */ + +/* "[ws]*(\d+)[ws]+(\d+)[ws]+obj" not followed by a regular character, at base. */ +static bool match_obj_hdr(const pdf_doc_t *d, size_t base, int64_t *num, size_t *end) { + const unsigned char *s = d->data; + size_t len = d->len; + size_t p = base; + while (p < len && is_ws(s[p])) { + p++; + } + size_t a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a) { + return false; + } + *num = sat_digits(s + a, p - a); + size_t w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w) { + return false; + } + a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a) { + return false; + } + w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w || p + 3 > len || memcmp(s + p, "obj", 3) != 0) { + return false; + } + p += 3; + if (p < len && is_reg(s[p])) { + return false; + } + *end = p; + return true; +} + +static pdf_xent_t *xent(pdf_doc_t *d, int64_t num) { + int64_t i = pdf_map_get(&d->xref.by_num, (uint64_t)num); + return i < 0 ? NULL : &d->xref.v[i]; +} + +/* "endstream" follows q, after whitespace, within the 32 bytes after it. */ +static bool endstream_follows(const pdf_doc_t *d, size_t q) { + size_t lim = d->len - q < PDF_TAIL_WINDOW ? d->len : q + PDF_TAIL_WINDOW; + for (size_t p = q; p < lim; p++) { + if (!is_ws(d->data[p])) { + return lim - p >= 9 && memcmp(d->data + p, "endstream", 9) == 0; + } + } + return false; +} + +/* The object at off (or off + hdr). NULL: no matching header there. */ +static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect, + bool *ok) { + *ok = false; + size_t at = 0; + bool found = false; + int64_t num = 0; + for (int k = 0; k < 2 && !found; k++) { + int64_t base = pdf_sat_add(off, k ? (int64_t)d->hdr : 0); + if (base < 0 || (uint64_t)base > d->len) { + continue; + } + size_t end; + if (match_obj_hdr(d, (size_t)base, &num, &end) && (!has_expect || num == expect)) { + at = end; + found = true; + } + } + if (!found) { + return NULL; + } + const unsigned char *s = d->data; + size_t len = d->len; + size_t p = at; + pdf_tok_t t; + pdf_lex(d, s, len, &p, &t); + pdf_val_t *val = pdf_parse_value(d, s, len, &p, &t, 0); + if (!val) { + return NULL; + } + *ok = true; + size_t p2 = p; + pdf_tok_t t2; + lex_peek(d, s, len, &p2, &t2); + if (!(pdf_kw_is(&t2, "stream") && val->kind == PV_DICT)) { + return val; + } + size_t st = p2; + if (st + 1 < len && s[st] == '\r' && s[st + 1] == '\n') { + st += 2; + } else if (st < len && (s[st] == '\n' || s[st] == '\r')) { + st += 1; + } + pdf_val_t *ln = pdf_dget_raw(val, "Length"); + if (ln && ln->kind == PV_REF) { + pdf_xent_t *e = xent(d, ln->ref_num); + ln = (e && e->loading) ? NULL : pdf_resolve(d, ln); + } + const unsigned char *raw = NULL; + size_t raw_len = 0; + if (ln && ln->kind == PV_NUM && ln->is_int && ln->inum >= 0 && + (uint64_t)ln->inum <= (uint64_t)(len - st)) { + if (endstream_follows(d, st + (size_t)ln->inum)) { + raw = s + st; + raw_len = (size_t)ln->inum; + } + } + if (!raw) { + d->counts->stream_length_recovered++; + const unsigned char *e = cbm_memmem(s + st, len - st, "endstream", 9); + size_t end = e ? (size_t)(e - s) : len; + raw = s + st; + raw_len = end - st; + if (raw_len >= 2 && raw[raw_len - 2] == '\r' && raw[raw_len - 1] == '\n') { + raw_len -= 2; + } else if (raw_len >= 1 && (raw[raw_len - 1] == '\n' || raw[raw_len - 1] == '\r')) { + raw_len -= 1; + } + } + pdf_stream_t *sm = (pdf_stream_t *)pdf_calloc(d, sizeof(*sm)); + pdf_val_t *sv = new_val(d, PV_STREAM); + if (!sm || !sv) { + return NULL; + } + sm->dict = val; + sm->raw = raw; + sm->raw_len = raw_len; + sv->stream = sm; + return sv; +} + +/* (number, position) pairs, sorted to find the first of a number. */ +static int numidx_cmp(const void *x, const void *y) { + const pdf_numidx_t *a = (const pdf_numidx_t *)x; + const pdf_numidx_t *b = (const pdf_numidx_t *)y; + if (a->num != b->num) { + return a->num < b->num ? -1 : 1; + } + return a->idx < b->idx ? -1 : a->idx > b->idx ? 1 : 0; +} + +/* The first pair of `num` in a sorted array; n when none. */ +static int numidx_first(const pdf_numidx_t *v, int n, int64_t num) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (v[mid].num < num) { + lo = mid + 1; + } else { + hi = mid; + } + } + return (lo < n && v[lo].num == num) ? lo : n; +} + +/* Every "N G obj" header in file order (the reconstruction and the fallback + * for objects missing at their offset both read it). */ +static bool build_hdrs(pdf_doc_t *d) { + if (d->hdrs_built) { + return true; + } + d->hdrs_built = true; + const unsigned char *s = d->data; + size_t len = d->len; + int cap = 0; + size_t from = 0; + while (from + 3 <= len) { + const unsigned char *o = cbm_memmem(s + from, len - from, "obj", 3); + if (!o) { + break; + } + size_t q = (size_t)(o - s); + from = q + 3; + size_t p = q; + while (p > 0 && is_ws(s[p - 1])) { + p--; + } + if (p == q) { + continue; + } + size_t ge = p; + while (p > 0 && is_digit(s[p - 1])) { + p--; + } + size_t gs = p; + if (gs == ge) { + continue; + } + size_t w = p; + while (p > 0 && is_ws(s[p - 1])) { + p--; + } + if (p == w) { + continue; + } + size_t ne = p; + while (p > 0 && is_digit(s[p - 1])) { + p--; + } + size_t ns = p; + if (ns == ne) { + continue; + } + if (d->nhdrs == cap) { + int ncap = cap ? cap * 2 : PDF_MAP_MIN; + pdf_hdr_t *g = + (pdf_hdr_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, d->hdrs, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + d->hdrs = g; + cap = ncap; + } + pdf_hdr_t *h = &d->hdrs[d->nhdrs++]; + h->num = sat_digits(s + ns, ne - ns); + h->gen = sat_digits(s + gs, ge - gs); + h->off = ns; + h->num_digits = (uint8_t)((ne - ns) > UINT8_MAX ? UINT8_MAX : (ne - ns)); + h->gen_digits = (uint8_t)((ge - gs) > UINT8_MAX ? UINT8_MAX : (ge - gs)); + h->canonical = (s[ns] != '0' || ne - ns == 1) && (s[gs] != '0' || ge - gs == 1); + h->reg_after = q + 3 < len && is_reg(s[q + 3]); + } + return true; +} + +/* The first header written exactly "num gen obj" (the prototype's search). */ +static int64_t hdr_first_exact(pdf_doc_t *d, int64_t num, int64_t gen) { + if (!build_hdrs(d) || !d->nhdrs) { + return -1; + } + if (!d->hdr_sorted) { + d->hdr_sorted = (pdf_numidx_t *)pdf_big(d, (size_t)d->nhdrs * sizeof(pdf_numidx_t)); + if (!d->hdr_sorted) { + return -1; + } + for (int i = 0; i < d->nhdrs; i++) { + d->hdr_sorted[i].num = d->hdrs[i].num; + d->hdr_sorted[i].idx = i; + } + qsort(d->hdr_sorted, (size_t)d->nhdrs, sizeof(pdf_numidx_t), numidx_cmp); + } + for (int k = numidx_first(d->hdr_sorted, d->nhdrs, num); + k < d->nhdrs && d->hdr_sorted[k].num == num; k++) { + const pdf_hdr_t *h = &d->hdrs[d->hdr_sorted[k].idx]; + if (h->gen == gen && h->canonical) { + return (int64_t)h->off; + } + } + return -1; +} + +static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum); + +struct pdf_objstm { + const unsigned char *data; + size_t len; + int64_t first; + int64_t *onum; /* member numbers, in stream order */ + int64_t *ooff; + int n; + pdf_numidx_t *sorted; /* (onum, order), built on first need */ +}; + +static int64_t objstm_find(pdf_doc_t *d, pdf_objstm_t *sm, int64_t num, int64_t idx) { + if (idx >= 0 && idx < sm->n && sm->onum[idx] == num) { + return sm->ooff[idx]; + } + if (!sm->sorted) { + sm->sorted = (pdf_numidx_t *)pdf_big(d, (size_t)(sm->n ? sm->n : 1) * sizeof(pdf_numidx_t)); + if (!sm->sorted) { + return INT64_MIN; + } + for (int i = 0; i < sm->n; i++) { + sm->sorted[i].num = sm->onum[i]; + sm->sorted[i].idx = i; + } + qsort(sm->sorted, (size_t)sm->n, sizeof(pdf_numidx_t), numidx_cmp); + } + int k = numidx_first(sm->sorted, sm->n, num); + return k < sm->n ? sm->ooff[sm->sorted[k].idx] : INT64_MIN; +} + +pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num) { + int64_t idx = pdf_map_get(&d->xref.by_num, (uint64_t)num); + if (idx < 0) { + return NULL; + } + pdf_xent_t *e = &d->xref.v[idx]; + if (e->cached) { + return e->obj; + } + if (e->loading) { + return NULL; + } + e->loading = true; + uint8_t type = e->type; + int64_t a = e->a; + int64_t b = e->b; + pdf_val_t *obj = NULL; + if (type == 1) { + bool ok; + obj = parse_indirect_at(d, a, true, num, &ok); + if (!ok && !d->reconstructed) { + int64_t off = hdr_first_exact(d, num, b); + if (off >= 0) { + obj = parse_indirect_at(d, off, true, num, &ok); + } + } + if (!ok) { + obj = NULL; + } + } else if (type == 2) { + pdf_objstm_t *sm = objstm(d, a); + if (sm) { + int64_t off = objstm_find(d, sm, num, b); + int64_t at = off == INT64_MIN ? -1 : pdf_sat_add(sm->first, off); + if (at >= 0 && (uint64_t)at <= (uint64_t)sm->len) { + size_t p = (size_t)at; + pdf_tok_t t; + pdf_lex(d, sm->data, sm->len, &p, &t); + obj = pdf_parse_value(d, sm->data, sm->len, &p, &t, 0); + } + } + } + e = &d->xref.v[idx]; + e->loading = false; + e->cached = true; + e->obj = obj; + return obj; +} + +pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v) { + int depth = 0; + while (v && v->kind == PV_REF) { + v = pdf_get(d, v->ref_num); + if (++depth > PDF_MAX_RESOLVE) { + return NULL; + } + } + return v; +} + +static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) { + pdf_xent_t *e = xent(d, stmnum); + if (e && e->stm) { + return e->stm; + } + pdf_val_t *st = pdf_get(d, stmnum); + if (!st || st->kind != PV_STREAM) { + return NULL; + } + size_t dlen = 0; + const unsigned char *data = pdf_stream_decode(d, st->stream, &dlen); + if (!data) { + data = (const unsigned char *)""; + dlen = 0; + } + pdf_val_t *nv = pdf_dget_raw(st->stream->dict, "N"); + pdf_val_t *fv = pdf_dget_raw(st->stream->dict, "First"); + int64_t n = (nv && nv->kind == PV_NUM && nv->is_int) ? nv->inum : 0; + pdf_objstm_t *sm = (pdf_objstm_t *)pdf_calloc(d, sizeof(*sm)); + if (!sm) { + return NULL; + } + sm->data = data; + sm->len = dlen; + sm->first = (fv && fv->kind == PV_NUM && fv->is_int) ? fv->inum : 0; + /* at most one pair per two tokens of the header: bounded by the data */ + int64_t cap = n < 0 ? 0 : n; + if (cap > (int64_t)(dlen / 2 + 1)) { + cap = (int64_t)(dlen / 2 + 1); + } + sm->onum = (int64_t *)pdf_big(d, (size_t)(cap ? cap : 1) * sizeof(int64_t)); + sm->ooff = (int64_t *)pdf_big(d, (size_t)(cap ? cap : 1) * sizeof(int64_t)); + if (!sm->onum || !sm->ooff) { + return NULL; + } + size_t p = 0; + for (int64_t i = 0; i < n && sm->n < cap; i++) { + pdf_tok_t t1; + pdf_tok_t t2; + lex_peek(d, data, dlen, &p, &t1); + lex_peek(d, data, dlen, &p, &t2); + if (t1.kind != PT_NUM || t2.kind != PT_NUM) { + break; + } + sm->onum[sm->n] = t1.is_int ? t1.inum : pdf_d2i(t1.num); + sm->ooff[sm->n] = t2.is_int ? t2.inum : pdf_d2i(t2.num); + sm->n++; + } + e = xent(d, stmnum); + if (e) { + e->stm = sm; + } + return sm; +} + +/* ── Cross-reference ─────────────────────────────────────────────── */ + +static void xref_free(pdf_xref_t *x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, x->v); + pdf_map_free(&x->by_num); + memset(x, 0, sizeof(*x)); +} + +/* The entry for num; created (zeroed) when absent. *fresh says which. */ +static pdf_xent_t *xref_slot(pdf_doc_t *d, pdf_xref_t *x, int64_t num, bool *fresh) { + int64_t i = pdf_map_get(&x->by_num, (uint64_t)num); + if (i >= 0) { + *fresh = false; + return &x->v[i]; + } + if (x->n == x->cap) { + int ncap = x->cap ? x->cap * 2 : PDF_MAP_MIN; + pdf_xent_t *g = + (pdf_xent_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, x->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return NULL; + } + x->v = g; + x->cap = ncap; + } + if (!pdf_map_put(&x->by_num, (uint64_t)num, x->n)) { + d->nomem = true; + return NULL; + } + pdf_xent_t *e = &x->v[x->n++]; + memset(e, 0, sizeof(*e)); + e->num = num; + *fresh = true; + return e; +} + +/* "[ws]*(\d{1,10})[ws]+(\d{1,5})[ws]*([nf])" at p. */ +static bool match_xref_entry(const pdf_doc_t *d, size_t p, int64_t *off, int64_t *gen, char *kind, + size_t *end) { + const unsigned char *s = d->data; + size_t len = d->len; + while (p < len && is_ws(s[p])) { + p++; + } + size_t a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a || p - a > 10) { + return false; + } + *off = sat_digits(s + a, p - a); + size_t w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w) { + return false; + } + a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a || p - a > 5) { + return false; + } + *gen = sat_digits(s + a, p - a); + while (p < len && is_ws(s[p])) { + p++; + } + if (p >= len || (s[p] != 'n' && s[p] != 'f')) { + return false; + } + *kind = (char)s[p]; + *end = p + 1; + return true; +} + +static bool xref_table(pdf_doc_t *d, size_t p, pdf_xref_t *sec, pdf_val_t **tr) { + const unsigned char *s = d->data; + size_t len = d->len; + for (;;) { + pdf_tok_t t; + size_t p2 = p; + pdf_lex(d, s, len, &p2, &t); + if (pdf_kw_is(&t, "trailer")) { + pdf_tok_t t2; + pdf_lex(d, s, len, &p2, &t2); + pdf_val_t *v = pdf_parse_value(d, s, len, &p2, &t2, 0); + if (!v) { + return false; + } + *tr = v->kind == PV_DICT ? v : NULL; + return true; + } + if (t.kind != PT_NUM) { + return false; + } + pdf_tok_t tc; + pdf_lex(d, s, len, &p2, &tc); + if (tc.kind != PT_NUM || !tc.is_int) { + return false; + } + int64_t start = t.is_int ? t.inum : pdf_d2i(t.num); + bool use = t.is_int || t.num == (double)start; + p = p2; + for (int64_t i = 0; i < tc.inum; i++) { + int64_t off; + int64_t gen; + char kind; + size_t end; + if (!match_xref_entry(d, p, &off, &gen, &kind, &end)) { + return false; + } + p = end; + if (!use) { + continue; + } + bool fresh; + pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh); + if (!e) { + return false; + } + if (fresh && kind == 'n') { + e->type = 1; + e->a = off; + e->b = gen; + } + } + } +} + +static int64_t be_field(const unsigned char *p, int w) { + uint64_t v = 0; + for (int i = 0; i < w; i++) { + v = (v << PDF_BITS_PER_BYTE) | p[i]; + } + return (int64_t)v; +} + +static bool num_to_int(const pdf_val_t *v, int64_t *out) { + if (!v || v->kind != PV_NUM) { + return false; + } + *out = v->is_int ? v->inum : pdf_d2i(v->num); + return true; +} + +static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_t **tr) { + bool ok; + pdf_val_t *v = parse_indirect_at(d, off, false, 0, &ok); + if (!v || v->kind != PV_STREAM) { + return false; + } + pdf_val_t *dict = v->stream->dict; + pdf_val_t *W = pdf_dget_raw(dict, "W"); + if (!W || W->kind != PV_ARR || W->count != 3) { + return false; + } + int64_t w[3]; + for (int i = 0; i < 3; i++) { + if (!num_to_int(W->items[i], &w[i]) || w[i] < 0 || w[i] > PDF_MAX_FIELD) { + return false; + } + } + pdf_val_t *size = pdf_dget_raw(dict, "Size"); + pdf_val_t *index = pdf_dget_raw(dict, "Index"); + bool dflt = !index || (index->kind == PV_ARR && index->count == 0) || + (index->kind == PV_NUM && index->num == 0.0) || + (index->kind == PV_BOOL && !index->b); + int64_t pairs[2]; + pdf_val_t **items = NULL; + int nitems = 0; + if (dflt) { + pairs[0] = 0; + pairs[1] = 0; + if (size && !num_to_int(size, &pairs[1])) { + return false; + } + nitems = 2; + } else if (index->kind == PV_ARR) { + items = index->items; + nitems = index->count; + } else { + return false; + } + size_t rlen = 0; + const unsigned char *raw = pdf_stream_decode(d, v->stream, &rlen); + if (!raw) { + return false; + } + int64_t rec = w[0] + w[1] + w[2]; + if (rec == 0) { + return false; + } + size_t pos = 0; + for (int j = 0; j + 1 < nitems; j += 2) { + int64_t start; + int64_t count; + if (items) { + if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) { + return false; + } + } else { + start = pairs[0]; + count = pairs[1]; + } + for (int64_t i = 0; i < count; i++) { + if (pos + (size_t)rec > rlen) { + break; + } + const unsigned char *r = raw + pos; + int64_t t = w[0] ? be_field(r, (int)w[0]) : 1; + int64_t f2 = be_field(r + w[0], (int)w[1]); + int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0; + pos += (size_t)rec; + bool fresh; + pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh); + if (!e) { + return false; + } + if (!fresh) { + continue; + } + if (t == 1 || t == 2) { + e->type = (uint8_t)t; + e->a = f2; + e->b = f3; + } + } + } + *tr = dict; + return true; +} + +static size_t skip_ws(const pdf_doc_t *d, size_t p) { + while (p < d->len && is_ws(d->data[p])) { + p++; + } + return p; +} + +static bool starts_with(const pdf_doc_t *d, size_t p, const char *lit) { + size_t n = strlen(lit); + return p <= d->len && d->len - p >= n && memcmp(d->data + p, lit, n) == 0; +} + +static bool find_xref_at(pdf_doc_t *d, int64_t off, size_t *out) { + for (int k = 0; k < 2; k++) { + int64_t base = pdf_sat_add(off, k ? (int64_t)d->hdr : 0); + if (base < 0 || (uint64_t)base > d->len) { + continue; + } + size_t p = skip_ws(d, (size_t)base); + if (starts_with(d, p, "xref")) { + *out = p; + return true; + } + int64_t num; + size_t end; + if (match_obj_hdr(d, (size_t)base, &num, &end)) { + *out = (size_t)base; + return true; + } + } + int64_t lo = pdf_sat_add(off, -PDF_XREF_NEAR); + lo = lo < 0 ? 0 : lo; + int64_t hi = pdf_sat_add(off, PDF_XREF_NEAR); + if ((uint64_t)hi > d->len) { + hi = (int64_t)d->len; + } + if (hi - lo >= 4 && (uint64_t)lo < d->len) { + const unsigned char *x = cbm_memmem(d->data + lo, (size_t)(hi - lo), "xref", 4); + if (x) { + d->counts->xref_offset_fixed++; + *out = (size_t)(x - d->data); + return true; + } + } + return false; +} + +static bool trailer_key_structural(const pdf_kv_t *kv) { + static const char *const keys[] = {"Prev", "XRefStm", "Type", "W", "Index", + "Filter", "DecodeParms", "Length", "Size"}; + for (size_t i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + size_t n = strlen(keys[i]); + if (kv->klen == n && memcmp(kv->key, keys[i], n) == 0) { + return true; + } + } + return false; +} + +static bool load_xref(pdf_doc_t *d) { + const unsigned char *s = d->data; + size_t len = d->len; + /* the last "startxref" */ + size_t i = len < 9 ? 0 : len - 9 + 1; + bool found = false; + while (i-- > 0) { + if (s[i] == 's' && memcmp(s + i, "startxref", 9) == 0) { + found = true; + break; + } + } + if (!found) { + return false; + } + size_t p = i + 9; + pdf_tok_t t; + pdf_lex(d, s, len, &p, &t); + if (t.kind != PT_NUM) { + return false; + } + pdf_map_t seen = {0}; + kv_list_t trail = {0}; + bool first = true; + bool ok = true; + pdf_val_t *offv = new_val(d, PV_NUM); + if (!offv) { + return false; + } + offv->is_int = t.is_int; + offv->inum = t.inum; + pdf_val_t *off = offv; + while (off && off->kind == PV_NUM && off->is_int && + pdf_map_get(&seen, (uint64_t)off->inum) < 0 && seen.n < PDF_MAX_PREV) { + if (!pdf_map_put(&seen, (uint64_t)off->inum, 1)) { + d->nomem = true; + ok = false; + break; + } + size_t at; + if (!find_xref_at(d, off->inum, &at)) { + ok = false; + break; + } + pdf_xref_t sec = {0}; + pdf_val_t *tr = NULL; + if (starts_with(d, at, "xref")) { + if (!xref_table(d, at + 4, &sec, &tr)) { + xref_free(&sec); + ok = false; + break; + } + pdf_val_t *xs = pdf_dget_raw(tr, "XRefStm"); + if (xs && xs->kind == PV_NUM && xs->is_int) { + pdf_xref_t sec2 = {0}; + pdf_val_t *tr2 = NULL; + if (xref_stream_at(d, xs->inum, &sec2, &tr2)) { + for (int k = 0; k < sec2.n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &sec, sec2.v[k].num, &fresh); + if (e && (fresh || e->type == 0)) { + e->type = sec2.v[k].type; + e->a = sec2.v[k].a; + e->b = sec2.v[k].b; + } + } + } + xref_free(&sec2); + } + } else if (!xref_stream_at(d, (int64_t)at, &sec, &tr)) { + xref_free(&sec); + ok = false; + break; + } + for (int k = 0; k < sec.n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, sec.v[k].num, &fresh); + if (e && fresh) { + e->type = sec.v[k].type; + e->a = sec.v[k].a; + e->b = sec.v[k].b; + } + } + xref_free(&sec); + for (int k = 0; tr && k < tr->nkv; k++) { + if (first || !trailer_key_structural(&tr->kv[k])) { + kl_push(&trail, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + } + first = false; + off = pdf_dget_raw(tr, "Prev"); + } + pdf_map_free(&seen); + d->trailer = kl_finish_dict(d, &trail, false); + if (!ok || d->nomem) { + return false; + } + return d->xref.n > 0; +} + +static int i64_cmp(const void *x, const void *y) { + int64_t a = *(const int64_t *)x; + int64_t b = *(const int64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +/* The xref's object numbers, ascending (a snapshot). */ +static int64_t *xref_sorted_nums(pdf_doc_t *d, int *n) { + *n = d->xref.n; + int64_t *v = (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(*n ? *n : 1) * sizeof(*v)); + if (!v) { + d->nomem = true; + *n = 0; + return NULL; + } + for (int i = 0; i < *n; i++) { + v[i] = d->xref.v[i].num; + } + qsort(v, (size_t)*n, sizeof(*v), i64_cmp); + return v; +} + +static void cache_clear(pdf_doc_t *d) { + for (int i = 0; i < d->xref.n; i++) { + d->xref.v[i].cached = false; + d->xref.v[i].obj = NULL; + } +} + +static void reconstruct(pdf_doc_t *d) { + d->reconstructed = true; + d->counts->xref_reconstructed++; + xref_free(&d->xref); + if (!build_hdrs(d)) { + return; + } + for (int i = 0; i < d->nhdrs; i++) { + const pdf_hdr_t *h = &d->hdrs[i]; + if (h->num_digits > 10 || h->gen_digits > 5 || h->reg_after) { + continue; + } + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, h->num, &fresh); + if (!e) { + return; + } + e->type = 1; + e->a = (int64_t)h->off; + e->b = h->gen; + } + /* every "trailer[ws]*<<" dict, the later overriding */ + const unsigned char *s = d->data; + size_t len = d->len; + kv_list_t trail = {0}; + size_t from = 0; + while (from + 7 <= len) { + const unsigned char *x = cbm_memmem(s + from, len - from, "trailer", 7); + if (!x) { + break; + } + size_t q = (size_t)(x - s) + 7; + from = q; + q = skip_ws(d, q); + if (!starts_with(d, q, "<<")) { + continue; + } + size_t p = q; + pdf_tok_t t; + pdf_lex(d, s, len, &p, &t); + pdf_val_t *v = pdf_parse_value(d, s, len, &p, &t, 0); + if (v && v->kind == PV_DICT) { + for (int k = 0; k < v->nkv; k++) { + kl_push(&trail, v->kv[k].key, v->kv[k].klen, v->kv[k].val); + } + } + from = q + 2 > from ? q + 2 : from; + } + pdf_val_t *tr = kl_finish_dict(d, &trail, true); + static const char *const xkeys[] = {"Root", "Info", "ID", "Encrypt"}; + bool taken[4] = {false, false, false, false}; + kv_list_t extra = {0}; + int n = 0; + int64_t *nums = xref_sorted_nums(d, &n); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (!o || o->kind != PV_STREAM) { + continue; + } + pdf_val_t *typ = pdf_dget_raw(o->stream->dict, "Type"); + if (pdf_is_name(typ, "ObjStm")) { + pdf_objstm_t *sm = objstm(d, nums[i]); + for (int k = 0; sm && k < sm->n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, sm->onum[k], &fresh); + if (e && fresh) { + e->type = 2; + e->a = nums[i]; + e->b = k; + } + } + } else if (pdf_is_name(typ, "XRef")) { + for (size_t k = 0; k < sizeof(xkeys) / sizeof(xkeys[0]); k++) { + pdf_val_t *v = dict_find(o->stream->dict, xkeys[k]); + if (v && !taken[k] && !dict_find(tr, xkeys[k])) { + kl_push(&extra, (const unsigned char *)xkeys[k], strlen(xkeys[k]), v); + taken[k] = true; + } + } + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); + if (extra.n && tr) { + for (int k = 0; k < tr->nkv; k++) { + kl_push(&extra, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + /* the scanned trailers come after: keep_last lets them win, and the + * stream keys were only added where the trailers had none */ + tr = kl_finish_dict(d, &extra, true); + } else { + cbm_free(CBM_MEM_CLASS_EXTRACT, extra.v); + } + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(tr, "Root")); + if (!root || root->kind != PV_DICT) { + nums = xref_sorted_nums(d, &n); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (o && o->kind == PV_DICT && pdf_is_name(pdf_dget_raw(o, "Type"), "Catalog")) { + pdf_val_t *r = new_val(d, PV_REF); + if (r) { + r->ref_num = nums[i]; + kv_list_t one = {0}; + for (int k = 0; tr && k < tr->nkv; k++) { + kl_push(&one, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + kl_push(&one, (const unsigned char *)"Root", 4, r); + tr = kl_finish_dict(d, &one, true); + } + break; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); + } + d->trailer = tr; + cache_clear(d); +} + +bool pdf_doc_open(pdf_doc_t *d) { + const unsigned char *h = cbm_memmem(d->data, d->len, "%PDF-", 5); + d->hdr = h ? (size_t)(h - d->data) : 0; + bool ok = load_xref(d); + if (ok) { + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(d->trailer, "Root")); + ok = root && root->kind == PV_DICT; + } + if (!ok && !d->nomem) { + reconstruct(d); + } + return !d->nomem; +} + +void pdf_doc_close(pdf_doc_t *d) { + if (d->scratch_arena_live) { + cbm_arena_destroy(&d->scratch_arena); + } + xref_free(&d->xref); + pdf_map_free(&d->fonts); + cbm_free(CBM_MEM_CLASS_EXTRACT, d->hdrs); + cbm_free(CBM_MEM_CLASS_EXTRACT, d->scratch); + for (int i = 0; i < d->nowned; i++) { + cbm_free(CBM_MEM_CLASS_EXTRACT, d->owned[i]); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)d->owned); + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)d->font_list); +} + +/* ── Filters ─────────────────────────────────────────────────────── */ + +/* Inflate src into out until the stream ends, an error, or the output passes + * the guard. chunk: feed the input in the prototype's 4096-byte steps, where + * an error drops the output of the step it happened in. */ +static void inflate_run(const unsigned char *src, size_t n, int wbits, bool chunk, pdf_buf_t *out, + bool *ended, bool *errored) { + *ended = false; + *errored = false; + z_stream z; + memset(&z, 0, sizeof(z)); + if (inflateInit2(&z, wbits) != Z_OK) { + *errored = true; + return; + } + size_t step = chunk ? PDF_INFLATE_CHUNK : (n ? n : 1); + for (size_t i = 0; i < n || (n == 0 && i == 0); i += step) { + size_t cl = (n - i < step) ? n - i : step; + size_t mark = out->n; + z.next_in = (Bytef *)(uintptr_t)(src + i); + z.avail_in = (uInt)cl; + bool bad = false; + for (;;) { + if (!buf_reserve(out, PDF_INFLATE_GROW)) { + bad = true; + break; + } + z.next_out = out->p + out->n; + z.avail_out = (uInt)(out->cap - out->n > UINT_MAX ? UINT_MAX : out->cap - out->n); + uInt before = z.avail_out; + int r = inflate(&z, Z_NO_FLUSH); + out->n += before - z.avail_out; + if (r == Z_STREAM_END) { + *ended = true; + break; + } + if (r == Z_BUF_ERROR) { + break; + } + if (r != Z_OK) { + bad = true; + break; + } + if (out->n > PDF_MAX_STREAM_OUT) { + break; + } + if (z.avail_in == 0 && z.avail_out != 0) { + break; + } + } + if (bad) { + out->n = mark; + *errored = true; + break; + } + if (*ended || out->n > PDF_MAX_STREAM_OUT || n == 0) { + break; + } + } + inflateEnd(&z); +} + +static bool f_flate(pdf_doc_t *d, const unsigned char *in, size_t n, pdf_buf_t *out) { + bool ended; + bool errored; + inflate_run(in, n, PDF_ZLIB_WBITS, false, out, &ended, &errored); + if (out->fail) { + return false; + } + if (ended || out->n > PDF_MAX_STREAM_OUT) { + return true; + } + d->counts->flate_recovered++; + for (int k = 0; k < 2; k++) { + out->n = 0; + const unsigned char *src = k ? in + (n >= 2 ? 2 : n) : in; + size_t sn = k ? (n >= 2 ? n - 2 : 0) : n; + inflate_run(src, sn, k ? -PDF_ZLIB_WBITS : PDF_ZLIB_WBITS, true, out, &ended, &errored); + if (out->fail) { + return false; + } + if (out->n) { + return true; + } + } + out->n = 0; + return true; +} + +/* PNG (>= 10) and TIFF (2) predictors. false: parameters it cannot apply. */ +static bool f_predict(pdf_doc_t *d, pdf_val_t *parms, pdf_buf_t *io) { + if (!parms || parms->kind != PV_DICT) { + return true; + } + pdf_val_t *pv = pdf_dget(d, parms, "Predictor"); + double pred = 1; + if (pv) { + if (pv->kind != PV_NUM) { + return false; + } + pred = pv->num == 0.0 ? 1 : pv->num; + } + if (pred == 1) { + return true; + } + static const char *const keys[] = {"Colors", "BitsPerComponent", "Columns"}; + static const int64_t dflt[] = {1, 8, 1}; + int64_t prm[3]; + bool ints = true; + for (int i = 0; i < 3; i++) { + pdf_val_t *v = pdf_dget(d, parms, keys[i]); + if (!v || (v->kind == PV_NUM && v->num == 0.0)) { + prm[i] = dflt[i]; + } else if (v->kind != PV_NUM) { + return false; + } else { + ints = ints && v->is_int; + prm[i] = v->is_int ? v->inum : pdf_d2i(v->num); + } + } + int64_t colors = prm[0]; + int64_t bpc = prm[1]; + int64_t cols = prm[2]; + if (pred < PDF_PNG_PRED && !(pred == PDF_TIFF_PRED && bpc == PDF_BITS_PER_BYTE)) { + return true; + } + if (!ints) { + return false; + } + /* products of hostile parameters: computed in double and bounded */ + double bits = (double)colors * (double)bpc; + double rowd = floor((bits * (double)cols + 7) / PDF_BITS_PER_BYTE); + double bppd = floor(bits / PDF_BITS_PER_BYTE); + if (rowd > (double)PDF_MAX_STREAM_OUT) { + return false; + } + int64_t rowlen = rowd < -2 ? -2 : (int64_t)rowd; + int64_t bpp = + bppd<1 ? 1 : bppd>(double) PDF_MAX_STREAM_OUT ? (int64_t)PDF_MAX_STREAM_OUT : (int64_t)bppd; + if (pred >= PDF_PNG_PRED) { + if (rowlen == -1) { + return false; + } + pdf_buf_t out = {0}; + if (rowlen < 0) { + buf_free(io); + *io = out; + return true; + } + size_t rl = (size_t)rowlen; + unsigned char *prev = (unsigned char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, rl ? rl : 1); + if (!prev) { + d->nomem = true; + return false; + } + for (size_t i = 0; i < io->n; i += rl + 1) { + unsigned ft = io->p[i]; + if (!buf_reserve(&out, rl ? rl : 1)) { + break; + } + unsigned char *row = out.p + out.n; + size_t have = io->n - (i + 1) < rl ? io->n - (i + 1) : rl; + if (have) { + memcpy(row, io->p + i + 1, have); + } + memset(row + have, 0, rl - have); + size_t b = (size_t)bpp; + for (size_t j = 0; j < rl; j++) { + unsigned a = j >= b ? row[j - b] : 0; + unsigned up = prev[j]; + unsigned c = j >= b ? prev[j - b] : 0; + switch (ft) { + case 1: + if (j >= b) { + row[j] = (unsigned char)(row[j] + a); + } + break; + case 2: + row[j] = (unsigned char)(row[j] + up); + break; + case 3: + row[j] = (unsigned char)(row[j] + ((a + up) >> 1)); + break; + case 4: { + int p = (int)a + (int)up - (int)c; + int pa = abs(p - (int)a); + int pb = abs(p - (int)up); + int pc = abs(p - (int)c); + unsigned pr = (pa <= pb && pa <= pc) ? a : (pb <= pc ? up : c); + row[j] = (unsigned char)(row[j] + pr); + break; + } + default: + break; + } + } + if (rl) { + memcpy(prev, row, rl); + } + out.n += rl; + if (out.n > PDF_MAX_STREAM_OUT) { + break; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, prev); + if (out.fail) { + buf_free(&out); + d->nomem = true; + return false; + } + buf_free(io); + *io = out; + return true; + } + /* TIFF predictor 2, 8 bits per component */ + if (rowlen == 0) { + return false; + } + if (rowlen < 0) { + return true; + } + size_t rl = (size_t)rowlen; + size_t b = (size_t)bpp; + for (size_t r = 0; r < io->n; r += rl) { + size_t lim = r + rl < io->n ? r + rl : io->n; + for (size_t j = r + b; j < lim; j++) { + io->p[j] = (unsigned char)(io->p[j] + io->p[j - b]); + } + } + return true; +} + +static void f_lzw(const unsigned char *in, size_t n, double early, pdf_buf_t *out) { + /* table entries >= 258 are runs of the output: (position, length) */ + size_t *pos = (size_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, PDF_LZW_TABLE * sizeof(size_t)); + size_t *lenv = (size_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, PDF_LZW_TABLE * sizeof(size_t)); + if (!pos || !lenv) { + out->fail = true; + cbm_free(CBM_MEM_CLASS_EXTRACT, pos); + cbm_free(CBM_MEM_CLASS_EXTRACT, lenv); + return; + } + int bits = PDF_LZW_MIN_BITS; + uint32_t buf = 0; + int nbits = 0; + int64_t table = PDF_LZW_FIRST; /* the prototype's table length (it never shrinks below) */ + bool have_prev = false; + size_t prev_pos = 0; + size_t prev_len = 0; + for (size_t i = 0; i < n; i++) { + buf = (buf << PDF_BITS_PER_BYTE) | in[i]; + nbits += PDF_BITS_PER_BYTE; + while (nbits >= bits) { + nbits -= bits; + int64_t code = (int64_t)((buf >> nbits) & ((1U << bits) - 1)); + if (code == PDF_LZW_CLEAR) { + table = PDF_LZW_FIRST; + bits = PDF_LZW_MIN_BITS; + have_prev = false; + continue; + } + if (code == PDF_LZW_EOD) { + goto done; + } + size_t at = out->n; + if (!have_prev) { + if (code >= table) { + goto done; + } + if (!buf_byte(out, (unsigned char)code)) { + goto done; + } + } else if (code < table) { + if (code < PDF_LZW_CLEAR) { + if (!buf_byte(out, (unsigned char)code)) { + goto done; + } + } else { + size_t sp = pos[code]; + size_t sl = lenv[code]; + if (!buf_reserve(out, sl)) { + goto done; + } + memmove(out->p + out->n, out->p + sp, sl); + out->n += sl; + } + if (table < PDF_LZW_TABLE) { + pos[table] = prev_pos; + lenv[table] = prev_len + 1; + } + table++; + } else if (code == table) { + size_t sl = prev_len + 1; + if (!buf_reserve(out, sl)) { + goto done; + } + for (size_t k = 0; k < sl; k++) { + out->p[out->n + k] = out->p[prev_pos + k]; + } + out->n += sl; + if (table < PDF_LZW_TABLE) { + pos[table] = prev_pos; + lenv[table] = sl; + } + table++; + } else { + goto done; + } + have_prev = true; + prev_pos = at; + prev_len = out->n - at; + if ((double)table + early >= (double)(1U << bits) && bits < PDF_LZW_MAX_BITS) { + bits++; + } + if (out->n > PDF_MAX_STREAM_OUT) { + goto done; + } + } + } +done: + cbm_free(CBM_MEM_CLASS_EXTRACT, pos); + cbm_free(CBM_MEM_CLASS_EXTRACT, lenv); +} + +/* Python's base64.a85decode: false on a character or group it rejects. */ +static bool a85_strict(const unsigned char *in, size_t n, pdf_buf_t *out) { + static const unsigned char pad[4] = {'u', 'u', 'u', 'u'}; + unsigned char cur[PDF_A85_GROUP]; + int nc = 0; + out->n = 0; + for (size_t i = 0; i < n + 4; i++) { + unsigned char x = i < n ? in[i] : pad[i - n]; + if (x >= '!' && x <= 'u') { + cur[nc++] = x; + if (nc == PDF_A85_GROUP) { + uint64_t acc = 0; + for (int k = 0; k < PDF_A85_GROUP; k++) { + acc = acc * PDF_A85_BASE + (uint64_t)(cur[k] - '!'); + } + if (acc > 0xFFFFFFFFULL) { + return false; + } + unsigned char b4[4] = {(unsigned char)(acc >> 24), (unsigned char)(acc >> 16), + (unsigned char)(acc >> 8), (unsigned char)acc}; + if (!buf_put(out, b4, 4)) { + return false; + } + nc = 0; + } + } else if (x == 'z') { + if (nc) { + return false; + } + static const unsigned char zero[4] = {0, 0, 0, 0}; + if (!buf_put(out, zero, 4)) { + return false; + } + } else if (x == ' ' || x == '\t' || x == '\n' || x == '\r' || x == '\v') { + continue; + } else { + return false; + } + } + size_t padding = (size_t)(4 - nc); + out->n = padding >= out->n ? 0 : out->n - padding; + return true; +} + +static void f_a85(pdf_doc_t *d, const unsigned char *in, size_t n, pdf_buf_t *out) { + unsigned char *t = (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n ? n : 1); + if (!t) { + d->nomem = true; + out->fail = true; + return; + } + size_t k = 0; + for (size_t i = 0; i < n; i++) { + if (!is_ws(in[i])) { + t[k++] = in[i]; + } + } + size_t s0 = 0; + if (k >= 2 && t[0] == '<' && t[1] == '~') { + s0 = 2; + } + const unsigned char *e = cbm_memmem(t + s0, k - s0, "~>", 2); + size_t m = e ? (size_t)(e - (t + s0)) : k - s0; + if (!a85_strict(t + s0, m, out) && !out->fail) { + if (!a85_strict(t + s0, m - m % PDF_A85_GROUP, out) && !out->fail) { + out->n = 0; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, t); +} + +static void f_ahx(const unsigned char *in, size_t n, pdf_buf_t *out) { + const unsigned char *gt = (const unsigned char *)memchr(in, '>', n); + size_t e = gt ? (size_t)(gt - in) : n; + int hi = -1; + for (size_t i = 0; i < e; i++) { + int v = hex_val(in[i]); + if (v < 0) { + continue; + } + if (hi < 0) { + hi = v; + } else if (!buf_byte(out, (unsigned char)(hi * PDF_HEX_BASE + v))) { + return; + } else { + hi = -1; + } + } + if (hi >= 0) { + buf_byte(out, (unsigned char)(hi * PDF_HEX_BASE)); + } +} + +static void f_rld(const unsigned char *in, size_t n, pdf_buf_t *out) { + size_t i = 0; + while (i < n) { + unsigned L = in[i]; + if (L == PDF_RLD_EOD) { + break; + } + if (L < PDF_RLD_EOD) { + size_t take = (size_t)L + 1; + size_t have = n - (i + 1) < take ? n - (i + 1) : take; + if (!buf_put(out, in + i + 1, have)) { + return; + } + i += (size_t)L + 2; + } else { + if (i + 1 < n) { + size_t rep = 257 - (size_t)L; + if (!buf_reserve(out, rep)) { + return; + } + memset(out->p + out->n, in[i + 1], rep); + out->n += rep; + } + i += 2; + } + if (out->n > PDF_MAX_STREAM_OUT) { + return; + } + } +} + +typedef enum { + FL_FLATE, + FL_LZW, + FL_A85, + FL_AHX, + FL_RLD, + FL_CRYPT, + FL_IMAGE, + FL_UNKNOWN, +} pdf_filter_t; + +static pdf_filter_t filter_kind(const pdf_val_t *f) { + if (!f || f->kind != PV_NAME) { + return FL_UNKNOWN; + } + static const struct { + const char *name; + pdf_filter_t kind; + } names[] = { + {"FlateDecode", FL_FLATE}, + {"Fl", FL_FLATE}, + {"LZWDecode", FL_LZW}, + {"LZW", FL_LZW}, + {"ASCII85Decode", FL_A85}, + {"A85", FL_A85}, + {"ASCIIHexDecode", FL_AHX}, + {"AHx", FL_AHX}, + {"RunLengthDecode", FL_RLD}, + {"RL", FL_RLD}, + {"Crypt", FL_CRYPT}, + {"DCTDecode", FL_IMAGE}, + {"DCT", FL_IMAGE}, + {"JPXDecode", FL_IMAGE}, + {"CCITTFaxDecode", FL_IMAGE}, + {"CCF", FL_IMAGE}, + {"JBIG2Decode", FL_IMAGE}, + }; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + if (pdf_is_name(f, names[i].name)) { + return names[i].kind; + } + } + return FL_UNKNOWN; +} + +const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) { + if (st->done) { + *len = st->dec_len; + return st->decoded; + } + st->done = true; + st->decoded = NULL; + st->dec_len = 0; + *len = 0; + pdf_val_t *filters = pdf_resolve(d, pdf_dget_raw(st->dict, "Filter")); + pdf_val_t *parms = pdf_resolve(d, pdf_dget_raw(st->dict, "DecodeParms")); + pdf_val_t *single[1] = {filters}; + pdf_val_t **fl = single; + int nf = 0; + if (filters && filters->kind == PV_ARR) { + fl = filters->items; + nf = filters->count; + } else if (filters && filters->kind != PV_NULL) { + nf = 1; + } + const unsigned char *cur = st->raw; + size_t cur_len = st->raw_len; + pdf_buf_t held = {0}; /* the buffer cur points into, when it is ours */ + for (int i = 0; i < nf; i++) { + pdf_filter_t kind = filter_kind(pdf_resolve(d, fl[i])); + pdf_val_t *p = NULL; + if (parms && parms->kind == PV_ARR) { + p = i < parms->count ? pdf_resolve(d, parms->items[i]) : NULL; + } else { + p = parms; + } + if (kind == FL_CRYPT) { + continue; + } + if (kind == FL_IMAGE || kind == FL_UNKNOWN) { + if (kind == FL_UNKNOWN) { + d->counts->filters_unsupported++; + } + buf_free(&held); + return NULL; + } + pdf_buf_t out = {0}; + bool ok = true; + switch (kind) { + case FL_FLATE: + ok = f_flate(d, cur, cur_len, &out) && f_predict(d, p, &out); + break; + case FL_LZW: { + double early = 1; + if (p && p->kind == PV_DICT) { + pdf_val_t *ev = pdf_dget(d, p, "EarlyChange"); + if (ev && ev->kind != PV_NUM) { + ok = false; + } else if (ev) { + early = ev->num; + } + } + if (ok) { + f_lzw(cur, cur_len, early, &out); + ok = f_predict(d, p, &out); + } + break; + } + case FL_A85: + f_a85(d, cur, cur_len, &out); + break; + case FL_AHX: + f_ahx(cur, cur_len, &out); + break; + case FL_RLD: + f_rld(cur, cur_len, &out); + break; + default: + break; + } + if (out.fail) { + d->nomem = true; + ok = false; + } + buf_free(&held); + if (!ok) { + buf_free(&out); + return NULL; + } + if (out.n > PDF_MAX_STREAM_OUT) { + d->counts->stream_guard_hits++; + out.n = PDF_MAX_STREAM_OUT; + } + held = out; + cur = held.p ? held.p : (const unsigned char *)""; + cur_len = held.n; + } + if (held.p) { + unsigned char *fit = + (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, held.p, held.n ? held.n : 1); + if (fit) { + held.p = fit; + } + if (!pdf_own(d, held.p)) { + return NULL; + } + cur = held.p; + } + st->decoded = cur; + st->dec_len = cur_len; + *len = cur_len; + return cur; +} diff --git a/internal/cbm/pdf/pdf_tables.c b/internal/cbm/pdf/pdf_tables.c new file mode 100644 index 0000000000..a892abf01f --- /dev/null +++ b/internal/cbm/pdf/pdf_tables.c @@ -0,0 +1,1619 @@ +/* + * pdf_tables.c — static tables of the PDF text-layer extractor (generated). + * + * Glyph names (the Adobe Glyph List subset every standard Latin encoding + * uses, common TeX and symbol names, and the accented Latin letters), + * the base encodings of ISO 32000-1 Annex D as code -> UTF-8, and + * approximate standard-14 widths. Generated from the field-tested tables; + * do not edit by hand. + */ +#include "pdf_internal.h" + +#include + +const pdf_glyph_t PDF_GLYPHS[] = { + {"A", "A"}, + {"AE", "\303\206"}, + {"Aacute", "\303\201"}, + {"Abreve", "\304\202"}, + {"Acircumflex", "\303\202"}, + {"Adieresis", "\303\204"}, + {"Agrave", "\303\200"}, + {"Alpha", "\316\221"}, + {"Amacron", "\304\200"}, + {"Aogonek", "\304\204"}, + {"Aring", "\303\205"}, + {"Atilde", "\303\203"}, + {"B", "B"}, + {"Beta", "\316\222"}, + {"C", "C"}, + {"Cacute", "\304\206"}, + {"Ccaron", "\304\214"}, + {"Ccedilla", "\303\207"}, + {"Ccircumflex", "\304\210"}, + {"Cdotaccent", "\304\212"}, + {"Chi", "\316\247"}, + {"D", "D"}, + {"Dcaron", "\304\216"}, + {"Dcroat", "\304\220"}, + {"Delta", "\316\224"}, + {"Dslash", "\304\220"}, + {"E", "E"}, + {"Eacute", "\303\211"}, + {"Ebreve", "\304\224"}, + {"Ecaron", "\304\232"}, + {"Ecircumflex", "\303\212"}, + {"Edieresis", "\303\213"}, + {"Edotaccent", "\304\226"}, + {"Egrave", "\303\210"}, + {"Emacron", "\304\222"}, + {"Eng", "\305\212"}, + {"Eogonek", "\304\230"}, + {"Epsilon", "\316\225"}, + {"Eta", "\316\227"}, + {"Eth", "\303\220"}, + {"Euro", "\342\202\254"}, + {"F", "F"}, + {"G", "G"}, + {"Gamma", "\316\223"}, + {"Gbreve", "\304\236"}, + {"Gcedilla", "\304\242"}, + {"Gcircumflex", "\304\234"}, + {"Gdotaccent", "\304\240"}, + {"H", "H"}, + {"Hbar", "\304\246"}, + {"Hcircumflex", "\304\244"}, + {"Hslash", "\304\246"}, + {"I", "I"}, + {"IJ", "\304\262"}, + {"Iacute", "\303\215"}, + {"Ibreve", "\304\254"}, + {"Icircumflex", "\303\216"}, + {"Idieresis", "\303\217"}, + {"Idotaccent", "\304\260"}, + {"Ifraktur", "\342\204\221"}, + {"Igrave", "\303\214"}, + {"Imacron", "\304\252"}, + {"Iogonek", "\304\256"}, + {"Iota", "\316\231"}, + {"Itilde", "\304\250"}, + {"J", "J"}, + {"Jcircumflex", "\304\264"}, + {"K", "K"}, + {"Kappa", "\316\232"}, + {"Kcedilla", "\304\266"}, + {"L", "L"}, + {"Lacute", "\304\271"}, + {"Lambda", "\316\233"}, + {"Lcaron", "\304\275"}, + {"Lcedilla", "\304\273"}, + {"Ldot", "\304\277"}, + {"Lslash", "\305\201"}, + {"M", "M"}, + {"Mu", "\316\234"}, + {"N", "N"}, + {"Nacute", "\305\203"}, + {"Ncaron", "\305\207"}, + {"Ncedilla", "\305\205"}, + {"Ntilde", "\303\221"}, + {"Nu", "\316\235"}, + {"O", "O"}, + {"OE", "\305\222"}, + {"Oacute", "\303\223"}, + {"Obreve", "\305\216"}, + {"Ocircumflex", "\303\224"}, + {"Odieresis", "\303\226"}, + {"Ograve", "\303\222"}, + {"Ohungarumlaut", "\305\220"}, + {"Omacron", "\305\214"}, + {"Omega", "\316\251"}, + {"Omicron", "\316\237"}, + {"Oslash", "\303\230"}, + {"Otilde", "\303\225"}, + {"P", "P"}, + {"Phi", "\316\246"}, + {"Pi", "\316\240"}, + {"Psi", "\316\250"}, + {"Q", "Q"}, + {"R", "R"}, + {"Racute", "\305\224"}, + {"Rcaron", "\305\230"}, + {"Rcedilla", "\305\226"}, + {"Rfraktur", "\342\204\234"}, + {"Rho", "\316\241"}, + {"S", "S"}, + {"Sacute", "\305\232"}, + {"Scaron", "\305\240"}, + {"Scedilla", "\305\236"}, + {"Scircumflex", "\305\234"}, + {"Scommaaccent", "\310\230"}, + {"Sigma", "\316\243"}, + {"T", "T"}, + {"Tau", "\316\244"}, + {"Tbar", "\305\246"}, + {"Tcaron", "\305\244"}, + {"Tcedilla", "\305\242"}, + {"Tcommaaccent", "\310\232"}, + {"Theta", "\316\230"}, + {"Thorn", "\303\236"}, + {"Tslash", "\305\246"}, + {"U", "U"}, + {"Uacute", "\303\232"}, + {"Ubreve", "\305\254"}, + {"Ucircumflex", "\303\233"}, + {"Udieresis", "\303\234"}, + {"Ugrave", "\303\231"}, + {"Uhungarumlaut", "\305\260"}, + {"Umacron", "\305\252"}, + {"Uogonek", "\305\262"}, + {"Upsilon", "\316\245"}, + {"Upsilon1", "\317\222"}, + {"Uring", "\305\256"}, + {"Utilde", "\305\250"}, + {"V", "V"}, + {"W", "W"}, + {"Wcircumflex", "\305\264"}, + {"X", "X"}, + {"Xi", "\316\236"}, + {"Y", "Y"}, + {"Yacute", "\303\235"}, + {"Ycircumflex", "\305\266"}, + {"Ydieresis", "\305\270"}, + {"Z", "Z"}, + {"Zacute", "\305\271"}, + {"Zcaron", "\305\275"}, + {"Zdotaccent", "\305\273"}, + {"Zeta", "\316\226"}, + {"a", "a"}, + {"aacute", "\303\241"}, + {"abreve", "\304\203"}, + {"acircumflex", "\303\242"}, + {"acute", "\302\264"}, + {"adieresis", "\303\244"}, + {"ae", "\303\246"}, + {"afii00208", "\342\200\225"}, + {"afii10017", "\320\220"}, + {"afii61289", "\342\204\223"}, + {"agrave", "\303\240"}, + {"aleph", "\342\204\265"}, + {"alpha", "\316\261"}, + {"amacron", "\304\201"}, + {"ampersand", "&"}, + {"angbracketleft", "\342\237\250"}, + {"angbracketright", "\342\237\251"}, + {"angle", "\342\210\240"}, + {"aogonek", "\304\205"}, + {"approxequal", "\342\211\210"}, + {"aring", "\303\245"}, + {"arrowboth", "\342\206\224"}, + {"arrowdblboth", "\342\207\224"}, + {"arrowdblleft", "\342\207\220"}, + {"arrowdblright", "\342\207\222"}, + {"arrowdown", "\342\206\223"}, + {"arrowleft", "\342\206\220"}, + {"arrowright", "\342\206\222"}, + {"arrowup", "\342\206\221"}, + {"asciicircum", "^"}, + {"asciitilde", "~"}, + {"asterisk", "*"}, + {"asteriskmath", "\342\210\227"}, + {"at", "@"}, + {"atilde", "\303\243"}, + {"b", "b"}, + {"backslash", "\134"}, + {"bar", "|"}, + {"beta", "\316\262"}, + {"braceleft", "{"}, + {"braceright", "}"}, + {"bracketleft", "["}, + {"bracketright", "]"}, + {"breve", "\313\230"}, + {"brokenbar", "\302\246"}, + {"bullet", "\342\200\242"}, + {"bulletmath", "\342\210\231"}, + {"c", "c"}, + {"cacute", "\304\207"}, + {"caron", "\313\207"}, + {"ccaron", "\304\215"}, + {"ccedilla", "\303\247"}, + {"ccircumflex", "\304\211"}, + {"cdotaccent", "\304\213"}, + {"cedilla", "\302\270"}, + {"cent", "\302\242"}, + {"checkmark", "\342\234\223"}, + {"chi", "\317\207"}, + {"circlemultiply", "\342\212\227"}, + {"circleplus", "\342\212\225"}, + {"circumflex", "\313\206"}, + {"club", "\342\231\243"}, + {"colon", ":"}, + {"comma", ","}, + {"commaaccent", "\314\246"}, + {"congruent", "\342\211\205"}, + {"copyright", "\302\251"}, + {"currency", "\302\244"}, + {"d", "d"}, + {"dagger", "\342\200\240"}, + {"daggerdbl", "\342\200\241"}, + {"dcaron", "\304\217"}, + {"dcroat", "\304\221"}, + {"degree", "\302\260"}, + {"delta", "\316\264"}, + {"diamond", "\342\231\246"}, + {"dieresis", "\302\250"}, + {"divide", "\303\267"}, + {"dollar", "$"}, + {"dotaccent", "\313\231"}, + {"dotlessi", "\304\261"}, + {"dotlessi.sc", "\304\261"}, + {"dotlessj", "\310\267"}, + {"dotmath", "\342\213\205"}, + {"dslash", "\304\221"}, + {"e", "e"}, + {"eacute", "\303\251"}, + {"ebreve", "\304\225"}, + {"ecaron", "\304\233"}, + {"ecircumflex", "\303\252"}, + {"edieresis", "\303\253"}, + {"edotaccent", "\304\227"}, + {"egrave", "\303\250"}, + {"eight", "8"}, + {"element", "\342\210\210"}, + {"ellipsis", "\342\200\246"}, + {"emacron", "\304\223"}, + {"emdash", "\342\200\224"}, + {"emptyset", "\342\210\205"}, + {"endash", "\342\200\223"}, + {"eng", "\305\213"}, + {"eogonek", "\304\231"}, + {"epsilon", "\316\265"}, + {"equal", "="}, + {"equivalence", "\342\211\241"}, + {"eta", "\316\267"}, + {"eth", "\303\260"}, + {"exclam", "!"}, + {"exclamdbl", "\342\200\274"}, + {"exclamdown", "\302\241"}, + {"existential", "\342\210\203"}, + {"f", "f"}, + {"ff", "ff"}, + {"ffi", "ffi"}, + {"ffl", "ffl"}, + {"fi", "fi"}, + {"figuredash", "\342\200\222"}, + {"five", "5"}, + {"fl", "fl"}, + {"florin", "\306\222"}, + {"four", "4"}, + {"fraction", "\342\201\204"}, + {"g", "g"}, + {"gamma", "\316\263"}, + {"gbreve", "\304\237"}, + {"gcedilla", "\304\243"}, + {"gcircumflex", "\304\235"}, + {"gdotaccent", "\304\241"}, + {"germandbls", "\303\237"}, + {"gradient", "\342\210\207"}, + {"grave", "`"}, + {"greater", ">"}, + {"greaterequal", "\342\211\245"}, + {"guillemotleft", "\302\253"}, + {"guillemotright", "\302\273"}, + {"guilsinglleft", "\342\200\271"}, + {"guilsinglright", "\342\200\272"}, + {"h", "h"}, + {"hbar", "\304\247"}, + {"hcircumflex", "\304\245"}, + {"heart", "\342\231\245"}, + {"horizontalbar", "\342\200\225"}, + {"hslash", "\304\247"}, + {"hungarumlaut", "\313\235"}, + {"hyphen", "-"}, + {"hyphen.cap", "-"}, + {"i", "i"}, + {"iacute", "\303\255"}, + {"ibreve", "\304\255"}, + {"icircumflex", "\303\256"}, + {"idieresis", "\303\257"}, + {"igrave", "\303\254"}, + {"ij", "\304\263"}, + {"imacron", "\304\253"}, + {"infinity", "\342\210\236"}, + {"integral", "\342\210\253"}, + {"intersection", "\342\210\251"}, + {"iogonek", "\304\257"}, + {"iota", "\316\271"}, + {"itilde", "\304\251"}, + {"j", "j"}, + {"jcircumflex", "\304\265"}, + {"k", "k"}, + {"kappa", "\316\272"}, + {"kcedilla", "\304\267"}, + {"kgreenlandic", "\304\270"}, + {"l", "l"}, + {"lacute", "\304\272"}, + {"lambda", "\316\273"}, + {"lcaron", "\304\276"}, + {"lcedilla", "\304\274"}, + {"ldot", "\305\200"}, + {"less", "<"}, + {"lessequal", "\342\211\244"}, + {"logicaland", "\342\210\247"}, + {"logicalnot", "\302\254"}, + {"logicalor", "\342\210\250"}, + {"longs", "\305\277"}, + {"lozenge", "\342\227\212"}, + {"lslash", "\305\202"}, + {"m", "m"}, + {"macron", "\302\257"}, + {"minus", "\342\210\222"}, + {"minute", "\342\200\262"}, + {"mu", "\302\265"}, + {"mu1", "\302\265"}, + {"multiply", "\303\227"}, + {"n", "n"}, + {"nacute", "\305\204"}, + {"napostrophe", "\305\211"}, + {"nbspace", "\302\240"}, + {"ncaron", "\305\210"}, + {"ncedilla", "\305\206"}, + {"nine", "9"}, + {"nobreakspace", "\302\240"}, + {"notelement", "\342\210\211"}, + {"notequal", "\342\211\240"}, + {"ntilde", "\303\261"}, + {"nu", "\316\275"}, + {"numbersign", "#"}, + {"o", "o"}, + {"oacute", "\303\263"}, + {"obreve", "\305\217"}, + {"ocircumflex", "\303\264"}, + {"odieresis", "\303\266"}, + {"oe", "\305\223"}, + {"ogonek", "\313\233"}, + {"ograve", "\303\262"}, + {"ohungarumlaut", "\305\221"}, + {"omacron", "\305\215"}, + {"omega", "\317\211"}, + {"omega1", "\317\226"}, + {"omicron", "\316\277"}, + {"one", "1"}, + {"onedotenleader", "\342\200\244"}, + {"onehalf", "\302\275"}, + {"onequarter", "\302\274"}, + {"onesuperior", "\302\271"}, + {"openbullet", "\342\227\246"}, + {"ordfeminine", "\302\252"}, + {"ordmasculine", "\302\272"}, + {"oslash", "\303\270"}, + {"otilde", "\303\265"}, + {"p", "p"}, + {"paragraph", "\302\266"}, + {"parenleft", "("}, + {"parenright", ")"}, + {"partialdiff", "\342\210\202"}, + {"percent", "%"}, + {"period", "."}, + {"periodcentered", "\302\267"}, + {"perpendicular", "\342\212\245"}, + {"perthousand", "\342\200\260"}, + {"phi", "\317\206"}, + {"phi1", "\317\225"}, + {"pi", "\317\200"}, + {"plus", "+"}, + {"plusminus", "\302\261"}, + {"prime", "\342\200\262"}, + {"product", "\342\210\217"}, + {"propersubset", "\342\212\202"}, + {"propersuperset", "\342\212\203"}, + {"proportional", "\342\210\235"}, + {"psi", "\317\210"}, + {"q", "q"}, + {"question", "\077"}, + {"questiondown", "\302\277"}, + {"quotedbl", "\042"}, + {"quotedblbase", "\342\200\236"}, + {"quotedblbase.cap", "\342\200\236"}, + {"quotedblleft", "\342\200\234"}, + {"quotedblright", "\342\200\235"}, + {"quoteleft", "\342\200\230"}, + {"quoteright", "\342\200\231"}, + {"quotesinglbase", "\342\200\232"}, + {"quotesingle", "'"}, + {"r", "r"}, + {"racute", "\305\225"}, + {"radical", "\342\210\232"}, + {"rcaron", "\305\231"}, + {"rcedilla", "\305\227"}, + {"reflexsubset", "\342\212\206"}, + {"reflexsuperset", "\342\212\207"}, + {"registered", "\302\256"}, + {"rho", "\317\201"}, + {"ring", "\313\232"}, + {"s", "s"}, + {"sacute", "\305\233"}, + {"scaron", "\305\241"}, + {"scedilla", "\305\237"}, + {"scircumflex", "\305\235"}, + {"scommaaccent", "\310\231"}, + {"second", "\342\200\263"}, + {"section", "\302\247"}, + {"semicolon", ";"}, + {"seven", "7"}, + {"sfthyphen", "\302\255"}, + {"sigma", "\317\203"}, + {"sigma1", "\317\202"}, + {"similar", "\342\210\274"}, + {"six", "6"}, + {"slash", "/"}, + {"space", " "}, + {"space.alt", " "}, + {"spade", "\342\231\240"}, + {"st", "st"}, + {"sterling", "\302\243"}, + {"summation", "\342\210\221"}, + {"suppress", ""}, + {"t", "t"}, + {"tau", "\317\204"}, + {"tbar", "\305\247"}, + {"tcaron", "\305\245"}, + {"tcedilla", "\305\243"}, + {"tcommaaccent", "\310\233"}, + {"therefore", "\342\210\264"}, + {"theta", "\316\270"}, + {"theta1", "\317\221"}, + {"thorn", "\303\276"}, + {"three", "3"}, + {"threequarters", "\302\276"}, + {"threesuperior", "\302\263"}, + {"tilde", "\313\234"}, + {"trademark", "\342\204\242"}, + {"tslash", "\305\247"}, + {"two", "2"}, + {"twodotenleader", "\342\200\245"}, + {"twosuperior", "\302\262"}, + {"u", "u"}, + {"uacute", "\303\272"}, + {"ubreve", "\305\255"}, + {"ucircumflex", "\303\273"}, + {"udieresis", "\303\274"}, + {"ugrave", "\303\271"}, + {"uhungarumlaut", "\305\261"}, + {"umacron", "\305\253"}, + {"underscore", "_"}, + {"underscoredbl", "\342\200\227"}, + {"uni00A0", "\302\240"}, + {"union", "\342\210\252"}, + {"universal", "\342\210\200"}, + {"uogonek", "\305\263"}, + {"upsilon", "\317\205"}, + {"uring", "\305\257"}, + {"utilde", "\305\251"}, + {"v", "v"}, + {"visiblespace", "\342\220\243"}, + {"w", "w"}, + {"wcircumflex", "\305\265"}, + {"weierstrass", "\342\204\230"}, + {"x", "x"}, + {"xi", "\316\276"}, + {"y", "y"}, + {"yacute", "\303\275"}, + {"ycircumflex", "\305\267"}, + {"ydieresis", "\303\277"}, + {"yen", "\302\245"}, + {"z", "z"}, + {"zacute", "\305\272"}, + {"zcaron", "\305\276"}, + {"zdotaccent", "\305\274"}, + {"zero", "0"}, + {"zeta", "\316\266"}, +}; +const int PDF_GLYPH_COUNT = (int)(sizeof(PDF_GLYPHS) / sizeof(PDF_GLYPHS[0])); + +const char *const PDF_ENC_STANDARD[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "\342\200\231", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "\342\200\230", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\302\241", + "\302\242", + "\302\243", + "\342\201\204", + "\302\245", + "\306\222", + "\302\247", + "\302\244", + "'", + "\342\200\234", + "\302\253", + "\342\200\271", + "\342\200\272", + "fi", + "fl", + NULL, + "\342\200\223", + "\342\200\240", + "\342\200\241", + "\302\267", + NULL, + "\302\266", + "\342\200\242", + "\342\200\232", + "\342\200\236", + "\342\200\235", + "\302\273", + "\342\200\246", + "\342\200\260", + NULL, + "\302\277", + NULL, + "`", + "\302\264", + "\313\206", + "\313\234", + "\302\257", + "\313\230", + "\313\231", + "\302\250", + NULL, + "\313\232", + "\302\270", + NULL, + "\313\235", + "\313\233", + "\313\207", + "\342\200\224", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\303\206", + NULL, + "\302\252", + NULL, + NULL, + NULL, + NULL, + "\305\201", + "\303\230", + "\305\222", + "\302\272", + NULL, + NULL, + NULL, + NULL, + NULL, + "\303\246", + NULL, + NULL, + NULL, + "\304\261", + NULL, + NULL, + "\305\202", + "\303\270", + "\305\223", + "\303\237", + NULL, + NULL, + NULL, + NULL, +}; + +const char *const PDF_ENC_WINANSI[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + "\342\202\254", + NULL, + "\342\200\232", + "\306\222", + "\342\200\236", + "\342\200\246", + "\342\200\240", + "\342\200\241", + "\313\206", + "\342\200\260", + "\305\240", + "\342\200\271", + "\305\222", + NULL, + "\305\275", + NULL, + NULL, + "\342\200\230", + "\342\200\231", + "\342\200\234", + "\342\200\235", + "\342\200\242", + "\342\200\223", + "\342\200\224", + "\313\234", + "\342\204\242", + "\305\241", + "\342\200\272", + "\305\223", + NULL, + "\305\276", + "\305\270", + " ", + "\302\241", + "\302\242", + "\302\243", + "\302\244", + "\302\245", + "\302\246", + "\302\247", + "\302\250", + "\302\251", + "\302\252", + "\302\253", + "\302\254", + "-", + "\302\256", + "\302\257", + "\302\260", + "\302\261", + "\302\262", + "\302\263", + "\302\264", + "\302\265", + "\302\266", + "\302\267", + "\302\270", + "\302\271", + "\302\272", + "\302\273", + "\302\274", + "\302\275", + "\302\276", + "\302\277", + "\303\200", + "\303\201", + "\303\202", + "\303\203", + "\303\204", + "\303\205", + "\303\206", + "\303\207", + "\303\210", + "\303\211", + "\303\212", + "\303\213", + "\303\214", + "\303\215", + "\303\216", + "\303\217", + "\303\220", + "\303\221", + "\303\222", + "\303\223", + "\303\224", + "\303\225", + "\303\226", + "\303\227", + "\303\230", + "\303\231", + "\303\232", + "\303\233", + "\303\234", + "\303\235", + "\303\236", + "\303\237", + "\303\240", + "\303\241", + "\303\242", + "\303\243", + "\303\244", + "\303\245", + "\303\246", + "\303\247", + "\303\250", + "\303\251", + "\303\252", + "\303\253", + "\303\254", + "\303\255", + "\303\256", + "\303\257", + "\303\260", + "\303\261", + "\303\262", + "\303\263", + "\303\264", + "\303\265", + "\303\266", + "\303\267", + "\303\270", + "\303\271", + "\303\272", + "\303\273", + "\303\274", + "\303\275", + "\303\276", + "\303\277", +}; + +const char *const PDF_ENC_MACROMAN[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + "\303\204", + "\303\205", + "\303\207", + "\303\211", + "\303\221", + "\303\226", + "\303\234", + "\303\241", + "\303\240", + "\303\242", + "\303\244", + "\303\243", + "\303\245", + "\303\247", + "\303\251", + "\303\250", + "\303\252", + "\303\253", + "\303\255", + "\303\254", + "\303\256", + "\303\257", + "\303\261", + "\303\263", + "\303\262", + "\303\264", + "\303\266", + "\303\265", + "\303\272", + "\303\271", + "\303\273", + "\303\274", + "\342\200\240", + "\302\260", + "\302\242", + "\302\243", + "\302\247", + "\342\200\242", + "\302\266", + "\303\237", + "\302\256", + "\302\251", + "\342\204\242", + "\302\264", + "\302\250", + "\342\211\240", + "\303\206", + "\303\230", + "\342\210\236", + "\302\261", + "\342\211\244", + "\342\211\245", + "\302\245", + "\302\265", + "\342\210\202", + "\342\210\221", + "\342\210\217", + "\317\200", + "\342\210\253", + "\302\252", + "\302\272", + "\316\251", + "\303\246", + "\303\270", + "\302\277", + "\302\241", + "\302\254", + "\342\210\232", + "\306\222", + "\342\211\210", + "\342\210\206", + "\302\253", + "\302\273", + "\342\200\246", + "\302\240", + "\303\200", + "\303\203", + "\303\225", + "\305\222", + "\305\223", + "\342\200\223", + "\342\200\224", + "\342\200\234", + "\342\200\235", + "\342\200\230", + "\342\200\231", + "\303\267", + "\342\227\212", + "\303\277", + "\305\270", + "\342\201\204", + "\342\202\254", + "\342\200\271", + "\342\200\272", + "\357\254\201", + "\357\254\202", + "\342\200\241", + "\302\267", + "\342\200\232", + "\342\200\236", + "\342\200\260", + "\303\202", + "\303\212", + "\303\201", + "\303\213", + "\303\210", + "\303\215", + "\303\216", + "\303\217", + "\303\214", + "\303\223", + "\303\224", + "\357\243\277", + "\303\222", + "\303\232", + "\303\233", + "\303\231", + "\304\261", + "\313\206", + "\313\234", + "\302\257", + "\313\230", + "\313\231", + "\313\232", + "\302\270", + "\313\235", + "\313\233", + "\313\207", +}; + +const char *const PDF_ENC_PDFDOC[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + " ", "!", "\042", "#", "$", "%", "&", "'", + "(", ")", "*", "+", ",", "-", ".", "/", + "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", ":", ";", "<", "=", ">", "\077", + "@", "A", "B", "C", "D", "E", "F", "G", + "H", "I", "J", "K", "L", "M", "N", "O", + "P", "Q", "R", "S", "T", "U", "V", "W", + "X", "Y", "Z", "[", "\134", "]", "^", "_", + "`", "a", "b", "c", "d", "e", "f", "g", + "h", "i", "j", "k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", + "x", "y", "z", "{", "|", "}", "~", NULL, + "\302\200", "\302\201", "\302\202", "\302\203", "\302\204", "\302\205", "\302\206", "\302\207", + "\302\210", "\302\211", "\302\212", "\302\213", "\302\214", "\302\215", "\302\216", "\302\217", + "\302\220", "\302\221", "\302\222", "\302\223", "\302\224", "\302\225", "\302\226", "\302\227", + "\302\230", "\302\231", "\302\232", "\302\233", "\302\234", "\302\235", "\302\236", "\302\237", + "\302\240", "\302\241", "\302\242", "\302\243", "\302\244", "\302\245", "\302\246", "\302\247", + "\302\250", "\302\251", "\302\252", "\302\253", "\302\254", "\302\255", "\302\256", "\302\257", + "\302\260", "\302\261", "\302\262", "\302\263", "\302\264", "\302\265", "\302\266", "\302\267", + "\302\270", "\302\271", "\302\272", "\302\273", "\302\274", "\302\275", "\302\276", "\302\277", + "\303\200", "\303\201", "\303\202", "\303\203", "\303\204", "\303\205", "\303\206", "\303\207", + "\303\210", "\303\211", "\303\212", "\303\213", "\303\214", "\303\215", "\303\216", "\303\217", + "\303\220", "\303\221", "\303\222", "\303\223", "\303\224", "\303\225", "\303\226", "\303\227", + "\303\230", "\303\231", "\303\232", "\303\233", "\303\234", "\303\235", "\303\236", "\303\237", + "\303\240", "\303\241", "\303\242", "\303\243", "\303\244", "\303\245", "\303\246", "\303\247", + "\303\250", "\303\251", "\303\252", "\303\253", "\303\254", "\303\255", "\303\256", "\303\257", + "\303\260", "\303\261", "\303\262", "\303\263", "\303\264", "\303\265", "\303\266", "\303\267", + "\303\270", "\303\271", "\303\272", "\303\273", "\303\274", "\303\275", "\303\276", "\303\277", +}; + +const char *const PDF_ENC_SYMBOL[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\342\210\200", + "#", + "\342\210\203", + "%", + "&", + NULL, + "(", + ")", + "\342\210\227", + "+", + ",", + "\342\210\222", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "\342\211\205", + "\316\221", + "\316\222", + "\316\247", + "\316\224", + "\316\225", + "\316\246", + "\316\223", + "\316\227", + "\316\231", + "\317\221", + "\316\232", + "\316\233", + "\316\234", + "\316\235", + "\316\237", + "\316\240", + "\316\230", + "\316\241", + "\316\243", + "\316\244", + "\316\245", + "\317\202", + "\316\251", + "\316\236", + "\316\250", + "\316\226", + "[", + "\342\210\264", + "]", + "\342\212\245", + "_", + NULL, + "\316\261", + "\316\262", + "\317\207", + "\316\264", + "\316\265", + "\317\206", + "\316\263", + "\316\267", + "\316\271", + "\317\225", + "\316\272", + "\316\273", + "\302\265", + "\316\275", + "\316\277", + "\317\200", + "\316\270", + "\317\201", + "\317\203", + "\317\204", + "\317\205", + "\317\226", + "\317\211", + "\316\276", + "\317\210", + "\316\266", + "{", + "|", + "}", + "\342\210\274", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\211\244", + NULL, + "\342\210\236", + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\206\224", + "\342\206\220", + "\342\206\221", + "\342\206\222", + "\342\206\223", + "\302\260", + "\302\261", + NULL, + "\342\211\245", + "\303\227", + "\342\210\235", + "\342\210\202", + "\342\200\242", + "\303\267", + "\342\211\240", + "\342\211\241", + "\342\211\210", + "\342\200\246", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\210", + NULL, + NULL, + "\342\210\207", + NULL, + NULL, + NULL, + NULL, + "\342\210\232", + "\342\213\205", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\221", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\253", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, +}; + +const unsigned short PDF_WIDTH_HELVETICA[95] = { + 278, 278, 355, 556, 556, 889, 667, 191, 333, 333, 389, 584, 278, 333, 278, 278, 556, 556, 556, + 556, 556, 556, 556, 556, 556, 556, 278, 278, 584, 584, 584, 556, 1015, 667, 667, 722, 722, 667, + 611, 778, 722, 278, 500, 667, 556, 833, 722, 778, 667, 778, 722, 667, 611, 722, 667, 944, 667, + 667, 611, 278, 278, 278, 469, 556, 333, 556, 556, 500, 556, 556, 278, 556, 556, 222, 222, 500, + 222, 833, 556, 556, 556, 556, 333, 500, 278, 556, 500, 722, 500, 500, 500, 334, 260, 334, 584}; +const unsigned short PDF_WIDTH_TIMES[95] = { + 250, 333, 408, 500, 500, 833, 778, 180, 333, 333, 500, 564, 250, 333, 250, 278, 500, 500, 500, + 500, 500, 500, 500, 500, 500, 500, 278, 278, 564, 564, 564, 444, 921, 722, 667, 667, 722, 611, + 556, 722, 722, 333, 389, 722, 611, 889, 722, 722, 556, 722, 667, 556, 611, 722, 722, 944, 722, + 722, 611, 333, 278, 333, 469, 500, 333, 444, 500, 444, 500, 444, 333, 500, 500, 278, 278, 500, + 278, 778, 500, 500, 500, 500, 333, 389, 278, 500, 500, 722, 500, 500, 444, 480, 200, 480, 541}; + +const char *const PDF_STD14[] = {"Arial", + "Arial,Bold", + "Arial,BoldItalic", + "Arial,Italic", + "Courier", + "Courier-Bold", + "Courier-BoldOblique", + "Courier-Oblique", + "CourierNew", + "CourierNew,Bold", + "Helvetica", + "Helvetica-Bold", + "Helvetica-BoldOblique", + "Helvetica-Oblique", + "Symbol", + "Times-Bold", + "Times-BoldItalic", + "Times-Italic", + "Times-Roman", + "TimesNewRoman", + "TimesNewRoman,Bold", + "ZapfDingbats", + NULL}; diff --git a/internal/cbm/pdf/pdf_text.c b/internal/cbm/pdf/pdf_text.c new file mode 100644 index 0000000000..7ccebddce6 --- /dev/null +++ b/internal/cbm/pdf/pdf_text.c @@ -0,0 +1,1040 @@ +/* + * pdf_text.c — content streams, layout, pages and the driver of the PDF + * text-layer extractor. + * + * Text comes out in content order. A word break is a gap of more than 0.15 em + * along the writing direction (or a step back of more than 0.6 em); a line + * break is a baseline shift of more than 0.5 em or a change of direction -- + * all measured on device-space glyph geometry. Runs of Hebrew/Arabic letters, + * drawn in visual order, are reversed into logical order. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" +#include "helpers.h" + +#include +#include +#include + +enum { + PDF_OPS_MIN = 16, + PDF_OPS_ARENA = 16 * 1024, + PDF_TEXT_MIN = 4096, +}; + +#define PDF_WORD_GAP 0.15 +#define PDF_LINE_SHIFT 0.5 +#define PDF_BACKSTEP 0.6 +#define PDF_SAME_DIR 0.95 + +static const double IDENT[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}; + +/* ── Arithmetic, exactly as the prototype ────────────────────────── */ + +static void mat_mul(const double *m, const double *n, double *o) { + double a = m[0]; + double b = m[1]; + double c = m[2]; + double d = m[3]; + double e = m[4]; + double f = m[5]; + double r[6]; + r[0] = a * n[0] + b * n[2]; + r[1] = a * n[1] + b * n[3]; + r[2] = c * n[0] + d * n[2]; + r[3] = c * n[1] + d * n[3]; + r[4] = e * n[0] + f * n[2] + n[4]; + r[5] = e * n[1] + f * n[3] + n[5]; + memcpy(o, r, sizeof(r)); +} + +typedef struct { + double hi; + double lo; +} dl_t; + +static dl_t dl_fast_sum(double a, double b) { + double x = a + b; + double z = x - a; + double y = b - z; + dl_t r = {x, y}; + return r; +} + +static dl_t dl_mul(double x, double y) { + double z = x * y; + double zz = fma(x, y, -z); + dl_t r = {z, zz}; + return r; +} + +/* CPython's math.hypot for two coordinates (vector_norm): the prototype's + * value to the last bit, on every platform. */ +static double vec_norm2(double x0, double x1, double max) { + int max_e; + frexp(max, &max_e); + if (max_e < -1023) { + return DBL_MIN * vec_norm2(x0 / DBL_MIN, x1 / DBL_MIN, max / DBL_MIN); + } + double scale = ldexp(1.0, -max_e); + double csum = 1.0; + double frac1 = 0.0; + double frac2 = 0.0; + double v[2] = {x0, x1}; + for (int i = 0; i < 2; i++) { + double x = v[i] * scale; + dl_t pr = dl_mul(x, x); + dl_t sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + } + double h = sqrt(csum - 1.0 + (frac1 + frac2)); + dl_t pr = dl_mul(-h, h); + dl_t sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + double x = csum - 1.0 + (frac1 + frac2); + h += x / (2.0 * h); + return h / scale; +} + +static double py_hypot(double a, double b) { + double x0 = fabs(a); + double x1 = fabs(b); + double max = 0.0; + bool nan = false; + if (isnan(x0)) { + nan = true; + } else if (x0 > max) { + max = x0; + } + if (isnan(x1)) { + nan = true; + } else if (x1 > max) { + max = x1; + } + if (isinf(max)) { + return max; + } + if (nan) { + return NAN; + } + if (max == 0.0) { + return max; + } + return vec_norm2(x0, x1, max); +} + +/* ── Page text ───────────────────────────────────────────────────── */ + +typedef struct { + char *p; + size_t n; + size_t cap; + bool fail; + bool has_last; + double lx; + double ly; + double ldx; + double ldy; + double lsize; + int glyphs; + int unmapped; +} pdf_out_t; + +static void out_put(pdf_out_t *o, const char *s, size_t n) { + if (o->fail || !n) { + return; + } + if (o->n + n + 1 > o->cap) { + size_t ncap = o->cap ? o->cap : PDF_TEXT_MIN; + while (ncap < o->n + n + 1) { + ncap *= 2; + } + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, o->p, ncap); + if (!g) { + o->fail = true; + return; + } + o->p = g; + o->cap = ncap; + } + memcpy(o->p + o->n, s, n); + o->n += n; +} + +static void emit(pdf_out_t *o, const char *u, double ox, double oy, double ex, double ey, double dx, + double dy, double size) { + if (o->has_last) { + if (dx * o->ldx + dy * o->ldy < PDF_SAME_DIR) { + out_put(o, "\n", 1); + } else { + double vx = ox - o->lx; + double vy = oy - o->ly; + double along = vx * o->ldx + vy * o->ldy; + double perp = vy * o->ldx - vx * o->ldy; + double L = size > o->lsize ? size : o->lsize; + if (L <= 0) { + L = 1e-6; + } + if (fabs(perp) > PDF_LINE_SHIFT * L) { + out_put(o, "\n", 1); + } else if (along > PDF_WORD_GAP * L || along < -PDF_BACKSTEP * L) { + out_put(o, " ", 1); + } + } + } + out_put(o, u, strlen(u)); + o->has_last = true; + o->lx = ex; + o->ly = ey; + o->ldx = dx; + o->ldy = dy; + o->lsize = size; +} + +/* One UTF-8 sequence at s[i] (C0 80 is the NUL marker): its code point and length. */ +static uint32_t u8_at(const char *s, size_t n, size_t i, size_t *len) { + unsigned char c = (unsigned char)s[i]; + if (c < 0x80) { + *len = 1; + return c; + } + size_t k = c >= 0xF0 ? 4 : c >= 0xE0 ? 3 : 2; + if (i + k > n) { + *len = 1; + return c; + } + uint32_t cp = c & (k == 2 ? 0x1FU : k == 3 ? 0x0FU : 0x07U); + for (size_t j = 1; j < k; j++) { + cp = (cp << 6) | ((unsigned char)s[i + j] & 0x3FU); + } + *len = k; + return cp; +} + +static bool is_rtl(uint32_t cp) { + return (cp >= 0x0590 && cp <= 0x05FF) || (cp >= 0x0600 && cp <= 0x06FF) || + (cp >= 0x0700 && cp <= 0x074F) || (cp >= 0x0750 && cp <= 0x077F) || + (cp >= 0x08A0 && cp <= 0x08FF) || (cp >= 0xFB1D && cp <= 0xFDFF) || + (cp >= 0xFE70 && cp <= 0xFEFF); +} + +/* The prototype's PageText.text(): [ \t ]+ -> " ", " ?\n ?" -> "\n", + * RTL runs reversed, spaces stripped at both ends; then the NUL markers go. */ +static char *page_text(pdf_doc_t *d, pdf_out_t *o, CBMArena *dst, size_t *out_len) { + const char *s = o->p ? o->p : ""; + size_t n = o->n; + char *a = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n + 1); + char *b = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n + 1); + if (!a || !b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, a); + cbm_free(CBM_MEM_CLASS_EXTRACT, b); + d->nomem = true; + return NULL; + } + /* 1: runs of space, tab, no-break space */ + size_t k = 0; + for (size_t i = 0; i < n;) { + bool sp = s[i] == ' ' || s[i] == '\t' || + (i + 1 < n && (unsigned char)s[i] == 0xC2 && (unsigned char)s[i + 1] == 0xA0); + if (!sp) { + a[k++] = s[i++]; + continue; + } + while (i < n && + (s[i] == ' ' || s[i] == '\t' || + (i + 1 < n && (unsigned char)s[i] == 0xC2 && (unsigned char)s[i + 1] == 0xA0))) { + i += s[i] == ' ' || s[i] == '\t' ? 1 : 2; + } + a[k++] = ' '; + } + /* 2: one space either side of a newline */ + size_t m = 0; + for (size_t i = 0; i < k;) { + if (a[i] == ' ' && i + 1 < k && a[i + 1] == '\n') { + b[m++] = '\n'; + i += 2; + if (i < k && a[i] == ' ') { + i++; + } + } else if (a[i] == '\n') { + b[m++] = '\n'; + i++; + if (i < k && a[i] == ' ') { + i++; + } + } else { + b[m++] = a[i++]; + } + } + /* 3: RTL runs ([RTL]+( [RTL]+)*) reversed by code point */ + k = 0; + for (size_t i = 0; i < m;) { + size_t l; + uint32_t cp = u8_at(b, m, i, &l); + if (!is_rtl(cp)) { + memcpy(a + k, b + i, l); + k += l; + i += l; + continue; + } + size_t j = i; + for (;;) { + size_t l2; + while (j < m && is_rtl(u8_at(b, m, j, &l2))) { + j += l2; + } + if (j + 1 < m && b[j] == ' ' && is_rtl(u8_at(b, m, j + 1, &l2))) { + j += 1; + continue; + } + break; + } + /* write the run's sequences in reverse order */ + size_t end = j; + size_t w = k + (end - i); + for (size_t q = i; q < end;) { + size_t l3; + u8_at(b, m, q, &l3); + w -= l3; + memcpy(a + w, b + q, l3); + q += l3; + } + k += end - i; + i = end; + } + /* 4: strip spaces; 5: drop NUL markers */ + size_t lo = 0; + size_t hi = k; + while (lo < hi && a[lo] == ' ') { + lo++; + } + while (hi > lo && a[hi - 1] == ' ') { + hi--; + } + char *res = (char *)cbm_arena_alloc(dst, hi - lo + 1); + size_t r = 0; + if (res) { + for (size_t i = lo; i < hi; i++) { + if ((unsigned char)a[i] == 0xC0 && i + 1 < hi && (unsigned char)a[i + 1] == 0x80) { + i++; + continue; + } + res[r++] = a[i]; + } + res[r] = '\0'; + } else { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, a); + cbm_free(CBM_MEM_CLASS_EXTRACT, b); + *out_len = r; + return res; +} + +/* ── Interpreter ─────────────────────────────────────────────────── */ + +typedef struct { + double ctm[6]; + pdf_font_t *font; + double fs; + double tc; + double tw; + double th; + double tl; + double rise; +} pdf_gs_t; + +typedef struct { + bool is_ref; + uint64_t key; +} pdf_formkey_t; + +typedef struct { + pdf_doc_t *d; + pdf_out_t out; /* the page being read */ + pdf_formkey_t forms[PDF_MAX_FORM_DEPTH + 2]; + int nforms; + pdf_glyph_out_t *gbuf; + size_t gcap; +} pdf_interp_t; + +static double onum(const pdf_val_t *v) { + return (v && v->kind == PV_NUM) ? v->num : 0.0; +} + +static void show(pdf_interp_t *it, const pdf_gs_t *gs, double *tm, const pdf_val_t *str) { + pdf_font_t *font = gs->font; + if (!font) { + return; + } + pdf_doc_t *d = it->d; + pdf_out_t *out = &it->out; + if (str->n > it->gcap) { + size_t ncap = str->n * 2; + pdf_glyph_out_t *g = + (pdf_glyph_out_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, it->gbuf, ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + it->gbuf = g; + it->gcap = ncap; + } + size_t ng = 0; + pdf_font_decode(d, font, str->s, str->n, it->gbuf, &ng); + double fs = gs->fs; + double tc = gs->tc; + double tw = gs->tw; + double th = gs->th; + double rise = gs->rise; + const double *ctm = gs->ctm; + double m[6]; + mat_mul(tm, ctm, m); + double a = m[0]; + double b = m[1]; + double size = fabs(fs) * py_hypot(m[2], m[3]) * pdf_font_em(font); + double nrm = py_hypot(a, b); + double dx = 1.0; + double dy = 0.0; + if (nrm > 0) { + dx = a / nrm; + dy = b / nrm; + } + if (th < 0) { + dx = -dx; + dy = -dy; + } + double e = tm[4]; + double f = tm[5]; + double ta = tm[0]; + double tb = tm[1]; + for (size_t i = 0; i < ng; i++) { + const pdf_glyph_out_t *g = &it->gbuf[i]; + double adv = (g->width * fs + tc + (g->space ? tw : 0.0)) * th; + out->glyphs++; + const char *u = g->utf8; + if (!u) { + out->unmapped++; + u = "\xEF\xBF\xBD"; + } + if (u[0]) { + /* the glyph's origin (0, rise) in text space, in device space */ + double ux = rise * tm[2] + e; + double uy = rise * tm[3] + f; + double ox = ux * ctm[0] + uy * ctm[2] + ctm[4]; + double oy = ux * ctm[1] + uy * ctm[3] + ctm[5]; + double ex = ox + adv * a; /* where the glyph ends */ + double ey = oy + adv * b; + emit(out, u, ox, oy, ex, ey, dx, dy, size); + } + e += adv * ta; + f += adv * tb; + } + tm[4] = e; + tm[5] = f; +} + +static bool run(pdf_interp_t *it, const unsigned char *data, size_t len, pdf_val_t *resources, + const double *ctm, int depth); + +static bool form_seen(const pdf_interp_t *it, pdf_formkey_t k) { + for (int i = 0; i < it->nforms; i++) { + if (it->forms[i].is_ref == k.is_ref && it->forms[i].key == k.key) { + return true; + } + } + return false; +} + +/* false: the form's content could not be read to its end */ +static bool do_xobject(pdf_interp_t *it, pdf_val_t *res, const pdf_val_t *name, const pdf_gs_t *gs, + int depth) { + pdf_doc_t *d = it->d; + pdf_val_t *xd = pdf_dget(d, res, "XObject"); + if (!xd || xd->kind != PV_DICT) { + return true; + } + pdf_val_t *ref = pdf_dfind_n(xd, name->s, name->n); + if (ref && ref->kind == PV_NULL) { + ref = NULL; + } + pdf_val_t *xo = pdf_resolve(d, ref); + if (!xo || xo->kind != PV_STREAM) { + return true; + } + pdf_val_t *sub = pdf_dget_raw(xo->stream->dict, "Subtype"); + if (!pdf_is_name(sub, "Form")) { + return true; + } + pdf_formkey_t key = {false, (uint64_t)(uintptr_t)xo}; + if (ref && ref->kind == PV_REF) { + key.is_ref = true; + key.key = (uint64_t)ref->ref_num; + } + if (form_seen(it, key)) { + return true; + } + size_t dlen = 0; + const unsigned char *data = pdf_stream_decode(d, xo->stream, &dlen); + if (!data || !dlen) { + return true; + } + pdf_val_t *mat = pdf_dget(d, xo->stream->dict, "Matrix"); + double m[6]; + memcpy(m, IDENT, sizeof(m)); + if (mat && mat->kind == PV_ARR && mat->count == 6) { + for (int i = 0; i < 6; i++) { + m[i] = onum(pdf_resolve(d, mat->items[i])); + } + } + pdf_val_t *fres = pdf_dget_raw(xo->stream->dict, "Resources"); + if (!fres) { + fres = res; + } + double ctm[6]; + mat_mul(m, gs->ctm, ctm); + if (it->nforms >= (int)(sizeof(it->forms) / sizeof(it->forms[0]))) { + return true; + } + it->forms[it->nforms++] = key; + bool ok = run(it, data, dlen, fres, ctm, depth + 1); + it->nforms--; + return ok; +} + +static bool kw_is(const pdf_tok_t *t, const char *s) { + return pdf_kw_is(t, s); +} + +static bool path_paint(const pdf_tok_t *t) { + return kw_is(t, "S") || kw_is(t, "s") || kw_is(t, "f") || kw_is(t, "F") || kw_is(t, "f*") || + kw_is(t, "B") || kw_is(t, "B*") || kw_is(t, "b") || kw_is(t, "b*") || kw_is(t, "sh"); +} + +/* "[ws]EI" followed by whitespace or the end, from p: the position after EI. */ +static size_t skip_inline_image(const unsigned char *data, size_t len, size_t from) { + for (size_t i = from; i + 3 <= len; i++) { + unsigned char c = data[i]; + bool ws = c == 0 || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' '; + if (!ws || data[i + 1] != 'E' || data[i + 2] != 'I') { + continue; + } + size_t e = i + 3; + if (e == len) { + return e; + } + unsigned char x = data[e]; + if (x == 0 || x == '\t' || x == '\n' || x == '\f' || x == '\r' || x == ' ') { + return e; + } + } + return len; +} + +typedef struct { + pdf_val_t **v; + int n; + int cap; +} ops_t; + +static bool ops_push(pdf_doc_t *d, ops_t *o, pdf_val_t *x) { + if (o->n == o->cap) { + int ncap = o->cap ? o->cap * 2 : PDF_OPS_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)o->v, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + o->v = g; + o->cap = ncap; + } + o->v[o->n++] = x; + return true; +} + +static void tlm_next_line(const pdf_gs_t *gs, double *tm, double *tlm) { + double e = -gs->tl * tlm[2] + tlm[4]; + double f = -gs->tl * tlm[3] + tlm[5]; + tlm[4] = e; + tlm[5] = f; + memcpy(tm, tlm, 6 * sizeof(double)); +} + +static bool run(pdf_interp_t *it, const unsigned char *data, size_t len, pdf_val_t *resources, + const double *ctm, int depth) { + if (depth > PDF_MAX_FORM_DEPTH) { + return true; + } + pdf_doc_t *d = it->d; + pdf_val_t *res = pdf_resolve(d, resources); + if (res && res->kind != PV_DICT) { + res = NULL; + } + pdf_gs_t gs; + memset(&gs, 0, sizeof(gs)); + memcpy(gs.ctm, ctm, sizeof(gs.ctm)); + gs.th = 1.0; + pdf_gs_t *stack = NULL; + int nstack = 0; + int stack_cap = 0; + double tm[6]; + double tlm[6]; + memcpy(tm, IDENT, sizeof(tm)); + memcpy(tlm, IDENT, sizeof(tlm)); + CBMArena opa; + cbm_arena_init_lazy(&opa, PDF_OPS_ARENA); + ops_t ops = {0}; + size_t pos = 0; + bool ok = true; + while (pos < len && !d->nomem) { + d->cur = &opa; + pdf_tok_t t; + pdf_lex(d, data, len, &pos, &t); + if (t.kind == PT_EOF) { + break; + } + pdf_val_t *v = NULL; + bool operand = true; + if (t.kind == PT_NUM || t.kind == PT_STR || t.kind == PT_NAME) { + v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = t.kind == PT_NUM ? PV_NUM : t.kind == PT_STR ? PV_STR : PV_NAME; + v->is_int = t.is_int; + v->num = t.num; + v->inum = t.inum; + v->s = t.s; + v->n = t.n; + } + } else if (t.kind == PT_AS) { + v = pdf_parse_content_array(d, data, len, &pos, 0); + if (!v) { + ok = false; + break; + } + } else if (t.kind == PT_DS) { + v = pdf_parse_value(d, data, len, &pos, &t, 0); + if (!v) { + ok = false; + break; + } + } else if (t.kind != PT_KW) { + continue; + } else { + operand = false; + } + d->cur = d->a; + if (operand) { + if (v) { + ops_push(d, &ops, v); + } + continue; + } + int n = ops.n; + pdf_val_t **o = ops.v; + if (kw_is(&t, "Tj") || kw_is(&t, "'") || kw_is(&t, "\"") || kw_is(&t, "TJ")) { + bool tj = kw_is(&t, "TJ"); + if (kw_is(&t, "\"") && n >= 3) { + gs.tw = onum(o[n - 3]); + gs.tc = onum(o[n - 2]); + } + if (!kw_is(&t, "Tj") && !tj) { + tlm_next_line(&gs, tm, tlm); + } + if (n == 0) { + /* nothing shown */ + } else if (tj) { + const pdf_val_t *arr = o[n - 1]; + for (int i = 0; arr && arr->kind == PV_ARR && i < arr->count; i++) { + const pdf_val_t *item = arr->items[i]; + if (item->kind == PV_STR) { + show(it, &gs, tm, item); + } else if (item->kind == PV_NUM) { + double tx = -item->num / 1000.0 * gs.fs * gs.th; + double e = tm[4] + tx * tm[0]; + double f = tm[5] + tx * tm[1]; + tm[4] = e; + tm[5] = f; + } + } + } else if (o[n - 1]->kind == PV_STR) { + show(it, &gs, tm, o[n - 1]); + } + } else if (kw_is(&t, "Td") || kw_is(&t, "TD")) { + if (n >= 2) { + double tx = onum(o[n - 2]); + double ty = onum(o[n - 1]); + if (kw_is(&t, "TD")) { + gs.tl = -ty; + } + double e = tx * tlm[0] + ty * tlm[2] + tlm[4]; + double f = tx * tlm[1] + ty * tlm[3] + tlm[5]; + tlm[4] = e; + tlm[5] = f; + memcpy(tm, tlm, sizeof(tm)); + } + } else if (kw_is(&t, "Tm")) { + if (n >= 6) { + for (int i = 0; i < 6; i++) { + tm[i] = onum(o[n - 6 + i]); + } + memcpy(tlm, tm, sizeof(tlm)); + } + } else if (kw_is(&t, "T*")) { + tlm_next_line(&gs, tm, tlm); + } else if (kw_is(&t, "Tf")) { + if (n >= 2) { + gs.fs = onum(o[n - 1]); + gs.font = o[n - 2]->kind == PV_NAME ? pdf_font_for(d, res, o[n - 2]) : NULL; + } + } else if (kw_is(&t, "BT")) { + memcpy(tm, IDENT, sizeof(tm)); + memcpy(tlm, IDENT, sizeof(tlm)); + } else if (kw_is(&t, "Tc")) { + if (n) { + gs.tc = onum(o[n - 1]); + } + } else if (kw_is(&t, "Tw")) { + if (n) { + gs.tw = onum(o[n - 1]); + } + } else if (kw_is(&t, "Tz")) { + if (n) { + gs.th = onum(o[n - 1]) / 100.0; + } + } else if (kw_is(&t, "TL")) { + if (n) { + gs.tl = onum(o[n - 1]); + } + } else if (kw_is(&t, "Ts")) { + if (n) { + gs.rise = onum(o[n - 1]); + } + } else if (kw_is(&t, "q")) { + if (nstack == stack_cap) { + int ncap = stack_cap ? stack_cap * 2 : PDF_OPS_MIN; + pdf_gs_t *g = (pdf_gs_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, stack, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + break; + } + stack = g; + stack_cap = ncap; + } + stack[nstack++] = gs; + } else if (kw_is(&t, "Q")) { + if (nstack) { + gs = stack[--nstack]; + } + } else if (kw_is(&t, "cm")) { + if (n >= 6) { + double m[6]; + for (int i = 0; i < 6; i++) { + m[i] = onum(o[n - 6 + i]); + } + mat_mul(m, gs.ctm, gs.ctm); + } + } else if (kw_is(&t, "Do")) { + if (n && o[n - 1]->kind == PV_NAME && !do_xobject(it, res, o[n - 1], &gs, depth)) { + d->counts->op_errors++; + } + } else if (kw_is(&t, "BI")) { + const unsigned char *id = + pos < len ? (const unsigned char *)cbm_memmem(data + pos, len - pos, "ID", 2) + : NULL; + if (!id) { + break; + } + pos = skip_inline_image(data, len, (size_t)(id - data) + 2); + } else if (path_paint(&t)) { + /* vector content: not text */ + } + ops.n = 0; + cbm_arena_rewind(&opa); + } + d->cur = d->a; + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)ops.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, stack); + cbm_arena_destroy(&opa); + return ok; +} + +/* ── Pages ───────────────────────────────────────────────────────── */ + +typedef struct { + pdf_val_t *node; + pdf_val_t *res; + pdf_val_t *box; +} pdf_pageref_t; + +typedef struct { + pdf_val_t *ref; + pdf_val_t *inh; + pdf_val_t *box; + int depth; +} pdf_walk_t; + +typedef struct { + pdf_pageref_t *v; + int n; + int cap; +} pagelist_t; + +static bool pl_push(pdf_doc_t *d, pagelist_t *l, pdf_val_t *node, pdf_val_t *res, pdf_val_t *box) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_OPS_MIN; + pdf_pageref_t *g = + (pdf_pageref_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].node = node; + l->v[l->n].res = res; + l->v[l->n].box = box; + l->n++; + return true; +} + +/* dict.get(key, dflt): a present key wins even when its value is null. */ +static pdf_val_t *dget_or(const pdf_val_t *dict, const char *key, pdf_val_t *dflt) { + return pdf_dhas(dict, key) ? pdf_dfind(dict, key) : dflt; +} + +static int i64cmp(const void *x, const void *y) { + int64_t a = *(const int64_t *)x; + int64_t b = *(const int64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +static void collect_pages(pdf_doc_t *d, pagelist_t *pl) { + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(d->trailer, "Root")); + if (root && root->kind == PV_DICT) { + pdf_map_t seen_ref = {0}; + pdf_map_t seen_ptr = {0}; + pdf_walk_t *stack = NULL; + int ns = 0; + int cap = 0; +#define WALK_PUSH(R, I, B, D) \ + do { \ + if (ns == cap) { \ + int nc = cap ? cap * 2 : PDF_OPS_MIN; \ + pdf_walk_t *g = \ + (pdf_walk_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, stack, (size_t)nc * sizeof(*g)); \ + if (!g) { \ + d->nomem = true; \ + goto done; \ + } \ + stack = g; \ + cap = nc; \ + } \ + stack[ns].ref = (R); \ + stack[ns].inh = (I); \ + stack[ns].box = (B); \ + stack[ns].depth = (D); \ + ns++; \ + } while (0) + WALK_PUSH(pdf_dget_raw(root, "Pages"), NULL, NULL, 0); + while (ns && !d->nomem) { + pdf_walk_t w = stack[--ns]; + bool seen; + if (w.ref && w.ref->kind == PV_REF) { + seen = pdf_map_get(&seen_ref, (uint64_t)w.ref->ref_num) >= 0; + if (!seen && !pdf_map_put(&seen_ref, (uint64_t)w.ref->ref_num, 1)) { + d->nomem = true; + } + } else { + uint64_t key = (uint64_t)(uintptr_t)w.ref; + seen = pdf_map_get(&seen_ptr, key) >= 0; + if (!seen && !pdf_map_put(&seen_ptr, key, 1)) { + d->nomem = true; + } + } + if (seen || w.depth > PDF_MAX_PAGE_DEPTH) { + continue; + } + pdf_val_t *node = pdf_resolve(d, w.ref); + if (!node || node->kind != PV_DICT) { + continue; + } + pdf_val_t *res = dget_or(node, "Resources", w.inh); + pdf_val_t *box = dget_or(node, "CropBox", dget_or(node, "MediaBox", w.box)); + pdf_val_t *kids = pdf_resolve(d, pdf_dget_raw(node, "Kids")); + pdf_val_t *typ = pdf_dget_raw(node, "Type"); + bool is_arr = kids && kids->kind == PV_ARR; + if (pdf_is_name(typ, "Pages") || (!pdf_is_name(typ, "Page") && is_arr)) { + for (int i = is_arr ? kids->count - 1 : -1; i >= 0; i--) { + WALK_PUSH(kids->items[i], res, box, w.depth + 1); + } + } else { + pl_push(d, pl, node, res, box); + } + } +#undef WALK_PUSH + done: + cbm_free(CBM_MEM_CLASS_EXTRACT, stack); + pdf_map_free(&seen_ref); + pdf_map_free(&seen_ptr); + } + if (pl->n || d->nomem) { + return; + } + int n = d->xref.n; + int64_t *nums = + (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1) * sizeof(int64_t)); + if (!nums) { + d->nomem = true; + return; + } + for (int i = 0; i < n; i++) { + nums[i] = d->xref.v[i].num; + } + qsort(nums, (size_t)n, sizeof(int64_t), i64cmp); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (o && o->kind == PV_DICT && pdf_is_name(pdf_dget_raw(o, "Type"), "Page")) { + pl_push(d, pl, o, pdf_dget_raw(o, "Resources"), + dget_or(o, "CropBox", pdf_dget_raw(o, "MediaBox"))); + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); +} + +static void page_extract(pdf_doc_t *d, pdf_interp_t *it, const pdf_pageref_t *pg, CBMArena *dst, + cbm_pdf_page_t *out) { + pdf_out_t *o = &it->out; + memset(o, 0, sizeof(*o)); + it->nforms = 0; + pdf_val_t *cont = pdf_dget(d, pg->node, "Contents"); + pdf_val_t *single[1] = {cont}; + pdf_val_t **parts = single; + int nparts = cont ? 1 : 0; + if (cont && cont->kind == PV_ARR) { + parts = cont->items; + nparts = cont->count; + } + /* the parts' decoded data joined by "\n" */ + const unsigned char *one = NULL; + size_t one_len = 0; + unsigned char *joined = NULL; + size_t jn = 0; + int nchunks = 0; + for (int i = 0; i < nparts && !d->nomem; i++) { + pdf_val_t *p = pdf_resolve(d, parts[i]); + if (!p || p->kind != PV_STREAM) { + continue; + } + size_t dl = 0; + const unsigned char *dd = pdf_stream_decode(d, p->stream, &dl); + if (!dd || !dl) { + continue; + } + if (nchunks == 0) { + one = dd; + one_len = dl; + } else { + size_t add = (nchunks == 1 ? one_len : 0) + 1 + dl; + unsigned char *g = + (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, joined, jn + add); + if (!g) { + d->nomem = true; + break; + } + joined = g; + if (nchunks == 1) { + memcpy(joined, one, one_len); + jn = one_len; + } + joined[jn++] = '\n'; + memcpy(joined + jn, dd, dl); + jn += dl; + } + nchunks++; + } + const unsigned char *content = nchunks > 1 ? joined : one; + size_t clen = nchunks > 1 ? jn : one_len; + if (!run(it, content ? content : (const unsigned char *)"", content ? clen : 0, pg->res, IDENT, + 0)) { + d->counts->page_errors++; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, joined); + size_t tl = 0; + out->text = page_text(d, o, dst, &tl); + out->len = tl; + out->glyphs = o->glyphs; + out->unmapped = o->unmapped; + if (!out->text) { + out->text = ""; + out->len = 0; + } + if (o->fail) { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, o->p); +} + +bool cbm_pdf_extract(const unsigned char *data, size_t len, CBMArena *out, cbm_pdf_result_t *res) { + memset(res, 0, sizeof(*res)); + const unsigned char *h = data ? (const unsigned char *)cbm_memmem(data, len, "%PDF-", 5) : NULL; + if (!h || (size_t)(h - data) > PDF_HEADER_WINDOW) { + res->status = CBM_PDF_NOT_PDF; + return true; + } + CBMArena arena; + cbm_arena_init(&arena); + pdf_doc_t d; + memset(&d, 0, sizeof(d)); + d.data = data; + d.len = len; + d.a = &arena; + d.cur = &arena; + d.counts = &res->counts; + pdf_doc_open(&d); + pagelist_t pl = {0}; + if (!d.nomem) { + pdf_val_t *enc = pdf_dget_raw(d.trailer, "Encrypt"); + pdf_val_t *encd = pdf_resolve(&d, enc); + if (encd && encd->kind == PV_DICT) { + res->status = CBM_PDF_ENCRYPTED; + pdf_doc_close(&d); + cbm_arena_destroy(&arena); + return true; + } + collect_pages(&d, &pl); + } + if (!d.nomem && pl.n) { + res->pages = (cbm_pdf_page_t *)cbm_arena_calloc(out, (size_t)pl.n * sizeof(*res->pages)); + if (!res->pages) { + d.nomem = true; + } + } + pdf_interp_t it; + memset(&it, 0, sizeof(it)); + it.d = &d; + for (int i = 0; i < pl.n && !d.nomem; i++) { + page_extract(&d, &it, &pl.v[i], out, &res->pages[i]); + res->npages = i + 1; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, it.gbuf); + cbm_free(CBM_MEM_CLASS_EXTRACT, pl.v); + bool nomem = d.nomem; + pdf_doc_close(&d); + cbm_arena_destroy(&arena); + if (nomem) { + res->status = CBM_PDF_NOMEM; + res->pages = NULL; + res->npages = 0; + return false; + } + res->status = res->npages ? CBM_PDF_OK : CBM_PDF_PARSE_FAILURE; + return true; +} diff --git a/internal/cbm/pdf/pdf_unicode.c b/internal/cbm/pdf/pdf_unicode.c new file mode 100644 index 0000000000..4ae6af9b48 --- /dev/null +++ b/internal/cbm/pdf/pdf_unicode.c @@ -0,0 +1,3124 @@ +/* + * pdf_unicode.c -- Unicode 16.0.0 facts for the PDF mention scanner (generated). + * + * The field test's scanner ran on CPython's str and re; these tables are what + * CPython 3.14.6 answers: NFKC of each code point, the canonical compositions, + * str.isspace, the re module's \w (str.isalnum or '_'), str.isalpha, and the + * one-character str.isupper / str.islower / titlecase. Generated by + * gen_pdf_unicode.py; do not edit by hand. + */ +#include "pdf_unicode.h" + +const pdf_nfkc_t PDF_NFKC[] = { + {0xA0, 0, 1}, {0xA8, 1, 2}, {0xAA, 3, 1}, {0xAF, 4, 2}, + {0xB2, 6, 1}, {0xB3, 7, 1}, {0xB4, 8, 2}, {0xB5, 10, 1}, + {0xB8, 11, 2}, {0xB9, 13, 1}, {0xBA, 14, 1}, {0xBC, 15, 3}, + {0xBD, 18, 3}, {0xBE, 21, 3}, {0x132, 24, 2}, {0x133, 26, 2}, + {0x13F, 28, 2}, {0x140, 30, 2}, {0x149, 32, 2}, {0x17F, 34, 1}, + {0x1C4, 35, 2}, {0x1C5, 37, 2}, {0x1C6, 39, 2}, {0x1C7, 41, 2}, + {0x1C8, 43, 2}, {0x1C9, 45, 2}, {0x1CA, 47, 2}, {0x1CB, 49, 2}, + {0x1CC, 51, 2}, {0x1F1, 53, 2}, {0x1F2, 55, 2}, {0x1F3, 57, 2}, + {0x2B0, 59, 1}, {0x2B1, 60, 1}, {0x2B2, 61, 1}, {0x2B3, 62, 1}, + {0x2B4, 63, 1}, {0x2B5, 64, 1}, {0x2B6, 65, 1}, {0x2B7, 66, 1}, + {0x2B8, 67, 1}, {0x2D8, 68, 2}, {0x2D9, 70, 2}, {0x2DA, 72, 2}, + {0x2DB, 74, 2}, {0x2DC, 76, 2}, {0x2DD, 78, 2}, {0x2E0, 80, 1}, + {0x2E1, 81, 1}, {0x2E2, 82, 1}, {0x2E3, 83, 1}, {0x2E4, 84, 1}, + {0x340, 85, 1}, {0x341, 86, 1}, {0x343, 87, 1}, {0x344, 88, 2}, + {0x374, 90, 1}, {0x37A, 91, 2}, {0x37E, 93, 1}, {0x384, 94, 2}, + {0x385, 96, 3}, {0x387, 99, 1}, {0x3D0, 100, 1}, {0x3D1, 101, 1}, + {0x3D2, 102, 1}, {0x3D3, 103, 1}, {0x3D4, 104, 1}, {0x3D5, 105, 1}, + {0x3D6, 106, 1}, {0x3F0, 107, 1}, {0x3F1, 108, 1}, {0x3F2, 109, 1}, + {0x3F4, 110, 1}, {0x3F5, 111, 1}, {0x3F9, 112, 1}, {0x587, 113, 2}, + {0x675, 115, 2}, {0x676, 117, 2}, {0x677, 119, 2}, {0x678, 121, 2}, + {0x958, 123, 2}, {0x959, 125, 2}, {0x95A, 127, 2}, {0x95B, 129, 2}, + {0x95C, 131, 2}, {0x95D, 133, 2}, {0x95E, 135, 2}, {0x95F, 137, 2}, + {0x9DC, 139, 2}, {0x9DD, 141, 2}, {0x9DF, 143, 2}, {0xA33, 145, 2}, + {0xA36, 147, 2}, {0xA59, 149, 2}, {0xA5A, 151, 2}, {0xA5B, 153, 2}, + {0xA5E, 155, 2}, {0xB5C, 157, 2}, {0xB5D, 159, 2}, {0xE33, 161, 2}, + {0xEB3, 163, 2}, {0xEDC, 165, 2}, {0xEDD, 167, 2}, {0xF0C, 169, 1}, + {0xF43, 170, 2}, {0xF4D, 172, 2}, {0xF52, 174, 2}, {0xF57, 176, 2}, + {0xF5C, 178, 2}, {0xF69, 180, 2}, {0xF73, 182, 2}, {0xF75, 184, 2}, + {0xF76, 186, 2}, {0xF77, 188, 3}, {0xF78, 191, 2}, {0xF79, 193, 3}, + {0xF81, 196, 2}, {0xF93, 198, 2}, {0xF9D, 200, 2}, {0xFA2, 202, 2}, + {0xFA7, 204, 2}, {0xFAC, 206, 2}, {0xFB9, 208, 2}, {0x10FC, 210, 1}, + {0x1D2C, 211, 1}, {0x1D2D, 212, 1}, {0x1D2E, 213, 1}, {0x1D30, 214, 1}, + {0x1D31, 215, 1}, {0x1D32, 216, 1}, {0x1D33, 217, 1}, {0x1D34, 218, 1}, + {0x1D35, 219, 1}, {0x1D36, 220, 1}, {0x1D37, 221, 1}, {0x1D38, 222, 1}, + {0x1D39, 223, 1}, {0x1D3A, 224, 1}, {0x1D3C, 225, 1}, {0x1D3D, 226, 1}, + {0x1D3E, 227, 1}, {0x1D3F, 228, 1}, {0x1D40, 229, 1}, {0x1D41, 230, 1}, + {0x1D42, 231, 1}, {0x1D43, 232, 1}, {0x1D44, 233, 1}, {0x1D45, 234, 1}, + {0x1D46, 235, 1}, {0x1D47, 236, 1}, {0x1D48, 237, 1}, {0x1D49, 238, 1}, + {0x1D4A, 239, 1}, {0x1D4B, 240, 1}, {0x1D4C, 241, 1}, {0x1D4D, 242, 1}, + {0x1D4F, 243, 1}, {0x1D50, 244, 1}, {0x1D51, 245, 1}, {0x1D52, 246, 1}, + {0x1D53, 247, 1}, {0x1D54, 248, 1}, {0x1D55, 249, 1}, {0x1D56, 250, 1}, + {0x1D57, 251, 1}, {0x1D58, 252, 1}, {0x1D59, 253, 1}, {0x1D5A, 254, 1}, + {0x1D5B, 255, 1}, {0x1D5C, 256, 1}, {0x1D5D, 257, 1}, {0x1D5E, 258, 1}, + {0x1D5F, 259, 1}, {0x1D60, 260, 1}, {0x1D61, 261, 1}, {0x1D62, 262, 1}, + {0x1D63, 263, 1}, {0x1D64, 264, 1}, {0x1D65, 265, 1}, {0x1D66, 266, 1}, + {0x1D67, 267, 1}, {0x1D68, 268, 1}, {0x1D69, 269, 1}, {0x1D6A, 270, 1}, + {0x1D78, 271, 1}, {0x1D9B, 272, 1}, {0x1D9C, 273, 1}, {0x1D9D, 274, 1}, + {0x1D9E, 275, 1}, {0x1D9F, 276, 1}, {0x1DA0, 277, 1}, {0x1DA1, 278, 1}, + {0x1DA2, 279, 1}, {0x1DA3, 280, 1}, {0x1DA4, 281, 1}, {0x1DA5, 282, 1}, + {0x1DA6, 283, 1}, {0x1DA7, 284, 1}, {0x1DA8, 285, 1}, {0x1DA9, 286, 1}, + {0x1DAA, 287, 1}, {0x1DAB, 288, 1}, {0x1DAC, 289, 1}, {0x1DAD, 290, 1}, + {0x1DAE, 291, 1}, {0x1DAF, 292, 1}, {0x1DB0, 293, 1}, {0x1DB1, 294, 1}, + {0x1DB2, 295, 1}, {0x1DB3, 296, 1}, {0x1DB4, 297, 1}, {0x1DB5, 298, 1}, + {0x1DB6, 299, 1}, {0x1DB7, 300, 1}, {0x1DB8, 301, 1}, {0x1DB9, 302, 1}, + {0x1DBA, 303, 1}, {0x1DBB, 304, 1}, {0x1DBC, 305, 1}, {0x1DBD, 306, 1}, + {0x1DBE, 307, 1}, {0x1DBF, 308, 1}, {0x1E9A, 309, 2}, {0x1E9B, 311, 1}, + {0x1F71, 312, 1}, {0x1F73, 313, 1}, {0x1F75, 314, 1}, {0x1F77, 315, 1}, + {0x1F79, 316, 1}, {0x1F7B, 317, 1}, {0x1F7D, 318, 1}, {0x1FBB, 319, 1}, + {0x1FBD, 320, 2}, {0x1FBE, 322, 1}, {0x1FBF, 323, 2}, {0x1FC0, 325, 2}, + {0x1FC1, 327, 3}, {0x1FC9, 330, 1}, {0x1FCB, 331, 1}, {0x1FCD, 332, 3}, + {0x1FCE, 335, 3}, {0x1FCF, 338, 3}, {0x1FD3, 341, 1}, {0x1FDB, 342, 1}, + {0x1FDD, 343, 3}, {0x1FDE, 346, 3}, {0x1FDF, 349, 3}, {0x1FE3, 352, 1}, + {0x1FEB, 353, 1}, {0x1FED, 354, 3}, {0x1FEE, 357, 3}, {0x1FEF, 360, 1}, + {0x1FF9, 361, 1}, {0x1FFB, 362, 1}, {0x1FFD, 363, 2}, {0x1FFE, 365, 2}, + {0x2000, 367, 1}, {0x2001, 368, 1}, {0x2002, 369, 1}, {0x2003, 370, 1}, + {0x2004, 371, 1}, {0x2005, 372, 1}, {0x2006, 373, 1}, {0x2007, 374, 1}, + {0x2008, 375, 1}, {0x2009, 376, 1}, {0x200A, 377, 1}, {0x2011, 378, 1}, + {0x2017, 379, 2}, {0x2024, 381, 1}, {0x2025, 382, 2}, {0x2026, 384, 3}, + {0x202F, 387, 1}, {0x2033, 388, 2}, {0x2034, 390, 3}, {0x2036, 393, 2}, + {0x2037, 395, 3}, {0x203C, 398, 2}, {0x203E, 400, 2}, {0x2047, 402, 2}, + {0x2048, 404, 2}, {0x2049, 406, 2}, {0x2057, 408, 4}, {0x205F, 412, 1}, + {0x2070, 413, 1}, {0x2071, 414, 1}, {0x2074, 415, 1}, {0x2075, 416, 1}, + {0x2076, 417, 1}, {0x2077, 418, 1}, {0x2078, 419, 1}, {0x2079, 420, 1}, + {0x207A, 421, 1}, {0x207B, 422, 1}, {0x207C, 423, 1}, {0x207D, 424, 1}, + {0x207E, 425, 1}, {0x207F, 426, 1}, {0x2080, 427, 1}, {0x2081, 428, 1}, + {0x2082, 429, 1}, {0x2083, 430, 1}, {0x2084, 431, 1}, {0x2085, 432, 1}, + {0x2086, 433, 1}, {0x2087, 434, 1}, {0x2088, 435, 1}, {0x2089, 436, 1}, + {0x208A, 437, 1}, {0x208B, 438, 1}, {0x208C, 439, 1}, {0x208D, 440, 1}, + {0x208E, 441, 1}, {0x2090, 442, 1}, {0x2091, 443, 1}, {0x2092, 444, 1}, + {0x2093, 445, 1}, {0x2094, 446, 1}, {0x2095, 447, 1}, {0x2096, 448, 1}, + {0x2097, 449, 1}, {0x2098, 450, 1}, {0x2099, 451, 1}, {0x209A, 452, 1}, + {0x209B, 453, 1}, {0x209C, 454, 1}, {0x20A8, 455, 2}, {0x2100, 457, 3}, + {0x2101, 460, 3}, {0x2102, 463, 1}, {0x2103, 464, 2}, {0x2105, 466, 3}, + {0x2106, 469, 3}, {0x2107, 472, 1}, {0x2109, 473, 2}, {0x210A, 475, 1}, + {0x210B, 476, 1}, {0x210C, 477, 1}, {0x210D, 478, 1}, {0x210E, 479, 1}, + {0x210F, 480, 1}, {0x2110, 481, 1}, {0x2111, 482, 1}, {0x2112, 483, 1}, + {0x2113, 484, 1}, {0x2115, 485, 1}, {0x2116, 486, 2}, {0x2119, 488, 1}, + {0x211A, 489, 1}, {0x211B, 490, 1}, {0x211C, 491, 1}, {0x211D, 492, 1}, + {0x2120, 493, 2}, {0x2121, 495, 3}, {0x2122, 498, 2}, {0x2124, 500, 1}, + {0x2126, 501, 1}, {0x2128, 502, 1}, {0x212A, 503, 1}, {0x212B, 504, 1}, + {0x212C, 505, 1}, {0x212D, 506, 1}, {0x212F, 507, 1}, {0x2130, 508, 1}, + {0x2131, 509, 1}, {0x2133, 510, 1}, {0x2134, 511, 1}, {0x2135, 512, 1}, + {0x2136, 513, 1}, {0x2137, 514, 1}, {0x2138, 515, 1}, {0x2139, 516, 1}, + {0x213B, 517, 3}, {0x213C, 520, 1}, {0x213D, 521, 1}, {0x213E, 522, 1}, + {0x213F, 523, 1}, {0x2140, 524, 1}, {0x2145, 525, 1}, {0x2146, 526, 1}, + {0x2147, 527, 1}, {0x2148, 528, 1}, {0x2149, 529, 1}, {0x2150, 530, 3}, + {0x2151, 533, 3}, {0x2152, 536, 4}, {0x2153, 540, 3}, {0x2154, 543, 3}, + {0x2155, 546, 3}, {0x2156, 549, 3}, {0x2157, 552, 3}, {0x2158, 555, 3}, + {0x2159, 558, 3}, {0x215A, 561, 3}, {0x215B, 564, 3}, {0x215C, 567, 3}, + {0x215D, 570, 3}, {0x215E, 573, 3}, {0x215F, 576, 2}, {0x2160, 578, 1}, + {0x2161, 579, 2}, {0x2162, 581, 3}, {0x2163, 584, 2}, {0x2164, 586, 1}, + {0x2165, 587, 2}, {0x2166, 589, 3}, {0x2167, 592, 4}, {0x2168, 596, 2}, + {0x2169, 598, 1}, {0x216A, 599, 2}, {0x216B, 601, 3}, {0x216C, 604, 1}, + {0x216D, 605, 1}, {0x216E, 606, 1}, {0x216F, 607, 1}, {0x2170, 608, 1}, + {0x2171, 609, 2}, {0x2172, 611, 3}, {0x2173, 614, 2}, {0x2174, 616, 1}, + {0x2175, 617, 2}, {0x2176, 619, 3}, {0x2177, 622, 4}, {0x2178, 626, 2}, + {0x2179, 628, 1}, {0x217A, 629, 2}, {0x217B, 631, 3}, {0x217C, 634, 1}, + {0x217D, 635, 1}, {0x217E, 636, 1}, {0x217F, 637, 1}, {0x2189, 638, 3}, + {0x222C, 641, 2}, {0x222D, 643, 3}, {0x222F, 646, 2}, {0x2230, 648, 3}, + {0x2329, 651, 1}, {0x232A, 652, 1}, {0x2460, 653, 1}, {0x2461, 654, 1}, + {0x2462, 655, 1}, {0x2463, 656, 1}, {0x2464, 657, 1}, {0x2465, 658, 1}, + {0x2466, 659, 1}, {0x2467, 660, 1}, {0x2468, 661, 1}, {0x2469, 662, 2}, + {0x246A, 664, 2}, {0x246B, 666, 2}, {0x246C, 668, 2}, {0x246D, 670, 2}, + {0x246E, 672, 2}, {0x246F, 674, 2}, {0x2470, 676, 2}, {0x2471, 678, 2}, + {0x2472, 680, 2}, {0x2473, 682, 2}, {0x2474, 684, 3}, {0x2475, 687, 3}, + {0x2476, 690, 3}, {0x2477, 693, 3}, {0x2478, 696, 3}, {0x2479, 699, 3}, + {0x247A, 702, 3}, {0x247B, 705, 3}, {0x247C, 708, 3}, {0x247D, 711, 4}, + {0x247E, 715, 4}, {0x247F, 719, 4}, {0x2480, 723, 4}, {0x2481, 727, 4}, + {0x2482, 731, 4}, {0x2483, 735, 4}, {0x2484, 739, 4}, {0x2485, 743, 4}, + {0x2486, 747, 4}, {0x2487, 751, 4}, {0x2488, 755, 2}, {0x2489, 757, 2}, + {0x248A, 759, 2}, {0x248B, 761, 2}, {0x248C, 763, 2}, {0x248D, 765, 2}, + {0x248E, 767, 2}, {0x248F, 769, 2}, {0x2490, 771, 2}, {0x2491, 773, 3}, + {0x2492, 776, 3}, {0x2493, 779, 3}, {0x2494, 782, 3}, {0x2495, 785, 3}, + {0x2496, 788, 3}, {0x2497, 791, 3}, {0x2498, 794, 3}, {0x2499, 797, 3}, + {0x249A, 800, 3}, {0x249B, 803, 3}, {0x249C, 806, 3}, {0x249D, 809, 3}, + {0x249E, 812, 3}, {0x249F, 815, 3}, {0x24A0, 818, 3}, {0x24A1, 821, 3}, + {0x24A2, 824, 3}, {0x24A3, 827, 3}, {0x24A4, 830, 3}, {0x24A5, 833, 3}, + {0x24A6, 836, 3}, {0x24A7, 839, 3}, {0x24A8, 842, 3}, {0x24A9, 845, 3}, + {0x24AA, 848, 3}, {0x24AB, 851, 3}, {0x24AC, 854, 3}, {0x24AD, 857, 3}, + {0x24AE, 860, 3}, {0x24AF, 863, 3}, {0x24B0, 866, 3}, {0x24B1, 869, 3}, + {0x24B2, 872, 3}, {0x24B3, 875, 3}, {0x24B4, 878, 3}, {0x24B5, 881, 3}, + {0x24B6, 884, 1}, {0x24B7, 885, 1}, {0x24B8, 886, 1}, {0x24B9, 887, 1}, + {0x24BA, 888, 1}, {0x24BB, 889, 1}, {0x24BC, 890, 1}, {0x24BD, 891, 1}, + {0x24BE, 892, 1}, {0x24BF, 893, 1}, {0x24C0, 894, 1}, {0x24C1, 895, 1}, + {0x24C2, 896, 1}, {0x24C3, 897, 1}, {0x24C4, 898, 1}, {0x24C5, 899, 1}, + {0x24C6, 900, 1}, {0x24C7, 901, 1}, {0x24C8, 902, 1}, {0x24C9, 903, 1}, + {0x24CA, 904, 1}, {0x24CB, 905, 1}, {0x24CC, 906, 1}, {0x24CD, 907, 1}, + {0x24CE, 908, 1}, {0x24CF, 909, 1}, {0x24D0, 910, 1}, {0x24D1, 911, 1}, + {0x24D2, 912, 1}, {0x24D3, 913, 1}, {0x24D4, 914, 1}, {0x24D5, 915, 1}, + {0x24D6, 916, 1}, {0x24D7, 917, 1}, {0x24D8, 918, 1}, {0x24D9, 919, 1}, + {0x24DA, 920, 1}, {0x24DB, 921, 1}, {0x24DC, 922, 1}, {0x24DD, 923, 1}, + {0x24DE, 924, 1}, {0x24DF, 925, 1}, {0x24E0, 926, 1}, {0x24E1, 927, 1}, + {0x24E2, 928, 1}, {0x24E3, 929, 1}, {0x24E4, 930, 1}, {0x24E5, 931, 1}, + {0x24E6, 932, 1}, {0x24E7, 933, 1}, {0x24E8, 934, 1}, {0x24E9, 935, 1}, + {0x24EA, 936, 1}, {0x2A0C, 937, 4}, {0x2A74, 941, 3}, {0x2A75, 944, 2}, + {0x2A76, 946, 3}, {0x2ADC, 949, 2}, {0x2C7C, 951, 1}, {0x2C7D, 952, 1}, + {0x2D6F, 953, 1}, {0x2E9F, 954, 1}, {0x2EF3, 955, 1}, {0x2F00, 956, 1}, + {0x2F01, 957, 1}, {0x2F02, 958, 1}, {0x2F03, 959, 1}, {0x2F04, 960, 1}, + {0x2F05, 961, 1}, {0x2F06, 962, 1}, {0x2F07, 963, 1}, {0x2F08, 964, 1}, + {0x2F09, 965, 1}, {0x2F0A, 966, 1}, {0x2F0B, 967, 1}, {0x2F0C, 968, 1}, + {0x2F0D, 969, 1}, {0x2F0E, 970, 1}, {0x2F0F, 971, 1}, {0x2F10, 972, 1}, + {0x2F11, 973, 1}, {0x2F12, 974, 1}, {0x2F13, 975, 1}, {0x2F14, 976, 1}, + {0x2F15, 977, 1}, {0x2F16, 978, 1}, {0x2F17, 979, 1}, {0x2F18, 980, 1}, + {0x2F19, 981, 1}, {0x2F1A, 982, 1}, {0x2F1B, 983, 1}, {0x2F1C, 984, 1}, + {0x2F1D, 985, 1}, {0x2F1E, 986, 1}, {0x2F1F, 987, 1}, {0x2F20, 988, 1}, + {0x2F21, 989, 1}, {0x2F22, 990, 1}, {0x2F23, 991, 1}, {0x2F24, 992, 1}, + {0x2F25, 993, 1}, {0x2F26, 994, 1}, {0x2F27, 995, 1}, {0x2F28, 996, 1}, + {0x2F29, 997, 1}, {0x2F2A, 998, 1}, {0x2F2B, 999, 1}, {0x2F2C, 1000, 1}, + {0x2F2D, 1001, 1}, {0x2F2E, 1002, 1}, {0x2F2F, 1003, 1}, {0x2F30, 1004, 1}, + {0x2F31, 1005, 1}, {0x2F32, 1006, 1}, {0x2F33, 1007, 1}, {0x2F34, 1008, 1}, + {0x2F35, 1009, 1}, {0x2F36, 1010, 1}, {0x2F37, 1011, 1}, {0x2F38, 1012, 1}, + {0x2F39, 1013, 1}, {0x2F3A, 1014, 1}, {0x2F3B, 1015, 1}, {0x2F3C, 1016, 1}, + {0x2F3D, 1017, 1}, {0x2F3E, 1018, 1}, {0x2F3F, 1019, 1}, {0x2F40, 1020, 1}, + {0x2F41, 1021, 1}, {0x2F42, 1022, 1}, {0x2F43, 1023, 1}, {0x2F44, 1024, 1}, + {0x2F45, 1025, 1}, {0x2F46, 1026, 1}, {0x2F47, 1027, 1}, {0x2F48, 1028, 1}, + {0x2F49, 1029, 1}, {0x2F4A, 1030, 1}, {0x2F4B, 1031, 1}, {0x2F4C, 1032, 1}, + {0x2F4D, 1033, 1}, {0x2F4E, 1034, 1}, {0x2F4F, 1035, 1}, {0x2F50, 1036, 1}, + {0x2F51, 1037, 1}, {0x2F52, 1038, 1}, {0x2F53, 1039, 1}, {0x2F54, 1040, 1}, + {0x2F55, 1041, 1}, {0x2F56, 1042, 1}, {0x2F57, 1043, 1}, {0x2F58, 1044, 1}, + {0x2F59, 1045, 1}, {0x2F5A, 1046, 1}, {0x2F5B, 1047, 1}, {0x2F5C, 1048, 1}, + {0x2F5D, 1049, 1}, {0x2F5E, 1050, 1}, {0x2F5F, 1051, 1}, {0x2F60, 1052, 1}, + {0x2F61, 1053, 1}, {0x2F62, 1054, 1}, {0x2F63, 1055, 1}, {0x2F64, 1056, 1}, + {0x2F65, 1057, 1}, {0x2F66, 1058, 1}, {0x2F67, 1059, 1}, {0x2F68, 1060, 1}, + {0x2F69, 1061, 1}, {0x2F6A, 1062, 1}, {0x2F6B, 1063, 1}, {0x2F6C, 1064, 1}, + {0x2F6D, 1065, 1}, {0x2F6E, 1066, 1}, {0x2F6F, 1067, 1}, {0x2F70, 1068, 1}, + {0x2F71, 1069, 1}, {0x2F72, 1070, 1}, {0x2F73, 1071, 1}, {0x2F74, 1072, 1}, + {0x2F75, 1073, 1}, {0x2F76, 1074, 1}, {0x2F77, 1075, 1}, {0x2F78, 1076, 1}, + {0x2F79, 1077, 1}, {0x2F7A, 1078, 1}, {0x2F7B, 1079, 1}, {0x2F7C, 1080, 1}, + {0x2F7D, 1081, 1}, {0x2F7E, 1082, 1}, {0x2F7F, 1083, 1}, {0x2F80, 1084, 1}, + {0x2F81, 1085, 1}, {0x2F82, 1086, 1}, {0x2F83, 1087, 1}, {0x2F84, 1088, 1}, + {0x2F85, 1089, 1}, {0x2F86, 1090, 1}, {0x2F87, 1091, 1}, {0x2F88, 1092, 1}, + {0x2F89, 1093, 1}, {0x2F8A, 1094, 1}, {0x2F8B, 1095, 1}, {0x2F8C, 1096, 1}, + {0x2F8D, 1097, 1}, {0x2F8E, 1098, 1}, {0x2F8F, 1099, 1}, {0x2F90, 1100, 1}, + {0x2F91, 1101, 1}, {0x2F92, 1102, 1}, {0x2F93, 1103, 1}, {0x2F94, 1104, 1}, + {0x2F95, 1105, 1}, {0x2F96, 1106, 1}, {0x2F97, 1107, 1}, {0x2F98, 1108, 1}, + {0x2F99, 1109, 1}, {0x2F9A, 1110, 1}, {0x2F9B, 1111, 1}, {0x2F9C, 1112, 1}, + {0x2F9D, 1113, 1}, {0x2F9E, 1114, 1}, {0x2F9F, 1115, 1}, {0x2FA0, 1116, 1}, + {0x2FA1, 1117, 1}, {0x2FA2, 1118, 1}, {0x2FA3, 1119, 1}, {0x2FA4, 1120, 1}, + {0x2FA5, 1121, 1}, {0x2FA6, 1122, 1}, {0x2FA7, 1123, 1}, {0x2FA8, 1124, 1}, + {0x2FA9, 1125, 1}, {0x2FAA, 1126, 1}, {0x2FAB, 1127, 1}, {0x2FAC, 1128, 1}, + {0x2FAD, 1129, 1}, {0x2FAE, 1130, 1}, {0x2FAF, 1131, 1}, {0x2FB0, 1132, 1}, + {0x2FB1, 1133, 1}, {0x2FB2, 1134, 1}, {0x2FB3, 1135, 1}, {0x2FB4, 1136, 1}, + {0x2FB5, 1137, 1}, {0x2FB6, 1138, 1}, {0x2FB7, 1139, 1}, {0x2FB8, 1140, 1}, + {0x2FB9, 1141, 1}, {0x2FBA, 1142, 1}, {0x2FBB, 1143, 1}, {0x2FBC, 1144, 1}, + {0x2FBD, 1145, 1}, {0x2FBE, 1146, 1}, {0x2FBF, 1147, 1}, {0x2FC0, 1148, 1}, + {0x2FC1, 1149, 1}, {0x2FC2, 1150, 1}, {0x2FC3, 1151, 1}, {0x2FC4, 1152, 1}, + {0x2FC5, 1153, 1}, {0x2FC6, 1154, 1}, {0x2FC7, 1155, 1}, {0x2FC8, 1156, 1}, + {0x2FC9, 1157, 1}, {0x2FCA, 1158, 1}, {0x2FCB, 1159, 1}, {0x2FCC, 1160, 1}, + {0x2FCD, 1161, 1}, {0x2FCE, 1162, 1}, {0x2FCF, 1163, 1}, {0x2FD0, 1164, 1}, + {0x2FD1, 1165, 1}, {0x2FD2, 1166, 1}, {0x2FD3, 1167, 1}, {0x2FD4, 1168, 1}, + {0x2FD5, 1169, 1}, {0x3000, 1170, 1}, {0x3036, 1171, 1}, {0x3038, 1172, 1}, + {0x3039, 1173, 1}, {0x303A, 1174, 1}, {0x309B, 1175, 2}, {0x309C, 1177, 2}, + {0x309F, 1179, 2}, {0x30FF, 1181, 2}, {0x3131, 1183, 1}, {0x3132, 1184, 1}, + {0x3133, 1185, 1}, {0x3134, 1186, 1}, {0x3135, 1187, 1}, {0x3136, 1188, 1}, + {0x3137, 1189, 1}, {0x3138, 1190, 1}, {0x3139, 1191, 1}, {0x313A, 1192, 1}, + {0x313B, 1193, 1}, {0x313C, 1194, 1}, {0x313D, 1195, 1}, {0x313E, 1196, 1}, + {0x313F, 1197, 1}, {0x3140, 1198, 1}, {0x3141, 1199, 1}, {0x3142, 1200, 1}, + {0x3143, 1201, 1}, {0x3144, 1202, 1}, {0x3145, 1203, 1}, {0x3146, 1204, 1}, + {0x3147, 1205, 1}, {0x3148, 1206, 1}, {0x3149, 1207, 1}, {0x314A, 1208, 1}, + {0x314B, 1209, 1}, {0x314C, 1210, 1}, {0x314D, 1211, 1}, {0x314E, 1212, 1}, + {0x314F, 1213, 1}, {0x3150, 1214, 1}, {0x3151, 1215, 1}, {0x3152, 1216, 1}, + {0x3153, 1217, 1}, {0x3154, 1218, 1}, {0x3155, 1219, 1}, {0x3156, 1220, 1}, + {0x3157, 1221, 1}, {0x3158, 1222, 1}, {0x3159, 1223, 1}, {0x315A, 1224, 1}, + {0x315B, 1225, 1}, {0x315C, 1226, 1}, {0x315D, 1227, 1}, {0x315E, 1228, 1}, + {0x315F, 1229, 1}, {0x3160, 1230, 1}, {0x3161, 1231, 1}, {0x3162, 1232, 1}, + {0x3163, 1233, 1}, {0x3164, 1234, 1}, {0x3165, 1235, 1}, {0x3166, 1236, 1}, + {0x3167, 1237, 1}, {0x3168, 1238, 1}, {0x3169, 1239, 1}, {0x316A, 1240, 1}, + {0x316B, 1241, 1}, {0x316C, 1242, 1}, {0x316D, 1243, 1}, {0x316E, 1244, 1}, + {0x316F, 1245, 1}, {0x3170, 1246, 1}, {0x3171, 1247, 1}, {0x3172, 1248, 1}, + {0x3173, 1249, 1}, {0x3174, 1250, 1}, {0x3175, 1251, 1}, {0x3176, 1252, 1}, + {0x3177, 1253, 1}, {0x3178, 1254, 1}, {0x3179, 1255, 1}, {0x317A, 1256, 1}, + {0x317B, 1257, 1}, {0x317C, 1258, 1}, {0x317D, 1259, 1}, {0x317E, 1260, 1}, + {0x317F, 1261, 1}, {0x3180, 1262, 1}, {0x3181, 1263, 1}, {0x3182, 1264, 1}, + {0x3183, 1265, 1}, {0x3184, 1266, 1}, {0x3185, 1267, 1}, {0x3186, 1268, 1}, + {0x3187, 1269, 1}, {0x3188, 1270, 1}, {0x3189, 1271, 1}, {0x318A, 1272, 1}, + {0x318B, 1273, 1}, {0x318C, 1274, 1}, {0x318D, 1275, 1}, {0x318E, 1276, 1}, + {0x3192, 1277, 1}, {0x3193, 1278, 1}, {0x3194, 1279, 1}, {0x3195, 1280, 1}, + {0x3196, 1281, 1}, {0x3197, 1282, 1}, {0x3198, 1283, 1}, {0x3199, 1284, 1}, + {0x319A, 1285, 1}, {0x319B, 1286, 1}, {0x319C, 1287, 1}, {0x319D, 1288, 1}, + {0x319E, 1289, 1}, {0x319F, 1290, 1}, {0x3200, 1291, 3}, {0x3201, 1294, 3}, + {0x3202, 1297, 3}, {0x3203, 1300, 3}, {0x3204, 1303, 3}, {0x3205, 1306, 3}, + {0x3206, 1309, 3}, {0x3207, 1312, 3}, {0x3208, 1315, 3}, {0x3209, 1318, 3}, + {0x320A, 1321, 3}, {0x320B, 1324, 3}, {0x320C, 1327, 3}, {0x320D, 1330, 3}, + {0x320E, 1333, 3}, {0x320F, 1336, 3}, {0x3210, 1339, 3}, {0x3211, 1342, 3}, + {0x3212, 1345, 3}, {0x3213, 1348, 3}, {0x3214, 1351, 3}, {0x3215, 1354, 3}, + {0x3216, 1357, 3}, {0x3217, 1360, 3}, {0x3218, 1363, 3}, {0x3219, 1366, 3}, + {0x321A, 1369, 3}, {0x321B, 1372, 3}, {0x321C, 1375, 3}, {0x321D, 1378, 4}, + {0x321E, 1382, 4}, {0x3220, 1386, 3}, {0x3221, 1389, 3}, {0x3222, 1392, 3}, + {0x3223, 1395, 3}, {0x3224, 1398, 3}, {0x3225, 1401, 3}, {0x3226, 1404, 3}, + {0x3227, 1407, 3}, {0x3228, 1410, 3}, {0x3229, 1413, 3}, {0x322A, 1416, 3}, + {0x322B, 1419, 3}, {0x322C, 1422, 3}, {0x322D, 1425, 3}, {0x322E, 1428, 3}, + {0x322F, 1431, 3}, {0x3230, 1434, 3}, {0x3231, 1437, 3}, {0x3232, 1440, 3}, + {0x3233, 1443, 3}, {0x3234, 1446, 3}, {0x3235, 1449, 3}, {0x3236, 1452, 3}, + {0x3237, 1455, 3}, {0x3238, 1458, 3}, {0x3239, 1461, 3}, {0x323A, 1464, 3}, + {0x323B, 1467, 3}, {0x323C, 1470, 3}, {0x323D, 1473, 3}, {0x323E, 1476, 3}, + {0x323F, 1479, 3}, {0x3240, 1482, 3}, {0x3241, 1485, 3}, {0x3242, 1488, 3}, + {0x3243, 1491, 3}, {0x3244, 1494, 1}, {0x3245, 1495, 1}, {0x3246, 1496, 1}, + {0x3247, 1497, 1}, {0x3250, 1498, 3}, {0x3251, 1501, 2}, {0x3252, 1503, 2}, + {0x3253, 1505, 2}, {0x3254, 1507, 2}, {0x3255, 1509, 2}, {0x3256, 1511, 2}, + {0x3257, 1513, 2}, {0x3258, 1515, 2}, {0x3259, 1517, 2}, {0x325A, 1519, 2}, + {0x325B, 1521, 2}, {0x325C, 1523, 2}, {0x325D, 1525, 2}, {0x325E, 1527, 2}, + {0x325F, 1529, 2}, {0x3260, 1531, 1}, {0x3261, 1532, 1}, {0x3262, 1533, 1}, + {0x3263, 1534, 1}, {0x3264, 1535, 1}, {0x3265, 1536, 1}, {0x3266, 1537, 1}, + {0x3267, 1538, 1}, {0x3268, 1539, 1}, {0x3269, 1540, 1}, {0x326A, 1541, 1}, + {0x326B, 1542, 1}, {0x326C, 1543, 1}, {0x326D, 1544, 1}, {0x326E, 1545, 1}, + {0x326F, 1546, 1}, {0x3270, 1547, 1}, {0x3271, 1548, 1}, {0x3272, 1549, 1}, + {0x3273, 1550, 1}, {0x3274, 1551, 1}, {0x3275, 1552, 1}, {0x3276, 1553, 1}, + {0x3277, 1554, 1}, {0x3278, 1555, 1}, {0x3279, 1556, 1}, {0x327A, 1557, 1}, + {0x327B, 1558, 1}, {0x327C, 1559, 2}, {0x327D, 1561, 2}, {0x327E, 1563, 1}, + {0x3280, 1564, 1}, {0x3281, 1565, 1}, {0x3282, 1566, 1}, {0x3283, 1567, 1}, + {0x3284, 1568, 1}, {0x3285, 1569, 1}, {0x3286, 1570, 1}, {0x3287, 1571, 1}, + {0x3288, 1572, 1}, {0x3289, 1573, 1}, {0x328A, 1574, 1}, {0x328B, 1575, 1}, + {0x328C, 1576, 1}, {0x328D, 1577, 1}, {0x328E, 1578, 1}, {0x328F, 1579, 1}, + {0x3290, 1580, 1}, {0x3291, 1581, 1}, {0x3292, 1582, 1}, {0x3293, 1583, 1}, + {0x3294, 1584, 1}, {0x3295, 1585, 1}, {0x3296, 1586, 1}, {0x3297, 1587, 1}, + {0x3298, 1588, 1}, {0x3299, 1589, 1}, {0x329A, 1590, 1}, {0x329B, 1591, 1}, + {0x329C, 1592, 1}, {0x329D, 1593, 1}, {0x329E, 1594, 1}, {0x329F, 1595, 1}, + {0x32A0, 1596, 1}, {0x32A1, 1597, 1}, {0x32A2, 1598, 1}, {0x32A3, 1599, 1}, + {0x32A4, 1600, 1}, {0x32A5, 1601, 1}, {0x32A6, 1602, 1}, {0x32A7, 1603, 1}, + {0x32A8, 1604, 1}, {0x32A9, 1605, 1}, {0x32AA, 1606, 1}, {0x32AB, 1607, 1}, + {0x32AC, 1608, 1}, {0x32AD, 1609, 1}, {0x32AE, 1610, 1}, {0x32AF, 1611, 1}, + {0x32B0, 1612, 1}, {0x32B1, 1613, 2}, {0x32B2, 1615, 2}, {0x32B3, 1617, 2}, + {0x32B4, 1619, 2}, {0x32B5, 1621, 2}, {0x32B6, 1623, 2}, {0x32B7, 1625, 2}, + {0x32B8, 1627, 2}, {0x32B9, 1629, 2}, {0x32BA, 1631, 2}, {0x32BB, 1633, 2}, + {0x32BC, 1635, 2}, {0x32BD, 1637, 2}, {0x32BE, 1639, 2}, {0x32BF, 1641, 2}, + {0x32C0, 1643, 2}, {0x32C1, 1645, 2}, {0x32C2, 1647, 2}, {0x32C3, 1649, 2}, + {0x32C4, 1651, 2}, {0x32C5, 1653, 2}, {0x32C6, 1655, 2}, {0x32C7, 1657, 2}, + {0x32C8, 1659, 2}, {0x32C9, 1661, 3}, {0x32CA, 1664, 3}, {0x32CB, 1667, 3}, + {0x32CC, 1670, 2}, {0x32CD, 1672, 3}, {0x32CE, 1675, 2}, {0x32CF, 1677, 3}, + {0x32D0, 1680, 1}, {0x32D1, 1681, 1}, {0x32D2, 1682, 1}, {0x32D3, 1683, 1}, + {0x32D4, 1684, 1}, {0x32D5, 1685, 1}, {0x32D6, 1686, 1}, {0x32D7, 1687, 1}, + {0x32D8, 1688, 1}, {0x32D9, 1689, 1}, {0x32DA, 1690, 1}, {0x32DB, 1691, 1}, + {0x32DC, 1692, 1}, {0x32DD, 1693, 1}, {0x32DE, 1694, 1}, {0x32DF, 1695, 1}, + {0x32E0, 1696, 1}, {0x32E1, 1697, 1}, {0x32E2, 1698, 1}, {0x32E3, 1699, 1}, + {0x32E4, 1700, 1}, {0x32E5, 1701, 1}, {0x32E6, 1702, 1}, {0x32E7, 1703, 1}, + {0x32E8, 1704, 1}, {0x32E9, 1705, 1}, {0x32EA, 1706, 1}, {0x32EB, 1707, 1}, + {0x32EC, 1708, 1}, {0x32ED, 1709, 1}, {0x32EE, 1710, 1}, {0x32EF, 1711, 1}, + {0x32F0, 1712, 1}, {0x32F1, 1713, 1}, {0x32F2, 1714, 1}, {0x32F3, 1715, 1}, + {0x32F4, 1716, 1}, {0x32F5, 1717, 1}, {0x32F6, 1718, 1}, {0x32F7, 1719, 1}, + {0x32F8, 1720, 1}, {0x32F9, 1721, 1}, {0x32FA, 1722, 1}, {0x32FB, 1723, 1}, + {0x32FC, 1724, 1}, {0x32FD, 1725, 1}, {0x32FE, 1726, 1}, {0x32FF, 1727, 2}, + {0x3300, 1729, 4}, {0x3301, 1733, 4}, {0x3302, 1737, 4}, {0x3303, 1741, 3}, + {0x3304, 1744, 4}, {0x3305, 1748, 3}, {0x3306, 1751, 3}, {0x3307, 1754, 5}, + {0x3308, 1759, 4}, {0x3309, 1763, 3}, {0x330A, 1766, 3}, {0x330B, 1769, 3}, + {0x330C, 1772, 4}, {0x330D, 1776, 4}, {0x330E, 1780, 3}, {0x330F, 1783, 3}, + {0x3310, 1786, 2}, {0x3311, 1788, 3}, {0x3312, 1791, 4}, {0x3313, 1795, 4}, + {0x3314, 1799, 2}, {0x3315, 1801, 5}, {0x3316, 1806, 6}, {0x3317, 1812, 5}, + {0x3318, 1817, 3}, {0x3319, 1820, 5}, {0x331A, 1825, 5}, {0x331B, 1830, 4}, + {0x331C, 1834, 3}, {0x331D, 1837, 3}, {0x331E, 1840, 3}, {0x331F, 1843, 4}, + {0x3320, 1847, 5}, {0x3321, 1852, 4}, {0x3322, 1856, 3}, {0x3323, 1859, 3}, + {0x3324, 1862, 3}, {0x3325, 1865, 2}, {0x3326, 1867, 2}, {0x3327, 1869, 2}, + {0x3328, 1871, 2}, {0x3329, 1873, 3}, {0x332A, 1876, 3}, {0x332B, 1879, 5}, + {0x332C, 1884, 3}, {0x332D, 1887, 4}, {0x332E, 1891, 5}, {0x332F, 1896, 3}, + {0x3330, 1899, 2}, {0x3331, 1901, 2}, {0x3332, 1903, 5}, {0x3333, 1908, 4}, + {0x3334, 1912, 5}, {0x3335, 1917, 3}, {0x3336, 1920, 5}, {0x3337, 1925, 2}, + {0x3338, 1927, 3}, {0x3339, 1930, 3}, {0x333A, 1933, 3}, {0x333B, 1936, 3}, + {0x333C, 1939, 3}, {0x333D, 1942, 4}, {0x333E, 1946, 3}, {0x333F, 1949, 2}, + {0x3340, 1951, 3}, {0x3341, 1954, 3}, {0x3342, 1957, 3}, {0x3343, 1960, 4}, + {0x3344, 1964, 3}, {0x3345, 1967, 3}, {0x3346, 1970, 3}, {0x3347, 1973, 5}, + {0x3348, 1978, 4}, {0x3349, 1982, 2}, {0x334A, 1984, 5}, {0x334B, 1989, 2}, + {0x334C, 1991, 4}, {0x334D, 1995, 4}, {0x334E, 1999, 3}, {0x334F, 2002, 3}, + {0x3350, 2005, 3}, {0x3351, 2008, 4}, {0x3352, 2012, 2}, {0x3353, 2014, 3}, + {0x3354, 2017, 4}, {0x3355, 2021, 2}, {0x3356, 2023, 5}, {0x3357, 2028, 3}, + {0x3358, 2031, 2}, {0x3359, 2033, 2}, {0x335A, 2035, 2}, {0x335B, 2037, 2}, + {0x335C, 2039, 2}, {0x335D, 2041, 2}, {0x335E, 2043, 2}, {0x335F, 2045, 2}, + {0x3360, 2047, 2}, {0x3361, 2049, 2}, {0x3362, 2051, 3}, {0x3363, 2054, 3}, + {0x3364, 2057, 3}, {0x3365, 2060, 3}, {0x3366, 2063, 3}, {0x3367, 2066, 3}, + {0x3368, 2069, 3}, {0x3369, 2072, 3}, {0x336A, 2075, 3}, {0x336B, 2078, 3}, + {0x336C, 2081, 3}, {0x336D, 2084, 3}, {0x336E, 2087, 3}, {0x336F, 2090, 3}, + {0x3370, 2093, 3}, {0x3371, 2096, 3}, {0x3372, 2099, 2}, {0x3373, 2101, 2}, + {0x3374, 2103, 3}, {0x3375, 2106, 2}, {0x3376, 2108, 2}, {0x3377, 2110, 2}, + {0x3378, 2112, 3}, {0x3379, 2115, 3}, {0x337A, 2118, 2}, {0x337B, 2120, 2}, + {0x337C, 2122, 2}, {0x337D, 2124, 2}, {0x337E, 2126, 2}, {0x337F, 2128, 4}, + {0x3380, 2132, 2}, {0x3381, 2134, 2}, {0x3382, 2136, 2}, {0x3383, 2138, 2}, + {0x3384, 2140, 2}, {0x3385, 2142, 2}, {0x3386, 2144, 2}, {0x3387, 2146, 2}, + {0x3388, 2148, 3}, {0x3389, 2151, 4}, {0x338A, 2155, 2}, {0x338B, 2157, 2}, + {0x338C, 2159, 2}, {0x338D, 2161, 2}, {0x338E, 2163, 2}, {0x338F, 2165, 2}, + {0x3390, 2167, 2}, {0x3391, 2169, 3}, {0x3392, 2172, 3}, {0x3393, 2175, 3}, + {0x3394, 2178, 3}, {0x3395, 2181, 2}, {0x3396, 2183, 2}, {0x3397, 2185, 2}, + {0x3398, 2187, 2}, {0x3399, 2189, 2}, {0x339A, 2191, 2}, {0x339B, 2193, 2}, + {0x339C, 2195, 2}, {0x339D, 2197, 2}, {0x339E, 2199, 2}, {0x339F, 2201, 3}, + {0x33A0, 2204, 3}, {0x33A1, 2207, 2}, {0x33A2, 2209, 3}, {0x33A3, 2212, 3}, + {0x33A4, 2215, 3}, {0x33A5, 2218, 2}, {0x33A6, 2220, 3}, {0x33A7, 2223, 3}, + {0x33A8, 2226, 4}, {0x33A9, 2230, 2}, {0x33AA, 2232, 3}, {0x33AB, 2235, 3}, + {0x33AC, 2238, 3}, {0x33AD, 2241, 3}, {0x33AE, 2244, 5}, {0x33AF, 2249, 6}, + {0x33B0, 2255, 2}, {0x33B1, 2257, 2}, {0x33B2, 2259, 2}, {0x33B3, 2261, 2}, + {0x33B4, 2263, 2}, {0x33B5, 2265, 2}, {0x33B6, 2267, 2}, {0x33B7, 2269, 2}, + {0x33B8, 2271, 2}, {0x33B9, 2273, 2}, {0x33BA, 2275, 2}, {0x33BB, 2277, 2}, + {0x33BC, 2279, 2}, {0x33BD, 2281, 2}, {0x33BE, 2283, 2}, {0x33BF, 2285, 2}, + {0x33C0, 2287, 2}, {0x33C1, 2289, 2}, {0x33C2, 2291, 4}, {0x33C3, 2295, 2}, + {0x33C4, 2297, 2}, {0x33C5, 2299, 2}, {0x33C6, 2301, 4}, {0x33C7, 2305, 3}, + {0x33C8, 2308, 2}, {0x33C9, 2310, 2}, {0x33CA, 2312, 2}, {0x33CB, 2314, 2}, + {0x33CC, 2316, 2}, {0x33CD, 2318, 2}, {0x33CE, 2320, 2}, {0x33CF, 2322, 2}, + {0x33D0, 2324, 2}, {0x33D1, 2326, 2}, {0x33D2, 2328, 3}, {0x33D3, 2331, 2}, + {0x33D4, 2333, 2}, {0x33D5, 2335, 3}, {0x33D6, 2338, 3}, {0x33D7, 2341, 2}, + {0x33D8, 2343, 4}, {0x33D9, 2347, 3}, {0x33DA, 2350, 2}, {0x33DB, 2352, 2}, + {0x33DC, 2354, 2}, {0x33DD, 2356, 2}, {0x33DE, 2358, 3}, {0x33DF, 2361, 3}, + {0x33E0, 2364, 2}, {0x33E1, 2366, 2}, {0x33E2, 2368, 2}, {0x33E3, 2370, 2}, + {0x33E4, 2372, 2}, {0x33E5, 2374, 2}, {0x33E6, 2376, 2}, {0x33E7, 2378, 2}, + {0x33E8, 2380, 2}, {0x33E9, 2382, 3}, {0x33EA, 2385, 3}, {0x33EB, 2388, 3}, + {0x33EC, 2391, 3}, {0x33ED, 2394, 3}, {0x33EE, 2397, 3}, {0x33EF, 2400, 3}, + {0x33F0, 2403, 3}, {0x33F1, 2406, 3}, {0x33F2, 2409, 3}, {0x33F3, 2412, 3}, + {0x33F4, 2415, 3}, {0x33F5, 2418, 3}, {0x33F6, 2421, 3}, {0x33F7, 2424, 3}, + {0x33F8, 2427, 3}, {0x33F9, 2430, 3}, {0x33FA, 2433, 3}, {0x33FB, 2436, 3}, + {0x33FC, 2439, 3}, {0x33FD, 2442, 3}, {0x33FE, 2445, 3}, {0x33FF, 2448, 3}, + {0xA69C, 2451, 1}, {0xA69D, 2452, 1}, {0xA770, 2453, 1}, {0xA7F2, 2454, 1}, + {0xA7F3, 2455, 1}, {0xA7F4, 2456, 1}, {0xA7F8, 2457, 1}, {0xA7F9, 2458, 1}, + {0xAB5C, 2459, 1}, {0xAB5D, 2460, 1}, {0xAB5E, 2461, 1}, {0xAB5F, 2462, 1}, + {0xAB69, 2463, 1}, {0xF900, 2464, 1}, {0xF901, 2465, 1}, {0xF902, 2466, 1}, + {0xF903, 2467, 1}, {0xF904, 2468, 1}, {0xF905, 2469, 1}, {0xF906, 2470, 1}, + {0xF907, 2471, 1}, {0xF908, 2472, 1}, {0xF909, 2473, 1}, {0xF90A, 2474, 1}, + {0xF90B, 2475, 1}, {0xF90C, 2476, 1}, {0xF90D, 2477, 1}, {0xF90E, 2478, 1}, + {0xF90F, 2479, 1}, {0xF910, 2480, 1}, {0xF911, 2481, 1}, {0xF912, 2482, 1}, + {0xF913, 2483, 1}, {0xF914, 2484, 1}, {0xF915, 2485, 1}, {0xF916, 2486, 1}, + {0xF917, 2487, 1}, {0xF918, 2488, 1}, {0xF919, 2489, 1}, {0xF91A, 2490, 1}, + {0xF91B, 2491, 1}, {0xF91C, 2492, 1}, {0xF91D, 2493, 1}, {0xF91E, 2494, 1}, + {0xF91F, 2495, 1}, {0xF920, 2496, 1}, {0xF921, 2497, 1}, {0xF922, 2498, 1}, + {0xF923, 2499, 1}, {0xF924, 2500, 1}, {0xF925, 2501, 1}, {0xF926, 2502, 1}, + {0xF927, 2503, 1}, {0xF928, 2504, 1}, {0xF929, 2505, 1}, {0xF92A, 2506, 1}, + {0xF92B, 2507, 1}, {0xF92C, 2508, 1}, {0xF92D, 2509, 1}, {0xF92E, 2510, 1}, + {0xF92F, 2511, 1}, {0xF930, 2512, 1}, {0xF931, 2513, 1}, {0xF932, 2514, 1}, + {0xF933, 2515, 1}, {0xF934, 2516, 1}, {0xF935, 2517, 1}, {0xF936, 2518, 1}, + {0xF937, 2519, 1}, {0xF938, 2520, 1}, {0xF939, 2521, 1}, {0xF93A, 2522, 1}, + {0xF93B, 2523, 1}, {0xF93C, 2524, 1}, {0xF93D, 2525, 1}, {0xF93E, 2526, 1}, + {0xF93F, 2527, 1}, {0xF940, 2528, 1}, {0xF941, 2529, 1}, {0xF942, 2530, 1}, + {0xF943, 2531, 1}, {0xF944, 2532, 1}, {0xF945, 2533, 1}, {0xF946, 2534, 1}, + {0xF947, 2535, 1}, {0xF948, 2536, 1}, {0xF949, 2537, 1}, {0xF94A, 2538, 1}, + {0xF94B, 2539, 1}, {0xF94C, 2540, 1}, {0xF94D, 2541, 1}, {0xF94E, 2542, 1}, + {0xF94F, 2543, 1}, {0xF950, 2544, 1}, {0xF951, 2545, 1}, {0xF952, 2546, 1}, + {0xF953, 2547, 1}, {0xF954, 2548, 1}, {0xF955, 2549, 1}, {0xF956, 2550, 1}, + {0xF957, 2551, 1}, {0xF958, 2552, 1}, {0xF959, 2553, 1}, {0xF95A, 2554, 1}, + {0xF95B, 2555, 1}, {0xF95C, 2556, 1}, {0xF95D, 2557, 1}, {0xF95E, 2558, 1}, + {0xF95F, 2559, 1}, {0xF960, 2560, 1}, {0xF961, 2561, 1}, {0xF962, 2562, 1}, + {0xF963, 2563, 1}, {0xF964, 2564, 1}, {0xF965, 2565, 1}, {0xF966, 2566, 1}, + {0xF967, 2567, 1}, {0xF968, 2568, 1}, {0xF969, 2569, 1}, {0xF96A, 2570, 1}, + {0xF96B, 2571, 1}, {0xF96C, 2572, 1}, {0xF96D, 2573, 1}, {0xF96E, 2574, 1}, + {0xF96F, 2575, 1}, {0xF970, 2576, 1}, {0xF971, 2577, 1}, {0xF972, 2578, 1}, + {0xF973, 2579, 1}, {0xF974, 2580, 1}, {0xF975, 2581, 1}, {0xF976, 2582, 1}, + {0xF977, 2583, 1}, {0xF978, 2584, 1}, {0xF979, 2585, 1}, {0xF97A, 2586, 1}, + {0xF97B, 2587, 1}, {0xF97C, 2588, 1}, {0xF97D, 2589, 1}, {0xF97E, 2590, 1}, + {0xF97F, 2591, 1}, {0xF980, 2592, 1}, {0xF981, 2593, 1}, {0xF982, 2594, 1}, + {0xF983, 2595, 1}, {0xF984, 2596, 1}, {0xF985, 2597, 1}, {0xF986, 2598, 1}, + {0xF987, 2599, 1}, {0xF988, 2600, 1}, {0xF989, 2601, 1}, {0xF98A, 2602, 1}, + {0xF98B, 2603, 1}, {0xF98C, 2604, 1}, {0xF98D, 2605, 1}, {0xF98E, 2606, 1}, + {0xF98F, 2607, 1}, {0xF990, 2608, 1}, {0xF991, 2609, 1}, {0xF992, 2610, 1}, + {0xF993, 2611, 1}, {0xF994, 2612, 1}, {0xF995, 2613, 1}, {0xF996, 2614, 1}, + {0xF997, 2615, 1}, {0xF998, 2616, 1}, {0xF999, 2617, 1}, {0xF99A, 2618, 1}, + {0xF99B, 2619, 1}, {0xF99C, 2620, 1}, {0xF99D, 2621, 1}, {0xF99E, 2622, 1}, + {0xF99F, 2623, 1}, {0xF9A0, 2624, 1}, {0xF9A1, 2625, 1}, {0xF9A2, 2626, 1}, + {0xF9A3, 2627, 1}, {0xF9A4, 2628, 1}, {0xF9A5, 2629, 1}, {0xF9A6, 2630, 1}, + {0xF9A7, 2631, 1}, {0xF9A8, 2632, 1}, {0xF9A9, 2633, 1}, {0xF9AA, 2634, 1}, + {0xF9AB, 2635, 1}, {0xF9AC, 2636, 1}, {0xF9AD, 2637, 1}, {0xF9AE, 2638, 1}, + {0xF9AF, 2639, 1}, {0xF9B0, 2640, 1}, {0xF9B1, 2641, 1}, {0xF9B2, 2642, 1}, + {0xF9B3, 2643, 1}, {0xF9B4, 2644, 1}, {0xF9B5, 2645, 1}, {0xF9B6, 2646, 1}, + {0xF9B7, 2647, 1}, {0xF9B8, 2648, 1}, {0xF9B9, 2649, 1}, {0xF9BA, 2650, 1}, + {0xF9BB, 2651, 1}, {0xF9BC, 2652, 1}, {0xF9BD, 2653, 1}, {0xF9BE, 2654, 1}, + {0xF9BF, 2655, 1}, {0xF9C0, 2656, 1}, {0xF9C1, 2657, 1}, {0xF9C2, 2658, 1}, + {0xF9C3, 2659, 1}, {0xF9C4, 2660, 1}, {0xF9C5, 2661, 1}, {0xF9C6, 2662, 1}, + {0xF9C7, 2663, 1}, {0xF9C8, 2664, 1}, {0xF9C9, 2665, 1}, {0xF9CA, 2666, 1}, + {0xF9CB, 2667, 1}, {0xF9CC, 2668, 1}, {0xF9CD, 2669, 1}, {0xF9CE, 2670, 1}, + {0xF9CF, 2671, 1}, {0xF9D0, 2672, 1}, {0xF9D1, 2673, 1}, {0xF9D2, 2674, 1}, + {0xF9D3, 2675, 1}, {0xF9D4, 2676, 1}, {0xF9D5, 2677, 1}, {0xF9D6, 2678, 1}, + {0xF9D7, 2679, 1}, {0xF9D8, 2680, 1}, {0xF9D9, 2681, 1}, {0xF9DA, 2682, 1}, + {0xF9DB, 2683, 1}, {0xF9DC, 2684, 1}, {0xF9DD, 2685, 1}, {0xF9DE, 2686, 1}, + {0xF9DF, 2687, 1}, {0xF9E0, 2688, 1}, {0xF9E1, 2689, 1}, {0xF9E2, 2690, 1}, + {0xF9E3, 2691, 1}, {0xF9E4, 2692, 1}, {0xF9E5, 2693, 1}, {0xF9E6, 2694, 1}, + {0xF9E7, 2695, 1}, {0xF9E8, 2696, 1}, {0xF9E9, 2697, 1}, {0xF9EA, 2698, 1}, + {0xF9EB, 2699, 1}, {0xF9EC, 2700, 1}, {0xF9ED, 2701, 1}, {0xF9EE, 2702, 1}, + {0xF9EF, 2703, 1}, {0xF9F0, 2704, 1}, {0xF9F1, 2705, 1}, {0xF9F2, 2706, 1}, + {0xF9F3, 2707, 1}, {0xF9F4, 2708, 1}, {0xF9F5, 2709, 1}, {0xF9F6, 2710, 1}, + {0xF9F7, 2711, 1}, {0xF9F8, 2712, 1}, {0xF9F9, 2713, 1}, {0xF9FA, 2714, 1}, + {0xF9FB, 2715, 1}, {0xF9FC, 2716, 1}, {0xF9FD, 2717, 1}, {0xF9FE, 2718, 1}, + {0xF9FF, 2719, 1}, {0xFA00, 2720, 1}, {0xFA01, 2721, 1}, {0xFA02, 2722, 1}, + {0xFA03, 2723, 1}, {0xFA04, 2724, 1}, {0xFA05, 2725, 1}, {0xFA06, 2726, 1}, + {0xFA07, 2727, 1}, {0xFA08, 2728, 1}, {0xFA09, 2729, 1}, {0xFA0A, 2730, 1}, + {0xFA0B, 2731, 1}, {0xFA0C, 2732, 1}, {0xFA0D, 2733, 1}, {0xFA10, 2734, 1}, + {0xFA12, 2735, 1}, {0xFA15, 2736, 1}, {0xFA16, 2737, 1}, {0xFA17, 2738, 1}, + {0xFA18, 2739, 1}, {0xFA19, 2740, 1}, {0xFA1A, 2741, 1}, {0xFA1B, 2742, 1}, + {0xFA1C, 2743, 1}, {0xFA1D, 2744, 1}, {0xFA1E, 2745, 1}, {0xFA20, 2746, 1}, + {0xFA22, 2747, 1}, {0xFA25, 2748, 1}, {0xFA26, 2749, 1}, {0xFA2A, 2750, 1}, + {0xFA2B, 2751, 1}, {0xFA2C, 2752, 1}, {0xFA2D, 2753, 1}, {0xFA2E, 2754, 1}, + {0xFA2F, 2755, 1}, {0xFA30, 2756, 1}, {0xFA31, 2757, 1}, {0xFA32, 2758, 1}, + {0xFA33, 2759, 1}, {0xFA34, 2760, 1}, {0xFA35, 2761, 1}, {0xFA36, 2762, 1}, + {0xFA37, 2763, 1}, {0xFA38, 2764, 1}, {0xFA39, 2765, 1}, {0xFA3A, 2766, 1}, + {0xFA3B, 2767, 1}, {0xFA3C, 2768, 1}, {0xFA3D, 2769, 1}, {0xFA3E, 2770, 1}, + {0xFA3F, 2771, 1}, {0xFA40, 2772, 1}, {0xFA41, 2773, 1}, {0xFA42, 2774, 1}, + {0xFA43, 2775, 1}, {0xFA44, 2776, 1}, {0xFA45, 2777, 1}, {0xFA46, 2778, 1}, + {0xFA47, 2779, 1}, {0xFA48, 2780, 1}, {0xFA49, 2781, 1}, {0xFA4A, 2782, 1}, + {0xFA4B, 2783, 1}, {0xFA4C, 2784, 1}, {0xFA4D, 2785, 1}, {0xFA4E, 2786, 1}, + {0xFA4F, 2787, 1}, {0xFA50, 2788, 1}, {0xFA51, 2789, 1}, {0xFA52, 2790, 1}, + {0xFA53, 2791, 1}, {0xFA54, 2792, 1}, {0xFA55, 2793, 1}, {0xFA56, 2794, 1}, + {0xFA57, 2795, 1}, {0xFA58, 2796, 1}, {0xFA59, 2797, 1}, {0xFA5A, 2798, 1}, + {0xFA5B, 2799, 1}, {0xFA5C, 2800, 1}, {0xFA5D, 2801, 1}, {0xFA5E, 2802, 1}, + {0xFA5F, 2803, 1}, {0xFA60, 2804, 1}, {0xFA61, 2805, 1}, {0xFA62, 2806, 1}, + {0xFA63, 2807, 1}, {0xFA64, 2808, 1}, {0xFA65, 2809, 1}, {0xFA66, 2810, 1}, + {0xFA67, 2811, 1}, {0xFA68, 2812, 1}, {0xFA69, 2813, 1}, {0xFA6A, 2814, 1}, + {0xFA6B, 2815, 1}, {0xFA6C, 2816, 1}, {0xFA6D, 2817, 1}, {0xFA70, 2818, 1}, + {0xFA71, 2819, 1}, {0xFA72, 2820, 1}, {0xFA73, 2821, 1}, {0xFA74, 2822, 1}, + {0xFA75, 2823, 1}, {0xFA76, 2824, 1}, {0xFA77, 2825, 1}, {0xFA78, 2826, 1}, + {0xFA79, 2827, 1}, {0xFA7A, 2828, 1}, {0xFA7B, 2829, 1}, {0xFA7C, 2830, 1}, + {0xFA7D, 2831, 1}, {0xFA7E, 2832, 1}, {0xFA7F, 2833, 1}, {0xFA80, 2834, 1}, + {0xFA81, 2835, 1}, {0xFA82, 2836, 1}, {0xFA83, 2837, 1}, {0xFA84, 2838, 1}, + {0xFA85, 2839, 1}, {0xFA86, 2840, 1}, {0xFA87, 2841, 1}, {0xFA88, 2842, 1}, + {0xFA89, 2843, 1}, {0xFA8A, 2844, 1}, {0xFA8B, 2845, 1}, {0xFA8C, 2846, 1}, + {0xFA8D, 2847, 1}, {0xFA8E, 2848, 1}, {0xFA8F, 2849, 1}, {0xFA90, 2850, 1}, + {0xFA91, 2851, 1}, {0xFA92, 2852, 1}, {0xFA93, 2853, 1}, {0xFA94, 2854, 1}, + {0xFA95, 2855, 1}, {0xFA96, 2856, 1}, {0xFA97, 2857, 1}, {0xFA98, 2858, 1}, + {0xFA99, 2859, 1}, {0xFA9A, 2860, 1}, {0xFA9B, 2861, 1}, {0xFA9C, 2862, 1}, + {0xFA9D, 2863, 1}, {0xFA9E, 2864, 1}, {0xFA9F, 2865, 1}, {0xFAA0, 2866, 1}, + {0xFAA1, 2867, 1}, {0xFAA2, 2868, 1}, {0xFAA3, 2869, 1}, {0xFAA4, 2870, 1}, + {0xFAA5, 2871, 1}, {0xFAA6, 2872, 1}, {0xFAA7, 2873, 1}, {0xFAA8, 2874, 1}, + {0xFAA9, 2875, 1}, {0xFAAA, 2876, 1}, {0xFAAB, 2877, 1}, {0xFAAC, 2878, 1}, + {0xFAAD, 2879, 1}, {0xFAAE, 2880, 1}, {0xFAAF, 2881, 1}, {0xFAB0, 2882, 1}, + {0xFAB1, 2883, 1}, {0xFAB2, 2884, 1}, {0xFAB3, 2885, 1}, {0xFAB4, 2886, 1}, + {0xFAB5, 2887, 1}, {0xFAB6, 2888, 1}, {0xFAB7, 2889, 1}, {0xFAB8, 2890, 1}, + {0xFAB9, 2891, 1}, {0xFABA, 2892, 1}, {0xFABB, 2893, 1}, {0xFABC, 2894, 1}, + {0xFABD, 2895, 1}, {0xFABE, 2896, 1}, {0xFABF, 2897, 1}, {0xFAC0, 2898, 1}, + {0xFAC1, 2899, 1}, {0xFAC2, 2900, 1}, {0xFAC3, 2901, 1}, {0xFAC4, 2902, 1}, + {0xFAC5, 2903, 1}, {0xFAC6, 2904, 1}, {0xFAC7, 2905, 1}, {0xFAC8, 2906, 1}, + {0xFAC9, 2907, 1}, {0xFACA, 2908, 1}, {0xFACB, 2909, 1}, {0xFACC, 2910, 1}, + {0xFACD, 2911, 1}, {0xFACE, 2912, 1}, {0xFACF, 2913, 1}, {0xFAD0, 2914, 1}, + {0xFAD1, 2915, 1}, {0xFAD2, 2916, 1}, {0xFAD3, 2917, 1}, {0xFAD4, 2918, 1}, + {0xFAD5, 2919, 1}, {0xFAD6, 2920, 1}, {0xFAD7, 2921, 1}, {0xFAD8, 2922, 1}, + {0xFAD9, 2923, 1}, {0xFB00, 2924, 2}, {0xFB01, 2926, 2}, {0xFB02, 2928, 2}, + {0xFB03, 2930, 3}, {0xFB04, 2933, 3}, {0xFB05, 2936, 2}, {0xFB06, 2938, 2}, + {0xFB13, 2940, 2}, {0xFB14, 2942, 2}, {0xFB15, 2944, 2}, {0xFB16, 2946, 2}, + {0xFB17, 2948, 2}, {0xFB1D, 2950, 2}, {0xFB1F, 2952, 2}, {0xFB20, 2954, 1}, + {0xFB21, 2955, 1}, {0xFB22, 2956, 1}, {0xFB23, 2957, 1}, {0xFB24, 2958, 1}, + {0xFB25, 2959, 1}, {0xFB26, 2960, 1}, {0xFB27, 2961, 1}, {0xFB28, 2962, 1}, + {0xFB29, 2963, 1}, {0xFB2A, 2964, 2}, {0xFB2B, 2966, 2}, {0xFB2C, 2968, 3}, + {0xFB2D, 2971, 3}, {0xFB2E, 2974, 2}, {0xFB2F, 2976, 2}, {0xFB30, 2978, 2}, + {0xFB31, 2980, 2}, {0xFB32, 2982, 2}, {0xFB33, 2984, 2}, {0xFB34, 2986, 2}, + {0xFB35, 2988, 2}, {0xFB36, 2990, 2}, {0xFB38, 2992, 2}, {0xFB39, 2994, 2}, + {0xFB3A, 2996, 2}, {0xFB3B, 2998, 2}, {0xFB3C, 3000, 2}, {0xFB3E, 3002, 2}, + {0xFB40, 3004, 2}, {0xFB41, 3006, 2}, {0xFB43, 3008, 2}, {0xFB44, 3010, 2}, + {0xFB46, 3012, 2}, {0xFB47, 3014, 2}, {0xFB48, 3016, 2}, {0xFB49, 3018, 2}, + {0xFB4A, 3020, 2}, {0xFB4B, 3022, 2}, {0xFB4C, 3024, 2}, {0xFB4D, 3026, 2}, + {0xFB4E, 3028, 2}, {0xFB4F, 3030, 2}, {0xFB50, 3032, 1}, {0xFB51, 3033, 1}, + {0xFB52, 3034, 1}, {0xFB53, 3035, 1}, {0xFB54, 3036, 1}, {0xFB55, 3037, 1}, + {0xFB56, 3038, 1}, {0xFB57, 3039, 1}, {0xFB58, 3040, 1}, {0xFB59, 3041, 1}, + {0xFB5A, 3042, 1}, {0xFB5B, 3043, 1}, {0xFB5C, 3044, 1}, {0xFB5D, 3045, 1}, + {0xFB5E, 3046, 1}, {0xFB5F, 3047, 1}, {0xFB60, 3048, 1}, {0xFB61, 3049, 1}, + {0xFB62, 3050, 1}, {0xFB63, 3051, 1}, {0xFB64, 3052, 1}, {0xFB65, 3053, 1}, + {0xFB66, 3054, 1}, {0xFB67, 3055, 1}, {0xFB68, 3056, 1}, {0xFB69, 3057, 1}, + {0xFB6A, 3058, 1}, {0xFB6B, 3059, 1}, {0xFB6C, 3060, 1}, {0xFB6D, 3061, 1}, + {0xFB6E, 3062, 1}, {0xFB6F, 3063, 1}, {0xFB70, 3064, 1}, {0xFB71, 3065, 1}, + {0xFB72, 3066, 1}, {0xFB73, 3067, 1}, {0xFB74, 3068, 1}, {0xFB75, 3069, 1}, + {0xFB76, 3070, 1}, {0xFB77, 3071, 1}, {0xFB78, 3072, 1}, {0xFB79, 3073, 1}, + {0xFB7A, 3074, 1}, {0xFB7B, 3075, 1}, {0xFB7C, 3076, 1}, {0xFB7D, 3077, 1}, + {0xFB7E, 3078, 1}, {0xFB7F, 3079, 1}, {0xFB80, 3080, 1}, {0xFB81, 3081, 1}, + {0xFB82, 3082, 1}, {0xFB83, 3083, 1}, {0xFB84, 3084, 1}, {0xFB85, 3085, 1}, + {0xFB86, 3086, 1}, {0xFB87, 3087, 1}, {0xFB88, 3088, 1}, {0xFB89, 3089, 1}, + {0xFB8A, 3090, 1}, {0xFB8B, 3091, 1}, {0xFB8C, 3092, 1}, {0xFB8D, 3093, 1}, + {0xFB8E, 3094, 1}, {0xFB8F, 3095, 1}, {0xFB90, 3096, 1}, {0xFB91, 3097, 1}, + {0xFB92, 3098, 1}, {0xFB93, 3099, 1}, {0xFB94, 3100, 1}, {0xFB95, 3101, 1}, + {0xFB96, 3102, 1}, {0xFB97, 3103, 1}, {0xFB98, 3104, 1}, {0xFB99, 3105, 1}, + {0xFB9A, 3106, 1}, {0xFB9B, 3107, 1}, {0xFB9C, 3108, 1}, {0xFB9D, 3109, 1}, + {0xFB9E, 3110, 1}, {0xFB9F, 3111, 1}, {0xFBA0, 3112, 1}, {0xFBA1, 3113, 1}, + {0xFBA2, 3114, 1}, {0xFBA3, 3115, 1}, {0xFBA4, 3116, 1}, {0xFBA5, 3117, 1}, + {0xFBA6, 3118, 1}, {0xFBA7, 3119, 1}, {0xFBA8, 3120, 1}, {0xFBA9, 3121, 1}, + {0xFBAA, 3122, 1}, {0xFBAB, 3123, 1}, {0xFBAC, 3124, 1}, {0xFBAD, 3125, 1}, + {0xFBAE, 3126, 1}, {0xFBAF, 3127, 1}, {0xFBB0, 3128, 1}, {0xFBB1, 3129, 1}, + {0xFBD3, 3130, 1}, {0xFBD4, 3131, 1}, {0xFBD5, 3132, 1}, {0xFBD6, 3133, 1}, + {0xFBD7, 3134, 1}, {0xFBD8, 3135, 1}, {0xFBD9, 3136, 1}, {0xFBDA, 3137, 1}, + {0xFBDB, 3138, 1}, {0xFBDC, 3139, 1}, {0xFBDD, 3140, 2}, {0xFBDE, 3142, 1}, + {0xFBDF, 3143, 1}, {0xFBE0, 3144, 1}, {0xFBE1, 3145, 1}, {0xFBE2, 3146, 1}, + {0xFBE3, 3147, 1}, {0xFBE4, 3148, 1}, {0xFBE5, 3149, 1}, {0xFBE6, 3150, 1}, + {0xFBE7, 3151, 1}, {0xFBE8, 3152, 1}, {0xFBE9, 3153, 1}, {0xFBEA, 3154, 2}, + {0xFBEB, 3156, 2}, {0xFBEC, 3158, 2}, {0xFBED, 3160, 2}, {0xFBEE, 3162, 2}, + {0xFBEF, 3164, 2}, {0xFBF0, 3166, 2}, {0xFBF1, 3168, 2}, {0xFBF2, 3170, 2}, + {0xFBF3, 3172, 2}, {0xFBF4, 3174, 2}, {0xFBF5, 3176, 2}, {0xFBF6, 3178, 2}, + {0xFBF7, 3180, 2}, {0xFBF8, 3182, 2}, {0xFBF9, 3184, 2}, {0xFBFA, 3186, 2}, + {0xFBFB, 3188, 2}, {0xFBFC, 3190, 1}, {0xFBFD, 3191, 1}, {0xFBFE, 3192, 1}, + {0xFBFF, 3193, 1}, {0xFC00, 3194, 2}, {0xFC01, 3196, 2}, {0xFC02, 3198, 2}, + {0xFC03, 3200, 2}, {0xFC04, 3202, 2}, {0xFC05, 3204, 2}, {0xFC06, 3206, 2}, + {0xFC07, 3208, 2}, {0xFC08, 3210, 2}, {0xFC09, 3212, 2}, {0xFC0A, 3214, 2}, + {0xFC0B, 3216, 2}, {0xFC0C, 3218, 2}, {0xFC0D, 3220, 2}, {0xFC0E, 3222, 2}, + {0xFC0F, 3224, 2}, {0xFC10, 3226, 2}, {0xFC11, 3228, 2}, {0xFC12, 3230, 2}, + {0xFC13, 3232, 2}, {0xFC14, 3234, 2}, {0xFC15, 3236, 2}, {0xFC16, 3238, 2}, + {0xFC17, 3240, 2}, {0xFC18, 3242, 2}, {0xFC19, 3244, 2}, {0xFC1A, 3246, 2}, + {0xFC1B, 3248, 2}, {0xFC1C, 3250, 2}, {0xFC1D, 3252, 2}, {0xFC1E, 3254, 2}, + {0xFC1F, 3256, 2}, {0xFC20, 3258, 2}, {0xFC21, 3260, 2}, {0xFC22, 3262, 2}, + {0xFC23, 3264, 2}, {0xFC24, 3266, 2}, {0xFC25, 3268, 2}, {0xFC26, 3270, 2}, + {0xFC27, 3272, 2}, {0xFC28, 3274, 2}, {0xFC29, 3276, 2}, {0xFC2A, 3278, 2}, + {0xFC2B, 3280, 2}, {0xFC2C, 3282, 2}, {0xFC2D, 3284, 2}, {0xFC2E, 3286, 2}, + {0xFC2F, 3288, 2}, {0xFC30, 3290, 2}, {0xFC31, 3292, 2}, {0xFC32, 3294, 2}, + {0xFC33, 3296, 2}, {0xFC34, 3298, 2}, {0xFC35, 3300, 2}, {0xFC36, 3302, 2}, + {0xFC37, 3304, 2}, {0xFC38, 3306, 2}, {0xFC39, 3308, 2}, {0xFC3A, 3310, 2}, + {0xFC3B, 3312, 2}, {0xFC3C, 3314, 2}, {0xFC3D, 3316, 2}, {0xFC3E, 3318, 2}, + {0xFC3F, 3320, 2}, {0xFC40, 3322, 2}, {0xFC41, 3324, 2}, {0xFC42, 3326, 2}, + {0xFC43, 3328, 2}, {0xFC44, 3330, 2}, {0xFC45, 3332, 2}, {0xFC46, 3334, 2}, + {0xFC47, 3336, 2}, {0xFC48, 3338, 2}, {0xFC49, 3340, 2}, {0xFC4A, 3342, 2}, + {0xFC4B, 3344, 2}, {0xFC4C, 3346, 2}, {0xFC4D, 3348, 2}, {0xFC4E, 3350, 2}, + {0xFC4F, 3352, 2}, {0xFC50, 3354, 2}, {0xFC51, 3356, 2}, {0xFC52, 3358, 2}, + {0xFC53, 3360, 2}, {0xFC54, 3362, 2}, {0xFC55, 3364, 2}, {0xFC56, 3366, 2}, + {0xFC57, 3368, 2}, {0xFC58, 3370, 2}, {0xFC59, 3372, 2}, {0xFC5A, 3374, 2}, + {0xFC5B, 3376, 2}, {0xFC5C, 3378, 2}, {0xFC5D, 3380, 2}, {0xFC5E, 3382, 3}, + {0xFC5F, 3385, 3}, {0xFC60, 3388, 3}, {0xFC61, 3391, 3}, {0xFC62, 3394, 3}, + {0xFC63, 3397, 3}, {0xFC64, 3400, 2}, {0xFC65, 3402, 2}, {0xFC66, 3404, 2}, + {0xFC67, 3406, 2}, {0xFC68, 3408, 2}, {0xFC69, 3410, 2}, {0xFC6A, 3412, 2}, + {0xFC6B, 3414, 2}, {0xFC6C, 3416, 2}, {0xFC6D, 3418, 2}, {0xFC6E, 3420, 2}, + {0xFC6F, 3422, 2}, {0xFC70, 3424, 2}, {0xFC71, 3426, 2}, {0xFC72, 3428, 2}, + {0xFC73, 3430, 2}, {0xFC74, 3432, 2}, {0xFC75, 3434, 2}, {0xFC76, 3436, 2}, + {0xFC77, 3438, 2}, {0xFC78, 3440, 2}, {0xFC79, 3442, 2}, {0xFC7A, 3444, 2}, + {0xFC7B, 3446, 2}, {0xFC7C, 3448, 2}, {0xFC7D, 3450, 2}, {0xFC7E, 3452, 2}, + {0xFC7F, 3454, 2}, {0xFC80, 3456, 2}, {0xFC81, 3458, 2}, {0xFC82, 3460, 2}, + {0xFC83, 3462, 2}, {0xFC84, 3464, 2}, {0xFC85, 3466, 2}, {0xFC86, 3468, 2}, + {0xFC87, 3470, 2}, {0xFC88, 3472, 2}, {0xFC89, 3474, 2}, {0xFC8A, 3476, 2}, + {0xFC8B, 3478, 2}, {0xFC8C, 3480, 2}, {0xFC8D, 3482, 2}, {0xFC8E, 3484, 2}, + {0xFC8F, 3486, 2}, {0xFC90, 3488, 2}, {0xFC91, 3490, 2}, {0xFC92, 3492, 2}, + {0xFC93, 3494, 2}, {0xFC94, 3496, 2}, {0xFC95, 3498, 2}, {0xFC96, 3500, 2}, + {0xFC97, 3502, 2}, {0xFC98, 3504, 2}, {0xFC99, 3506, 2}, {0xFC9A, 3508, 2}, + {0xFC9B, 3510, 2}, {0xFC9C, 3512, 2}, {0xFC9D, 3514, 2}, {0xFC9E, 3516, 2}, + {0xFC9F, 3518, 2}, {0xFCA0, 3520, 2}, {0xFCA1, 3522, 2}, {0xFCA2, 3524, 2}, + {0xFCA3, 3526, 2}, {0xFCA4, 3528, 2}, {0xFCA5, 3530, 2}, {0xFCA6, 3532, 2}, + {0xFCA7, 3534, 2}, {0xFCA8, 3536, 2}, {0xFCA9, 3538, 2}, {0xFCAA, 3540, 2}, + {0xFCAB, 3542, 2}, {0xFCAC, 3544, 2}, {0xFCAD, 3546, 2}, {0xFCAE, 3548, 2}, + {0xFCAF, 3550, 2}, {0xFCB0, 3552, 2}, {0xFCB1, 3554, 2}, {0xFCB2, 3556, 2}, + {0xFCB3, 3558, 2}, {0xFCB4, 3560, 2}, {0xFCB5, 3562, 2}, {0xFCB6, 3564, 2}, + {0xFCB7, 3566, 2}, {0xFCB8, 3568, 2}, {0xFCB9, 3570, 2}, {0xFCBA, 3572, 2}, + {0xFCBB, 3574, 2}, {0xFCBC, 3576, 2}, {0xFCBD, 3578, 2}, {0xFCBE, 3580, 2}, + {0xFCBF, 3582, 2}, {0xFCC0, 3584, 2}, {0xFCC1, 3586, 2}, {0xFCC2, 3588, 2}, + {0xFCC3, 3590, 2}, {0xFCC4, 3592, 2}, {0xFCC5, 3594, 2}, {0xFCC6, 3596, 2}, + {0xFCC7, 3598, 2}, {0xFCC8, 3600, 2}, {0xFCC9, 3602, 2}, {0xFCCA, 3604, 2}, + {0xFCCB, 3606, 2}, {0xFCCC, 3608, 2}, {0xFCCD, 3610, 2}, {0xFCCE, 3612, 2}, + {0xFCCF, 3614, 2}, {0xFCD0, 3616, 2}, {0xFCD1, 3618, 2}, {0xFCD2, 3620, 2}, + {0xFCD3, 3622, 2}, {0xFCD4, 3624, 2}, {0xFCD5, 3626, 2}, {0xFCD6, 3628, 2}, + {0xFCD7, 3630, 2}, {0xFCD8, 3632, 2}, {0xFCD9, 3634, 2}, {0xFCDA, 3636, 2}, + {0xFCDB, 3638, 2}, {0xFCDC, 3640, 2}, {0xFCDD, 3642, 2}, {0xFCDE, 3644, 2}, + {0xFCDF, 3646, 2}, {0xFCE0, 3648, 2}, {0xFCE1, 3650, 2}, {0xFCE2, 3652, 2}, + {0xFCE3, 3654, 2}, {0xFCE4, 3656, 2}, {0xFCE5, 3658, 2}, {0xFCE6, 3660, 2}, + {0xFCE7, 3662, 2}, {0xFCE8, 3664, 2}, {0xFCE9, 3666, 2}, {0xFCEA, 3668, 2}, + {0xFCEB, 3670, 2}, {0xFCEC, 3672, 2}, {0xFCED, 3674, 2}, {0xFCEE, 3676, 2}, + {0xFCEF, 3678, 2}, {0xFCF0, 3680, 2}, {0xFCF1, 3682, 2}, {0xFCF2, 3684, 3}, + {0xFCF3, 3687, 3}, {0xFCF4, 3690, 3}, {0xFCF5, 3693, 2}, {0xFCF6, 3695, 2}, + {0xFCF7, 3697, 2}, {0xFCF8, 3699, 2}, {0xFCF9, 3701, 2}, {0xFCFA, 3703, 2}, + {0xFCFB, 3705, 2}, {0xFCFC, 3707, 2}, {0xFCFD, 3709, 2}, {0xFCFE, 3711, 2}, + {0xFCFF, 3713, 2}, {0xFD00, 3715, 2}, {0xFD01, 3717, 2}, {0xFD02, 3719, 2}, + {0xFD03, 3721, 2}, {0xFD04, 3723, 2}, {0xFD05, 3725, 2}, {0xFD06, 3727, 2}, + {0xFD07, 3729, 2}, {0xFD08, 3731, 2}, {0xFD09, 3733, 2}, {0xFD0A, 3735, 2}, + {0xFD0B, 3737, 2}, {0xFD0C, 3739, 2}, {0xFD0D, 3741, 2}, {0xFD0E, 3743, 2}, + {0xFD0F, 3745, 2}, {0xFD10, 3747, 2}, {0xFD11, 3749, 2}, {0xFD12, 3751, 2}, + {0xFD13, 3753, 2}, {0xFD14, 3755, 2}, {0xFD15, 3757, 2}, {0xFD16, 3759, 2}, + {0xFD17, 3761, 2}, {0xFD18, 3763, 2}, {0xFD19, 3765, 2}, {0xFD1A, 3767, 2}, + {0xFD1B, 3769, 2}, {0xFD1C, 3771, 2}, {0xFD1D, 3773, 2}, {0xFD1E, 3775, 2}, + {0xFD1F, 3777, 2}, {0xFD20, 3779, 2}, {0xFD21, 3781, 2}, {0xFD22, 3783, 2}, + {0xFD23, 3785, 2}, {0xFD24, 3787, 2}, {0xFD25, 3789, 2}, {0xFD26, 3791, 2}, + {0xFD27, 3793, 2}, {0xFD28, 3795, 2}, {0xFD29, 3797, 2}, {0xFD2A, 3799, 2}, + {0xFD2B, 3801, 2}, {0xFD2C, 3803, 2}, {0xFD2D, 3805, 2}, {0xFD2E, 3807, 2}, + {0xFD2F, 3809, 2}, {0xFD30, 3811, 2}, {0xFD31, 3813, 2}, {0xFD32, 3815, 2}, + {0xFD33, 3817, 2}, {0xFD34, 3819, 2}, {0xFD35, 3821, 2}, {0xFD36, 3823, 2}, + {0xFD37, 3825, 2}, {0xFD38, 3827, 2}, {0xFD39, 3829, 2}, {0xFD3A, 3831, 2}, + {0xFD3B, 3833, 2}, {0xFD3C, 3835, 2}, {0xFD3D, 3837, 2}, {0xFD50, 3839, 3}, + {0xFD51, 3842, 3}, {0xFD52, 3845, 3}, {0xFD53, 3848, 3}, {0xFD54, 3851, 3}, + {0xFD55, 3854, 3}, {0xFD56, 3857, 3}, {0xFD57, 3860, 3}, {0xFD58, 3863, 3}, + {0xFD59, 3866, 3}, {0xFD5A, 3869, 3}, {0xFD5B, 3872, 3}, {0xFD5C, 3875, 3}, + {0xFD5D, 3878, 3}, {0xFD5E, 3881, 3}, {0xFD5F, 3884, 3}, {0xFD60, 3887, 3}, + {0xFD61, 3890, 3}, {0xFD62, 3893, 3}, {0xFD63, 3896, 3}, {0xFD64, 3899, 3}, + {0xFD65, 3902, 3}, {0xFD66, 3905, 3}, {0xFD67, 3908, 3}, {0xFD68, 3911, 3}, + {0xFD69, 3914, 3}, {0xFD6A, 3917, 3}, {0xFD6B, 3920, 3}, {0xFD6C, 3923, 3}, + {0xFD6D, 3926, 3}, {0xFD6E, 3929, 3}, {0xFD6F, 3932, 3}, {0xFD70, 3935, 3}, + {0xFD71, 3938, 3}, {0xFD72, 3941, 3}, {0xFD73, 3944, 3}, {0xFD74, 3947, 3}, + {0xFD75, 3950, 3}, {0xFD76, 3953, 3}, {0xFD77, 3956, 3}, {0xFD78, 3959, 3}, + {0xFD79, 3962, 3}, {0xFD7A, 3965, 3}, {0xFD7B, 3968, 3}, {0xFD7C, 3971, 3}, + {0xFD7D, 3974, 3}, {0xFD7E, 3977, 3}, {0xFD7F, 3980, 3}, {0xFD80, 3983, 3}, + {0xFD81, 3986, 3}, {0xFD82, 3989, 3}, {0xFD83, 3992, 3}, {0xFD84, 3995, 3}, + {0xFD85, 3998, 3}, {0xFD86, 4001, 3}, {0xFD87, 4004, 3}, {0xFD88, 4007, 3}, + {0xFD89, 4010, 3}, {0xFD8A, 4013, 3}, {0xFD8B, 4016, 3}, {0xFD8C, 4019, 3}, + {0xFD8D, 4022, 3}, {0xFD8E, 4025, 3}, {0xFD8F, 4028, 3}, {0xFD92, 4031, 3}, + {0xFD93, 4034, 3}, {0xFD94, 4037, 3}, {0xFD95, 4040, 3}, {0xFD96, 4043, 3}, + {0xFD97, 4046, 3}, {0xFD98, 4049, 3}, {0xFD99, 4052, 3}, {0xFD9A, 4055, 3}, + {0xFD9B, 4058, 3}, {0xFD9C, 4061, 3}, {0xFD9D, 4064, 3}, {0xFD9E, 4067, 3}, + {0xFD9F, 4070, 3}, {0xFDA0, 4073, 3}, {0xFDA1, 4076, 3}, {0xFDA2, 4079, 3}, + {0xFDA3, 4082, 3}, {0xFDA4, 4085, 3}, {0xFDA5, 4088, 3}, {0xFDA6, 4091, 3}, + {0xFDA7, 4094, 3}, {0xFDA8, 4097, 3}, {0xFDA9, 4100, 3}, {0xFDAA, 4103, 3}, + {0xFDAB, 4106, 3}, {0xFDAC, 4109, 3}, {0xFDAD, 4112, 3}, {0xFDAE, 4115, 3}, + {0xFDAF, 4118, 3}, {0xFDB0, 4121, 3}, {0xFDB1, 4124, 3}, {0xFDB2, 4127, 3}, + {0xFDB3, 4130, 3}, {0xFDB4, 4133, 3}, {0xFDB5, 4136, 3}, {0xFDB6, 4139, 3}, + {0xFDB7, 4142, 3}, {0xFDB8, 4145, 3}, {0xFDB9, 4148, 3}, {0xFDBA, 4151, 3}, + {0xFDBB, 4154, 3}, {0xFDBC, 4157, 3}, {0xFDBD, 4160, 3}, {0xFDBE, 4163, 3}, + {0xFDBF, 4166, 3}, {0xFDC0, 4169, 3}, {0xFDC1, 4172, 3}, {0xFDC2, 4175, 3}, + {0xFDC3, 4178, 3}, {0xFDC4, 4181, 3}, {0xFDC5, 4184, 3}, {0xFDC6, 4187, 3}, + {0xFDC7, 4190, 3}, {0xFDF0, 4193, 3}, {0xFDF1, 4196, 3}, {0xFDF2, 4199, 4}, + {0xFDF3, 4203, 4}, {0xFDF4, 4207, 4}, {0xFDF5, 4211, 4}, {0xFDF6, 4215, 4}, + {0xFDF7, 4219, 4}, {0xFDF8, 4223, 4}, {0xFDF9, 4227, 3}, {0xFDFA, 4230, 18}, + {0xFDFB, 4248, 8}, {0xFDFC, 4256, 4}, {0xFE10, 4260, 1}, {0xFE11, 4261, 1}, + {0xFE12, 4262, 1}, {0xFE13, 4263, 1}, {0xFE14, 4264, 1}, {0xFE15, 4265, 1}, + {0xFE16, 4266, 1}, {0xFE17, 4267, 1}, {0xFE18, 4268, 1}, {0xFE19, 4269, 3}, + {0xFE30, 4272, 2}, {0xFE31, 4274, 1}, {0xFE32, 4275, 1}, {0xFE33, 4276, 1}, + {0xFE34, 4277, 1}, {0xFE35, 4278, 1}, {0xFE36, 4279, 1}, {0xFE37, 4280, 1}, + {0xFE38, 4281, 1}, {0xFE39, 4282, 1}, {0xFE3A, 4283, 1}, {0xFE3B, 4284, 1}, + {0xFE3C, 4285, 1}, {0xFE3D, 4286, 1}, {0xFE3E, 4287, 1}, {0xFE3F, 4288, 1}, + {0xFE40, 4289, 1}, {0xFE41, 4290, 1}, {0xFE42, 4291, 1}, {0xFE43, 4292, 1}, + {0xFE44, 4293, 1}, {0xFE47, 4294, 1}, {0xFE48, 4295, 1}, {0xFE49, 4296, 2}, + {0xFE4A, 4298, 2}, {0xFE4B, 4300, 2}, {0xFE4C, 4302, 2}, {0xFE4D, 4304, 1}, + {0xFE4E, 4305, 1}, {0xFE4F, 4306, 1}, {0xFE50, 4307, 1}, {0xFE51, 4308, 1}, + {0xFE52, 4309, 1}, {0xFE54, 4310, 1}, {0xFE55, 4311, 1}, {0xFE56, 4312, 1}, + {0xFE57, 4313, 1}, {0xFE58, 4314, 1}, {0xFE59, 4315, 1}, {0xFE5A, 4316, 1}, + {0xFE5B, 4317, 1}, {0xFE5C, 4318, 1}, {0xFE5D, 4319, 1}, {0xFE5E, 4320, 1}, + {0xFE5F, 4321, 1}, {0xFE60, 4322, 1}, {0xFE61, 4323, 1}, {0xFE62, 4324, 1}, + {0xFE63, 4325, 1}, {0xFE64, 4326, 1}, {0xFE65, 4327, 1}, {0xFE66, 4328, 1}, + {0xFE68, 4329, 1}, {0xFE69, 4330, 1}, {0xFE6A, 4331, 1}, {0xFE6B, 4332, 1}, + {0xFE70, 4333, 2}, {0xFE71, 4335, 2}, {0xFE72, 4337, 2}, {0xFE74, 4339, 2}, + {0xFE76, 4341, 2}, {0xFE77, 4343, 2}, {0xFE78, 4345, 2}, {0xFE79, 4347, 2}, + {0xFE7A, 4349, 2}, {0xFE7B, 4351, 2}, {0xFE7C, 4353, 2}, {0xFE7D, 4355, 2}, + {0xFE7E, 4357, 2}, {0xFE7F, 4359, 2}, {0xFE80, 4361, 1}, {0xFE81, 4362, 1}, + {0xFE82, 4363, 1}, {0xFE83, 4364, 1}, {0xFE84, 4365, 1}, {0xFE85, 4366, 1}, + {0xFE86, 4367, 1}, {0xFE87, 4368, 1}, {0xFE88, 4369, 1}, {0xFE89, 4370, 1}, + {0xFE8A, 4371, 1}, {0xFE8B, 4372, 1}, {0xFE8C, 4373, 1}, {0xFE8D, 4374, 1}, + {0xFE8E, 4375, 1}, {0xFE8F, 4376, 1}, {0xFE90, 4377, 1}, {0xFE91, 4378, 1}, + {0xFE92, 4379, 1}, {0xFE93, 4380, 1}, {0xFE94, 4381, 1}, {0xFE95, 4382, 1}, + {0xFE96, 4383, 1}, {0xFE97, 4384, 1}, {0xFE98, 4385, 1}, {0xFE99, 4386, 1}, + {0xFE9A, 4387, 1}, {0xFE9B, 4388, 1}, {0xFE9C, 4389, 1}, {0xFE9D, 4390, 1}, + {0xFE9E, 4391, 1}, {0xFE9F, 4392, 1}, {0xFEA0, 4393, 1}, {0xFEA1, 4394, 1}, + {0xFEA2, 4395, 1}, {0xFEA3, 4396, 1}, {0xFEA4, 4397, 1}, {0xFEA5, 4398, 1}, + {0xFEA6, 4399, 1}, {0xFEA7, 4400, 1}, {0xFEA8, 4401, 1}, {0xFEA9, 4402, 1}, + {0xFEAA, 4403, 1}, {0xFEAB, 4404, 1}, {0xFEAC, 4405, 1}, {0xFEAD, 4406, 1}, + {0xFEAE, 4407, 1}, {0xFEAF, 4408, 1}, {0xFEB0, 4409, 1}, {0xFEB1, 4410, 1}, + {0xFEB2, 4411, 1}, {0xFEB3, 4412, 1}, {0xFEB4, 4413, 1}, {0xFEB5, 4414, 1}, + {0xFEB6, 4415, 1}, {0xFEB7, 4416, 1}, {0xFEB8, 4417, 1}, {0xFEB9, 4418, 1}, + {0xFEBA, 4419, 1}, {0xFEBB, 4420, 1}, {0xFEBC, 4421, 1}, {0xFEBD, 4422, 1}, + {0xFEBE, 4423, 1}, {0xFEBF, 4424, 1}, {0xFEC0, 4425, 1}, {0xFEC1, 4426, 1}, + {0xFEC2, 4427, 1}, {0xFEC3, 4428, 1}, {0xFEC4, 4429, 1}, {0xFEC5, 4430, 1}, + {0xFEC6, 4431, 1}, {0xFEC7, 4432, 1}, {0xFEC8, 4433, 1}, {0xFEC9, 4434, 1}, + {0xFECA, 4435, 1}, {0xFECB, 4436, 1}, {0xFECC, 4437, 1}, {0xFECD, 4438, 1}, + {0xFECE, 4439, 1}, {0xFECF, 4440, 1}, {0xFED0, 4441, 1}, {0xFED1, 4442, 1}, + {0xFED2, 4443, 1}, {0xFED3, 4444, 1}, {0xFED4, 4445, 1}, {0xFED5, 4446, 1}, + {0xFED6, 4447, 1}, {0xFED7, 4448, 1}, {0xFED8, 4449, 1}, {0xFED9, 4450, 1}, + {0xFEDA, 4451, 1}, {0xFEDB, 4452, 1}, {0xFEDC, 4453, 1}, {0xFEDD, 4454, 1}, + {0xFEDE, 4455, 1}, {0xFEDF, 4456, 1}, {0xFEE0, 4457, 1}, {0xFEE1, 4458, 1}, + {0xFEE2, 4459, 1}, {0xFEE3, 4460, 1}, {0xFEE4, 4461, 1}, {0xFEE5, 4462, 1}, + {0xFEE6, 4463, 1}, {0xFEE7, 4464, 1}, {0xFEE8, 4465, 1}, {0xFEE9, 4466, 1}, + {0xFEEA, 4467, 1}, {0xFEEB, 4468, 1}, {0xFEEC, 4469, 1}, {0xFEED, 4470, 1}, + {0xFEEE, 4471, 1}, {0xFEEF, 4472, 1}, {0xFEF0, 4473, 1}, {0xFEF1, 4474, 1}, + {0xFEF2, 4475, 1}, {0xFEF3, 4476, 1}, {0xFEF4, 4477, 1}, {0xFEF5, 4478, 2}, + {0xFEF6, 4480, 2}, {0xFEF7, 4482, 2}, {0xFEF8, 4484, 2}, {0xFEF9, 4486, 2}, + {0xFEFA, 4488, 2}, {0xFEFB, 4490, 2}, {0xFEFC, 4492, 2}, {0xFF01, 4494, 1}, + {0xFF02, 4495, 1}, {0xFF03, 4496, 1}, {0xFF04, 4497, 1}, {0xFF05, 4498, 1}, + {0xFF06, 4499, 1}, {0xFF07, 4500, 1}, {0xFF08, 4501, 1}, {0xFF09, 4502, 1}, + {0xFF0A, 4503, 1}, {0xFF0B, 4504, 1}, {0xFF0C, 4505, 1}, {0xFF0D, 4506, 1}, + {0xFF0E, 4507, 1}, {0xFF0F, 4508, 1}, {0xFF10, 4509, 1}, {0xFF11, 4510, 1}, + {0xFF12, 4511, 1}, {0xFF13, 4512, 1}, {0xFF14, 4513, 1}, {0xFF15, 4514, 1}, + {0xFF16, 4515, 1}, {0xFF17, 4516, 1}, {0xFF18, 4517, 1}, {0xFF19, 4518, 1}, + {0xFF1A, 4519, 1}, {0xFF1B, 4520, 1}, {0xFF1C, 4521, 1}, {0xFF1D, 4522, 1}, + {0xFF1E, 4523, 1}, {0xFF1F, 4524, 1}, {0xFF20, 4525, 1}, {0xFF21, 4526, 1}, + {0xFF22, 4527, 1}, {0xFF23, 4528, 1}, {0xFF24, 4529, 1}, {0xFF25, 4530, 1}, + {0xFF26, 4531, 1}, {0xFF27, 4532, 1}, {0xFF28, 4533, 1}, {0xFF29, 4534, 1}, + {0xFF2A, 4535, 1}, {0xFF2B, 4536, 1}, {0xFF2C, 4537, 1}, {0xFF2D, 4538, 1}, + {0xFF2E, 4539, 1}, {0xFF2F, 4540, 1}, {0xFF30, 4541, 1}, {0xFF31, 4542, 1}, + {0xFF32, 4543, 1}, {0xFF33, 4544, 1}, {0xFF34, 4545, 1}, {0xFF35, 4546, 1}, + {0xFF36, 4547, 1}, {0xFF37, 4548, 1}, {0xFF38, 4549, 1}, {0xFF39, 4550, 1}, + {0xFF3A, 4551, 1}, {0xFF3B, 4552, 1}, {0xFF3C, 4553, 1}, {0xFF3D, 4554, 1}, + {0xFF3E, 4555, 1}, {0xFF3F, 4556, 1}, {0xFF40, 4557, 1}, {0xFF41, 4558, 1}, + {0xFF42, 4559, 1}, {0xFF43, 4560, 1}, {0xFF44, 4561, 1}, {0xFF45, 4562, 1}, + {0xFF46, 4563, 1}, {0xFF47, 4564, 1}, {0xFF48, 4565, 1}, {0xFF49, 4566, 1}, + {0xFF4A, 4567, 1}, {0xFF4B, 4568, 1}, {0xFF4C, 4569, 1}, {0xFF4D, 4570, 1}, + {0xFF4E, 4571, 1}, {0xFF4F, 4572, 1}, {0xFF50, 4573, 1}, {0xFF51, 4574, 1}, + {0xFF52, 4575, 1}, {0xFF53, 4576, 1}, {0xFF54, 4577, 1}, {0xFF55, 4578, 1}, + {0xFF56, 4579, 1}, {0xFF57, 4580, 1}, {0xFF58, 4581, 1}, {0xFF59, 4582, 1}, + {0xFF5A, 4583, 1}, {0xFF5B, 4584, 1}, {0xFF5C, 4585, 1}, {0xFF5D, 4586, 1}, + {0xFF5E, 4587, 1}, {0xFF5F, 4588, 1}, {0xFF60, 4589, 1}, {0xFF61, 4590, 1}, + {0xFF62, 4591, 1}, {0xFF63, 4592, 1}, {0xFF64, 4593, 1}, {0xFF65, 4594, 1}, + {0xFF66, 4595, 1}, {0xFF67, 4596, 1}, {0xFF68, 4597, 1}, {0xFF69, 4598, 1}, + {0xFF6A, 4599, 1}, {0xFF6B, 4600, 1}, {0xFF6C, 4601, 1}, {0xFF6D, 4602, 1}, + {0xFF6E, 4603, 1}, {0xFF6F, 4604, 1}, {0xFF70, 4605, 1}, {0xFF71, 4606, 1}, + {0xFF72, 4607, 1}, {0xFF73, 4608, 1}, {0xFF74, 4609, 1}, {0xFF75, 4610, 1}, + {0xFF76, 4611, 1}, {0xFF77, 4612, 1}, {0xFF78, 4613, 1}, {0xFF79, 4614, 1}, + {0xFF7A, 4615, 1}, {0xFF7B, 4616, 1}, {0xFF7C, 4617, 1}, {0xFF7D, 4618, 1}, + {0xFF7E, 4619, 1}, {0xFF7F, 4620, 1}, {0xFF80, 4621, 1}, {0xFF81, 4622, 1}, + {0xFF82, 4623, 1}, {0xFF83, 4624, 1}, {0xFF84, 4625, 1}, {0xFF85, 4626, 1}, + {0xFF86, 4627, 1}, {0xFF87, 4628, 1}, {0xFF88, 4629, 1}, {0xFF89, 4630, 1}, + {0xFF8A, 4631, 1}, {0xFF8B, 4632, 1}, {0xFF8C, 4633, 1}, {0xFF8D, 4634, 1}, + {0xFF8E, 4635, 1}, {0xFF8F, 4636, 1}, {0xFF90, 4637, 1}, {0xFF91, 4638, 1}, + {0xFF92, 4639, 1}, {0xFF93, 4640, 1}, {0xFF94, 4641, 1}, {0xFF95, 4642, 1}, + {0xFF96, 4643, 1}, {0xFF97, 4644, 1}, {0xFF98, 4645, 1}, {0xFF99, 4646, 1}, + {0xFF9A, 4647, 1}, {0xFF9B, 4648, 1}, {0xFF9C, 4649, 1}, {0xFF9D, 4650, 1}, + {0xFF9E, 4651, 1}, {0xFF9F, 4652, 1}, {0xFFA0, 4653, 1}, {0xFFA1, 4654, 1}, + {0xFFA2, 4655, 1}, {0xFFA3, 4656, 1}, {0xFFA4, 4657, 1}, {0xFFA5, 4658, 1}, + {0xFFA6, 4659, 1}, {0xFFA7, 4660, 1}, {0xFFA8, 4661, 1}, {0xFFA9, 4662, 1}, + {0xFFAA, 4663, 1}, {0xFFAB, 4664, 1}, {0xFFAC, 4665, 1}, {0xFFAD, 4666, 1}, + {0xFFAE, 4667, 1}, {0xFFAF, 4668, 1}, {0xFFB0, 4669, 1}, {0xFFB1, 4670, 1}, + {0xFFB2, 4671, 1}, {0xFFB3, 4672, 1}, {0xFFB4, 4673, 1}, {0xFFB5, 4674, 1}, + {0xFFB6, 4675, 1}, {0xFFB7, 4676, 1}, {0xFFB8, 4677, 1}, {0xFFB9, 4678, 1}, + {0xFFBA, 4679, 1}, {0xFFBB, 4680, 1}, {0xFFBC, 4681, 1}, {0xFFBD, 4682, 1}, + {0xFFBE, 4683, 1}, {0xFFC2, 4684, 1}, {0xFFC3, 4685, 1}, {0xFFC4, 4686, 1}, + {0xFFC5, 4687, 1}, {0xFFC6, 4688, 1}, {0xFFC7, 4689, 1}, {0xFFCA, 4690, 1}, + {0xFFCB, 4691, 1}, {0xFFCC, 4692, 1}, {0xFFCD, 4693, 1}, {0xFFCE, 4694, 1}, + {0xFFCF, 4695, 1}, {0xFFD2, 4696, 1}, {0xFFD3, 4697, 1}, {0xFFD4, 4698, 1}, + {0xFFD5, 4699, 1}, {0xFFD6, 4700, 1}, {0xFFD7, 4701, 1}, {0xFFDA, 4702, 1}, + {0xFFDB, 4703, 1}, {0xFFDC, 4704, 1}, {0xFFE0, 4705, 1}, {0xFFE1, 4706, 1}, + {0xFFE2, 4707, 1}, {0xFFE3, 4708, 2}, {0xFFE4, 4710, 1}, {0xFFE5, 4711, 1}, + {0xFFE6, 4712, 1}, {0xFFE8, 4713, 1}, {0xFFE9, 4714, 1}, {0xFFEA, 4715, 1}, + {0xFFEB, 4716, 1}, {0xFFEC, 4717, 1}, {0xFFED, 4718, 1}, {0xFFEE, 4719, 1}, + {0x10781, 4720, 1}, {0x10782, 4721, 1}, {0x10783, 4722, 1}, {0x10784, 4723, 1}, + {0x10785, 4724, 1}, {0x10787, 4725, 1}, {0x10788, 4726, 1}, {0x10789, 4727, 1}, + {0x1078A, 4728, 1}, {0x1078B, 4729, 1}, {0x1078C, 4730, 1}, {0x1078D, 4731, 1}, + {0x1078E, 4732, 1}, {0x1078F, 4733, 1}, {0x10790, 4734, 1}, {0x10791, 4735, 1}, + {0x10792, 4736, 1}, {0x10793, 4737, 1}, {0x10794, 4738, 1}, {0x10795, 4739, 1}, + {0x10796, 4740, 1}, {0x10797, 4741, 1}, {0x10798, 4742, 1}, {0x10799, 4743, 1}, + {0x1079A, 4744, 1}, {0x1079B, 4745, 1}, {0x1079C, 4746, 1}, {0x1079D, 4747, 1}, + {0x1079E, 4748, 1}, {0x1079F, 4749, 1}, {0x107A0, 4750, 1}, {0x107A1, 4751, 1}, + {0x107A2, 4752, 1}, {0x107A3, 4753, 1}, {0x107A4, 4754, 1}, {0x107A5, 4755, 1}, + {0x107A6, 4756, 1}, {0x107A7, 4757, 1}, {0x107A8, 4758, 1}, {0x107A9, 4759, 1}, + {0x107AA, 4760, 1}, {0x107AB, 4761, 1}, {0x107AC, 4762, 1}, {0x107AD, 4763, 1}, + {0x107AE, 4764, 1}, {0x107AF, 4765, 1}, {0x107B0, 4766, 1}, {0x107B2, 4767, 1}, + {0x107B3, 4768, 1}, {0x107B4, 4769, 1}, {0x107B5, 4770, 1}, {0x107B6, 4771, 1}, + {0x107B7, 4772, 1}, {0x107B8, 4773, 1}, {0x107B9, 4774, 1}, {0x107BA, 4775, 1}, + {0x1CCD6, 4776, 1}, {0x1CCD7, 4777, 1}, {0x1CCD8, 4778, 1}, {0x1CCD9, 4779, 1}, + {0x1CCDA, 4780, 1}, {0x1CCDB, 4781, 1}, {0x1CCDC, 4782, 1}, {0x1CCDD, 4783, 1}, + {0x1CCDE, 4784, 1}, {0x1CCDF, 4785, 1}, {0x1CCE0, 4786, 1}, {0x1CCE1, 4787, 1}, + {0x1CCE2, 4788, 1}, {0x1CCE3, 4789, 1}, {0x1CCE4, 4790, 1}, {0x1CCE5, 4791, 1}, + {0x1CCE6, 4792, 1}, {0x1CCE7, 4793, 1}, {0x1CCE8, 4794, 1}, {0x1CCE9, 4795, 1}, + {0x1CCEA, 4796, 1}, {0x1CCEB, 4797, 1}, {0x1CCEC, 4798, 1}, {0x1CCED, 4799, 1}, + {0x1CCEE, 4800, 1}, {0x1CCEF, 4801, 1}, {0x1CCF0, 4802, 1}, {0x1CCF1, 4803, 1}, + {0x1CCF2, 4804, 1}, {0x1CCF3, 4805, 1}, {0x1CCF4, 4806, 1}, {0x1CCF5, 4807, 1}, + {0x1CCF6, 4808, 1}, {0x1CCF7, 4809, 1}, {0x1CCF8, 4810, 1}, {0x1CCF9, 4811, 1}, + {0x1D15E, 4812, 2}, {0x1D15F, 4814, 2}, {0x1D160, 4816, 3}, {0x1D161, 4819, 3}, + {0x1D162, 4822, 3}, {0x1D163, 4825, 3}, {0x1D164, 4828, 3}, {0x1D1BB, 4831, 2}, + {0x1D1BC, 4833, 2}, {0x1D1BD, 4835, 3}, {0x1D1BE, 4838, 3}, {0x1D1BF, 4841, 3}, + {0x1D1C0, 4844, 3}, {0x1D400, 4847, 1}, {0x1D401, 4848, 1}, {0x1D402, 4849, 1}, + {0x1D403, 4850, 1}, {0x1D404, 4851, 1}, {0x1D405, 4852, 1}, {0x1D406, 4853, 1}, + {0x1D407, 4854, 1}, {0x1D408, 4855, 1}, {0x1D409, 4856, 1}, {0x1D40A, 4857, 1}, + {0x1D40B, 4858, 1}, {0x1D40C, 4859, 1}, {0x1D40D, 4860, 1}, {0x1D40E, 4861, 1}, + {0x1D40F, 4862, 1}, {0x1D410, 4863, 1}, {0x1D411, 4864, 1}, {0x1D412, 4865, 1}, + {0x1D413, 4866, 1}, {0x1D414, 4867, 1}, {0x1D415, 4868, 1}, {0x1D416, 4869, 1}, + {0x1D417, 4870, 1}, {0x1D418, 4871, 1}, {0x1D419, 4872, 1}, {0x1D41A, 4873, 1}, + {0x1D41B, 4874, 1}, {0x1D41C, 4875, 1}, {0x1D41D, 4876, 1}, {0x1D41E, 4877, 1}, + {0x1D41F, 4878, 1}, {0x1D420, 4879, 1}, {0x1D421, 4880, 1}, {0x1D422, 4881, 1}, + {0x1D423, 4882, 1}, {0x1D424, 4883, 1}, {0x1D425, 4884, 1}, {0x1D426, 4885, 1}, + {0x1D427, 4886, 1}, {0x1D428, 4887, 1}, {0x1D429, 4888, 1}, {0x1D42A, 4889, 1}, + {0x1D42B, 4890, 1}, {0x1D42C, 4891, 1}, {0x1D42D, 4892, 1}, {0x1D42E, 4893, 1}, + {0x1D42F, 4894, 1}, {0x1D430, 4895, 1}, {0x1D431, 4896, 1}, {0x1D432, 4897, 1}, + {0x1D433, 4898, 1}, {0x1D434, 4899, 1}, {0x1D435, 4900, 1}, {0x1D436, 4901, 1}, + {0x1D437, 4902, 1}, {0x1D438, 4903, 1}, {0x1D439, 4904, 1}, {0x1D43A, 4905, 1}, + {0x1D43B, 4906, 1}, {0x1D43C, 4907, 1}, {0x1D43D, 4908, 1}, {0x1D43E, 4909, 1}, + {0x1D43F, 4910, 1}, {0x1D440, 4911, 1}, {0x1D441, 4912, 1}, {0x1D442, 4913, 1}, + {0x1D443, 4914, 1}, {0x1D444, 4915, 1}, {0x1D445, 4916, 1}, {0x1D446, 4917, 1}, + {0x1D447, 4918, 1}, {0x1D448, 4919, 1}, {0x1D449, 4920, 1}, {0x1D44A, 4921, 1}, + {0x1D44B, 4922, 1}, {0x1D44C, 4923, 1}, {0x1D44D, 4924, 1}, {0x1D44E, 4925, 1}, + {0x1D44F, 4926, 1}, {0x1D450, 4927, 1}, {0x1D451, 4928, 1}, {0x1D452, 4929, 1}, + {0x1D453, 4930, 1}, {0x1D454, 4931, 1}, {0x1D456, 4932, 1}, {0x1D457, 4933, 1}, + {0x1D458, 4934, 1}, {0x1D459, 4935, 1}, {0x1D45A, 4936, 1}, {0x1D45B, 4937, 1}, + {0x1D45C, 4938, 1}, {0x1D45D, 4939, 1}, {0x1D45E, 4940, 1}, {0x1D45F, 4941, 1}, + {0x1D460, 4942, 1}, {0x1D461, 4943, 1}, {0x1D462, 4944, 1}, {0x1D463, 4945, 1}, + {0x1D464, 4946, 1}, {0x1D465, 4947, 1}, {0x1D466, 4948, 1}, {0x1D467, 4949, 1}, + {0x1D468, 4950, 1}, {0x1D469, 4951, 1}, {0x1D46A, 4952, 1}, {0x1D46B, 4953, 1}, + {0x1D46C, 4954, 1}, {0x1D46D, 4955, 1}, {0x1D46E, 4956, 1}, {0x1D46F, 4957, 1}, + {0x1D470, 4958, 1}, {0x1D471, 4959, 1}, {0x1D472, 4960, 1}, {0x1D473, 4961, 1}, + {0x1D474, 4962, 1}, {0x1D475, 4963, 1}, {0x1D476, 4964, 1}, {0x1D477, 4965, 1}, + {0x1D478, 4966, 1}, {0x1D479, 4967, 1}, {0x1D47A, 4968, 1}, {0x1D47B, 4969, 1}, + {0x1D47C, 4970, 1}, {0x1D47D, 4971, 1}, {0x1D47E, 4972, 1}, {0x1D47F, 4973, 1}, + {0x1D480, 4974, 1}, {0x1D481, 4975, 1}, {0x1D482, 4976, 1}, {0x1D483, 4977, 1}, + {0x1D484, 4978, 1}, {0x1D485, 4979, 1}, {0x1D486, 4980, 1}, {0x1D487, 4981, 1}, + {0x1D488, 4982, 1}, {0x1D489, 4983, 1}, {0x1D48A, 4984, 1}, {0x1D48B, 4985, 1}, + {0x1D48C, 4986, 1}, {0x1D48D, 4987, 1}, {0x1D48E, 4988, 1}, {0x1D48F, 4989, 1}, + {0x1D490, 4990, 1}, {0x1D491, 4991, 1}, {0x1D492, 4992, 1}, {0x1D493, 4993, 1}, + {0x1D494, 4994, 1}, {0x1D495, 4995, 1}, {0x1D496, 4996, 1}, {0x1D497, 4997, 1}, + {0x1D498, 4998, 1}, {0x1D499, 4999, 1}, {0x1D49A, 5000, 1}, {0x1D49B, 5001, 1}, + {0x1D49C, 5002, 1}, {0x1D49E, 5003, 1}, {0x1D49F, 5004, 1}, {0x1D4A2, 5005, 1}, + {0x1D4A5, 5006, 1}, {0x1D4A6, 5007, 1}, {0x1D4A9, 5008, 1}, {0x1D4AA, 5009, 1}, + {0x1D4AB, 5010, 1}, {0x1D4AC, 5011, 1}, {0x1D4AE, 5012, 1}, {0x1D4AF, 5013, 1}, + {0x1D4B0, 5014, 1}, {0x1D4B1, 5015, 1}, {0x1D4B2, 5016, 1}, {0x1D4B3, 5017, 1}, + {0x1D4B4, 5018, 1}, {0x1D4B5, 5019, 1}, {0x1D4B6, 5020, 1}, {0x1D4B7, 5021, 1}, + {0x1D4B8, 5022, 1}, {0x1D4B9, 5023, 1}, {0x1D4BB, 5024, 1}, {0x1D4BD, 5025, 1}, + {0x1D4BE, 5026, 1}, {0x1D4BF, 5027, 1}, {0x1D4C0, 5028, 1}, {0x1D4C1, 5029, 1}, + {0x1D4C2, 5030, 1}, {0x1D4C3, 5031, 1}, {0x1D4C5, 5032, 1}, {0x1D4C6, 5033, 1}, + {0x1D4C7, 5034, 1}, {0x1D4C8, 5035, 1}, {0x1D4C9, 5036, 1}, {0x1D4CA, 5037, 1}, + {0x1D4CB, 5038, 1}, {0x1D4CC, 5039, 1}, {0x1D4CD, 5040, 1}, {0x1D4CE, 5041, 1}, + {0x1D4CF, 5042, 1}, {0x1D4D0, 5043, 1}, {0x1D4D1, 5044, 1}, {0x1D4D2, 5045, 1}, + {0x1D4D3, 5046, 1}, {0x1D4D4, 5047, 1}, {0x1D4D5, 5048, 1}, {0x1D4D6, 5049, 1}, + {0x1D4D7, 5050, 1}, {0x1D4D8, 5051, 1}, {0x1D4D9, 5052, 1}, {0x1D4DA, 5053, 1}, + {0x1D4DB, 5054, 1}, {0x1D4DC, 5055, 1}, {0x1D4DD, 5056, 1}, {0x1D4DE, 5057, 1}, + {0x1D4DF, 5058, 1}, {0x1D4E0, 5059, 1}, {0x1D4E1, 5060, 1}, {0x1D4E2, 5061, 1}, + {0x1D4E3, 5062, 1}, {0x1D4E4, 5063, 1}, {0x1D4E5, 5064, 1}, {0x1D4E6, 5065, 1}, + {0x1D4E7, 5066, 1}, {0x1D4E8, 5067, 1}, {0x1D4E9, 5068, 1}, {0x1D4EA, 5069, 1}, + {0x1D4EB, 5070, 1}, {0x1D4EC, 5071, 1}, {0x1D4ED, 5072, 1}, {0x1D4EE, 5073, 1}, + {0x1D4EF, 5074, 1}, {0x1D4F0, 5075, 1}, {0x1D4F1, 5076, 1}, {0x1D4F2, 5077, 1}, + {0x1D4F3, 5078, 1}, {0x1D4F4, 5079, 1}, {0x1D4F5, 5080, 1}, {0x1D4F6, 5081, 1}, + {0x1D4F7, 5082, 1}, {0x1D4F8, 5083, 1}, {0x1D4F9, 5084, 1}, {0x1D4FA, 5085, 1}, + {0x1D4FB, 5086, 1}, {0x1D4FC, 5087, 1}, {0x1D4FD, 5088, 1}, {0x1D4FE, 5089, 1}, + {0x1D4FF, 5090, 1}, {0x1D500, 5091, 1}, {0x1D501, 5092, 1}, {0x1D502, 5093, 1}, + {0x1D503, 5094, 1}, {0x1D504, 5095, 1}, {0x1D505, 5096, 1}, {0x1D507, 5097, 1}, + {0x1D508, 5098, 1}, {0x1D509, 5099, 1}, {0x1D50A, 5100, 1}, {0x1D50D, 5101, 1}, + {0x1D50E, 5102, 1}, {0x1D50F, 5103, 1}, {0x1D510, 5104, 1}, {0x1D511, 5105, 1}, + {0x1D512, 5106, 1}, {0x1D513, 5107, 1}, {0x1D514, 5108, 1}, {0x1D516, 5109, 1}, + {0x1D517, 5110, 1}, {0x1D518, 5111, 1}, {0x1D519, 5112, 1}, {0x1D51A, 5113, 1}, + {0x1D51B, 5114, 1}, {0x1D51C, 5115, 1}, {0x1D51E, 5116, 1}, {0x1D51F, 5117, 1}, + {0x1D520, 5118, 1}, {0x1D521, 5119, 1}, {0x1D522, 5120, 1}, {0x1D523, 5121, 1}, + {0x1D524, 5122, 1}, {0x1D525, 5123, 1}, {0x1D526, 5124, 1}, {0x1D527, 5125, 1}, + {0x1D528, 5126, 1}, {0x1D529, 5127, 1}, {0x1D52A, 5128, 1}, {0x1D52B, 5129, 1}, + {0x1D52C, 5130, 1}, {0x1D52D, 5131, 1}, {0x1D52E, 5132, 1}, {0x1D52F, 5133, 1}, + {0x1D530, 5134, 1}, {0x1D531, 5135, 1}, {0x1D532, 5136, 1}, {0x1D533, 5137, 1}, + {0x1D534, 5138, 1}, {0x1D535, 5139, 1}, {0x1D536, 5140, 1}, {0x1D537, 5141, 1}, + {0x1D538, 5142, 1}, {0x1D539, 5143, 1}, {0x1D53B, 5144, 1}, {0x1D53C, 5145, 1}, + {0x1D53D, 5146, 1}, {0x1D53E, 5147, 1}, {0x1D540, 5148, 1}, {0x1D541, 5149, 1}, + {0x1D542, 5150, 1}, {0x1D543, 5151, 1}, {0x1D544, 5152, 1}, {0x1D546, 5153, 1}, + {0x1D54A, 5154, 1}, {0x1D54B, 5155, 1}, {0x1D54C, 5156, 1}, {0x1D54D, 5157, 1}, + {0x1D54E, 5158, 1}, {0x1D54F, 5159, 1}, {0x1D550, 5160, 1}, {0x1D552, 5161, 1}, + {0x1D553, 5162, 1}, {0x1D554, 5163, 1}, {0x1D555, 5164, 1}, {0x1D556, 5165, 1}, + {0x1D557, 5166, 1}, {0x1D558, 5167, 1}, {0x1D559, 5168, 1}, {0x1D55A, 5169, 1}, + {0x1D55B, 5170, 1}, {0x1D55C, 5171, 1}, {0x1D55D, 5172, 1}, {0x1D55E, 5173, 1}, + {0x1D55F, 5174, 1}, {0x1D560, 5175, 1}, {0x1D561, 5176, 1}, {0x1D562, 5177, 1}, + {0x1D563, 5178, 1}, {0x1D564, 5179, 1}, {0x1D565, 5180, 1}, {0x1D566, 5181, 1}, + {0x1D567, 5182, 1}, {0x1D568, 5183, 1}, {0x1D569, 5184, 1}, {0x1D56A, 5185, 1}, + {0x1D56B, 5186, 1}, {0x1D56C, 5187, 1}, {0x1D56D, 5188, 1}, {0x1D56E, 5189, 1}, + {0x1D56F, 5190, 1}, {0x1D570, 5191, 1}, {0x1D571, 5192, 1}, {0x1D572, 5193, 1}, + {0x1D573, 5194, 1}, {0x1D574, 5195, 1}, {0x1D575, 5196, 1}, {0x1D576, 5197, 1}, + {0x1D577, 5198, 1}, {0x1D578, 5199, 1}, {0x1D579, 5200, 1}, {0x1D57A, 5201, 1}, + {0x1D57B, 5202, 1}, {0x1D57C, 5203, 1}, {0x1D57D, 5204, 1}, {0x1D57E, 5205, 1}, + {0x1D57F, 5206, 1}, {0x1D580, 5207, 1}, {0x1D581, 5208, 1}, {0x1D582, 5209, 1}, + {0x1D583, 5210, 1}, {0x1D584, 5211, 1}, {0x1D585, 5212, 1}, {0x1D586, 5213, 1}, + {0x1D587, 5214, 1}, {0x1D588, 5215, 1}, {0x1D589, 5216, 1}, {0x1D58A, 5217, 1}, + {0x1D58B, 5218, 1}, {0x1D58C, 5219, 1}, {0x1D58D, 5220, 1}, {0x1D58E, 5221, 1}, + {0x1D58F, 5222, 1}, {0x1D590, 5223, 1}, {0x1D591, 5224, 1}, {0x1D592, 5225, 1}, + {0x1D593, 5226, 1}, {0x1D594, 5227, 1}, {0x1D595, 5228, 1}, {0x1D596, 5229, 1}, + {0x1D597, 5230, 1}, {0x1D598, 5231, 1}, {0x1D599, 5232, 1}, {0x1D59A, 5233, 1}, + {0x1D59B, 5234, 1}, {0x1D59C, 5235, 1}, {0x1D59D, 5236, 1}, {0x1D59E, 5237, 1}, + {0x1D59F, 5238, 1}, {0x1D5A0, 5239, 1}, {0x1D5A1, 5240, 1}, {0x1D5A2, 5241, 1}, + {0x1D5A3, 5242, 1}, {0x1D5A4, 5243, 1}, {0x1D5A5, 5244, 1}, {0x1D5A6, 5245, 1}, + {0x1D5A7, 5246, 1}, {0x1D5A8, 5247, 1}, {0x1D5A9, 5248, 1}, {0x1D5AA, 5249, 1}, + {0x1D5AB, 5250, 1}, {0x1D5AC, 5251, 1}, {0x1D5AD, 5252, 1}, {0x1D5AE, 5253, 1}, + {0x1D5AF, 5254, 1}, {0x1D5B0, 5255, 1}, {0x1D5B1, 5256, 1}, {0x1D5B2, 5257, 1}, + {0x1D5B3, 5258, 1}, {0x1D5B4, 5259, 1}, {0x1D5B5, 5260, 1}, {0x1D5B6, 5261, 1}, + {0x1D5B7, 5262, 1}, {0x1D5B8, 5263, 1}, {0x1D5B9, 5264, 1}, {0x1D5BA, 5265, 1}, + {0x1D5BB, 5266, 1}, {0x1D5BC, 5267, 1}, {0x1D5BD, 5268, 1}, {0x1D5BE, 5269, 1}, + {0x1D5BF, 5270, 1}, {0x1D5C0, 5271, 1}, {0x1D5C1, 5272, 1}, {0x1D5C2, 5273, 1}, + {0x1D5C3, 5274, 1}, {0x1D5C4, 5275, 1}, {0x1D5C5, 5276, 1}, {0x1D5C6, 5277, 1}, + {0x1D5C7, 5278, 1}, {0x1D5C8, 5279, 1}, {0x1D5C9, 5280, 1}, {0x1D5CA, 5281, 1}, + {0x1D5CB, 5282, 1}, {0x1D5CC, 5283, 1}, {0x1D5CD, 5284, 1}, {0x1D5CE, 5285, 1}, + {0x1D5CF, 5286, 1}, {0x1D5D0, 5287, 1}, {0x1D5D1, 5288, 1}, {0x1D5D2, 5289, 1}, + {0x1D5D3, 5290, 1}, {0x1D5D4, 5291, 1}, {0x1D5D5, 5292, 1}, {0x1D5D6, 5293, 1}, + {0x1D5D7, 5294, 1}, {0x1D5D8, 5295, 1}, {0x1D5D9, 5296, 1}, {0x1D5DA, 5297, 1}, + {0x1D5DB, 5298, 1}, {0x1D5DC, 5299, 1}, {0x1D5DD, 5300, 1}, {0x1D5DE, 5301, 1}, + {0x1D5DF, 5302, 1}, {0x1D5E0, 5303, 1}, {0x1D5E1, 5304, 1}, {0x1D5E2, 5305, 1}, + {0x1D5E3, 5306, 1}, {0x1D5E4, 5307, 1}, {0x1D5E5, 5308, 1}, {0x1D5E6, 5309, 1}, + {0x1D5E7, 5310, 1}, {0x1D5E8, 5311, 1}, {0x1D5E9, 5312, 1}, {0x1D5EA, 5313, 1}, + {0x1D5EB, 5314, 1}, {0x1D5EC, 5315, 1}, {0x1D5ED, 5316, 1}, {0x1D5EE, 5317, 1}, + {0x1D5EF, 5318, 1}, {0x1D5F0, 5319, 1}, {0x1D5F1, 5320, 1}, {0x1D5F2, 5321, 1}, + {0x1D5F3, 5322, 1}, {0x1D5F4, 5323, 1}, {0x1D5F5, 5324, 1}, {0x1D5F6, 5325, 1}, + {0x1D5F7, 5326, 1}, {0x1D5F8, 5327, 1}, {0x1D5F9, 5328, 1}, {0x1D5FA, 5329, 1}, + {0x1D5FB, 5330, 1}, {0x1D5FC, 5331, 1}, {0x1D5FD, 5332, 1}, {0x1D5FE, 5333, 1}, + {0x1D5FF, 5334, 1}, {0x1D600, 5335, 1}, {0x1D601, 5336, 1}, {0x1D602, 5337, 1}, + {0x1D603, 5338, 1}, {0x1D604, 5339, 1}, {0x1D605, 5340, 1}, {0x1D606, 5341, 1}, + {0x1D607, 5342, 1}, {0x1D608, 5343, 1}, {0x1D609, 5344, 1}, {0x1D60A, 5345, 1}, + {0x1D60B, 5346, 1}, {0x1D60C, 5347, 1}, {0x1D60D, 5348, 1}, {0x1D60E, 5349, 1}, + {0x1D60F, 5350, 1}, {0x1D610, 5351, 1}, {0x1D611, 5352, 1}, {0x1D612, 5353, 1}, + {0x1D613, 5354, 1}, {0x1D614, 5355, 1}, {0x1D615, 5356, 1}, {0x1D616, 5357, 1}, + {0x1D617, 5358, 1}, {0x1D618, 5359, 1}, {0x1D619, 5360, 1}, {0x1D61A, 5361, 1}, + {0x1D61B, 5362, 1}, {0x1D61C, 5363, 1}, {0x1D61D, 5364, 1}, {0x1D61E, 5365, 1}, + {0x1D61F, 5366, 1}, {0x1D620, 5367, 1}, {0x1D621, 5368, 1}, {0x1D622, 5369, 1}, + {0x1D623, 5370, 1}, {0x1D624, 5371, 1}, {0x1D625, 5372, 1}, {0x1D626, 5373, 1}, + {0x1D627, 5374, 1}, {0x1D628, 5375, 1}, {0x1D629, 5376, 1}, {0x1D62A, 5377, 1}, + {0x1D62B, 5378, 1}, {0x1D62C, 5379, 1}, {0x1D62D, 5380, 1}, {0x1D62E, 5381, 1}, + {0x1D62F, 5382, 1}, {0x1D630, 5383, 1}, {0x1D631, 5384, 1}, {0x1D632, 5385, 1}, + {0x1D633, 5386, 1}, {0x1D634, 5387, 1}, {0x1D635, 5388, 1}, {0x1D636, 5389, 1}, + {0x1D637, 5390, 1}, {0x1D638, 5391, 1}, {0x1D639, 5392, 1}, {0x1D63A, 5393, 1}, + {0x1D63B, 5394, 1}, {0x1D63C, 5395, 1}, {0x1D63D, 5396, 1}, {0x1D63E, 5397, 1}, + {0x1D63F, 5398, 1}, {0x1D640, 5399, 1}, {0x1D641, 5400, 1}, {0x1D642, 5401, 1}, + {0x1D643, 5402, 1}, {0x1D644, 5403, 1}, {0x1D645, 5404, 1}, {0x1D646, 5405, 1}, + {0x1D647, 5406, 1}, {0x1D648, 5407, 1}, {0x1D649, 5408, 1}, {0x1D64A, 5409, 1}, + {0x1D64B, 5410, 1}, {0x1D64C, 5411, 1}, {0x1D64D, 5412, 1}, {0x1D64E, 5413, 1}, + {0x1D64F, 5414, 1}, {0x1D650, 5415, 1}, {0x1D651, 5416, 1}, {0x1D652, 5417, 1}, + {0x1D653, 5418, 1}, {0x1D654, 5419, 1}, {0x1D655, 5420, 1}, {0x1D656, 5421, 1}, + {0x1D657, 5422, 1}, {0x1D658, 5423, 1}, {0x1D659, 5424, 1}, {0x1D65A, 5425, 1}, + {0x1D65B, 5426, 1}, {0x1D65C, 5427, 1}, {0x1D65D, 5428, 1}, {0x1D65E, 5429, 1}, + {0x1D65F, 5430, 1}, {0x1D660, 5431, 1}, {0x1D661, 5432, 1}, {0x1D662, 5433, 1}, + {0x1D663, 5434, 1}, {0x1D664, 5435, 1}, {0x1D665, 5436, 1}, {0x1D666, 5437, 1}, + {0x1D667, 5438, 1}, {0x1D668, 5439, 1}, {0x1D669, 5440, 1}, {0x1D66A, 5441, 1}, + {0x1D66B, 5442, 1}, {0x1D66C, 5443, 1}, {0x1D66D, 5444, 1}, {0x1D66E, 5445, 1}, + {0x1D66F, 5446, 1}, {0x1D670, 5447, 1}, {0x1D671, 5448, 1}, {0x1D672, 5449, 1}, + {0x1D673, 5450, 1}, {0x1D674, 5451, 1}, {0x1D675, 5452, 1}, {0x1D676, 5453, 1}, + {0x1D677, 5454, 1}, {0x1D678, 5455, 1}, {0x1D679, 5456, 1}, {0x1D67A, 5457, 1}, + {0x1D67B, 5458, 1}, {0x1D67C, 5459, 1}, {0x1D67D, 5460, 1}, {0x1D67E, 5461, 1}, + {0x1D67F, 5462, 1}, {0x1D680, 5463, 1}, {0x1D681, 5464, 1}, {0x1D682, 5465, 1}, + {0x1D683, 5466, 1}, {0x1D684, 5467, 1}, {0x1D685, 5468, 1}, {0x1D686, 5469, 1}, + {0x1D687, 5470, 1}, {0x1D688, 5471, 1}, {0x1D689, 5472, 1}, {0x1D68A, 5473, 1}, + {0x1D68B, 5474, 1}, {0x1D68C, 5475, 1}, {0x1D68D, 5476, 1}, {0x1D68E, 5477, 1}, + {0x1D68F, 5478, 1}, {0x1D690, 5479, 1}, {0x1D691, 5480, 1}, {0x1D692, 5481, 1}, + {0x1D693, 5482, 1}, {0x1D694, 5483, 1}, {0x1D695, 5484, 1}, {0x1D696, 5485, 1}, + {0x1D697, 5486, 1}, {0x1D698, 5487, 1}, {0x1D699, 5488, 1}, {0x1D69A, 5489, 1}, + {0x1D69B, 5490, 1}, {0x1D69C, 5491, 1}, {0x1D69D, 5492, 1}, {0x1D69E, 5493, 1}, + {0x1D69F, 5494, 1}, {0x1D6A0, 5495, 1}, {0x1D6A1, 5496, 1}, {0x1D6A2, 5497, 1}, + {0x1D6A3, 5498, 1}, {0x1D6A4, 5499, 1}, {0x1D6A5, 5500, 1}, {0x1D6A8, 5501, 1}, + {0x1D6A9, 5502, 1}, {0x1D6AA, 5503, 1}, {0x1D6AB, 5504, 1}, {0x1D6AC, 5505, 1}, + {0x1D6AD, 5506, 1}, {0x1D6AE, 5507, 1}, {0x1D6AF, 5508, 1}, {0x1D6B0, 5509, 1}, + {0x1D6B1, 5510, 1}, {0x1D6B2, 5511, 1}, {0x1D6B3, 5512, 1}, {0x1D6B4, 5513, 1}, + {0x1D6B5, 5514, 1}, {0x1D6B6, 5515, 1}, {0x1D6B7, 5516, 1}, {0x1D6B8, 5517, 1}, + {0x1D6B9, 5518, 1}, {0x1D6BA, 5519, 1}, {0x1D6BB, 5520, 1}, {0x1D6BC, 5521, 1}, + {0x1D6BD, 5522, 1}, {0x1D6BE, 5523, 1}, {0x1D6BF, 5524, 1}, {0x1D6C0, 5525, 1}, + {0x1D6C1, 5526, 1}, {0x1D6C2, 5527, 1}, {0x1D6C3, 5528, 1}, {0x1D6C4, 5529, 1}, + {0x1D6C5, 5530, 1}, {0x1D6C6, 5531, 1}, {0x1D6C7, 5532, 1}, {0x1D6C8, 5533, 1}, + {0x1D6C9, 5534, 1}, {0x1D6CA, 5535, 1}, {0x1D6CB, 5536, 1}, {0x1D6CC, 5537, 1}, + {0x1D6CD, 5538, 1}, {0x1D6CE, 5539, 1}, {0x1D6CF, 5540, 1}, {0x1D6D0, 5541, 1}, + {0x1D6D1, 5542, 1}, {0x1D6D2, 5543, 1}, {0x1D6D3, 5544, 1}, {0x1D6D4, 5545, 1}, + {0x1D6D5, 5546, 1}, {0x1D6D6, 5547, 1}, {0x1D6D7, 5548, 1}, {0x1D6D8, 5549, 1}, + {0x1D6D9, 5550, 1}, {0x1D6DA, 5551, 1}, {0x1D6DB, 5552, 1}, {0x1D6DC, 5553, 1}, + {0x1D6DD, 5554, 1}, {0x1D6DE, 5555, 1}, {0x1D6DF, 5556, 1}, {0x1D6E0, 5557, 1}, + {0x1D6E1, 5558, 1}, {0x1D6E2, 5559, 1}, {0x1D6E3, 5560, 1}, {0x1D6E4, 5561, 1}, + {0x1D6E5, 5562, 1}, {0x1D6E6, 5563, 1}, {0x1D6E7, 5564, 1}, {0x1D6E8, 5565, 1}, + {0x1D6E9, 5566, 1}, {0x1D6EA, 5567, 1}, {0x1D6EB, 5568, 1}, {0x1D6EC, 5569, 1}, + {0x1D6ED, 5570, 1}, {0x1D6EE, 5571, 1}, {0x1D6EF, 5572, 1}, {0x1D6F0, 5573, 1}, + {0x1D6F1, 5574, 1}, {0x1D6F2, 5575, 1}, {0x1D6F3, 5576, 1}, {0x1D6F4, 5577, 1}, + {0x1D6F5, 5578, 1}, {0x1D6F6, 5579, 1}, {0x1D6F7, 5580, 1}, {0x1D6F8, 5581, 1}, + {0x1D6F9, 5582, 1}, {0x1D6FA, 5583, 1}, {0x1D6FB, 5584, 1}, {0x1D6FC, 5585, 1}, + {0x1D6FD, 5586, 1}, {0x1D6FE, 5587, 1}, {0x1D6FF, 5588, 1}, {0x1D700, 5589, 1}, + {0x1D701, 5590, 1}, {0x1D702, 5591, 1}, {0x1D703, 5592, 1}, {0x1D704, 5593, 1}, + {0x1D705, 5594, 1}, {0x1D706, 5595, 1}, {0x1D707, 5596, 1}, {0x1D708, 5597, 1}, + {0x1D709, 5598, 1}, {0x1D70A, 5599, 1}, {0x1D70B, 5600, 1}, {0x1D70C, 5601, 1}, + {0x1D70D, 5602, 1}, {0x1D70E, 5603, 1}, {0x1D70F, 5604, 1}, {0x1D710, 5605, 1}, + {0x1D711, 5606, 1}, {0x1D712, 5607, 1}, {0x1D713, 5608, 1}, {0x1D714, 5609, 1}, + {0x1D715, 5610, 1}, {0x1D716, 5611, 1}, {0x1D717, 5612, 1}, {0x1D718, 5613, 1}, + {0x1D719, 5614, 1}, {0x1D71A, 5615, 1}, {0x1D71B, 5616, 1}, {0x1D71C, 5617, 1}, + {0x1D71D, 5618, 1}, {0x1D71E, 5619, 1}, {0x1D71F, 5620, 1}, {0x1D720, 5621, 1}, + {0x1D721, 5622, 1}, {0x1D722, 5623, 1}, {0x1D723, 5624, 1}, {0x1D724, 5625, 1}, + {0x1D725, 5626, 1}, {0x1D726, 5627, 1}, {0x1D727, 5628, 1}, {0x1D728, 5629, 1}, + {0x1D729, 5630, 1}, {0x1D72A, 5631, 1}, {0x1D72B, 5632, 1}, {0x1D72C, 5633, 1}, + {0x1D72D, 5634, 1}, {0x1D72E, 5635, 1}, {0x1D72F, 5636, 1}, {0x1D730, 5637, 1}, + {0x1D731, 5638, 1}, {0x1D732, 5639, 1}, {0x1D733, 5640, 1}, {0x1D734, 5641, 1}, + {0x1D735, 5642, 1}, {0x1D736, 5643, 1}, {0x1D737, 5644, 1}, {0x1D738, 5645, 1}, + {0x1D739, 5646, 1}, {0x1D73A, 5647, 1}, {0x1D73B, 5648, 1}, {0x1D73C, 5649, 1}, + {0x1D73D, 5650, 1}, {0x1D73E, 5651, 1}, {0x1D73F, 5652, 1}, {0x1D740, 5653, 1}, + {0x1D741, 5654, 1}, {0x1D742, 5655, 1}, {0x1D743, 5656, 1}, {0x1D744, 5657, 1}, + {0x1D745, 5658, 1}, {0x1D746, 5659, 1}, {0x1D747, 5660, 1}, {0x1D748, 5661, 1}, + {0x1D749, 5662, 1}, {0x1D74A, 5663, 1}, {0x1D74B, 5664, 1}, {0x1D74C, 5665, 1}, + {0x1D74D, 5666, 1}, {0x1D74E, 5667, 1}, {0x1D74F, 5668, 1}, {0x1D750, 5669, 1}, + {0x1D751, 5670, 1}, {0x1D752, 5671, 1}, {0x1D753, 5672, 1}, {0x1D754, 5673, 1}, + {0x1D755, 5674, 1}, {0x1D756, 5675, 1}, {0x1D757, 5676, 1}, {0x1D758, 5677, 1}, + {0x1D759, 5678, 1}, {0x1D75A, 5679, 1}, {0x1D75B, 5680, 1}, {0x1D75C, 5681, 1}, + {0x1D75D, 5682, 1}, {0x1D75E, 5683, 1}, {0x1D75F, 5684, 1}, {0x1D760, 5685, 1}, + {0x1D761, 5686, 1}, {0x1D762, 5687, 1}, {0x1D763, 5688, 1}, {0x1D764, 5689, 1}, + {0x1D765, 5690, 1}, {0x1D766, 5691, 1}, {0x1D767, 5692, 1}, {0x1D768, 5693, 1}, + {0x1D769, 5694, 1}, {0x1D76A, 5695, 1}, {0x1D76B, 5696, 1}, {0x1D76C, 5697, 1}, + {0x1D76D, 5698, 1}, {0x1D76E, 5699, 1}, {0x1D76F, 5700, 1}, {0x1D770, 5701, 1}, + {0x1D771, 5702, 1}, {0x1D772, 5703, 1}, {0x1D773, 5704, 1}, {0x1D774, 5705, 1}, + {0x1D775, 5706, 1}, {0x1D776, 5707, 1}, {0x1D777, 5708, 1}, {0x1D778, 5709, 1}, + {0x1D779, 5710, 1}, {0x1D77A, 5711, 1}, {0x1D77B, 5712, 1}, {0x1D77C, 5713, 1}, + {0x1D77D, 5714, 1}, {0x1D77E, 5715, 1}, {0x1D77F, 5716, 1}, {0x1D780, 5717, 1}, + {0x1D781, 5718, 1}, {0x1D782, 5719, 1}, {0x1D783, 5720, 1}, {0x1D784, 5721, 1}, + {0x1D785, 5722, 1}, {0x1D786, 5723, 1}, {0x1D787, 5724, 1}, {0x1D788, 5725, 1}, + {0x1D789, 5726, 1}, {0x1D78A, 5727, 1}, {0x1D78B, 5728, 1}, {0x1D78C, 5729, 1}, + {0x1D78D, 5730, 1}, {0x1D78E, 5731, 1}, {0x1D78F, 5732, 1}, {0x1D790, 5733, 1}, + {0x1D791, 5734, 1}, {0x1D792, 5735, 1}, {0x1D793, 5736, 1}, {0x1D794, 5737, 1}, + {0x1D795, 5738, 1}, {0x1D796, 5739, 1}, {0x1D797, 5740, 1}, {0x1D798, 5741, 1}, + {0x1D799, 5742, 1}, {0x1D79A, 5743, 1}, {0x1D79B, 5744, 1}, {0x1D79C, 5745, 1}, + {0x1D79D, 5746, 1}, {0x1D79E, 5747, 1}, {0x1D79F, 5748, 1}, {0x1D7A0, 5749, 1}, + {0x1D7A1, 5750, 1}, {0x1D7A2, 5751, 1}, {0x1D7A3, 5752, 1}, {0x1D7A4, 5753, 1}, + {0x1D7A5, 5754, 1}, {0x1D7A6, 5755, 1}, {0x1D7A7, 5756, 1}, {0x1D7A8, 5757, 1}, + {0x1D7A9, 5758, 1}, {0x1D7AA, 5759, 1}, {0x1D7AB, 5760, 1}, {0x1D7AC, 5761, 1}, + {0x1D7AD, 5762, 1}, {0x1D7AE, 5763, 1}, {0x1D7AF, 5764, 1}, {0x1D7B0, 5765, 1}, + {0x1D7B1, 5766, 1}, {0x1D7B2, 5767, 1}, {0x1D7B3, 5768, 1}, {0x1D7B4, 5769, 1}, + {0x1D7B5, 5770, 1}, {0x1D7B6, 5771, 1}, {0x1D7B7, 5772, 1}, {0x1D7B8, 5773, 1}, + {0x1D7B9, 5774, 1}, {0x1D7BA, 5775, 1}, {0x1D7BB, 5776, 1}, {0x1D7BC, 5777, 1}, + {0x1D7BD, 5778, 1}, {0x1D7BE, 5779, 1}, {0x1D7BF, 5780, 1}, {0x1D7C0, 5781, 1}, + {0x1D7C1, 5782, 1}, {0x1D7C2, 5783, 1}, {0x1D7C3, 5784, 1}, {0x1D7C4, 5785, 1}, + {0x1D7C5, 5786, 1}, {0x1D7C6, 5787, 1}, {0x1D7C7, 5788, 1}, {0x1D7C8, 5789, 1}, + {0x1D7C9, 5790, 1}, {0x1D7CA, 5791, 1}, {0x1D7CB, 5792, 1}, {0x1D7CE, 5793, 1}, + {0x1D7CF, 5794, 1}, {0x1D7D0, 5795, 1}, {0x1D7D1, 5796, 1}, {0x1D7D2, 5797, 1}, + {0x1D7D3, 5798, 1}, {0x1D7D4, 5799, 1}, {0x1D7D5, 5800, 1}, {0x1D7D6, 5801, 1}, + {0x1D7D7, 5802, 1}, {0x1D7D8, 5803, 1}, {0x1D7D9, 5804, 1}, {0x1D7DA, 5805, 1}, + {0x1D7DB, 5806, 1}, {0x1D7DC, 5807, 1}, {0x1D7DD, 5808, 1}, {0x1D7DE, 5809, 1}, + {0x1D7DF, 5810, 1}, {0x1D7E0, 5811, 1}, {0x1D7E1, 5812, 1}, {0x1D7E2, 5813, 1}, + {0x1D7E3, 5814, 1}, {0x1D7E4, 5815, 1}, {0x1D7E5, 5816, 1}, {0x1D7E6, 5817, 1}, + {0x1D7E7, 5818, 1}, {0x1D7E8, 5819, 1}, {0x1D7E9, 5820, 1}, {0x1D7EA, 5821, 1}, + {0x1D7EB, 5822, 1}, {0x1D7EC, 5823, 1}, {0x1D7ED, 5824, 1}, {0x1D7EE, 5825, 1}, + {0x1D7EF, 5826, 1}, {0x1D7F0, 5827, 1}, {0x1D7F1, 5828, 1}, {0x1D7F2, 5829, 1}, + {0x1D7F3, 5830, 1}, {0x1D7F4, 5831, 1}, {0x1D7F5, 5832, 1}, {0x1D7F6, 5833, 1}, + {0x1D7F7, 5834, 1}, {0x1D7F8, 5835, 1}, {0x1D7F9, 5836, 1}, {0x1D7FA, 5837, 1}, + {0x1D7FB, 5838, 1}, {0x1D7FC, 5839, 1}, {0x1D7FD, 5840, 1}, {0x1D7FE, 5841, 1}, + {0x1D7FF, 5842, 1}, {0x1E030, 5843, 1}, {0x1E031, 5844, 1}, {0x1E032, 5845, 1}, + {0x1E033, 5846, 1}, {0x1E034, 5847, 1}, {0x1E035, 5848, 1}, {0x1E036, 5849, 1}, + {0x1E037, 5850, 1}, {0x1E038, 5851, 1}, {0x1E039, 5852, 1}, {0x1E03A, 5853, 1}, + {0x1E03B, 5854, 1}, {0x1E03C, 5855, 1}, {0x1E03D, 5856, 1}, {0x1E03E, 5857, 1}, + {0x1E03F, 5858, 1}, {0x1E040, 5859, 1}, {0x1E041, 5860, 1}, {0x1E042, 5861, 1}, + {0x1E043, 5862, 1}, {0x1E044, 5863, 1}, {0x1E045, 5864, 1}, {0x1E046, 5865, 1}, + {0x1E047, 5866, 1}, {0x1E048, 5867, 1}, {0x1E049, 5868, 1}, {0x1E04A, 5869, 1}, + {0x1E04B, 5870, 1}, {0x1E04C, 5871, 1}, {0x1E04D, 5872, 1}, {0x1E04E, 5873, 1}, + {0x1E04F, 5874, 1}, {0x1E050, 5875, 1}, {0x1E051, 5876, 1}, {0x1E052, 5877, 1}, + {0x1E053, 5878, 1}, {0x1E054, 5879, 1}, {0x1E055, 5880, 1}, {0x1E056, 5881, 1}, + {0x1E057, 5882, 1}, {0x1E058, 5883, 1}, {0x1E059, 5884, 1}, {0x1E05A, 5885, 1}, + {0x1E05B, 5886, 1}, {0x1E05C, 5887, 1}, {0x1E05D, 5888, 1}, {0x1E05E, 5889, 1}, + {0x1E05F, 5890, 1}, {0x1E060, 5891, 1}, {0x1E061, 5892, 1}, {0x1E062, 5893, 1}, + {0x1E063, 5894, 1}, {0x1E064, 5895, 1}, {0x1E065, 5896, 1}, {0x1E066, 5897, 1}, + {0x1E067, 5898, 1}, {0x1E068, 5899, 1}, {0x1E069, 5900, 1}, {0x1E06A, 5901, 1}, + {0x1E06B, 5902, 1}, {0x1E06C, 5903, 1}, {0x1E06D, 5904, 1}, {0x1EE00, 5905, 1}, + {0x1EE01, 5906, 1}, {0x1EE02, 5907, 1}, {0x1EE03, 5908, 1}, {0x1EE05, 5909, 1}, + {0x1EE06, 5910, 1}, {0x1EE07, 5911, 1}, {0x1EE08, 5912, 1}, {0x1EE09, 5913, 1}, + {0x1EE0A, 5914, 1}, {0x1EE0B, 5915, 1}, {0x1EE0C, 5916, 1}, {0x1EE0D, 5917, 1}, + {0x1EE0E, 5918, 1}, {0x1EE0F, 5919, 1}, {0x1EE10, 5920, 1}, {0x1EE11, 5921, 1}, + {0x1EE12, 5922, 1}, {0x1EE13, 5923, 1}, {0x1EE14, 5924, 1}, {0x1EE15, 5925, 1}, + {0x1EE16, 5926, 1}, {0x1EE17, 5927, 1}, {0x1EE18, 5928, 1}, {0x1EE19, 5929, 1}, + {0x1EE1A, 5930, 1}, {0x1EE1B, 5931, 1}, {0x1EE1C, 5932, 1}, {0x1EE1D, 5933, 1}, + {0x1EE1E, 5934, 1}, {0x1EE1F, 5935, 1}, {0x1EE21, 5936, 1}, {0x1EE22, 5937, 1}, + {0x1EE24, 5938, 1}, {0x1EE27, 5939, 1}, {0x1EE29, 5940, 1}, {0x1EE2A, 5941, 1}, + {0x1EE2B, 5942, 1}, {0x1EE2C, 5943, 1}, {0x1EE2D, 5944, 1}, {0x1EE2E, 5945, 1}, + {0x1EE2F, 5946, 1}, {0x1EE30, 5947, 1}, {0x1EE31, 5948, 1}, {0x1EE32, 5949, 1}, + {0x1EE34, 5950, 1}, {0x1EE35, 5951, 1}, {0x1EE36, 5952, 1}, {0x1EE37, 5953, 1}, + {0x1EE39, 5954, 1}, {0x1EE3B, 5955, 1}, {0x1EE42, 5956, 1}, {0x1EE47, 5957, 1}, + {0x1EE49, 5958, 1}, {0x1EE4B, 5959, 1}, {0x1EE4D, 5960, 1}, {0x1EE4E, 5961, 1}, + {0x1EE4F, 5962, 1}, {0x1EE51, 5963, 1}, {0x1EE52, 5964, 1}, {0x1EE54, 5965, 1}, + {0x1EE57, 5966, 1}, {0x1EE59, 5967, 1}, {0x1EE5B, 5968, 1}, {0x1EE5D, 5969, 1}, + {0x1EE5F, 5970, 1}, {0x1EE61, 5971, 1}, {0x1EE62, 5972, 1}, {0x1EE64, 5973, 1}, + {0x1EE67, 5974, 1}, {0x1EE68, 5975, 1}, {0x1EE69, 5976, 1}, {0x1EE6A, 5977, 1}, + {0x1EE6C, 5978, 1}, {0x1EE6D, 5979, 1}, {0x1EE6E, 5980, 1}, {0x1EE6F, 5981, 1}, + {0x1EE70, 5982, 1}, {0x1EE71, 5983, 1}, {0x1EE72, 5984, 1}, {0x1EE74, 5985, 1}, + {0x1EE75, 5986, 1}, {0x1EE76, 5987, 1}, {0x1EE77, 5988, 1}, {0x1EE79, 5989, 1}, + {0x1EE7A, 5990, 1}, {0x1EE7B, 5991, 1}, {0x1EE7C, 5992, 1}, {0x1EE7E, 5993, 1}, + {0x1EE80, 5994, 1}, {0x1EE81, 5995, 1}, {0x1EE82, 5996, 1}, {0x1EE83, 5997, 1}, + {0x1EE84, 5998, 1}, {0x1EE85, 5999, 1}, {0x1EE86, 6000, 1}, {0x1EE87, 6001, 1}, + {0x1EE88, 6002, 1}, {0x1EE89, 6003, 1}, {0x1EE8B, 6004, 1}, {0x1EE8C, 6005, 1}, + {0x1EE8D, 6006, 1}, {0x1EE8E, 6007, 1}, {0x1EE8F, 6008, 1}, {0x1EE90, 6009, 1}, + {0x1EE91, 6010, 1}, {0x1EE92, 6011, 1}, {0x1EE93, 6012, 1}, {0x1EE94, 6013, 1}, + {0x1EE95, 6014, 1}, {0x1EE96, 6015, 1}, {0x1EE97, 6016, 1}, {0x1EE98, 6017, 1}, + {0x1EE99, 6018, 1}, {0x1EE9A, 6019, 1}, {0x1EE9B, 6020, 1}, {0x1EEA1, 6021, 1}, + {0x1EEA2, 6022, 1}, {0x1EEA3, 6023, 1}, {0x1EEA5, 6024, 1}, {0x1EEA6, 6025, 1}, + {0x1EEA7, 6026, 1}, {0x1EEA8, 6027, 1}, {0x1EEA9, 6028, 1}, {0x1EEAB, 6029, 1}, + {0x1EEAC, 6030, 1}, {0x1EEAD, 6031, 1}, {0x1EEAE, 6032, 1}, {0x1EEAF, 6033, 1}, + {0x1EEB0, 6034, 1}, {0x1EEB1, 6035, 1}, {0x1EEB2, 6036, 1}, {0x1EEB3, 6037, 1}, + {0x1EEB4, 6038, 1}, {0x1EEB5, 6039, 1}, {0x1EEB6, 6040, 1}, {0x1EEB7, 6041, 1}, + {0x1EEB8, 6042, 1}, {0x1EEB9, 6043, 1}, {0x1EEBA, 6044, 1}, {0x1EEBB, 6045, 1}, + {0x1F100, 6046, 2}, {0x1F101, 6048, 2}, {0x1F102, 6050, 2}, {0x1F103, 6052, 2}, + {0x1F104, 6054, 2}, {0x1F105, 6056, 2}, {0x1F106, 6058, 2}, {0x1F107, 6060, 2}, + {0x1F108, 6062, 2}, {0x1F109, 6064, 2}, {0x1F10A, 6066, 2}, {0x1F110, 6068, 3}, + {0x1F111, 6071, 3}, {0x1F112, 6074, 3}, {0x1F113, 6077, 3}, {0x1F114, 6080, 3}, + {0x1F115, 6083, 3}, {0x1F116, 6086, 3}, {0x1F117, 6089, 3}, {0x1F118, 6092, 3}, + {0x1F119, 6095, 3}, {0x1F11A, 6098, 3}, {0x1F11B, 6101, 3}, {0x1F11C, 6104, 3}, + {0x1F11D, 6107, 3}, {0x1F11E, 6110, 3}, {0x1F11F, 6113, 3}, {0x1F120, 6116, 3}, + {0x1F121, 6119, 3}, {0x1F122, 6122, 3}, {0x1F123, 6125, 3}, {0x1F124, 6128, 3}, + {0x1F125, 6131, 3}, {0x1F126, 6134, 3}, {0x1F127, 6137, 3}, {0x1F128, 6140, 3}, + {0x1F129, 6143, 3}, {0x1F12A, 6146, 3}, {0x1F12B, 6149, 1}, {0x1F12C, 6150, 1}, + {0x1F12D, 6151, 2}, {0x1F12E, 6153, 2}, {0x1F130, 6155, 1}, {0x1F131, 6156, 1}, + {0x1F132, 6157, 1}, {0x1F133, 6158, 1}, {0x1F134, 6159, 1}, {0x1F135, 6160, 1}, + {0x1F136, 6161, 1}, {0x1F137, 6162, 1}, {0x1F138, 6163, 1}, {0x1F139, 6164, 1}, + {0x1F13A, 6165, 1}, {0x1F13B, 6166, 1}, {0x1F13C, 6167, 1}, {0x1F13D, 6168, 1}, + {0x1F13E, 6169, 1}, {0x1F13F, 6170, 1}, {0x1F140, 6171, 1}, {0x1F141, 6172, 1}, + {0x1F142, 6173, 1}, {0x1F143, 6174, 1}, {0x1F144, 6175, 1}, {0x1F145, 6176, 1}, + {0x1F146, 6177, 1}, {0x1F147, 6178, 1}, {0x1F148, 6179, 1}, {0x1F149, 6180, 1}, + {0x1F14A, 6181, 2}, {0x1F14B, 6183, 2}, {0x1F14C, 6185, 2}, {0x1F14D, 6187, 2}, + {0x1F14E, 6189, 3}, {0x1F14F, 6192, 2}, {0x1F16A, 6194, 2}, {0x1F16B, 6196, 2}, + {0x1F16C, 6198, 2}, {0x1F190, 6200, 2}, {0x1F200, 6202, 2}, {0x1F201, 6204, 2}, + {0x1F202, 6206, 1}, {0x1F210, 6207, 1}, {0x1F211, 6208, 1}, {0x1F212, 6209, 1}, + {0x1F213, 6210, 1}, {0x1F214, 6211, 1}, {0x1F215, 6212, 1}, {0x1F216, 6213, 1}, + {0x1F217, 6214, 1}, {0x1F218, 6215, 1}, {0x1F219, 6216, 1}, {0x1F21A, 6217, 1}, + {0x1F21B, 6218, 1}, {0x1F21C, 6219, 1}, {0x1F21D, 6220, 1}, {0x1F21E, 6221, 1}, + {0x1F21F, 6222, 1}, {0x1F220, 6223, 1}, {0x1F221, 6224, 1}, {0x1F222, 6225, 1}, + {0x1F223, 6226, 1}, {0x1F224, 6227, 1}, {0x1F225, 6228, 1}, {0x1F226, 6229, 1}, + {0x1F227, 6230, 1}, {0x1F228, 6231, 1}, {0x1F229, 6232, 1}, {0x1F22A, 6233, 1}, + {0x1F22B, 6234, 1}, {0x1F22C, 6235, 1}, {0x1F22D, 6236, 1}, {0x1F22E, 6237, 1}, + {0x1F22F, 6238, 1}, {0x1F230, 6239, 1}, {0x1F231, 6240, 1}, {0x1F232, 6241, 1}, + {0x1F233, 6242, 1}, {0x1F234, 6243, 1}, {0x1F235, 6244, 1}, {0x1F236, 6245, 1}, + {0x1F237, 6246, 1}, {0x1F238, 6247, 1}, {0x1F239, 6248, 1}, {0x1F23A, 6249, 1}, + {0x1F23B, 6250, 1}, {0x1F240, 6251, 3}, {0x1F241, 6254, 3}, {0x1F242, 6257, 3}, + {0x1F243, 6260, 3}, {0x1F244, 6263, 3}, {0x1F245, 6266, 3}, {0x1F246, 6269, 3}, + {0x1F247, 6272, 3}, {0x1F248, 6275, 3}, {0x1F250, 6278, 1}, {0x1F251, 6279, 1}, + {0x1FBF0, 6280, 1}, {0x1FBF1, 6281, 1}, {0x1FBF2, 6282, 1}, {0x1FBF3, 6283, 1}, + {0x1FBF4, 6284, 1}, {0x1FBF5, 6285, 1}, {0x1FBF6, 6286, 1}, {0x1FBF7, 6287, 1}, + {0x1FBF8, 6288, 1}, {0x1FBF9, 6289, 1}, {0x2F800, 6290, 1}, {0x2F801, 6291, 1}, + {0x2F802, 6292, 1}, {0x2F803, 6293, 1}, {0x2F804, 6294, 1}, {0x2F805, 6295, 1}, + {0x2F806, 6296, 1}, {0x2F807, 6297, 1}, {0x2F808, 6298, 1}, {0x2F809, 6299, 1}, + {0x2F80A, 6300, 1}, {0x2F80B, 6301, 1}, {0x2F80C, 6302, 1}, {0x2F80D, 6303, 1}, + {0x2F80E, 6304, 1}, {0x2F80F, 6305, 1}, {0x2F810, 6306, 1}, {0x2F811, 6307, 1}, + {0x2F812, 6308, 1}, {0x2F813, 6309, 1}, {0x2F814, 6310, 1}, {0x2F815, 6311, 1}, + {0x2F816, 6312, 1}, {0x2F817, 6313, 1}, {0x2F818, 6314, 1}, {0x2F819, 6315, 1}, + {0x2F81A, 6316, 1}, {0x2F81B, 6317, 1}, {0x2F81C, 6318, 1}, {0x2F81D, 6319, 1}, + {0x2F81E, 6320, 1}, {0x2F81F, 6321, 1}, {0x2F820, 6322, 1}, {0x2F821, 6323, 1}, + {0x2F822, 6324, 1}, {0x2F823, 6325, 1}, {0x2F824, 6326, 1}, {0x2F825, 6327, 1}, + {0x2F826, 6328, 1}, {0x2F827, 6329, 1}, {0x2F828, 6330, 1}, {0x2F829, 6331, 1}, + {0x2F82A, 6332, 1}, {0x2F82B, 6333, 1}, {0x2F82C, 6334, 1}, {0x2F82D, 6335, 1}, + {0x2F82E, 6336, 1}, {0x2F82F, 6337, 1}, {0x2F830, 6338, 1}, {0x2F831, 6339, 1}, + {0x2F832, 6340, 1}, {0x2F833, 6341, 1}, {0x2F834, 6342, 1}, {0x2F835, 6343, 1}, + {0x2F836, 6344, 1}, {0x2F837, 6345, 1}, {0x2F838, 6346, 1}, {0x2F839, 6347, 1}, + {0x2F83A, 6348, 1}, {0x2F83B, 6349, 1}, {0x2F83C, 6350, 1}, {0x2F83D, 6351, 1}, + {0x2F83E, 6352, 1}, {0x2F83F, 6353, 1}, {0x2F840, 6354, 1}, {0x2F841, 6355, 1}, + {0x2F842, 6356, 1}, {0x2F843, 6357, 1}, {0x2F844, 6358, 1}, {0x2F845, 6359, 1}, + {0x2F846, 6360, 1}, {0x2F847, 6361, 1}, {0x2F848, 6362, 1}, {0x2F849, 6363, 1}, + {0x2F84A, 6364, 1}, {0x2F84B, 6365, 1}, {0x2F84C, 6366, 1}, {0x2F84D, 6367, 1}, + {0x2F84E, 6368, 1}, {0x2F84F, 6369, 1}, {0x2F850, 6370, 1}, {0x2F851, 6371, 1}, + {0x2F852, 6372, 1}, {0x2F853, 6373, 1}, {0x2F854, 6374, 1}, {0x2F855, 6375, 1}, + {0x2F856, 6376, 1}, {0x2F857, 6377, 1}, {0x2F858, 6378, 1}, {0x2F859, 6379, 1}, + {0x2F85A, 6380, 1}, {0x2F85B, 6381, 1}, {0x2F85C, 6382, 1}, {0x2F85D, 6383, 1}, + {0x2F85E, 6384, 1}, {0x2F85F, 6385, 1}, {0x2F860, 6386, 1}, {0x2F861, 6387, 1}, + {0x2F862, 6388, 1}, {0x2F863, 6389, 1}, {0x2F864, 6390, 1}, {0x2F865, 6391, 1}, + {0x2F866, 6392, 1}, {0x2F867, 6393, 1}, {0x2F868, 6394, 1}, {0x2F869, 6395, 1}, + {0x2F86A, 6396, 1}, {0x2F86B, 6397, 1}, {0x2F86C, 6398, 1}, {0x2F86D, 6399, 1}, + {0x2F86E, 6400, 1}, {0x2F86F, 6401, 1}, {0x2F870, 6402, 1}, {0x2F871, 6403, 1}, + {0x2F872, 6404, 1}, {0x2F873, 6405, 1}, {0x2F874, 6406, 1}, {0x2F875, 6407, 1}, + {0x2F876, 6408, 1}, {0x2F877, 6409, 1}, {0x2F878, 6410, 1}, {0x2F879, 6411, 1}, + {0x2F87A, 6412, 1}, {0x2F87B, 6413, 1}, {0x2F87C, 6414, 1}, {0x2F87D, 6415, 1}, + {0x2F87E, 6416, 1}, {0x2F87F, 6417, 1}, {0x2F880, 6418, 1}, {0x2F881, 6419, 1}, + {0x2F882, 6420, 1}, {0x2F883, 6421, 1}, {0x2F884, 6422, 1}, {0x2F885, 6423, 1}, + {0x2F886, 6424, 1}, {0x2F887, 6425, 1}, {0x2F888, 6426, 1}, {0x2F889, 6427, 1}, + {0x2F88A, 6428, 1}, {0x2F88B, 6429, 1}, {0x2F88C, 6430, 1}, {0x2F88D, 6431, 1}, + {0x2F88E, 6432, 1}, {0x2F88F, 6433, 1}, {0x2F890, 6434, 1}, {0x2F891, 6435, 1}, + {0x2F892, 6436, 1}, {0x2F893, 6437, 1}, {0x2F894, 6438, 1}, {0x2F895, 6439, 1}, + {0x2F896, 6440, 1}, {0x2F897, 6441, 1}, {0x2F898, 6442, 1}, {0x2F899, 6443, 1}, + {0x2F89A, 6444, 1}, {0x2F89B, 6445, 1}, {0x2F89C, 6446, 1}, {0x2F89D, 6447, 1}, + {0x2F89E, 6448, 1}, {0x2F89F, 6449, 1}, {0x2F8A0, 6450, 1}, {0x2F8A1, 6451, 1}, + {0x2F8A2, 6452, 1}, {0x2F8A3, 6453, 1}, {0x2F8A4, 6454, 1}, {0x2F8A5, 6455, 1}, + {0x2F8A6, 6456, 1}, {0x2F8A7, 6457, 1}, {0x2F8A8, 6458, 1}, {0x2F8A9, 6459, 1}, + {0x2F8AA, 6460, 1}, {0x2F8AB, 6461, 1}, {0x2F8AC, 6462, 1}, {0x2F8AD, 6463, 1}, + {0x2F8AE, 6464, 1}, {0x2F8AF, 6465, 1}, {0x2F8B0, 6466, 1}, {0x2F8B1, 6467, 1}, + {0x2F8B2, 6468, 1}, {0x2F8B3, 6469, 1}, {0x2F8B4, 6470, 1}, {0x2F8B5, 6471, 1}, + {0x2F8B6, 6472, 1}, {0x2F8B7, 6473, 1}, {0x2F8B8, 6474, 1}, {0x2F8B9, 6475, 1}, + {0x2F8BA, 6476, 1}, {0x2F8BB, 6477, 1}, {0x2F8BC, 6478, 1}, {0x2F8BD, 6479, 1}, + {0x2F8BE, 6480, 1}, {0x2F8BF, 6481, 1}, {0x2F8C0, 6482, 1}, {0x2F8C1, 6483, 1}, + {0x2F8C2, 6484, 1}, {0x2F8C3, 6485, 1}, {0x2F8C4, 6486, 1}, {0x2F8C5, 6487, 1}, + {0x2F8C6, 6488, 1}, {0x2F8C7, 6489, 1}, {0x2F8C8, 6490, 1}, {0x2F8C9, 6491, 1}, + {0x2F8CA, 6492, 1}, {0x2F8CB, 6493, 1}, {0x2F8CC, 6494, 1}, {0x2F8CD, 6495, 1}, + {0x2F8CE, 6496, 1}, {0x2F8CF, 6497, 1}, {0x2F8D0, 6498, 1}, {0x2F8D1, 6499, 1}, + {0x2F8D2, 6500, 1}, {0x2F8D3, 6501, 1}, {0x2F8D4, 6502, 1}, {0x2F8D5, 6503, 1}, + {0x2F8D6, 6504, 1}, {0x2F8D7, 6505, 1}, {0x2F8D8, 6506, 1}, {0x2F8D9, 6507, 1}, + {0x2F8DA, 6508, 1}, {0x2F8DB, 6509, 1}, {0x2F8DC, 6510, 1}, {0x2F8DD, 6511, 1}, + {0x2F8DE, 6512, 1}, {0x2F8DF, 6513, 1}, {0x2F8E0, 6514, 1}, {0x2F8E1, 6515, 1}, + {0x2F8E2, 6516, 1}, {0x2F8E3, 6517, 1}, {0x2F8E4, 6518, 1}, {0x2F8E5, 6519, 1}, + {0x2F8E6, 6520, 1}, {0x2F8E7, 6521, 1}, {0x2F8E8, 6522, 1}, {0x2F8E9, 6523, 1}, + {0x2F8EA, 6524, 1}, {0x2F8EB, 6525, 1}, {0x2F8EC, 6526, 1}, {0x2F8ED, 6527, 1}, + {0x2F8EE, 6528, 1}, {0x2F8EF, 6529, 1}, {0x2F8F0, 6530, 1}, {0x2F8F1, 6531, 1}, + {0x2F8F2, 6532, 1}, {0x2F8F3, 6533, 1}, {0x2F8F4, 6534, 1}, {0x2F8F5, 6535, 1}, + {0x2F8F6, 6536, 1}, {0x2F8F7, 6537, 1}, {0x2F8F8, 6538, 1}, {0x2F8F9, 6539, 1}, + {0x2F8FA, 6540, 1}, {0x2F8FB, 6541, 1}, {0x2F8FC, 6542, 1}, {0x2F8FD, 6543, 1}, + {0x2F8FE, 6544, 1}, {0x2F8FF, 6545, 1}, {0x2F900, 6546, 1}, {0x2F901, 6547, 1}, + {0x2F902, 6548, 1}, {0x2F903, 6549, 1}, {0x2F904, 6550, 1}, {0x2F905, 6551, 1}, + {0x2F906, 6552, 1}, {0x2F907, 6553, 1}, {0x2F908, 6554, 1}, {0x2F909, 6555, 1}, + {0x2F90A, 6556, 1}, {0x2F90B, 6557, 1}, {0x2F90C, 6558, 1}, {0x2F90D, 6559, 1}, + {0x2F90E, 6560, 1}, {0x2F90F, 6561, 1}, {0x2F910, 6562, 1}, {0x2F911, 6563, 1}, + {0x2F912, 6564, 1}, {0x2F913, 6565, 1}, {0x2F914, 6566, 1}, {0x2F915, 6567, 1}, + {0x2F916, 6568, 1}, {0x2F917, 6569, 1}, {0x2F918, 6570, 1}, {0x2F919, 6571, 1}, + {0x2F91A, 6572, 1}, {0x2F91B, 6573, 1}, {0x2F91C, 6574, 1}, {0x2F91D, 6575, 1}, + {0x2F91E, 6576, 1}, {0x2F91F, 6577, 1}, {0x2F920, 6578, 1}, {0x2F921, 6579, 1}, + {0x2F922, 6580, 1}, {0x2F923, 6581, 1}, {0x2F924, 6582, 1}, {0x2F925, 6583, 1}, + {0x2F926, 6584, 1}, {0x2F927, 6585, 1}, {0x2F928, 6586, 1}, {0x2F929, 6587, 1}, + {0x2F92A, 6588, 1}, {0x2F92B, 6589, 1}, {0x2F92C, 6590, 1}, {0x2F92D, 6591, 1}, + {0x2F92E, 6592, 1}, {0x2F92F, 6593, 1}, {0x2F930, 6594, 1}, {0x2F931, 6595, 1}, + {0x2F932, 6596, 1}, {0x2F933, 6597, 1}, {0x2F934, 6598, 1}, {0x2F935, 6599, 1}, + {0x2F936, 6600, 1}, {0x2F937, 6601, 1}, {0x2F938, 6602, 1}, {0x2F939, 6603, 1}, + {0x2F93A, 6604, 1}, {0x2F93B, 6605, 1}, {0x2F93C, 6606, 1}, {0x2F93D, 6607, 1}, + {0x2F93E, 6608, 1}, {0x2F93F, 6609, 1}, {0x2F940, 6610, 1}, {0x2F941, 6611, 1}, + {0x2F942, 6612, 1}, {0x2F943, 6613, 1}, {0x2F944, 6614, 1}, {0x2F945, 6615, 1}, + {0x2F946, 6616, 1}, {0x2F947, 6617, 1}, {0x2F948, 6618, 1}, {0x2F949, 6619, 1}, + {0x2F94A, 6620, 1}, {0x2F94B, 6621, 1}, {0x2F94C, 6622, 1}, {0x2F94D, 6623, 1}, + {0x2F94E, 6624, 1}, {0x2F94F, 6625, 1}, {0x2F950, 6626, 1}, {0x2F951, 6627, 1}, + {0x2F952, 6628, 1}, {0x2F953, 6629, 1}, {0x2F954, 6630, 1}, {0x2F955, 6631, 1}, + {0x2F956, 6632, 1}, {0x2F957, 6633, 1}, {0x2F958, 6634, 1}, {0x2F959, 6635, 1}, + {0x2F95A, 6636, 1}, {0x2F95B, 6637, 1}, {0x2F95C, 6638, 1}, {0x2F95D, 6639, 1}, + {0x2F95E, 6640, 1}, {0x2F95F, 6641, 1}, {0x2F960, 6642, 1}, {0x2F961, 6643, 1}, + {0x2F962, 6644, 1}, {0x2F963, 6645, 1}, {0x2F964, 6646, 1}, {0x2F965, 6647, 1}, + {0x2F966, 6648, 1}, {0x2F967, 6649, 1}, {0x2F968, 6650, 1}, {0x2F969, 6651, 1}, + {0x2F96A, 6652, 1}, {0x2F96B, 6653, 1}, {0x2F96C, 6654, 1}, {0x2F96D, 6655, 1}, + {0x2F96E, 6656, 1}, {0x2F96F, 6657, 1}, {0x2F970, 6658, 1}, {0x2F971, 6659, 1}, + {0x2F972, 6660, 1}, {0x2F973, 6661, 1}, {0x2F974, 6662, 1}, {0x2F975, 6663, 1}, + {0x2F976, 6664, 1}, {0x2F977, 6665, 1}, {0x2F978, 6666, 1}, {0x2F979, 6667, 1}, + {0x2F97A, 6668, 1}, {0x2F97B, 6669, 1}, {0x2F97C, 6670, 1}, {0x2F97D, 6671, 1}, + {0x2F97E, 6672, 1}, {0x2F97F, 6673, 1}, {0x2F980, 6674, 1}, {0x2F981, 6675, 1}, + {0x2F982, 6676, 1}, {0x2F983, 6677, 1}, {0x2F984, 6678, 1}, {0x2F985, 6679, 1}, + {0x2F986, 6680, 1}, {0x2F987, 6681, 1}, {0x2F988, 6682, 1}, {0x2F989, 6683, 1}, + {0x2F98A, 6684, 1}, {0x2F98B, 6685, 1}, {0x2F98C, 6686, 1}, {0x2F98D, 6687, 1}, + {0x2F98E, 6688, 1}, {0x2F98F, 6689, 1}, {0x2F990, 6690, 1}, {0x2F991, 6691, 1}, + {0x2F992, 6692, 1}, {0x2F993, 6693, 1}, {0x2F994, 6694, 1}, {0x2F995, 6695, 1}, + {0x2F996, 6696, 1}, {0x2F997, 6697, 1}, {0x2F998, 6698, 1}, {0x2F999, 6699, 1}, + {0x2F99A, 6700, 1}, {0x2F99B, 6701, 1}, {0x2F99C, 6702, 1}, {0x2F99D, 6703, 1}, + {0x2F99E, 6704, 1}, {0x2F99F, 6705, 1}, {0x2F9A0, 6706, 1}, {0x2F9A1, 6707, 1}, + {0x2F9A2, 6708, 1}, {0x2F9A3, 6709, 1}, {0x2F9A4, 6710, 1}, {0x2F9A5, 6711, 1}, + {0x2F9A6, 6712, 1}, {0x2F9A7, 6713, 1}, {0x2F9A8, 6714, 1}, {0x2F9A9, 6715, 1}, + {0x2F9AA, 6716, 1}, {0x2F9AB, 6717, 1}, {0x2F9AC, 6718, 1}, {0x2F9AD, 6719, 1}, + {0x2F9AE, 6720, 1}, {0x2F9AF, 6721, 1}, {0x2F9B0, 6722, 1}, {0x2F9B1, 6723, 1}, + {0x2F9B2, 6724, 1}, {0x2F9B3, 6725, 1}, {0x2F9B4, 6726, 1}, {0x2F9B5, 6727, 1}, + {0x2F9B6, 6728, 1}, {0x2F9B7, 6729, 1}, {0x2F9B8, 6730, 1}, {0x2F9B9, 6731, 1}, + {0x2F9BA, 6732, 1}, {0x2F9BB, 6733, 1}, {0x2F9BC, 6734, 1}, {0x2F9BD, 6735, 1}, + {0x2F9BE, 6736, 1}, {0x2F9BF, 6737, 1}, {0x2F9C0, 6738, 1}, {0x2F9C1, 6739, 1}, + {0x2F9C2, 6740, 1}, {0x2F9C3, 6741, 1}, {0x2F9C4, 6742, 1}, {0x2F9C5, 6743, 1}, + {0x2F9C6, 6744, 1}, {0x2F9C7, 6745, 1}, {0x2F9C8, 6746, 1}, {0x2F9C9, 6747, 1}, + {0x2F9CA, 6748, 1}, {0x2F9CB, 6749, 1}, {0x2F9CC, 6750, 1}, {0x2F9CD, 6751, 1}, + {0x2F9CE, 6752, 1}, {0x2F9CF, 6753, 1}, {0x2F9D0, 6754, 1}, {0x2F9D1, 6755, 1}, + {0x2F9D2, 6756, 1}, {0x2F9D3, 6757, 1}, {0x2F9D4, 6758, 1}, {0x2F9D5, 6759, 1}, + {0x2F9D6, 6760, 1}, {0x2F9D7, 6761, 1}, {0x2F9D8, 6762, 1}, {0x2F9D9, 6763, 1}, + {0x2F9DA, 6764, 1}, {0x2F9DB, 6765, 1}, {0x2F9DC, 6766, 1}, {0x2F9DD, 6767, 1}, + {0x2F9DE, 6768, 1}, {0x2F9DF, 6769, 1}, {0x2F9E0, 6770, 1}, {0x2F9E1, 6771, 1}, + {0x2F9E2, 6772, 1}, {0x2F9E3, 6773, 1}, {0x2F9E4, 6774, 1}, {0x2F9E5, 6775, 1}, + {0x2F9E6, 6776, 1}, {0x2F9E7, 6777, 1}, {0x2F9E8, 6778, 1}, {0x2F9E9, 6779, 1}, + {0x2F9EA, 6780, 1}, {0x2F9EB, 6781, 1}, {0x2F9EC, 6782, 1}, {0x2F9ED, 6783, 1}, + {0x2F9EE, 6784, 1}, {0x2F9EF, 6785, 1}, {0x2F9F0, 6786, 1}, {0x2F9F1, 6787, 1}, + {0x2F9F2, 6788, 1}, {0x2F9F3, 6789, 1}, {0x2F9F4, 6790, 1}, {0x2F9F5, 6791, 1}, + {0x2F9F6, 6792, 1}, {0x2F9F7, 6793, 1}, {0x2F9F8, 6794, 1}, {0x2F9F9, 6795, 1}, + {0x2F9FA, 6796, 1}, {0x2F9FB, 6797, 1}, {0x2F9FC, 6798, 1}, {0x2F9FD, 6799, 1}, + {0x2F9FE, 6800, 1}, {0x2F9FF, 6801, 1}, {0x2FA00, 6802, 1}, {0x2FA01, 6803, 1}, + {0x2FA02, 6804, 1}, {0x2FA03, 6805, 1}, {0x2FA04, 6806, 1}, {0x2FA05, 6807, 1}, + {0x2FA06, 6808, 1}, {0x2FA07, 6809, 1}, {0x2FA08, 6810, 1}, {0x2FA09, 6811, 1}, + {0x2FA0A, 6812, 1}, {0x2FA0B, 6813, 1}, {0x2FA0C, 6814, 1}, {0x2FA0D, 6815, 1}, + {0x2FA0E, 6816, 1}, {0x2FA0F, 6817, 1}, {0x2FA10, 6818, 1}, {0x2FA11, 6819, 1}, + {0x2FA12, 6820, 1}, {0x2FA13, 6821, 1}, {0x2FA14, 6822, 1}, {0x2FA15, 6823, 1}, + {0x2FA16, 6824, 1}, {0x2FA17, 6825, 1}, {0x2FA18, 6826, 1}, {0x2FA19, 6827, 1}, + {0x2FA1A, 6828, 1}, {0x2FA1B, 6829, 1}, {0x2FA1C, 6830, 1}, {0x2FA1D, 6831, 1}, +}; +const int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0])); + +const uint32_t PDF_NFKC_SEQ[] = { + 0x20, 0x20, 0x308, 0x61, 0x20, 0x304, 0x32, 0x33, 0x20, 0x301, + 0x3BC, 0x20, 0x327, 0x31, 0x6F, 0x31, 0x2044, 0x34, 0x31, 0x2044, + 0x32, 0x33, 0x2044, 0x34, 0x49, 0x4A, 0x69, 0x6A, 0x4C, 0xB7, + 0x6C, 0xB7, 0x2BC, 0x6E, 0x73, 0x44, 0x17D, 0x44, 0x17E, 0x64, + 0x17E, 0x4C, 0x4A, 0x4C, 0x6A, 0x6C, 0x6A, 0x4E, 0x4A, 0x4E, + 0x6A, 0x6E, 0x6A, 0x44, 0x5A, 0x44, 0x7A, 0x64, 0x7A, 0x68, + 0x266, 0x6A, 0x72, 0x279, 0x27B, 0x281, 0x77, 0x79, 0x20, 0x306, + 0x20, 0x307, 0x20, 0x30A, 0x20, 0x328, 0x20, 0x303, 0x20, 0x30B, + 0x263, 0x6C, 0x73, 0x78, 0x295, 0x300, 0x301, 0x313, 0x308, 0x301, + 0x2B9, 0x20, 0x345, 0x3B, 0x20, 0x301, 0x20, 0x308, 0x301, 0xB7, + 0x3B2, 0x3B8, 0x3A5, 0x38E, 0x3AB, 0x3C6, 0x3C0, 0x3BA, 0x3C1, 0x3C2, + 0x398, 0x3B5, 0x3A3, 0x565, 0x582, 0x627, 0x674, 0x648, 0x674, 0x6C7, + 0x674, 0x64A, 0x674, 0x915, 0x93C, 0x916, 0x93C, 0x917, 0x93C, 0x91C, + 0x93C, 0x921, 0x93C, 0x922, 0x93C, 0x92B, 0x93C, 0x92F, 0x93C, 0x9A1, + 0x9BC, 0x9A2, 0x9BC, 0x9AF, 0x9BC, 0xA32, 0xA3C, 0xA38, 0xA3C, 0xA16, + 0xA3C, 0xA17, 0xA3C, 0xA1C, 0xA3C, 0xA2B, 0xA3C, 0xB21, 0xB3C, 0xB22, + 0xB3C, 0xE4D, 0xE32, 0xECD, 0xEB2, 0xEAB, 0xE99, 0xEAB, 0xEA1, 0xF0B, + 0xF42, 0xFB7, 0xF4C, 0xFB7, 0xF51, 0xFB7, 0xF56, 0xFB7, 0xF5B, 0xFB7, + 0xF40, 0xFB5, 0xF71, 0xF72, 0xF71, 0xF74, 0xFB2, 0xF80, 0xFB2, 0xF71, + 0xF80, 0xFB3, 0xF80, 0xFB3, 0xF71, 0xF80, 0xF71, 0xF80, 0xF92, 0xFB7, + 0xF9C, 0xFB7, 0xFA1, 0xFB7, 0xFA6, 0xFB7, 0xFAB, 0xFB7, 0xF90, 0xFB5, + 0x10DC, 0x41, 0xC6, 0x42, 0x44, 0x45, 0x18E, 0x47, 0x48, 0x49, + 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x222, 0x50, 0x52, 0x54, + 0x55, 0x57, 0x61, 0x250, 0x251, 0x1D02, 0x62, 0x64, 0x65, 0x259, + 0x25B, 0x25C, 0x67, 0x6B, 0x6D, 0x14B, 0x6F, 0x254, 0x1D16, 0x1D17, + 0x70, 0x74, 0x75, 0x1D1D, 0x26F, 0x76, 0x1D25, 0x3B2, 0x3B3, 0x3B4, + 0x3C6, 0x3C7, 0x69, 0x72, 0x75, 0x76, 0x3B2, 0x3B3, 0x3C1, 0x3C6, + 0x3C7, 0x43D, 0x252, 0x63, 0x255, 0xF0, 0x25C, 0x66, 0x25F, 0x261, + 0x265, 0x268, 0x269, 0x26A, 0x1D7B, 0x29D, 0x26D, 0x1D85, 0x29F, 0x271, + 0x270, 0x272, 0x273, 0x274, 0x275, 0x278, 0x282, 0x283, 0x1AB, 0x289, + 0x28A, 0x1D1C, 0x28B, 0x28C, 0x7A, 0x290, 0x291, 0x292, 0x3B8, 0x61, + 0x2BE, 0x1E61, 0x3AC, 0x3AD, 0x3AE, 0x3AF, 0x3CC, 0x3CD, 0x3CE, 0x386, + 0x20, 0x313, 0x3B9, 0x20, 0x313, 0x20, 0x342, 0x20, 0x308, 0x342, + 0x388, 0x389, 0x20, 0x313, 0x300, 0x20, 0x313, 0x301, 0x20, 0x313, + 0x342, 0x390, 0x38A, 0x20, 0x314, 0x300, 0x20, 0x314, 0x301, 0x20, + 0x314, 0x342, 0x3B0, 0x38E, 0x20, 0x308, 0x300, 0x20, 0x308, 0x301, + 0x60, 0x38C, 0x38F, 0x20, 0x301, 0x20, 0x314, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2010, 0x20, + 0x333, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x20, 0x2032, 0x2032, + 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, 0x2035, 0x2035, 0x21, 0x21, + 0x20, 0x305, 0x3F, 0x3F, 0x3F, 0x21, 0x21, 0x3F, 0x2032, 0x2032, + 0x2032, 0x2032, 0x20, 0x30, 0x69, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x2B, 0x2212, 0x3D, 0x28, 0x29, 0x6E, 0x30, 0x31, 0x32, + 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2212, 0x3D, + 0x28, 0x29, 0x61, 0x65, 0x6F, 0x78, 0x259, 0x68, 0x6B, 0x6C, + 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x52, 0x73, 0x61, 0x2F, 0x63, + 0x61, 0x2F, 0x73, 0x43, 0xB0, 0x43, 0x63, 0x2F, 0x6F, 0x63, + 0x2F, 0x75, 0x190, 0xB0, 0x46, 0x67, 0x48, 0x48, 0x48, 0x68, + 0x127, 0x49, 0x49, 0x4C, 0x6C, 0x4E, 0x4E, 0x6F, 0x50, 0x51, + 0x52, 0x52, 0x52, 0x53, 0x4D, 0x54, 0x45, 0x4C, 0x54, 0x4D, + 0x5A, 0x3A9, 0x5A, 0x4B, 0xC5, 0x42, 0x43, 0x65, 0x45, 0x46, + 0x4D, 0x6F, 0x5D0, 0x5D1, 0x5D2, 0x5D3, 0x69, 0x46, 0x41, 0x58, + 0x3C0, 0x3B3, 0x393, 0x3A0, 0x2211, 0x44, 0x64, 0x65, 0x69, 0x6A, + 0x31, 0x2044, 0x37, 0x31, 0x2044, 0x39, 0x31, 0x2044, 0x31, 0x30, + 0x31, 0x2044, 0x33, 0x32, 0x2044, 0x33, 0x31, 0x2044, 0x35, 0x32, + 0x2044, 0x35, 0x33, 0x2044, 0x35, 0x34, 0x2044, 0x35, 0x31, 0x2044, + 0x36, 0x35, 0x2044, 0x36, 0x31, 0x2044, 0x38, 0x33, 0x2044, 0x38, + 0x35, 0x2044, 0x38, 0x37, 0x2044, 0x38, 0x31, 0x2044, 0x49, 0x49, + 0x49, 0x49, 0x49, 0x49, 0x49, 0x56, 0x56, 0x56, 0x49, 0x56, + 0x49, 0x49, 0x56, 0x49, 0x49, 0x49, 0x49, 0x58, 0x58, 0x58, + 0x49, 0x58, 0x49, 0x49, 0x4C, 0x43, 0x44, 0x4D, 0x69, 0x69, + 0x69, 0x69, 0x69, 0x69, 0x69, 0x76, 0x76, 0x76, 0x69, 0x76, + 0x69, 0x69, 0x76, 0x69, 0x69, 0x69, 0x69, 0x78, 0x78, 0x78, + 0x69, 0x78, 0x69, 0x69, 0x6C, 0x63, 0x64, 0x6D, 0x30, 0x2044, + 0x33, 0x222B, 0x222B, 0x222B, 0x222B, 0x222B, 0x222E, 0x222E, 0x222E, 0x222E, + 0x222E, 0x3008, 0x3009, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x31, 0x30, 0x31, 0x31, 0x31, 0x32, 0x31, 0x33, + 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, 0x31, 0x38, + 0x31, 0x39, 0x32, 0x30, 0x28, 0x31, 0x29, 0x28, 0x32, 0x29, + 0x28, 0x33, 0x29, 0x28, 0x34, 0x29, 0x28, 0x35, 0x29, 0x28, + 0x36, 0x29, 0x28, 0x37, 0x29, 0x28, 0x38, 0x29, 0x28, 0x39, + 0x29, 0x28, 0x31, 0x30, 0x29, 0x28, 0x31, 0x31, 0x29, 0x28, + 0x31, 0x32, 0x29, 0x28, 0x31, 0x33, 0x29, 0x28, 0x31, 0x34, + 0x29, 0x28, 0x31, 0x35, 0x29, 0x28, 0x31, 0x36, 0x29, 0x28, + 0x31, 0x37, 0x29, 0x28, 0x31, 0x38, 0x29, 0x28, 0x31, 0x39, + 0x29, 0x28, 0x32, 0x30, 0x29, 0x31, 0x2E, 0x32, 0x2E, 0x33, + 0x2E, 0x34, 0x2E, 0x35, 0x2E, 0x36, 0x2E, 0x37, 0x2E, 0x38, + 0x2E, 0x39, 0x2E, 0x31, 0x30, 0x2E, 0x31, 0x31, 0x2E, 0x31, + 0x32, 0x2E, 0x31, 0x33, 0x2E, 0x31, 0x34, 0x2E, 0x31, 0x35, + 0x2E, 0x31, 0x36, 0x2E, 0x31, 0x37, 0x2E, 0x31, 0x38, 0x2E, + 0x31, 0x39, 0x2E, 0x32, 0x30, 0x2E, 0x28, 0x61, 0x29, 0x28, + 0x62, 0x29, 0x28, 0x63, 0x29, 0x28, 0x64, 0x29, 0x28, 0x65, + 0x29, 0x28, 0x66, 0x29, 0x28, 0x67, 0x29, 0x28, 0x68, 0x29, + 0x28, 0x69, 0x29, 0x28, 0x6A, 0x29, 0x28, 0x6B, 0x29, 0x28, + 0x6C, 0x29, 0x28, 0x6D, 0x29, 0x28, 0x6E, 0x29, 0x28, 0x6F, + 0x29, 0x28, 0x70, 0x29, 0x28, 0x71, 0x29, 0x28, 0x72, 0x29, + 0x28, 0x73, 0x29, 0x28, 0x74, 0x29, 0x28, 0x75, 0x29, 0x28, + 0x76, 0x29, 0x28, 0x77, 0x29, 0x28, 0x78, 0x29, 0x28, 0x79, + 0x29, 0x28, 0x7A, 0x29, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, + 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x30, 0x222B, 0x222B, 0x222B, + 0x222B, 0x3A, 0x3A, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x2ADD, + 0x338, 0x6A, 0x56, 0x2D61, 0x6BCD, 0x9F9F, 0x4E00, 0x4E28, 0x4E36, 0x4E3F, + 0x4E59, 0x4E85, 0x4E8C, 0x4EA0, 0x4EBA, 0x513F, 0x5165, 0x516B, 0x5182, 0x5196, + 0x51AB, 0x51E0, 0x51F5, 0x5200, 0x529B, 0x52F9, 0x5315, 0x531A, 0x5338, 0x5341, + 0x535C, 0x5369, 0x5382, 0x53B6, 0x53C8, 0x53E3, 0x56D7, 0x571F, 0x58EB, 0x5902, + 0x590A, 0x5915, 0x5927, 0x5973, 0x5B50, 0x5B80, 0x5BF8, 0x5C0F, 0x5C22, 0x5C38, + 0x5C6E, 0x5C71, 0x5DDB, 0x5DE5, 0x5DF1, 0x5DFE, 0x5E72, 0x5E7A, 0x5E7F, 0x5EF4, + 0x5EFE, 0x5F0B, 0x5F13, 0x5F50, 0x5F61, 0x5F73, 0x5FC3, 0x6208, 0x6236, 0x624B, + 0x652F, 0x6534, 0x6587, 0x6597, 0x65A4, 0x65B9, 0x65E0, 0x65E5, 0x66F0, 0x6708, + 0x6728, 0x6B20, 0x6B62, 0x6B79, 0x6BB3, 0x6BCB, 0x6BD4, 0x6BDB, 0x6C0F, 0x6C14, + 0x6C34, 0x706B, 0x722A, 0x7236, 0x723B, 0x723F, 0x7247, 0x7259, 0x725B, 0x72AC, + 0x7384, 0x7389, 0x74DC, 0x74E6, 0x7518, 0x751F, 0x7528, 0x7530, 0x758B, 0x7592, + 0x7676, 0x767D, 0x76AE, 0x76BF, 0x76EE, 0x77DB, 0x77E2, 0x77F3, 0x793A, 0x79B8, + 0x79BE, 0x7A74, 0x7ACB, 0x7AF9, 0x7C73, 0x7CF8, 0x7F36, 0x7F51, 0x7F8A, 0x7FBD, + 0x8001, 0x800C, 0x8012, 0x8033, 0x807F, 0x8089, 0x81E3, 0x81EA, 0x81F3, 0x81FC, + 0x820C, 0x821B, 0x821F, 0x826E, 0x8272, 0x8278, 0x864D, 0x866B, 0x8840, 0x884C, + 0x8863, 0x897E, 0x898B, 0x89D2, 0x8A00, 0x8C37, 0x8C46, 0x8C55, 0x8C78, 0x8C9D, + 0x8D64, 0x8D70, 0x8DB3, 0x8EAB, 0x8ECA, 0x8F9B, 0x8FB0, 0x8FB5, 0x9091, 0x9149, + 0x91C6, 0x91CC, 0x91D1, 0x9577, 0x9580, 0x961C, 0x96B6, 0x96B9, 0x96E8, 0x9751, + 0x975E, 0x9762, 0x9769, 0x97CB, 0x97ED, 0x97F3, 0x9801, 0x98A8, 0x98DB, 0x98DF, + 0x9996, 0x9999, 0x99AC, 0x9AA8, 0x9AD8, 0x9ADF, 0x9B25, 0x9B2F, 0x9B32, 0x9B3C, + 0x9B5A, 0x9CE5, 0x9E75, 0x9E7F, 0x9EA5, 0x9EBB, 0x9EC3, 0x9ECD, 0x9ED1, 0x9EF9, + 0x9EFD, 0x9F0E, 0x9F13, 0x9F20, 0x9F3B, 0x9F4A, 0x9F52, 0x9F8D, 0x9F9C, 0x9FA0, + 0x20, 0x3012, 0x5341, 0x5344, 0x5345, 0x20, 0x3099, 0x20, 0x309A, 0x3088, + 0x308A, 0x30B3, 0x30C8, 0x1100, 0x1101, 0x11AA, 0x1102, 0x11AC, 0x11AD, 0x1103, + 0x1104, 0x1105, 0x11B0, 0x11B1, 0x11B2, 0x11B3, 0x11B4, 0x11B5, 0x111A, 0x1106, + 0x1107, 0x1108, 0x1121, 0x1109, 0x110A, 0x110B, 0x110C, 0x110D, 0x110E, 0x110F, + 0x1110, 0x1111, 0x1112, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, + 0x1168, 0x1169, 0x116A, 0x116B, 0x116C, 0x116D, 0x116E, 0x116F, 0x1170, 0x1171, + 0x1172, 0x1173, 0x1174, 0x1175, 0x1160, 0x1114, 0x1115, 0x11C7, 0x11C8, 0x11CC, + 0x11CE, 0x11D3, 0x11D7, 0x11D9, 0x111C, 0x11DD, 0x11DF, 0x111D, 0x111E, 0x1120, + 0x1122, 0x1123, 0x1127, 0x1129, 0x112B, 0x112C, 0x112D, 0x112E, 0x112F, 0x1132, + 0x1136, 0x1140, 0x1147, 0x114C, 0x11F1, 0x11F2, 0x1157, 0x1158, 0x1159, 0x1184, + 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119E, 0x11A1, 0x4E00, 0x4E8C, 0x4E09, + 0x56DB, 0x4E0A, 0x4E2D, 0x4E0B, 0x7532, 0x4E59, 0x4E19, 0x4E01, 0x5929, 0x5730, + 0x4EBA, 0x28, 0x1100, 0x29, 0x28, 0x1102, 0x29, 0x28, 0x1103, 0x29, + 0x28, 0x1105, 0x29, 0x28, 0x1106, 0x29, 0x28, 0x1107, 0x29, 0x28, + 0x1109, 0x29, 0x28, 0x110B, 0x29, 0x28, 0x110C, 0x29, 0x28, 0x110E, + 0x29, 0x28, 0x110F, 0x29, 0x28, 0x1110, 0x29, 0x28, 0x1111, 0x29, + 0x28, 0x1112, 0x29, 0x28, 0xAC00, 0x29, 0x28, 0xB098, 0x29, 0x28, + 0xB2E4, 0x29, 0x28, 0xB77C, 0x29, 0x28, 0xB9C8, 0x29, 0x28, 0xBC14, + 0x29, 0x28, 0xC0AC, 0x29, 0x28, 0xC544, 0x29, 0x28, 0xC790, 0x29, + 0x28, 0xCC28, 0x29, 0x28, 0xCE74, 0x29, 0x28, 0xD0C0, 0x29, 0x28, + 0xD30C, 0x29, 0x28, 0xD558, 0x29, 0x28, 0xC8FC, 0x29, 0x28, 0xC624, + 0xC804, 0x29, 0x28, 0xC624, 0xD6C4, 0x29, 0x28, 0x4E00, 0x29, 0x28, + 0x4E8C, 0x29, 0x28, 0x4E09, 0x29, 0x28, 0x56DB, 0x29, 0x28, 0x4E94, + 0x29, 0x28, 0x516D, 0x29, 0x28, 0x4E03, 0x29, 0x28, 0x516B, 0x29, + 0x28, 0x4E5D, 0x29, 0x28, 0x5341, 0x29, 0x28, 0x6708, 0x29, 0x28, + 0x706B, 0x29, 0x28, 0x6C34, 0x29, 0x28, 0x6728, 0x29, 0x28, 0x91D1, + 0x29, 0x28, 0x571F, 0x29, 0x28, 0x65E5, 0x29, 0x28, 0x682A, 0x29, + 0x28, 0x6709, 0x29, 0x28, 0x793E, 0x29, 0x28, 0x540D, 0x29, 0x28, + 0x7279, 0x29, 0x28, 0x8CA1, 0x29, 0x28, 0x795D, 0x29, 0x28, 0x52B4, + 0x29, 0x28, 0x4EE3, 0x29, 0x28, 0x547C, 0x29, 0x28, 0x5B66, 0x29, + 0x28, 0x76E3, 0x29, 0x28, 0x4F01, 0x29, 0x28, 0x8CC7, 0x29, 0x28, + 0x5354, 0x29, 0x28, 0x796D, 0x29, 0x28, 0x4F11, 0x29, 0x28, 0x81EA, + 0x29, 0x28, 0x81F3, 0x29, 0x554F, 0x5E7C, 0x6587, 0x7B8F, 0x50, 0x54, + 0x45, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, 0x32, 0x34, 0x32, + 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, 0x33, + 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, + 0x35, 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110B, 0x110C, + 0x110E, 0x110F, 0x1110, 0x1111, 0x1112, 0xAC00, 0xB098, 0xB2E4, 0xB77C, 0xB9C8, + 0xBC14, 0xC0AC, 0xC544, 0xC790, 0xCC28, 0xCE74, 0xD0C0, 0xD30C, 0xD558, 0xCC38, + 0xACE0, 0xC8FC, 0xC758, 0xC6B0, 0x4E00, 0x4E8C, 0x4E09, 0x56DB, 0x4E94, 0x516D, + 0x4E03, 0x516B, 0x4E5D, 0x5341, 0x6708, 0x706B, 0x6C34, 0x6728, 0x91D1, 0x571F, + 0x65E5, 0x682A, 0x6709, 0x793E, 0x540D, 0x7279, 0x8CA1, 0x795D, 0x52B4, 0x79D8, + 0x7537, 0x5973, 0x9069, 0x512A, 0x5370, 0x6CE8, 0x9805, 0x4F11, 0x5199, 0x6B63, + 0x4E0A, 0x4E2D, 0x4E0B, 0x5DE6, 0x53F3, 0x533B, 0x5B97, 0x5B66, 0x76E3, 0x4F01, + 0x8CC7, 0x5354, 0x591C, 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, + 0x39, 0x34, 0x30, 0x34, 0x31, 0x34, 0x32, 0x34, 0x33, 0x34, + 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, 0x34, 0x38, 0x34, + 0x39, 0x35, 0x30, 0x31, 0x6708, 0x32, 0x6708, 0x33, 0x6708, 0x34, + 0x6708, 0x35, 0x6708, 0x36, 0x6708, 0x37, 0x6708, 0x38, 0x6708, 0x39, + 0x6708, 0x31, 0x30, 0x6708, 0x31, 0x31, 0x6708, 0x31, 0x32, 0x6708, + 0x48, 0x67, 0x65, 0x72, 0x67, 0x65, 0x56, 0x4C, 0x54, 0x44, + 0x30A2, 0x30A4, 0x30A6, 0x30A8, 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3, + 0x30B5, 0x30B7, 0x30B9, 0x30BB, 0x30BD, 0x30BF, 0x30C1, 0x30C4, 0x30C6, 0x30C8, + 0x30CA, 0x30CB, 0x30CC, 0x30CD, 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, + 0x30DE, 0x30DF, 0x30E0, 0x30E1, 0x30E2, 0x30E4, 0x30E6, 0x30E8, 0x30E9, 0x30EA, + 0x30EB, 0x30EC, 0x30ED, 0x30EF, 0x30F0, 0x30F1, 0x30F2, 0x4EE4, 0x548C, 0x30A2, + 0x30D1, 0x30FC, 0x30C8, 0x30A2, 0x30EB, 0x30D5, 0x30A1, 0x30A2, 0x30F3, 0x30DA, + 0x30A2, 0x30A2, 0x30FC, 0x30EB, 0x30A4, 0x30CB, 0x30F3, 0x30B0, 0x30A4, 0x30F3, + 0x30C1, 0x30A6, 0x30A9, 0x30F3, 0x30A8, 0x30B9, 0x30AF, 0x30FC, 0x30C9, 0x30A8, + 0x30FC, 0x30AB, 0x30FC, 0x30AA, 0x30F3, 0x30B9, 0x30AA, 0x30FC, 0x30E0, 0x30AB, + 0x30A4, 0x30EA, 0x30AB, 0x30E9, 0x30C3, 0x30C8, 0x30AB, 0x30ED, 0x30EA, 0x30FC, + 0x30AC, 0x30ED, 0x30F3, 0x30AC, 0x30F3, 0x30DE, 0x30AE, 0x30AC, 0x30AE, 0x30CB, + 0x30FC, 0x30AD, 0x30E5, 0x30EA, 0x30FC, 0x30AE, 0x30EB, 0x30C0, 0x30FC, 0x30AD, + 0x30ED, 0x30AD, 0x30ED, 0x30B0, 0x30E9, 0x30E0, 0x30AD, 0x30ED, 0x30E1, 0x30FC, + 0x30C8, 0x30EB, 0x30AD, 0x30ED, 0x30EF, 0x30C3, 0x30C8, 0x30B0, 0x30E9, 0x30E0, + 0x30B0, 0x30E9, 0x30E0, 0x30C8, 0x30F3, 0x30AF, 0x30EB, 0x30BC, 0x30A4, 0x30ED, + 0x30AF, 0x30ED, 0x30FC, 0x30CD, 0x30B1, 0x30FC, 0x30B9, 0x30B3, 0x30EB, 0x30CA, + 0x30B3, 0x30FC, 0x30DD, 0x30B5, 0x30A4, 0x30AF, 0x30EB, 0x30B5, 0x30F3, 0x30C1, + 0x30FC, 0x30E0, 0x30B7, 0x30EA, 0x30F3, 0x30B0, 0x30BB, 0x30F3, 0x30C1, 0x30BB, + 0x30F3, 0x30C8, 0x30C0, 0x30FC, 0x30B9, 0x30C7, 0x30B7, 0x30C9, 0x30EB, 0x30C8, + 0x30F3, 0x30CA, 0x30CE, 0x30CE, 0x30C3, 0x30C8, 0x30CF, 0x30A4, 0x30C4, 0x30D1, + 0x30FC, 0x30BB, 0x30F3, 0x30C8, 0x30D1, 0x30FC, 0x30C4, 0x30D0, 0x30FC, 0x30EC, + 0x30EB, 0x30D4, 0x30A2, 0x30B9, 0x30C8, 0x30EB, 0x30D4, 0x30AF, 0x30EB, 0x30D4, + 0x30B3, 0x30D3, 0x30EB, 0x30D5, 0x30A1, 0x30E9, 0x30C3, 0x30C9, 0x30D5, 0x30A3, + 0x30FC, 0x30C8, 0x30D6, 0x30C3, 0x30B7, 0x30A7, 0x30EB, 0x30D5, 0x30E9, 0x30F3, + 0x30D8, 0x30AF, 0x30BF, 0x30FC, 0x30EB, 0x30DA, 0x30BD, 0x30DA, 0x30CB, 0x30D2, + 0x30D8, 0x30EB, 0x30C4, 0x30DA, 0x30F3, 0x30B9, 0x30DA, 0x30FC, 0x30B8, 0x30D9, + 0x30FC, 0x30BF, 0x30DD, 0x30A4, 0x30F3, 0x30C8, 0x30DC, 0x30EB, 0x30C8, 0x30DB, + 0x30F3, 0x30DD, 0x30F3, 0x30C9, 0x30DB, 0x30FC, 0x30EB, 0x30DB, 0x30FC, 0x30F3, + 0x30DE, 0x30A4, 0x30AF, 0x30ED, 0x30DE, 0x30A4, 0x30EB, 0x30DE, 0x30C3, 0x30CF, + 0x30DE, 0x30EB, 0x30AF, 0x30DE, 0x30F3, 0x30B7, 0x30E7, 0x30F3, 0x30DF, 0x30AF, + 0x30ED, 0x30F3, 0x30DF, 0x30EA, 0x30DF, 0x30EA, 0x30D0, 0x30FC, 0x30EB, 0x30E1, + 0x30AC, 0x30E1, 0x30AC, 0x30C8, 0x30F3, 0x30E1, 0x30FC, 0x30C8, 0x30EB, 0x30E4, + 0x30FC, 0x30C9, 0x30E4, 0x30FC, 0x30EB, 0x30E6, 0x30A2, 0x30F3, 0x30EA, 0x30C3, + 0x30C8, 0x30EB, 0x30EA, 0x30E9, 0x30EB, 0x30D4, 0x30FC, 0x30EB, 0x30FC, 0x30D6, + 0x30EB, 0x30EC, 0x30E0, 0x30EC, 0x30F3, 0x30C8, 0x30B2, 0x30F3, 0x30EF, 0x30C3, + 0x30C8, 0x30, 0x70B9, 0x31, 0x70B9, 0x32, 0x70B9, 0x33, 0x70B9, 0x34, + 0x70B9, 0x35, 0x70B9, 0x36, 0x70B9, 0x37, 0x70B9, 0x38, 0x70B9, 0x39, + 0x70B9, 0x31, 0x30, 0x70B9, 0x31, 0x31, 0x70B9, 0x31, 0x32, 0x70B9, + 0x31, 0x33, 0x70B9, 0x31, 0x34, 0x70B9, 0x31, 0x35, 0x70B9, 0x31, + 0x36, 0x70B9, 0x31, 0x37, 0x70B9, 0x31, 0x38, 0x70B9, 0x31, 0x39, + 0x70B9, 0x32, 0x30, 0x70B9, 0x32, 0x31, 0x70B9, 0x32, 0x32, 0x70B9, + 0x32, 0x33, 0x70B9, 0x32, 0x34, 0x70B9, 0x68, 0x50, 0x61, 0x64, + 0x61, 0x41, 0x55, 0x62, 0x61, 0x72, 0x6F, 0x56, 0x70, 0x63, + 0x64, 0x6D, 0x64, 0x6D, 0x32, 0x64, 0x6D, 0x33, 0x49, 0x55, + 0x5E73, 0x6210, 0x662D, 0x548C, 0x5927, 0x6B63, 0x660E, 0x6CBB, 0x682A, 0x5F0F, + 0x4F1A, 0x793E, 0x70, 0x41, 0x6E, 0x41, 0x3BC, 0x41, 0x6D, 0x41, + 0x6B, 0x41, 0x4B, 0x42, 0x4D, 0x42, 0x47, 0x42, 0x63, 0x61, + 0x6C, 0x6B, 0x63, 0x61, 0x6C, 0x70, 0x46, 0x6E, 0x46, 0x3BC, + 0x46, 0x3BC, 0x67, 0x6D, 0x67, 0x6B, 0x67, 0x48, 0x7A, 0x6B, + 0x48, 0x7A, 0x4D, 0x48, 0x7A, 0x47, 0x48, 0x7A, 0x54, 0x48, + 0x7A, 0x3BC, 0x6C, 0x6D, 0x6C, 0x64, 0x6C, 0x6B, 0x6C, 0x66, + 0x6D, 0x6E, 0x6D, 0x3BC, 0x6D, 0x6D, 0x6D, 0x63, 0x6D, 0x6B, + 0x6D, 0x6D, 0x6D, 0x32, 0x63, 0x6D, 0x32, 0x6D, 0x32, 0x6B, + 0x6D, 0x32, 0x6D, 0x6D, 0x33, 0x63, 0x6D, 0x33, 0x6D, 0x33, + 0x6B, 0x6D, 0x33, 0x6D, 0x2215, 0x73, 0x6D, 0x2215, 0x73, 0x32, + 0x50, 0x61, 0x6B, 0x50, 0x61, 0x4D, 0x50, 0x61, 0x47, 0x50, + 0x61, 0x72, 0x61, 0x64, 0x72, 0x61, 0x64, 0x2215, 0x73, 0x72, + 0x61, 0x64, 0x2215, 0x73, 0x32, 0x70, 0x73, 0x6E, 0x73, 0x3BC, + 0x73, 0x6D, 0x73, 0x70, 0x56, 0x6E, 0x56, 0x3BC, 0x56, 0x6D, + 0x56, 0x6B, 0x56, 0x4D, 0x56, 0x70, 0x57, 0x6E, 0x57, 0x3BC, + 0x57, 0x6D, 0x57, 0x6B, 0x57, 0x4D, 0x57, 0x6B, 0x3A9, 0x4D, + 0x3A9, 0x61, 0x2E, 0x6D, 0x2E, 0x42, 0x71, 0x63, 0x63, 0x63, + 0x64, 0x43, 0x2215, 0x6B, 0x67, 0x43, 0x6F, 0x2E, 0x64, 0x42, + 0x47, 0x79, 0x68, 0x61, 0x48, 0x50, 0x69, 0x6E, 0x4B, 0x4B, + 0x4B, 0x4D, 0x6B, 0x74, 0x6C, 0x6D, 0x6C, 0x6E, 0x6C, 0x6F, + 0x67, 0x6C, 0x78, 0x6D, 0x62, 0x6D, 0x69, 0x6C, 0x6D, 0x6F, + 0x6C, 0x50, 0x48, 0x70, 0x2E, 0x6D, 0x2E, 0x50, 0x50, 0x4D, + 0x50, 0x52, 0x73, 0x72, 0x53, 0x76, 0x57, 0x62, 0x56, 0x2215, + 0x6D, 0x41, 0x2215, 0x6D, 0x31, 0x65E5, 0x32, 0x65E5, 0x33, 0x65E5, + 0x34, 0x65E5, 0x35, 0x65E5, 0x36, 0x65E5, 0x37, 0x65E5, 0x38, 0x65E5, + 0x39, 0x65E5, 0x31, 0x30, 0x65E5, 0x31, 0x31, 0x65E5, 0x31, 0x32, + 0x65E5, 0x31, 0x33, 0x65E5, 0x31, 0x34, 0x65E5, 0x31, 0x35, 0x65E5, + 0x31, 0x36, 0x65E5, 0x31, 0x37, 0x65E5, 0x31, 0x38, 0x65E5, 0x31, + 0x39, 0x65E5, 0x32, 0x30, 0x65E5, 0x32, 0x31, 0x65E5, 0x32, 0x32, + 0x65E5, 0x32, 0x33, 0x65E5, 0x32, 0x34, 0x65E5, 0x32, 0x35, 0x65E5, + 0x32, 0x36, 0x65E5, 0x32, 0x37, 0x65E5, 0x32, 0x38, 0x65E5, 0x32, + 0x39, 0x65E5, 0x33, 0x30, 0x65E5, 0x33, 0x31, 0x65E5, 0x67, 0x61, + 0x6C, 0x44A, 0x44C, 0xA76F, 0x43, 0x46, 0x51, 0x126, 0x153, 0xA727, + 0xAB37, 0x26B, 0xAB52, 0x28D, 0x8C48, 0x66F4, 0x8ECA, 0x8CC8, 0x6ED1, 0x4E32, + 0x53E5, 0x9F9C, 0x9F9C, 0x5951, 0x91D1, 0x5587, 0x5948, 0x61F6, 0x7669, 0x7F85, + 0x863F, 0x87BA, 0x88F8, 0x908F, 0x6A02, 0x6D1B, 0x70D9, 0x73DE, 0x843D, 0x916A, + 0x99F1, 0x4E82, 0x5375, 0x6B04, 0x721B, 0x862D, 0x9E1E, 0x5D50, 0x6FEB, 0x85CD, + 0x8964, 0x62C9, 0x81D8, 0x881F, 0x5ECA, 0x6717, 0x6D6A, 0x72FC, 0x90CE, 0x4F86, + 0x51B7, 0x52DE, 0x64C4, 0x6AD3, 0x7210, 0x76E7, 0x8001, 0x8606, 0x865C, 0x8DEF, + 0x9732, 0x9B6F, 0x9DFA, 0x788C, 0x797F, 0x7DA0, 0x83C9, 0x9304, 0x9E7F, 0x8AD6, + 0x58DF, 0x5F04, 0x7C60, 0x807E, 0x7262, 0x78CA, 0x8CC2, 0x96F7, 0x58D8, 0x5C62, + 0x6A13, 0x6DDA, 0x6F0F, 0x7D2F, 0x7E37, 0x964B, 0x52D2, 0x808B, 0x51DC, 0x51CC, + 0x7A1C, 0x7DBE, 0x83F1, 0x9675, 0x8B80, 0x62CF, 0x6A02, 0x8AFE, 0x4E39, 0x5BE7, + 0x6012, 0x7387, 0x7570, 0x5317, 0x78FB, 0x4FBF, 0x5FA9, 0x4E0D, 0x6CCC, 0x6578, + 0x7D22, 0x53C3, 0x585E, 0x7701, 0x8449, 0x8AAA, 0x6BBA, 0x8FB0, 0x6C88, 0x62FE, + 0x82E5, 0x63A0, 0x7565, 0x4EAE, 0x5169, 0x51C9, 0x6881, 0x7CE7, 0x826F, 0x8AD2, + 0x91CF, 0x52F5, 0x5442, 0x5973, 0x5EEC, 0x65C5, 0x6FFE, 0x792A, 0x95AD, 0x9A6A, + 0x9E97, 0x9ECE, 0x529B, 0x66C6, 0x6B77, 0x8F62, 0x5E74, 0x6190, 0x6200, 0x649A, + 0x6F23, 0x7149, 0x7489, 0x79CA, 0x7DF4, 0x806F, 0x8F26, 0x84EE, 0x9023, 0x934A, + 0x5217, 0x52A3, 0x54BD, 0x70C8, 0x88C2, 0x8AAA, 0x5EC9, 0x5FF5, 0x637B, 0x6BAE, + 0x7C3E, 0x7375, 0x4EE4, 0x56F9, 0x5BE7, 0x5DBA, 0x601C, 0x73B2, 0x7469, 0x7F9A, + 0x8046, 0x9234, 0x96F6, 0x9748, 0x9818, 0x4F8B, 0x79AE, 0x91B4, 0x96B8, 0x60E1, + 0x4E86, 0x50DA, 0x5BEE, 0x5C3F, 0x6599, 0x6A02, 0x71CE, 0x7642, 0x84FC, 0x907C, + 0x9F8D, 0x6688, 0x962E, 0x5289, 0x677B, 0x67F3, 0x6D41, 0x6E9C, 0x7409, 0x7559, + 0x786B, 0x7D10, 0x985E, 0x516D, 0x622E, 0x9678, 0x502B, 0x5D19, 0x6DEA, 0x8F2A, + 0x5F8B, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540F, 0x5C65, 0x6613, 0x674E, + 0x68A8, 0x6CE5, 0x7406, 0x75E2, 0x7F79, 0x88CF, 0x88E1, 0x91CC, 0x96E2, 0x533F, + 0x6EBA, 0x541D, 0x71D0, 0x7498, 0x85FA, 0x96A3, 0x9C57, 0x9E9F, 0x6797, 0x6DCB, + 0x81E8, 0x7ACB, 0x7B20, 0x7C92, 0x72C0, 0x7099, 0x8B58, 0x4EC0, 0x8336, 0x523A, + 0x5207, 0x5EA6, 0x62D3, 0x7CD6, 0x5B85, 0x6D1E, 0x66B4, 0x8F3B, 0x884C, 0x964D, + 0x898B, 0x5ED3, 0x5140, 0x55C0, 0x585A, 0x6674, 0x51DE, 0x732A, 0x76CA, 0x793C, + 0x795E, 0x7965, 0x798F, 0x9756, 0x7CBE, 0x7FBD, 0x8612, 0x8AF8, 0x9038, 0x90FD, + 0x98EF, 0x98FC, 0x9928, 0x9DB4, 0x90DE, 0x96B7, 0x4FAE, 0x50E7, 0x514D, 0x52C9, + 0x52E4, 0x5351, 0x559D, 0x5606, 0x5668, 0x5840, 0x58A8, 0x5C64, 0x5C6E, 0x6094, + 0x6168, 0x618E, 0x61F2, 0x654F, 0x65E2, 0x6691, 0x6885, 0x6D77, 0x6E1A, 0x6F22, + 0x716E, 0x722B, 0x7422, 0x7891, 0x793E, 0x7949, 0x7948, 0x7950, 0x7956, 0x795D, + 0x798D, 0x798E, 0x7A40, 0x7A81, 0x7BC0, 0x7DF4, 0x7E09, 0x7E41, 0x7F72, 0x8005, + 0x81ED, 0x8279, 0x8279, 0x8457, 0x8910, 0x8996, 0x8B01, 0x8B39, 0x8CD3, 0x8D08, + 0x8FB6, 0x9038, 0x96E3, 0x97FF, 0x983B, 0x6075, 0x242EE, 0x8218, 0x4E26, 0x51B5, + 0x5168, 0x4F80, 0x5145, 0x5180, 0x52C7, 0x52FA, 0x559D, 0x5555, 0x5599, 0x55E2, + 0x585A, 0x58B3, 0x5944, 0x5954, 0x5A62, 0x5B28, 0x5ED2, 0x5ED9, 0x5F69, 0x5FAD, + 0x60D8, 0x614E, 0x6108, 0x618E, 0x6160, 0x61F2, 0x6234, 0x63C4, 0x641C, 0x6452, + 0x6556, 0x6674, 0x6717, 0x671B, 0x6756, 0x6B79, 0x6BBA, 0x6D41, 0x6EDB, 0x6ECB, + 0x6F22, 0x701E, 0x716E, 0x77A7, 0x7235, 0x72AF, 0x732A, 0x7471, 0x7506, 0x753B, + 0x761D, 0x761F, 0x76CA, 0x76DB, 0x76F4, 0x774A, 0x7740, 0x78CC, 0x7AB1, 0x7BC0, + 0x7C7B, 0x7D5B, 0x7DF4, 0x7F3E, 0x8005, 0x8352, 0x83EF, 0x8779, 0x8941, 0x8986, + 0x8996, 0x8ABF, 0x8AF8, 0x8ACB, 0x8B01, 0x8AFE, 0x8AED, 0x8B39, 0x8B8A, 0x8D08, + 0x8F38, 0x9072, 0x9199, 0x9276, 0x967C, 0x96E3, 0x9756, 0x97DB, 0x97FF, 0x980B, + 0x983B, 0x9B12, 0x9F9C, 0x2284A, 0x22844, 0x233D5, 0x3B9D, 0x4018, 0x4039, 0x25249, + 0x25CD0, 0x27ED3, 0x9F43, 0x9F8E, 0x66, 0x66, 0x66, 0x69, 0x66, 0x6C, + 0x66, 0x66, 0x69, 0x66, 0x66, 0x6C, 0x73, 0x74, 0x73, 0x74, + 0x574, 0x576, 0x574, 0x565, 0x574, 0x56B, 0x57E, 0x576, 0x574, 0x56D, + 0x5D9, 0x5B4, 0x5F2, 0x5B7, 0x5E2, 0x5D0, 0x5D3, 0x5D4, 0x5DB, 0x5DC, + 0x5DD, 0x5E8, 0x5EA, 0x2B, 0x5E9, 0x5C1, 0x5E9, 0x5C2, 0x5E9, 0x5BC, + 0x5C1, 0x5E9, 0x5BC, 0x5C2, 0x5D0, 0x5B7, 0x5D0, 0x5B8, 0x5D0, 0x5BC, + 0x5D1, 0x5BC, 0x5D2, 0x5BC, 0x5D3, 0x5BC, 0x5D4, 0x5BC, 0x5D5, 0x5BC, + 0x5D6, 0x5BC, 0x5D8, 0x5BC, 0x5D9, 0x5BC, 0x5DA, 0x5BC, 0x5DB, 0x5BC, + 0x5DC, 0x5BC, 0x5DE, 0x5BC, 0x5E0, 0x5BC, 0x5E1, 0x5BC, 0x5E3, 0x5BC, + 0x5E4, 0x5BC, 0x5E6, 0x5BC, 0x5E7, 0x5BC, 0x5E8, 0x5BC, 0x5E9, 0x5BC, + 0x5EA, 0x5BC, 0x5D5, 0x5B9, 0x5D1, 0x5BF, 0x5DB, 0x5BF, 0x5E4, 0x5BF, + 0x5D0, 0x5DC, 0x671, 0x671, 0x67B, 0x67B, 0x67B, 0x67B, 0x67E, 0x67E, + 0x67E, 0x67E, 0x680, 0x680, 0x680, 0x680, 0x67A, 0x67A, 0x67A, 0x67A, + 0x67F, 0x67F, 0x67F, 0x67F, 0x679, 0x679, 0x679, 0x679, 0x6A4, 0x6A4, + 0x6A4, 0x6A4, 0x6A6, 0x6A6, 0x6A6, 0x6A6, 0x684, 0x684, 0x684, 0x684, + 0x683, 0x683, 0x683, 0x683, 0x686, 0x686, 0x686, 0x686, 0x687, 0x687, + 0x687, 0x687, 0x68D, 0x68D, 0x68C, 0x68C, 0x68E, 0x68E, 0x688, 0x688, + 0x698, 0x698, 0x691, 0x691, 0x6A9, 0x6A9, 0x6A9, 0x6A9, 0x6AF, 0x6AF, + 0x6AF, 0x6AF, 0x6B3, 0x6B3, 0x6B3, 0x6B3, 0x6B1, 0x6B1, 0x6B1, 0x6B1, + 0x6BA, 0x6BA, 0x6BB, 0x6BB, 0x6BB, 0x6BB, 0x6C0, 0x6C0, 0x6C1, 0x6C1, + 0x6C1, 0x6C1, 0x6BE, 0x6BE, 0x6BE, 0x6BE, 0x6D2, 0x6D2, 0x6D3, 0x6D3, + 0x6AD, 0x6AD, 0x6AD, 0x6AD, 0x6C7, 0x6C7, 0x6C6, 0x6C6, 0x6C8, 0x6C8, + 0x6C7, 0x674, 0x6CB, 0x6CB, 0x6C5, 0x6C5, 0x6C9, 0x6C9, 0x6D0, 0x6D0, + 0x6D0, 0x6D0, 0x649, 0x649, 0x626, 0x627, 0x626, 0x627, 0x626, 0x6D5, + 0x626, 0x6D5, 0x626, 0x648, 0x626, 0x648, 0x626, 0x6C7, 0x626, 0x6C7, + 0x626, 0x6C6, 0x626, 0x6C6, 0x626, 0x6C8, 0x626, 0x6C8, 0x626, 0x6D0, + 0x626, 0x6D0, 0x626, 0x6D0, 0x626, 0x649, 0x626, 0x649, 0x626, 0x649, + 0x6CC, 0x6CC, 0x6CC, 0x6CC, 0x626, 0x62C, 0x626, 0x62D, 0x626, 0x645, + 0x626, 0x649, 0x626, 0x64A, 0x628, 0x62C, 0x628, 0x62D, 0x628, 0x62E, + 0x628, 0x645, 0x628, 0x649, 0x628, 0x64A, 0x62A, 0x62C, 0x62A, 0x62D, + 0x62A, 0x62E, 0x62A, 0x645, 0x62A, 0x649, 0x62A, 0x64A, 0x62B, 0x62C, + 0x62B, 0x645, 0x62B, 0x649, 0x62B, 0x64A, 0x62C, 0x62D, 0x62C, 0x645, + 0x62D, 0x62C, 0x62D, 0x645, 0x62E, 0x62C, 0x62E, 0x62D, 0x62E, 0x645, + 0x633, 0x62C, 0x633, 0x62D, 0x633, 0x62E, 0x633, 0x645, 0x635, 0x62D, + 0x635, 0x645, 0x636, 0x62C, 0x636, 0x62D, 0x636, 0x62E, 0x636, 0x645, + 0x637, 0x62D, 0x637, 0x645, 0x638, 0x645, 0x639, 0x62C, 0x639, 0x645, + 0x63A, 0x62C, 0x63A, 0x645, 0x641, 0x62C, 0x641, 0x62D, 0x641, 0x62E, + 0x641, 0x645, 0x641, 0x649, 0x641, 0x64A, 0x642, 0x62D, 0x642, 0x645, + 0x642, 0x649, 0x642, 0x64A, 0x643, 0x627, 0x643, 0x62C, 0x643, 0x62D, + 0x643, 0x62E, 0x643, 0x644, 0x643, 0x645, 0x643, 0x649, 0x643, 0x64A, + 0x644, 0x62C, 0x644, 0x62D, 0x644, 0x62E, 0x644, 0x645, 0x644, 0x649, + 0x644, 0x64A, 0x645, 0x62C, 0x645, 0x62D, 0x645, 0x62E, 0x645, 0x645, + 0x645, 0x649, 0x645, 0x64A, 0x646, 0x62C, 0x646, 0x62D, 0x646, 0x62E, + 0x646, 0x645, 0x646, 0x649, 0x646, 0x64A, 0x647, 0x62C, 0x647, 0x645, + 0x647, 0x649, 0x647, 0x64A, 0x64A, 0x62C, 0x64A, 0x62D, 0x64A, 0x62E, + 0x64A, 0x645, 0x64A, 0x649, 0x64A, 0x64A, 0x630, 0x670, 0x631, 0x670, + 0x649, 0x670, 0x20, 0x64C, 0x651, 0x20, 0x64D, 0x651, 0x20, 0x64E, + 0x651, 0x20, 0x64F, 0x651, 0x20, 0x650, 0x651, 0x20, 0x651, 0x670, + 0x626, 0x631, 0x626, 0x632, 0x626, 0x645, 0x626, 0x646, 0x626, 0x649, + 0x626, 0x64A, 0x628, 0x631, 0x628, 0x632, 0x628, 0x645, 0x628, 0x646, + 0x628, 0x649, 0x628, 0x64A, 0x62A, 0x631, 0x62A, 0x632, 0x62A, 0x645, + 0x62A, 0x646, 0x62A, 0x649, 0x62A, 0x64A, 0x62B, 0x631, 0x62B, 0x632, + 0x62B, 0x645, 0x62B, 0x646, 0x62B, 0x649, 0x62B, 0x64A, 0x641, 0x649, + 0x641, 0x64A, 0x642, 0x649, 0x642, 0x64A, 0x643, 0x627, 0x643, 0x644, + 0x643, 0x645, 0x643, 0x649, 0x643, 0x64A, 0x644, 0x645, 0x644, 0x649, + 0x644, 0x64A, 0x645, 0x627, 0x645, 0x645, 0x646, 0x631, 0x646, 0x632, + 0x646, 0x645, 0x646, 0x646, 0x646, 0x649, 0x646, 0x64A, 0x649, 0x670, + 0x64A, 0x631, 0x64A, 0x632, 0x64A, 0x645, 0x64A, 0x646, 0x64A, 0x649, + 0x64A, 0x64A, 0x626, 0x62C, 0x626, 0x62D, 0x626, 0x62E, 0x626, 0x645, + 0x626, 0x647, 0x628, 0x62C, 0x628, 0x62D, 0x628, 0x62E, 0x628, 0x645, + 0x628, 0x647, 0x62A, 0x62C, 0x62A, 0x62D, 0x62A, 0x62E, 0x62A, 0x645, + 0x62A, 0x647, 0x62B, 0x645, 0x62C, 0x62D, 0x62C, 0x645, 0x62D, 0x62C, + 0x62D, 0x645, 0x62E, 0x62C, 0x62E, 0x645, 0x633, 0x62C, 0x633, 0x62D, + 0x633, 0x62E, 0x633, 0x645, 0x635, 0x62D, 0x635, 0x62E, 0x635, 0x645, + 0x636, 0x62C, 0x636, 0x62D, 0x636, 0x62E, 0x636, 0x645, 0x637, 0x62D, + 0x638, 0x645, 0x639, 0x62C, 0x639, 0x645, 0x63A, 0x62C, 0x63A, 0x645, + 0x641, 0x62C, 0x641, 0x62D, 0x641, 0x62E, 0x641, 0x645, 0x642, 0x62D, + 0x642, 0x645, 0x643, 0x62C, 0x643, 0x62D, 0x643, 0x62E, 0x643, 0x644, + 0x643, 0x645, 0x644, 0x62C, 0x644, 0x62D, 0x644, 0x62E, 0x644, 0x645, + 0x644, 0x647, 0x645, 0x62C, 0x645, 0x62D, 0x645, 0x62E, 0x645, 0x645, + 0x646, 0x62C, 0x646, 0x62D, 0x646, 0x62E, 0x646, 0x645, 0x646, 0x647, + 0x647, 0x62C, 0x647, 0x645, 0x647, 0x670, 0x64A, 0x62C, 0x64A, 0x62D, + 0x64A, 0x62E, 0x64A, 0x645, 0x64A, 0x647, 0x626, 0x645, 0x626, 0x647, + 0x628, 0x645, 0x628, 0x647, 0x62A, 0x645, 0x62A, 0x647, 0x62B, 0x645, + 0x62B, 0x647, 0x633, 0x645, 0x633, 0x647, 0x634, 0x645, 0x634, 0x647, + 0x643, 0x644, 0x643, 0x645, 0x644, 0x645, 0x646, 0x645, 0x646, 0x647, + 0x64A, 0x645, 0x64A, 0x647, 0x640, 0x64E, 0x651, 0x640, 0x64F, 0x651, + 0x640, 0x650, 0x651, 0x637, 0x649, 0x637, 0x64A, 0x639, 0x649, 0x639, + 0x64A, 0x63A, 0x649, 0x63A, 0x64A, 0x633, 0x649, 0x633, 0x64A, 0x634, + 0x649, 0x634, 0x64A, 0x62D, 0x649, 0x62D, 0x64A, 0x62C, 0x649, 0x62C, + 0x64A, 0x62E, 0x649, 0x62E, 0x64A, 0x635, 0x649, 0x635, 0x64A, 0x636, + 0x649, 0x636, 0x64A, 0x634, 0x62C, 0x634, 0x62D, 0x634, 0x62E, 0x634, + 0x645, 0x634, 0x631, 0x633, 0x631, 0x635, 0x631, 0x636, 0x631, 0x637, + 0x649, 0x637, 0x64A, 0x639, 0x649, 0x639, 0x64A, 0x63A, 0x649, 0x63A, + 0x64A, 0x633, 0x649, 0x633, 0x64A, 0x634, 0x649, 0x634, 0x64A, 0x62D, + 0x649, 0x62D, 0x64A, 0x62C, 0x649, 0x62C, 0x64A, 0x62E, 0x649, 0x62E, + 0x64A, 0x635, 0x649, 0x635, 0x64A, 0x636, 0x649, 0x636, 0x64A, 0x634, + 0x62C, 0x634, 0x62D, 0x634, 0x62E, 0x634, 0x645, 0x634, 0x631, 0x633, + 0x631, 0x635, 0x631, 0x636, 0x631, 0x634, 0x62C, 0x634, 0x62D, 0x634, + 0x62E, 0x634, 0x645, 0x633, 0x647, 0x634, 0x647, 0x637, 0x645, 0x633, + 0x62C, 0x633, 0x62D, 0x633, 0x62E, 0x634, 0x62C, 0x634, 0x62D, 0x634, + 0x62E, 0x637, 0x645, 0x638, 0x645, 0x627, 0x64B, 0x627, 0x64B, 0x62A, + 0x62C, 0x645, 0x62A, 0x62D, 0x62C, 0x62A, 0x62D, 0x62C, 0x62A, 0x62D, + 0x645, 0x62A, 0x62E, 0x645, 0x62A, 0x645, 0x62C, 0x62A, 0x645, 0x62D, + 0x62A, 0x645, 0x62E, 0x62C, 0x645, 0x62D, 0x62C, 0x645, 0x62D, 0x62D, + 0x645, 0x64A, 0x62D, 0x645, 0x649, 0x633, 0x62D, 0x62C, 0x633, 0x62C, + 0x62D, 0x633, 0x62C, 0x649, 0x633, 0x645, 0x62D, 0x633, 0x645, 0x62D, + 0x633, 0x645, 0x62C, 0x633, 0x645, 0x645, 0x633, 0x645, 0x645, 0x635, + 0x62D, 0x62D, 0x635, 0x62D, 0x62D, 0x635, 0x645, 0x645, 0x634, 0x62D, + 0x645, 0x634, 0x62D, 0x645, 0x634, 0x62C, 0x64A, 0x634, 0x645, 0x62E, + 0x634, 0x645, 0x62E, 0x634, 0x645, 0x645, 0x634, 0x645, 0x645, 0x636, + 0x62D, 0x649, 0x636, 0x62E, 0x645, 0x636, 0x62E, 0x645, 0x637, 0x645, + 0x62D, 0x637, 0x645, 0x62D, 0x637, 0x645, 0x645, 0x637, 0x645, 0x64A, + 0x639, 0x62C, 0x645, 0x639, 0x645, 0x645, 0x639, 0x645, 0x645, 0x639, + 0x645, 0x649, 0x63A, 0x645, 0x645, 0x63A, 0x645, 0x64A, 0x63A, 0x645, + 0x649, 0x641, 0x62E, 0x645, 0x641, 0x62E, 0x645, 0x642, 0x645, 0x62D, + 0x642, 0x645, 0x645, 0x644, 0x62D, 0x645, 0x644, 0x62D, 0x64A, 0x644, + 0x62D, 0x649, 0x644, 0x62C, 0x62C, 0x644, 0x62C, 0x62C, 0x644, 0x62E, + 0x645, 0x644, 0x62E, 0x645, 0x644, 0x645, 0x62D, 0x644, 0x645, 0x62D, + 0x645, 0x62D, 0x62C, 0x645, 0x62D, 0x645, 0x645, 0x62D, 0x64A, 0x645, + 0x62C, 0x62D, 0x645, 0x62C, 0x645, 0x645, 0x62E, 0x62C, 0x645, 0x62E, + 0x645, 0x645, 0x62C, 0x62E, 0x647, 0x645, 0x62C, 0x647, 0x645, 0x645, + 0x646, 0x62D, 0x645, 0x646, 0x62D, 0x649, 0x646, 0x62C, 0x645, 0x646, + 0x62C, 0x645, 0x646, 0x62C, 0x649, 0x646, 0x645, 0x64A, 0x646, 0x645, + 0x649, 0x64A, 0x645, 0x645, 0x64A, 0x645, 0x645, 0x628, 0x62E, 0x64A, + 0x62A, 0x62C, 0x64A, 0x62A, 0x62C, 0x649, 0x62A, 0x62E, 0x64A, 0x62A, + 0x62E, 0x649, 0x62A, 0x645, 0x64A, 0x62A, 0x645, 0x649, 0x62C, 0x645, + 0x64A, 0x62C, 0x62D, 0x649, 0x62C, 0x645, 0x649, 0x633, 0x62E, 0x649, + 0x635, 0x62D, 0x64A, 0x634, 0x62D, 0x64A, 0x636, 0x62D, 0x64A, 0x644, + 0x62C, 0x64A, 0x644, 0x645, 0x64A, 0x64A, 0x62D, 0x64A, 0x64A, 0x62C, + 0x64A, 0x64A, 0x645, 0x64A, 0x645, 0x645, 0x64A, 0x642, 0x645, 0x64A, + 0x646, 0x62D, 0x64A, 0x642, 0x645, 0x62D, 0x644, 0x62D, 0x645, 0x639, + 0x645, 0x64A, 0x643, 0x645, 0x64A, 0x646, 0x62C, 0x62D, 0x645, 0x62E, + 0x64A, 0x644, 0x62C, 0x645, 0x643, 0x645, 0x645, 0x644, 0x62C, 0x645, + 0x646, 0x62C, 0x62D, 0x62C, 0x62D, 0x64A, 0x62D, 0x62C, 0x64A, 0x645, + 0x62C, 0x64A, 0x641, 0x645, 0x64A, 0x628, 0x62D, 0x64A, 0x643, 0x645, + 0x645, 0x639, 0x62C, 0x645, 0x635, 0x645, 0x645, 0x633, 0x62E, 0x64A, + 0x646, 0x62C, 0x64A, 0x635, 0x644, 0x6D2, 0x642, 0x644, 0x6D2, 0x627, + 0x644, 0x644, 0x647, 0x627, 0x643, 0x628, 0x631, 0x645, 0x62D, 0x645, + 0x62F, 0x635, 0x644, 0x639, 0x645, 0x631, 0x633, 0x648, 0x644, 0x639, + 0x644, 0x64A, 0x647, 0x648, 0x633, 0x644, 0x645, 0x635, 0x644, 0x649, + 0x635, 0x644, 0x649, 0x20, 0x627, 0x644, 0x644, 0x647, 0x20, 0x639, + 0x644, 0x64A, 0x647, 0x20, 0x648, 0x633, 0x644, 0x645, 0x62C, 0x644, + 0x20, 0x62C, 0x644, 0x627, 0x644, 0x647, 0x631, 0x6CC, 0x627, 0x644, + 0x2C, 0x3001, 0x3002, 0x3A, 0x3B, 0x21, 0x3F, 0x3016, 0x3017, 0x2E, + 0x2E, 0x2E, 0x2E, 0x2E, 0x2014, 0x2013, 0x5F, 0x5F, 0x28, 0x29, + 0x7B, 0x7D, 0x3014, 0x3015, 0x3010, 0x3011, 0x300A, 0x300B, 0x3008, 0x3009, + 0x300C, 0x300D, 0x300E, 0x300F, 0x5B, 0x5D, 0x20, 0x305, 0x20, 0x305, + 0x20, 0x305, 0x20, 0x305, 0x5F, 0x5F, 0x5F, 0x2C, 0x3001, 0x2E, + 0x3B, 0x3A, 0x3F, 0x21, 0x2014, 0x28, 0x29, 0x7B, 0x7D, 0x3014, + 0x3015, 0x23, 0x26, 0x2A, 0x2B, 0x2D, 0x3C, 0x3E, 0x3D, 0x5C, + 0x24, 0x25, 0x40, 0x20, 0x64B, 0x640, 0x64B, 0x20, 0x64C, 0x20, + 0x64D, 0x20, 0x64E, 0x640, 0x64E, 0x20, 0x64F, 0x640, 0x64F, 0x20, + 0x650, 0x640, 0x650, 0x20, 0x651, 0x640, 0x651, 0x20, 0x652, 0x640, + 0x652, 0x621, 0x622, 0x622, 0x623, 0x623, 0x624, 0x624, 0x625, 0x625, + 0x626, 0x626, 0x626, 0x626, 0x627, 0x627, 0x628, 0x628, 0x628, 0x628, + 0x629, 0x629, 0x62A, 0x62A, 0x62A, 0x62A, 0x62B, 0x62B, 0x62B, 0x62B, + 0x62C, 0x62C, 0x62C, 0x62C, 0x62D, 0x62D, 0x62D, 0x62D, 0x62E, 0x62E, + 0x62E, 0x62E, 0x62F, 0x62F, 0x630, 0x630, 0x631, 0x631, 0x632, 0x632, + 0x633, 0x633, 0x633, 0x633, 0x634, 0x634, 0x634, 0x634, 0x635, 0x635, + 0x635, 0x635, 0x636, 0x636, 0x636, 0x636, 0x637, 0x637, 0x637, 0x637, + 0x638, 0x638, 0x638, 0x638, 0x639, 0x639, 0x639, 0x639, 0x63A, 0x63A, + 0x63A, 0x63A, 0x641, 0x641, 0x641, 0x641, 0x642, 0x642, 0x642, 0x642, + 0x643, 0x643, 0x643, 0x643, 0x644, 0x644, 0x644, 0x644, 0x645, 0x645, + 0x645, 0x645, 0x646, 0x646, 0x646, 0x646, 0x647, 0x647, 0x647, 0x647, + 0x648, 0x648, 0x649, 0x649, 0x64A, 0x64A, 0x64A, 0x64A, 0x644, 0x622, + 0x644, 0x622, 0x644, 0x623, 0x644, 0x623, 0x644, 0x625, 0x644, 0x625, + 0x644, 0x627, 0x644, 0x627, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, + 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, + 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, + 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 0x60, 0x61, 0x62, + 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, + 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, + 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x2985, 0x2986, + 0x3002, 0x300C, 0x300D, 0x3001, 0x30FB, 0x30F2, 0x30A1, 0x30A3, 0x30A5, 0x30A7, + 0x30A9, 0x30E3, 0x30E5, 0x30E7, 0x30C3, 0x30FC, 0x30A2, 0x30A4, 0x30A6, 0x30A8, + 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3, 0x30B5, 0x30B7, 0x30B9, 0x30BB, + 0x30BD, 0x30BF, 0x30C1, 0x30C4, 0x30C6, 0x30C8, 0x30CA, 0x30CB, 0x30CC, 0x30CD, + 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, 0x30DE, 0x30DF, 0x30E0, 0x30E1, + 0x30E2, 0x30E4, 0x30E6, 0x30E8, 0x30E9, 0x30EA, 0x30EB, 0x30EC, 0x30ED, 0x30EF, + 0x30F3, 0x3099, 0x309A, 0x1160, 0x1100, 0x1101, 0x11AA, 0x1102, 0x11AC, 0x11AD, + 0x1103, 0x1104, 0x1105, 0x11B0, 0x11B1, 0x11B2, 0x11B3, 0x11B4, 0x11B5, 0x111A, + 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110A, 0x110B, 0x110C, 0x110D, 0x110E, + 0x110F, 0x1110, 0x1111, 0x1112, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, + 0x1167, 0x1168, 0x1169, 0x116A, 0x116B, 0x116C, 0x116D, 0x116E, 0x116F, 0x1170, + 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0xA2, 0xA3, 0xAC, 0x20, 0x304, + 0xA6, 0xA5, 0x20A9, 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25A0, 0x25CB, + 0x2D0, 0x2D1, 0xE6, 0x299, 0x253, 0x2A3, 0xAB66, 0x2A5, 0x2A4, 0x256, + 0x257, 0x1D91, 0x258, 0x25E, 0x2A9, 0x264, 0x262, 0x260, 0x29B, 0x127, + 0x29C, 0x267, 0x284, 0x2AA, 0x2AB, 0x26C, 0x1DF04, 0xA78E, 0x26E, 0x1DF05, + 0x28E, 0x1DF06, 0xF8, 0x276, 0x277, 0x71, 0x27A, 0x1DF08, 0x27D, 0x27E, + 0x280, 0x2A8, 0x2A6, 0xAB67, 0x2A7, 0x288, 0x2C71, 0x28F, 0x2A1, 0x2A2, + 0x298, 0x1C0, 0x1C1, 0x1C2, 0x1DF0A, 0x1DF1E, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, + 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x1D157, 0x1D165, 0x1D158, 0x1D165, 0x1D158, 0x1D165, 0x1D16E, 0x1D158, + 0x1D165, 0x1D16F, 0x1D158, 0x1D165, 0x1D170, 0x1D158, 0x1D165, 0x1D171, 0x1D158, 0x1D165, + 0x1D172, 0x1D1B9, 0x1D165, 0x1D1BA, 0x1D165, 0x1D1B9, 0x1D165, 0x1D16E, 0x1D1BA, 0x1D165, + 0x1D16E, 0x1D1B9, 0x1D165, 0x1D16F, 0x1D1BA, 0x1D165, 0x1D16F, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, + 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, + 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7A, 0x41, 0x43, 0x44, 0x47, 0x4A, 0x4B, 0x4E, 0x4F, + 0x50, 0x51, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, + 0x61, 0x62, 0x63, 0x64, 0x66, 0x68, 0x69, 0x6A, 0x6B, 0x6C, + 0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x44, 0x45, 0x46, + 0x47, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, + 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7A, 0x41, 0x42, 0x44, 0x45, 0x46, 0x47, 0x49, 0x4A, + 0x4B, 0x4C, 0x4D, 0x4F, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, + 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, + 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x131, + 0x237, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, + 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, + 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, + 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, + 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, + 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, + 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, + 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, + 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, + 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, + 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, + 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, + 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, + 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, + 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, + 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, + 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, + 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, 0x394, 0x395, + 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, + 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, + 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, + 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, + 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, + 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, + 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, + 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, + 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, + 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, + 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, + 0x3C0, 0x3DC, 0x3DD, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x430, 0x431, 0x432, 0x433, 0x434, 0x435, 0x436, + 0x437, 0x438, 0x43A, 0x43B, 0x43C, 0x43E, 0x43F, 0x440, 0x441, 0x442, + 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x44B, 0x44D, 0x44E, 0xA689, + 0x4D9, 0x456, 0x458, 0x4E9, 0x4AF, 0x4CF, 0x430, 0x431, 0x432, 0x433, + 0x434, 0x435, 0x436, 0x437, 0x438, 0x43A, 0x43B, 0x43E, 0x43F, 0x441, + 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x44A, 0x44B, 0x491, 0x456, + 0x455, 0x45F, 0x4AB, 0xA651, 0x4B1, 0x627, 0x628, 0x62C, 0x62F, 0x648, + 0x632, 0x62D, 0x637, 0x64A, 0x643, 0x644, 0x645, 0x646, 0x633, 0x639, + 0x641, 0x635, 0x642, 0x631, 0x634, 0x62A, 0x62B, 0x62E, 0x630, 0x636, + 0x638, 0x63A, 0x66E, 0x6BA, 0x6A1, 0x66F, 0x628, 0x62C, 0x647, 0x62D, + 0x64A, 0x643, 0x644, 0x645, 0x646, 0x633, 0x639, 0x641, 0x635, 0x642, + 0x634, 0x62A, 0x62B, 0x62E, 0x636, 0x63A, 0x62C, 0x62D, 0x64A, 0x644, + 0x646, 0x633, 0x639, 0x635, 0x642, 0x634, 0x62E, 0x636, 0x63A, 0x6BA, + 0x66F, 0x628, 0x62C, 0x647, 0x62D, 0x637, 0x64A, 0x643, 0x645, 0x646, + 0x633, 0x639, 0x641, 0x635, 0x642, 0x634, 0x62A, 0x62B, 0x62E, 0x636, + 0x638, 0x63A, 0x66E, 0x6A1, 0x627, 0x628, 0x62C, 0x62F, 0x647, 0x648, + 0x632, 0x62D, 0x637, 0x64A, 0x644, 0x645, 0x646, 0x633, 0x639, 0x641, + 0x635, 0x642, 0x631, 0x634, 0x62A, 0x62B, 0x62E, 0x630, 0x636, 0x638, + 0x63A, 0x628, 0x62C, 0x62F, 0x648, 0x632, 0x62D, 0x637, 0x64A, 0x644, + 0x645, 0x646, 0x633, 0x639, 0x641, 0x635, 0x642, 0x631, 0x634, 0x62A, + 0x62B, 0x62E, 0x630, 0x636, 0x638, 0x63A, 0x30, 0x2E, 0x30, 0x2C, + 0x31, 0x2C, 0x32, 0x2C, 0x33, 0x2C, 0x34, 0x2C, 0x35, 0x2C, + 0x36, 0x2C, 0x37, 0x2C, 0x38, 0x2C, 0x39, 0x2C, 0x28, 0x41, + 0x29, 0x28, 0x42, 0x29, 0x28, 0x43, 0x29, 0x28, 0x44, 0x29, + 0x28, 0x45, 0x29, 0x28, 0x46, 0x29, 0x28, 0x47, 0x29, 0x28, + 0x48, 0x29, 0x28, 0x49, 0x29, 0x28, 0x4A, 0x29, 0x28, 0x4B, + 0x29, 0x28, 0x4C, 0x29, 0x28, 0x4D, 0x29, 0x28, 0x4E, 0x29, + 0x28, 0x4F, 0x29, 0x28, 0x50, 0x29, 0x28, 0x51, 0x29, 0x28, + 0x52, 0x29, 0x28, 0x53, 0x29, 0x28, 0x54, 0x29, 0x28, 0x55, + 0x29, 0x28, 0x56, 0x29, 0x28, 0x57, 0x29, 0x28, 0x58, 0x29, + 0x28, 0x59, 0x29, 0x28, 0x5A, 0x29, 0x3014, 0x53, 0x3015, 0x43, + 0x52, 0x43, 0x44, 0x57, 0x5A, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5A, 0x48, 0x56, 0x4D, 0x56, 0x53, 0x44, 0x53, 0x53, 0x50, + 0x50, 0x56, 0x57, 0x43, 0x4D, 0x43, 0x4D, 0x44, 0x4D, 0x52, + 0x44, 0x4A, 0x307B, 0x304B, 0x30B3, 0x30B3, 0x30B5, 0x624B, 0x5B57, 0x53CC, + 0x30C7, 0x4E8C, 0x591A, 0x89E3, 0x5929, 0x4EA4, 0x6620, 0x7121, 0x6599, 0x524D, + 0x5F8C, 0x518D, 0x65B0, 0x521D, 0x7D42, 0x751F, 0x8CA9, 0x58F0, 0x5439, 0x6F14, + 0x6295, 0x6355, 0x4E00, 0x4E09, 0x904A, 0x5DE6, 0x4E2D, 0x53F3, 0x6307, 0x8D70, + 0x6253, 0x7981, 0x7A7A, 0x5408, 0x6E80, 0x6709, 0x6708, 0x7533, 0x5272, 0x55B6, + 0x914D, 0x3014, 0x672C, 0x3015, 0x3014, 0x4E09, 0x3015, 0x3014, 0x4E8C, 0x3015, + 0x3014, 0x5B89, 0x3015, 0x3014, 0x70B9, 0x3015, 0x3014, 0x6253, 0x3015, 0x3014, + 0x76D7, 0x3015, 0x3014, 0x52DD, 0x3015, 0x3014, 0x6557, 0x3015, 0x5F97, 0x53EF, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x4E3D, 0x4E38, 0x4E41, 0x20122, 0x4F60, 0x4FAE, 0x4FBB, 0x5002, 0x507A, 0x5099, + 0x50E7, 0x50CF, 0x349E, 0x2063A, 0x514D, 0x5154, 0x5164, 0x5177, 0x2051C, 0x34B9, + 0x5167, 0x518D, 0x2054B, 0x5197, 0x51A4, 0x4ECC, 0x51AC, 0x51B5, 0x291DF, 0x51F5, + 0x5203, 0x34DF, 0x523B, 0x5246, 0x5272, 0x5277, 0x3515, 0x52C7, 0x52C9, 0x52E4, + 0x52FA, 0x5305, 0x5306, 0x5317, 0x5349, 0x5351, 0x535A, 0x5373, 0x537D, 0x537F, + 0x537F, 0x537F, 0x20A2C, 0x7070, 0x53CA, 0x53DF, 0x20B63, 0x53EB, 0x53F1, 0x5406, + 0x549E, 0x5438, 0x5448, 0x5468, 0x54A2, 0x54F6, 0x5510, 0x5553, 0x5563, 0x5584, + 0x5584, 0x5599, 0x55AB, 0x55B3, 0x55C2, 0x5716, 0x5606, 0x5717, 0x5651, 0x5674, + 0x5207, 0x58EE, 0x57CE, 0x57F4, 0x580D, 0x578B, 0x5832, 0x5831, 0x58AC, 0x214E4, + 0x58F2, 0x58F7, 0x5906, 0x591A, 0x5922, 0x5962, 0x216A8, 0x216EA, 0x59EC, 0x5A1B, + 0x5A27, 0x59D8, 0x5A66, 0x36EE, 0x36FC, 0x5B08, 0x5B3E, 0x5B3E, 0x219C8, 0x5BC3, + 0x5BD8, 0x5BE7, 0x5BF3, 0x21B18, 0x5BFF, 0x5C06, 0x5F53, 0x5C22, 0x3781, 0x5C60, + 0x5C6E, 0x5CC0, 0x5C8D, 0x21DE4, 0x5D43, 0x21DE6, 0x5D6E, 0x5D6B, 0x5D7C, 0x5DE1, + 0x5DE2, 0x382F, 0x5DFD, 0x5E28, 0x5E3D, 0x5E69, 0x3862, 0x22183, 0x387C, 0x5EB0, + 0x5EB3, 0x5EB6, 0x5ECA, 0x2A392, 0x5EFE, 0x22331, 0x22331, 0x8201, 0x5F22, 0x5F22, + 0x38C7, 0x232B8, 0x261DA, 0x5F62, 0x5F6B, 0x38E3, 0x5F9A, 0x5FCD, 0x5FD7, 0x5FF9, + 0x6081, 0x393A, 0x391C, 0x6094, 0x226D4, 0x60C7, 0x6148, 0x614C, 0x614E, 0x614C, + 0x617A, 0x618E, 0x61B2, 0x61A4, 0x61AF, 0x61DE, 0x61F2, 0x61F6, 0x6210, 0x621B, + 0x625D, 0x62B1, 0x62D4, 0x6350, 0x22B0C, 0x633D, 0x62FC, 0x6368, 0x6383, 0x63E4, + 0x22BF1, 0x6422, 0x63C5, 0x63A9, 0x3A2E, 0x6469, 0x647E, 0x649D, 0x6477, 0x3A6C, + 0x654F, 0x656C, 0x2300A, 0x65E3, 0x66F8, 0x6649, 0x3B19, 0x6691, 0x3B08, 0x3AE4, + 0x5192, 0x5195, 0x6700, 0x669C, 0x80AD, 0x43D9, 0x6717, 0x671B, 0x6721, 0x675E, + 0x6753, 0x233C3, 0x3B49, 0x67FA, 0x6785, 0x6852, 0x6885, 0x2346D, 0x688E, 0x681F, + 0x6914, 0x3B9D, 0x6942, 0x69A3, 0x69EA, 0x6AA8, 0x236A3, 0x6ADB, 0x3C18, 0x6B21, + 0x238A7, 0x6B54, 0x3C4E, 0x6B72, 0x6B9F, 0x6BBA, 0x6BBB, 0x23A8D, 0x21D0B, 0x23AFA, + 0x6C4E, 0x23CBC, 0x6CBF, 0x6CCD, 0x6C67, 0x6D16, 0x6D3E, 0x6D77, 0x6D41, 0x6D69, + 0x6D78, 0x6D85, 0x23D1E, 0x6D34, 0x6E2F, 0x6E6E, 0x3D33, 0x6ECB, 0x6EC7, 0x23ED1, + 0x6DF9, 0x6F6E, 0x23F5E, 0x23F8E, 0x6FC6, 0x7039, 0x701E, 0x701B, 0x3D96, 0x704A, + 0x707D, 0x7077, 0x70AD, 0x20525, 0x7145, 0x24263, 0x719C, 0x243AB, 0x7228, 0x7235, + 0x7250, 0x24608, 0x7280, 0x7295, 0x24735, 0x24814, 0x737A, 0x738B, 0x3EAC, 0x73A5, + 0x3EB8, 0x3EB8, 0x7447, 0x745C, 0x7471, 0x7485, 0x74CA, 0x3F1B, 0x7524, 0x24C36, + 0x753E, 0x24C92, 0x7570, 0x2219F, 0x7610, 0x24FA1, 0x24FB8, 0x25044, 0x3FFC, 0x4008, + 0x76F4, 0x250F3, 0x250F2, 0x25119, 0x25133, 0x771E, 0x771F, 0x771F, 0x774A, 0x4039, + 0x778B, 0x4046, 0x4096, 0x2541D, 0x784E, 0x788C, 0x78CC, 0x40E3, 0x25626, 0x7956, + 0x2569A, 0x256C5, 0x798F, 0x79EB, 0x412F, 0x7A40, 0x7A4A, 0x7A4F, 0x2597C, 0x25AA7, + 0x25AA7, 0x7AEE, 0x4202, 0x25BAB, 0x7BC6, 0x7BC9, 0x4227, 0x25C80, 0x7CD2, 0x42A0, + 0x7CE8, 0x7CE3, 0x7D00, 0x25F86, 0x7D63, 0x4301, 0x7DC7, 0x7E02, 0x7E45, 0x4334, + 0x26228, 0x26247, 0x4359, 0x262D9, 0x7F7A, 0x2633E, 0x7F95, 0x7FFA, 0x8005, 0x264DA, + 0x26523, 0x8060, 0x265A8, 0x8070, 0x2335F, 0x43D5, 0x80B2, 0x8103, 0x440B, 0x813E, + 0x5AB5, 0x267A7, 0x267B5, 0x23393, 0x2339C, 0x8201, 0x8204, 0x8F9E, 0x446B, 0x8291, + 0x828B, 0x829D, 0x52B3, 0x82B1, 0x82B3, 0x82BD, 0x82E6, 0x26B3C, 0x82E5, 0x831D, + 0x8363, 0x83AD, 0x8323, 0x83BD, 0x83E7, 0x8457, 0x8353, 0x83CA, 0x83CC, 0x83DC, + 0x26C36, 0x26D6B, 0x26CD5, 0x452B, 0x84F1, 0x84F3, 0x8516, 0x273CA, 0x8564, 0x26F2C, + 0x455D, 0x4561, 0x26FB1, 0x270D2, 0x456B, 0x8650, 0x865C, 0x8667, 0x8669, 0x86A9, + 0x8688, 0x870E, 0x86E2, 0x8779, 0x8728, 0x876B, 0x8786, 0x45D7, 0x87E1, 0x8801, + 0x45F9, 0x8860, 0x8863, 0x27667, 0x88D7, 0x88DE, 0x4635, 0x88FA, 0x34BB, 0x278AE, + 0x27966, 0x46BE, 0x46C7, 0x8AA0, 0x8AED, 0x8B8A, 0x8C55, 0x27CA8, 0x8CAB, 0x8CC1, + 0x8D1B, 0x8D77, 0x27F2F, 0x20804, 0x8DCB, 0x8DBC, 0x8DF0, 0x208DE, 0x8ED4, 0x8F38, + 0x285D2, 0x285ED, 0x9094, 0x90F1, 0x9111, 0x2872E, 0x911B, 0x9238, 0x92D7, 0x92D8, + 0x927C, 0x93F9, 0x9415, 0x28BFA, 0x958B, 0x4995, 0x95B7, 0x28D77, 0x49E6, 0x96C3, + 0x5DB2, 0x9723, 0x29145, 0x2921A, 0x4A6E, 0x4A76, 0x97E0, 0x2940A, 0x4AB2, 0x29496, + 0x980B, 0x980B, 0x9829, 0x295B6, 0x98E2, 0x4B33, 0x9929, 0x99A7, 0x99C2, 0x99FE, + 0x4BCE, 0x29B30, 0x9B12, 0x9C40, 0x9CFD, 0x4CCE, 0x4CED, 0x9D67, 0x2A0CE, 0x4CF8, + 0x2A105, 0x2A20E, 0x2A291, 0x9EBB, 0x4D56, 0x9EF9, 0x9EFE, 0x9F05, 0x9F0F, 0x9F16, + 0x9F3B, 0x2A600, +}; + +const pdf_ucomp_t PDF_UCOMP[] = { + {0x3C, 0x338, 0x226E}, {0x3D, 0x338, 0x2260}, {0x3E, 0x338, 0x226F}, + {0x41, 0x300, 0xC0}, {0x41, 0x301, 0xC1}, {0x41, 0x302, 0xC2}, + {0x41, 0x303, 0xC3}, {0x41, 0x304, 0x100}, {0x41, 0x306, 0x102}, + {0x41, 0x307, 0x226}, {0x41, 0x308, 0xC4}, {0x41, 0x309, 0x1EA2}, + {0x41, 0x30A, 0xC5}, {0x41, 0x30C, 0x1CD}, {0x41, 0x30F, 0x200}, + {0x41, 0x311, 0x202}, {0x41, 0x323, 0x1EA0}, {0x41, 0x325, 0x1E00}, + {0x41, 0x328, 0x104}, {0x42, 0x307, 0x1E02}, {0x42, 0x323, 0x1E04}, + {0x42, 0x331, 0x1E06}, {0x43, 0x301, 0x106}, {0x43, 0x302, 0x108}, + {0x43, 0x307, 0x10A}, {0x43, 0x30C, 0x10C}, {0x43, 0x327, 0xC7}, + {0x44, 0x307, 0x1E0A}, {0x44, 0x30C, 0x10E}, {0x44, 0x323, 0x1E0C}, + {0x44, 0x327, 0x1E10}, {0x44, 0x32D, 0x1E12}, {0x44, 0x331, 0x1E0E}, + {0x45, 0x300, 0xC8}, {0x45, 0x301, 0xC9}, {0x45, 0x302, 0xCA}, + {0x45, 0x303, 0x1EBC}, {0x45, 0x304, 0x112}, {0x45, 0x306, 0x114}, + {0x45, 0x307, 0x116}, {0x45, 0x308, 0xCB}, {0x45, 0x309, 0x1EBA}, + {0x45, 0x30C, 0x11A}, {0x45, 0x30F, 0x204}, {0x45, 0x311, 0x206}, + {0x45, 0x323, 0x1EB8}, {0x45, 0x327, 0x228}, {0x45, 0x328, 0x118}, + {0x45, 0x32D, 0x1E18}, {0x45, 0x330, 0x1E1A}, {0x46, 0x307, 0x1E1E}, + {0x47, 0x301, 0x1F4}, {0x47, 0x302, 0x11C}, {0x47, 0x304, 0x1E20}, + {0x47, 0x306, 0x11E}, {0x47, 0x307, 0x120}, {0x47, 0x30C, 0x1E6}, + {0x47, 0x327, 0x122}, {0x48, 0x302, 0x124}, {0x48, 0x307, 0x1E22}, + {0x48, 0x308, 0x1E26}, {0x48, 0x30C, 0x21E}, {0x48, 0x323, 0x1E24}, + {0x48, 0x327, 0x1E28}, {0x48, 0x32E, 0x1E2A}, {0x49, 0x300, 0xCC}, + {0x49, 0x301, 0xCD}, {0x49, 0x302, 0xCE}, {0x49, 0x303, 0x128}, + {0x49, 0x304, 0x12A}, {0x49, 0x306, 0x12C}, {0x49, 0x307, 0x130}, + {0x49, 0x308, 0xCF}, {0x49, 0x309, 0x1EC8}, {0x49, 0x30C, 0x1CF}, + {0x49, 0x30F, 0x208}, {0x49, 0x311, 0x20A}, {0x49, 0x323, 0x1ECA}, + {0x49, 0x328, 0x12E}, {0x49, 0x330, 0x1E2C}, {0x4A, 0x302, 0x134}, + {0x4B, 0x301, 0x1E30}, {0x4B, 0x30C, 0x1E8}, {0x4B, 0x323, 0x1E32}, + {0x4B, 0x327, 0x136}, {0x4B, 0x331, 0x1E34}, {0x4C, 0x301, 0x139}, + {0x4C, 0x30C, 0x13D}, {0x4C, 0x323, 0x1E36}, {0x4C, 0x327, 0x13B}, + {0x4C, 0x32D, 0x1E3C}, {0x4C, 0x331, 0x1E3A}, {0x4D, 0x301, 0x1E3E}, + {0x4D, 0x307, 0x1E40}, {0x4D, 0x323, 0x1E42}, {0x4E, 0x300, 0x1F8}, + {0x4E, 0x301, 0x143}, {0x4E, 0x303, 0xD1}, {0x4E, 0x307, 0x1E44}, + {0x4E, 0x30C, 0x147}, {0x4E, 0x323, 0x1E46}, {0x4E, 0x327, 0x145}, + {0x4E, 0x32D, 0x1E4A}, {0x4E, 0x331, 0x1E48}, {0x4F, 0x300, 0xD2}, + {0x4F, 0x301, 0xD3}, {0x4F, 0x302, 0xD4}, {0x4F, 0x303, 0xD5}, + {0x4F, 0x304, 0x14C}, {0x4F, 0x306, 0x14E}, {0x4F, 0x307, 0x22E}, + {0x4F, 0x308, 0xD6}, {0x4F, 0x309, 0x1ECE}, {0x4F, 0x30B, 0x150}, + {0x4F, 0x30C, 0x1D1}, {0x4F, 0x30F, 0x20C}, {0x4F, 0x311, 0x20E}, + {0x4F, 0x31B, 0x1A0}, {0x4F, 0x323, 0x1ECC}, {0x4F, 0x328, 0x1EA}, + {0x50, 0x301, 0x1E54}, {0x50, 0x307, 0x1E56}, {0x52, 0x301, 0x154}, + {0x52, 0x307, 0x1E58}, {0x52, 0x30C, 0x158}, {0x52, 0x30F, 0x210}, + {0x52, 0x311, 0x212}, {0x52, 0x323, 0x1E5A}, {0x52, 0x327, 0x156}, + {0x52, 0x331, 0x1E5E}, {0x53, 0x301, 0x15A}, {0x53, 0x302, 0x15C}, + {0x53, 0x307, 0x1E60}, {0x53, 0x30C, 0x160}, {0x53, 0x323, 0x1E62}, + {0x53, 0x326, 0x218}, {0x53, 0x327, 0x15E}, {0x54, 0x307, 0x1E6A}, + {0x54, 0x30C, 0x164}, {0x54, 0x323, 0x1E6C}, {0x54, 0x326, 0x21A}, + {0x54, 0x327, 0x162}, {0x54, 0x32D, 0x1E70}, {0x54, 0x331, 0x1E6E}, + {0x55, 0x300, 0xD9}, {0x55, 0x301, 0xDA}, {0x55, 0x302, 0xDB}, + {0x55, 0x303, 0x168}, {0x55, 0x304, 0x16A}, {0x55, 0x306, 0x16C}, + {0x55, 0x308, 0xDC}, {0x55, 0x309, 0x1EE6}, {0x55, 0x30A, 0x16E}, + {0x55, 0x30B, 0x170}, {0x55, 0x30C, 0x1D3}, {0x55, 0x30F, 0x214}, + {0x55, 0x311, 0x216}, {0x55, 0x31B, 0x1AF}, {0x55, 0x323, 0x1EE4}, + {0x55, 0x324, 0x1E72}, {0x55, 0x328, 0x172}, {0x55, 0x32D, 0x1E76}, + {0x55, 0x330, 0x1E74}, {0x56, 0x303, 0x1E7C}, {0x56, 0x323, 0x1E7E}, + {0x57, 0x300, 0x1E80}, {0x57, 0x301, 0x1E82}, {0x57, 0x302, 0x174}, + {0x57, 0x307, 0x1E86}, {0x57, 0x308, 0x1E84}, {0x57, 0x323, 0x1E88}, + {0x58, 0x307, 0x1E8A}, {0x58, 0x308, 0x1E8C}, {0x59, 0x300, 0x1EF2}, + {0x59, 0x301, 0xDD}, {0x59, 0x302, 0x176}, {0x59, 0x303, 0x1EF8}, + {0x59, 0x304, 0x232}, {0x59, 0x307, 0x1E8E}, {0x59, 0x308, 0x178}, + {0x59, 0x309, 0x1EF6}, {0x59, 0x323, 0x1EF4}, {0x5A, 0x301, 0x179}, + {0x5A, 0x302, 0x1E90}, {0x5A, 0x307, 0x17B}, {0x5A, 0x30C, 0x17D}, + {0x5A, 0x323, 0x1E92}, {0x5A, 0x331, 0x1E94}, {0x61, 0x300, 0xE0}, + {0x61, 0x301, 0xE1}, {0x61, 0x302, 0xE2}, {0x61, 0x303, 0xE3}, + {0x61, 0x304, 0x101}, {0x61, 0x306, 0x103}, {0x61, 0x307, 0x227}, + {0x61, 0x308, 0xE4}, {0x61, 0x309, 0x1EA3}, {0x61, 0x30A, 0xE5}, + {0x61, 0x30C, 0x1CE}, {0x61, 0x30F, 0x201}, {0x61, 0x311, 0x203}, + {0x61, 0x323, 0x1EA1}, {0x61, 0x325, 0x1E01}, {0x61, 0x328, 0x105}, + {0x62, 0x307, 0x1E03}, {0x62, 0x323, 0x1E05}, {0x62, 0x331, 0x1E07}, + {0x63, 0x301, 0x107}, {0x63, 0x302, 0x109}, {0x63, 0x307, 0x10B}, + {0x63, 0x30C, 0x10D}, {0x63, 0x327, 0xE7}, {0x64, 0x307, 0x1E0B}, + {0x64, 0x30C, 0x10F}, {0x64, 0x323, 0x1E0D}, {0x64, 0x327, 0x1E11}, + {0x64, 0x32D, 0x1E13}, {0x64, 0x331, 0x1E0F}, {0x65, 0x300, 0xE8}, + {0x65, 0x301, 0xE9}, {0x65, 0x302, 0xEA}, {0x65, 0x303, 0x1EBD}, + {0x65, 0x304, 0x113}, {0x65, 0x306, 0x115}, {0x65, 0x307, 0x117}, + {0x65, 0x308, 0xEB}, {0x65, 0x309, 0x1EBB}, {0x65, 0x30C, 0x11B}, + {0x65, 0x30F, 0x205}, {0x65, 0x311, 0x207}, {0x65, 0x323, 0x1EB9}, + {0x65, 0x327, 0x229}, {0x65, 0x328, 0x119}, {0x65, 0x32D, 0x1E19}, + {0x65, 0x330, 0x1E1B}, {0x66, 0x307, 0x1E1F}, {0x67, 0x301, 0x1F5}, + {0x67, 0x302, 0x11D}, {0x67, 0x304, 0x1E21}, {0x67, 0x306, 0x11F}, + {0x67, 0x307, 0x121}, {0x67, 0x30C, 0x1E7}, {0x67, 0x327, 0x123}, + {0x68, 0x302, 0x125}, {0x68, 0x307, 0x1E23}, {0x68, 0x308, 0x1E27}, + {0x68, 0x30C, 0x21F}, {0x68, 0x323, 0x1E25}, {0x68, 0x327, 0x1E29}, + {0x68, 0x32E, 0x1E2B}, {0x68, 0x331, 0x1E96}, {0x69, 0x300, 0xEC}, + {0x69, 0x301, 0xED}, {0x69, 0x302, 0xEE}, {0x69, 0x303, 0x129}, + {0x69, 0x304, 0x12B}, {0x69, 0x306, 0x12D}, {0x69, 0x308, 0xEF}, + {0x69, 0x309, 0x1EC9}, {0x69, 0x30C, 0x1D0}, {0x69, 0x30F, 0x209}, + {0x69, 0x311, 0x20B}, {0x69, 0x323, 0x1ECB}, {0x69, 0x328, 0x12F}, + {0x69, 0x330, 0x1E2D}, {0x6A, 0x302, 0x135}, {0x6A, 0x30C, 0x1F0}, + {0x6B, 0x301, 0x1E31}, {0x6B, 0x30C, 0x1E9}, {0x6B, 0x323, 0x1E33}, + {0x6B, 0x327, 0x137}, {0x6B, 0x331, 0x1E35}, {0x6C, 0x301, 0x13A}, + {0x6C, 0x30C, 0x13E}, {0x6C, 0x323, 0x1E37}, {0x6C, 0x327, 0x13C}, + {0x6C, 0x32D, 0x1E3D}, {0x6C, 0x331, 0x1E3B}, {0x6D, 0x301, 0x1E3F}, + {0x6D, 0x307, 0x1E41}, {0x6D, 0x323, 0x1E43}, {0x6E, 0x300, 0x1F9}, + {0x6E, 0x301, 0x144}, {0x6E, 0x303, 0xF1}, {0x6E, 0x307, 0x1E45}, + {0x6E, 0x30C, 0x148}, {0x6E, 0x323, 0x1E47}, {0x6E, 0x327, 0x146}, + {0x6E, 0x32D, 0x1E4B}, {0x6E, 0x331, 0x1E49}, {0x6F, 0x300, 0xF2}, + {0x6F, 0x301, 0xF3}, {0x6F, 0x302, 0xF4}, {0x6F, 0x303, 0xF5}, + {0x6F, 0x304, 0x14D}, {0x6F, 0x306, 0x14F}, {0x6F, 0x307, 0x22F}, + {0x6F, 0x308, 0xF6}, {0x6F, 0x309, 0x1ECF}, {0x6F, 0x30B, 0x151}, + {0x6F, 0x30C, 0x1D2}, {0x6F, 0x30F, 0x20D}, {0x6F, 0x311, 0x20F}, + {0x6F, 0x31B, 0x1A1}, {0x6F, 0x323, 0x1ECD}, {0x6F, 0x328, 0x1EB}, + {0x70, 0x301, 0x1E55}, {0x70, 0x307, 0x1E57}, {0x72, 0x301, 0x155}, + {0x72, 0x307, 0x1E59}, {0x72, 0x30C, 0x159}, {0x72, 0x30F, 0x211}, + {0x72, 0x311, 0x213}, {0x72, 0x323, 0x1E5B}, {0x72, 0x327, 0x157}, + {0x72, 0x331, 0x1E5F}, {0x73, 0x301, 0x15B}, {0x73, 0x302, 0x15D}, + {0x73, 0x307, 0x1E61}, {0x73, 0x30C, 0x161}, {0x73, 0x323, 0x1E63}, + {0x73, 0x326, 0x219}, {0x73, 0x327, 0x15F}, {0x74, 0x307, 0x1E6B}, + {0x74, 0x308, 0x1E97}, {0x74, 0x30C, 0x165}, {0x74, 0x323, 0x1E6D}, + {0x74, 0x326, 0x21B}, {0x74, 0x327, 0x163}, {0x74, 0x32D, 0x1E71}, + {0x74, 0x331, 0x1E6F}, {0x75, 0x300, 0xF9}, {0x75, 0x301, 0xFA}, + {0x75, 0x302, 0xFB}, {0x75, 0x303, 0x169}, {0x75, 0x304, 0x16B}, + {0x75, 0x306, 0x16D}, {0x75, 0x308, 0xFC}, {0x75, 0x309, 0x1EE7}, + {0x75, 0x30A, 0x16F}, {0x75, 0x30B, 0x171}, {0x75, 0x30C, 0x1D4}, + {0x75, 0x30F, 0x215}, {0x75, 0x311, 0x217}, {0x75, 0x31B, 0x1B0}, + {0x75, 0x323, 0x1EE5}, {0x75, 0x324, 0x1E73}, {0x75, 0x328, 0x173}, + {0x75, 0x32D, 0x1E77}, {0x75, 0x330, 0x1E75}, {0x76, 0x303, 0x1E7D}, + {0x76, 0x323, 0x1E7F}, {0x77, 0x300, 0x1E81}, {0x77, 0x301, 0x1E83}, + {0x77, 0x302, 0x175}, {0x77, 0x307, 0x1E87}, {0x77, 0x308, 0x1E85}, + {0x77, 0x30A, 0x1E98}, {0x77, 0x323, 0x1E89}, {0x78, 0x307, 0x1E8B}, + {0x78, 0x308, 0x1E8D}, {0x79, 0x300, 0x1EF3}, {0x79, 0x301, 0xFD}, + {0x79, 0x302, 0x177}, {0x79, 0x303, 0x1EF9}, {0x79, 0x304, 0x233}, + {0x79, 0x307, 0x1E8F}, {0x79, 0x308, 0xFF}, {0x79, 0x309, 0x1EF7}, + {0x79, 0x30A, 0x1E99}, {0x79, 0x323, 0x1EF5}, {0x7A, 0x301, 0x17A}, + {0x7A, 0x302, 0x1E91}, {0x7A, 0x307, 0x17C}, {0x7A, 0x30C, 0x17E}, + {0x7A, 0x323, 0x1E93}, {0x7A, 0x331, 0x1E95}, {0xA8, 0x300, 0x1FED}, + {0xA8, 0x301, 0x385}, {0xA8, 0x342, 0x1FC1}, {0xC2, 0x300, 0x1EA6}, + {0xC2, 0x301, 0x1EA4}, {0xC2, 0x303, 0x1EAA}, {0xC2, 0x309, 0x1EA8}, + {0xC4, 0x304, 0x1DE}, {0xC5, 0x301, 0x1FA}, {0xC6, 0x301, 0x1FC}, + {0xC6, 0x304, 0x1E2}, {0xC7, 0x301, 0x1E08}, {0xCA, 0x300, 0x1EC0}, + {0xCA, 0x301, 0x1EBE}, {0xCA, 0x303, 0x1EC4}, {0xCA, 0x309, 0x1EC2}, + {0xCF, 0x301, 0x1E2E}, {0xD4, 0x300, 0x1ED2}, {0xD4, 0x301, 0x1ED0}, + {0xD4, 0x303, 0x1ED6}, {0xD4, 0x309, 0x1ED4}, {0xD5, 0x301, 0x1E4C}, + {0xD5, 0x304, 0x22C}, {0xD5, 0x308, 0x1E4E}, {0xD6, 0x304, 0x22A}, + {0xD8, 0x301, 0x1FE}, {0xDC, 0x300, 0x1DB}, {0xDC, 0x301, 0x1D7}, + {0xDC, 0x304, 0x1D5}, {0xDC, 0x30C, 0x1D9}, {0xE2, 0x300, 0x1EA7}, + {0xE2, 0x301, 0x1EA5}, {0xE2, 0x303, 0x1EAB}, {0xE2, 0x309, 0x1EA9}, + {0xE4, 0x304, 0x1DF}, {0xE5, 0x301, 0x1FB}, {0xE6, 0x301, 0x1FD}, + {0xE6, 0x304, 0x1E3}, {0xE7, 0x301, 0x1E09}, {0xEA, 0x300, 0x1EC1}, + {0xEA, 0x301, 0x1EBF}, {0xEA, 0x303, 0x1EC5}, {0xEA, 0x309, 0x1EC3}, + {0xEF, 0x301, 0x1E2F}, {0xF4, 0x300, 0x1ED3}, {0xF4, 0x301, 0x1ED1}, + {0xF4, 0x303, 0x1ED7}, {0xF4, 0x309, 0x1ED5}, {0xF5, 0x301, 0x1E4D}, + {0xF5, 0x304, 0x22D}, {0xF5, 0x308, 0x1E4F}, {0xF6, 0x304, 0x22B}, + {0xF8, 0x301, 0x1FF}, {0xFC, 0x300, 0x1DC}, {0xFC, 0x301, 0x1D8}, + {0xFC, 0x304, 0x1D6}, {0xFC, 0x30C, 0x1DA}, {0x102, 0x300, 0x1EB0}, + {0x102, 0x301, 0x1EAE}, {0x102, 0x303, 0x1EB4}, {0x102, 0x309, 0x1EB2}, + {0x103, 0x300, 0x1EB1}, {0x103, 0x301, 0x1EAF}, {0x103, 0x303, 0x1EB5}, + {0x103, 0x309, 0x1EB3}, {0x112, 0x300, 0x1E14}, {0x112, 0x301, 0x1E16}, + {0x113, 0x300, 0x1E15}, {0x113, 0x301, 0x1E17}, {0x14C, 0x300, 0x1E50}, + {0x14C, 0x301, 0x1E52}, {0x14D, 0x300, 0x1E51}, {0x14D, 0x301, 0x1E53}, + {0x15A, 0x307, 0x1E64}, {0x15B, 0x307, 0x1E65}, {0x160, 0x307, 0x1E66}, + {0x161, 0x307, 0x1E67}, {0x168, 0x301, 0x1E78}, {0x169, 0x301, 0x1E79}, + {0x16A, 0x308, 0x1E7A}, {0x16B, 0x308, 0x1E7B}, {0x17F, 0x307, 0x1E9B}, + {0x1A0, 0x300, 0x1EDC}, {0x1A0, 0x301, 0x1EDA}, {0x1A0, 0x303, 0x1EE0}, + {0x1A0, 0x309, 0x1EDE}, {0x1A0, 0x323, 0x1EE2}, {0x1A1, 0x300, 0x1EDD}, + {0x1A1, 0x301, 0x1EDB}, {0x1A1, 0x303, 0x1EE1}, {0x1A1, 0x309, 0x1EDF}, + {0x1A1, 0x323, 0x1EE3}, {0x1AF, 0x300, 0x1EEA}, {0x1AF, 0x301, 0x1EE8}, + {0x1AF, 0x303, 0x1EEE}, {0x1AF, 0x309, 0x1EEC}, {0x1AF, 0x323, 0x1EF0}, + {0x1B0, 0x300, 0x1EEB}, {0x1B0, 0x301, 0x1EE9}, {0x1B0, 0x303, 0x1EEF}, + {0x1B0, 0x309, 0x1EED}, {0x1B0, 0x323, 0x1EF1}, {0x1B7, 0x30C, 0x1EE}, + {0x1EA, 0x304, 0x1EC}, {0x1EB, 0x304, 0x1ED}, {0x226, 0x304, 0x1E0}, + {0x227, 0x304, 0x1E1}, {0x228, 0x306, 0x1E1C}, {0x229, 0x306, 0x1E1D}, + {0x22E, 0x304, 0x230}, {0x22F, 0x304, 0x231}, {0x292, 0x30C, 0x1EF}, + {0x391, 0x300, 0x1FBA}, {0x391, 0x301, 0x386}, {0x391, 0x304, 0x1FB9}, + {0x391, 0x306, 0x1FB8}, {0x391, 0x313, 0x1F08}, {0x391, 0x314, 0x1F09}, + {0x391, 0x345, 0x1FBC}, {0x395, 0x300, 0x1FC8}, {0x395, 0x301, 0x388}, + {0x395, 0x313, 0x1F18}, {0x395, 0x314, 0x1F19}, {0x397, 0x300, 0x1FCA}, + {0x397, 0x301, 0x389}, {0x397, 0x313, 0x1F28}, {0x397, 0x314, 0x1F29}, + {0x397, 0x345, 0x1FCC}, {0x399, 0x300, 0x1FDA}, {0x399, 0x301, 0x38A}, + {0x399, 0x304, 0x1FD9}, {0x399, 0x306, 0x1FD8}, {0x399, 0x308, 0x3AA}, + {0x399, 0x313, 0x1F38}, {0x399, 0x314, 0x1F39}, {0x39F, 0x300, 0x1FF8}, + {0x39F, 0x301, 0x38C}, {0x39F, 0x313, 0x1F48}, {0x39F, 0x314, 0x1F49}, + {0x3A1, 0x314, 0x1FEC}, {0x3A5, 0x300, 0x1FEA}, {0x3A5, 0x301, 0x38E}, + {0x3A5, 0x304, 0x1FE9}, {0x3A5, 0x306, 0x1FE8}, {0x3A5, 0x308, 0x3AB}, + {0x3A5, 0x314, 0x1F59}, {0x3A9, 0x300, 0x1FFA}, {0x3A9, 0x301, 0x38F}, + {0x3A9, 0x313, 0x1F68}, {0x3A9, 0x314, 0x1F69}, {0x3A9, 0x345, 0x1FFC}, + {0x3AC, 0x345, 0x1FB4}, {0x3AE, 0x345, 0x1FC4}, {0x3B1, 0x300, 0x1F70}, + {0x3B1, 0x301, 0x3AC}, {0x3B1, 0x304, 0x1FB1}, {0x3B1, 0x306, 0x1FB0}, + {0x3B1, 0x313, 0x1F00}, {0x3B1, 0x314, 0x1F01}, {0x3B1, 0x342, 0x1FB6}, + {0x3B1, 0x345, 0x1FB3}, {0x3B5, 0x300, 0x1F72}, {0x3B5, 0x301, 0x3AD}, + {0x3B5, 0x313, 0x1F10}, {0x3B5, 0x314, 0x1F11}, {0x3B7, 0x300, 0x1F74}, + {0x3B7, 0x301, 0x3AE}, {0x3B7, 0x313, 0x1F20}, {0x3B7, 0x314, 0x1F21}, + {0x3B7, 0x342, 0x1FC6}, {0x3B7, 0x345, 0x1FC3}, {0x3B9, 0x300, 0x1F76}, + {0x3B9, 0x301, 0x3AF}, {0x3B9, 0x304, 0x1FD1}, {0x3B9, 0x306, 0x1FD0}, + {0x3B9, 0x308, 0x3CA}, {0x3B9, 0x313, 0x1F30}, {0x3B9, 0x314, 0x1F31}, + {0x3B9, 0x342, 0x1FD6}, {0x3BF, 0x300, 0x1F78}, {0x3BF, 0x301, 0x3CC}, + {0x3BF, 0x313, 0x1F40}, {0x3BF, 0x314, 0x1F41}, {0x3C1, 0x313, 0x1FE4}, + {0x3C1, 0x314, 0x1FE5}, {0x3C5, 0x300, 0x1F7A}, {0x3C5, 0x301, 0x3CD}, + {0x3C5, 0x304, 0x1FE1}, {0x3C5, 0x306, 0x1FE0}, {0x3C5, 0x308, 0x3CB}, + {0x3C5, 0x313, 0x1F50}, {0x3C5, 0x314, 0x1F51}, {0x3C5, 0x342, 0x1FE6}, + {0x3C9, 0x300, 0x1F7C}, {0x3C9, 0x301, 0x3CE}, {0x3C9, 0x313, 0x1F60}, + {0x3C9, 0x314, 0x1F61}, {0x3C9, 0x342, 0x1FF6}, {0x3C9, 0x345, 0x1FF3}, + {0x3CA, 0x300, 0x1FD2}, {0x3CA, 0x301, 0x390}, {0x3CA, 0x342, 0x1FD7}, + {0x3CB, 0x300, 0x1FE2}, {0x3CB, 0x301, 0x3B0}, {0x3CB, 0x342, 0x1FE7}, + {0x3CE, 0x345, 0x1FF4}, {0x3D2, 0x301, 0x3D3}, {0x3D2, 0x308, 0x3D4}, + {0x406, 0x308, 0x407}, {0x410, 0x306, 0x4D0}, {0x410, 0x308, 0x4D2}, + {0x413, 0x301, 0x403}, {0x415, 0x300, 0x400}, {0x415, 0x306, 0x4D6}, + {0x415, 0x308, 0x401}, {0x416, 0x306, 0x4C1}, {0x416, 0x308, 0x4DC}, + {0x417, 0x308, 0x4DE}, {0x418, 0x300, 0x40D}, {0x418, 0x304, 0x4E2}, + {0x418, 0x306, 0x419}, {0x418, 0x308, 0x4E4}, {0x41A, 0x301, 0x40C}, + {0x41E, 0x308, 0x4E6}, {0x423, 0x304, 0x4EE}, {0x423, 0x306, 0x40E}, + {0x423, 0x308, 0x4F0}, {0x423, 0x30B, 0x4F2}, {0x427, 0x308, 0x4F4}, + {0x42B, 0x308, 0x4F8}, {0x42D, 0x308, 0x4EC}, {0x430, 0x306, 0x4D1}, + {0x430, 0x308, 0x4D3}, {0x433, 0x301, 0x453}, {0x435, 0x300, 0x450}, + {0x435, 0x306, 0x4D7}, {0x435, 0x308, 0x451}, {0x436, 0x306, 0x4C2}, + {0x436, 0x308, 0x4DD}, {0x437, 0x308, 0x4DF}, {0x438, 0x300, 0x45D}, + {0x438, 0x304, 0x4E3}, {0x438, 0x306, 0x439}, {0x438, 0x308, 0x4E5}, + {0x43A, 0x301, 0x45C}, {0x43E, 0x308, 0x4E7}, {0x443, 0x304, 0x4EF}, + {0x443, 0x306, 0x45E}, {0x443, 0x308, 0x4F1}, {0x443, 0x30B, 0x4F3}, + {0x447, 0x308, 0x4F5}, {0x44B, 0x308, 0x4F9}, {0x44D, 0x308, 0x4ED}, + {0x456, 0x308, 0x457}, {0x474, 0x30F, 0x476}, {0x475, 0x30F, 0x477}, + {0x4D8, 0x308, 0x4DA}, {0x4D9, 0x308, 0x4DB}, {0x4E8, 0x308, 0x4EA}, + {0x4E9, 0x308, 0x4EB}, {0x627, 0x653, 0x622}, {0x627, 0x654, 0x623}, + {0x627, 0x655, 0x625}, {0x648, 0x654, 0x624}, {0x64A, 0x654, 0x626}, + {0x6C1, 0x654, 0x6C2}, {0x6D2, 0x654, 0x6D3}, {0x6D5, 0x654, 0x6C0}, + {0x928, 0x93C, 0x929}, {0x930, 0x93C, 0x931}, {0x933, 0x93C, 0x934}, + {0x9C7, 0x9BE, 0x9CB}, {0x9C7, 0x9D7, 0x9CC}, {0xB47, 0xB3E, 0xB4B}, + {0xB47, 0xB56, 0xB48}, {0xB47, 0xB57, 0xB4C}, {0xB92, 0xBD7, 0xB94}, + {0xBC6, 0xBBE, 0xBCA}, {0xBC6, 0xBD7, 0xBCC}, {0xBC7, 0xBBE, 0xBCB}, + {0xC46, 0xC56, 0xC48}, {0xCBF, 0xCD5, 0xCC0}, {0xCC6, 0xCC2, 0xCCA}, + {0xCC6, 0xCD5, 0xCC7}, {0xCC6, 0xCD6, 0xCC8}, {0xCCA, 0xCD5, 0xCCB}, + {0xD46, 0xD3E, 0xD4A}, {0xD46, 0xD57, 0xD4C}, {0xD47, 0xD3E, 0xD4B}, + {0xDD9, 0xDCA, 0xDDA}, {0xDD9, 0xDCF, 0xDDC}, {0xDD9, 0xDDF, 0xDDE}, + {0xDDC, 0xDCA, 0xDDD}, {0x1025, 0x102E, 0x1026}, {0x1100, 0x1161, 0xAC00}, + {0x1100, 0x1162, 0xAC1C}, {0x1100, 0x1163, 0xAC38}, {0x1100, 0x1164, 0xAC54}, + {0x1100, 0x1165, 0xAC70}, {0x1100, 0x1166, 0xAC8C}, {0x1100, 0x1167, 0xACA8}, + {0x1100, 0x1168, 0xACC4}, {0x1100, 0x1169, 0xACE0}, {0x1100, 0x116A, 0xACFC}, + {0x1100, 0x116B, 0xAD18}, {0x1100, 0x116C, 0xAD34}, {0x1100, 0x116D, 0xAD50}, + {0x1100, 0x116E, 0xAD6C}, {0x1100, 0x116F, 0xAD88}, {0x1100, 0x1170, 0xADA4}, + {0x1100, 0x1171, 0xADC0}, {0x1100, 0x1172, 0xADDC}, {0x1100, 0x1173, 0xADF8}, + {0x1100, 0x1174, 0xAE14}, {0x1100, 0x1175, 0xAE30}, {0x1101, 0x1161, 0xAE4C}, + {0x1101, 0x1162, 0xAE68}, {0x1101, 0x1163, 0xAE84}, {0x1101, 0x1164, 0xAEA0}, + {0x1101, 0x1165, 0xAEBC}, {0x1101, 0x1166, 0xAED8}, {0x1101, 0x1167, 0xAEF4}, + {0x1101, 0x1168, 0xAF10}, {0x1101, 0x1169, 0xAF2C}, {0x1101, 0x116A, 0xAF48}, + {0x1101, 0x116B, 0xAF64}, {0x1101, 0x116C, 0xAF80}, {0x1101, 0x116D, 0xAF9C}, + {0x1101, 0x116E, 0xAFB8}, {0x1101, 0x116F, 0xAFD4}, {0x1101, 0x1170, 0xAFF0}, + {0x1101, 0x1171, 0xB00C}, {0x1101, 0x1172, 0xB028}, {0x1101, 0x1173, 0xB044}, + {0x1101, 0x1174, 0xB060}, {0x1101, 0x1175, 0xB07C}, {0x1102, 0x1161, 0xB098}, + {0x1102, 0x1162, 0xB0B4}, {0x1102, 0x1163, 0xB0D0}, {0x1102, 0x1164, 0xB0EC}, + {0x1102, 0x1165, 0xB108}, {0x1102, 0x1166, 0xB124}, {0x1102, 0x1167, 0xB140}, + {0x1102, 0x1168, 0xB15C}, {0x1102, 0x1169, 0xB178}, {0x1102, 0x116A, 0xB194}, + {0x1102, 0x116B, 0xB1B0}, {0x1102, 0x116C, 0xB1CC}, {0x1102, 0x116D, 0xB1E8}, + {0x1102, 0x116E, 0xB204}, {0x1102, 0x116F, 0xB220}, {0x1102, 0x1170, 0xB23C}, + {0x1102, 0x1171, 0xB258}, {0x1102, 0x1172, 0xB274}, {0x1102, 0x1173, 0xB290}, + {0x1102, 0x1174, 0xB2AC}, {0x1102, 0x1175, 0xB2C8}, {0x1103, 0x1161, 0xB2E4}, + {0x1103, 0x1162, 0xB300}, {0x1103, 0x1163, 0xB31C}, {0x1103, 0x1164, 0xB338}, + {0x1103, 0x1165, 0xB354}, {0x1103, 0x1166, 0xB370}, {0x1103, 0x1167, 0xB38C}, + {0x1103, 0x1168, 0xB3A8}, {0x1103, 0x1169, 0xB3C4}, {0x1103, 0x116A, 0xB3E0}, + {0x1103, 0x116B, 0xB3FC}, {0x1103, 0x116C, 0xB418}, {0x1103, 0x116D, 0xB434}, + {0x1103, 0x116E, 0xB450}, {0x1103, 0x116F, 0xB46C}, {0x1103, 0x1170, 0xB488}, + {0x1103, 0x1171, 0xB4A4}, {0x1103, 0x1172, 0xB4C0}, {0x1103, 0x1173, 0xB4DC}, + {0x1103, 0x1174, 0xB4F8}, {0x1103, 0x1175, 0xB514}, {0x1104, 0x1161, 0xB530}, + {0x1104, 0x1162, 0xB54C}, {0x1104, 0x1163, 0xB568}, {0x1104, 0x1164, 0xB584}, + {0x1104, 0x1165, 0xB5A0}, {0x1104, 0x1166, 0xB5BC}, {0x1104, 0x1167, 0xB5D8}, + {0x1104, 0x1168, 0xB5F4}, {0x1104, 0x1169, 0xB610}, {0x1104, 0x116A, 0xB62C}, + {0x1104, 0x116B, 0xB648}, {0x1104, 0x116C, 0xB664}, {0x1104, 0x116D, 0xB680}, + {0x1104, 0x116E, 0xB69C}, {0x1104, 0x116F, 0xB6B8}, {0x1104, 0x1170, 0xB6D4}, + {0x1104, 0x1171, 0xB6F0}, {0x1104, 0x1172, 0xB70C}, {0x1104, 0x1173, 0xB728}, + {0x1104, 0x1174, 0xB744}, {0x1104, 0x1175, 0xB760}, {0x1105, 0x1161, 0xB77C}, + {0x1105, 0x1162, 0xB798}, {0x1105, 0x1163, 0xB7B4}, {0x1105, 0x1164, 0xB7D0}, + {0x1105, 0x1165, 0xB7EC}, {0x1105, 0x1166, 0xB808}, {0x1105, 0x1167, 0xB824}, + {0x1105, 0x1168, 0xB840}, {0x1105, 0x1169, 0xB85C}, {0x1105, 0x116A, 0xB878}, + {0x1105, 0x116B, 0xB894}, {0x1105, 0x116C, 0xB8B0}, {0x1105, 0x116D, 0xB8CC}, + {0x1105, 0x116E, 0xB8E8}, {0x1105, 0x116F, 0xB904}, {0x1105, 0x1170, 0xB920}, + {0x1105, 0x1171, 0xB93C}, {0x1105, 0x1172, 0xB958}, {0x1105, 0x1173, 0xB974}, + {0x1105, 0x1174, 0xB990}, {0x1105, 0x1175, 0xB9AC}, {0x1106, 0x1161, 0xB9C8}, + {0x1106, 0x1162, 0xB9E4}, {0x1106, 0x1163, 0xBA00}, {0x1106, 0x1164, 0xBA1C}, + {0x1106, 0x1165, 0xBA38}, {0x1106, 0x1166, 0xBA54}, {0x1106, 0x1167, 0xBA70}, + {0x1106, 0x1168, 0xBA8C}, {0x1106, 0x1169, 0xBAA8}, {0x1106, 0x116A, 0xBAC4}, + {0x1106, 0x116B, 0xBAE0}, {0x1106, 0x116C, 0xBAFC}, {0x1106, 0x116D, 0xBB18}, + {0x1106, 0x116E, 0xBB34}, {0x1106, 0x116F, 0xBB50}, {0x1106, 0x1170, 0xBB6C}, + {0x1106, 0x1171, 0xBB88}, {0x1106, 0x1172, 0xBBA4}, {0x1106, 0x1173, 0xBBC0}, + {0x1106, 0x1174, 0xBBDC}, {0x1106, 0x1175, 0xBBF8}, {0x1107, 0x1161, 0xBC14}, + {0x1107, 0x1162, 0xBC30}, {0x1107, 0x1163, 0xBC4C}, {0x1107, 0x1164, 0xBC68}, + {0x1107, 0x1165, 0xBC84}, {0x1107, 0x1166, 0xBCA0}, {0x1107, 0x1167, 0xBCBC}, + {0x1107, 0x1168, 0xBCD8}, {0x1107, 0x1169, 0xBCF4}, {0x1107, 0x116A, 0xBD10}, + {0x1107, 0x116B, 0xBD2C}, {0x1107, 0x116C, 0xBD48}, {0x1107, 0x116D, 0xBD64}, + {0x1107, 0x116E, 0xBD80}, {0x1107, 0x116F, 0xBD9C}, {0x1107, 0x1170, 0xBDB8}, + {0x1107, 0x1171, 0xBDD4}, {0x1107, 0x1172, 0xBDF0}, {0x1107, 0x1173, 0xBE0C}, + {0x1107, 0x1174, 0xBE28}, {0x1107, 0x1175, 0xBE44}, {0x1108, 0x1161, 0xBE60}, + {0x1108, 0x1162, 0xBE7C}, {0x1108, 0x1163, 0xBE98}, {0x1108, 0x1164, 0xBEB4}, + {0x1108, 0x1165, 0xBED0}, {0x1108, 0x1166, 0xBEEC}, {0x1108, 0x1167, 0xBF08}, + {0x1108, 0x1168, 0xBF24}, {0x1108, 0x1169, 0xBF40}, {0x1108, 0x116A, 0xBF5C}, + {0x1108, 0x116B, 0xBF78}, {0x1108, 0x116C, 0xBF94}, {0x1108, 0x116D, 0xBFB0}, + {0x1108, 0x116E, 0xBFCC}, {0x1108, 0x116F, 0xBFE8}, {0x1108, 0x1170, 0xC004}, + {0x1108, 0x1171, 0xC020}, {0x1108, 0x1172, 0xC03C}, {0x1108, 0x1173, 0xC058}, + {0x1108, 0x1174, 0xC074}, {0x1108, 0x1175, 0xC090}, {0x1109, 0x1161, 0xC0AC}, + {0x1109, 0x1162, 0xC0C8}, {0x1109, 0x1163, 0xC0E4}, {0x1109, 0x1164, 0xC100}, + {0x1109, 0x1165, 0xC11C}, {0x1109, 0x1166, 0xC138}, {0x1109, 0x1167, 0xC154}, + {0x1109, 0x1168, 0xC170}, {0x1109, 0x1169, 0xC18C}, {0x1109, 0x116A, 0xC1A8}, + {0x1109, 0x116B, 0xC1C4}, {0x1109, 0x116C, 0xC1E0}, {0x1109, 0x116D, 0xC1FC}, + {0x1109, 0x116E, 0xC218}, {0x1109, 0x116F, 0xC234}, {0x1109, 0x1170, 0xC250}, + {0x1109, 0x1171, 0xC26C}, {0x1109, 0x1172, 0xC288}, {0x1109, 0x1173, 0xC2A4}, + {0x1109, 0x1174, 0xC2C0}, {0x1109, 0x1175, 0xC2DC}, {0x110A, 0x1161, 0xC2F8}, + {0x110A, 0x1162, 0xC314}, {0x110A, 0x1163, 0xC330}, {0x110A, 0x1164, 0xC34C}, + {0x110A, 0x1165, 0xC368}, {0x110A, 0x1166, 0xC384}, {0x110A, 0x1167, 0xC3A0}, + {0x110A, 0x1168, 0xC3BC}, {0x110A, 0x1169, 0xC3D8}, {0x110A, 0x116A, 0xC3F4}, + {0x110A, 0x116B, 0xC410}, {0x110A, 0x116C, 0xC42C}, {0x110A, 0x116D, 0xC448}, + {0x110A, 0x116E, 0xC464}, {0x110A, 0x116F, 0xC480}, {0x110A, 0x1170, 0xC49C}, + {0x110A, 0x1171, 0xC4B8}, {0x110A, 0x1172, 0xC4D4}, {0x110A, 0x1173, 0xC4F0}, + {0x110A, 0x1174, 0xC50C}, {0x110A, 0x1175, 0xC528}, {0x110B, 0x1161, 0xC544}, + {0x110B, 0x1162, 0xC560}, {0x110B, 0x1163, 0xC57C}, {0x110B, 0x1164, 0xC598}, + {0x110B, 0x1165, 0xC5B4}, {0x110B, 0x1166, 0xC5D0}, {0x110B, 0x1167, 0xC5EC}, + {0x110B, 0x1168, 0xC608}, {0x110B, 0x1169, 0xC624}, {0x110B, 0x116A, 0xC640}, + {0x110B, 0x116B, 0xC65C}, {0x110B, 0x116C, 0xC678}, {0x110B, 0x116D, 0xC694}, + {0x110B, 0x116E, 0xC6B0}, {0x110B, 0x116F, 0xC6CC}, {0x110B, 0x1170, 0xC6E8}, + {0x110B, 0x1171, 0xC704}, {0x110B, 0x1172, 0xC720}, {0x110B, 0x1173, 0xC73C}, + {0x110B, 0x1174, 0xC758}, {0x110B, 0x1175, 0xC774}, {0x110C, 0x1161, 0xC790}, + {0x110C, 0x1162, 0xC7AC}, {0x110C, 0x1163, 0xC7C8}, {0x110C, 0x1164, 0xC7E4}, + {0x110C, 0x1165, 0xC800}, {0x110C, 0x1166, 0xC81C}, {0x110C, 0x1167, 0xC838}, + {0x110C, 0x1168, 0xC854}, {0x110C, 0x1169, 0xC870}, {0x110C, 0x116A, 0xC88C}, + {0x110C, 0x116B, 0xC8A8}, {0x110C, 0x116C, 0xC8C4}, {0x110C, 0x116D, 0xC8E0}, + {0x110C, 0x116E, 0xC8FC}, {0x110C, 0x116F, 0xC918}, {0x110C, 0x1170, 0xC934}, + {0x110C, 0x1171, 0xC950}, {0x110C, 0x1172, 0xC96C}, {0x110C, 0x1173, 0xC988}, + {0x110C, 0x1174, 0xC9A4}, {0x110C, 0x1175, 0xC9C0}, {0x110D, 0x1161, 0xC9DC}, + {0x110D, 0x1162, 0xC9F8}, {0x110D, 0x1163, 0xCA14}, {0x110D, 0x1164, 0xCA30}, + {0x110D, 0x1165, 0xCA4C}, {0x110D, 0x1166, 0xCA68}, {0x110D, 0x1167, 0xCA84}, + {0x110D, 0x1168, 0xCAA0}, {0x110D, 0x1169, 0xCABC}, {0x110D, 0x116A, 0xCAD8}, + {0x110D, 0x116B, 0xCAF4}, {0x110D, 0x116C, 0xCB10}, {0x110D, 0x116D, 0xCB2C}, + {0x110D, 0x116E, 0xCB48}, {0x110D, 0x116F, 0xCB64}, {0x110D, 0x1170, 0xCB80}, + {0x110D, 0x1171, 0xCB9C}, {0x110D, 0x1172, 0xCBB8}, {0x110D, 0x1173, 0xCBD4}, + {0x110D, 0x1174, 0xCBF0}, {0x110D, 0x1175, 0xCC0C}, {0x110E, 0x1161, 0xCC28}, + {0x110E, 0x1162, 0xCC44}, {0x110E, 0x1163, 0xCC60}, {0x110E, 0x1164, 0xCC7C}, + {0x110E, 0x1165, 0xCC98}, {0x110E, 0x1166, 0xCCB4}, {0x110E, 0x1167, 0xCCD0}, + {0x110E, 0x1168, 0xCCEC}, {0x110E, 0x1169, 0xCD08}, {0x110E, 0x116A, 0xCD24}, + {0x110E, 0x116B, 0xCD40}, {0x110E, 0x116C, 0xCD5C}, {0x110E, 0x116D, 0xCD78}, + {0x110E, 0x116E, 0xCD94}, {0x110E, 0x116F, 0xCDB0}, {0x110E, 0x1170, 0xCDCC}, + {0x110E, 0x1171, 0xCDE8}, {0x110E, 0x1172, 0xCE04}, {0x110E, 0x1173, 0xCE20}, + {0x110E, 0x1174, 0xCE3C}, {0x110E, 0x1175, 0xCE58}, {0x110F, 0x1161, 0xCE74}, + {0x110F, 0x1162, 0xCE90}, {0x110F, 0x1163, 0xCEAC}, {0x110F, 0x1164, 0xCEC8}, + {0x110F, 0x1165, 0xCEE4}, {0x110F, 0x1166, 0xCF00}, {0x110F, 0x1167, 0xCF1C}, + {0x110F, 0x1168, 0xCF38}, {0x110F, 0x1169, 0xCF54}, {0x110F, 0x116A, 0xCF70}, + {0x110F, 0x116B, 0xCF8C}, {0x110F, 0x116C, 0xCFA8}, {0x110F, 0x116D, 0xCFC4}, + {0x110F, 0x116E, 0xCFE0}, {0x110F, 0x116F, 0xCFFC}, {0x110F, 0x1170, 0xD018}, + {0x110F, 0x1171, 0xD034}, {0x110F, 0x1172, 0xD050}, {0x110F, 0x1173, 0xD06C}, + {0x110F, 0x1174, 0xD088}, {0x110F, 0x1175, 0xD0A4}, {0x1110, 0x1161, 0xD0C0}, + {0x1110, 0x1162, 0xD0DC}, {0x1110, 0x1163, 0xD0F8}, {0x1110, 0x1164, 0xD114}, + {0x1110, 0x1165, 0xD130}, {0x1110, 0x1166, 0xD14C}, {0x1110, 0x1167, 0xD168}, + {0x1110, 0x1168, 0xD184}, {0x1110, 0x1169, 0xD1A0}, {0x1110, 0x116A, 0xD1BC}, + {0x1110, 0x116B, 0xD1D8}, {0x1110, 0x116C, 0xD1F4}, {0x1110, 0x116D, 0xD210}, + {0x1110, 0x116E, 0xD22C}, {0x1110, 0x116F, 0xD248}, {0x1110, 0x1170, 0xD264}, + {0x1110, 0x1171, 0xD280}, {0x1110, 0x1172, 0xD29C}, {0x1110, 0x1173, 0xD2B8}, + {0x1110, 0x1174, 0xD2D4}, {0x1110, 0x1175, 0xD2F0}, {0x1111, 0x1161, 0xD30C}, + {0x1111, 0x1162, 0xD328}, {0x1111, 0x1163, 0xD344}, {0x1111, 0x1164, 0xD360}, + {0x1111, 0x1165, 0xD37C}, {0x1111, 0x1166, 0xD398}, {0x1111, 0x1167, 0xD3B4}, + {0x1111, 0x1168, 0xD3D0}, {0x1111, 0x1169, 0xD3EC}, {0x1111, 0x116A, 0xD408}, + {0x1111, 0x116B, 0xD424}, {0x1111, 0x116C, 0xD440}, {0x1111, 0x116D, 0xD45C}, + {0x1111, 0x116E, 0xD478}, {0x1111, 0x116F, 0xD494}, {0x1111, 0x1170, 0xD4B0}, + {0x1111, 0x1171, 0xD4CC}, {0x1111, 0x1172, 0xD4E8}, {0x1111, 0x1173, 0xD504}, + {0x1111, 0x1174, 0xD520}, {0x1111, 0x1175, 0xD53C}, {0x1112, 0x1161, 0xD558}, + {0x1112, 0x1162, 0xD574}, {0x1112, 0x1163, 0xD590}, {0x1112, 0x1164, 0xD5AC}, + {0x1112, 0x1165, 0xD5C8}, {0x1112, 0x1166, 0xD5E4}, {0x1112, 0x1167, 0xD600}, + {0x1112, 0x1168, 0xD61C}, {0x1112, 0x1169, 0xD638}, {0x1112, 0x116A, 0xD654}, + {0x1112, 0x116B, 0xD670}, {0x1112, 0x116C, 0xD68C}, {0x1112, 0x116D, 0xD6A8}, + {0x1112, 0x116E, 0xD6C4}, {0x1112, 0x116F, 0xD6E0}, {0x1112, 0x1170, 0xD6FC}, + {0x1112, 0x1171, 0xD718}, {0x1112, 0x1172, 0xD734}, {0x1112, 0x1173, 0xD750}, + {0x1112, 0x1174, 0xD76C}, {0x1112, 0x1175, 0xD788}, {0x1B05, 0x1B35, 0x1B06}, + {0x1B07, 0x1B35, 0x1B08}, {0x1B09, 0x1B35, 0x1B0A}, {0x1B0B, 0x1B35, 0x1B0C}, + {0x1B0D, 0x1B35, 0x1B0E}, {0x1B11, 0x1B35, 0x1B12}, {0x1B3A, 0x1B35, 0x1B3B}, + {0x1B3C, 0x1B35, 0x1B3D}, {0x1B3E, 0x1B35, 0x1B40}, {0x1B3F, 0x1B35, 0x1B41}, + {0x1B42, 0x1B35, 0x1B43}, {0x1E36, 0x304, 0x1E38}, {0x1E37, 0x304, 0x1E39}, + {0x1E5A, 0x304, 0x1E5C}, {0x1E5B, 0x304, 0x1E5D}, {0x1E62, 0x307, 0x1E68}, + {0x1E63, 0x307, 0x1E69}, {0x1EA0, 0x302, 0x1EAC}, {0x1EA0, 0x306, 0x1EB6}, + {0x1EA1, 0x302, 0x1EAD}, {0x1EA1, 0x306, 0x1EB7}, {0x1EB8, 0x302, 0x1EC6}, + {0x1EB9, 0x302, 0x1EC7}, {0x1ECC, 0x302, 0x1ED8}, {0x1ECD, 0x302, 0x1ED9}, + {0x1F00, 0x300, 0x1F02}, {0x1F00, 0x301, 0x1F04}, {0x1F00, 0x342, 0x1F06}, + {0x1F00, 0x345, 0x1F80}, {0x1F01, 0x300, 0x1F03}, {0x1F01, 0x301, 0x1F05}, + {0x1F01, 0x342, 0x1F07}, {0x1F01, 0x345, 0x1F81}, {0x1F02, 0x345, 0x1F82}, + {0x1F03, 0x345, 0x1F83}, {0x1F04, 0x345, 0x1F84}, {0x1F05, 0x345, 0x1F85}, + {0x1F06, 0x345, 0x1F86}, {0x1F07, 0x345, 0x1F87}, {0x1F08, 0x300, 0x1F0A}, + {0x1F08, 0x301, 0x1F0C}, {0x1F08, 0x342, 0x1F0E}, {0x1F08, 0x345, 0x1F88}, + {0x1F09, 0x300, 0x1F0B}, {0x1F09, 0x301, 0x1F0D}, {0x1F09, 0x342, 0x1F0F}, + {0x1F09, 0x345, 0x1F89}, {0x1F0A, 0x345, 0x1F8A}, {0x1F0B, 0x345, 0x1F8B}, + {0x1F0C, 0x345, 0x1F8C}, {0x1F0D, 0x345, 0x1F8D}, {0x1F0E, 0x345, 0x1F8E}, + {0x1F0F, 0x345, 0x1F8F}, {0x1F10, 0x300, 0x1F12}, {0x1F10, 0x301, 0x1F14}, + {0x1F11, 0x300, 0x1F13}, {0x1F11, 0x301, 0x1F15}, {0x1F18, 0x300, 0x1F1A}, + {0x1F18, 0x301, 0x1F1C}, {0x1F19, 0x300, 0x1F1B}, {0x1F19, 0x301, 0x1F1D}, + {0x1F20, 0x300, 0x1F22}, {0x1F20, 0x301, 0x1F24}, {0x1F20, 0x342, 0x1F26}, + {0x1F20, 0x345, 0x1F90}, {0x1F21, 0x300, 0x1F23}, {0x1F21, 0x301, 0x1F25}, + {0x1F21, 0x342, 0x1F27}, {0x1F21, 0x345, 0x1F91}, {0x1F22, 0x345, 0x1F92}, + {0x1F23, 0x345, 0x1F93}, {0x1F24, 0x345, 0x1F94}, {0x1F25, 0x345, 0x1F95}, + {0x1F26, 0x345, 0x1F96}, {0x1F27, 0x345, 0x1F97}, {0x1F28, 0x300, 0x1F2A}, + {0x1F28, 0x301, 0x1F2C}, {0x1F28, 0x342, 0x1F2E}, {0x1F28, 0x345, 0x1F98}, + {0x1F29, 0x300, 0x1F2B}, {0x1F29, 0x301, 0x1F2D}, {0x1F29, 0x342, 0x1F2F}, + {0x1F29, 0x345, 0x1F99}, {0x1F2A, 0x345, 0x1F9A}, {0x1F2B, 0x345, 0x1F9B}, + {0x1F2C, 0x345, 0x1F9C}, {0x1F2D, 0x345, 0x1F9D}, {0x1F2E, 0x345, 0x1F9E}, + {0x1F2F, 0x345, 0x1F9F}, {0x1F30, 0x300, 0x1F32}, {0x1F30, 0x301, 0x1F34}, + {0x1F30, 0x342, 0x1F36}, {0x1F31, 0x300, 0x1F33}, {0x1F31, 0x301, 0x1F35}, + {0x1F31, 0x342, 0x1F37}, {0x1F38, 0x300, 0x1F3A}, {0x1F38, 0x301, 0x1F3C}, + {0x1F38, 0x342, 0x1F3E}, {0x1F39, 0x300, 0x1F3B}, {0x1F39, 0x301, 0x1F3D}, + {0x1F39, 0x342, 0x1F3F}, {0x1F40, 0x300, 0x1F42}, {0x1F40, 0x301, 0x1F44}, + {0x1F41, 0x300, 0x1F43}, {0x1F41, 0x301, 0x1F45}, {0x1F48, 0x300, 0x1F4A}, + {0x1F48, 0x301, 0x1F4C}, {0x1F49, 0x300, 0x1F4B}, {0x1F49, 0x301, 0x1F4D}, + {0x1F50, 0x300, 0x1F52}, {0x1F50, 0x301, 0x1F54}, {0x1F50, 0x342, 0x1F56}, + {0x1F51, 0x300, 0x1F53}, {0x1F51, 0x301, 0x1F55}, {0x1F51, 0x342, 0x1F57}, + {0x1F59, 0x300, 0x1F5B}, {0x1F59, 0x301, 0x1F5D}, {0x1F59, 0x342, 0x1F5F}, + {0x1F60, 0x300, 0x1F62}, {0x1F60, 0x301, 0x1F64}, {0x1F60, 0x342, 0x1F66}, + {0x1F60, 0x345, 0x1FA0}, {0x1F61, 0x300, 0x1F63}, {0x1F61, 0x301, 0x1F65}, + {0x1F61, 0x342, 0x1F67}, {0x1F61, 0x345, 0x1FA1}, {0x1F62, 0x345, 0x1FA2}, + {0x1F63, 0x345, 0x1FA3}, {0x1F64, 0x345, 0x1FA4}, {0x1F65, 0x345, 0x1FA5}, + {0x1F66, 0x345, 0x1FA6}, {0x1F67, 0x345, 0x1FA7}, {0x1F68, 0x300, 0x1F6A}, + {0x1F68, 0x301, 0x1F6C}, {0x1F68, 0x342, 0x1F6E}, {0x1F68, 0x345, 0x1FA8}, + {0x1F69, 0x300, 0x1F6B}, {0x1F69, 0x301, 0x1F6D}, {0x1F69, 0x342, 0x1F6F}, + {0x1F69, 0x345, 0x1FA9}, {0x1F6A, 0x345, 0x1FAA}, {0x1F6B, 0x345, 0x1FAB}, + {0x1F6C, 0x345, 0x1FAC}, {0x1F6D, 0x345, 0x1FAD}, {0x1F6E, 0x345, 0x1FAE}, + {0x1F6F, 0x345, 0x1FAF}, {0x1F70, 0x345, 0x1FB2}, {0x1F74, 0x345, 0x1FC2}, + {0x1F7C, 0x345, 0x1FF2}, {0x1FB6, 0x345, 0x1FB7}, {0x1FBF, 0x300, 0x1FCD}, + {0x1FBF, 0x301, 0x1FCE}, {0x1FBF, 0x342, 0x1FCF}, {0x1FC6, 0x345, 0x1FC7}, + {0x1FF6, 0x345, 0x1FF7}, {0x1FFE, 0x300, 0x1FDD}, {0x1FFE, 0x301, 0x1FDE}, + {0x1FFE, 0x342, 0x1FDF}, {0x2190, 0x338, 0x219A}, {0x2192, 0x338, 0x219B}, + {0x2194, 0x338, 0x21AE}, {0x21D0, 0x338, 0x21CD}, {0x21D2, 0x338, 0x21CF}, + {0x21D4, 0x338, 0x21CE}, {0x2203, 0x338, 0x2204}, {0x2208, 0x338, 0x2209}, + {0x220B, 0x338, 0x220C}, {0x2223, 0x338, 0x2224}, {0x2225, 0x338, 0x2226}, + {0x223C, 0x338, 0x2241}, {0x2243, 0x338, 0x2244}, {0x2245, 0x338, 0x2247}, + {0x2248, 0x338, 0x2249}, {0x224D, 0x338, 0x226D}, {0x2261, 0x338, 0x2262}, + {0x2264, 0x338, 0x2270}, {0x2265, 0x338, 0x2271}, {0x2272, 0x338, 0x2274}, + {0x2273, 0x338, 0x2275}, {0x2276, 0x338, 0x2278}, {0x2277, 0x338, 0x2279}, + {0x227A, 0x338, 0x2280}, {0x227B, 0x338, 0x2281}, {0x227C, 0x338, 0x22E0}, + {0x227D, 0x338, 0x22E1}, {0x2282, 0x338, 0x2284}, {0x2283, 0x338, 0x2285}, + {0x2286, 0x338, 0x2288}, {0x2287, 0x338, 0x2289}, {0x2291, 0x338, 0x22E2}, + {0x2292, 0x338, 0x22E3}, {0x22A2, 0x338, 0x22AC}, {0x22A8, 0x338, 0x22AD}, + {0x22A9, 0x338, 0x22AE}, {0x22AB, 0x338, 0x22AF}, {0x22B2, 0x338, 0x22EA}, + {0x22B3, 0x338, 0x22EB}, {0x22B4, 0x338, 0x22EC}, {0x22B5, 0x338, 0x22ED}, + {0x3046, 0x3099, 0x3094}, {0x304B, 0x3099, 0x304C}, {0x304D, 0x3099, 0x304E}, + {0x304F, 0x3099, 0x3050}, {0x3051, 0x3099, 0x3052}, {0x3053, 0x3099, 0x3054}, + {0x3055, 0x3099, 0x3056}, {0x3057, 0x3099, 0x3058}, {0x3059, 0x3099, 0x305A}, + {0x305B, 0x3099, 0x305C}, {0x305D, 0x3099, 0x305E}, {0x305F, 0x3099, 0x3060}, + {0x3061, 0x3099, 0x3062}, {0x3064, 0x3099, 0x3065}, {0x3066, 0x3099, 0x3067}, + {0x3068, 0x3099, 0x3069}, {0x306F, 0x3099, 0x3070}, {0x306F, 0x309A, 0x3071}, + {0x3072, 0x3099, 0x3073}, {0x3072, 0x309A, 0x3074}, {0x3075, 0x3099, 0x3076}, + {0x3075, 0x309A, 0x3077}, {0x3078, 0x3099, 0x3079}, {0x3078, 0x309A, 0x307A}, + {0x307B, 0x3099, 0x307C}, {0x307B, 0x309A, 0x307D}, {0x309D, 0x3099, 0x309E}, + {0x30A6, 0x3099, 0x30F4}, {0x30AB, 0x3099, 0x30AC}, {0x30AD, 0x3099, 0x30AE}, + {0x30AF, 0x3099, 0x30B0}, {0x30B1, 0x3099, 0x30B2}, {0x30B3, 0x3099, 0x30B4}, + {0x30B5, 0x3099, 0x30B6}, {0x30B7, 0x3099, 0x30B8}, {0x30B9, 0x3099, 0x30BA}, + {0x30BB, 0x3099, 0x30BC}, {0x30BD, 0x3099, 0x30BE}, {0x30BF, 0x3099, 0x30C0}, + {0x30C1, 0x3099, 0x30C2}, {0x30C4, 0x3099, 0x30C5}, {0x30C6, 0x3099, 0x30C7}, + {0x30C8, 0x3099, 0x30C9}, {0x30CF, 0x3099, 0x30D0}, {0x30CF, 0x309A, 0x30D1}, + {0x30D2, 0x3099, 0x30D3}, {0x30D2, 0x309A, 0x30D4}, {0x30D5, 0x3099, 0x30D6}, + {0x30D5, 0x309A, 0x30D7}, {0x30D8, 0x3099, 0x30D9}, {0x30D8, 0x309A, 0x30DA}, + {0x30DB, 0x3099, 0x30DC}, {0x30DB, 0x309A, 0x30DD}, {0x30EF, 0x3099, 0x30F7}, + {0x30F0, 0x3099, 0x30F8}, {0x30F1, 0x3099, 0x30F9}, {0x30F2, 0x3099, 0x30FA}, + {0x30FD, 0x3099, 0x30FE}, {0x105D2, 0x307, 0x105C9}, {0x105DA, 0x307, 0x105E4}, + {0x11099, 0x110BA, 0x1109A}, {0x1109B, 0x110BA, 0x1109C}, {0x110A5, 0x110BA, 0x110AB}, + {0x11131, 0x11127, 0x1112E}, {0x11132, 0x11127, 0x1112F}, {0x11347, 0x1133E, 0x1134B}, + {0x11347, 0x11357, 0x1134C}, {0x11382, 0x113C9, 0x11383}, {0x11384, 0x113BB, 0x11385}, + {0x1138B, 0x113C2, 0x1138E}, {0x11390, 0x113C9, 0x11391}, {0x113C2, 0x113B8, 0x113C7}, + {0x113C2, 0x113C2, 0x113C5}, {0x113C2, 0x113C9, 0x113C8}, {0x114B9, 0x114B0, 0x114BC}, + {0x114B9, 0x114BA, 0x114BB}, {0x114B9, 0x114BD, 0x114BE}, {0x115B8, 0x115AF, 0x115BA}, + {0x115B9, 0x115AF, 0x115BB}, {0x11935, 0x11930, 0x11938}, {0x1611E, 0x1611E, 0x16121}, + {0x1611E, 0x1611F, 0x16123}, {0x1611E, 0x16120, 0x16125}, {0x1611E, 0x16129, 0x16122}, + {0x16121, 0x1611F, 0x16126}, {0x16121, 0x16120, 0x16128}, {0x16122, 0x1611F, 0x16127}, + {0x16129, 0x1611F, 0x16124}, {0x16D63, 0x16D67, 0x16D69}, {0x16D67, 0x16D67, 0x16D68}, + {0x16D69, 0x16D67, 0x16D6A}, +}; +const int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0])); + +const pdf_urange_t PDF_U_SPACE[] = { + {0x9, 0xD}, {0x1C, 0x20}, {0x85, 0x85}, {0xA0, 0xA0}, {0x1680, 0x1680}, + {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, +}; +const int PDF_U_SPACE_COUNT = (int)(sizeof(PDF_U_SPACE) / sizeof(PDF_U_SPACE[0])); + +const pdf_urange_t PDF_U_WORD[] = { + {0x30, 0x39}, {0x41, 0x5A}, {0x5F, 0x5F}, {0x61, 0x7A}, + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, + {0xBC, 0xBE}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, + {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, + {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, + {0x559, 0x559}, {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, + {0x620, 0x64A}, {0x660, 0x669}, {0x66E, 0x66F}, {0x671, 0x6D3}, + {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, {0x6FF, 0x6FF}, + {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, + {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, + {0x860, 0x86A}, {0x870, 0x887}, {0x889, 0x88E}, {0x8A0, 0x8C9}, + {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, + {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, + {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, + {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, {0x9FC, 0x9FC}, {0xA05, 0xA0A}, + {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, {0xA32, 0xA33}, + {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, + {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, + {0xABD, 0xABD}, {0xAD0, 0xAD0}, {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, + {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, + {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, + {0xB83, 0xB83}, {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, + {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, + {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xBE6, 0xBF2}, + {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, + {0xC66, 0xC6F}, {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, + {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, + {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, {0xCE6, 0xCEF}, + {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, + {0xD66, 0xD78}, {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, + {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, + {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, {0xE50, 0xE59}, + {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, + {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, + {0xF00, 0xF00}, {0xF20, 0xF33}, {0xF40, 0xF47}, {0xF49, 0xF6C}, + {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, {0x1050, 0x1055}, + {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, + {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, + {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, + {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, + {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, + {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, + {0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, + {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, + {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, + {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, + {0x1900, 0x191E}, {0x1946, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB}, + {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, + {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, + {0x1C00, 0x1C23}, {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C8A}, + {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, + {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, + {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, + {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, + {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, {0x2074, 0x2079}, + {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, + {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, + {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, {0x2150, 0x2189}, + {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, {0x2C00, 0x2CE4}, + {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, + {0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, + {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, + {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, {0x3021, 0x3029}, + {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, + {0x3192, 0x3195}, {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, + {0x3248, 0x324F}, {0x3251, 0x325F}, {0x3280, 0x3289}, {0x32B1, 0x32BF}, + {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, + {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CD}, {0xA7D0, 0xA7D1}, + {0xA7D3, 0xA7D3}, {0xA7D5, 0xA7DC}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, + {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA830, 0xA835}, {0xA840, 0xA873}, + {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, + {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, + {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, + {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, + {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, + {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, + {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, + {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, + {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, + {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, + {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, + {0x10107, 0x10133}, {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A}, + {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, + {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, + {0x1057C, 0x1058A}, {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, + {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, + {0x10858, 0x10876}, {0x10879, 0x1089E}, {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, + {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, + {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, + {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D23}, + {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, {0x10D6F, 0x10D85}, {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC4}, {0x10F00, 0x10F27}, + {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, + {0x11075, 0x11075}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, + {0x11103, 0x11126}, {0x11136, 0x1113F}, {0x11144, 0x11144}, {0x11147, 0x11147}, + {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, + {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, + {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x112F0, 0x112F9}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, + {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, + {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11380, 0x11389}, {0x1138B, 0x1138B}, + {0x1138E, 0x1138E}, {0x11390, 0x113B5}, {0x113B7, 0x113B7}, {0x113D1, 0x113D1}, + {0x113D3, 0x113D3}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x11450, 0x11459}, + {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, + {0x11644, 0x11644}, {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, + {0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A}, {0x11730, 0x1173B}, + {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, + {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, + {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, + {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, + {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11BF0, 0x11BF9}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, + {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, + {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, + {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, + {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, {0x12000, 0x12399}, {0x12400, 0x1246E}, + {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, {0x13441, 0x13446}, + {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, {0x16130, 0x16139}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A70, 0x16ABE}, + {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, + {0x16D40, 0x16D6C}, {0x16D70, 0x16D79}, {0x16E40, 0x16E96}, {0x16F00, 0x16F4A}, + {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18CFF, 0x18D08}, {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, + {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, + {0x1CCF0, 0x1CCF9}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, + {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, + {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, + {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, {0x1DF00, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, + {0x1E2F0, 0x1E2F9}, {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E5D0, 0x1E5ED}, + {0x1E5F0, 0x1E5FA}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, + {0x1E94B, 0x1E94B}, {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, + {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, + {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, + {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1F100, 0x1F10C}, {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, + {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, + {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, +}; +const int PDF_U_WORD_COUNT = (int)(sizeof(PDF_U_WORD) / sizeof(PDF_U_WORD[0])); + +const pdf_urange_t PDF_U_ALPHA[] = { + {0x41, 0x5A}, {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, + {0xBA, 0xBA}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, + {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, + {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, + {0x559, 0x559}, {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, + {0x620, 0x64A}, {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, + {0x6E5, 0x6E6}, {0x6EE, 0x6EF}, {0x6FA, 0x6FC}, {0x6FF, 0x6FF}, + {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7CA, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, + {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, + {0x860, 0x86A}, {0x870, 0x887}, {0x889, 0x88E}, {0x8A0, 0x8C9}, + {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, + {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, + {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9F0, 0x9F1}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, + {0xA2A, 0xA30}, {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, + {0xA59, 0xA5C}, {0xA5E, 0xA5E}, {0xA72, 0xA74}, {0xA85, 0xA8D}, + {0xA8F, 0xA91}, {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, + {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, {0xAE0, 0xAE1}, + {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, + {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB71, 0xB71}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, + {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, + {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, + {0xC12, 0xC28}, {0xC2A, 0xC39}, {0xC3D, 0xC3D}, {0xC58, 0xC5A}, + {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC80, 0xC80}, {0xC85, 0xC8C}, + {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, + {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, {0xCF1, 0xCF2}, + {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, + {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD5F, 0xD61}, {0xD7A, 0xD7F}, + {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, + {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, + {0x103F, 0x103F}, {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, + {0x1065, 0x1066}, {0x106E, 0x1070}, {0x1075, 0x1081}, {0x108E, 0x108E}, + {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, + {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, + {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, + {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, + {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, + {0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16F1, 0x16F8}, + {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, + {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, + {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1950, 0x196D}, {0x1970, 0x1974}, + {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, + {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4C}, {0x1B83, 0x1BA0}, + {0x1BAE, 0x1BAF}, {0x1BBA, 0x1BE5}, {0x1C00, 0x1C23}, {0x1C4D, 0x1C4F}, + {0x1C5A, 0x1C7D}, {0x1C80, 0x1C8A}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, + {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, + {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, + {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, + {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, {0x2102, 0x2102}, + {0x2107, 0x2107}, {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, + {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, + {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, + {0x2183, 0x2184}, {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, + {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, + {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, + {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3006}, + {0x3031, 0x3035}, {0x303B, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, + {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA61F}, {0xA62A, 0xA62B}, + {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6E5}, {0xA717, 0xA71F}, + {0xA722, 0xA788}, {0xA78B, 0xA7CD}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7DC}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, + {0xA80C, 0xA822}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA90A, 0xA925}, {0xA930, 0xA946}, + {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9CF}, {0xA9E0, 0xA9E4}, + {0xA9E6, 0xA9EF}, {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, + {0xAA44, 0xAA4B}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, + {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, + {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, + {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, + {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, + {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, + {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, + {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, + {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x10300, 0x1031F}, {0x1032D, 0x10340}, {0x10342, 0x10349}, + {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, + {0x10400, 0x1049D}, {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, + {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, + {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0}, {0x107B2, 0x107BA}, + {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838}, + {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10860, 0x10876}, {0x10880, 0x1089E}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x10900, 0x10915}, {0x10920, 0x10939}, + {0x10980, 0x109B7}, {0x109BE, 0x109BF}, {0x10A00, 0x10A00}, {0x10A10, 0x10A13}, + {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A60, 0x10A7C}, {0x10A80, 0x10A9C}, + {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, + {0x10B60, 0x10B72}, {0x10B80, 0x10B91}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, + {0x10CC0, 0x10CF2}, {0x10D00, 0x10D23}, {0x10D4A, 0x10D65}, {0x10D6F, 0x10D85}, + {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC4}, {0x10F00, 0x10F1C}, + {0x10F27, 0x10F27}, {0x10F30, 0x10F45}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FC4}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11071, 0x11072}, {0x11075, 0x11075}, + {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x11103, 0x11126}, {0x11144, 0x11144}, + {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, + {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, + {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, + {0x1135D, 0x11361}, {0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, + {0x11390, 0x113B5}, {0x113B7, 0x113B7}, {0x113D1, 0x113D1}, {0x113D3, 0x113D3}, + {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, + {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, + {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, + {0x11700, 0x1171A}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118DF}, + {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, + {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, + {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, + {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, + {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11C00, 0x11C08}, + {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, + {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65}, + {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, + {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11FB0, 0x11FB0}, + {0x12000, 0x12399}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, + {0x13441, 0x13446}, {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A70, 0x16ABE}, {0x16AD0, 0x16AED}, + {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, + {0x16D40, 0x16D6C}, {0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, + {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, {0x18CFF, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, + {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, + {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, + {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7CB}, {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, + {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, + {0x1E2C0, 0x1E2EB}, {0x1E4D0, 0x1E4EB}, {0x1E5D0, 0x1E5ED}, {0x1E5F0, 0x1E5F0}, + {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, + {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, + {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, + {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, + {0x31350, 0x323AF}, +}; +const int PDF_U_ALPHA_COUNT = (int)(sizeof(PDF_U_ALPHA) / sizeof(PDF_U_ALPHA[0])); + +const pdf_urange_t PDF_U_UPPER[] = { + {0x41, 0x5A}, {0xC0, 0xD6}, {0xD8, 0xDE}, {0x100, 0x100}, + {0x102, 0x102}, {0x104, 0x104}, {0x106, 0x106}, {0x108, 0x108}, + {0x10A, 0x10A}, {0x10C, 0x10C}, {0x10E, 0x10E}, {0x110, 0x110}, + {0x112, 0x112}, {0x114, 0x114}, {0x116, 0x116}, {0x118, 0x118}, + {0x11A, 0x11A}, {0x11C, 0x11C}, {0x11E, 0x11E}, {0x120, 0x120}, + {0x122, 0x122}, {0x124, 0x124}, {0x126, 0x126}, {0x128, 0x128}, + {0x12A, 0x12A}, {0x12C, 0x12C}, {0x12E, 0x12E}, {0x130, 0x130}, + {0x132, 0x132}, {0x134, 0x134}, {0x136, 0x136}, {0x139, 0x139}, + {0x13B, 0x13B}, {0x13D, 0x13D}, {0x13F, 0x13F}, {0x141, 0x141}, + {0x143, 0x143}, {0x145, 0x145}, {0x147, 0x147}, {0x14A, 0x14A}, + {0x14C, 0x14C}, {0x14E, 0x14E}, {0x150, 0x150}, {0x152, 0x152}, + {0x154, 0x154}, {0x156, 0x156}, {0x158, 0x158}, {0x15A, 0x15A}, + {0x15C, 0x15C}, {0x15E, 0x15E}, {0x160, 0x160}, {0x162, 0x162}, + {0x164, 0x164}, {0x166, 0x166}, {0x168, 0x168}, {0x16A, 0x16A}, + {0x16C, 0x16C}, {0x16E, 0x16E}, {0x170, 0x170}, {0x172, 0x172}, + {0x174, 0x174}, {0x176, 0x176}, {0x178, 0x179}, {0x17B, 0x17B}, + {0x17D, 0x17D}, {0x181, 0x182}, {0x184, 0x184}, {0x186, 0x187}, + {0x189, 0x18B}, {0x18E, 0x191}, {0x193, 0x194}, {0x196, 0x198}, + {0x19C, 0x19D}, {0x19F, 0x1A0}, {0x1A2, 0x1A2}, {0x1A4, 0x1A4}, + {0x1A6, 0x1A7}, {0x1A9, 0x1A9}, {0x1AC, 0x1AC}, {0x1AE, 0x1AF}, + {0x1B1, 0x1B3}, {0x1B5, 0x1B5}, {0x1B7, 0x1B8}, {0x1BC, 0x1BC}, + {0x1C4, 0x1C4}, {0x1C7, 0x1C7}, {0x1CA, 0x1CA}, {0x1CD, 0x1CD}, + {0x1CF, 0x1CF}, {0x1D1, 0x1D1}, {0x1D3, 0x1D3}, {0x1D5, 0x1D5}, + {0x1D7, 0x1D7}, {0x1D9, 0x1D9}, {0x1DB, 0x1DB}, {0x1DE, 0x1DE}, + {0x1E0, 0x1E0}, {0x1E2, 0x1E2}, {0x1E4, 0x1E4}, {0x1E6, 0x1E6}, + {0x1E8, 0x1E8}, {0x1EA, 0x1EA}, {0x1EC, 0x1EC}, {0x1EE, 0x1EE}, + {0x1F1, 0x1F1}, {0x1F4, 0x1F4}, {0x1F6, 0x1F8}, {0x1FA, 0x1FA}, + {0x1FC, 0x1FC}, {0x1FE, 0x1FE}, {0x200, 0x200}, {0x202, 0x202}, + {0x204, 0x204}, {0x206, 0x206}, {0x208, 0x208}, {0x20A, 0x20A}, + {0x20C, 0x20C}, {0x20E, 0x20E}, {0x210, 0x210}, {0x212, 0x212}, + {0x214, 0x214}, {0x216, 0x216}, {0x218, 0x218}, {0x21A, 0x21A}, + {0x21C, 0x21C}, {0x21E, 0x21E}, {0x220, 0x220}, {0x222, 0x222}, + {0x224, 0x224}, {0x226, 0x226}, {0x228, 0x228}, {0x22A, 0x22A}, + {0x22C, 0x22C}, {0x22E, 0x22E}, {0x230, 0x230}, {0x232, 0x232}, + {0x23A, 0x23B}, {0x23D, 0x23E}, {0x241, 0x241}, {0x243, 0x246}, + {0x248, 0x248}, {0x24A, 0x24A}, {0x24C, 0x24C}, {0x24E, 0x24E}, + {0x370, 0x370}, {0x372, 0x372}, {0x376, 0x376}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x38F}, + {0x391, 0x3A1}, {0x3A3, 0x3AB}, {0x3CF, 0x3CF}, {0x3D2, 0x3D4}, + {0x3D8, 0x3D8}, {0x3DA, 0x3DA}, {0x3DC, 0x3DC}, {0x3DE, 0x3DE}, + {0x3E0, 0x3E0}, {0x3E2, 0x3E2}, {0x3E4, 0x3E4}, {0x3E6, 0x3E6}, + {0x3E8, 0x3E8}, {0x3EA, 0x3EA}, {0x3EC, 0x3EC}, {0x3EE, 0x3EE}, + {0x3F4, 0x3F4}, {0x3F7, 0x3F7}, {0x3F9, 0x3FA}, {0x3FD, 0x42F}, + {0x460, 0x460}, {0x462, 0x462}, {0x464, 0x464}, {0x466, 0x466}, + {0x468, 0x468}, {0x46A, 0x46A}, {0x46C, 0x46C}, {0x46E, 0x46E}, + {0x470, 0x470}, {0x472, 0x472}, {0x474, 0x474}, {0x476, 0x476}, + {0x478, 0x478}, {0x47A, 0x47A}, {0x47C, 0x47C}, {0x47E, 0x47E}, + {0x480, 0x480}, {0x48A, 0x48A}, {0x48C, 0x48C}, {0x48E, 0x48E}, + {0x490, 0x490}, {0x492, 0x492}, {0x494, 0x494}, {0x496, 0x496}, + {0x498, 0x498}, {0x49A, 0x49A}, {0x49C, 0x49C}, {0x49E, 0x49E}, + {0x4A0, 0x4A0}, {0x4A2, 0x4A2}, {0x4A4, 0x4A4}, {0x4A6, 0x4A6}, + {0x4A8, 0x4A8}, {0x4AA, 0x4AA}, {0x4AC, 0x4AC}, {0x4AE, 0x4AE}, + {0x4B0, 0x4B0}, {0x4B2, 0x4B2}, {0x4B4, 0x4B4}, {0x4B6, 0x4B6}, + {0x4B8, 0x4B8}, {0x4BA, 0x4BA}, {0x4BC, 0x4BC}, {0x4BE, 0x4BE}, + {0x4C0, 0x4C1}, {0x4C3, 0x4C3}, {0x4C5, 0x4C5}, {0x4C7, 0x4C7}, + {0x4C9, 0x4C9}, {0x4CB, 0x4CB}, {0x4CD, 0x4CD}, {0x4D0, 0x4D0}, + {0x4D2, 0x4D2}, {0x4D4, 0x4D4}, {0x4D6, 0x4D6}, {0x4D8, 0x4D8}, + {0x4DA, 0x4DA}, {0x4DC, 0x4DC}, {0x4DE, 0x4DE}, {0x4E0, 0x4E0}, + {0x4E2, 0x4E2}, {0x4E4, 0x4E4}, {0x4E6, 0x4E6}, {0x4E8, 0x4E8}, + {0x4EA, 0x4EA}, {0x4EC, 0x4EC}, {0x4EE, 0x4EE}, {0x4F0, 0x4F0}, + {0x4F2, 0x4F2}, {0x4F4, 0x4F4}, {0x4F6, 0x4F6}, {0x4F8, 0x4F8}, + {0x4FA, 0x4FA}, {0x4FC, 0x4FC}, {0x4FE, 0x4FE}, {0x500, 0x500}, + {0x502, 0x502}, {0x504, 0x504}, {0x506, 0x506}, {0x508, 0x508}, + {0x50A, 0x50A}, {0x50C, 0x50C}, {0x50E, 0x50E}, {0x510, 0x510}, + {0x512, 0x512}, {0x514, 0x514}, {0x516, 0x516}, {0x518, 0x518}, + {0x51A, 0x51A}, {0x51C, 0x51C}, {0x51E, 0x51E}, {0x520, 0x520}, + {0x522, 0x522}, {0x524, 0x524}, {0x526, 0x526}, {0x528, 0x528}, + {0x52A, 0x52A}, {0x52C, 0x52C}, {0x52E, 0x52E}, {0x531, 0x556}, + {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x13A0, 0x13F5}, + {0x1C89, 0x1C89}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1E00, 0x1E00}, + {0x1E02, 0x1E02}, {0x1E04, 0x1E04}, {0x1E06, 0x1E06}, {0x1E08, 0x1E08}, + {0x1E0A, 0x1E0A}, {0x1E0C, 0x1E0C}, {0x1E0E, 0x1E0E}, {0x1E10, 0x1E10}, + {0x1E12, 0x1E12}, {0x1E14, 0x1E14}, {0x1E16, 0x1E16}, {0x1E18, 0x1E18}, + {0x1E1A, 0x1E1A}, {0x1E1C, 0x1E1C}, {0x1E1E, 0x1E1E}, {0x1E20, 0x1E20}, + {0x1E22, 0x1E22}, {0x1E24, 0x1E24}, {0x1E26, 0x1E26}, {0x1E28, 0x1E28}, + {0x1E2A, 0x1E2A}, {0x1E2C, 0x1E2C}, {0x1E2E, 0x1E2E}, {0x1E30, 0x1E30}, + {0x1E32, 0x1E32}, {0x1E34, 0x1E34}, {0x1E36, 0x1E36}, {0x1E38, 0x1E38}, + {0x1E3A, 0x1E3A}, {0x1E3C, 0x1E3C}, {0x1E3E, 0x1E3E}, {0x1E40, 0x1E40}, + {0x1E42, 0x1E42}, {0x1E44, 0x1E44}, {0x1E46, 0x1E46}, {0x1E48, 0x1E48}, + {0x1E4A, 0x1E4A}, {0x1E4C, 0x1E4C}, {0x1E4E, 0x1E4E}, {0x1E50, 0x1E50}, + {0x1E52, 0x1E52}, {0x1E54, 0x1E54}, {0x1E56, 0x1E56}, {0x1E58, 0x1E58}, + {0x1E5A, 0x1E5A}, {0x1E5C, 0x1E5C}, {0x1E5E, 0x1E5E}, {0x1E60, 0x1E60}, + {0x1E62, 0x1E62}, {0x1E64, 0x1E64}, {0x1E66, 0x1E66}, {0x1E68, 0x1E68}, + {0x1E6A, 0x1E6A}, {0x1E6C, 0x1E6C}, {0x1E6E, 0x1E6E}, {0x1E70, 0x1E70}, + {0x1E72, 0x1E72}, {0x1E74, 0x1E74}, {0x1E76, 0x1E76}, {0x1E78, 0x1E78}, + {0x1E7A, 0x1E7A}, {0x1E7C, 0x1E7C}, {0x1E7E, 0x1E7E}, {0x1E80, 0x1E80}, + {0x1E82, 0x1E82}, {0x1E84, 0x1E84}, {0x1E86, 0x1E86}, {0x1E88, 0x1E88}, + {0x1E8A, 0x1E8A}, {0x1E8C, 0x1E8C}, {0x1E8E, 0x1E8E}, {0x1E90, 0x1E90}, + {0x1E92, 0x1E92}, {0x1E94, 0x1E94}, {0x1E9E, 0x1E9E}, {0x1EA0, 0x1EA0}, + {0x1EA2, 0x1EA2}, {0x1EA4, 0x1EA4}, {0x1EA6, 0x1EA6}, {0x1EA8, 0x1EA8}, + {0x1EAA, 0x1EAA}, {0x1EAC, 0x1EAC}, {0x1EAE, 0x1EAE}, {0x1EB0, 0x1EB0}, + {0x1EB2, 0x1EB2}, {0x1EB4, 0x1EB4}, {0x1EB6, 0x1EB6}, {0x1EB8, 0x1EB8}, + {0x1EBA, 0x1EBA}, {0x1EBC, 0x1EBC}, {0x1EBE, 0x1EBE}, {0x1EC0, 0x1EC0}, + {0x1EC2, 0x1EC2}, {0x1EC4, 0x1EC4}, {0x1EC6, 0x1EC6}, {0x1EC8, 0x1EC8}, + {0x1ECA, 0x1ECA}, {0x1ECC, 0x1ECC}, {0x1ECE, 0x1ECE}, {0x1ED0, 0x1ED0}, + {0x1ED2, 0x1ED2}, {0x1ED4, 0x1ED4}, {0x1ED6, 0x1ED6}, {0x1ED8, 0x1ED8}, + {0x1EDA, 0x1EDA}, {0x1EDC, 0x1EDC}, {0x1EDE, 0x1EDE}, {0x1EE0, 0x1EE0}, + {0x1EE2, 0x1EE2}, {0x1EE4, 0x1EE4}, {0x1EE6, 0x1EE6}, {0x1EE8, 0x1EE8}, + {0x1EEA, 0x1EEA}, {0x1EEC, 0x1EEC}, {0x1EEE, 0x1EEE}, {0x1EF0, 0x1EF0}, + {0x1EF2, 0x1EF2}, {0x1EF4, 0x1EF4}, {0x1EF6, 0x1EF6}, {0x1EF8, 0x1EF8}, + {0x1EFA, 0x1EFA}, {0x1EFC, 0x1EFC}, {0x1EFE, 0x1EFE}, {0x1F08, 0x1F0F}, + {0x1F18, 0x1F1D}, {0x1F28, 0x1F2F}, {0x1F38, 0x1F3F}, {0x1F48, 0x1F4D}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F5F}, + {0x1F68, 0x1F6F}, {0x1FB8, 0x1FBB}, {0x1FC8, 0x1FCB}, {0x1FD8, 0x1FDB}, + {0x1FE8, 0x1FEC}, {0x1FF8, 0x1FFB}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210B, 0x210D}, {0x2110, 0x2112}, {0x2115, 0x2115}, {0x2119, 0x211D}, + {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, + {0x2130, 0x2133}, {0x213E, 0x213F}, {0x2145, 0x2145}, {0x2160, 0x216F}, + {0x2183, 0x2183}, {0x24B6, 0x24CF}, {0x2C00, 0x2C2F}, {0x2C60, 0x2C60}, + {0x2C62, 0x2C64}, {0x2C67, 0x2C67}, {0x2C69, 0x2C69}, {0x2C6B, 0x2C6B}, + {0x2C6D, 0x2C70}, {0x2C72, 0x2C72}, {0x2C75, 0x2C75}, {0x2C7E, 0x2C80}, + {0x2C82, 0x2C82}, {0x2C84, 0x2C84}, {0x2C86, 0x2C86}, {0x2C88, 0x2C88}, + {0x2C8A, 0x2C8A}, {0x2C8C, 0x2C8C}, {0x2C8E, 0x2C8E}, {0x2C90, 0x2C90}, + {0x2C92, 0x2C92}, {0x2C94, 0x2C94}, {0x2C96, 0x2C96}, {0x2C98, 0x2C98}, + {0x2C9A, 0x2C9A}, {0x2C9C, 0x2C9C}, {0x2C9E, 0x2C9E}, {0x2CA0, 0x2CA0}, + {0x2CA2, 0x2CA2}, {0x2CA4, 0x2CA4}, {0x2CA6, 0x2CA6}, {0x2CA8, 0x2CA8}, + {0x2CAA, 0x2CAA}, {0x2CAC, 0x2CAC}, {0x2CAE, 0x2CAE}, {0x2CB0, 0x2CB0}, + {0x2CB2, 0x2CB2}, {0x2CB4, 0x2CB4}, {0x2CB6, 0x2CB6}, {0x2CB8, 0x2CB8}, + {0x2CBA, 0x2CBA}, {0x2CBC, 0x2CBC}, {0x2CBE, 0x2CBE}, {0x2CC0, 0x2CC0}, + {0x2CC2, 0x2CC2}, {0x2CC4, 0x2CC4}, {0x2CC6, 0x2CC6}, {0x2CC8, 0x2CC8}, + {0x2CCA, 0x2CCA}, {0x2CCC, 0x2CCC}, {0x2CCE, 0x2CCE}, {0x2CD0, 0x2CD0}, + {0x2CD2, 0x2CD2}, {0x2CD4, 0x2CD4}, {0x2CD6, 0x2CD6}, {0x2CD8, 0x2CD8}, + {0x2CDA, 0x2CDA}, {0x2CDC, 0x2CDC}, {0x2CDE, 0x2CDE}, {0x2CE0, 0x2CE0}, + {0x2CE2, 0x2CE2}, {0x2CEB, 0x2CEB}, {0x2CED, 0x2CED}, {0x2CF2, 0x2CF2}, + {0xA640, 0xA640}, {0xA642, 0xA642}, {0xA644, 0xA644}, {0xA646, 0xA646}, + {0xA648, 0xA648}, {0xA64A, 0xA64A}, {0xA64C, 0xA64C}, {0xA64E, 0xA64E}, + {0xA650, 0xA650}, {0xA652, 0xA652}, {0xA654, 0xA654}, {0xA656, 0xA656}, + {0xA658, 0xA658}, {0xA65A, 0xA65A}, {0xA65C, 0xA65C}, {0xA65E, 0xA65E}, + {0xA660, 0xA660}, {0xA662, 0xA662}, {0xA664, 0xA664}, {0xA666, 0xA666}, + {0xA668, 0xA668}, {0xA66A, 0xA66A}, {0xA66C, 0xA66C}, {0xA680, 0xA680}, + {0xA682, 0xA682}, {0xA684, 0xA684}, {0xA686, 0xA686}, {0xA688, 0xA688}, + {0xA68A, 0xA68A}, {0xA68C, 0xA68C}, {0xA68E, 0xA68E}, {0xA690, 0xA690}, + {0xA692, 0xA692}, {0xA694, 0xA694}, {0xA696, 0xA696}, {0xA698, 0xA698}, + {0xA69A, 0xA69A}, {0xA722, 0xA722}, {0xA724, 0xA724}, {0xA726, 0xA726}, + {0xA728, 0xA728}, {0xA72A, 0xA72A}, {0xA72C, 0xA72C}, {0xA72E, 0xA72E}, + {0xA732, 0xA732}, {0xA734, 0xA734}, {0xA736, 0xA736}, {0xA738, 0xA738}, + {0xA73A, 0xA73A}, {0xA73C, 0xA73C}, {0xA73E, 0xA73E}, {0xA740, 0xA740}, + {0xA742, 0xA742}, {0xA744, 0xA744}, {0xA746, 0xA746}, {0xA748, 0xA748}, + {0xA74A, 0xA74A}, {0xA74C, 0xA74C}, {0xA74E, 0xA74E}, {0xA750, 0xA750}, + {0xA752, 0xA752}, {0xA754, 0xA754}, {0xA756, 0xA756}, {0xA758, 0xA758}, + {0xA75A, 0xA75A}, {0xA75C, 0xA75C}, {0xA75E, 0xA75E}, {0xA760, 0xA760}, + {0xA762, 0xA762}, {0xA764, 0xA764}, {0xA766, 0xA766}, {0xA768, 0xA768}, + {0xA76A, 0xA76A}, {0xA76C, 0xA76C}, {0xA76E, 0xA76E}, {0xA779, 0xA779}, + {0xA77B, 0xA77B}, {0xA77D, 0xA77E}, {0xA780, 0xA780}, {0xA782, 0xA782}, + {0xA784, 0xA784}, {0xA786, 0xA786}, {0xA78B, 0xA78B}, {0xA78D, 0xA78D}, + {0xA790, 0xA790}, {0xA792, 0xA792}, {0xA796, 0xA796}, {0xA798, 0xA798}, + {0xA79A, 0xA79A}, {0xA79C, 0xA79C}, {0xA79E, 0xA79E}, {0xA7A0, 0xA7A0}, + {0xA7A2, 0xA7A2}, {0xA7A4, 0xA7A4}, {0xA7A6, 0xA7A6}, {0xA7A8, 0xA7A8}, + {0xA7AA, 0xA7AE}, {0xA7B0, 0xA7B4}, {0xA7B6, 0xA7B6}, {0xA7B8, 0xA7B8}, + {0xA7BA, 0xA7BA}, {0xA7BC, 0xA7BC}, {0xA7BE, 0xA7BE}, {0xA7C0, 0xA7C0}, + {0xA7C2, 0xA7C2}, {0xA7C4, 0xA7C7}, {0xA7C9, 0xA7C9}, {0xA7CB, 0xA7CC}, + {0xA7D0, 0xA7D0}, {0xA7D6, 0xA7D6}, {0xA7D8, 0xA7D8}, {0xA7DA, 0xA7DA}, + {0xA7DC, 0xA7DC}, {0xA7F5, 0xA7F5}, {0xFF21, 0xFF3A}, {0x10400, 0x10427}, + {0x104B0, 0x104D3}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, + {0x10594, 0x10595}, {0x10C80, 0x10CB2}, {0x10D50, 0x10D65}, {0x118A0, 0x118BF}, + {0x16E40, 0x16E5F}, {0x1D400, 0x1D419}, {0x1D434, 0x1D44D}, {0x1D468, 0x1D481}, + {0x1D49C, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B5}, {0x1D4D0, 0x1D4E9}, {0x1D504, 0x1D505}, + {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D538, 0x1D539}, + {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D56C, 0x1D585}, {0x1D5A0, 0x1D5B9}, {0x1D5D4, 0x1D5ED}, {0x1D608, 0x1D621}, + {0x1D63C, 0x1D655}, {0x1D670, 0x1D689}, {0x1D6A8, 0x1D6C0}, {0x1D6E2, 0x1D6FA}, + {0x1D71C, 0x1D734}, {0x1D756, 0x1D76E}, {0x1D790, 0x1D7A8}, {0x1D7CA, 0x1D7CA}, + {0x1E900, 0x1E921}, {0x1F130, 0x1F149}, {0x1F150, 0x1F169}, {0x1F170, 0x1F189}, +}; +const int PDF_U_UPPER_COUNT = (int)(sizeof(PDF_U_UPPER) / sizeof(PDF_U_UPPER[0])); + +const pdf_urange_t PDF_U_LOWER[] = { + {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, {0xBA, 0xBA}, + {0xDF, 0xF6}, {0xF8, 0xFF}, {0x101, 0x101}, {0x103, 0x103}, + {0x105, 0x105}, {0x107, 0x107}, {0x109, 0x109}, {0x10B, 0x10B}, + {0x10D, 0x10D}, {0x10F, 0x10F}, {0x111, 0x111}, {0x113, 0x113}, + {0x115, 0x115}, {0x117, 0x117}, {0x119, 0x119}, {0x11B, 0x11B}, + {0x11D, 0x11D}, {0x11F, 0x11F}, {0x121, 0x121}, {0x123, 0x123}, + {0x125, 0x125}, {0x127, 0x127}, {0x129, 0x129}, {0x12B, 0x12B}, + {0x12D, 0x12D}, {0x12F, 0x12F}, {0x131, 0x131}, {0x133, 0x133}, + {0x135, 0x135}, {0x137, 0x138}, {0x13A, 0x13A}, {0x13C, 0x13C}, + {0x13E, 0x13E}, {0x140, 0x140}, {0x142, 0x142}, {0x144, 0x144}, + {0x146, 0x146}, {0x148, 0x149}, {0x14B, 0x14B}, {0x14D, 0x14D}, + {0x14F, 0x14F}, {0x151, 0x151}, {0x153, 0x153}, {0x155, 0x155}, + {0x157, 0x157}, {0x159, 0x159}, {0x15B, 0x15B}, {0x15D, 0x15D}, + {0x15F, 0x15F}, {0x161, 0x161}, {0x163, 0x163}, {0x165, 0x165}, + {0x167, 0x167}, {0x169, 0x169}, {0x16B, 0x16B}, {0x16D, 0x16D}, + {0x16F, 0x16F}, {0x171, 0x171}, {0x173, 0x173}, {0x175, 0x175}, + {0x177, 0x177}, {0x17A, 0x17A}, {0x17C, 0x17C}, {0x17E, 0x180}, + {0x183, 0x183}, {0x185, 0x185}, {0x188, 0x188}, {0x18C, 0x18D}, + {0x192, 0x192}, {0x195, 0x195}, {0x199, 0x19B}, {0x19E, 0x19E}, + {0x1A1, 0x1A1}, {0x1A3, 0x1A3}, {0x1A5, 0x1A5}, {0x1A8, 0x1A8}, + {0x1AA, 0x1AB}, {0x1AD, 0x1AD}, {0x1B0, 0x1B0}, {0x1B4, 0x1B4}, + {0x1B6, 0x1B6}, {0x1B9, 0x1BA}, {0x1BD, 0x1BF}, {0x1C6, 0x1C6}, + {0x1C9, 0x1C9}, {0x1CC, 0x1CC}, {0x1CE, 0x1CE}, {0x1D0, 0x1D0}, + {0x1D2, 0x1D2}, {0x1D4, 0x1D4}, {0x1D6, 0x1D6}, {0x1D8, 0x1D8}, + {0x1DA, 0x1DA}, {0x1DC, 0x1DD}, {0x1DF, 0x1DF}, {0x1E1, 0x1E1}, + {0x1E3, 0x1E3}, {0x1E5, 0x1E5}, {0x1E7, 0x1E7}, {0x1E9, 0x1E9}, + {0x1EB, 0x1EB}, {0x1ED, 0x1ED}, {0x1EF, 0x1F0}, {0x1F3, 0x1F3}, + {0x1F5, 0x1F5}, {0x1F9, 0x1F9}, {0x1FB, 0x1FB}, {0x1FD, 0x1FD}, + {0x1FF, 0x1FF}, {0x201, 0x201}, {0x203, 0x203}, {0x205, 0x205}, + {0x207, 0x207}, {0x209, 0x209}, {0x20B, 0x20B}, {0x20D, 0x20D}, + {0x20F, 0x20F}, {0x211, 0x211}, {0x213, 0x213}, {0x215, 0x215}, + {0x217, 0x217}, {0x219, 0x219}, {0x21B, 0x21B}, {0x21D, 0x21D}, + {0x21F, 0x21F}, {0x221, 0x221}, {0x223, 0x223}, {0x225, 0x225}, + {0x227, 0x227}, {0x229, 0x229}, {0x22B, 0x22B}, {0x22D, 0x22D}, + {0x22F, 0x22F}, {0x231, 0x231}, {0x233, 0x239}, {0x23C, 0x23C}, + {0x23F, 0x240}, {0x242, 0x242}, {0x247, 0x247}, {0x249, 0x249}, + {0x24B, 0x24B}, {0x24D, 0x24D}, {0x24F, 0x293}, {0x295, 0x2B8}, + {0x2C0, 0x2C1}, {0x2E0, 0x2E4}, {0x345, 0x345}, {0x371, 0x371}, + {0x373, 0x373}, {0x377, 0x377}, {0x37A, 0x37D}, {0x390, 0x390}, + {0x3AC, 0x3CE}, {0x3D0, 0x3D1}, {0x3D5, 0x3D7}, {0x3D9, 0x3D9}, + {0x3DB, 0x3DB}, {0x3DD, 0x3DD}, {0x3DF, 0x3DF}, {0x3E1, 0x3E1}, + {0x3E3, 0x3E3}, {0x3E5, 0x3E5}, {0x3E7, 0x3E7}, {0x3E9, 0x3E9}, + {0x3EB, 0x3EB}, {0x3ED, 0x3ED}, {0x3EF, 0x3F3}, {0x3F5, 0x3F5}, + {0x3F8, 0x3F8}, {0x3FB, 0x3FC}, {0x430, 0x45F}, {0x461, 0x461}, + {0x463, 0x463}, {0x465, 0x465}, {0x467, 0x467}, {0x469, 0x469}, + {0x46B, 0x46B}, {0x46D, 0x46D}, {0x46F, 0x46F}, {0x471, 0x471}, + {0x473, 0x473}, {0x475, 0x475}, {0x477, 0x477}, {0x479, 0x479}, + {0x47B, 0x47B}, {0x47D, 0x47D}, {0x47F, 0x47F}, {0x481, 0x481}, + {0x48B, 0x48B}, {0x48D, 0x48D}, {0x48F, 0x48F}, {0x491, 0x491}, + {0x493, 0x493}, {0x495, 0x495}, {0x497, 0x497}, {0x499, 0x499}, + {0x49B, 0x49B}, {0x49D, 0x49D}, {0x49F, 0x49F}, {0x4A1, 0x4A1}, + {0x4A3, 0x4A3}, {0x4A5, 0x4A5}, {0x4A7, 0x4A7}, {0x4A9, 0x4A9}, + {0x4AB, 0x4AB}, {0x4AD, 0x4AD}, {0x4AF, 0x4AF}, {0x4B1, 0x4B1}, + {0x4B3, 0x4B3}, {0x4B5, 0x4B5}, {0x4B7, 0x4B7}, {0x4B9, 0x4B9}, + {0x4BB, 0x4BB}, {0x4BD, 0x4BD}, {0x4BF, 0x4BF}, {0x4C2, 0x4C2}, + {0x4C4, 0x4C4}, {0x4C6, 0x4C6}, {0x4C8, 0x4C8}, {0x4CA, 0x4CA}, + {0x4CC, 0x4CC}, {0x4CE, 0x4CF}, {0x4D1, 0x4D1}, {0x4D3, 0x4D3}, + {0x4D5, 0x4D5}, {0x4D7, 0x4D7}, {0x4D9, 0x4D9}, {0x4DB, 0x4DB}, + {0x4DD, 0x4DD}, {0x4DF, 0x4DF}, {0x4E1, 0x4E1}, {0x4E3, 0x4E3}, + {0x4E5, 0x4E5}, {0x4E7, 0x4E7}, {0x4E9, 0x4E9}, {0x4EB, 0x4EB}, + {0x4ED, 0x4ED}, {0x4EF, 0x4EF}, {0x4F1, 0x4F1}, {0x4F3, 0x4F3}, + {0x4F5, 0x4F5}, {0x4F7, 0x4F7}, {0x4F9, 0x4F9}, {0x4FB, 0x4FB}, + {0x4FD, 0x4FD}, {0x4FF, 0x4FF}, {0x501, 0x501}, {0x503, 0x503}, + {0x505, 0x505}, {0x507, 0x507}, {0x509, 0x509}, {0x50B, 0x50B}, + {0x50D, 0x50D}, {0x50F, 0x50F}, {0x511, 0x511}, {0x513, 0x513}, + {0x515, 0x515}, {0x517, 0x517}, {0x519, 0x519}, {0x51B, 0x51B}, + {0x51D, 0x51D}, {0x51F, 0x51F}, {0x521, 0x521}, {0x523, 0x523}, + {0x525, 0x525}, {0x527, 0x527}, {0x529, 0x529}, {0x52B, 0x52B}, + {0x52D, 0x52D}, {0x52F, 0x52F}, {0x560, 0x588}, {0x10D0, 0x10FA}, + {0x10FC, 0x10FF}, {0x13F8, 0x13FD}, {0x1C80, 0x1C88}, {0x1C8A, 0x1C8A}, + {0x1D00, 0x1DBF}, {0x1E01, 0x1E01}, {0x1E03, 0x1E03}, {0x1E05, 0x1E05}, + {0x1E07, 0x1E07}, {0x1E09, 0x1E09}, {0x1E0B, 0x1E0B}, {0x1E0D, 0x1E0D}, + {0x1E0F, 0x1E0F}, {0x1E11, 0x1E11}, {0x1E13, 0x1E13}, {0x1E15, 0x1E15}, + {0x1E17, 0x1E17}, {0x1E19, 0x1E19}, {0x1E1B, 0x1E1B}, {0x1E1D, 0x1E1D}, + {0x1E1F, 0x1E1F}, {0x1E21, 0x1E21}, {0x1E23, 0x1E23}, {0x1E25, 0x1E25}, + {0x1E27, 0x1E27}, {0x1E29, 0x1E29}, {0x1E2B, 0x1E2B}, {0x1E2D, 0x1E2D}, + {0x1E2F, 0x1E2F}, {0x1E31, 0x1E31}, {0x1E33, 0x1E33}, {0x1E35, 0x1E35}, + {0x1E37, 0x1E37}, {0x1E39, 0x1E39}, {0x1E3B, 0x1E3B}, {0x1E3D, 0x1E3D}, + {0x1E3F, 0x1E3F}, {0x1E41, 0x1E41}, {0x1E43, 0x1E43}, {0x1E45, 0x1E45}, + {0x1E47, 0x1E47}, {0x1E49, 0x1E49}, {0x1E4B, 0x1E4B}, {0x1E4D, 0x1E4D}, + {0x1E4F, 0x1E4F}, {0x1E51, 0x1E51}, {0x1E53, 0x1E53}, {0x1E55, 0x1E55}, + {0x1E57, 0x1E57}, {0x1E59, 0x1E59}, {0x1E5B, 0x1E5B}, {0x1E5D, 0x1E5D}, + {0x1E5F, 0x1E5F}, {0x1E61, 0x1E61}, {0x1E63, 0x1E63}, {0x1E65, 0x1E65}, + {0x1E67, 0x1E67}, {0x1E69, 0x1E69}, {0x1E6B, 0x1E6B}, {0x1E6D, 0x1E6D}, + {0x1E6F, 0x1E6F}, {0x1E71, 0x1E71}, {0x1E73, 0x1E73}, {0x1E75, 0x1E75}, + {0x1E77, 0x1E77}, {0x1E79, 0x1E79}, {0x1E7B, 0x1E7B}, {0x1E7D, 0x1E7D}, + {0x1E7F, 0x1E7F}, {0x1E81, 0x1E81}, {0x1E83, 0x1E83}, {0x1E85, 0x1E85}, + {0x1E87, 0x1E87}, {0x1E89, 0x1E89}, {0x1E8B, 0x1E8B}, {0x1E8D, 0x1E8D}, + {0x1E8F, 0x1E8F}, {0x1E91, 0x1E91}, {0x1E93, 0x1E93}, {0x1E95, 0x1E9D}, + {0x1E9F, 0x1E9F}, {0x1EA1, 0x1EA1}, {0x1EA3, 0x1EA3}, {0x1EA5, 0x1EA5}, + {0x1EA7, 0x1EA7}, {0x1EA9, 0x1EA9}, {0x1EAB, 0x1EAB}, {0x1EAD, 0x1EAD}, + {0x1EAF, 0x1EAF}, {0x1EB1, 0x1EB1}, {0x1EB3, 0x1EB3}, {0x1EB5, 0x1EB5}, + {0x1EB7, 0x1EB7}, {0x1EB9, 0x1EB9}, {0x1EBB, 0x1EBB}, {0x1EBD, 0x1EBD}, + {0x1EBF, 0x1EBF}, {0x1EC1, 0x1EC1}, {0x1EC3, 0x1EC3}, {0x1EC5, 0x1EC5}, + {0x1EC7, 0x1EC7}, {0x1EC9, 0x1EC9}, {0x1ECB, 0x1ECB}, {0x1ECD, 0x1ECD}, + {0x1ECF, 0x1ECF}, {0x1ED1, 0x1ED1}, {0x1ED3, 0x1ED3}, {0x1ED5, 0x1ED5}, + {0x1ED7, 0x1ED7}, {0x1ED9, 0x1ED9}, {0x1EDB, 0x1EDB}, {0x1EDD, 0x1EDD}, + {0x1EDF, 0x1EDF}, {0x1EE1, 0x1EE1}, {0x1EE3, 0x1EE3}, {0x1EE5, 0x1EE5}, + {0x1EE7, 0x1EE7}, {0x1EE9, 0x1EE9}, {0x1EEB, 0x1EEB}, {0x1EED, 0x1EED}, + {0x1EEF, 0x1EEF}, {0x1EF1, 0x1EF1}, {0x1EF3, 0x1EF3}, {0x1EF5, 0x1EF5}, + {0x1EF7, 0x1EF7}, {0x1EF9, 0x1EF9}, {0x1EFB, 0x1EFB}, {0x1EFD, 0x1EFD}, + {0x1EFF, 0x1F07}, {0x1F10, 0x1F15}, {0x1F20, 0x1F27}, {0x1F30, 0x1F37}, + {0x1F40, 0x1F45}, {0x1F50, 0x1F57}, {0x1F60, 0x1F67}, {0x1F70, 0x1F7D}, + {0x1F80, 0x1F87}, {0x1F90, 0x1F97}, {0x1FA0, 0x1FA7}, {0x1FB0, 0x1FB4}, + {0x1FB6, 0x1FB7}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FC7}, + {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FD7}, {0x1FE0, 0x1FE7}, {0x1FF2, 0x1FF4}, + {0x1FF6, 0x1FF7}, {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, + {0x210A, 0x210A}, {0x210E, 0x210F}, {0x2113, 0x2113}, {0x212F, 0x212F}, + {0x2134, 0x2134}, {0x2139, 0x2139}, {0x213C, 0x213D}, {0x2146, 0x2149}, + {0x214E, 0x214E}, {0x2170, 0x217F}, {0x2184, 0x2184}, {0x24D0, 0x24E9}, + {0x2C30, 0x2C5F}, {0x2C61, 0x2C61}, {0x2C65, 0x2C66}, {0x2C68, 0x2C68}, + {0x2C6A, 0x2C6A}, {0x2C6C, 0x2C6C}, {0x2C71, 0x2C71}, {0x2C73, 0x2C74}, + {0x2C76, 0x2C7D}, {0x2C81, 0x2C81}, {0x2C83, 0x2C83}, {0x2C85, 0x2C85}, + {0x2C87, 0x2C87}, {0x2C89, 0x2C89}, {0x2C8B, 0x2C8B}, {0x2C8D, 0x2C8D}, + {0x2C8F, 0x2C8F}, {0x2C91, 0x2C91}, {0x2C93, 0x2C93}, {0x2C95, 0x2C95}, + {0x2C97, 0x2C97}, {0x2C99, 0x2C99}, {0x2C9B, 0x2C9B}, {0x2C9D, 0x2C9D}, + {0x2C9F, 0x2C9F}, {0x2CA1, 0x2CA1}, {0x2CA3, 0x2CA3}, {0x2CA5, 0x2CA5}, + {0x2CA7, 0x2CA7}, {0x2CA9, 0x2CA9}, {0x2CAB, 0x2CAB}, {0x2CAD, 0x2CAD}, + {0x2CAF, 0x2CAF}, {0x2CB1, 0x2CB1}, {0x2CB3, 0x2CB3}, {0x2CB5, 0x2CB5}, + {0x2CB7, 0x2CB7}, {0x2CB9, 0x2CB9}, {0x2CBB, 0x2CBB}, {0x2CBD, 0x2CBD}, + {0x2CBF, 0x2CBF}, {0x2CC1, 0x2CC1}, {0x2CC3, 0x2CC3}, {0x2CC5, 0x2CC5}, + {0x2CC7, 0x2CC7}, {0x2CC9, 0x2CC9}, {0x2CCB, 0x2CCB}, {0x2CCD, 0x2CCD}, + {0x2CCF, 0x2CCF}, {0x2CD1, 0x2CD1}, {0x2CD3, 0x2CD3}, {0x2CD5, 0x2CD5}, + {0x2CD7, 0x2CD7}, {0x2CD9, 0x2CD9}, {0x2CDB, 0x2CDB}, {0x2CDD, 0x2CDD}, + {0x2CDF, 0x2CDF}, {0x2CE1, 0x2CE1}, {0x2CE3, 0x2CE4}, {0x2CEC, 0x2CEC}, + {0x2CEE, 0x2CEE}, {0x2CF3, 0x2CF3}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0xA641, 0xA641}, {0xA643, 0xA643}, {0xA645, 0xA645}, + {0xA647, 0xA647}, {0xA649, 0xA649}, {0xA64B, 0xA64B}, {0xA64D, 0xA64D}, + {0xA64F, 0xA64F}, {0xA651, 0xA651}, {0xA653, 0xA653}, {0xA655, 0xA655}, + {0xA657, 0xA657}, {0xA659, 0xA659}, {0xA65B, 0xA65B}, {0xA65D, 0xA65D}, + {0xA65F, 0xA65F}, {0xA661, 0xA661}, {0xA663, 0xA663}, {0xA665, 0xA665}, + {0xA667, 0xA667}, {0xA669, 0xA669}, {0xA66B, 0xA66B}, {0xA66D, 0xA66D}, + {0xA681, 0xA681}, {0xA683, 0xA683}, {0xA685, 0xA685}, {0xA687, 0xA687}, + {0xA689, 0xA689}, {0xA68B, 0xA68B}, {0xA68D, 0xA68D}, {0xA68F, 0xA68F}, + {0xA691, 0xA691}, {0xA693, 0xA693}, {0xA695, 0xA695}, {0xA697, 0xA697}, + {0xA699, 0xA699}, {0xA69B, 0xA69D}, {0xA723, 0xA723}, {0xA725, 0xA725}, + {0xA727, 0xA727}, {0xA729, 0xA729}, {0xA72B, 0xA72B}, {0xA72D, 0xA72D}, + {0xA72F, 0xA731}, {0xA733, 0xA733}, {0xA735, 0xA735}, {0xA737, 0xA737}, + {0xA739, 0xA739}, {0xA73B, 0xA73B}, {0xA73D, 0xA73D}, {0xA73F, 0xA73F}, + {0xA741, 0xA741}, {0xA743, 0xA743}, {0xA745, 0xA745}, {0xA747, 0xA747}, + {0xA749, 0xA749}, {0xA74B, 0xA74B}, {0xA74D, 0xA74D}, {0xA74F, 0xA74F}, + {0xA751, 0xA751}, {0xA753, 0xA753}, {0xA755, 0xA755}, {0xA757, 0xA757}, + {0xA759, 0xA759}, {0xA75B, 0xA75B}, {0xA75D, 0xA75D}, {0xA75F, 0xA75F}, + {0xA761, 0xA761}, {0xA763, 0xA763}, {0xA765, 0xA765}, {0xA767, 0xA767}, + {0xA769, 0xA769}, {0xA76B, 0xA76B}, {0xA76D, 0xA76D}, {0xA76F, 0xA778}, + {0xA77A, 0xA77A}, {0xA77C, 0xA77C}, {0xA77F, 0xA77F}, {0xA781, 0xA781}, + {0xA783, 0xA783}, {0xA785, 0xA785}, {0xA787, 0xA787}, {0xA78C, 0xA78C}, + {0xA78E, 0xA78E}, {0xA791, 0xA791}, {0xA793, 0xA795}, {0xA797, 0xA797}, + {0xA799, 0xA799}, {0xA79B, 0xA79B}, {0xA79D, 0xA79D}, {0xA79F, 0xA79F}, + {0xA7A1, 0xA7A1}, {0xA7A3, 0xA7A3}, {0xA7A5, 0xA7A5}, {0xA7A7, 0xA7A7}, + {0xA7A9, 0xA7A9}, {0xA7AF, 0xA7AF}, {0xA7B5, 0xA7B5}, {0xA7B7, 0xA7B7}, + {0xA7B9, 0xA7B9}, {0xA7BB, 0xA7BB}, {0xA7BD, 0xA7BD}, {0xA7BF, 0xA7BF}, + {0xA7C1, 0xA7C1}, {0xA7C3, 0xA7C3}, {0xA7C8, 0xA7C8}, {0xA7CA, 0xA7CA}, + {0xA7CD, 0xA7CD}, {0xA7D1, 0xA7D1}, {0xA7D3, 0xA7D3}, {0xA7D5, 0xA7D5}, + {0xA7D7, 0xA7D7}, {0xA7D9, 0xA7D9}, {0xA7DB, 0xA7DB}, {0xA7F2, 0xA7F4}, + {0xA7F6, 0xA7F6}, {0xA7F8, 0xA7FA}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, + {0xAB70, 0xABBF}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFF41, 0xFF5A}, + {0x10428, 0x1044F}, {0x104D8, 0x104FB}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, + {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x10780, 0x10780}, {0x10783, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10CC0, 0x10CF2}, {0x10D70, 0x10D85}, + {0x118C0, 0x118DF}, {0x16E60, 0x16E7F}, {0x1D41A, 0x1D433}, {0x1D44E, 0x1D454}, + {0x1D456, 0x1D467}, {0x1D482, 0x1D49B}, {0x1D4B6, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D4CF}, {0x1D4EA, 0x1D503}, {0x1D51E, 0x1D537}, + {0x1D552, 0x1D56B}, {0x1D586, 0x1D59F}, {0x1D5BA, 0x1D5D3}, {0x1D5EE, 0x1D607}, + {0x1D622, 0x1D63B}, {0x1D656, 0x1D66F}, {0x1D68A, 0x1D6A5}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6E1}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D71B}, {0x1D736, 0x1D74E}, + {0x1D750, 0x1D755}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D78F}, {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7C9}, {0x1D7CB, 0x1D7CB}, {0x1DF00, 0x1DF09}, {0x1DF0B, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E922, 0x1E943}, +}; +const int PDF_U_LOWER_COUNT = (int)(sizeof(PDF_U_LOWER) / sizeof(PDF_U_LOWER[0])); + +const pdf_urange_t PDF_U_TITLE[] = { + {0x1C5, 0x1C5}, {0x1C8, 0x1C8}, {0x1CB, 0x1CB}, {0x1F2, 0x1F2}, {0x1F88, 0x1F8F}, + {0x1F98, 0x1F9F}, {0x1FA8, 0x1FAF}, {0x1FBC, 0x1FBC}, {0x1FCC, 0x1FCC}, {0x1FFC, 0x1FFC}, +}; +const int PDF_U_TITLE_COUNT = (int)(sizeof(PDF_U_TITLE) / sizeof(PDF_U_TITLE[0])); diff --git a/internal/cbm/pdf/pdf_unicode.h b/internal/cbm/pdf/pdf_unicode.h new file mode 100644 index 0000000000..12194b9c55 --- /dev/null +++ b/internal/cbm/pdf/pdf_unicode.h @@ -0,0 +1,103 @@ +/* + * pdf_unicode.h — the Unicode facts the PDF mention scanner needs, exactly + * as CPython answers them (pdf_unicode.c is generated from CPython's + * unicodedata; the field-tested scanner ran on CPython's str and re). + */ +#ifndef CBM_PDF_UNICODE_H +#define CBM_PDF_UNICODE_H + +#include +#include + +typedef struct { + uint32_t cp; + uint16_t off; /* into PDF_NFKC_SEQ */ + uint8_t len; +} pdf_nfkc_t; + +typedef struct { + uint32_t first; + uint32_t second; + uint32_t composite; +} pdf_ucomp_t; + +typedef struct { + uint32_t lo; + uint32_t hi; +} pdf_urange_t; + +extern const pdf_nfkc_t PDF_NFKC[]; /* sorted by cp: NFKC of one code point where it differs */ +extern const int PDF_NFKC_COUNT; +extern const uint32_t PDF_NFKC_SEQ[]; +extern const pdf_ucomp_t PDF_UCOMP[]; /* sorted by (first, second): canonical compositions */ +extern const int PDF_UCOMP_COUNT; +extern const pdf_urange_t PDF_U_SPACE[]; /* str.isspace */ +extern const int PDF_U_SPACE_COUNT; +extern const pdf_urange_t PDF_U_WORD[]; /* re's \w: str.isalnum or '_' */ +extern const int PDF_U_WORD_COUNT; +extern const pdf_urange_t PDF_U_ALPHA[]; /* str.isalpha */ +extern const int PDF_U_ALPHA_COUNT; +extern const pdf_urange_t PDF_U_UPPER[]; /* one-character str.isupper */ +extern const int PDF_U_UPPER_COUNT; +extern const pdf_urange_t PDF_U_LOWER[]; /* one-character str.islower */ +extern const int PDF_U_LOWER_COUNT; +extern const pdf_urange_t PDF_U_TITLE[]; /* titlecase letters (neither upper nor lower) */ +extern const int PDF_U_TITLE_COUNT; + +static inline bool pdf_u_in(const pdf_urange_t *r, int n, uint32_t cp) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (r[mid].hi < cp) { + lo = mid + 1; + } else if (r[mid].lo > cp) { + hi = mid - 1; + } else { + return true; + } + } + return false; +} + +static inline bool pdf_u_space(uint32_t c) { + if (c < 0x80) { + return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F); + } + return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, c); +} + +static inline bool pdf_u_word(uint32_t c) { + if (c < 0x80) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_'; + } + return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, c); +} + +static inline bool pdf_u_alpha(uint32_t c) { + if (c < 0x80) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + return pdf_u_in(PDF_U_ALPHA, PDF_U_ALPHA_COUNT, c); +} + +static inline bool pdf_u_upper(uint32_t c) { + if (c < 0x80) { + return c >= 'A' && c <= 'Z'; + } + return pdf_u_in(PDF_U_UPPER, PDF_U_UPPER_COUNT, c); +} + +static inline bool pdf_u_lower(uint32_t c) { + if (c < 0x80) { + return c >= 'a' && c <= 'z'; + } + return pdf_u_in(PDF_U_LOWER, PDF_U_LOWER_COUNT, c); +} + +static inline bool pdf_u_title(uint32_t c) { + return c >= 0x80 && pdf_u_in(PDF_U_TITLE, PDF_U_TITLE_COUNT, c); +} + +#endif /* CBM_PDF_UNICODE_H */ diff --git a/src/discover/language.c b/src/discover/language.c index 116cec0245..c15a1d43d8 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -618,6 +618,7 @@ static const ext_entry_t EXT_TABLE[] = { {".clsp", CBM_LANG_CHIALISP}, {".clib", CBM_LANG_CHIALISP}, {".clinc", CBM_LANG_CHIALISP}, + {".pdf", CBM_LANG_PDF}, /* Slang */ {".slang", CBM_LANG_SLANG}, @@ -829,6 +830,7 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_NIM] = "Nim", [CBM_LANG_SCHEME] = "Scheme", [CBM_LANG_CHIALISP] = "Chialisp", + [CBM_LANG_PDF] = "PDF", [CBM_LANG_FENNEL] = "Fennel", [CBM_LANG_FISH] = "Fish", [CBM_LANG_AWK] = "AWK", diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index 8f906ef071..7b1f3d2b59 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -160,6 +160,7 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = { {"mojo", CBM_LANG_MOJO}, {"plsql", CBM_LANG_PLSQL}, {"chialisp", CBM_LANG_CHIALISP}, + {"pdf", CBM_LANG_PDF}, {"lean", CBM_LANG_LEAN}, {"form", CBM_LANG_FORM}, {"magma", CBM_LANG_MAGMA}, diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 12c4455f02..bd4196eea6 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -12535,6 +12535,36 @@ static char *resolve_path_owner(const char *path, char *derived) { return owner; } +/* A PDF is read as its text layer at index time (doc_pdf.c); its bytes are no + * source text. Its page Sections carry the page's text as their docstring. */ +static bool snippet_is_pdf(const char *file_path) { + size_t n = file_path ? strlen(file_path) : 0; + return n >= 4 && file_path[n - 4] == '.' && (file_path[n - 3] | 0x20) == 'p' && + (file_path[n - 2] | 0x20) == 'd' && (file_path[n - 1] | 0x20) == 'f'; +} + +static char *snippet_pdf_text(const cbm_node_t *node) { + if (!node->properties_json) { + return NULL; + } + yyjson_doc *pd = yyjson_read(node->properties_json, strlen(node->properties_json), 0); + if (!pd) { + return NULL; + } + yyjson_val *ds = yyjson_obj_get(yyjson_doc_get_root(pd), "docstring"); + const char *text = yyjson_is_str(ds) ? yyjson_get_str(ds) : NULL; + char *out = NULL; + if (text && text[0]) { + /* the same builder read_file_lines returns its text from */ + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_sb_append_n(&sb, text, strlen(text)); + out = cbm_sb_finish(&sb); + } + yyjson_doc_free(pd); + return out; +} + static char *resolve_snippet_source(const char *root_path, const char *file_path, int start, int end, char **out_abs_path) { *out_abs_path = NULL; @@ -12885,8 +12915,14 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, snippet_clipped = true; } char *abs_path = NULL; - char *source = - outline ? NULL : resolve_snippet_source(root_path, node->file_path, start, end, &abs_path); + char *source = NULL; + if (outline) { + source = NULL; + } else if (snippet_is_pdf(node->file_path)) { + source = snippet_pdf_text(node); /* never the file's bytes */ + } else { + source = resolve_snippet_source(root_path, node->file_path, start, end, &abs_path); + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root_obj = yyjson_mut_obj(doc); diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index df012626f8..035fbf79a4 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -47,6 +47,7 @@ const char *cbm_doclink_reason_name(int reason) { static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, + &cbm_doclink_pdf_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; @@ -462,7 +463,12 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe * file-level doc resolves */ const cbm_gbuf_node_t *file_node = NULL; bool file_node_looked_up = false; - void *state = (R && index && R->file_begin) ? R->file_begin(index, file_idx) : NULL; + void *state = NULL; + if (R && index && R->file_prepare) { + state = R->file_prepare(index, file_idx, result->doc_links.items, n, graph); + } else if (R && index && R->file_begin) { + state = R->file_begin(index, file_idx); + } for (int i = 0; i < n; i++) { const CBMDocLink *link = &result->doc_links.items[i]; cbm_doclink_outcome_t out = {.kind = CBM_DOCLINK_UNRESOLVED, diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index a51ada8c2c..ef09d15e6d 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -198,6 +198,12 @@ typedef struct { * of the file, released after its last. A NULL state (no hook, or memory * ran out) changes the cost of resolving, never an outcome. */ void *(*file_begin)(const void *index, int run_file); + /* Optional, instead of file_begin: the file's state built from ALL its + * references before any is resolved (a PDF's cross-line joins decide + * which line fragments stand). `links` stays valid until file_end, and + * resolve's `link` points into it. */ + void *(*file_prepare)(const void *index, int run_file, const CBMDocLink *links, int n, + const cbm_gbuf_t *graph); void (*file_end)(void *state); /* Resolve one reference of run file `run_file` (its own scope is in the * index). Thread-safe: the index is read-only after build; `state` is the @@ -234,6 +240,8 @@ typedef struct { extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver; /* Markdown documents (doc_links_md.c): via "markdown". */ extern const cbm_doclink_resolver_t cbm_doclink_md_resolver; +/* PDF documents (doc_links_pdf.c): via "pdf". */ +extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver; #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS /* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver diff --git a/src/pipeline/doc_links_pdf.c b/src/pipeline/doc_links_pdf.c new file mode 100644 index 0000000000..bac9081789 --- /dev/null +++ b/src/pipeline/doc_links_pdf.c @@ -0,0 +1,807 @@ +/* + * doc_links_pdf.c — PDF mentions -> MENTIONS edges: the resolving half. + * + * A PDF page (doc_pdf.c) names code structurally: a path or a qualified name. + * The tiers are the field-tested ones (E6, frozen resolver), EXACT or UNIQUE + * only: + * path-exact the mention is a File/Folder node's repository path + * path-suffix exactly one File/Folder whose path ends with the mention at a + * '/' boundary + * qn-exact exactly one node whose qualified name (project prefix off) + * is the mention + * qn-suffix exactly one node whose qualified name ends with the mention + * at a component boundary -- or whose owner-qualified name + * does (parent_class + name: a Go method's QN has no receiver) + * Two candidates that are one entity collapse first (a class and its + * constructors; duplicate nodes of one definition). A cross-line join is + * resolved before anything else in its file: when it resolves EXACT or + * UNIQUE it replaces the line fragments it was built from. + * + * Hygiene (the field test's R2 and R4): a reference that resolves to a data + * file (.json, .yaml, ...) is no code reference; one that resolves into a + * test, mock or fixture file is a test_only_target row. + * + * Unresolved: ambiguous (several entities); missing (nothing, but the + * mention's first component is a name or directory of this repository). + * Anything else names code outside the repository: LOCAL, no row. + * + * Targets exclude the labels the incremental route does not proxy (Macro, + * Comment, Section, Branch, Commit, Tag) and the field test's non-code labels + * (Project, Route, Channel, Decorator), so a re-resolved document binds what a + * full build binds. + */ +#include "pipeline/doc_links.h" + +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + PDFR_TABLE_INIT = 1024, + PDFR_KEY_CAP = 1024, /* the longest mention form looked up */ + PDFR_MAX_COMPS = 64, + PDFR_MAX_FORMS = 2, + PDFR_MEMO_INIT = 64, +}; + +static const char *const PDFR_EXCLUDED_LABELS[] = { + "Section", "Project", "Branch", "Route", "Channel", "Decorator", "Macro", + "Comment", "Commit", "Tag", "File", "Folder", NULL}; + +static const char *const PDFR_CONTAINERS[] = {"Class", "Struct", "Interface", "Enum", + "Trait", "Type", "Record", "Object", + "Protocol", "Union", NULL}; + +static const char *const PDFR_DATA_EXT[] = {".json", ".yaml", ".yml", ".toml", + ".csv", ".tsv", ".lock", NULL}; + +/* R4: the frozen test-file rule, with the directories the field test found it + * missing (Foundry *.t.sol, test/, mocks/, testutil/, testing/, testdata/). */ +static const char *const PDFR_TEST_DIRS[] = {"tests", "__tests__", "test", "mocks", + "testutil", "testing", "testdata", NULL}; + +static bool in_list(const char *s, const char *const *list) { + for (int i = 0; list[i]; i++) { + if (strcmp(s, list[i]) == 0) { + return true; + } + } + return false; +} + +static bool ends_with(const char *s, const char *suf) { + size_t n = strlen(s); + size_t m = strlen(suf); + return n >= m && memcmp(s + n - m, suf, m) == 0; +} + +static bool starts_with(const char *s, const char *pre) { + return strncmp(s, pre, strlen(pre)) == 0; +} + +/* ── Index ───────────────────────────────────────────────────────── */ + +typedef struct pdfr_list { + const cbm_gbuf_node_t *node; + struct pdfr_list *next; +} pdfr_list_t; + +typedef struct { + const char *project; + size_t project_len; + CBMArena arena; + CBMHashTable *paths; /* File/Folder repository path -> pdfr_list_t* */ + CBMHashTable *basenames; /* last path component -> pdfr_list_t* */ + CBMHashTable *dirs; /* every directory component of an indexed path (value 1) */ + /* Code nodes whose qualified name does not end with their name (a Module + * is named by its file path), keyed by the QN's last component: the + * field test matched qualified names by their components, not by name. */ + CBMHashTable *irregular; + bool failed; +} pdfr_index_t; + +static void pdfr_destroy(void *index); + +static bool list_add(pdfr_index_t *x, CBMHashTable *t, const char *key, const cbm_gbuf_node_t *n) { + pdfr_list_t *e = (pdfr_list_t *)cbm_arena_alloc(&x->arena, sizeof(*e)); + if (!e) { + return false; + } + e->node = n; + e->next = (pdfr_list_t *)cbm_ht_get(t, key); + cbm_ht_set(t, key, e); + return true; +} + +static bool index_paths(pdfr_index_t *x, const cbm_gbuf_t *graph, const char *label) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, label, &nodes, &count) != 0) { + return true; + } + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !fp[0] || fp[0] == '<') { + continue; + } + while (*fp == '/') { + fp++; + } + size_t n = strlen(fp); + while (n > 0 && fp[n - 1] == '/') { + n--; + } + char *p = cbm_arena_strndup(&x->arena, fp, n); + if (!p || !list_add(x, x->paths, p, nodes[i])) { + return false; + } + const char *slash = strrchr(p, '/'); + if (!list_add(x, x->basenames, slash ? slash + 1 : p, nodes[i])) { + return false; + } + /* directory components, for "is this a name of the repository" */ + const char *s = p; + for (const char *q = p;; q++) { + if (*q == '/' || *q == '\0') { + if (q > s) { + char *c = cbm_arena_strndup(&x->arena, s, (size_t)(q - s)); + if (!c) { + return false; + } + cbm_ht_set(x->dirs, c, (void *)(uintptr_t)1); + } + if (!*q) { + break; + } + s = q + 1; + } + } + } + return true; +} + +static bool code_node(const pdfr_index_t *x, const cbm_gbuf_node_t *n); +static const char *rel_qn(const pdfr_index_t *x, const char *qn); + +static void index_irregular(const cbm_gbuf_node_t *n, void *ud) { + pdfr_index_t *x = (pdfr_index_t *)ud; + if (x->failed || !code_node(x, n)) { + return; + } + const char *rel = rel_qn(x, n->qualified_name); + const char *dot = strrchr(rel, '.'); + const char *last = dot ? dot + 1 : rel; + if (n->name && strcmp(last, n->name) == 0) { + return; + } + if (!list_add(x, x->irregular, last, n)) { + x->failed = true; + } +} + +static void *pdfr_build(const cbm_doclink_build_in_t *in) { + if (in->file_count == 0) { + return NULL; /* no PDF in the repository: nothing to resolve */ + } + pdfr_index_t *x = (pdfr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->project_len = x->project ? strlen(x->project) : 0; + x->paths = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT); + x->basenames = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT); + x->dirs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT); + x->irregular = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT); + if (!x->project || !x->paths || !x->basenames || !x->dirs || !x->irregular || + !index_paths(x, in->graph, "File") || !index_paths(x, in->graph, "Folder")) { + pdfr_destroy(x); + return NULL; + } + cbm_gbuf_foreach_node(in->graph, index_irregular, x); + if (x->failed) { + pdfr_destroy(x); + return NULL; + } + return x; +} + +static void pdfr_destroy(void *index) { + pdfr_index_t *x = (pdfr_index_t *)index; + if (!x) { + return; + } + cbm_ht_free(x->paths); + cbm_ht_free(x->basenames); + cbm_ht_free(x->dirs); + cbm_ht_free(x->irregular); + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* ── Candidate sets ──────────────────────────────────────────────── */ + +typedef struct { + const cbm_gbuf_node_t **v; + int n; + int cap; + bool fail; +} nodeset_t; + +static void ns_add(nodeset_t *s, const cbm_gbuf_node_t *n) { + for (int i = 0; i < s->n; i++) { + if (s->v[i] == n) { + return; + } + } + if (s->fail) { + return; + } + if (s->n == s->cap) { + int ncap = s->cap ? s->cap * 2 : 8; + const cbm_gbuf_node_t **g = (const cbm_gbuf_node_t **)cbm_realloc( + CBM_MEM_CLASS_OTHER, (void *)s->v, (size_t)ncap * sizeof(*g)); + if (!g) { + s->fail = true; + return; + } + s->v = g; + s->cap = ncap; + } + s->v[s->n++] = n; +} + +static void ns_free(nodeset_t *s) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)s->v); + memset(s, 0, sizeof(*s)); +} + +/* The qualified name without the project prefix. */ +static const char *rel_qn(const pdfr_index_t *x, const char *qn) { + if (qn && x->project_len && strncmp(qn, x->project, x->project_len) == 0 && + qn[x->project_len] == '.') { + return qn + x->project_len + 1; + } + return qn ? qn : ""; +} + +static bool code_node(const pdfr_index_t *x, const cbm_gbuf_node_t *n) { + if (!n->label || in_list(n->label, PDFR_EXCLUDED_LABELS) || !n->qualified_name || + !n->qualified_name[0]) { + return false; + } + if (n->file_path && n->file_path[0] == '<') { + return false; /* builtins */ + } + return strncmp(rel_qn(x, n->qualified_name), "__", 2) != 0; /* dependency stand-ins */ +} + +/* s is a component-aligned suffix of q ('.'-separated) */ +static bool comp_suffix(const char *q, const char *s) { + size_t ql = strlen(q); + size_t sl = strlen(s); + if (sl > ql || memcmp(q + ql - sl, s, sl) != 0) { + return false; + } + return sl == ql || q[ql - sl - 1] == '.'; +} + +/* The owner-qualified name of a node: rel(parent_class) + "." + name, when the + * node has a parent_class (properties JSON). */ +static bool owner_name(const pdfr_index_t *x, const cbm_gbuf_node_t *n, char *buf, size_t cap) { + const char *p = n->properties_json ? strstr(n->properties_json, "\"parent_class\":\"") : NULL; + if (!p) { + return false; + } + p += strlen("\"parent_class\":\""); + char pc[PDFR_KEY_CAP]; + size_t k = 0; + while (*p && *p != '"' && k + 1 < sizeof(pc)) { + if (*p == '\\' && p[1]) { + p++; + } + pc[k++] = *p++; + } + pc[k] = '\0'; + if (*p != '"') { + return false; + } + const char *prel = rel_qn(x, pc); + int w = snprintf(buf, cap, "%s.%s", prel, n->name ? n->name : ""); + return w > 0 && (size_t)w < cap; +} + +static void qn_candidate(const pdfr_index_t *x, const cbm_gbuf_node_t *n, const char *s, + nodeset_t *out, nodeset_t *exact) { + if (!code_node(x, n)) { + return; + } + const char *rel = rel_qn(x, n->qualified_name); + char alt[PDFR_KEY_CAP]; + bool via_rel = comp_suffix(rel, s); + bool via_alt = !via_rel && owner_name(x, n, alt, sizeof(alt)) && strcmp(alt, rel) != 0 && + comp_suffix(alt, s); + if (via_rel || via_alt) { + ns_add(out, n); + if (via_rel && strcmp(rel, s) == 0) { + ns_add(exact, n); + } + } +} + +/* Nodes whose QN or owner-qualified name ends with the mention s (>= 2 + * components). A node's name may itself hold dots, so every component suffix + * of s is tried as a name. */ +static void qn_suffix_set(const pdfr_index_t *x, const cbm_gbuf_t *graph, const char *s, + nodeset_t *out, nodeset_t *exact) { + const char *starts[PDFR_MAX_COMPS]; + int nc = 0; + starts[nc++] = s; + for (const char *q = s; *q && nc < PDFR_MAX_COMPS; q++) { + if (*q == '.') { + starts[nc++] = q + 1; + } + } + if (nc < 2) { + return; + } + for (int j = nc - 1; j >= 0; j--) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, starts[j], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + qn_candidate(x, nodes[i], s, out, exact); + } + } + for (const pdfr_list_t *l = (const pdfr_list_t *)cbm_ht_get(x->irregular, starts[nc - 1]); l; + l = l->next) { + qn_candidate(x, l->node, s, out, exact); + } +} + +static int node_id_cmp(const void *a, const void *b) { + const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b; + return x->id < y->id ? -1 : x->id > y->id ? 1 : 0; +} + +/* The field test's _collapse: one entity's several nodes become one. */ +static const cbm_gbuf_node_t *collapse(const pdfr_index_t *x, nodeset_t *s) { + if (s->n == 1) { + return s->v[0]; + } + if (s->n == 0) { + return NULL; + } + qsort((void *)s->v, (size_t)s->n, sizeof(*s->v), node_id_cmp); + const cbm_gbuf_node_t *c = NULL; + int containers = 0; + for (int i = 0; i < s->n; i++) { + if (s->v[i]->label && in_list(s->v[i]->label, PDFR_CONTAINERS)) { + c = s->v[i]; + containers++; + } + } + if (containers == 1) { + const char *crel = rel_qn(x, c->qualified_name); + size_t cl = strlen(crel); + bool all = true; + for (int i = 0; i < s->n && all; i++) { + const cbm_gbuf_node_t *n = s->v[i]; + if (n == c) { + continue; + } + const char *rel = rel_qn(x, n->qualified_name); + all = n->name && c->name && strcmp(n->name, c->name) == 0 && + strncmp(rel, crel, cl) == 0 && rel[cl] == '.'; + } + if (all) { + return c; /* a class and its constructors */ + } + } + const cbm_gbuf_node_t *f = s->v[0]; + for (int i = 1; i < s->n; i++) { + const cbm_gbuf_node_t *n = s->v[i]; + if (strcmp(n->file_path ? n->file_path : "", f->file_path ? f->file_path : "") != 0 || + n->start_line != f->start_line || + strcmp(n->name ? n->name : "", f->name ? f->name : "") != 0) { + return NULL; + } + } + return f; /* duplicate nodes of one definition: the lowest id */ +} + +/* ── One mention form ────────────────────────────────────────────── */ + +typedef enum { RES_NONE = 0, RES_AMBIGUOUS, RES_HIT } res_kind_t; + +typedef struct { + res_kind_t kind; + bool exact; + const cbm_gbuf_node_t *node; +} res_t; + +static res_t outcome(const pdfr_index_t *x, nodeset_t *s, bool exact) { + res_t r = {RES_NONE, false, NULL}; + if (s->n == 0) { + return r; + } + const cbm_gbuf_node_t *n = collapse(x, s); + if (n) { + r.kind = RES_HIT; + r.exact = exact; + r.node = n; + } else { + r.kind = RES_AMBIGUOUS; + } + return r; +} + +static void list_to_set(const pdfr_list_t *l, nodeset_t *s) { + for (; l; l = l->next) { + ns_add(s, l->node); + } +} + +static res_t resolve_form(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax, + const char *form) { + res_t none = {RES_NONE, false, NULL}; + if (syntax == CBM_DOCLINK_PDF_PATH || syntax == CBM_DOCLINK_PDF_FILE) { + const char *p = form; + while (*p == '/') { + p++; + } + char key[PDFR_KEY_CAP]; + size_t n = strlen(p); + while (n > 0 && p[n - 1] == '/') { + n--; + } + if (n == 0 || n >= sizeof(key)) { + return none; + } + memcpy(key, p, n); + key[n] = '\0'; + nodeset_t s = {0}; + list_to_set((const pdfr_list_t *)cbm_ht_get(x->paths, key), &s); + if (s.n) { + res_t r = outcome(x, &s, true); + ns_free(&s); + return r; + } + const char *slash = strrchr(key, '/'); + for (const pdfr_list_t *l = + (const pdfr_list_t *)cbm_ht_get(x->basenames, slash ? slash + 1 : key); + l; l = l->next) { + const char *fp = l->node->file_path; + while (*fp == '/') { + fp++; + } + size_t fl = strlen(fp); + while (fl > 0 && fp[fl - 1] == '/') { + fl--; + } + if (fl >= n && memcmp(fp + fl - n, key, n) == 0 && (fl == n || fp[fl - n - 1] == '/')) { + ns_add(&s, l->node); + } + } + if (s.n) { + res_t r = outcome(x, &s, false); + ns_free(&s); + return r; + } + ns_free(&s); + if (syntax == CBM_DOCLINK_PDF_PATH) { + return none; + } + } + if (syntax == CBM_DOCLINK_PDF_QN || syntax == CBM_DOCLINK_PDF_FILE) { + nodeset_t all = {0}; + nodeset_t ex = {0}; + qn_suffix_set(x, graph, form, &all, &ex); + res_t r = none; + if (ex.n == 1) { + r.kind = RES_HIT; + r.exact = true; + r.node = ex.v[0]; + } else if (all.n) { + r = outcome(x, &all, false); + } + ns_free(&all); + ns_free(&ex); + return r; + } + if (syntax == CBM_DOCLINK_PDF_NAME) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + nodeset_t s = {0}; + if (cbm_gbuf_find_by_name(graph, form, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + if (code_node(x, nodes[i])) { + ns_add(&s, nodes[i]); + } + } + } + res_t r = outcome(x, &s, false); + ns_free(&s); + return r; + } + return none; +} + +/* A join's forms: the first, plus the hyphen-kept alternative. One node hit + * by the forms is the outcome; two different nodes are ambiguous. */ +static res_t resolve_join(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax, + const char *raw) { + char buf[PDFR_KEY_CAP]; + const char *end = strchr(raw, '\x1e'); + size_t len = end ? (size_t)(end - raw) : strlen(raw); + if (len >= sizeof(buf)) { + res_t none = {RES_NONE, false, NULL}; + return none; + } + memcpy(buf, raw, len); + buf[len] = '\0'; + char *forms[PDFR_MAX_FORMS]; + int nf = 0; + forms[nf++] = buf; + char *sep = strchr(buf, '\x1f'); + if (sep) { + *sep = '\0'; + forms[nf++] = sep + 1; + } + res_t first = resolve_form(x, graph, syntax, forms[0]); + if (nf == 1) { + return first; + } + res_t second = resolve_form(x, graph, syntax, forms[1]); + if (first.kind == RES_HIT && second.kind == RES_HIT) { + if (first.node == second.node) { + return first; + } + res_t amb = {RES_AMBIGUOUS, false, NULL}; + return amb; + } + if (first.kind == RES_HIT) { + return first; + } + if (second.kind == RES_HIT) { + return second; + } + return first; +} + +/* ── Hygiene ─────────────────────────────────────────────────────── */ + +static bool data_target(const char *fp) { + char low[PDFR_KEY_CAP]; + size_t n = strlen(fp); + if (n >= sizeof(low)) { + return false; + } + for (size_t i = 0; i <= n; i++) { + low[i] = (char)(fp[i] >= 'A' && fp[i] <= 'Z' ? fp[i] + ('a' - 'A') : fp[i]); + } + for (int i = 0; PDFR_DATA_EXT[i]; i++) { + if (ends_with(low, PDFR_DATA_EXT[i])) { + return true; + } + } + return false; +} + +static bool test_target(const char *fp) { + const char *base = strrchr(fp, '/'); + base = base ? base + 1 : fp; + if ((starts_with(base, "test_") && ends_with(base, ".py")) || ends_with(base, "_test.py") || + ends_with(base, "_test.go") || ends_with(base, "Test.cs") || ends_with(base, "Tests.cs") || + ends_with(base, "Test.java") || ends_with(base, "Tests.java") || + strcmp(base, "conftest.py") == 0 || ends_with(base, ".t.sol")) { + return true; + } + static const char *const js[] = {".test.js", ".test.jsx", ".test.ts", ".test.tsx", ".spec.js", + ".spec.jsx", ".spec.ts", ".spec.tsx", NULL}; + for (int i = 0; js[i]; i++) { + if (ends_with(base, js[i]) && strlen(base) > strlen(js[i])) { + return true; + } + } + const char *s = fp; + for (const char *q = fp; q < base; q++) { + if (*q == '/') { + char comp[PDFR_KEY_CAP]; + size_t n = (size_t)(q - s); + if (n < sizeof(comp)) { + memcpy(comp, s, n); + comp[n] = '\0'; + if (in_list(comp, PDFR_TEST_DIRS)) { + return true; + } + } + s = q + 1; + } + } + return false; +} + +/* ── Per file ────────────────────────────────────────────────────── */ + +typedef struct { + const CBMDocLink *base; + int n; + uint8_t *suppressed; /* line-local tokens a resolved join replaces */ + res_t *join; /* each join token's own outcome */ + /* A page set repeats its paths and names (an audit names one file on + * every page): mention text -> 1 + the index of the token whose outcome + * res[] holds, reused for a token of the same family. */ + CBMHashTable *memo; + res_t *res; +} pdfr_state_t; + +static void pdfr_end(void *state); + +static void *pdfr_prepare(const void *index, int run_file, const CBMDocLink *links, int n, + const cbm_gbuf_t *graph) { + (void)run_file; + const pdfr_index_t *x = (const pdfr_index_t *)index; + pdfr_state_t *st = (pdfr_state_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*st)); + if (!st) { + return NULL; + } + st->base = links; + st->n = n; + st->suppressed = (uint8_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1)); + st->join = (res_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1) * sizeof(res_t)); + st->res = (res_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1) * sizeof(res_t)); + st->memo = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_MEMO_INIT); + if (!st->suppressed || !st->join || !st->res || !st->memo) { + pdfr_end(st); + return NULL; + } + for (int i = 0; i < n; i++) { + const CBMDocLink *l = &links[i]; + if (!(l->flags & CBM_DOCLINK_FLAG_JOIN) || !l->raw) { + continue; + } + st->join[i] = resolve_join(x, graph, l->syntax, l->raw); + if (st->join[i].kind != RES_HIT) { + continue; + } + const char *f = strchr(l->raw, '\x1e'); + while (f && *f) { + f++; + char *endp = NULL; + long k = strtol(f, &endp, 10); + if (endp == f) { + break; + } + if (k >= 0 && k < n) { + st->suppressed[k] = 1; + } + f = (*endp == ',') ? endp : NULL; + } + } + return st; +} + +static void pdfr_end(void *state) { + pdfr_state_t *st = (pdfr_state_t *)state; + if (!st) { + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, st->suppressed); + cbm_free(CBM_MEM_CLASS_OTHER, st->join); + cbm_free(CBM_MEM_CLASS_OTHER, st->res); + cbm_ht_free(st->memo); + cbm_free(CBM_MEM_CLASS_OTHER, st); +} + +/* The mention's first component names something of this repository. */ +static bool first_known(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax, + const char *raw) { + char first[PDFR_KEY_CAP]; + size_t n = 0; + const char *p = raw; + while (*p == '/') { + p++; + } + char sep = syntax == CBM_DOCLINK_PDF_PATH ? '/' : '.'; + while (p[n] && p[n] != sep && n + 1 < sizeof(first)) { + first[n] = p[n]; + n++; + } + first[n] = '\0'; + if (!n) { + return false; + } + if (cbm_ht_get(x->dirs, first) || cbm_ht_get(x->basenames, first)) { + return true; + } + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + return cbm_gbuf_find_by_name(graph, first, &nodes, &count) == 0 && count > 0; +} + +static void finish_hit(cbm_doclink_outcome_t *out, res_t r) { + const char *fp = r.node->file_path ? r.node->file_path : ""; + if (data_target(fp)) { + out->kind = CBM_DOCLINK_LOCAL; /* R2: a data file, not code */ + return; + } + if (test_target(fp)) { + out->kind = CBM_DOCLINK_UNRESOLVED; /* R4 */ + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + out->kind = CBM_DOCLINK_EDGE; + out->target = r.node; + out->exact = r.exact; +} + +static void pdfr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)run_file; + const pdfr_index_t *x = (const pdfr_index_t *)index; + pdfr_state_t *st = (pdfr_state_t *)state; + out->kind = CBM_DOCLINK_LOCAL; + if (!x || !link->raw) { + return; + } + int i = st ? (int)(link - st->base) : -1; + bool in_file = st && i >= 0 && i < st->n; + if (link->flags & CBM_DOCLINK_FLAG_JOIN) { + /* a join stands only when it resolved; a bare-name join never links */ + if (in_file && st->join[i].kind == RES_HIT && link->syntax != CBM_DOCLINK_PDF_NAME) { + finish_hit(out, st->join[i]); + } + return; + } + if (in_file && st->suppressed[i]) { + return; /* a resolved join replaced this fragment */ + } + if (link->syntax == CBM_DOCLINK_PDF_NAME) { + return; + } + res_t r; + uintptr_t seen = in_file ? (uintptr_t)cbm_ht_get(st->memo, link->raw) : 0; + if (seen && st->base[seen - 1].syntax == link->syntax) { + r = st->res[seen - 1]; + } else { + r = resolve_form(x, graph, link->syntax, link->raw); + if (in_file && !seen) { + st->res[i] = r; + cbm_ht_set(st->memo, link->raw, (void *)(uintptr_t)(i + 1)); + } + } + if (r.kind == RES_HIT) { + finish_hit(out, r); + } else if (r.kind == RES_AMBIGUOUS) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + } else if (first_known(x, graph, link->syntax, link->raw)) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + } +} + +const cbm_doclink_resolver_t cbm_doclink_pdf_resolver = { + .langs = {CBM_LANG_PDF}, + .lang_count = 1, + .scope_tag = NULL, + .build = pdfr_build, + .destroy = pdfr_destroy, + .file_prepare = pdfr_prepare, + .file_end = pdfr_end, + .resolve = pdfr_resolve, + .via = "pdf", +}; diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index a237881ec8..e7f5ee40a1 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -37,6 +37,7 @@ #include #include "foundation/compat_fs.h" +#include "foundation/mem_core.h" #include "foundation/log.h" #include "sqlite3.h" #include "store/store.h" @@ -254,6 +255,50 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * * guessing. The proxy load stays edge-free and property-free, which is * where the old full load actually spent its time. */ +/* `qn` and `parent` without the project prefix. */ +static const char *delta_rel(const char *project, const char *qn) { + size_t pl = project ? strlen(project) : 0; + if (qn && pl && strncmp(qn, project, pl) == 0 && qn[pl] == '.') { + return qn + pl + 1; + } + return qn ? qn : ""; +} + +/* A proxy's properties carrying `parent` when owner + "." + name is not the + * node's own (relative) QN, merged with the http client keys if any; NULL + * when the owner adds nothing (CBM_MEM_CLASS_OTHER; the caller frees). */ +static char *delta_owner_props(const char *project, const char *name, const char *qn, + const char *parent, const char *client_props) { + const char *rel = delta_rel(project, qn); + const char *prel = delta_rel(project, parent); + size_t pl = strlen(prel); + size_t nl = name ? strlen(name) : 0; + if (strlen(rel) == pl + 1 + nl && strncmp(rel, prel, pl) == 0 && rel[pl] == '.' && + (nl == 0 || memcmp(rel + pl + 1, name, nl) == 0)) { + return NULL; + } + size_t cap = 2 * strlen(parent) + (client_props ? strlen(client_props) : 0) + 32; + char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, cap); + if (!out) { + return NULL; + } + size_t k = 0; + k += (size_t)snprintf(out + k, cap - k, "{\"parent_class\":\""); + for (const char *p = parent; *p && k + 3 < cap; p++) { + if (*p == '"' || *p == '\\') { + out[k++] = '\\'; + } + out[k++] = *p; + } + out[k++] = '"'; + if (client_props && client_props[0] == '{' && client_props[1] && client_props[1] != '}') { + snprintf(out + k, cap - k, ",%s", client_props + 1); + } else { + snprintf(out + k, cap - k, "}"); + } + return out; +} + int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf) { sqlite3 *db = cbm_store_get_db(store); if (!db) { @@ -292,7 +337,13 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g * compose `api.get('/p')` against its unchanged wrapper exactly as a full * build does. Only the two keys, only on the rare Module/Variable rows * that carry them; every other proxy stays "{}". Proxies are never - * written back (cbm_delta_patch skips id <= max_db_id). */ + * written back (cbm_delta_patch skips id <= max_db_id). + * + * The other: a node's parent_class where it changes the name the node + * goes by -- a Go method's QN has no receiver, a class-body variable's QN + * no class. A document naming `Order.Reset` binds such a node through + * its owner (doc_links_pdf.c), so a re-resolved document needs it too. + * Only where owner + name differs from the QN (delta_owner_props). */ sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2( db, @@ -301,7 +352,10 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g " AND instr(properties, '\"http_client\"') > 0" " THEN json_object('http_client', json_extract(properties, '$.http_client')," " 'http_base_url', json_extract(properties, '$.http_base_url')) END," - " start_line, end_line" + " start_line, end_line," + " CASE WHEN label IN ('Method','Variable')" + " AND instr(properties, '\"parent_class\"') > 0" + " THEN json_extract(properties, '$.parent_class') END" " FROM nodes" " WHERE project = ?1 AND label NOT IN" " ('Macro','Comment','Section','Branch','Commit','Tag')" @@ -324,10 +378,15 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g * in a full build. */ int start_line = sqlite3_column_int(stmt, 6); int end_line = sqlite3_column_int(stmt, 7); + const char *parent = (const char *)sqlite3_column_text(stmt, 8); + char *owner_props = + parent ? delta_owner_props(project, name, qn, parent, client_props) : NULL; /* Pin the gbuf id to the database id: proxies ARE their rows. */ cbm_gbuf_set_next_id(gbuf, id); - int64_t got = cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", start_line, - end_line, client_props ? client_props : "{}"); + int64_t got = + cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", start_line, end_line, + owner_props ? owner_props : (client_props ? client_props : "{}")); + cbm_free(CBM_MEM_CLASS_OTHER, owner_props); if (got != id) { /* A QN collision inside the preseed set would silently split * identity between RAM and disk; the run cannot be trusted. */ diff --git a/tests/repro/repro_call_node_manifest.c b/tests/repro/repro_call_node_manifest.c index 1e9a418c19..fe6e3eda5a 100644 --- a/tests/repro/repro_call_node_manifest.c +++ b/tests/repro/repro_call_node_manifest.c @@ -547,9 +547,11 @@ TEST(repro_call_node_manifest_live_specs_contain_only_expected_primary_entries) CBMLanguage language = (CBMLanguage)value; const CBMLangSpec *spec = cbm_lang_spec(language); - /* The registry reproduction owns Nim's stale enum and Studio Export's - * transform-only, intentionally grammar-free classification. */ - if (language == CBM_LANG_NIM || language == CBM_LANG_OBJECTSCRIPT_EXPORT) { + /* The registry reproduction owns Nim's stale enum and the + * intentionally grammar-free classifications: Studio Export's + * transform-only one and PDF's DOCUMENT one (doc_pdf.c). */ + if (language == CBM_LANG_NIM || language == CBM_LANG_OBJECTSCRIPT_EXPORT || + language == CBM_LANG_PDF) { continue; } if (!spec || spec->language != language) { diff --git a/tests/repro/repro_language_registry.c b/tests/repro/repro_language_registry.c index ddb1f9e7e4..67496eb7fe 100644 --- a/tests/repro/repro_language_registry.c +++ b/tests/repro/repro_language_registry.c @@ -18,6 +18,7 @@ typedef enum { CAP_NO_CALL, CAP_TRANSFORM_ONLY, CAP_UNSUPPORTED, + CAP_DOCUMENT, CAP_COUNT } LanguageCapability; @@ -43,6 +44,9 @@ size_t repro_call_argument_matrix_b_copy_language_ids(CBMLanguage *language_ids, "source is transformed into another registered language first"} #define UNSUPPORTED(lang) \ [CBM_LANG_##lang] = {CAP_UNSUPPORTED, #lang, "enum retained without a registered grammar"} +#define DOCUMENT(lang) \ + [CBM_LANG_##lang] = {CAP_DOCUMENT, #lang, \ + "a document format read without a grammar; no code, no calls"} /* Explicit ledger: no default initializer is intentional. A new enum remains * CAP_UNSET until its call-metadata/reference-vocabulary behavior is audited. */ @@ -213,6 +217,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = { CALL_WITH_REFERENCE_VOCAB(ARKTS), CALL_WITHOUT_REFERENCE_VOCAB(PLSQL), CALL_WITHOUT_REFERENCE_VOCAB(CHIALISP), + DOCUMENT(PDF), }; #undef CALL_WITH_REFERENCE_VOCAB @@ -220,6 +225,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = { #undef NO_CALL #undef TRANSFORM_ONLY #undef UNSUPPORTED +#undef DOCUMENT _Static_assert(sizeof(LANGUAGE_CAPABILITIES) / sizeof(LANGUAGE_CAPABILITIES[0]) == CBM_LANG_COUNT, "language capability ledger must track CBM_LANG_COUNT"); @@ -271,7 +277,8 @@ TEST(repro_language_capability_ledger_covers_every_enum) { } counts[entry->capability]++; - if (entry->capability == CAP_UNSUPPORTED || entry->capability == CAP_TRANSFORM_ONLY) { + if (entry->capability == CAP_UNSUPPORTED || entry->capability == CAP_TRANSFORM_ONLY || + entry->capability == CAP_DOCUMENT) { if (spec) { fprintf(stderr, " [language-registry] lang=%s " @@ -303,12 +310,14 @@ TEST(repro_language_capability_ledger_covers_every_enum) { if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 88 || counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 49 || - counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) { + counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1 || + counts[CAP_DOCUMENT] != 1) { fprintf(stderr, " [language-registry] invariant=capability_partition call_ref_vocab=%d ref_gap=%d " - "no_call=%d transform_only=%d unsupported=%d\n", + "no_call=%d transform_only=%d unsupported=%d document=%d\n", counts[CAP_CALL_WITH_REFERENCE_VOCAB], counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB], - counts[CAP_NO_CALL], counts[CAP_TRANSFORM_ONLY], counts[CAP_UNSUPPORTED]); + counts[CAP_NO_CALL], counts[CAP_TRANSFORM_ONLY], counts[CAP_UNSUPPORTED], + counts[CAP_DOCUMENT]); failures++; } @@ -321,7 +330,7 @@ TEST(repro_call_argument_matrices_equal_call_capability_ledger) { EXPECTED_MATRIX_A_ROWS = 68, EXPECTED_MATRIX_B_ROWS = 49, EXPECTED_CALL_CAPABLE_LANGUAGES = 115, - EXPECTED_NON_CALL_LANGUAGES = 51, + EXPECTED_NON_CALL_LANGUAGES = 52, /* 49 no-call + transform-only + unsupported + PDF */ EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2, }; CBMLanguage matrix_a_ids[CBM_LANG_COUNT]; @@ -432,11 +441,11 @@ TEST(repro_call_argument_matrices_equal_call_capability_ledger) { } if (entry->capability != CAP_NO_CALL && entry->capability != CAP_TRANSFORM_ONLY && - entry->capability != CAP_UNSUPPORTED) { + entry->capability != CAP_UNSUPPORTED && entry->capability != CAP_DOCUMENT) { fprintf(stderr, " [language-registry] lang=%s id=%d " "invariant=remaining_capability_explicit " - "expected=NO_CALL_or_TRANSFORM_ONLY_or_UNSUPPORTED " + "expected=NO_CALL_or_TRANSFORM_ONLY_or_UNSUPPORTED_or_DOCUMENT " "actual=%d\n", language_name, language_id, (int)entry->capability); failures++; diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c new file mode 100644 index 0000000000..0ce880d59e --- /dev/null +++ b/tests/test_doc_links_pdf.c @@ -0,0 +1,793 @@ +/* + * test_doc_links_pdf.c — PDF documents -> Sections and MENTIONS edges. + * + * The text-layer extractor (internal/cbm/pdf: fonts, encodings, CMaps, + * cross-reference forms, filters, layout, hostile input), the mention scanner + * (doc_pdf.c, through its test seam: expected tokens are the field-test + * scanner's own output for the same text), the resolver (doc_links_pdf.c: + * tiers, owner-qualified Go methods, hygiene, unresolved reasons), the + * snippet of a page, and incremental == full. PDFs are built here, byte by + * byte, with their cross-reference offsets computed. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "mcp/mcp.h" +#include "pdf/pdf.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include +#include + +/* ── building PDFs ───────────────────────────────────────────────── */ + +typedef struct { + unsigned char *p; + size_t n; + size_t cap; +} tp_buf_t; + +static void tp_put(tp_buf_t *b, const void *s, size_t n) { + if (b->n + n + 1 > b->cap) { + size_t cap = b->cap ? b->cap * 2 : 4096; + while (cap < b->n + n + 1) { + cap *= 2; + } + b->p = (unsigned char *)realloc(b->p, cap); + b->cap = cap; + } + memcpy(b->p + b->n, s, n); + b->n += n; + b->p[b->n] = 0; +} + +static void tp_puts(tp_buf_t *b, const char *s) { + tp_put(b, s, strlen(s)); +} + +/* One object's body: a value, or a dict plus stream data. */ +typedef struct { + const char *dict; /* the value, or the stream's dict without /Length */ + const void *data; /* stream data, or NULL */ + size_t len; +} tp_obj_t; + +/* A PDF of objects 1..n (objs[i] is object i+1) with a classic xref table; + * `trailer_extra` is added to the trailer dict. Free with free(). */ +static unsigned char *tp_pdf(const tp_obj_t *objs, int n, const char *trailer_extra, + size_t *out_len) { + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n%\xe2\xe3\xcf\xd3\n"); + size_t *off = (size_t *)calloc((size_t)n + 1, sizeof(size_t)); + char line[256]; + for (int i = 0; i < n; i++) { + off[i + 1] = b.n; + snprintf(line, sizeof(line), "%d 0 obj\n", i + 1); + tp_puts(&b, line); + if (objs[i].data) { + snprintf(line, sizeof(line), "<< /Length %zu ", objs[i].len); + tp_puts(&b, line); + tp_puts(&b, objs[i].dict); + tp_puts(&b, " >>\nstream\n"); + tp_put(&b, objs[i].data, objs[i].len); + tp_puts(&b, "\nendstream"); + } else { + tp_puts(&b, objs[i].dict); + } + tp_puts(&b, "\nendobj\n"); + } + size_t xref = b.n; + snprintf(line, sizeof(line), "xref\n0 %d\n0000000000 65535 f \n", n + 1); + tp_puts(&b, line); + for (int i = 1; i <= n; i++) { + snprintf(line, sizeof(line), "%010zu 00000 n \n", off[i]); + tp_puts(&b, line); + } + snprintf(line, sizeof(line), + "trailer\n<< /Size %d /Root 1 0 R %s >>\nstartxref\n%zu\n%%%%EOF\n", n + 1, + trailer_extra ? trailer_extra : "", xref); + tp_puts(&b, line); + free(off); + *out_len = b.n; + return b.p; +} + +static unsigned char *tp_flate(const void *src, size_t n, size_t *out_len) { + uLongf cap = compressBound((uLong)n); + unsigned char *out = (unsigned char *)malloc(cap); + compress(out, &cap, (const Bytef *)src, (uLong)n); + *out_len = cap; + return out; +} + +#define TP_CATALOG "<< /Type /Catalog /Pages 2 0 R >>" +#define TP_HELV "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" + +/* A one-page PDF whose content is `content` with font F1 = Helvetica. */ +static unsigned char *tp_simple(const char *content, size_t *len) { + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 " + "0 R >> >> /Contents 5 0 R >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", content, strlen(content)}, + }; + return tp_pdf(o, 5, NULL, len); +} + +typedef struct { + CBMArena arena; + cbm_pdf_result_t r; +} tp_text_t; + +static bool tp_extract(const unsigned char *pdf, size_t len, tp_text_t *t) { + cbm_arena_init(&t->arena); + return cbm_pdf_extract(pdf, len, &t->arena, &t->r); +} + +/* ── extractor ───────────────────────────────────────────────────── */ + +TEST(pdf_extract_layout) { + /* inherited resources, two pages, a line break by Td, a word break by TJ + * kerning, a new text object further down */ + const char *c1 = "BT /F1 12 Tf 72 700 Td (Hello src/auth/handler.go) Tj 0 -14 Td (second line) " + "Tj ET\nBT /F1 12 Tf 72 600 Td [(word)-1000(gap)] TJ ET"; + const char *c2 = "BT /F1 12 Tf 72 700 Td (page two) Tj ET"; + tp_obj_t o[7] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 /Resources << /Font << /F1 4 0 R >> >> " + "/MediaBox [0 0 612 792] >>", + NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Contents 5 0 R >>", NULL, 0}, + {TP_HELV, NULL, 0}, + {"", c1, strlen(c1)}, + {"<< /Type /Page /Parent 2 0 R /Contents 7 0 R >>", NULL, 0}, + {"", c2, strlen(c2)}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 7, NULL, &len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 2); + ASSERT_STR_EQ(t.r.pages[0].text, "Hello src/auth/handler.go\nsecond line\nword gap"); + ASSERT_STR_EQ(t.r.pages[1].text, "page two"); + ASSERT_EQ(t.r.pages[0].unmapped, 0); + ASSERT_TRUE(t.r.pages[0].glyphs > 30); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +TEST(pdf_extract_fonts) { + /* Type0 Identity-H through a ToUnicode CMap (bfchar, a ligature, a + * bfrange, an unmapped code), and a simple font with /Differences */ + static const char cmap[] = "/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + "1 begincodespacerange <0000> endcodespacerange\n" + "2 beginbfchar <0001> <0041> <0002> <00660069> endbfchar\n" + "1 beginbfrange <0003> <0005> <0061> endbfrange\n" + "endcmap CMapName currentdict /CMap defineresource pop end end\n"; + const char *c = "BT /F1 10 Tf 10 700 Td <000100020003000400050006> Tj ET\n" + "BT /F2 10 Tf 10 600 Td (AB) Tj ET"; + tp_obj_t o[8] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 " + "R /F2 8 0 R >> >> /Contents 5 0 R >>", + NULL, 0}, + {"<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEE+Foo /Encoding /Identity-H " + "/DescendantFonts [6 0 R] /ToUnicode 7 0 R >>", + NULL, 0}, + {"", c, strlen(c)}, + {"<< /Type /Font /Subtype /CIDFontType2 /BaseFont /Foo /DW 500 /W [1 [600 600] 3 5 " + "700] >>", + NULL, 0}, + {"", cmap, sizeof(cmap) - 1}, + {"<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding << /BaseEncoding " + "/WinAnsiEncoding /Differences [65 /B /A] >> >>", + NULL, 0}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 8, NULL, &len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "Afiabc\xEF\xBF\xBD\nBA"); + ASSERT_EQ(t.r.pages[0].unmapped, 1); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* A PDF whose page and font live in a Flate object stream, indexed by a + * PNG-predicted xref stream, with no classic table at all. */ +static unsigned char *tp_compressed(size_t *out_len) { + const char *content = "BT /F1 12 Tf 72 700 Td (from an object stream) Tj ET"; + size_t clen; + unsigned char *cz = tp_flate(content, strlen(content), &clen); + /* object stream: objects 3 (page) and 4 (font) */ + const char *o3 = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << " + "/F1 4 0 R >> >> /Contents 5 0 R >>"; + const char *o4 = TP_HELV; + char hdr[64]; + snprintf(hdr, sizeof(hdr), "3 0 4 %zu ", strlen(o3) + 1); + tp_buf_t stm = {0}; + tp_puts(&stm, hdr); + size_t first = stm.n; + tp_puts(&stm, o3); + tp_puts(&stm, " "); + tp_puts(&stm, o4); + size_t sz; + unsigned char *stmz = tp_flate(stm.p, stm.n, &sz); + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + size_t off[8] = {0}; + char line[256]; + off[1] = b.n; + tp_puts(&b, "1 0 obj\n" TP_CATALOG "\nendobj\n"); + off[2] = b.n; + tp_puts(&b, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"); + off[5] = b.n; + snprintf(line, sizeof(line), "5 0 obj\n<< /Length %zu /Filter /FlateDecode >>\nstream\n", clen); + tp_puts(&b, line); + tp_put(&b, cz, clen); + tp_puts(&b, "\nendstream\nendobj\n"); + off[6] = b.n; + snprintf(line, sizeof(line), + "6 0 obj\n<< /Type /ObjStm /N 2 /First %zu /Length %zu /Filter /FlateDecode " + ">>\nstream\n", + first, sz); + tp_puts(&b, line); + tp_put(&b, stmz, sz); + tp_puts(&b, "\nendstream\nendobj\n"); + off[7] = b.n; + /* xref stream rows: W [1 2 1], PNG Up/None rows of 4 bytes + filter byte */ + unsigned char rows[8][5]; + for (int i = 0; i < 8; i++) { + unsigned type = (i == 0) ? 0 : (i == 3 || i == 4) ? 2 : 1; + unsigned f2 = (type == 1) ? (unsigned)off[i] : (type == 2) ? 6U : 0U; + unsigned f3 = (i == 4) ? 1U : 0U; + rows[i][0] = 0; /* PNG filter: None */ + rows[i][1] = (unsigned char)type; + rows[i][2] = (unsigned char)(f2 >> 8); + rows[i][3] = (unsigned char)f2; + rows[i][4] = (unsigned char)f3; + } + size_t xz; + unsigned char *xrz = tp_flate(rows, sizeof(rows), &xz); + snprintf(line, sizeof(line), + "7 0 obj\n<< /Type /XRef /Size 8 /W [1 2 1] /Root 1 0 R /Length %zu /Filter " + "/FlateDecode /DecodeParms << /Predictor 12 /Columns 4 >> >>\nstream\n", + xz); + tp_puts(&b, line); + tp_put(&b, xrz, xz); + tp_puts(&b, "\nendstream\nendobj\n"); + snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", off[7]); + tp_puts(&b, line); + free(cz); + free(stm.p); + free(stmz); + free(xrz); + *out_len = b.n; + return b.p; +} + +TEST(pdf_extract_structure) { + size_t len; + unsigned char *pdf = tp_compressed(&len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "from an object stream"); + ASSERT_EQ(t.r.counts.xref_reconstructed, 0); + cbm_arena_destroy(&t.arena); + free(pdf); + + /* a wrong startxref: the xref section is found near it */ + unsigned char *ok = tp_simple("BT /F1 12 Tf 72 700 Td (rebuilt) Tj ET", &len); + char *sx = strstr((char *)ok, "startxref\n"); + ASSERT_NOT_NULL(sx); + sx[strlen("startxref\n")] = '9'; + ASSERT_TRUE(tp_extract(ok, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "rebuilt"); + ASSERT_EQ(t.r.counts.xref_offset_fixed, 1); + ASSERT_EQ(t.r.counts.xref_reconstructed, 0); + cbm_arena_destroy(&t.arena); + /* no xref section at all: rebuilt from the object headers, same text */ + char *xr = strstr((char *)ok, "xref\n0 "); + ASSERT_NOT_NULL(xr); + memcpy(xr, "XXXX", 4); + ASSERT_TRUE(tp_extract(ok, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "rebuilt"); + ASSERT_EQ(t.r.counts.xref_reconstructed, 1); + cbm_arena_destroy(&t.arena); + free(ok); + + /* a Form XObject with its own resources, a self-referencing form, an + * inline image with binary data, content split over two streams */ + const char *page_a = "q 1 0 0 1 100 500 cm /Fm1 Do Q /Fm2 Do\n"; + const char *page_b = "BI /W 2 /H 2 /BPC 8 /CS /G ID \x01\xff(junk)\xfe EI\n" + "BT /F1 12 Tf 72 400 Td (after image) Tj ET"; + const char *form = "BT /F1 12 Tf 0 0 Td (in form) Tj ET"; + const char *self = "/Fm2 Do BT /F1 12 Tf 0 300 Td (self) Tj ET"; + tp_obj_t o[9] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /XObject << /Fm1 6 " + "0 R /Fm2 9 0 R >> /Font << /F1 4 0 R >> >> /Contents [5 0 R 7 0 R] >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", page_a, strlen(page_a)}, + {"/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Resources << /Font << /F1 8 0 R " + ">> >>", + form, strlen(form)}, + {"", page_b, strlen(page_b)}, + {TP_HELV, NULL, 0}, + {"/Type /XObject /Subtype /Form /BBox [0 0 200 200]", self, strlen(self)}, + }; + unsigned char *fp = tp_pdf(o, 9, NULL, &len); + ASSERT_TRUE(tp_extract(fp, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "in form")); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "self")); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "after image")); + ASSERT_NULL(strstr(t.r.pages[0].text, "junk")); + cbm_arena_destroy(&t.arena); + free(fp); + PASS(); +} + +static void tp_ahx(tp_buf_t *b, const char *s) { + static const char hex[] = "0123456789abcdef"; + for (; *s; s++) { + char h[3] = {hex[(unsigned char)*s >> 4], hex[(unsigned char)*s & 15], ' '}; + tp_put(b, h, 3); + } + tp_puts(b, ">"); +} + +static void tp_a85(tp_buf_t *b, const unsigned char *s, size_t n) { + tp_puts(b, "<~"); + for (size_t i = 0; i < n; i += 4) { + unsigned char g[4] = {0, 0, 0, 0}; + size_t k = n - i < 4 ? n - i : 4; + memcpy(g, s + i, k); + uint32_t v = ((uint32_t)g[0] << 24) | ((uint32_t)g[1] << 16) | ((uint32_t)g[2] << 8) | g[3]; + char out[5]; + for (int j = 4; j >= 0; j--) { + out[j] = (char)('!' + v % 85); + v /= 85; + } + tp_put(b, out, k + 1); + } + tp_puts(b, "~>"); +} + +TEST(pdf_extract_filters_and_hostile) { + size_t len; + tp_text_t t; + const char *content = "BT /F1 12 Tf 72 700 Td (filtered text) Tj ET"; + /* ASCIIHex, then ASCII85 over Flate */ + tp_buf_t h = {0}; + tp_ahx(&h, content); + size_t zl; + unsigned char *z = tp_flate(content, strlen(content), &zl); + tp_buf_t a = {0}; + tp_a85(&a, z, zl); + const char *dicts[2] = {"/Filter /ASCIIHexDecode", "/Filter [/ASCII85Decode /FlateDecode]"}; + const tp_buf_t *datas[2] = {&h, &a}; + for (int i = 0; i < 2; i++) { + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 " + "R >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {dicts[i], datas[i]->p, datas[i]->n}, + }; + unsigned char *pdf = tp_pdf(o, 5, NULL, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "filtered text"); + cbm_arena_destroy(&t.arena); + free(pdf); + } + free(h.p); + free(a.p); + free(z); + + /* not a PDF; encrypted; truncated anywhere; nested a hundred thousand deep */ + ASSERT_TRUE(tp_extract((const unsigned char *)"hello", 5, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_NOT_PDF); + cbm_arena_destroy(&t.arena); + tp_obj_t enc[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Contents 5 0 R >>", NULL, 0}, + {"<< /Filter /Standard /V 1 /R 2 /O <00> /U <00> /P -4 >>", NULL, 0}, + {"", "BT ET", 5}, + }; + unsigned char *pdf = tp_pdf(enc, 5, "/Encrypt 4 0 R", &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_ENCRYPTED); + ASSERT_EQ(t.r.npages, 0); + cbm_arena_destroy(&t.arena); + free(pdf); + pdf = tp_simple("BT /F1 12 Tf 72 700 Td (cut short) Tj ET", &len); + for (size_t cut = 0; cut < len; cut += 7) { + ASSERT_TRUE(tp_extract(pdf, cut, &t)); /* any status; never a crash */ + cbm_arena_destroy(&t.arena); + } + free(pdf); + enum { DEEP = 100000 }; + char *deep = (char *)malloc(DEEP * 2 + 64); + memset(deep, '[', DEEP); + strcpy(deep + DEEP, " BT /F1 12 Tf (x) Tj ET"); + pdf = tp_simple(deep, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.counts.page_errors, 1); /* the content stops at the nesting limit */ + cbm_arena_destroy(&t.arena); + free(pdf); + free(deep); + /* a page tree that contains itself, a /Prev loop */ + tp_obj_t loop[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [2 0 R 3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R " + ">>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", "BT /F1 12 Tf (loops) Tj ET", 26}, + }; + pdf = tp_pdf(loop, 5, "/Prev 9", &len); + char *sx = strstr((char *)pdf, "startxref\n"); + char prev[32]; + snprintf(prev, sizeof(prev), "/Prev %ld", strtol(sx + strlen("startxref\n"), NULL, 10)); + char *pv = strstr((char *)pdf, "/Prev 9"); + ASSERT_NOT_NULL(pv); + /* the same width: point /Prev at this very section (a loop) */ + if (strlen(prev) <= strlen("/Prev 9") + 6) { + tp_buf_t nb = {0}; + tp_put(&nb, pdf, (size_t)(pv - (char *)pdf)); + tp_puts(&nb, prev); + tp_puts(&nb, pv + strlen("/Prev 9")); + free(pdf); + pdf = nb.p; + len = nb.n; + } + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "loops"); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* ── scanner (expected: the field-test scanner on the same text) ─── */ + +TEST(pdf_scan_mentions) { + const char *page = "See src/auth/handler.go and ./pkg/x.py, http://example.com/a/b.py and " + "and/or 2024/01/02.\n" + "Call pkg.Config.Name(), Handler::run and obj->field, e.g. x.y and " + "example.com.\n" + "The file config.py and setup.cfg; f(a, opts.debug=True).\n" + "long/path/to/mod-\n" + "ule.py and Handler::\n" + "serve here\n" + "\xEF\xAC\x81le_utils.py ligature\n"; + static const struct { + uint32_t line; + int syntax; + uint16_t flags; + const char *raw; + } want[] = { + {1, CBM_DOCLINK_PDF_PATH, 0, "src/auth/handler.go"}, + {1, CBM_DOCLINK_PDF_PATH, 0, "pkg/x.py"}, + {2, CBM_DOCLINK_PDF_QN, 0, "pkg.Config.Name"}, + {2, CBM_DOCLINK_PDF_QN, 0, "Handler.run"}, + {2, CBM_DOCLINK_PDF_QN, 0, "obj.field"}, + {3, CBM_DOCLINK_PDF_FILE, 0, "config.py"}, + {3, CBM_DOCLINK_PDF_FILE, 0, "setup.cfg"}, + {4, CBM_DOCLINK_PDF_PATH, 0, "long/path/to/mod"}, + {5, CBM_DOCLINK_PDF_FILE, 0, "ule.py"}, + {7, CBM_DOCLINK_PDF_FILE, 0, "file_utils.py"}, + {4, CBM_DOCLINK_PDF_PATH, CBM_DOCLINK_FLAG_JOIN, + "long/path/to/module.py\x1flong/path/to/mod-ule.py\x1e" + "7,8"}, + {5, CBM_DOCLINK_PDF_QN, CBM_DOCLINK_FLAG_JOIN, "Handler.serve\x1e"}, + }; + CBMFileResult *r = (CBMFileResult *)calloc(1, sizeof(*r)); + cbm_arena_init(&r->arena); + CBMExtractCtx ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.arena = &r->arena; + ctx.result = r; + cbm_pdf_test_scan_page(&ctx, page, strlen(page), 1, "p.doc.page_1"); + ASSERT_FALSE(r->doc_links.failed); + ASSERT_EQ(r->doc_links.count, (int)(sizeof(want) / sizeof(want[0]))); + for (int i = 0; i < r->doc_links.count; i++) { + const CBMDocLink *l = &r->doc_links.items[i]; + ASSERT_EQ(l->line, want[i].line); + ASSERT_EQ(l->syntax, want[i].syntax); + ASSERT_EQ(l->flags, want[i].flags); + ASSERT_STR_EQ(l->raw, want[i].raw); + ASSERT_EQ(l->def_line, 1); + ASSERT_STR_EQ(l->source_qn, "p.doc.page_1"); + } + cbm_free_result(r); + PASS(); +} + +/* ── the pipeline ────────────────────────────────────────────────── */ + +/* A PDF whose pages each show the given lines (Helvetica, one Tj a line). */ +static void tp_write_pdf(const char *path, const char *const *pages, int npages) { + int n = 3 + 2 * npages; + tp_obj_t *o = (tp_obj_t *)calloc((size_t)n, sizeof(*o)); + char **bufs = (char **)calloc((size_t)npages * 2 + 1, sizeof(char *)); + o[0].dict = TP_CATALOG; + tp_buf_t kids = {0}; + tp_puts(&kids, "<< /Type /Pages /Count 1 /Kids ["); + for (int i = 0; i < npages; i++) { + char k[32]; + snprintf(k, sizeof(k), "%d 0 R ", 4 + 2 * i); + tp_puts(&kids, k); + } + tp_puts(&kids, "] >>"); + o[1].dict = (const char *)kids.p; + o[2].dict = TP_HELV; + for (int i = 0; i < npages; i++) { + char *pg = (char *)malloc(256); + snprintf(pg, 256, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 " + "3 0 R >> >> /Contents %d 0 R >>", + 5 + 2 * i); + bufs[2 * i] = pg; + o[3 + 2 * i].dict = pg; + tp_buf_t c = {0}; + tp_puts(&c, "BT /F1 10 Tf 72 720 Td 12 TL\n"); + for (const char *s = pages[i]; *s;) { + const char *e = strchr(s, '\n'); + size_t k = e ? (size_t)(e - s) : strlen(s); + tp_puts(&c, "("); + tp_put(&c, s, k); + tp_puts(&c, ") Tj T*\n"); + s += k + (e ? 1 : 0); + } + tp_puts(&c, "ET"); + bufs[2 * i + 1] = (char *)c.p; + o[4 + 2 * i].dict = ""; + o[4 + 2 * i].data = c.p; + o[4 + 2 * i].len = c.n; + } + size_t len; + unsigned char *pdf = tp_pdf(o, n, NULL, &len); + char dir[512]; + snprintf(dir, sizeof(dir), "%s", path); + char *slash = strrchr(dir, '/'); + if (slash) { + *slash = '\0'; + cbm_mkdir_p(dir, 0755); + } + FILE *f = fopen(path, "wb"); + fwrite(pdf, 1, len, f); + fclose(f); + free(pdf); + free(kids.p); + for (int i = 0; i < npages * 2; i++) { + free(bufs[i]); + } + free(bufs); + free(o); +} + +static const char *const DESIGN_P1 = + "Design: pkg/server/handler.go serves via server.Handler.Serve\n" + "and app.models.User.save stores users; see\n" + "tests/test_api.py and config/settings.json.\n" + "Gone: pkg/server/gone.go and other/thing.go."; +static const char *const DESIGN_P2 = + "Factory server.NewHandler and models.py build it; app.models holds it."; + +static void tp_write_repo(const char *repo, const char *p2) { + th_write_file(TH_PATH(repo, "pkg/server/handler.go"), + "package server\n" + "\n" + "type Handler struct{}\n" + "\n" + "func (h *Handler) Serve() {}\n" + "\n" + "func NewHandler() *Handler { return nil }\n"); + th_write_file(TH_PATH(repo, "app/__init__.py"), ""); + th_write_file(TH_PATH(repo, "app/models.py"), "class User:\n" + " def save(self):\n" + " return None\n"); + th_write_file(TH_PATH(repo, "tests/test_api.py"), "def test_x():\n" + " return None\n"); + th_write_file(TH_PATH(repo, "config/settings.json"), "{\"a\": 1}\n"); + char path[1024]; + snprintf(path, sizeof(path), "%s/docs/design.pdf", repo); + const char *pages[2] = {DESIGN_P1, p2}; + tp_write_pdf(path, pages, 2); + snprintf(path, sizeof(path), "%s/tests/fixtures/sample.pdf", repo); + const char *fx[1] = {"Fixture names pkg/server/handler.go too."}; + tp_write_pdf(path, fx, 1); +} + +TEST(pdf_links_pipeline) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdf_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + tp_write_repo(repo, DESIGN_P2); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* one Section per page, the page's text as its docstring */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND " + "file_path='docs/design.pdf'"), + 2); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND name='page 1' " + "AND file_path='docs/design.pdf' AND properties LIKE " + "'%pkg/server/handler.go serves%'"), + 1); + char props[512]; + int n; + /* path-exact */ + dm_edge(db, "docs.design.page_1", "pkg.server.handler.go.__file__", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"via\":\"pdf\"")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_path\"")); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":1")); + /* a Go method through its receiver (its QN has none) */ + dm_edge(db, "docs.design.page_1", "pkg.server.Serve", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"unique\"")); + /* qn-exact */ + dm_edge(db, "docs.design.page_1", "app.models.User.save", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":2")); + /* page 2: a Go function, a file name */ + dm_edge(db, "docs.design.page_2", "pkg.server.NewHandler", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + dm_edge(db, "docs.design.page_2", "app.models.py.__file__", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_file\"")); + /* a module by its qualified name (a Module node is named by its path) */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.label='Module' AND t.file_path='app/models.py' AND s.name='page 2' " + "AND e.properties LIKE '%\"tier\":\"exact\"%'"), + 1); + /* hygiene: a test target is a row, a data file nothing; a fixture PDF links nothing */ + char reason[64]; + dm_row(db, "docs/design.pdf", "tests/test_api.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "test_only_target"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path='config/settings.json'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "raw='config/settings.json'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id WHERE " + "e.type='MENTIONS' AND s.file_path='tests/fixtures/sample.pdf'"), + 0); + /* unresolved: a missing file under a known directory; another repo's path is no row */ + dm_row(db, "docs/design.pdf", "pkg/server/gone.go", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw='other/thing.go'"), + 0); + /* back navigation: the method knows the page that names it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND t.name='Serve' " + "AND s.name='page 1'"), + 1); + th_cleanup(tmp); + PASS(); +} + +TEST(pdf_snippet_is_page_text) { + char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + char cache[400]; + char db[1024]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(cache, sizeof(cache), "%s/cache", tmp); + tp_write_repo(repo, DESIGN_P2); + cbm_mkdir_p(cache, 0700); + char *project = cbm_project_name_from_path(repo); + ASSERT_NOT_NULL(project); + const char *saved = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved ? strdup(saved) : NULL; + snprintf(db, sizeof(db), "%s/%s.db", cache, project); + cbm_setenv("CBM_CACHE_DIR", cache, 1); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + char args[1024]; + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"qualified_name\":\"%s.docs.design.page_2\"}", project, project); + char *resp = cbm_mcp_handle_tool(srv, "get_code_snippet", args); + char *text = dm_tool_text(resp); + ASSERT_NOT_NULL(text); + ASSERT_NOT_NULL( + strstr(text, "Factory server.NewHandler and models.py build it; app.models holds it.")); + ASSERT_NULL(strstr(text, "%PDF")); + free(text); + free(resp); + cbm_mcp_server_free(srv); + if (saved_copy) { + cbm_setenv("CBM_CACHE_DIR", saved_copy, 1); + } else { + cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_copy); + free(project); + th_cleanup(tmp); + PASS(); +} + +TEST(pdf_links_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdf_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + tp_write_repo(repo, DESIGN_P2); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a body edit of a linked code file: the document re-resolves */ + th_write_file(TH_PATH(repo, "app/models.py"), "class User:\n" + " def save(self):\n" + " return 1\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "code body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + /* the document changes while the Go file does not: its method binds + * through the owner a proxy carries */ + char path[1024]; + snprintf(path, sizeof(path), "%s/docs/design.pdf", repo); + const char *pages[2] = {DESIGN_P1, "Now only server.Handler.Serve and pkg/server/new.go."}; + tp_write_pdf(path, pages, 2); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* the missing file appears: the row becomes an edge, as in a full index */ + th_write_file(TH_PATH(repo, "pkg/server/gone.go"), "package server\n" + "\n" + "func Gone() {}\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "missing file added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_pdf) { + RUN_TEST(pdf_extract_layout); + RUN_TEST(pdf_extract_fonts); + RUN_TEST(pdf_extract_structure); + RUN_TEST(pdf_extract_filters_and_hostile); + RUN_TEST(pdf_scan_mentions); + RUN_TEST(pdf_links_pipeline); + RUN_TEST(pdf_snippet_is_page_text); + RUN_TEST(pdf_links_incremental); +} diff --git a/tests/test_main.c b/tests/test_main.c index a4fa4a154f..cad85c4bac 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1822,6 +1822,7 @@ extern void suite_doc_mentions(void); extern void suite_doc_mentions_cost(void); extern void suite_doc_mentions_msbuild(void); extern void suite_doc_links_md(void); +extern void suite_doc_links_pdf(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2290,6 +2291,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_cost); RUN_SELECTED_SUITE(doc_mentions_msbuild); RUN_SELECTED_SUITE(doc_links_md); + RUN_SELECTED_SUITE(doc_links_pdf); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From 1b444c6b61c80d52cf13cb38b4c9d831f096d789 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 00:03:42 +0200 Subject: [PATCH 04/25] doc-links(rst): reStructuredText sections linked to the code Sphinx markup names A reST document now gets a Section per title (the text up to the next title as its docstring, its lines from the title to the next one) and MENTIONS edges, via "rst", from each section to the code it names: role :class:`pkg.Model`, :meth:`~x.Y.z`, :func:`.f`, :c:func:`f`, and extlink roles whose URL is a repository blob path (:source:`django/db/models/base.py`) object .. py:class:: / .. method:: / .. function:: ... and their C-domain twins autodoc .. automodule:: / .. autoclass:: ... include, .. include:: / .. kernel-include:: of a code file, literalinclude .. literalinclude:: (its :lines: bind the innermost definition holding them, kept as target_lines; its :pyobject: the named definition) kernel_doc .. kernel-doc:: path and its :identifiers: / :functions: code_path, inline literals naming a path or a qualified name (the code_name Markdown code-span rules) Structure is the field test's line model (H3): titles by adornment style, literal blocks after `::` and in code directives, comments, directive arguments and options, Sphinx's module / currentmodule / class contexts, CPython's str whitespace and \w. Text before the first title belongs to the file. A Sphinx documentation set's conf.py is read as text, never run: its source_suffix makes discovery index e.g. Django's `.txt` documents as reST in that set only, its primary_domain decides undomained roles, its extlinks and intersphinx names are known to the resolver. Python names resolve the way Sphinx imports them: the longest prefix that is a module's import name (named from its topmost package directory), the rest by qualified name, a package's re-exports (`from .base import Model` in its __init__.py) followed. The re-exports and the conf.py settings travel as Python scope blobs (doclink_py.c), so an incremental run resolves as a full one; one resolver serves reST and Python. Forms in Sphinx's order (T, C.T, M.T, M.C.T; a leading `.` reverses it). EXACT when a form names a node, otherwise UNIQUE by name with the field test's fixes: a bare member (:meth:`save`, :attr:`x`) is never searched (37 of 40 such results were wrong), an object directive's target lies in its declared module, a module never answers a member's role (`:meth:`dispatch`` is not tests/dispatch), test code is no target of a search but binds when the document writes its name in full (`django.test.TestCase`). C identifiers are whole names: the one C/C++ definition of that name and kind. Unresolved: missing when the name or path is the repository's, ambiguous, test_only_target; another project's name (stdlib, intersphinx) is no row. reST architecture decision records: the ADR detector reads reST records like Markdown ones (titles underlined with = or -): course-discovery's 34 decision records become ADR nodes. Proof on Django's documentation (the field test's corpus and commit), C against the frozen H3 harvester over the same graph, per (document, section, target): 6,695 of H3's 7,238 links identical; H3-only 543 = links C attributes to the document because a section title repeats in it (the Markdown rule; the same document -> code link exists), deliberate refusals (bare members, directive searches without a module) and H3's own audited error class (router protocol methods -> ConnectionRouter, forms Field -> model Field); C-only = the same attribution mirrored, plus 1,053 inline literal links (sample of 30: 30 correct). 6,574 reST sections; 12 s for the whole index. Tests: suite doc_links_rst (structure and tokens, the scope blobs, the pipeline's rules, source_suffix in discovery, reST ADRs, incremental == full). Revert check: without the module-kind guard and without the source_suffix mapping exactly their assertions fail (observed). Two expectations of the old behaviour now name the new one: the reST grammar golden and probe have the document's Sections (they recorded "reST titles are no Sections", the gap this closes). Signed-off-by: Martin Vogel --- Makefile.cbm | 5 +- internal/cbm/doc_adr.c | 9 +- internal/cbm/doclink.c | 13 +- internal/cbm/doclink.h | 68 +- internal/cbm/doclink_py.c | 419 ++++++++ internal/cbm/doclink_rst.c | 1791 ++++++++++++++++++++++++++++++++++ src/discover/discover.c | 163 +++- src/pipeline/doc_links.c | 1 + src/pipeline/doc_links.h | 12 + src/pipeline/doc_links_md.c | 15 + src/pipeline/doc_links_rst.c | 1713 ++++++++++++++++++++++++++++++++ tests/test_doc_links_rst.c | 440 +++++++++ tests/test_grammar_labels.c | 2 +- tests/test_grammar_probe_g.c | 15 +- tests/test_main.c | 2 + 15 files changed, 4632 insertions(+), 36 deletions(-) create mode 100644 internal/cbm/doclink_py.c create mode 100644 internal/cbm/doclink_rst.c create mode 100644 src/pipeline/doc_links_rst.c create mode 100644 tests/test_doc_links_rst.c diff --git a/Makefile.cbm b/Makefile.cbm index 0c9eca047a..9e1a7ac94c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -376,6 +376,8 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink.c \ $(CBM_DIR)/doclink_cs.c \ $(CBM_DIR)/doclink_md.c \ + $(CBM_DIR)/doclink_rst.c \ + $(CBM_DIR)/doclink_py.c \ $(CBM_DIR)/doc_adr.c \ $(CBM_DIR)/doc_pdf.c \ $(CBM_DIR)/pdf/pdf_obj.c \ @@ -507,6 +509,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_cs.c \ src/pipeline/doc_links_md.c \ src/pipeline/doc_links_pdf.c \ + src/pipeline/doc_links_rst.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -769,7 +772,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doc_adr.c b/internal/cbm/doc_adr.c index 6913e340dd..68ac3826a6 100644 --- a/internal/cbm/doc_adr.c +++ b/internal/cbm/doc_adr.c @@ -1151,13 +1151,16 @@ static bool adr_ends_ci(const char *s, const char *suffix) { return n > k && adr_eq_ci(s + n - k, k, suffix); } -static bool adr_markdown_path(const char *rel) { - return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown"); +/* A Markdown or reStructuredText record (reST titles underlined with `=` or + * `-` read like setext headings). */ +static bool adr_document_path(const char *rel) { + return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown") || + adr_ends_ci(rel, ".rst"); } void cbm_adr_extract(CBMExtractCtx *ctx) { if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0 || !ctx->rel_path || - !adr_markdown_path(ctx->rel_path)) { + !adr_document_path(ctx->rel_path)) { return; } const char *rel = ctx->rel_path; diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index a67364e34e..1a3d06b3c4 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -169,6 +169,12 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scope_tag = CBM_DOCLINK_CS_SCOPE_TAG, .portable_scope = cbm_doclink_cs_portable_scope}, {.lang = CBM_LANG_MARKDOWN, .scan_file = cbm_doclink_md_scan_file}, + {.lang = CBM_LANG_RST, .scan_file = cbm_doclink_rst_scan_file}, + /* Python: no doc references yet, only the scope the reST resolver reads + * (package re-exports, Sphinx conf.py settings) */ + {.lang = CBM_LANG_PYTHON, + .scan_scope = cbm_doclink_py_scan_scope, + .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { @@ -181,7 +187,8 @@ static const doclink_lang_t *doclink_lang(CBMLanguage lang) { } bool cbm_doclink_lang_supported(CBMLanguage lang) { - return doclink_lang(lang) != NULL; + const doclink_lang_t *L = doclink_lang(lang); + return L && (L->parse_doc || L->scan_file); } /* ── Doc-line side map ───────────────────────────────────────────── */ @@ -402,6 +409,10 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) { L->scan_file(ctx); return; } + if (!L->parse_doc) { /* a scope, and no references (Python) */ + ctx->result->doc_scope = L->scan_scope ? L->scan_scope(ctx) : NULL; + return; + } int twin_count = 0; doclink_twin_key_t *twins = L->twin_label ? twin_keys(ctx, L->twin_of, &twin_count) : NULL; CBMDefArray *defs = &ctx->result->defs; diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index ecdae767cd..741b95aaaa 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -64,26 +64,35 @@ * * The enum below and the family table (doclink.c) are both generated from this * list, so a value cannot exist without its name and its gate. */ -#define CBM_DOCLINK_FAMILY_LIST(X) \ - /* csharp */ \ - X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ - /* markdown (doclink_md.c): what a document's text names explicitly */ \ - X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ - /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ - X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ - /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ - /* names) only decides which line fragments a cross-line join replaces */ \ - X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ - /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ +#define CBM_DOCLINK_FAMILY_LIST(X) \ + /* csharp */ \ + X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ + /* markdown (doclink_md.c): what a document's text names explicitly */ \ + X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ + /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ + X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ + /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ + /* names) only decides which line fragments a cross-line join replaces */ \ + X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ + /* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \ + X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \ + /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") typedef enum { @@ -305,6 +314,25 @@ void cbm_adr_extract(CBMExtractCtx *ctx); /* The qualified-name tail of an ADR node: "" + this. */ #define CBM_ADR_QN_NAME "__adr__" +/* ── reStructuredText (doclink_rst.c) and Python's scope (doclink_py.c) ── */ + +/* The scan_file hook of a reStructuredText document: a Section per title + * (the text up to the next title as its docstring) and its references to + * code -- Sphinx roles and object directives, autodoc, include / + * literalinclude / kernel-doc paths, inline literals naming a path or a + * qualified name -- each with the section it is written in as its source. */ +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx); + +/* The scope blob of a Python file, which the reST resolver reads: a package + * `__init__.py`'s top-level explicit `from x import y [as z]` (what the + * package re-exports), and a Sphinx `conf.py`'s primary domain, extlink roles + * to repository paths and intersphinx names (read as text, never run). NULL + * for every other file. */ +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx); + +/* The Python scope blob starts with this tag line. */ +#define CBM_DOCLINK_PY_SCOPE_TAG "py1" + /* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's * text as docstring, property page) and the pages' structural code mentions * as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */ diff --git a/internal/cbm/doclink_py.c b/internal/cbm/doclink_py.c new file mode 100644 index 0000000000..d762ab2e35 --- /dev/null +++ b/internal/cbm/doclink_py.c @@ -0,0 +1,419 @@ +/* + * doclink_py.c — the doc-link scope of a Python file: what the reST resolver + * (src/pipeline/doc_links_rst.c) needs from Python sources to resolve Sphinx + * references the way Sphinx does. + * + * I \t level \t module \t name \t alias a package `__init__.py`'s top-level + * `from module import name [as alias]`: + * the package re-exports `alias` + * (`django.db.models.Model` is + * django/db/models/base.py's Model). + * Star imports are not read (the field + * test's resolver did not follow them). + * C the file is a `conf.py`: documents in + * and below its directory are one + * Sphinx documentation set + * D \t domain its primary_domain + * X \t role an extlink role whose URL is a + * repository blob/tree path + * (:source:`django/db/x.py`) + * S \t name an intersphinx name (`python:` in + * :class:`python:dict` is external) + * + * conf.py is read as text, with the field test's patterns, and never run. + * Every other Python file has no scope (NULL), so a change to it never + * changes another file's resolution through this blob. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" + +#include +#include +#include +#include + +enum { + PY_SCOPE_FIELD_MAX = 256, /* a module, name or setting longer than this is skipped */ + PY_SB_INIT = 256, + PY_INT_DIGITS = 16, +}; + +/* A small arena string builder (the blob lives in the result arena). */ +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} py_sb_t; + +static void py_put(py_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + 1 > sb->cap) { + size_t ncap = sb->cap ? sb->cap : PY_SB_INIT; + while (ncap < sb->len + n + 1) { + ncap *= 2; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +/* One record: its kind, then each field after a TAB, then a newline. */ +static void py_rec(py_sb_t *sb, const char *kind, int nfields, const char *const *f, + const size_t *fl) { + py_put(sb, kind, strlen(kind)); + for (int i = 0; i < nfields; i++) { + py_put(sb, "\t", 1); + py_put(sb, f[i], fl[i]); + } + py_put(sb, "\n", 1); +} + +static const char *py_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + 1 : path; +} + +static bool py_field_ok(const char *s, size_t n) { + if (n == 0 || n > PY_SCOPE_FIELD_MAX) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '\t' || c == '\n' || c == '\r' || c < ' ') { + return false; + } + } + return true; +} + +static void py_text(const CBMExtractCtx *ctx, TSNode n, const char **s, size_t *len) { + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (b > (uint32_t)ctx->source_len || a > b) { + *s = ""; + *len = 0; + return; + } + *s = ctx->source + a; + *len = b - a; +} + +/* One `from module import names` statement of the package's top level. */ +static void py_from_import(const CBMExtractCtx *ctx, TSNode stmt, py_sb_t *sb) { + TSNode mod = ts_node_child_by_field_name(stmt, "module_name", (uint32_t)strlen("module_name")); + if (ts_node_is_null(mod)) { + return; + } + int level = 0; + const char *ms = ""; + size_t ml = 0; + if (strcmp(ts_node_type(mod), "relative_import") == 0) { + uint32_t nc = ts_node_child_count(mod); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(mod, i); + const char *k = ts_node_type(c); + if (strcmp(k, "import_prefix") == 0) { + const char *ps; + size_t pl; + py_text(ctx, c, &ps, &pl); + for (size_t j = 0; j < pl; j++) { + level += ps[j] == '.'; + } + } else if (strcmp(k, "dotted_name") == 0) { + py_text(ctx, c, &ms, &ml); + } + } + } else { + py_text(ctx, mod, &ms, &ml); + } + if (ml > 0 && !py_field_ok(ms, ml)) { + return; + } + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + const char *field = ts_node_field_name_for_child(stmt, i); + if (!field || strcmp(field, "name") != 0) { + continue; + } + TSNode c = ts_node_child(stmt, i); + TSNode name = c; + TSNode alias = c; + if (strcmp(ts_node_type(c), "aliased_import") == 0) { + name = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name")); + alias = ts_node_child_by_field_name(c, "alias", (uint32_t)strlen("alias")); + if (ts_node_is_null(name) || ts_node_is_null(alias)) { + continue; + } + } + const char *ns; + size_t nl; + const char *as; + size_t al; + py_text(ctx, name, &ns, &nl); + py_text(ctx, alias, &as, &al); + if (!py_field_ok(ns, nl) || !py_field_ok(as, al)) { + continue; + } + char lv[PY_INT_DIGITS]; + int lvn = snprintf(lv, sizeof(lv), "%d", level); + const char *f[] = {lv, ms, ns, as}; + const size_t fl[] = {lvn > 0 ? (size_t)lvn : 0, ml, nl, al}; + py_rec(sb, "I", 4, f, fl); + } +} + +/* ── conf.py, read as text (the field test's patterns) ───────────── */ + +static bool py_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool py_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +static size_t py_skip_ws(const char *s, size_t n, size_t i) { + while (i < n && py_ws(s[i])) { + i++; + } + return i; +} + +/* `^name\s*=\s*` at a line start: the index after it, or 0. */ +static size_t py_assign_at(const char *s, size_t n, size_t line, const char *name) { + size_t nl = strlen(name); + if (line + nl > n || memcmp(s + line, name, nl) != 0) { + return 0; + } + size_t i = py_skip_ws(s, n, line + nl); + if (i >= n || s[i] != '=') { + return 0; + } + return py_skip_ws(s, n, i + 1); +} + +/* conf_scalar: ^name\s*=\s*["']([^"']*)["'] (first match). */ +static bool py_conf_scalar(const char *s, size_t n, const char *name, const char **v, size_t *vl) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && (s[i] == '"' || s[i] == '\'')) { + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j < n) { + *v = s + i + 1; + *vl = j - i - 1; + return true; + } + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* conf_dict_block: ^name\s*=\s*\{(.*?)^\} -- the text between, or false. */ +static bool py_conf_block(const char *s, size_t n, const char *name, size_t *a, size_t *b) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && s[i] == '{') { + size_t k = i + 1; + while (k < n) { + if (s[k] == '}' && (k == 0 || s[k - 1] == '\n')) { + *a = i + 1; + *b = k; + return true; + } + k++; + } + return false; + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* ["']([\w.-]+)["'] at s[i] (extra: also '.'): its end after the closing quote, or 0. */ +static size_t py_quoted_key(const char *s, size_t n, size_t i, bool dots, size_t *ka, size_t *kb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return 0; + } + size_t j = i + 1; + while (j < n && (py_word(s[j]) || s[j] == '-' || (dots && s[j] == '.'))) { + j++; + } + if (j == i + 1 || j >= n || (s[j] != '"' && s[j] != '\'')) { + return 0; + } + *ka = i + 1; + *kb = j; + return j + 1; +} + +/* ["']([^"']+)["'] at s[i]: the value span, or false. */ +static bool py_quoted_value(const char *s, size_t n, size_t i, size_t *va, size_t *vb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return false; + } + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j == i + 1 || j >= n) { + return false; + } + *va = i + 1; + *vb = j; + return true; +} + +/* /(blob|tree)/[^/]+/%s in the URL: a repository path pattern. */ +static bool py_repo_url(const char *u, size_t n) { + static const char *const kinds[] = {"/blob/", "/tree/", NULL}; + for (int k = 0; kinds[k]; k++) { + size_t kl = strlen(kinds[k]); + for (size_t i = 0; i + kl <= n; i++) { + if (memcmp(u + i, kinds[k], kl) != 0) { + continue; + } + size_t j = i + kl; + size_t ref = j; + while (j < n && u[j] != '/') { + j++; + } + if (j > ref && j + 3 <= n && memcmp(u + j, "/%s", 3) == 0) { + return true; + } + } + } + return false; +} + +static void py_extlink(py_sb_t *sb, const char *s, size_t ka, size_t kb, size_t va, size_t vb) { + if (py_repo_url(s + va, vb - va) && py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "X", 1, f, fl); + } +} + +static void py_conf(const char *s, size_t n, py_sb_t *sb) { + py_rec(sb, "C", 0, NULL, NULL); + const char *v; + size_t vl; + if (py_conf_scalar(s, n, "primary_domain", &v, &vl) && py_field_ok(v, vl)) { + const char *f[] = {v}; + const size_t fl[] = {vl}; + py_rec(sb, "D", 1, f, fl); + } + size_t a; + size_t b; + if (py_conf_block(s, n, "intersphinx_mapping", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, true, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "S", 1, f, fl); + } + i = e - 1; + } + } + /* extlinks = {'role': ('url%s', ...)} and extlinks['role'] = ('url%s', ...) */ + if (py_conf_block(s, n, "extlinks", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, false, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + size_t va; + size_t vb; + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_quoted_value(s, b, py_skip_ws(s, b, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + i = e - 1; + } + } + static const char key_open[] = "extlinks["; + size_t kol = strlen(key_open); + for (size_t at = 0; at + kol <= n; at++) { + if (memcmp(s + at, key_open, kol) != 0) { + continue; + } + size_t i = at + kol; + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, n, i, false, &ka, &kb); + if (e == 0 || e >= n || s[e] != ']') { + continue; + } + size_t c = py_skip_ws(s, n, e + 1); + size_t va; + size_t vb; + if (c < n && s[c] == '=' && (c = py_skip_ws(s, n, c + 1)) < n && s[c] == '(' && + py_quoted_value(s, n, py_skip_ws(s, n, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + } +} + +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->rel_path || !ctx->source) { + return NULL; + } + const char *base = py_base(ctx->rel_path); + bool init = strcmp(base, "__init__.py") == 0; + bool conf = strcmp(base, "conf.py") == 0; + if (!init && !conf) { + return NULL; + } + py_sb_t sb = {.a = ctx->arena}; + py_put(&sb, CBM_DOCLINK_PY_SCOPE_TAG "\n", strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n")); + if (conf) { + py_conf(ctx->source, (size_t)ctx->source_len, &sb); + } + if (init && !ts_node_is_null(ctx->root)) { + uint32_t nc = ts_node_child_count(ctx->root); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(ctx->root, i); + if (strcmp(ts_node_type(c), "import_from_statement") == 0) { + py_from_import(ctx, c, &sb); + } + } + } + if (sb.failed) { + ctx->result->doc_links.failed = true; + return NULL; + } + return sb.buf; +} diff --git a/internal/cbm/doclink_rst.c b/internal/cbm/doclink_rst.c new file mode 100644 index 0000000000..a50498f144 --- /dev/null +++ b/internal/cbm/doclink_rst.c @@ -0,0 +1,1791 @@ +/* + * doclink_rst.c — reStructuredText documents: their sections, and the code + * their text names (the extraction half; src/pipeline/doc_links_rst.c + * resolves). + * + * Structure: the field test's line model (H3 RstDoc) with CPython's str + * semantics -- tabs expanded to 8 columns, str.strip / str.split whitespace + * and re's \s and \w from the Unicode tables the PDF scanner uses. Every line + * gets a kind: title, adornment, prose, directive head / argument / option, + * literal block (after `::` or in a code directive), comment, hyperlink + * target. Titles are found by adornment style (over+under or under only; the + * level is the style's first appearance). Each title becomes a Section + * definition: name the title, docstring the text up to the next title, lines + * from its title to the line before the next one. Text before the first title + * belongs to the file. + * + * References become tokens whose `raw` is the reference AS WRITTEN followed by + * TAB-separated fields the resolver reads (a doc_link_unresolved row shows the + * written form first; whitespace inside a field is collapsed, so no field + * holds a TAB): + * role :py:class:`x`, :func:`~a.b`, :c:type:`t`, :source:`p` + * written, role, content, module, class, C namespace + * object .. py:class:: sig, .. function:: sig (one per signature) + * written, directive, signature, module, class, C namespace, + * the Python full name in its class ("" when Sphinx's + * signature grammar does not read it) + * autodoc .. autoclass:: a.b + * written, directive, argument, module, class + * include .. include:: / .. kernel-include:: of a code file + * written, directive, path + * literalinclude .. literalinclude:: path + * written, path, :lines:, :pyobject:, "1" when a + * :start-after: / :end-before: / :start-at: / :end-at: + * bounds it by text (the resolver binds the file then) + * kernel_doc .. kernel-doc:: path, and each name of :identifiers: / + * :functions: -- written, path, name ("" for the file) + * code_path, an inline literal ``...`` that names a path or a qualified + * code_name name (the Markdown code-span classifier); raw is its text + * A role or directive without a domain is taken for Python's when this file + * sets contexts (`module`, `currentmodule`, class directives: Sphinx's + * default domain); which domain it belongs to is the resolver's call (the + * documentation set's conf.py). Roles of no code domain are tokens only when + * their content is a path (an extlink such as :source:`django/db/x.py`). + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "helpers.h" +#include "pdf/pdf_unicode.h" + +#include +#include +#include +#include + +enum { + RST_TAB = 8, + RST_SHORT_ADORN = 4, /* an underline this long fits any title; a transition */ + RST_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */ + RST_SPAN_MAX = 150, /* an inline literal the span classifier reads (MD_SPAN_MAX) */ + RST_INIT_CAP = 16, + RST_REPLACEMENT = 0xFFFD, + RST_ASCII_END = 0x80, + RST_LEAD2_MIN = 0xC2, /* UTF-8 lead bytes (no overlong two-byte form) */ + RST_LEAD3_MIN = 0xE0, + RST_LEAD3_MAX = 0xEF, + RST_LEAD4_MIN = 0xF0, + RST_LEAD4_MAX = 0xF4, + RST_NEED3 = 2, + RST_NEED4 = 3, + RST_BULLET = 0x2022, + RST_SIG_ARROW = 2, /* "->" */ +}; + +#define RST_SEP '\t' + +typedef enum { + K_BLANK = 0, + K_TITLE, + K_ADORN, + K_DHEAD, + K_DHEAD_PROSE, + K_DARG, + K_DARG_PROSE, + K_DOPT, + K_LITERAL, + K_PLIT, + K_SKIP, + K_TARGET, + K_COMMENT, + K_PROSE, +} rst_kind_t; + +typedef struct { + char *s; /* the line, tabs expanded, NUL-terminated */ + int len; /* bytes */ + int indent; /* leading spaces (H3 indent_of) */ + bool blank; /* nothing but whitespace */ +} rst_line_t; + +typedef struct { + const char *key; + const char *val; +} rst_opt_t; + +typedef struct { + int line; + int indent; + const char *name; /* as written, lower-cased, with its domain */ + const char *dom; /* "py", "c", "std" or the written domain */ + const char *dname; /* without the domain */ + const char **args; + int nargs; + rst_opt_t *opts; + int nopts; + int cs; /* content lines [cs, ce) */ + int ce; +} rst_dir_t; + +typedef struct { + int first; /* the title block's first line (an overline, or the title) */ + int idx; /* the title text's line */ + int next; /* the line after the title block */ + const char *text; + const char *qn; + bool shared_qn; /* another title of the file has the same QN: the file is the source */ +} rst_title_t; + +typedef struct { + int line; /* 0-based */ + int col; /* byte offset in its line; directives use their indent */ + int syntax; + const char *raw; +} rst_tok_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *scratch; + rst_line_t *L; + int n; + unsigned char *K; + rst_dir_t *dirs; + int ndirs; + int cap_dirs; + rst_title_t *titles; + int ntitles; + int cap_titles; + char styles[2 * RST_INIT_CAP * RST_INIT_CAP]; /* (char, overline) pairs in first-seen order */ + int nstyles; + const char **mod_at; + const char **cls_at; + const char **ns_at; + rst_tok_t *toks; + int ntoks; + int cap_toks; + char *span_buf; + bool failed; +} rst_doc_t; + +/* ── Unicode, as CPython's str and re see it ─────────────────────── */ + +/* The code point at s[i] and its length; a malformed byte is U+FFFD of one + * byte (decode(errors='replace')). */ +static uint32_t rst_cp(const char *s, int len, int i, int *n) { + unsigned char c = (unsigned char)s[i]; + if (c < RST_ASCII_END) { + *n = 1; + return c; + } + int need = 0; /* continuation bytes of a well-formed lead byte */ + if (c >= RST_LEAD4_MIN && c <= RST_LEAD4_MAX) { + need = RST_NEED4; + } else if (c >= RST_LEAD3_MIN && c <= RST_LEAD3_MAX) { + need = RST_NEED3; + } else if (c >= RST_LEAD2_MIN && c < RST_LEAD3_MIN) { + need = 1; + } + if (need == 0 || i + need >= len) { + *n = 1; + return RST_REPLACEMENT; + } + uint32_t cp = c & (0x3F >> need); + for (int k = 1; k <= need; k++) { + unsigned char cc = (unsigned char)s[i + k]; + if ((cc & 0xC0) != 0x80) { + *n = 1; + return RST_REPLACEMENT; + } + cp = (cp << 6) | (cc & 0x3F); + } + static const uint32_t min_cp[] = {0, 0x80, 0x800, 0x10000}; + if (cp < min_cp[need] || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + *n = 1; + return RST_REPLACEMENT; + } + *n = need + 1; + return cp; +} + +static bool rst_space_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return cp == ' ' || (cp >= '\t' && cp <= '\r') || (cp >= 0x1C && cp <= 0x1F); + } + return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, cp); +} + +static bool rst_word_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z') || (cp >= '0' && cp <= '9') || + cp == '_'; + } + return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, cp); +} + +static bool rst_alpha_ascii(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool rst_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +/* Whitespace code points from s[i]: the index after them. */ +static int rst_skip_ws(const char *s, int len, int i) { + while (i < len) { + int n; + if (!rst_space_cp(rst_cp(s, len, i, &n))) { + break; + } + i += n; + } + return i; +} + +/* str.strip(): [*a, *b) without leading and trailing whitespace. */ +static void rst_strip(const char *s, int len, int *a, int *b) { + int i = rst_skip_ws(s, len, 0); + int end = i; + int k = i; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + k += n; + if (!rst_space_cp(cp)) { + end = k; + } + } + *a = i; + *b = end; +} + +/* Code points in s[a, b). */ +static int rst_cps(const char *s, int a, int b) { + int count = 0; + while (a < b) { + int n; + (void)rst_cp(s, b, a, &n); + a += n; + count++; + } + return count; +} + +/* ' '.join(s.split()), into the arena. */ +static char *rst_collapse(CBMArena *a, const char *s, int len) { + char *out = (char *)cbm_arena_alloc(a, (size_t)len + 1); + if (!out) { + return NULL; + } + int w = 0; + int i = 0; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + out[w] = '\0'; + return out; +} + +/* ── Growing arrays in the scratch arena ─────────────────────────── */ + +static void *rst_grow(rst_doc_t *d, void *items, int count, int *cap, size_t size) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * 2 : RST_INIT_CAP; + void *neu = cbm_arena_alloc(d->scratch, (size_t)ncap * size); + if (!neu) { + d->failed = true; + return NULL; + } + if (items && count) { + memcpy(neu, items, (size_t)count * size); + } + *cap = ncap; + return neu; +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +/* text.expandtabs(8).split('\n'): the column restarts after '\r' and '\n'. */ +static bool rst_lines(rst_doc_t *d, const char *src, int n) { + int count = 1; + for (int i = 0; i < n; i++) { + count += src[i] == '\n'; + } + d->L = (rst_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L)); + d->K = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count); + if (!d->L || !d->K) { + return false; + } + memset(d->K, 0, (size_t)count); + int pos = 0; + for (int li = 0; li < count; li++) { + const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); + int end = nl ? (int)(nl - src) : n; + int tabs = 0; + for (int i = pos; i < end; i++) { + tabs += src[i] == '\t'; + } + int cap = (end - pos) + tabs * RST_TAB + 1; + char *out = (char *)cbm_arena_alloc(d->scratch, (size_t)cap); + if (!out) { + return false; + } + int w = 0; + int col = 0; + int i = pos; + while (i < end) { + int cn; + uint32_t cp = rst_cp(src, end, i, &cn); + if (cp == '\t') { + int fill = RST_TAB - (col % RST_TAB); + memset(out + w, ' ', (size_t)fill); + w += fill; + col += fill; + } else { + memcpy(out + w, src + i, (size_t)cn); + w += cn; + col = cp == '\r' ? 0 : col + 1; + } + i += cn; + } + out[w] = '\0'; + rst_line_t *ln = &d->L[li]; + ln->s = out; + ln->len = w; + int sp = 0; + while (sp < w && out[sp] == ' ') { + sp++; + } + ln->indent = sp; + int a; + int b; + rst_strip(out, w, &a, &b); + ln->blank = a >= b; + pos = end + 1; + } + d->n = count; + return true; +} + +/* ── Line patterns (H3's regexes) ────────────────────────────────── */ + +static bool rst_punct(unsigned char c) { + return (c >= '!' && c <= '/') || (c >= ':' && c <= '@') || (c >= '[' && c <= '`') || + (c >= '{' && c <= '~'); +} + +/* ADORN_RE ^([!-/:-@\[-`{-~])\1*\s*$ */ +static bool rst_adorn(const char *s, int len) { + if (len == 0 || !rst_punct((unsigned char)s[0])) { + return false; + } + int i = 1; + while (i < len && s[i] == s[0]) { + i++; + } + return rst_skip_ws(s, len, i) == len; +} + +/* s.rstrip() length. */ +static int rst_rlen(const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + return a >= b ? 0 : b; +} + +static bool rst_style_index(rst_doc_t *d, char ch, bool over, int *level) { + for (int i = 0; i < d->nstyles; i++) { + if (d->styles[2 * i] == ch && d->styles[(2 * i) + 1] == (char)over) { + *level = i; + return true; + } + } + if ((size_t)(2 * (d->nstyles + 1)) > sizeof(d->styles)) { + return false; + } + d->styles[2 * d->nstyles] = ch; + d->styles[(2 * d->nstyles) + 1] = (char)over; + *level = d->nstyles++; + return true; +} + +/* H3 _title_at: the title's line, its style and the line after the block. */ +static bool rst_title_at(const rst_doc_t *d, int i, int *tidx, char *ch, bool *over, int *next) { + const rst_line_t *L = d->L; + int n = d->n; + int sl = rst_rlen(L[i].s, L[i].len); + int sa; + int sb; + rst_strip(L[i].s, sl, &sa, &sb); + if (rst_adorn(L[i].s, sl)) { + if (sb - sa == 2 && L[i].s[sa] == ':' && L[i].s[sa + 1] == ':') { + return false; + } + if (i + 2 < n) { + const rst_line_t *t = &L[i + 1]; + int ul = rst_rlen(L[i + 2].s, L[i + 2].len); + const char *u = L[i + 2].s; + if (!t->blank && !rst_adorn(t->s, t->len) && rst_adorn(u, ul) && u[0] == L[i].s[sa]) { + *tidx = i + 1; + *ch = L[i].s[sa]; + *over = true; + *next = i + 3; + return true; + } + } + return false; + } + if (i + 1 < n && !(sl >= 2 && L[i].s[0] == '.' && L[i].s[1] == '.')) { + const char *u = L[i + 1].s; + int ul = rst_rlen(u, L[i + 1].len); + if (ul > 0 && !rst_space_cp((unsigned char)u[0]) && rst_adorn(u, ul)) { + int ua; + int ub; + rst_strip(u, ul, &ua, &ub); + bool dcolon = ub - ua == 2 && u[ua] == ':' && u[ua + 1] == ':'; + int ulen = ub - ua; /* ASCII punctuation: bytes are code points */ + if (!dcolon && (ulen >= rst_cps(L[i].s, sa, sb) || ulen >= RST_SHORT_ADORN)) { + *tidx = i; + *ch = u[ua]; + *over = false; + *next = i + 2; + return true; + } + } + } + return false; +} + +static void rst_mark(rst_doc_t *d, int a, int b, rst_kind_t kind) { + for (int k = a; k < b; k++) { + d->K[k] = (unsigned char)(d->L[k].blank ? K_BLANK : kind); + } +} + +static int rst_block_end(const rst_doc_t *d, int j, int ind) { + int k = j; + while (k < d->n && (d->L[k].blank || d->L[k].indent > ind)) { + k++; + } + return k; +} + +static bool rst_name_char(const char *s, int len, int i, int *n) { + uint32_t cp = rst_cp(s, len, i, n); + return cp == ':' || cp == '.' || cp == '+' || cp == '-' || rst_word_cp(cp); +} + +/* DIRECTIVE_RE: ^(\s*)\.\.\s+(?:\|[^|]+\|\s+)?([A-Za-z0-9][\w:.+-]*?)::(?:\s+(.*?))?\s*$ + * Fills the indent (code points), the name span and the first argument + * span (stripped). */ +static bool rst_directive_match(const char *s, int len, int *ind, int *na, int *nb, int *aa, + int *ab) { + int p = rst_skip_ws(s, len, 0); + *ind = rst_cps(s, 0, p); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + int q = rst_skip_ws(s, len, p + 2); + if (q == p + 2) { + return false; + } + p = q; + if (p < len && s[p] == '|') { + const char *bar = memchr(s + p + 1, '|', (size_t)(len - p - 1)); + if (bar && bar > s + p + 1) { + int after = (int)(bar - s) + 1; + int w = rst_skip_ws(s, len, after); + if (w > after) { + p = w; + } + } + } + if (p >= len || !(rst_alpha_ascii((unsigned char)s[p]) || rst_digit((unsigned char)s[p]))) { + return false; + } + int e = p + 1; + for (;;) { + if (e + 2 <= len && s[e] == ':' && s[e + 1] == ':') { + int t = e + 2; + int w = rst_skip_ws(s, len, t); + if (t == len || w > t) { + *na = p; + *nb = e; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *aa = w + a; + *ab = w + (b > a ? b : a); + return true; + } + } + int cn; + if (e >= len || !rst_name_char(s, len, e, &cn)) { + return false; + } + e += cn; + } +} + +/* EXPLICIT_RE ^(\s*)\.\.(?:\s|$) */ +static bool rst_explicit_match(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + if (p + 2 == len) { + return true; + } + int n; + return rst_space_cp(rst_cp(s, len, p + 2, &n)); +} + +/* OPTION_LINE_RE ^(\s*):([A-Za-z][\w-]*):(?:\s+(.*?))?\s*$ */ +static bool rst_option_match(const char *s, int len, int *ka, int *kb, int *va, int *vb) { + int p = rst_skip_ws(s, len, 0); + if (p >= len || s[p] != ':' || p + 1 >= len || !rst_alpha_ascii((unsigned char)s[p + 1])) { + return false; + } + int k = p + 2; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + if (cp != '-' && !rst_word_cp(cp)) { + break; + } + k += n; + } + if (k >= len || s[k] != ':') { + return false; + } + int t = k + 1; + int w = rst_skip_ws(s, len, t); + if (t != len && w == t) { + return false; + } + *ka = p + 1; + *kb = k; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *va = w + a; + *vb = w + (b > a ? b : a); + return true; +} + +static bool rst_roman(unsigned char c) { + return strchr("ivxlcdmIVXLCDM", c) != NULL && c != '\0'; +} + +/* LIST_MARKER_RE: the match length in code points, or -1. */ +static int rst_list_marker(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p >= len) { + return -1; + } + int n; + uint32_t cp = rst_cp(s, len, p, &n); + int q = -1; + if (cp == '-' || cp == '*' || cp == '+' || cp == RST_BULLET) { + q = p + n; + } else { + int r = p + (s[p] == '('); + int e = -1; + if (r < len && rst_digit((unsigned char)s[r])) { + e = r; + while (e < len && rst_digit((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e < 0 && r + 1 < len && rst_alpha_ascii((unsigned char)s[r]) && + (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r + 1 < len && s[r] == '#' && (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r < len && rst_roman((unsigned char)s[r])) { + e = r; + while (e < len && rst_roman((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e >= 0) { + q = e + 1; + } + } + if (q < 0) { + return -1; + } + int w = rst_skip_ws(s, len, q); + if (w == q || w >= len) { + return -1; + } + return rst_cps(s, 0, w); +} + +/* ── Block structure (H3 _scan) ──────────────────────────────────── */ + +static const char *const PY_DOMAIN_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", "currentmodule", NULL}; +static const char *const LITERAL_CONTENT[] = { + "code-block", "code", "sourcecode", "literalinclude", "raw", "math", + "graphviz", "digraph", "graph", "productionlist", "console", "doctest", + "testcode", "testoutput", "testsetup", "testcleanup", "highlight", "kernel-render", + "prompt", "kernel-include", "include", "kernel-abi", "kernel-feat", "uml", + "ipython", "mermaid", NULL}; +static const char *const SKIP_CONTENT[] = {"toctree", "index", "tabularcolumns", NULL}; +static const char *const ARG_PROSE[] = {"note", "warning", "tip", + "hint", "important", "caution", + "attention", "danger", "error", + "seealso", "admonition", "versionadded", + "versionchanged", "deprecated", "versionremoved", + "rubric", "topic", "sidebar", + "replace", "centered", NULL}; +static const char *const PY_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", NULL}; +static const char *const C_OBJ[] = {"function", "macro", "type", "struct", "union", + "enum", "enumerator", "member", "var", NULL}; +static const char *const AUTODOC[] = { + "automodule", "autoclass", "autofunction", "automethod", "autoattribute", + "autoexception", "autodata", "autodecorator", "autoproperty", NULL}; +static const char *const PY_ROLES[] = {"func", "class", "meth", "attr", "mod", "exc", + "data", "obj", "const", "deco", NULL}; +static const char *const C_ROLES[] = {"func", "macro", "struct", "union", "enum", "enumerator", + "type", "member", "data", "var", NULL}; +/* include / kernel-include of these is code (H3 CODE_EXT) */ +static const char *const CODE_EXT[] = {".c", ".h", ".s", ".py", ".sh", ".rs", ".pl", + ".awk", ".cpp", ".cc", ".hpp", ".go", ".js", ".ts", + ".lds", ".dts", ".dtsi", ".yaml", ".yml", ".json", ".toml", + ".mk", ".cmake", ".bpf", ".tc", NULL}; + +static bool rst_in(const char *s, const char *const *list) { + for (int i = 0; s && list[i]; i++) { + if (strcmp(s, list[i]) == 0) { + return true; + } + } + return false; +} + +/* H3 split_domain with Python as the primary domain. */ +static void rst_split_domain(rst_doc_t *d, const char *name, const char *const *py_members, + const char **dom, const char **rest) { + const char *colon = strchr(name, ':'); + if (colon) { + *dom = cbm_arena_strndup(d->scratch, name, (size_t)(colon - name)); + *rest = colon + 1; + if (!*dom) { + d->failed = true; + *dom = ""; + } + return; + } + *dom = rst_in(name, py_members) ? "py" : "std"; + *rest = name; +} + +static char *rst_dup_stripped(rst_doc_t *d, const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + char *out = cbm_arena_strndup(d->scratch, s + a, (size_t)(b > a ? b - a : 0)); + if (!out) { + d->failed = true; + } + return out; +} + +static int rst_directive(rst_doc_t *d, int i, int ind, int na, int nb, int aa, int ab) { + rst_line_t *L = d->L; + int n = d->n; + char *name = cbm_arena_strndup(d->scratch, L[i].s + na, (size_t)(nb - na)); + if (!name) { + d->failed = true; + return n; + } + for (char *c = name; *c; c++) { + if (*c >= 'A' && *c <= 'Z') { + *c = (char)(*c + ('a' - 'A')); + } + } + rst_dir_t dir = {.line = i, .indent = ind, .name = name}; + rst_split_domain(d, name, PY_DOMAIN_OBJ, &dir.dom, &dir.dname); + bool prose_arg = rst_in(dir.dname, ARG_PROSE); + d->K[i] = prose_arg ? K_DHEAD_PROSE : K_DHEAD; + int cap_args = 0; + if (ab > aa) { + dir.args = (const char **)rst_grow(d, NULL, 0, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs] = rst_dup_stripped(d, L[i].s + aa, ab - aa); + dir.nargs++; + } + int j = i + 1; + int ka; + int kb; + int va; + int vb; + while (j < n && !L[j].blank && L[j].indent > ind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + dir.args = (const char **)rst_grow(d, dir.args, dir.nargs, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs++] = rst_dup_stripped(d, L[j].s, L[j].len); + d->K[j] = prose_arg ? K_DARG_PROSE : K_DARG; + j++; + } + int cap_opts = 0; + while (j < n && !L[j].blank && L[j].indent > ind && + rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *key = cbm_arena_strndup(d->scratch, L[j].s + ka, (size_t)(kb - ka)); + char *val = cbm_arena_strndup(d->scratch, L[j].s + va, (size_t)(vb - va)); + int oind = L[j].indent; + d->K[j] = K_DOPT; + j++; + while (j < n && !L[j].blank && L[j].indent > oind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *more = rst_dup_stripped(d, L[j].s, L[j].len); + if (val && more) { + val = cbm_arena_sprintf(d->scratch, "%s%s%s", val, val[0] && more[0] ? " " : "", + more); + } + d->K[j] = K_DOPT; + j++; + } + dir.opts = (rst_opt_t *)rst_grow(d, dir.opts, dir.nopts, &cap_opts, sizeof(rst_opt_t)); + if (!key || !val || !dir.opts) { + d->failed = true; + return n; + } + /* options[key] = val: a repeated key keeps its first place, the last value */ + bool found = false; + for (int k = 0; k < dir.nopts; k++) { + if (strcmp(dir.opts[k].key, key) == 0) { + dir.opts[k].val = val; + found = true; + } + } + if (!found) { + dir.opts[dir.nopts++] = (rst_opt_t){key, val}; + } + } + dir.cs = j; + dir.ce = rst_block_end(d, j, ind); + d->dirs = (rst_dir_t *)rst_grow(d, d->dirs, d->ndirs, &d->cap_dirs, sizeof(rst_dir_t)); + if (!d->dirs) { + return n; + } + d->dirs[d->ndirs++] = dir; + if (rst_in(dir.dname, LITERAL_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_LITERAL); + return dir.ce; + } + if (strcmp(dir.dname, "parsed-literal") == 0) { + rst_mark(d, dir.cs, dir.ce, K_PLIT); + return dir.ce; + } + if (rst_in(dir.dname, SKIP_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_SKIP); + return dir.ce; + } + return dir.cs; +} + +static int rst_explicit(rst_doc_t *d, int i, int ind) { + rst_line_t *L = d->L; + int a; + int b; + rst_strip(L[i].s, L[i].len, &a, &b); + const char *s = L[i].s + a; + int sl = b - a; + if ((sl >= 4 && strncmp(s, ".. _", 4) == 0) || (sl >= 3 && strncmp(s, "__ ", 3) == 0)) { + d->K[i] = K_TARGET; + int j = i + 1; + while (j < d->n && !L[j].blank && L[j].indent > ind) { + d->K[j] = K_TARGET; + j++; + } + return j; + } + /* \.\.\s+\[[^\]]+\] : a footnote or citation */ + int p = rst_skip_ws(s, sl, 2); + if (p > 2 && p < sl && s[p] == '[') { + const char *close = memchr(s + p + 1, ']', (size_t)(sl - p - 1)); + if (close && close > s + p + 1) { + d->K[i] = K_PROSE; + return i + 1; + } + } + if (sl == 2 && (i + 1 >= d->n || L[i + 1].blank)) { + d->K[i] = K_COMMENT; + return i + 1; + } + int j = rst_block_end(d, i + 1, ind); + rst_mark(d, i, j, K_COMMENT); + return j; +} + +static void rst_scan(rst_doc_t *d) { + rst_line_t *L = d->L; + int n = d->n; + int i = 0; + int para_indent = 0; + int literal_pending = -1; + while (i < n && !d->failed) { + if (L[i].blank) { + d->K[i] = K_BLANK; + i++; + continue; + } + bool prev_blank = i == 0 || L[i - 1].blank; + int ind = L[i].indent; + if (literal_pending >= 0) { + int lp = literal_pending; + literal_pending = -1; + if (prev_blank && ind > lp) { + int j = rst_block_end(d, i, lp); + rst_mark(d, i, j, K_LITERAL); + i = j; + continue; + } + } + if (ind == 0 && prev_blank) { + int tidx; + char ch; + bool over; + int next; + if (rst_title_at(d, i, &tidx, &ch, &over, &next)) { + int level; + d->titles = (rst_title_t *)rst_grow(d, d->titles, d->ntitles, &d->cap_titles, + sizeof(rst_title_t)); + char *text = rst_collapse(d->scratch, L[tidx].s, L[tidx].len); + if (!d->titles || !text || !rst_style_index(d, ch, over, &level)) { + d->failed = true; + return; + } + (void)level; + d->titles[d->ntitles++] = + (rst_title_t){.first = i, .idx = tidx, .next = next, .text = text}; + rst_mark(d, i, next, K_ADORN); + d->K[tidx] = K_TITLE; + i = next; + continue; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (rst_adorn(L[i].s, L[i].len) && rst_cps(L[i].s, sa, sb) >= RST_SHORT_ADORN && + (i + 1 >= n || L[i + 1].blank)) { + d->K[i] = K_ADORN; + i++; + continue; + } + } + int dind; + int na; + int nb; + int aa; + int ab; + if (rst_directive_match(L[i].s, L[i].len, &dind, &na, &nb, &aa, &ab)) { + i = rst_directive(d, i, dind, na, nb, aa, ab); + continue; + } + if (rst_explicit_match(L[i].s, L[i].len)) { + i = rst_explicit(d, i, ind); + continue; + } + d->K[i] = K_PROSE; + if (prev_blank) { + int mm = rst_list_marker(L[i].s, L[i].len); + para_indent = mm >= 0 ? mm : ind; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (sb - sa >= 2 && L[i].s[sb - 1] == ':' && L[i].s[sb - 2] == ':') { + literal_pending = para_indent; + } + i++; + } +} + +/* ── Python signatures (Sphinx py_sig_re, H3 join_sig_lines / py_fullname) ── */ + +/* One signature per argument line; a trailing backslash continues it. */ +static int rst_join_sigs(rst_doc_t *d, const rst_dir_t *dir, const char ***out) { + const char **sigs = + (const char **)cbm_arena_alloc(d->scratch, (size_t)(dir->nargs + 1) * sizeof(char *)); + if (!sigs) { + d->failed = true; + return 0; + } + int ns = 0; + const char *cur = ""; + bool open = false; + for (int k = 0; k < dir->nargs; k++) { + const char *a = dir->args[k]; + size_t al = strlen(a); + if (al > 0 && a[al - 1] == '\\') { + cur = cbm_arena_sprintf(d->scratch, "%s%.*s ", cur, (int)(al - 1), a); + open = true; + } else { + sigs[ns++] = cbm_arena_sprintf(d->scratch, "%s%s", cur, a); + cur = ""; + open = false; + } + if (!cur || (ns > 0 && !sigs[ns - 1])) { + d->failed = true; + return 0; + } + } + if (open) { + sigs[ns++] = cur; + } + *out = sigs; + return ns; +} + +/* After the name: \s* (\[ .* \])? \s* ( \( .* \) (\s* -> .*)? )? $ */ +static bool rst_sig_paren_ok(const char *s, int len, int i) { + if (i == len) { + return true; + } + if (s[i] != '(') { + return false; + } + for (int q = len - 1; q > i; q--) { + if (s[q] != ')') { + continue; + } + if (q + 1 == len) { + return true; + } + int w = rst_skip_ws(s, len, q + 1); + if (w + RST_SIG_ARROW <= len && s[w] == '-' && s[w + 1] == '>') { + return true; + } + } + return false; +} + +static bool rst_sig_rest_ok(const char *s, int len, int i) { + i = rst_skip_ws(s, len, i); + if (i == len) { + return true; + } + if (s[i] == '[') { + for (int q = len - 1; q > i; q--) { + if (s[q] == ']' && rst_sig_paren_ok(s, len, rst_skip_ws(s, len, q + 1))) { + return true; + } + } + return false; + } + return rst_sig_paren_ok(s, len, i); +} + +/* PY_SIG_RE on sig.strip(): the prefix ("a.b." or "") and the name. */ +static bool rst_py_sig(rst_doc_t *d, const char *sig, const char **prefix, const char **name) { + int a; + int b; + rst_strip(sig, (int)strlen(sig), &a, &b); + const char *s = sig + a; + int len = b > a ? b - a : 0; + int r = 0; + int last_dot = -1; + while (r < len) { + int n; + uint32_t cp = rst_cp(s, len, r, &n); + if (cp == '.') { + last_dot = r; + } else if (!rst_word_cp(cp)) { + break; + } + r += n; + } + int ns = last_dot + 1; + if (ns >= r || !rst_sig_rest_ok(s, len, r)) { + return false; + } + *prefix = cbm_arena_strndup(d->scratch, s, (size_t)ns); + *name = cbm_arena_strndup(d->scratch, s + ns, (size_t)(r - ns)); + if (!*prefix || !*name) { + d->failed = true; + return false; + } + return true; +} + +/* Sphinx PyObject.handle_signature's fullname, and the remaining prefix. */ +static void rst_py_fullname(rst_doc_t *d, const char *prefix, const char *name, const char *cls, + const char **full, const char **rem) { + if (cls && cls[0]) { + size_t cl = strlen(cls); + if (prefix[0] && + (strcmp(prefix, cls) == 0 || (strncmp(prefix, cls, cl) == 0 && prefix[cl] == '.'))) { + const char *r = prefix + cl; + while (*r == '.') { + r++; + } + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = r; + } else if (prefix[0]) { + *full = cbm_arena_sprintf(d->scratch, "%s.%s%s", cls, prefix, name); + *rem = prefix; + } else { + *full = cbm_arena_sprintf(d->scratch, "%s.%s", cls, name); + *rem = ""; + } + } else { + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = prefix; + } + if (!*full) { + d->failed = true; + *full = ""; + } +} + +static const char *rst_opt(const rst_dir_t *dir, const char *key) { + for (int k = 0; k < dir->nopts; k++) { + if (strcmp(dir->opts[k].key, key) == 0) { + return dir->opts[k].val; + } + } + return NULL; +} + +/* H3 _contexts: the module, C namespace and class in force on every line. */ +static void rst_contexts(rst_doc_t *d) { + int n = d->n; + d->mod_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->ns_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->cls_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + if (!d->mod_at || !d->ns_at || !d->cls_at) { + d->failed = true; + return; + } + memset(d->cls_at, 0, (size_t)n * sizeof(char *)); + const char *cm = NULL; + const char *cn = NULL; + int k = 0; + /* directives are in line order (the scan appends them so) */ + for (int di = 0; di <= d->ndirs; di++) { + int upto = di < d->ndirs ? d->dirs[di].line : n; + for (; k < upto; k++) { + d->mod_at[k] = cm; + d->ns_at[k] = cn; + } + if (di == d->ndirs) { + break; + } + const rst_dir_t *dir = &d->dirs[di]; + const char *v = dir->nargs ? dir->args[0] : NULL; + if (strcmp(dir->dom, "py") == 0 && + (strcmp(dir->dname, "module") == 0 || strcmp(dir->dname, "currentmodule") == 0)) { + cm = (!v || !v[0] || strcmp(v, "None") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && + (strcmp(dir->dname, "namespace") == 0 || strcmp(dir->dname, "namespace-push") == 0)) { + cn = (!v || !v[0] || strcmp(v, "NULL") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && strcmp(dir->dname, "namespace-pop") == 0) { + cn = NULL; + } + } + for (int di = 0; di < d->ndirs && !d->failed; di++) { + const rst_dir_t *dir = &d->dirs[di]; + if (strcmp(dir->dom, "py") != 0 || !rst_in(dir->dname, PY_OBJ) || + strcmp(dir->dname, "module") == 0 || dir->cs >= dir->ce) { + continue; + } + const char *outer = d->cls_at[dir->line]; + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + const char *content_cls = outer; + const char *prefix; + const char *name; + if (ns > 0 && rst_py_sig(d, sigs[ns - 1], &prefix, &name)) { + const char *full; + const char *rem; + rst_py_fullname(d, prefix, name, outer, &full, &rem); + if (strcmp(dir->dname, "class") == 0 || strcmp(dir->dname, "exception") == 0) { + content_cls = full; + } else if (rem[0]) { + size_t rl = strlen(rem); + const char *r = rem; + while (*r == '.') { + r++; + rl--; + } + while (rl > 0 && r[rl - 1] == '.') { + rl--; + } + content_cls = cbm_arena_strndup(d->scratch, r, rl); + } + } + const char *mod = rst_opt(dir, "module"); + if (mod) { + for (int j = dir->cs; j < dir->ce; j++) { + d->mod_at[j] = mod; + } + } + for (int j = dir->cs; j < dir->ce; j++) { + d->cls_at[j] = content_cls; + } + } +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +static void rst_tok(rst_doc_t *d, int line, int col, int syntax, const char *raw) { + if (!raw) { + d->failed = true; + return; + } + d->toks = (rst_tok_t *)rst_grow(d, d->toks, d->ntoks, &d->cap_toks, sizeof(rst_tok_t)); + if (!d->toks) { + return; + } + d->toks[d->ntoks++] = (rst_tok_t){.line = line, .col = col, .syntax = syntax, .raw = raw}; +} + +static const char *rst_s(const char *s) { + return s ? s : ""; +} + +/* The written form of a directive: its head line, whitespace collapsed. */ +static const char *rst_written_dir(rst_doc_t *d, const rst_dir_t *dir) { + const rst_line_t *ln = &d->L[dir->line]; + return rst_collapse(d->ctx->arena, ln->s, ln->len); +} + +static bool rst_code_ext(const char *path) { + const char *base = strrchr(path, '/'); + base = base ? base + 1 : path; + if (strncmp(base, "Makefile", strlen("Makefile")) == 0 || + strncmp(base, "Kconfig", strlen("Kconfig")) == 0) { + return true; + } + const char *dot = strrchr(base, '.'); + if (!dot) { + return false; + } + char ext[16]; + size_t el = strlen(dot); + if (el >= sizeof(ext)) { + return false; + } + for (size_t i = 0; i <= el; i++) { + char c = dot[i]; + ext[i] = (char)(c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c); + } + return rst_in(ext, CODE_EXT); +} + +static void rst_directive_tokens(rst_doc_t *d, const rst_dir_t *dir) { + CBMArena *a = d->ctx->arena; + const char *dn = dir->dname; + const char *dom = dir->dom; + bool py_obj = (strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(dn, PY_OBJ); + bool c_obj = (strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0 || strcmp(dom, "py") == 0) && + rst_in(dn, C_OBJ); + if (strcmp(dom, "py") != 0 && strcmp(dom, "c") != 0 && strcmp(dom, "std") != 0) { + py_obj = false; + c_obj = false; + } + if (strcmp(dom, "c") == 0) { + py_obj = false; + } + int line = dir->line; + const char *written = NULL; + if (py_obj || c_obj) { + written = rst_written_dir(d, dir); + const char *mod = rst_opt(dir, "module"); + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + if (ns == 0 && !d->failed) { + sigs = (const char **)cbm_arena_alloc(d->scratch, sizeof(char *)); + if (!sigs) { + d->failed = true; + return; + } + sigs[0] = ""; + ns = 1; + } + bool attr_like = + strcmp(dn, "attribute") == 0 || strcmp(dn, "data") == 0 || strcmp(dn, "property") == 0; + for (int k = 0; k < ns && !d->failed; k++) { + char *sig = rst_collapse(d->scratch, sigs[k], (int)strlen(sigs[k])); + if (!sig) { + d->failed = true; + return; + } + /* Python's reading (H3 py_directive): an attribute's type or + * value is no part of its name; the full name in the class */ + char *py_sig = sig; + if (attr_like) { + size_t cut = strcspn(sig, ":="); + while (cut > 0 && sig[cut - 1] == ' ') { + cut--; + } + py_sig = cbm_arena_strndup(d->scratch, sig, cut); + } + const char *prefix; + const char *name; + const char *full = ""; + const char *rem; + if (py_sig && strcmp(dn, "module") != 0 && rst_py_sig(d, py_sig, &prefix, &name)) { + rst_py_fullname(d, prefix, name, d->cls_at[line], &full, &rem); + } + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_OBJECT, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s\t%s", rst_s(written), dir->name, + sig, rst_s(mod ? mod : d->mod_at[line]), + rst_s(d->cls_at[line]), rst_s(d->ns_at[line]), full)); + } + return; + } + const char *arg0 = dir->nargs ? dir->args[0] : ""; + if (rst_in(dn, AUTODOC)) { + written = rst_written_dir(d, dir); + char *arg = rst_collapse(d->scratch, arg0, (int)strlen(arg0)); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_AUTODOC, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), dn, rst_s(arg), + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]))); + return; + } + if (strcmp(dn, "literalinclude") == 0) { + written = rst_written_dir(d, dir); + const char *lines = rst_opt(dir, "lines"); + const char *pyobject = rst_opt(dir, "pyobject"); + bool bounded = rst_opt(dir, "start-after") || rst_opt(dir, "end-before") || + rst_opt(dir, "start-at") || rst_opt(dir, "end-at"); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_LITERALINCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), arg0, rst_s(lines), + rst_s(pyobject), bounded ? "1" : "")); + return; + } + if (strcmp(dn, "include") == 0 || strcmp(dn, "kernel-include") == 0) { + if (!arg0[0] || arg0[0] == '<' || !rst_code_ext(arg0)) { + return; /* a document or text include: not code */ + } + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_INCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s", rst_s(written), dn, arg0)); + return; + } + if (strcmp(dn, "kernel-doc") == 0) { + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t", rst_s(written), arg0)); + const char *ids = rst_opt(dir, "identifiers"); + if (!ids) { + ids = rst_opt(dir, "functions"); + } + if (!ids || rst_opt(dir, "export") || rst_opt(dir, "internal") || rst_opt(dir, "doc")) { + return; /* the names come from the source file's own kernel-doc comments */ + } + /* each name, a trailing backslash dropped (kerneldoc.py) */ + const char *p = ids; + while (*p && !d->failed) { + while (*p == ' ') { + p++; + } + const char *e = p; + while (*e && *e != ' ') { + e++; + } + size_t nl = (size_t)(e - p); + while (nl > 0 && p[nl - 1] == '\\') { + nl--; + } + if (nl > 0) { + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t%.*s", rst_s(written), arg0, (int)nl, p)); + } + p = e; + } + } +} + +/* ── Inline text: roles and inline literals ──────────────────────── */ + +static bool rst_inline_kind(unsigned char k) { + return k == K_PROSE || k == K_TITLE || k == K_DHEAD_PROSE || k == K_DARG_PROSE || k == K_PLIT; +} + +typedef struct { + char *text; /* the run's lines joined by '\n' */ + char *mask; /* inline literals blanked (H3 mask) */ + int len; + int a; /* first line */ + int *offs; + int nlines; +} rst_run_t; + +static int rst_run_line(const rst_run_t *r, int pos) { + int lo = 0; + int hi = r->nlines; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (r->offs[mid] <= pos) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo - 1; +} + +/* Is the code point before byte i a role's left neighbour that blocks it + * ((? 0 && ((unsigned char)s[k] & 0xC0) == 0x80) { + k--; + } + int n; + uint32_t cp = rst_cp(s, i, k, &n); + return cp == '`' || cp == '\\' || rst_word_cp(cp); +} + +/* [A-Za-z][\w.+-]* at s[i]: its end, or -1. */ +static int rst_role_name(const char *s, int len, int i) { + if (i >= len || !rst_alpha_ascii((unsigned char)s[i])) { + return -1; + } + i++; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (cp != '.' && cp != '+' && cp != '-' && !rst_word_cp(cp)) { + break; + } + i += n; + } + return i; +} + +static bool rst_role_end_ok(const char *s, int len, int i) { + if (i >= len) { + return true; + } + int n; + uint32_t cp = rst_cp(s, len, i, &n); + return cp != '`' && !rst_word_cp(cp); +} + +static void rst_emit_role(rst_doc_t *d, const rst_run_t *r, int start, int role_a, int role_b, + int ca, int cb) { + CBMArena *a = d->ctx->arena; + char *role = cbm_arena_strndup(d->scratch, r->text + role_a, (size_t)(role_b - role_a)); + if (!role) { + d->failed = true; + return; + } + const char *dom; + const char *rname; + rst_split_domain(d, role, PY_ROLES, &dom, &rname); + bool code = ((strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, PY_ROLES)) || + ((strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, C_ROLES)); + char *content = rst_collapse(d->scratch, r->text + ca, cb - ca); + if (!content) { + d->failed = true; + return; + } + if (!code) { + /* an extlink to a repository path (:source:`django/db/x.py`, or with a + * title, :source:`x `): only a role without a domain + * whose target is a path */ + const char *target = content; + size_t tl = strlen(content); + const char *lt = strrchr(content, '<'); + if (tl > 0 && content[tl - 1] == '>' && lt && lt > content) { + target = lt + 1; + tl = (size_t)(content + tl - 1 - target); + } + CBMDocLinkMdPath p; + if (strchr(role, ':') || + !cbm_doclink_md_classify_span(target, tl, d->span_buf, RST_SPAN_MAX + 1, &p) || + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + return; + } + } + int li = rst_run_line(r, start); + int line = r->a + li; + char *written = rst_collapse(a, r->text + start, cb + 1 - start); + rst_tok(d, line, start - r->offs[li], CBM_DOCLINK_RST_ROLE, + written ? cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s", written, role, content, + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]), + rst_s(d->ns_at[line])) + : NULL); +} + +/* ROLE_RE over the masked text. */ +static void rst_roles(rst_doc_t *d, const rst_run_t *r) { + const char *m = r->mask; + int len = r->len; + int i = 0; + while (i < len && !d->failed) { + const char *colon = memchr(m + i, ':', (size_t)(len - i)); + if (!colon) { + break; + } + i = (int)(colon - m); + if (rst_role_blocked(m, i)) { + i++; + continue; + } + int e1 = rst_role_name(m, len, i + 1); + int role_end = -1; + if (e1 > 0 && e1 < len && m[e1] == ':') { + int e2 = rst_role_name(m, len, e1 + 1); + if (e2 > 0 && e2 + 1 < len && m[e2] == ':' && m[e2 + 1] == '`') { + role_end = e2; + } else if (e1 + 1 < len && m[e1 + 1] == '`') { + role_end = e1; + } + } + if (role_end < 0) { + i++; + continue; + } + int open = role_end + 1; + const char *close = memchr(m + open + 1, '`', (size_t)(len - open - 1)); + if (!close || close == m + open + 1 || !rst_role_end_ok(m, len, (int)(close - m) + 1)) { + i++; + continue; + } + int cb = (int)(close - m); + rst_emit_role(d, r, i, i + 1, role_end, open + 1, cb); + i = cb + 1; + } +} + +static void rst_literal_token(rst_doc_t *d, const rst_run_t *r, int i, int a, int b) { + if (b <= a || b - a > RST_SPAN_MAX || memchr(r->text + a, '\n', (size_t)(b - a))) { + return; + } + CBMDocLinkMdPath p; + if (!cbm_doclink_md_classify_span(r->text + a, (size_t)(b - a), d->span_buf, RST_SPAN_MAX + 1, + &p)) { + return; + } + int li = rst_run_line(r, i); + rst_tok(d, r->a + li, i - r->offs[li], + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_RST_CODE_NAME + : CBM_DOCLINK_RST_CODE_PATH, + cbm_arena_strndup(d->ctx->arena, r->text + a, (size_t)(b - a))); +} + +/* INLINE_LIT_RE ``.+?`` (DOTALL): blank them in the mask, and take each as a + * possible code_path / code_name. */ +static void rst_literals(rst_doc_t *d, rst_run_t *r) { + const char *t = r->text; + int len = r->len; + int i = 0; + while (i + 1 < len && !d->failed) { + if (t[i] != '`' || t[i + 1] != '`') { + i++; + continue; + } + int j = i + 3; + while (j + 1 < len && !(t[j] == '`' && t[j + 1] == '`')) { + j++; + } + if (j + 1 >= len) { + i++; + continue; + } + for (int k = i; k < j + 2; k++) { + if (r->mask[k] != '\n') { + r->mask[k] = ' '; + } + } + rst_literal_token(d, r, i, i + 2, j); + i = j + 2; + } +} + +static void rst_inline(rst_doc_t *d) { + int i = 0; + while (i < d->n && !d->failed) { + if (!rst_inline_kind(d->K[i])) { + i++; + continue; + } + int j = i; + int len = 0; + while (j < d->n && rst_inline_kind(d->K[j])) { + len += d->L[j].len + 1; + j++; + } + rst_run_t r = {.a = i, .nlines = j - i}; + r.text = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.mask = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.offs = (int *)cbm_arena_alloc(d->scratch, (size_t)(j - i) * sizeof(int)); + if (!r.text || !r.mask || !r.offs) { + d->failed = true; + return; + } + int pos = 0; + for (int k = i; k < j; k++) { + r.offs[k - i] = pos; + memcpy(r.text + pos, d->L[k].s, (size_t)d->L[k].len); + pos += d->L[k].len; + if (k + 1 < j) { + r.text[pos++] = '\n'; + } + } + r.text[pos] = '\0'; + r.len = pos; + memcpy(r.mask, r.text, (size_t)pos + 1); + rst_literals(d, &r); + rst_roles(d, &r); + i = j; + } +} + +/* ── Sections ────────────────────────────────────────────────────── */ + +/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */ +static const char *rst_qn_segment(CBMArena *a, const char *name) { + char *out = cbm_arena_strdup(a, name); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = false; + for (const char *p = name; *p; p++) { + if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = '-'; + } + in_ws = false; + out[w++] = *p; + } + out[w] = '\0'; + return out; +} + +/* The section's own text (title block to the next title), whitespace + * collapsed, at most RST_BODY_MAX bytes and never a cut character. */ +static const char *rst_body(rst_doc_t *d, int from, int to) { + char *out = (char *)cbm_arena_alloc(d->ctx->arena, RST_BODY_MAX + 1); + if (!out) { + return NULL; + } + int w = 0; + for (int k = from; k < to && w < RST_BODY_MAX; k++) { + const char *s = d->L[k].s; + int len = d->L[k].len; + int i = 0; + while (i < len && w < RST_BODY_MAX) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + if (w + n > RST_BODY_MAX) { + w = RST_BODY_MAX + 1; /* stop: the next character does not fit */ + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + } + if (w > RST_BODY_MAX) { + w = RST_BODY_MAX; + while (w > 0 && ((unsigned char)out[w] & 0xC0) == 0x80) { + w--; + } + } + while (w > 0 && out[w - 1] == ' ') { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +static int rst_title_qn_cmp(const void *a, const void *b) { + const rst_title_t *x = *(const rst_title_t *const *)a; + const rst_title_t *y = *(const rst_title_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static void rst_sections(rst_doc_t *d) { + CBMExtractCtx *ctx = d->ctx; + CBMArena *a = ctx->arena; + for (int t = 0; t < d->ntitles; t++) { + rst_title_t *ti = &d->titles[t]; + const char *seg = rst_qn_segment(a, ti->text); + ti->qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL; + if (!ti->qn) { + d->failed = true; + return; + } + } + if (d->ntitles > 1) { + rst_title_t **by_qn = + (rst_title_t **)cbm_arena_alloc(d->scratch, (size_t)d->ntitles * sizeof(*by_qn)); + if (!by_qn) { + d->failed = true; + return; + } + for (int t = 0; t < d->ntitles; t++) { + by_qn[t] = &d->titles[t]; + } + qsort(by_qn, (size_t)d->ntitles, sizeof(*by_qn), rst_title_qn_cmp); + for (int t = 1; t < d->ntitles; t++) { + if (strcmp(by_qn[t]->qn, by_qn[t - 1]->qn) == 0) { + by_qn[t]->shared_qn = true; + by_qn[t - 1]->shared_qn = true; + } + } + } + for (int t = 0; t < d->ntitles; t++) { + const rst_title_t *ti = &d->titles[t]; + int end = t + 1 < d->ntitles ? d->titles[t + 1].first : d->n; + int last = end - 1; + while (last > ti->idx && d->L[last].blank) { + last--; + } + char *name = cbm_arena_strdup(a, ti->text); + if (!name) { + d->failed = true; + return; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = name; + def.qualified_name = ti->qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = (uint32_t)ti->idx + 1; + def.end_line = (uint32_t)last + 1; + def.is_exported = true; + def.docstring = rst_body(d, ti->next, end); + cbm_defs_push(&ctx->result->defs, a, def); + } +} + +/* ── Driver ──────────────────────────────────────────────────────── */ + +static int rst_tok_cmp(const void *a, const void *b) { + const rst_tok_t *x = (const rst_tok_t *)a; + const rst_tok_t *y = (const rst_tok_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + if (x->col != y->col) { + return x->col < y->col ? -1 : 1; + } + if (x->syntax != y->syntax) { + return x->syntax < y->syntax ? -1 : 1; + } + return strcmp(x->raw, y->raw); +} + +/* The section a 0-based line is written in (by its title block's first line). */ +static const rst_title_t *rst_section_of(const rst_doc_t *d, int line) { + int lo = 0; + int hi = d->ntitles; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (d->titles[mid].first <= line) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo > 0 ? &d->titles[lo - 1] : NULL; +} + +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + rst_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena}; + d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(RST_SPAN_MAX + 1) * 2); + if (!d.span_buf || !rst_lines(&d, ctx->source, ctx->source_len)) { + ctx->result->doc_links.failed = true; + return; + } + rst_scan(&d); + if (!d.failed) { + rst_contexts(&d); + } + if (!d.failed) { + rst_sections(&d); + } + for (int i = 0; i < d.ndirs && !d.failed; i++) { + rst_directive_tokens(&d, &d.dirs[i]); + } + if (!d.failed) { + rst_inline(&d); + } + if (d.failed) { + ctx->result->doc_links.failed = true; + return; + } + qsort(d.toks, (size_t)d.ntoks, sizeof(*d.toks), rst_tok_cmp); + for (int i = 0; i < d.ntoks; i++) { + const rst_tok_t *t = &d.toks[i]; + const rst_title_t *sec = rst_section_of(&d, t->line); + if (sec && sec->shared_qn) { + sec = NULL; /* the file, not the node another title owns */ + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""), + .raw = t->raw, + .line = (uint32_t)t->line + 1, + .def_line = sec ? (uint32_t)sec->idx + 1 : 1, + .syntax = (uint16_t)t->syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link); + } + /* an architecture decision record in reST: its node, facts and supersedes */ + cbm_adr_extract(ctx); +} diff --git a/src/discover/discover.c b/src/discover/discover.c index d2b02e217d..26a5ab4d48 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -16,6 +16,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/mem_core.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -888,13 +889,159 @@ static int safe_stat(const char *abs_path, struct stat *st, bool *is_symlink) { #endif } +/* ── Sphinx documentation sets ──────────────────────────────────── + * A directory with a conf.py is a Sphinx documentation set, and its + * `source_suffix` says which file suffixes are reStructuredText (Django's + * docs are `.txt`). A file of such a suffix in the set's subtree, of no + * language by its name, is reST; a nested conf.py starts its own set. conf.py + * is read as text (the first SPHINX_CONF_READ bytes) and never run. */ + +enum { + SPHINX_SUFFIXES = 4, /* extra reST suffixes one set declares */ + SPHINX_SUFFIX_LEN = 16, /* ".txt", ".rest": a longer one is no suffix */ + SPHINX_CONF_READ = 65536, +}; + +typedef struct { + char ext[SPHINX_SUFFIXES][SPHINX_SUFFIX_LEN]; + int count; +} sphinx_suffixes_t; + +/* The quoted string at s[i] ('...' or "..."): its span; false otherwise. */ +static bool sphinx_quoted(const char *s, size_t n, size_t i, size_t *a, size_t *b) { + if (i >= n || (s[i] != '\'' && s[i] != '"')) { + return false; + } + const char *close = memchr(s + i + SKIP_ONE, s[i], n - i - SKIP_ONE); + if (!close) { + return false; + } + *a = i + SKIP_ONE; + *b = (size_t)(close - s); + return true; +} + +static void sphinx_add(sphinx_suffixes_t *out, const char *s, size_t n) { + if (n < PAIR_LEN || n >= SPHINX_SUFFIX_LEN || s[0] != '.' || out->count >= SPHINX_SUFFIXES || + (n == strlen(".rst") && memcmp(s, ".rst", n) == 0)) { + return; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!isalnum((unsigned char)s[i]) && s[i] != '_' && s[i] != '-') { + return; + } + } + memcpy(out->ext[out->count], s, n); + out->ext[out->count][n] = '\0'; + out->count++; +} + +/* `source_suffix = '.txt'`, `= ['.rst', '.txt']` or + * `= {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}` (a dict + * entry counts only when it maps to restructuredtext). */ +static void sphinx_parse(const char *s, size_t n, sphinx_suffixes_t *out) { + static const char key[] = "source_suffix"; + size_t kl = strlen(key); + for (size_t line = 0; line < n;) { + const char *nl = memchr(s + line, '\n', n - line); + size_t end = nl ? (size_t)(nl - s) : n; + size_t i = line + kl; + if (end >= i && memcmp(s + line, key, kl) == 0) { + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + if (i < n && s[i] == '=') { + i++; + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + char open = i < n ? s[i] : '\0'; + char close = '\0'; + if (open == '[' || open == '(' || open == '{') { + close = open == '[' ? ']' : (open == '(' ? ')' : '}'); + } + /* the value: up to its closing bracket, or this line */ + size_t stop = end; + if (close) { + const char *c = memchr(s + i, close, n - i); + stop = c ? (size_t)(c - s) : end; + } + size_t k = i; + while (k < stop) { + size_t a; + size_t b; + if (!sphinx_quoted(s, stop, k, &a, &b)) { + k++; + continue; + } + k = b + SKIP_ONE; + if (open != '{') { + sphinx_add(out, s + a, b - a); + continue; + } + while (k < stop && (s[k] == ' ' || s[k] == ':')) { + k++; + } + size_t va; + size_t vb; + if (sphinx_quoted(s, stop, k, &va, &vb)) { + if (vb - va == strlen("restructuredtext") && + memcmp(s + va, "restructuredtext", vb - va) == 0) { + sphinx_add(out, s + a, b - a); + } + k = vb + SKIP_ONE; + } + } + return; /* the first assignment */ + } + } + line = end + SKIP_ONE; + } +} + +/* The documentation set this directory starts, if it has a conf.py. */ +static bool sphinx_load(const char *dir, sphinx_suffixes_t *out) { + char conf[CBM_SZ_4K]; + if (snprintf(conf, sizeof(conf), "%s/conf.py", dir) >= (int)sizeof(conf)) { + return false; + } + struct stat st; + if (wide_stat(conf, &st) != 0 || !S_ISREG(st.st_mode)) { + return false; + } + FILE *f = cbm_fopen(conf, "rb"); + if (!f) { + return false; + } + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, SPHINX_CONF_READ); + size_t n = buf ? fread(buf, SKIP_ONE, SPHINX_CONF_READ, f) : 0; + (void)fclose(f); + memset(out, 0, sizeof(*out)); + if (buf) { + sphinx_parse(buf, n, out); + } + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return true; +} + +static bool sphinx_is_rst(const sphinx_suffixes_t *set, const char *name) { + size_t nl = strlen(name); + for (int i = 0; set && i < set->count; i++) { + size_t el = strlen(set->ext[i]); + if (nl > el && strcmp(name + nl - el, set->ext[i]) == 0) { + return true; + } + } + return false; +} + /* Process a single regular file entry during directory walk. */ static void walk_dir_process_file(const char *abs_path, const char *rel_path, const char *name, const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, const struct stat *st, - file_list_t *out) { + const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx, + const struct stat *st, file_list_t *out) { const char *skip_reason = file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); if (skip_reason) { @@ -906,6 +1053,9 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co return; } CBMLanguage lang = detect_file_language(name, abs_path); + if (lang == CBM_LANG_COUNT && sphinx_is_rst(sphinx, name)) { + lang = CBM_LANG_RST; /* its documentation set's source_suffix */ + } if (lang == CBM_LANG_COUNT) { return; } @@ -916,6 +1066,7 @@ typedef struct { char dir[CBM_SZ_4K]; char prefix[CBM_SZ_4K]; const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */ + sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */ } walk_frame_t; /* Initial capacity only — the stack grows on demand. A single directory can * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855 @@ -973,6 +1124,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char return; } slot->ignore_chain = parent->ignore_chain; + slot->sphinx = parent->sphinx; ws->top++; } @@ -1021,7 +1173,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, &st, out); + cbmignore, &frame->sphinx, &st, out); } } @@ -1115,6 +1267,11 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis frame.ignore_chain = link; } + sphinx_suffixes_t set; + if (sphinx_load(frame.dir, &set)) { + frame.sphinx = set; /* a (nested) documentation set starts here */ + } + cbm_dir_t *d = cbm_opendir(frame.dir); if (!d) { if (out->count_only) { diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 035fbf79a4..f3b8f228d0 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -48,6 +48,7 @@ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, + &cbm_doclink_rst_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index ef09d15e6d..8ad2384d41 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -242,6 +242,18 @@ extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver; extern const cbm_doclink_resolver_t cbm_doclink_md_resolver; /* PDF documents (doc_links_pdf.c): via "pdf". */ extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver; +/* reStructuredText documents and the Python scope they read (doc_links_rst.c): + * via "rst". */ +extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver; + +/* Over a Markdown resolver index (cbm_doclink_md_resolver.build), for the + * other document resolvers: the innermost code definition of `path` holding + * lines [first, last] (or the one named `member`), NULL for none; the Folder + * node at `path`; whether `path` is test code (a test directory or file). */ +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member); +const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path); +bool cbm_doclink_md_test_path(const char *path); #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS /* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 07db1d4f3a..b2a76594e7 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -1198,6 +1198,21 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD mdr_resolve_path(x, graph, doc, &r, out); } +/* ── Shared with the other document resolvers (doc_links.h) ──────── */ + +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member) { + return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; +} + +const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path) { + return md_index && path ? mdr_folder((const mdr_index_t *)md_index, NULL, path) : NULL; +} + +bool cbm_doclink_md_test_path(const char *path) { + return path && mdr_test_path(path); +} + const cbm_doclink_resolver_t cbm_doclink_md_resolver = { .langs = {CBM_LANG_MARKDOWN}, .lang_count = 1, diff --git a/src/pipeline/doc_links_rst.c b/src/pipeline/doc_links_rst.c new file mode 100644 index 0000000000..f0c5377f51 --- /dev/null +++ b/src/pipeline/doc_links_rst.c @@ -0,0 +1,1713 @@ +/* + * doc_links_rst.c — reStructuredText references -> MENTIONS edges, via "rst" + * (the resolving half of internal/cbm/doclink_rst.c). + * + * Python domain (roles, object directives, autodoc) as the field test's + * resolver (H3 PyResolver) reads it. A dotted name is read like Python's + * import system: its longest prefix that is a module's import name (a .py + * file, or a package's __init__.py, named from its topmost package + * directory down) and the rest as a qualified name in that module; a + * package's re-export (`from .base import Model` in its __init__.py, read + * from the Python scope blobs of doclink_py.c) continues the lookup at its + * source. Forms in Sphinx's search order: T, Class.T, module.T, + * module.Class.T (a leading `.` searches the most specific first). EXACT + * when one form names a node. Otherwise UNIQUE by name: the one Python + * definition named like T's last piece whose qualified name ends with T -- + * for a T of two or more pieces only (a bare name is never searched: the + * field test's errors were bare names), under a module context the one + * inside the module when several are, and for an object directive (which + * declares its object IN its module) only inside it. + * + * C domain: a C identifier is the whole name -- the one C or C++ definition + * of that name and kind (H3 CResolver), a member through its struct. + * + * Paths: include / literalinclude (relative to the document; `/x` relative + * to its documentation set: the directory of the nearest conf.py), + * kernel-include and kernel-doc (relative to the repository root, as the + * kernel's Sphinx extensions read them), extlink roles whose URL is a + * repository blob path. A literalinclude's :lines: bind the innermost + * definition holding them, its :pyobject: the named definition. Inline + * literals: the Markdown resolver's code-span rules (doc_links_md.c). + * + * Unresolved: ambiguous (several), missing (the name's first piece or the + * path's directory is the repository's), test_only_target (only test code has + * it), unparseable. Anything else is LOCAL: another project's name (the + * standard library, an intersphinx name), a bare name no context resolves. + */ +#include "pipeline/doc_links.h" + +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + RR_PATH_CAP = 1024, /* the longest path or dotted name looked up */ + RR_RAW_CAP = 4096, /* the longest token record read */ + RR_FIELDS = 8, /* fields of a token record */ + RR_PIECES = 64, /* pieces of a dotted name */ + RR_ALIAS_DEPTH = 12, /* re-exports followed (H3) */ + RR_TABLE_INIT = 1024, +}; + +/* ── Index ───────────────────────────────────────────────────────── */ + +/* A Python module by its import name. */ +typedef struct rr_mod { + const char *file; /* its .py path */ + const char *qn_base; /* the QN prefix of its definitions: project + dotted path */ + bool is_pkg; /* an __init__.py */ + struct rr_mod *next; /* another module of the same import name */ +} rr_mod_t; + +/* A package's re-export: `from src import orig as alias` (src absolute). */ +typedef struct rr_alias { + const char *alias; + int level; + const char *src; /* as written (relative when level > 0) */ + const char *orig; + struct rr_alias *next; +} rr_alias_t; + +/* A Sphinx documentation set: the directory of its conf.py. */ +typedef struct { + const char *dir; + const char *primary; /* primary_domain ("py" when unset) */ + const char *const *ext; /* extlink roles to repository paths */ + int next_count; + const char *const *isx; /* intersphinx names */ + int nisx; +} rr_conf_t; + +typedef struct { + const char *project; + size_t project_len; + void *md; /* the Markdown resolver's index: code spans, segments, folders */ + const char **run_paths; + int run_count; + CBMArena arena; + CBMHashTable *mods; /* import name -> rr_mod_t list */ + CBMHashTable *roots; /* first piece of every import name */ + CBMHashTable *aliases; /* __init__.py path -> rr_alias_t list (first per alias) */ + CBMHashTable *modnode; /* .py path -> its Module node */ + rr_conf_t *confs; /* sorted by dir */ + int nconfs; +} rr_index_t; + +static void rr_destroy(void *index); + +static bool rr_ends(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n >= k && strcmp(s + n - k, suffix) == 0; +} + +static const char *rr_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +/* A python source file and its import name (from its topmost package + * directory: the directories above it holding an __init__.py). */ +static bool rr_add_module(rr_index_t *x, const CBMHashTable *pkgdirs, const char *path) { + size_t n = strlen(path); + if (n < strlen(".py") + 1 || n >= RR_PATH_CAP) { + return true; + } + char dotted[RR_PATH_CAP]; + size_t stem = n - strlen(".py"); + memcpy(dotted, path, stem); + dotted[stem] = '\0'; + bool is_pkg = strcmp(rr_base(path), "__init__.py") == 0; + if (is_pkg) { + char *slash = strrchr(dotted, '/'); + if (!slash) { + return true; /* a top-level __init__.py has no package name */ + } + *slash = '\0'; + } + /* the topmost package directory: walk up while the parent is a package */ + char dir[RR_PATH_CAP]; + snprintf(dir, sizeof(dir), "%s", dotted); + char *cut = is_pkg ? dir + strlen(dir) : strrchr(dir, '/'); + size_t start = 0; + if (!is_pkg && !cut) { + start = 0; /* a top-level module */ + } else { + if (cut) { + *cut = '\0'; + } + for (;;) { + if (!cbm_ht_get(pkgdirs, dir)) { + start = strlen(dir) + (dir[0] ? SKIP_ONE : 0); + break; + } + char *up = strrchr(dir, '/'); + if (!up) { + start = 0; + break; + } + *up = '\0'; + } + } + if (start >= strlen(dotted)) { + return true; + } + char *name = cbm_arena_strdup(&x->arena, dotted + start); + char *qn = cbm_arena_sprintf(&x->arena, "%s.%s", x->project, dotted); + rr_mod_t *m = (rr_mod_t *)cbm_arena_alloc(&x->arena, sizeof(*m)); + if (!name || !qn || !m) { + return false; + } + for (char *c = name; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + for (char *c = qn + x->project_len + SKIP_ONE; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + m->file = cbm_arena_strdup(&x->arena, path); + m->qn_base = qn; + m->is_pkg = is_pkg; + m->next = (rr_mod_t *)cbm_ht_get(x->mods, name); + if (!m->file) { + return false; + } + cbm_ht_set(x->mods, name, m); + size_t root_len = strcspn(name, "."); + char *root = cbm_arena_strndup(&x->arena, name, root_len); + if (!root) { + return false; + } + cbm_ht_set(x->roots, root, root); + return true; +} + +static bool rr_build_modules(rr_index_t *x, const cbm_gbuf_t *graph) { + const cbm_gbuf_node_t **files = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, "File", &files, &count) != 0) { + return true; + } + CBMHashTable *pkgdirs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + if (!pkgdirs) { + return false; + } + bool ok = true; + for (int i = 0; ok && i < count; i++) { + const char *fp = files[i]->file_path; + if (fp && strcmp(rr_base(fp), "__init__.py") == 0 && rr_base(fp) > fp) { + char *d = cbm_arena_strndup(&x->arena, fp, (size_t)(rr_base(fp) - fp - SKIP_ONE)); + ok = d != NULL; + if (ok) { + cbm_ht_set(pkgdirs, d, d); + } + } + } + for (int i = 0; ok && i < count; i++) { + const char *fp = files[i]->file_path; + if (fp && rr_ends(fp, ".py")) { + ok = rr_add_module(x, pkgdirs, fp); + } + } + cbm_ht_free(pkgdirs); + const cbm_gbuf_node_t **mods = NULL; + int nmods = 0; + if (ok && cbm_gbuf_find_by_label(graph, "Module", &mods, &nmods) == 0) { + for (int i = 0; i < nmods; i++) { + const char *fp = mods[i]->file_path; + if (fp && rr_ends(fp, ".py") && !cbm_ht_get(x->modnode, fp)) { + cbm_ht_set(x->modnode, fp, (void *)mods[i]); /* the graph outlives the index */ + } + } + } + return ok; +} + +/* Split a blob line into TAB-separated fields (in place). */ +static int rr_split(char *line, char **f, int cap) { + int n = 0; + char *p = line; + while (n < cap) { + f[n++] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + p = tab + SKIP_ONE; + } + return n; +} + +static int rr_conf_cmp(const void *a, const void *b) { + return strcmp(((const rr_conf_t *)a)->dir, ((const rr_conf_t *)b)->dir); +} + +/* One Python scope blob (doclink_py.c): re-exports and conf.py settings. */ +static bool rr_read_scope(rr_index_t *x, const char *path, const char *scope, int *conf_cap) { + char *copy = cbm_arena_strdup(&x->arena, scope); + if (!copy) { + return false; + } + rr_conf_t conf = {0}; + bool is_conf = false; + const char **ext = NULL; + const char **isx = NULL; + int next = 0; + int nisx = 0; + char *save = NULL; + for (char *line = strtok_r(copy, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { + char *f[RR_FIELDS]; + int nf = rr_split(line, f, RR_FIELDS); + if (strcmp(f[0], "I") == 0 && nf == 5) { + rr_alias_t *a = (rr_alias_t *)cbm_arena_alloc(&x->arena, sizeof(*a)); + if (!a) { + return false; + } + *a = (rr_alias_t){.alias = f[4], .level = atoi(f[1]), .src = f[2], .orig = f[3]}; + rr_alias_t *head = (rr_alias_t *)cbm_ht_get(x->aliases, path); + if (!head) { + cbm_ht_set(x->aliases, path, a); + } else { + rr_alias_t *t = head; + bool seen = false; + for (;; t = t->next) { + seen = seen || strcmp(t->alias, a->alias) == 0; + if (!t->next) { + break; + } + } + if (!seen) { + t->next = a; /* the first import of a name binds it (setdefault) */ + } + } + } else if (strcmp(f[0], "C") == 0) { + is_conf = true; + } else if (strcmp(f[0], "D") == 0 && nf == 2) { + conf.primary = f[1]; + } else if ((strcmp(f[0], "X") == 0 || strcmp(f[0], "S") == 0) && nf == 2) { + bool is_ext = f[0][0] == 'X'; + const char ***arr = is_ext ? &ext : &isx; + int *cnt = is_ext ? &next : &nisx; + const char **grown = + (const char **)cbm_arena_alloc(&x->arena, (size_t)(*cnt + 1) * sizeof(char *)); + if (!grown) { + return false; + } + if (*cnt > 0) { + memcpy(grown, *arr, (size_t)*cnt * sizeof(char *)); + } + grown[(*cnt)++] = f[1]; + *arr = grown; + } + } + if (!is_conf) { + return true; + } + if (x->nconfs >= *conf_cap) { + int ncap = *conf_cap ? *conf_cap * PAIR_LEN : RR_FIELDS; + rr_conf_t *grown = (rr_conf_t *)cbm_arena_alloc(&x->arena, (size_t)ncap * sizeof(*grown)); + if (!grown) { + return false; + } + if (x->nconfs > 0) { + memcpy(grown, x->confs, (size_t)x->nconfs * sizeof(*grown)); + } + x->confs = grown; + *conf_cap = ncap; + } + const char *b = rr_base(path); + conf.dir = cbm_arena_strndup(&x->arena, path, b > path ? (size_t)(b - path - SKIP_ONE) : 0); + if (!conf.dir) { + return false; + } + conf.primary = conf.primary ? conf.primary : "py"; + conf.ext = ext; + conf.next_count = next; + conf.isx = isx; + conf.nisx = nisx; + x->confs[x->nconfs++] = conf; + return true; +} + +static void *rr_build(const cbm_doclink_build_in_t *in) { + rr_index_t *x = (rr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->project_len = x->project ? strlen(x->project) : 0; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->mods = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->roots = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->aliases = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->modnode = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->md = cbm_doclink_md_resolver.build(in); + if ((in->run_file_count > 0 && !x->run_paths) || !x->mods || !x->roots || !x->aliases || + !x->modnode || !x->md || !x->project || !rr_build_modules(x, in->graph)) { + rr_destroy(x); + return NULL; + } + int conf_cap = 0; + for (int i = 0; i < in->file_count; i++) { + const cbm_doclink_file_t *f = &in->files[i]; + if (f->run_file >= 0 && f->run_file < x->run_count) { + x->run_paths[f->run_file] = f->rel_path; /* valid until destroy */ + } + if (f->scope && strncmp(f->scope, CBM_DOCLINK_PY_SCOPE_TAG "\n", + strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n")) == 0) { + const char *path = cbm_arena_strdup(&x->arena, f->rel_path); + if (!path || !rr_read_scope(x, path, f->scope, &conf_cap)) { + rr_destroy(x); + return NULL; + } + } + } + if (x->nconfs > SKIP_ONE) { + qsort(x->confs, (size_t)x->nconfs, sizeof(*x->confs), rr_conf_cmp); + } + return x; +} + +static void rr_destroy(void *index) { + rr_index_t *x = (rr_index_t *)index; + if (!x) { + return; + } + if (x->md) { + cbm_doclink_md_resolver.destroy(x->md); + } + cbm_ht_free(x->mods); + cbm_ht_free(x->roots); + cbm_ht_free(x->aliases); + cbm_ht_free(x->modnode); + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* The documentation set a document belongs to: the nearest conf.py at or + * above its directory; NULL for none. */ +static const rr_conf_t *rr_conf_for(const rr_index_t *x, const char *doc) { + char dir[RR_PATH_CAP]; + const char *b = rr_base(doc); + size_t dl = b > doc ? (size_t)(b - doc - SKIP_ONE) : 0; + if (dl >= sizeof(dir)) { + return NULL; + } + memcpy(dir, doc, dl); + dir[dl] = '\0'; + for (;;) { + rr_conf_t key = {.dir = dir}; + const rr_conf_t *hit = (const rr_conf_t *)bsearch(&key, x->confs, (size_t)x->nconfs, + sizeof(*x->confs), rr_conf_cmp); + if (hit) { + return hit; + } + if (!dir[0]) { + return NULL; + } + char *up = strrchr(dir, '/'); + if (up) { + *up = '\0'; + } else { + dir[0] = '\0'; + } + } +} + +static bool rr_listed(const char *const *list, int n, const char *s) { + for (int i = 0; i < n; i++) { + if (strcmp(list[i], s) == 0) { + return true; + } + } + return false; +} + +/* ── Outcomes ────────────────────────────────────────────────────── */ + +typedef enum { RR_NONE = 0, RR_HIT, RR_AMBIGUOUS, RR_TEST_ONLY } rr_kind_t; + +typedef struct { + rr_kind_t kind; + const cbm_gbuf_node_t *node; + bool exact; +} rr_res_t; + +static void rr_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *n, bool exact, + uint32_t first, uint32_t last) { + out->kind = CBM_DOCLINK_EDGE; + out->target = n; + out->exact = exact; + out->target_first = first; + out->target_last = last; +} + +static void rr_unresolved(cbm_doclink_outcome_t *out, int reason) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = reason; +} + +/* The outcome of a lookup. A name the document writes in full binds even + * test code (`django.test.TestCase` is Django's product); the name searches + * leave test code out themselves (R4). */ +static void rr_finish(cbm_doclink_outcome_t *out, rr_res_t r) { + if (r.kind == RR_HIT && r.node) { + rr_edge(out, r.node, r.exact, 0, 0); + } else if (r.kind == RR_AMBIGUOUS) { + rr_unresolved(out, CBM_DOCLINK_REASON_AMBIGUOUS); + } else if (r.kind == RR_TEST_ONLY) { + rr_unresolved(out, CBM_DOCLINK_REASON_TEST_ONLY); + } +} + +/* ── Python names ────────────────────────────────────────────────── */ + +static bool rr_word(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || + c >= 0x80; +} + +/* PY_TARGET_RE ^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$ */ +static bool rr_py_target(const char *t) { + bool start = true; + if (!t[0]) { + return false; + } + for (const char *p = t; *p; p++) { + unsigned char c = (unsigned char)*p; + if (c == '.') { + if (start) { + return false; + } + start = true; + continue; + } + if (start && c >= '0' && c <= '9') { + return false; + } + if (!rr_word(c)) { + return false; + } + start = false; + } + return !start; +} + +static const rr_mod_t *rr_module(const rr_index_t *x, const char *name, int *count) { + const rr_mod_t *m = (const rr_mod_t *)cbm_ht_get(x->mods, name); + *count = 0; + for (const rr_mod_t *k = m; k; k = k->next) { + (*count)++; + } + return m; +} + +/* One lookup of H3 resolve_dotted: the longest module prefix, the rest by + * qualified name. RR_NONE with `next` set: a package re-export names the + * dotted name the lookup continues with. */ +static rr_res_t rr_dotted_step(const rr_index_t *x, const cbm_gbuf_t *graph, const char *dotted, + char *next, size_t next_cap) { + rr_res_t none = {RR_NONE, NULL, false}; + next[0] = '\0'; + size_t dlen = strlen(dotted); + if (dlen == 0 || dlen >= RR_PATH_CAP) { + return none; + } + char buf[RR_PATH_CAP]; + memcpy(buf, dotted, dlen + SKIP_ONE); + size_t ends[RR_PIECES]; + int np = 0; + for (size_t i = 0; i <= dlen && np < RR_PIECES; i++) { + if (buf[i] == '.' || buf[i] == '\0') { + ends[np++] = i; + } + } + const rr_mod_t *mod = NULL; + int k = np; + for (; k > 0; k--) { + char c = buf[ends[k - 1]]; + buf[ends[k - 1]] = '\0'; + int count = 0; + mod = rr_module(x, buf, &count); + buf[ends[k - 1]] = c; + if (count > SKIP_ONE) { + return (rr_res_t){RR_AMBIGUOUS, NULL, false}; + } + if (mod) { + break; + } + } + if (!mod) { + return none; + } + const char *rest = ends[k - 1] < dlen ? dotted + ends[k - 1] + SKIP_ONE : ""; + if (!rest[0]) { + const cbm_gbuf_node_t *n = (const cbm_gbuf_node_t *)cbm_ht_get(x->modnode, mod->file); + if (!n && mod->is_pkg) { + char dir[RR_PATH_CAP]; + const char *b = rr_base(mod->file); + size_t l = (size_t)(b - mod->file - SKIP_ONE); + memcpy(dir, mod->file, l); + dir[l] = '\0'; + n = cbm_doclink_md_folder(x->md, dir); + } + return n ? (rr_res_t){RR_HIT, n, true} : none; + } + char qn[RR_PATH_CAP * PAIR_LEN]; + snprintf(qn, sizeof(qn), "%s.%s", mod->qn_base, rest); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(graph, qn); + if (n) { + return (rr_res_t){RR_HIT, n, true}; + } + if (!mod->is_pkg) { + return none; + } + size_t first_len = strcspn(rest, "."); + for (const rr_alias_t *a = (const rr_alias_t *)cbm_ht_get(x->aliases, mod->file); a; + a = a->next) { + if (strlen(a->alias) != first_len || strncmp(a->alias, rest, first_len) != 0) { + continue; + } + /* the source module, absolute: a relative import counts from the + * package as written (H3 init_imports) */ + char pkg[RR_PATH_CAP]; + size_t pl = ends[k - 1]; + memcpy(pkg, dotted, pl); + pkg[pl] = '\0'; + char src[RR_PATH_CAP]; + if (a->level > 0) { + for (int up = a->level - SKIP_ONE; up > 0; up--) { + char *dot = strrchr(pkg, '.'); + if (!dot) { + pkg[0] = '\0'; + break; + } + *dot = '\0'; + } + snprintf(src, sizeof(src), "%s%s%s", pkg, pkg[0] && a->src[0] ? "." : "", a->src); + } else { + snprintf(src, sizeof(src), "%s", a->src); + } + const char *tail = rest + first_len; + int w = snprintf(next, next_cap, "%s%s%s%s", src, src[0] ? "." : "", a->orig, tail); + if (w <= 0 || (size_t)w >= next_cap) { + next[0] = '\0'; + } + return none; /* the lookup continues at the re-export's source */ + } + return none; +} + +/* H3 resolve_dotted: a module prefix, the rest by qualified name, a package's + * re-exports followed (at most RR_ALIAS_DEPTH of them, as H3). */ +static rr_res_t rr_dotted(const rr_index_t *x, const cbm_gbuf_t *graph, const char *dotted) { + rr_res_t none = {RR_NONE, NULL, false}; + char cur[RR_PATH_CAP]; + char next[RR_PATH_CAP]; + size_t n = strlen(dotted); + if (n >= sizeof(cur)) { + return none; + } + memcpy(cur, dotted, n + SKIP_ONE); + for (int hop = 0; hop <= RR_ALIAS_DEPTH; hop++) { + rr_res_t r = rr_dotted_step(x, graph, cur, next, sizeof(next)); + if (r.kind != RR_NONE || !next[0]) { + return r; + } + memcpy(cur, next, strlen(next) + SKIP_ONE); + } + return none; +} + +static const char *const RR_PY_LABELS[] = {"Class", "Function", "Method", "Variable", "Module", + "Field", "Enum", "Interface", "Type", NULL}; + +/* PY_KIND_LABELS: the labels a role or directive kind may bind by name. */ +static bool rr_kind_label(const char *kind, const char *label) { + static const struct { + const char *kind; + const char *labels[3]; + } map[] = {{"class", {"Class"}}, + {"exc", {"Class"}}, + {"exception", {"Class"}}, + {"meth", {"Method"}}, + {"method", {"Method"}}, + {"classmethod", {"Method"}}, + {"staticmethod", {"Method"}}, + {"func", {"Function"}}, + {"function", {"Function"}}, + {"decorator", {"Function"}}, + {"deco", {"Function"}}, + {"decoratormethod", {"Method"}}, + {"attr", {"Variable", "Method"}}, + {"attribute", {"Variable", "Method"}}, + {"property", {"Method", "Variable"}}, + {"data", {"Variable"}}, + {"const", {"Variable"}}, + {"mod", {"Module"}}, + {"module", {"Module"}}}; + for (size_t i = 0; i < sizeof(map) / sizeof(map[0]); i++) { + if (strcmp(map[i].kind, kind) != 0) { + continue; + } + for (int j = 0; j < 3 && map[i].labels[j]; j++) { + if (strcmp(map[i].labels[j], label) == 0) { + return true; + } + } + return false; + } + return true; /* obj: any code label */ +} + +static bool rr_in_labels(const char *label, const char *const *labels) { + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* The QN prefix of module M's definitions, written as M; NULL when M is no + * module of the repository. */ +static const char *rr_module_base(const rr_index_t *x, const char *m) { + int count = 0; + const rr_mod_t *mod = m && m[0] ? rr_module(x, m, &count) : NULL; + return count == SKIP_ONE ? mod->qn_base : NULL; +} + +static bool rr_under(const char *qn, const char *base) { + size_t bl = strlen(base); + return strncmp(qn, base, bl) == 0 && qn[bl] == '.'; +} + +/* Kinds whose one-piece name may be searched: a class, exception, function or + * module-level datum is named by itself. A bare member (`:meth:`save``, + * `:attr:`name``) names a member of SOME class: the field test found 37 of 40 + * such search results wrong without a class context, so it never is. */ +static bool rr_bare_searchable(const char *kind) { + static const char *const kinds[] = {"class", "exc", "func", "data", "const", "deco", NULL}; + for (int i = 0; kinds[i]; i++) { + if (strcmp(kinds[i], kind) == 0) { + return true; + } + } + return false; +} + +/* H3 unique_candidates, with the field test's fixes: a bare member name is + * never searched (rr_bare_searchable); an object directive's target lies + * inside the module it is declared in (its one-piece name only there); + * test code is no target of a search (R4). */ +static rr_res_t rr_unique(const rr_index_t *x, const cbm_gbuf_t *graph, const char *t, + const char *kind, const char *m, bool directive) { + rr_res_t none = {RR_NONE, NULL, false}; + const char *last = strrchr(t, '.'); + const char *mbase = rr_module_base(x, m); + if (!last && !(directive ? mbase != NULL : rr_bare_searchable(kind))) { + return none; + } + const char *name = last ? last + SKIP_ONE : t; + char suffix[RR_PATH_CAP]; + if (snprintf(suffix, sizeof(suffix), ".%s", t) >= (int)sizeof(suffix)) { + return none; + } + size_t sl = strlen(suffix); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return none; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *in_mod = NULL; + int in_mod_hits = 0; + int test_hits = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *qn = n->qualified_name; + if (!rr_in_labels(n->label, RR_PY_LABELS) || !rr_kind_label(kind, n->label) || + !n->file_path || !rr_ends(n->file_path, ".py") || !qn || strlen(qn) < sl || + strcmp(qn + strlen(qn) - sl, suffix) != 0) { + continue; + } + if (cbm_doclink_md_test_path(n->file_path)) { + test_hits++; + continue; + } + hit = n; + hits++; + if (mbase && rr_under(qn, mbase)) { + in_mod = n; + in_mod_hits++; + } + } + if (mbase && (directive || hits > SKIP_ONE) && in_mod_hits > 0) { + hit = in_mod; + hits = in_mod_hits; + } else if (directive && mbase) { + hits = 0; /* declared in its module: never one found outside it */ + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, false}; + } + if (hits > SKIP_ONE) { + return (rr_res_t){RR_AMBIGUOUS, NULL, false}; + } + return test_hits > 0 ? (rr_res_t){RR_TEST_ONLY, NULL, false} : none; +} + +/* H3 PyResolver.resolve: the forms, then the name search. */ +static rr_res_t rr_py_resolve(const rr_index_t *x, const cbm_gbuf_t *graph, const char *t, + const char *kind, const char *m, const char *c, bool refspecific, + bool directive) { + char forms[4][RR_PATH_CAP]; + int nf = 0; + bool has_m = m && m[0]; + bool has_c = c && c[0]; + if (refspecific) { + if (has_m && has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s.%s", m, c, t); + } + if (has_m) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", m, t); + } + snprintf(forms[nf++], RR_PATH_CAP, "%s", t); + } else { + snprintf(forms[nf++], RR_PATH_CAP, "%s", t); + if (has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", c, t); + } + if (has_m) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", m, t); + } + if (has_m && has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s.%s", m, c, t); + } + } + /* a module answers only a module's kind: `:meth:`dispatch`` is never the + * package tests/dispatch (whose import name is `dispatch`) */ + bool module_kind = + strcmp(kind, "mod") == 0 || strcmp(kind, "module") == 0 || strcmp(kind, "obj") == 0; + for (int i = 0; i < nf; i++) { + rr_res_t r = rr_dotted(x, graph, forms[i]); + if (r.kind == RR_HIT && + (module_kind || !r.node->label || + (strcmp(r.node->label, "Module") != 0 && strcmp(r.node->label, "Folder") != 0))) { + return r; + } + } + return rr_unique(x, graph, t, kind, m, directive); +} + +/* Not found: the repository's name (a row) or another project's (LOCAL). */ +static void rr_py_unfound(const rr_index_t *x, const char *t, cbm_doclink_outcome_t *out) { + char root[RR_PATH_CAP]; + size_t rl = strcspn(t, "."); + if (!strchr(t, '.') || rl >= sizeof(root)) { + out->kind = CBM_DOCLINK_LOCAL; /* a bare name no context resolved */ + return; + } + memcpy(root, t, rl); + root[rl] = '\0'; + if (cbm_ht_get(x->roots, root)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + out->kind = CBM_DOCLINK_LOCAL; +} + +/* ── Token records ───────────────────────────────────────────────── */ + +typedef struct { + char buf[RR_RAW_CAP]; + char *f[RR_FIELDS]; + int n; +} rr_rec_t; + +static bool rr_record(const char *raw, rr_rec_t *r) { + size_t n = strlen(raw); + if (n >= sizeof(r->buf)) { + return false; + } + memcpy(r->buf, raw, n + SKIP_ONE); + r->n = rr_split(r->buf, r->f, RR_FIELDS); + for (int i = r->n; i < RR_FIELDS; i++) { + r->f[i] = ""; + } + return true; +} + +/* H3 split_domain. */ +static void rr_split_domain(const char *name, const char *primary, const char *const *py_members, + const char *const *c_members, char *dom, size_t cap, + const char **rest) { + const char *colon = strchr(name, ':'); + if (colon) { + size_t l = (size_t)(colon - name); + snprintf(dom, cap, "%.*s", (int)(l < cap ? l : cap - SKIP_ONE), name); + *rest = colon + SKIP_ONE; + return; + } + const char *const *members = strcmp(primary, "c") == 0 ? c_members + : strcmp(primary, "py") == 0 ? py_members + : NULL; + bool in = false; + for (int i = 0; members && members[i]; i++) { + in = in || strcmp(members[i], name) == 0; + } + snprintf(dom, cap, "%s", in ? primary : "std"); + *rest = name; +} + +static const char *const RR_PY_ROLES[] = {"func", "class", "meth", "attr", "mod", "exc", + "data", "obj", "const", "deco", NULL}; +static const char *const RR_C_ROLES[] = {"func", "macro", "struct", "union", "enum", "enumerator", + "type", "member", "data", "var", NULL}; +static const char *const RR_PY_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", NULL}; +static const char *const RR_PY_DOMAIN_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", "currentmodule", NULL}; +static const char *const RR_C_OBJ[] = {"function", "macro", "type", "struct", "union", + "enum", "enumerator", "member", "var", NULL}; +static const char *const RR_C_DOMAIN_OBJ[] = { + "function", "macro", "type", "struct", "union", "enum", "enumerator", + "member", "var", "namespace", "namespace-push", "namespace-pop", NULL}; + +static bool rr_member_of(const char *s, const char *const *list) { + for (int i = 0; list[i]; i++) { + if (strcmp(list[i], s) == 0) { + return true; + } + } + return false; +} + +/* H3 parse_target: title , then !, ~, a leading . and a trailing (). */ +typedef struct { + char t[RR_PATH_CAP]; + bool bang; + bool refspecific; +} rr_target_t; + +static bool rr_parse_target(const char *content, rr_target_t *out) { + char c[RR_PATH_CAP]; + size_t w = 0; + for (const char *p = content; *p && w + SKIP_ONE < sizeof(c); p++) { + if (*p == '\\' && (p[1] == ' ' || p[1] == '\t')) { + p++; /* an escaped space joins its neighbours */ + continue; + } + c[w++] = *p; + } + c[w] = '\0'; + const char *t = c; + size_t tl = w; + if (w > 0 && c[w - SKIP_ONE] == '>') { + char *lt = strrchr(c, '<'); + if (lt && lt > c) { + /* a title before it: ^(.*?)\s*<([^<>]+)>$ */ + size_t inner = (size_t)(c + w - SKIP_ONE - (lt + SKIP_ONE)); + if (inner > 0 && !memchr(lt + SKIP_ONE, '>', inner)) { + t = lt + SKIP_ONE; + tl = inner; + while (tl > 0 && t[0] == ' ') { + t++; + tl--; + } + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + tl--; + } + } + } + } + out->bang = false; + out->refspecific = false; + for (int round = 0; round < PAIR_LEN; round++) { + if (tl > 0 && t[0] == '!') { + out->bang = true; + t++; + tl--; + } + if (tl > 0 && t[0] == '~') { + t++; + tl--; + } + } + if (tl > 0 && t[0] == '.') { + out->refspecific = true; + t++; + tl--; + } + if (tl >= PAIR_LEN && t[tl - PAIR_LEN] == '(' && t[tl - SKIP_ONE] == ')') { + tl -= PAIR_LEN; + } + while (tl > 0 && t[0] == ' ') { + t++; + tl--; + } + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + tl--; + } + if (tl >= sizeof(out->t)) { + return false; + } + memcpy(out->t, t, tl); + out->t[tl] = '\0'; + return true; +} + +/* ── C domain ────────────────────────────────────────────────────── */ + +static bool rr_c_file(const char *path) { + static const char *const exts[] = {".c", ".h", ".cc", ".cpp", ".cxx", + ".hh", ".hpp", ".hxx", NULL}; + for (int i = 0; path && exts[i]; i++) { + if (rr_ends(path, exts[i])) { + return true; + } + } + return false; +} + +/* H3 c_is_def: a definition, not a usage or a forward declaration. */ +static bool rr_c_def(const cbm_gbuf_node_t *n) { + static const char *const defining[] = {"Function", "Type", "Variable", "Field", "Method", NULL}; + return rr_in_labels(n->label, defining) || n->end_line > n->start_line; +} + +/* Macro nodes are not proxied on the incremental route (pipeline_delta.c), so + * a C role never binds one: a re-resolved document binds what a full build + * does. */ +static const char *const RR_C_LABELS[] = {"Function", "Class", "Enum", "Struct", "Type", NULL}; + +static rr_res_t rr_c_name(const cbm_gbuf_t *graph, const char *name, bool data, + const char *in_file) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + rr_res_t none = {RR_NONE, NULL, false}; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return none; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *def = NULL; + int defs = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *fp = n->file_path; + bool label = rr_in_labels(n->label, RR_C_LABELS) || + (data && n->label && strcmp(n->label, "Variable") == 0) || + (n->label && strcmp(n->label, "Method") == 0 && fp && + (rr_ends(fp, ".c") || rr_ends(fp, ".h"))); + if (!label || !rr_c_file(fp) || (in_file && strcmp(fp, in_file) != 0) || + (!in_file && cbm_doclink_md_test_path(fp))) { + continue; + } + if (!in_file && !rr_c_def(n)) { + continue; + } + hit = n; + hits++; + if (rr_c_def(n)) { + def = n; + defs++; + } + } + if (in_file && hits > SKIP_ONE && defs == SKIP_ONE) { + return (rr_res_t){RR_HIT, def, true}; /* H3 kd_node: the one definition */ + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, in_file != NULL}; + } + return hits > SKIP_ONE ? (rr_res_t){RR_AMBIGUOUS, NULL, false} : none; +} + +/* A member through its struct: the one Field `struct.member`. */ +static rr_res_t rr_c_member(const cbm_gbuf_t *graph, const char *owner, const char *member) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + rr_res_t none = {RR_NONE, NULL, false}; + if (cbm_gbuf_find_by_name(graph, member, &nodes, &count) != 0) { + return none; + } + char suffix[RR_PATH_CAP]; + snprintf(suffix, sizeof(suffix), ".%s.%s", owner ? owner : "", member); + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (!n->label || strcmp(n->label, "Field") != 0 || !rr_c_file(n->file_path) || + cbm_doclink_md_test_path(n->file_path)) { + continue; + } + if (owner && (!n->qualified_name || !rr_ends(n->qualified_name, suffix))) { + continue; + } + hit = n; + hits++; + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, false}; + } + return hits > SKIP_ONE ? (rr_res_t){RR_AMBIGUOUS, NULL, false} : none; +} + +static bool rr_c_ident(const char *s) { + if (!s[0] || (s[0] >= '0' && s[0] <= '9')) { + return false; + } + for (const char *p = s; *p; p++) { + unsigned char c = (unsigned char)*p; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_')) { + return false; + } + } + return true; +} + +/* Is the name the repository's C code at all (a row rather than LOCAL)? */ +static bool rr_c_known(const cbm_gbuf_t *graph, const char *name) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return false; + } + for (int i = 0; i < count; i++) { + if (rr_c_file(nodes[i]->file_path)) { + return true; + } + } + return false; +} + +/* H3 c_clean then c_role: struct/union/enum/typedef dropped, -> read as ., + * a trailing * or () dropped; a member through its struct. */ +static void rr_c_role(const cbm_gbuf_t *graph, const char *doc, const char *rname, + const char *target, cbm_doclink_outcome_t *out) { + char t[RR_PATH_CAP]; + const char *s = target; + static const char *const kws[] = {"struct ", "union ", "enum ", "typedef ", NULL}; + for (int i = 0; kws[i]; i++) { + if (strncmp(s, kws[i], strlen(kws[i])) == 0) { + s += strlen(kws[i]); + break; + } + } + size_t w = 0; + for (const char *p = s; *p && w + SKIP_ONE < sizeof(t); p++) { + if (p[0] == '-' && p[1] == '>') { + t[w++] = '.'; + p++; + } else { + t[w++] = *p; + } + } + while (w > 0 && (t[w - SKIP_ONE] == ' ' || t[w - SKIP_ONE] == '*')) { + w--; + } + t[w] = '\0'; + if (w >= PAIR_LEN && t[w - PAIR_LEN] == '(' && t[w - SKIP_ONE] == ')') { + t[w - PAIR_LEN] = '\0'; + } + char *start = t; + while (*start == ' ') { + start++; + } + rr_res_t r; + const char *name = start; + if (strcmp(rname, "member") == 0 || strchr(start, '.')) { + char *dot = strrchr(start, '.'); + const char *owner = NULL; + if (dot) { + *dot = '\0'; + char *prev = strrchr(start, '.'); + owner = prev ? prev + SKIP_ONE : start; + name = dot + SKIP_ONE; + } + if (!rr_c_ident(name) || (owner && !rr_c_ident(owner))) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + r = rr_c_member(graph, owner, name); + } else { + if (!rr_c_ident(name)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + bool data = strcmp(rname, "data") == 0 || strcmp(rname, "var") == 0; + r = rr_c_name(graph, name, data, NULL); + } + if (r.kind == RR_NONE) { + if (rr_c_known(graph, name)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + out->kind = CBM_DOCLINK_LOCAL; + } + return; + } + rr_finish(out, r); +} + +/* H3 c_sig_name: the declared name of a C object directive's signature. */ +static bool rr_c_sig_name(const char *dname, const char *sig, char *out, size_t cap) { + if (strcmp(dname, "function") == 0) { + size_t n = strlen(sig); + /* (*name)( : a function pointer */ + for (size_t i = 0; i < n; i++) { + if (sig[i] != '(') { + continue; + } + size_t j = i + SKIP_ONE; + while (j < n && sig[j] == ' ') { + j++; + } + if (j >= n || sig[j] != '*') { + continue; + } + j++; + while (j < n && sig[j] == ' ') { + j++; + } + size_t a = j; + while (j < n && (rr_word((unsigned char)sig[j]) && (unsigned char)sig[j] < 0x80)) { + j++; + } + size_t b = j; + while (j < n && sig[j] == ' ') { + j++; + } + if (b > a && j < n && sig[j] == ')') { + j++; + while (j < n && sig[j] == ' ') { + j++; + } + if (j < n && sig[j] == '(') { + snprintf(out, cap, "%.*s", (int)(b - a), sig + a); + return true; + } + } + } + /* ([A-Za-z_]\w*)\s*\( : the first identifier before a parenthesis */ + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)sig[i]; + bool start = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + if (!start || (i > 0 && rr_word((unsigned char)sig[i - SKIP_ONE]) && + (unsigned char)sig[i - SKIP_ONE] < 0x80)) { + continue; + } + size_t j = i; + while (j < n && rr_word((unsigned char)sig[j]) && (unsigned char)sig[j] < 0x80) { + j++; + } + size_t k = j; + while (k < n && sig[k] == ' ') { + k++; + } + if (k < n && sig[k] == '(') { + snprintf(out, cap, "%.*s", (int)(j - i), sig + i); + return true; + } + i = j; + } + return false; + } + const char *s = sig; + if (strcmp(dname, "macro") != 0) { + static const char *const kws[] = {"struct ", "union ", "enum ", "typedef ", NULL}; + for (int i = 0; kws[i]; i++) { + if (strncmp(s, kws[i], strlen(kws[i])) == 0) { + s += strlen(kws[i]); + break; + } + } + } + /* macro: the leading identifier; others: the last identifier */ + const char *best = NULL; + size_t best_len = 0; + for (const char *p = s; *p;) { + unsigned char c = (unsigned char)*p; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { + const char *e = p; + while (*e && rr_word((unsigned char)*e) && (unsigned char)*e < 0x80) { + e++; + } + best = p; + best_len = (size_t)(e - p); + if (strcmp(dname, "macro") == 0) { + break; + } + p = e; + } else { + if (strcmp(dname, "macro") == 0) { + break; + } + if (rr_word(c) && c < 0x80) { /* digits: part of no identifier start */ + while (*p && rr_word((unsigned char)*p) && (unsigned char)*p < 0x80) { + p++; + } + continue; + } + p++; + } + } + if (!best || best_len >= cap) { + return false; + } + memcpy(out, best, best_len); + out[best_len] = '\0'; + return true; +} + +/* ── Paths ───────────────────────────────────────────────────────── */ + +/* os.path.normpath(base/p) relative to the repository; false when it leaves + * the root. */ +static bool rr_norm(const char *base, const char *p, char *out, size_t cap) { + char tmp[RR_PATH_CAP * PAIR_LEN]; + snprintf(tmp, sizeof(tmp), "%s%s%s", base, base[0] && p[0] ? "/" : "", p); + const char *seg[RR_PIECES * PAIR_LEN]; + size_t len[RR_PIECES * PAIR_LEN]; + int n = 0; + for (char *s = tmp; *s;) { + char *e = strchr(s, '/'); + size_t l = e ? (size_t)(e - s) : strlen(s); + if (l == 0 || (l == SKIP_ONE && s[0] == '.')) { + /* nothing */ + } else if (l == PAIR_LEN && s[0] == '.' && s[1] == '.') { + if (n == 0) { + return false; + } + n--; + } else if (n < (int)(sizeof(seg) / sizeof(seg[0]))) { + seg[n] = s; + len[n] = l; + n++; + } else { + return false; + } + if (!e) { + break; + } + s = e + SKIP_ONE; + } + size_t w = 0; + for (int i = 0; i < n; i++) { + if (w + len[i] + PAIR_LEN > cap) { + return false; + } + if (i > 0) { + out[w++] = '/'; + } + memcpy(out + w, seg[i], len[i]); + w += len[i]; + } + out[w] = '\0'; + return true; +} + +static void rr_doc_dir(const char *doc, char *out, size_t cap) { + const char *b = rr_base(doc); + size_t l = b > doc ? (size_t)(b - doc - SKIP_ONE) : 0; + snprintf(out, cap, "%.*s", (int)(l < cap ? l : cap - SKIP_ONE), doc); +} + +/* The File node at `rel`; else a row when its directory is the repository's + * (doc rot), LOCAL otherwise. */ +static const cbm_gbuf_node_t *rr_file(const rr_index_t *x, const cbm_gbuf_t *graph, const char *rel, + bool folder_too, cbm_doclink_outcome_t *out) { + const cbm_gbuf_node_t *f = rel[0] ? cbm_pipeline_file_node(graph, x->project, rel) : NULL; + if (!f && folder_too && rel[0]) { + f = cbm_doclink_md_folder(x->md, rel); + } + if (f) { + return f; + } + char dir[RR_PATH_CAP]; + rr_doc_dir(rel, dir, sizeof(dir)); + if (dir[0] && cbm_doclink_md_folder(x->md, dir)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + out->kind = CBM_DOCLINK_LOCAL; + } + return NULL; +} + +/* include / literalinclude: Sphinx reads `/x` from the documentation set's + * directory and everything else from the document's. */ +static bool rr_doc_path(const rr_index_t *x, const char *doc, const char *arg, char *out, + size_t cap) { + if (arg[0] == '/') { + const rr_conf_t *conf = rr_conf_for(x, doc); + return rr_norm(conf ? conf->dir : "", arg + SKIP_ONE, out, cap); + } + char dir[RR_PATH_CAP]; + rr_doc_dir(doc, dir, sizeof(dir)); + return rr_norm(dir, arg, out, cap); +} + +static void rr_literalinclude(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + const char *arg = r->f[1]; + if (!arg[0] || strchr(arg, '$')) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char rel[RR_PATH_CAP]; + if (!rr_doc_path(x, doc, arg, rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *file = rr_file(x, graph, rel, false, out); + if (!file) { + return; + } + const char *lines = r->f[2]; + const char *pyobject = r->f[3]; + if (pyobject[0]) { + /* H3: the definition `pyobject` of this .py file, by its QN */ + const cbm_gbuf_node_t *n = NULL; + if (rr_ends(rel, ".py")) { + char qn[RR_PATH_CAP * PAIR_LEN]; + char dotted[RR_PATH_CAP]; + size_t l = strlen(rel) - strlen(".py"); + snprintf(dotted, sizeof(dotted), "%.*s", (int)l, rel); + if (rr_ends(dotted, "/__init__")) { + dotted[strlen(dotted) - strlen("/__init__")] = '\0'; + } + for (char *c = dotted; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + snprintf(qn, sizeof(qn), "%s.%s.%s", x->project, dotted, pyobject); + n = cbm_gbuf_find_by_qn(graph, qn); + } + if (n) { + rr_edge(out, n, true, 0, 0); + } else { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } + return; + } + uint32_t a = 0; + uint32_t b = 0; + for (const char *p = lines; *p;) { + if (*p >= '0' && *p <= '9') { + uint32_t v = 0; + while (*p >= '0' && *p <= '9') { + v = v < UINT32_MAX / 10 ? v * 10 + (uint32_t)(*p - '0') : UINT32_MAX; + p++; + } + a = (a == 0 || v < a) ? v : a; + b = v > b ? v : b; + } else { + p++; + } + } + if (a > 0) { + const cbm_gbuf_node_t *seg = cbm_doclink_md_segment(x->md, rel, a, b, NULL); + rr_edge(out, seg ? seg : file, true, a, b); + return; + } + rr_edge(out, file, true, 0, 0); +} + +/* kernel-doc: the file, or one of its names (H3 kd_node). */ +static void rr_kernel_doc(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + char rel[RR_PATH_CAP]; + if (!r->f[1][0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (!rr_norm("", r->f[1], rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *file = rr_file(x, graph, rel, false, out); + if (!file) { + return; + } + const char *name = r->f[2]; + if (!name[0]) { + rr_edge(out, file, true, 0, 0); + return; + } + rr_res_t res = rr_c_name(graph, name, true, rel); + if (res.kind == RR_NONE && rr_ends(rel, ".h")) { + /* the comment documents a prototype: its one definition elsewhere */ + res = rr_c_name(graph, name, false, NULL); + } + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +/* ── Resolve ─────────────────────────────────────────────────────── */ + +static void rr_py_role(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_conf_t *conf, const char *rname, const rr_rec_t *r, + cbm_doclink_outcome_t *out) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (tg.bang) { + out->kind = CBM_DOCLINK_LOCAL; /* `!x`: no link wanted */ + return; + } + /* an intersphinx name: `python:dict` */ + const char *colon = strchr(tg.t, ':'); + if (colon && conf) { + char key[RR_PATH_CAP]; + snprintf(key, sizeof(key), "%.*s", (int)(colon - tg.t), tg.t); + if (rr_listed(conf->isx, conf->nisx, key)) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + } + if (!rr_py_target(tg.t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + rr_res_t res = rr_py_resolve(x, graph, tg.t, rname, r->f[3], r->f[4], tg.refspecific, false); + if (res.kind == RR_NONE) { + rr_py_unfound(x, tg.t, out); + return; + } + rr_finish(out, res); +} + +static void rr_py_object(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const char *dname, const rr_rec_t *r, cbm_doclink_outcome_t *out) { + const char *m = r->f[3]; + if (strcmp(dname, "module") == 0) { + const char *t = r->f[2]; + if (!rr_py_target(t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + rr_res_t res = rr_dotted(x, graph, t); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); + return; + } + const char *fullname = r->f[6]; + if (!fullname[0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char full[RR_PATH_CAP]; + snprintf(full, sizeof(full), "%s%s%s", m, m[0] ? "." : "", fullname); + rr_res_t res = rr_dotted(x, graph, full); + if (res.kind != RR_HIT) { + res = rr_py_resolve(x, graph, fullname, dname, m, NULL, false, true); + } + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +static void rr_autodoc(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + char t[RR_PATH_CAP]; + snprintf(t, sizeof(t), "%s", r->f[2]); + char *paren = strchr(t, '('); + if (paren) { + *paren = '\0'; + } + size_t tl = strlen(t); + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + t[--tl] = '\0'; + } + if (!rr_py_target(t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + const char *m = r->f[3]; + char full[RR_PATH_CAP]; + if (strcmp(r->f[1], "automodule") == 0 || !m[0]) { + snprintf(full, sizeof(full), "%s", t); + } else { + snprintf(full, sizeof(full), "%s.%s", m, t); + } + rr_res_t res = rr_dotted(x, graph, full); + if (res.kind != RR_HIT && strcmp(full, t) != 0) { + res = rr_dotted(x, graph, t); + } + if (res.kind != RR_HIT) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +static void rr_extlink(const rr_index_t *x, const cbm_gbuf_t *graph, const rr_rec_t *r, + cbm_doclink_outcome_t *out) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg) || !tg.t[0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char rel[RR_PATH_CAP]; + const char *p = tg.t; + while (*p == '/') { + p++; + } + if (!rr_norm("", p, rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *f = rr_file(x, graph, rel, true, out); + if (f) { + rr_edge(out, f, true, 0, 0); + } +} + +static void rr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const rr_index_t *x = (const rr_index_t *)index; + out->kind = CBM_DOCLINK_LOCAL; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (link->syntax == CBM_DOCLINK_MD_SUPERSEDES) { + /* a reST ADR's own supersedes statement (doc_adr.c) */ + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, link, graph, out); + return; + } + if (link->syntax == CBM_DOCLINK_RST_CODE_PATH || link->syntax == CBM_DOCLINK_RST_CODE_NAME) { + CBMDocLink md = *link; + md.syntax = link->syntax == CBM_DOCLINK_RST_CODE_PATH ? CBM_DOCLINK_MD_CODE_PATH + : CBM_DOCLINK_MD_CODE_NAME; + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, &md, graph, out); + return; + } + rr_rec_t *r = (rr_rec_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*r)); + if (!r) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (!rr_record(link->raw, r)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + cbm_free(CBM_MEM_CLASS_OTHER, r); + return; + } + const rr_conf_t *conf = rr_conf_for(x, doc); + const char *primary = conf ? conf->primary : "py"; + char dom[RR_PATH_CAP]; + const char *rest = ""; + switch (link->syntax) { + case CBM_DOCLINK_RST_ROLE: + rr_split_domain(r->f[1], primary, RR_PY_ROLES, RR_C_ROLES, dom, sizeof(dom), &rest); + if (strcmp(dom, "py") == 0 && rr_member_of(rest, RR_PY_ROLES)) { + rr_py_role(x, graph, doc, conf, rest, r, out); + } else if (strcmp(dom, "c") == 0 && rr_member_of(rest, RR_C_ROLES)) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (!tg.bang) { + rr_c_role(graph, doc, rest, tg.t, out); + } + } else if (strcmp(dom, "std") == 0 && conf && + rr_listed(conf->ext, conf->next_count, rest)) { + rr_extlink(x, graph, r, out); + } + break; + case CBM_DOCLINK_RST_OBJECT: + rr_split_domain(r->f[1], primary, RR_PY_DOMAIN_OBJ, RR_C_DOMAIN_OBJ, dom, sizeof(dom), + &rest); + if (strcmp(dom, "py") == 0 && rr_member_of(rest, RR_PY_OBJ)) { + rr_py_object(x, graph, doc, rest, r, out); + } else if (strcmp(dom, "c") == 0 && rr_member_of(rest, RR_C_OBJ)) { + char name[RR_PATH_CAP]; + if (!rr_c_sig_name(rest, r->f[2], name, sizeof(name))) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (strcmp(rest, "member") == 0) { + rr_res_t res = rr_c_member(graph, NULL, name); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + rr_finish(out, res); + } + } else { + bool data = strcmp(rest, "var") == 0; + rr_res_t res = rr_c_name(graph, name, data, NULL); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + rr_finish(out, res); + } + } + } + break; + case CBM_DOCLINK_RST_AUTODOC: + rr_autodoc(x, graph, doc, r, out); + break; + case CBM_DOCLINK_RST_INCLUDE: { + char rel[RR_PATH_CAP]; + const char *arg = r->f[2]; + bool ok = strcmp(r->f[1], "kernel-include") == 0 + ? rr_norm("", arg, rel, sizeof(rel)) + : rr_doc_path(x, doc, arg, rel, sizeof(rel)); + if (strchr(arg, '$')) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (ok) { + const cbm_gbuf_node_t *f = rr_file(x, graph, rel, false, out); + if (f) { + rr_edge(out, f, true, 0, 0); + } + } + break; + } + case CBM_DOCLINK_RST_LITERALINCLUDE: + rr_literalinclude(x, graph, doc, r, out); + break; + case CBM_DOCLINK_RST_KERNEL_DOC: + rr_kernel_doc(x, graph, doc, r, out); + break; + default: + break; + } + cbm_free(CBM_MEM_CLASS_OTHER, r); +} + +const cbm_doclink_resolver_t cbm_doclink_rst_resolver = { + .langs = {CBM_LANG_RST, CBM_LANG_PYTHON}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG, + .build = rr_build, + .destroy = rr_destroy, + .resolve = rr_resolve, + .via = "rst", +}; diff --git a/tests/test_doc_links_rst.c b/tests/test_doc_links_rst.c new file mode 100644 index 0000000000..160dd55812 --- /dev/null +++ b/tests/test_doc_links_rst.c @@ -0,0 +1,440 @@ +/* + * test_doc_links_rst.c — reStructuredText documents -> Sections and MENTIONS + * edges (doclink_rst.c, doclink_py.c, doc_links_rst.c). + * + * The line model (titles, literal blocks, directives, comments), the tokens + * and their resolution contexts, the Python scope blob (package re-exports, + * Sphinx conf.py), the resolver's rules as the field test measured them + * (forms and re-exports, a module never answers a member's role, bare members + * are never searched, a name written in full binds even test code while a + * search leaves it out, intersphinx and other projects are no rows), the C + * domain, include / literalinclude / kernel-doc, extlinks, inline literals, + * Sphinx's source_suffix in discovery, reST ADRs, incremental == full. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include + +/* ── the scanner ─────────────────────────────────────────────────── */ + +static const char *const RST_DOC = + "Intro names :class:`pkg.Top` before any title.\n" /* 1 */ + "\n" /* 2 */ + "=========\n" /* 3 */ + " The API\n" /* 4 */ + "=========\n" /* 5 */ + "\n" /* 6 */ + ".. module:: pkg\n" /* 7 */ + "\n" /* 8 */ + "Use :class:`Model` and :meth:`~pkg.Model.save`::\n" /* 9 */ + "\n" /* 10 */ + " :class:`NotARole` in a literal block\n" /* 11 */ + " Fake Title\n" /* 12 */ + " ==========\n" /* 13 */ + "\n" /* 14 */ + ".. code-block:: python\n" /* 15 */ + "\n" /* 16 */ + " x = :func:`nope`\n" /* 17 */ + "\n" /* 18 */ + ".. class:: Model(name)\n" /* 19 */ + "\n" /* 20 */ + " .. method:: save(force=False)\n" /* 21 */ + "\n" /* 22 */ + " Saves; see :attr:`name`.\n" /* 23 */ + "\n" /* 24 */ + "Usage\n" /* 25 */ + "-----\n" /* 26 */ + "\n" /* 27 */ + ".. autofunction:: pkg.base.top\n" /* 28 */ + "\n" /* 29 */ + ".. literalinclude:: ../pkg/base.py\n" /* 30 */ + " :lines: 2-3\n" /* 31 */ + "\n" /* 32 */ + ".. include:: ../pkg/snippet.py\n" /* 33 */ + "\n" /* 34 */ + ".. include:: other.rst\n" /* 35 */ + "\n" /* 36 */ + ".. kernel-doc:: drivers/x.c\n" /* 37 */ + " :identifiers: foo bar\n" /* 38 */ + "\n" /* 39 */ + ".. a comment with :class:`Hidden`\n" /* 40 */ + "\n" /* 41 */ + "Text ``see :class:`Hidden2` there`` and ``pkg/base.py``,\n" /* 42 */ + "``pkg.base.top`` and :source:`a file `.\n"; /* 43 */ + +static int rst_count(const CBMFileResult *r, int syntax) { + int n = 0; + for (int i = 0; i < r->doc_links.count; i++) { + n += r->doc_links.items[i].syntax == syntax; + } + return n; +} + +static bool rst_raw_has(const CBMFileResult *r, const char *needle) { + for (int i = 0; i < r->doc_links.count; i++) { + if (strstr(r->doc_links.items[i].raw, needle)) { + return true; + } + } + return false; +} + +static const CBMDefinition *rst_def(const CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && strcmp(d->label, label) == 0 && d->name && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +TEST(rst_scan_structure) { + CBMFileResult *r = dm_extract(RST_DOC, CBM_LANG_RST, "docs/api.rst"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->doc_links.failed); + /* titles: over+under and under only; a title shape in a literal block is none */ + const CBMDefinition *api = rst_def(r, "Section", "The API"); + const CBMDefinition *usage = rst_def(r, "Section", "Usage"); + ASSERT_NOT_NULL(api); + ASSERT_NOT_NULL(usage); + ASSERT_NULL(rst_def(r, "Section", "Fake Title")); + ASSERT_EQ(api->start_line, 4); + ASSERT_EQ(api->end_line, 23); + ASSERT_EQ(usage->start_line, 25); + ASSERT_STR_EQ(api->qualified_name, "p.docs.api.The-API"); + ASSERT_NOT_NULL(api->docstring); + ASSERT_NOT_NULL(strstr(api->docstring, "Use :class:`Model`")); + /* nothing from a literal block, a code directive, a comment or an inline literal */ + ASSERT_FALSE(rst_raw_has(r, "NotARole")); + ASSERT_FALSE(rst_raw_has(r, "nope")); + ASSERT_FALSE(rst_raw_has(r, "Hidden")); + /* text before the first title belongs to the file */ + const CBMDocLink *top = dm_find_token(r, ":class:`pkg.Top`\tclass\tpkg.Top\t\t\t"); + ASSERT_NOT_NULL(top); + ASSERT_EQ(top->flags, CBM_DOCLINK_FLAG_FILE); + ASSERT_EQ(top->line, 1); + /* roles carry the module and class in force */ + const CBMDocLink *model = dm_find_token(r, ":class:`Model`\tclass\tModel\tpkg\t\t"); + ASSERT_NOT_NULL(model); + ASSERT_STR_EQ(model->source_qn, "p.docs.api.The-API"); + ASSERT_EQ(model->line, 9); + ASSERT_NOT_NULL(dm_find_token(r, ":attr:`name`\tattr\tname\tpkg\tModel\t")); + /* object directives: the signature and Python's full name in its class */ + ASSERT_NOT_NULL(dm_find_token(r, ".. class:: Model(name)\tclass\tModel(name)\tpkg\t\t\tModel")); + ASSERT_NOT_NULL(dm_find_token( + r, ".. method:: save(force=False)\tmethod\tsave(force=False)\tpkg\tModel\t\tModel.save")); + ASSERT_NOT_NULL(dm_find_token(r, ".. module:: pkg\tmodule\tpkg\tpkg\t\t\t")); + /* autodoc, literalinclude, an include of code (not of a document), kernel-doc names */ + ASSERT_NOT_NULL( + dm_find_token(r, ".. autofunction:: pkg.base.top\tautofunction\tpkg.base.top\tpkg\t")); + ASSERT_NOT_NULL( + dm_find_token(r, ".. literalinclude:: ../pkg/base.py\t../pkg/base.py\t2-3\t\t")); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_INCLUDE), 1); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_KERNEL_DOC), 3); + ASSERT_NOT_NULL(dm_find_token(r, ".. kernel-doc:: drivers/x.c\tdrivers/x.c\tbar")); + /* inline literals naming a path or a qualified name; a titled extlink */ + ASSERT_NOT_NULL(dm_find_token(r, "pkg/base.py")); + ASSERT_EQ(dm_find_token(r, "pkg/base.py")->syntax, CBM_DOCLINK_RST_CODE_PATH); + ASSERT_EQ(dm_find_token(r, "pkg.base.top")->syntax, CBM_DOCLINK_RST_CODE_NAME); + ASSERT_NOT_NULL( + dm_find_token(r, ":source:`a file `\tsource\ta file \tpkg\t\t")); + cbm_free_result(r); + PASS(); +} + +/* ── Python's scope ──────────────────────────────────────────────── */ + +TEST(rst_python_scope_blob) { + CBMFileResult *r = dm_extract("from .base import Model\n" + "from .sub.thing import helper as aid, other\n" + "from ..up import x\n" + "from os import *\n" + "from django.db import models\n" + "try:\n" + " from .opt import Nested\n" + "except ImportError:\n" + " pass\n", + CBM_LANG_PYTHON, "pkg/__init__.py"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "I\t1\tbase\tModel\tModel\n" + "I\t1\tsub.thing\thelper\taid\n" + "I\t1\tsub.thing\tother\tother\n" + "I\t2\tup\tx\tx\n" + "I\t0\tdjango.db\tmodels\tmodels\n"); + ASSERT_EQ(r->doc_links.count, 0); + cbm_free_result(r); + r = dm_extract("project = 'x'\n" + "primary_domain = 'c'\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + " \"sphinx\": (\"https://www.sphinx-doc.org/en/master/\", None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + " 'issue': ('https://github.com/o/r/issues/%s', '#%s'),\n" + "}\n" + "extlinks['tree'] = ('https://github.com/o/r/tree/main/%s', '%s')\n", + CBM_LANG_PYTHON, "docs/conf.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "C\n" + "D\tc\n" + "S\tpython\n" + "S\tsphinx\n" + "X\tsource\n" + "X\ttree\n"); + cbm_free_result(r); + r = dm_extract("from .x import y\n", CBM_LANG_PYTHON, "pkg/mod.py"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(r->doc_scope); + cbm_free_result(r); + PASS(); +} + +/* ── the pipeline ────────────────────────────────────────────────── */ + +static const char *const API_RST = + "Models\n" /* 1 */ + "======\n" /* 2 */ + "\n" /* 3 */ + ".. module:: pkg\n" /* 4 */ + "\n" /* 5 */ + "Use :class:`Model`, :func:`pkg.aid` and :meth:`save`.\n" /* 6 */ + "Not :class:`python:dict`, :class:`otherlib.Thing`; gone\n" /* 7 */ + ":class:`pkg.Missing`.\n" /* 8 */ + "\n" /* 9 */ + ".. currentmodule:: pkg.views\n" /* 10 */ + "\n" /* 11 */ + ".. class:: View\n" /* 12 */ + "\n" /* 13 */ + " Calls :meth:`dispatch`.\n" /* 14 */ + "\n" /* 15 */ + "Testing\n" /* 16 */ + "=======\n" /* 17 */ + "\n" /* 18 */ + "Run :class:`pkg.test.runner.Runner`, not :class:`Runner`.\n" /* 19 */ + "\n" /* 20 */ + "Code\n" /* 21 */ + "====\n" /* 22 */ + "\n" /* 23 */ + "See :source:`pkg/base.py`, ``pkg.base.top`` and :c:func:`c_func`.\n" /* 24 */ + "\n" /* 25 */ + ".. autofunction:: pkg.base.top\n" /* 26 */ + "\n" /* 27 */ + ".. literalinclude:: ../pkg/base.py\n" /* 28 */ + " :lines: 2-3\n" /* 29 */ + "\n" /* 30 */ + ".. include:: /../pkg/sub/thing.py\n" /* 31 */ + "\n" /* 32 */ + ".. kernel-doc:: csrc/lib.c\n" /* 33 */ + " :identifiers: c_func\n"; /* 34 */ + +static void rst_write_repo(const char *repo, const char *init) { + th_write_file(TH_PATH(repo, "pkg/__init__.py"), init); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " def save(self):\n" + " return None\n" + "\n" + "\n" + "def top():\n" + " return 1\n"); + th_write_file(TH_PATH(repo, "pkg/other.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "pkg/sub/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/sub/thing.py"), "def helper():\n" + " return 2\n"); + th_write_file(TH_PATH(repo, "pkg/views.py"), "class View:\n" + " def dispatch(self):\n" + " return None\n"); + th_write_file(TH_PATH(repo, "pkg/test/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/test/runner.py"), "class Runner:\n" + " pass\n"); + /* tests/ is no package: tests/dispatch's import name is `dispatch` */ + th_write_file(TH_PATH(repo, "tests/dispatch/__init__.py"), ""); + th_write_file(TH_PATH(repo, "tests/dispatch/test_x.py"), "def test_x():\n" + " return None\n"); + th_write_file(TH_PATH(repo, "csrc/lib.c"), "int c_func(void) {\n" + " return 0;\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/conf.py"), + "project = 'x'\n" + "source_suffix = {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/api.rst"), API_RST); + th_write_file(TH_PATH(repo, "docs/topics/guide.txt"), "Guide\n" + "=====\n" + "\n" + "The :class:`pkg.views.View`.\n"); + th_write_file(TH_PATH(repo, "notes/readme.txt"), "Title\n" + "=====\n" + "\n" + "Plain :class:`pkg.views.View`.\n"); +} + +static const char *const INIT_PY = "from .base import Model\n" + "from .sub.thing import helper as aid\n"; + +static bool rst_edge(const char *db, const char *src, const char *tgt, const char *needle) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + return n == 1 && (!needle || strstr(props, needle)); +} + +static int rst_rows(const char *db, const char *raw_prefix, const char *reason) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE '%s%%' AND reason='%s'", + raw_prefix, reason); + return dm_count(db, sql); +} + +TEST(rst_links_pipeline) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a re-export through the module context: pkg.Model is pkg/base.py's */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", + "{\"via\":\"rst\",\"syntax\":\"role\",\"tier\":\"exact\",\"line\":6")); + /* an aliased re-export: pkg.aid is helper */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.sub.thing.helper", "\"tier\":\"exact\"")); + /* a bare member is never searched; intersphinx and another project: no rows */ + ASSERT_EQ(dm_count(db, + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE ':meth:`save`%' " + "OR raw LIKE ':class:`python:dict`%' OR raw LIKE " + "':class:`otherlib.Thing`%'"), + 0); + ASSERT_EQ(rst_rows(db, ":class:`pkg.Missing`", "missing"), 1); + /* a module never answers :meth: -- the class context does */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View.dispatch", "\"tier\":\"exact\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path LIKE 'tests/%'"), + 0); + /* the object directive declares View in its module */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\"")); + /* a name written in full binds test code; a search for it does not */ + ASSERT_TRUE(rst_edge(db, "docs.api.Testing", "pkg.test.runner.Runner", "\"tier\":\"exact\"")); + ASSERT_EQ(rst_rows(db, ":class:`Runner`", "test_only_target"), 1); + /* extlink, inline literal, C domain, autodoc */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.py.__file__", "\"syntax\":\"role\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.top", "\"syntax\":\"code_name\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c_func", NULL)); + /* literalinclude :lines: the innermost definition, the lines on the edge */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* include from the documentation set's directory; kernel-doc file */ + ASSERT_TRUE( + rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", "\"syntax\":\"include\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c.__file__", "\"syntax\":\"kernel_doc\"")); + /* conf.py's source_suffix: .txt is reST in its documentation set only */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND " + "file_path='docs/topics/guide.txt' AND name='Guide'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.topics.guide.Guide", "pkg.views.View", "\"via\":\"rst\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE file_path='notes/readme.txt'"), 0); + /* back navigation: the class knows the sections that name it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.name='View' AND t.label='Class' AND s.label='Section'"), + 2); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_adr_record) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_adr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "docs/decisions/0002-use-models.rst"), "2. Use models\n" + "=============\n" + "\n" + "Status\n" + "------\n" + "\n" + "Accepted\n" + "\n" + "Context\n" + "-------\n" + "\n" + "We store records.\n" + "\n" + "Decision\n" + "--------\n" + "\n" + "Use ``pkg.base.Model``.\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='ADR' AND name='ADR-2' AND " + "json_extract(properties,'$.status')='accepted'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.decisions.0002-use-models.Decision", "pkg.base.Model", + "\"syntax\":\"code_name\"")); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_links_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* the document changes alone: it re-resolves through the stored re-exports */ + th_write_file(TH_PATH(repo, "docs/api.rst"), "Models\n" + "======\n" + "\n" + ".. module:: pkg\n" + "\n" + "Only :class:`Model` and :func:`aid`.\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", NULL)); + /* the package re-exports another Model: the unchanged document follows */ + th_write_file(TH_PATH(repo, "pkg/__init__.py"), "from .other import Model\n" + "from .sub.thing import helper as aid\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "re-export change", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.other.Model", NULL)); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_rst) { + RUN_TEST(rst_scan_structure); + RUN_TEST(rst_python_scope_blob); + RUN_TEST(rst_links_pipeline); + RUN_TEST(rst_adr_record); + RUN_TEST(rst_links_incremental); +} diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 7818c4c445..7246e5b958 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -195,7 +195,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"css", "Module:1"}, {"scss", "Module:1,Variable:1"}, {"markdown", "Module:1,Section:1"}, - {"rst", "Module:1"}, + {"rst", "Module:1,Section:1"}, {"dockerfile", "Module:1"}, {"makefile", "Function:1,Module:1"}, {"cmake", "Function:1,Module:1"}, diff --git a/tests/test_grammar_probe_g.c b/tests/test_grammar_probe_g.c index d0f99d4e0e..215b4acee0 100644 --- a/tests/test_grammar_probe_g.c +++ b/tests/test_grammar_probe_g.c @@ -399,12 +399,13 @@ TEST(probe_markdown_no_class_variable) { /* ══════════════════════════════════════════════════════════════════ * GROUP 6 — RST (.rst) * - * RST golden histogram: Module:1 (pure-data). - * reStructuredText headings are NOT extracted as Section nodes (unlike Markdown). + * RST golden histogram: Module:1,Section:1. + * reStructuredText titles are Section nodes, like Markdown headings + * (doclink_rst.c). * ══════════════════════════════════════════════════════════════════ */ -/* RST: document with sections → Module:1, no Section nodes. */ -TEST(probe_rst_module_only) { +/* RST: a document with two titles → Module:1 and a Section per title. */ +TEST(probe_rst_sections) { GpgMetrics m = gpg_metrics("docs.rst", "Project Title\n" "=============\n" "\n" @@ -415,9 +416,9 @@ TEST(probe_rst_module_only) { "\n" "Run ``pip install mypackage``.\n"); ASSERT_TRUE(m.ok); - /* GREEN: RST produces only a Module node (no Section nodes extracted). */ + /* GREEN: a Module node and the two titles' Section nodes. */ ASSERT_TRUE(m.modules == 1); - ASSERT_TRUE(m.sections == 0); + ASSERT_TRUE(m.sections == 2); PASS(); } @@ -1063,7 +1064,7 @@ SUITE(grammar_probe_g) { RUN_TEST(probe_markdown_no_class_variable); /* RST */ - RUN_TEST(probe_rst_module_only); + RUN_TEST(probe_rst_sections); RUN_TEST(probe_rst_no_crash); /* JSON */ diff --git a/tests/test_main.c b/tests/test_main.c index cad85c4bac..542df5b48e 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1823,6 +1823,7 @@ extern void suite_doc_mentions_cost(void); extern void suite_doc_mentions_msbuild(void); extern void suite_doc_links_md(void); extern void suite_doc_links_pdf(void); +extern void suite_doc_links_rst(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2292,6 +2293,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_msbuild); RUN_SELECTED_SUITE(doc_links_md); RUN_SELECTED_SUITE(doc_links_pdf); + RUN_SELECTED_SUITE(doc_links_rst); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); From c2c87f6202d91649414225e0c8b56865159d4b1f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 00:20:54 +0200 Subject: [PATCH 05/25] incremental(doc-links): documents bound by lines re-resolve on a body edit The closure route treats a body-only edit (the file's surface, which holds no line numbers, unchanged) as "the file re-resolves, nobody else does". That holds for references that bind names, but a document can bind lines: Markdown `app/routing.py#L8-L9` and reST `.. literalinclude:: :lines:` bind the innermost definition holding those lines. A body edit that moves code left such a document on its old target on an incremental run, while a full index bound the new one (observed: the doc_links_md and doc_links_rst "lines moved" steps, incremental != full). The documents (Markdown, reST: cbm_doclinks_binds_lines) with edges into a body-edited file now join the repair closure, from one more dependent-files query over the body-edited files. Code files stay out: their references bind names, which a body edit keeps. The query shares the existing dependents query's block and failure path (no new allocation site). Tests: a "lines moved" step in doc_links_md_incremental and in rst_links_incremental; with the predicate disabled exactly these two steps fail (observed). doc_links_md/rst/pdf, doc_mentions, incremental, pipeline pass. Signed-off-by: Martin Vogel --- src/pipeline/doc_links.c | 4 ++ src/pipeline/doc_links.h | 6 ++ src/pipeline/pipeline_incremental.c | 45 ++++++++++-- tests/test_doc_links_md.c | 105 ++++++++++++++++------------ tests/test_doc_links_rst.c | 14 ++++ 5 files changed, 121 insertions(+), 53 deletions(-) diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index f3b8f228d0..71159220d6 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -92,6 +92,10 @@ static const cbm_doclink_resolver_t *resolver_of_scope(const char *scope) { /* ── Incremental scope rules ─────────────────────────────────────── */ +bool cbm_doclinks_binds_lines(CBMLanguage lang) { + return lang == CBM_LANG_MARKDOWN || lang == CBM_LANG_RST; +} + bool cbm_doclinks_is_scope_input(const char *rel_path) { for (int i = 0; rel_path && i < DOCLINK_RESOLVER_COUNT; i++) { if (DOCLINK_RESOLVERS[i]->scope_input && DOCLINK_RESOLVERS[i]->scope_input(rel_path)) { diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 8ad2384d41..ec1416f274 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -277,6 +277,12 @@ bool cbm_doclink_cs_test_scope_parses(const char *scope); void cbm_doclink_cs_test_unit_memo(bool on); #endif +/* True for a document language whose references may bind the LINES of a file + * (Markdown `file#L3-L9`, reST :lines:): a body edit of that file keeps its + * names but can move what those lines hold, so an incremental run + * re-resolves such documents with the edited file. */ +bool cbm_doclinks_binds_lines(CBMLanguage lang); + /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 03ccf415bb..bc3beb7e62 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2363,6 +2363,12 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p char **changed_paths = NULL; char **dependents = NULL; int dependent_count = 0; + /* body-edited files (surface unchanged) and the documents bound to their + * lines, which re-resolve with them */ + const char **body_targets = NULL; + int body_target_count = 0; + char **line_docs = NULL; + int line_doc_count = 0; cbm_lsp_surface_row_t *probe_rows = NULL; int probe_count = 0; cbm_file_info_t *probe_files = NULL; @@ -2529,13 +2535,15 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p /* Early cutoff per changed file; surface-changed files seed the * dependent query. Deleted files always seed it. */ int n_surface_changed = 0; + /* dep_targets, then the body-edited files (body_targets) in the same block */ const char **dep_targets = - (const char **)calloc((size_t)(n_changed + n_deleted), sizeof(char *)); + (const char **)calloc((size_t)(n_changed + n_deleted + n_changed), sizeof(char *)); if (!dep_targets) { cbm_ht_free(rows_by_path); decline = "alloc"; goto done; } + body_targets = dep_targets + n_changed + n_deleted; int dep_target_count = 0; for (int i = 0; i < n_changed; i++) { const cbm_lsp_surface_row_t *stored_row = cbm_ht_get(rows_by_path, changed_paths[i]); @@ -2553,7 +2561,10 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } if (strcmp(stored_row->surface_sha, fresh_row->surface_sha) == 0) { - continue; /* body edit: the file re-resolves, nobody else does */ + /* body edit: the file re-resolves, and of the others only the + * documents bound to its lines (below) */ + body_targets[body_target_count++] = changed_paths[i]; + continue; } bool added = false; const char *why = NULL; @@ -2583,14 +2594,23 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p } cbm_ht_free(rows_by_path); - if (dep_target_count > 0 && - cbm_store_get_dependent_files(store, project, dep_targets, dep_target_count, &dependents, - &dependent_count) != CBM_STORE_OK) { - free(dep_targets); + /* A body edit keeps a file's names but moves its lines: a document that + * bound a reference by lines (`file#L3-L9`, :lines:) would keep a stale + * target, so the documents with edges into a body-edited file re-resolve + * too (added to the closure below). */ + bool dep_failed = + (dep_target_count > 0 && + cbm_store_get_dependent_files(store, project, dep_targets, dep_target_count, &dependents, + &dependent_count) != CBM_STORE_OK) || + (body_target_count > 0 && + cbm_store_get_dependent_files(store, project, body_targets, body_target_count, &line_docs, + &line_doc_count) != CBM_STORE_OK); + free(dep_targets); /* body_targets with it */ + body_targets = NULL; + if (dep_failed) { decline = "dependent_query_failed"; goto done; } - free(dep_targets); if (gone_scope_changed) { decline = "doc_scope_changed"; goto done; @@ -2618,6 +2638,16 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p if (decline) { goto done; } + /* the documents bound to a body-edited file's lines; code files stay out: + * their references bind names, which a body edit keeps */ + for (int i = 0; i < line_doc_count; i++) { + const cbm_file_info_t *info = cbm_ht_get(files_by_path, line_docs[i]); + if (info && cbm_doclinks_binds_lines(info->language) && + !cbm_ht_get(closure_set, line_docs[i])) { + cbm_ht_set(closure_set, line_docs[i], line_docs[i]); + n_row_dependents++; + } + } /* Files whose unresolved doc-link references name something a changed or * deleted file no longer declares re-resolve too. */ doc_row_name_dependents(plan->doc_rows, plan->doc_row_count, removed_names, row_deps); @@ -2665,6 +2695,7 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p free(probe_files); cbm_store_free_lsp_surfaces(probe_rows, probe_count); cbm_store_free_dependent_files(dependents, dependent_count); + cbm_store_free_dependent_files(line_docs, line_doc_count); free(changed_paths); cbm_ht_free(fresh_by_path); cbm_ht_free(files_by_path); diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 4ec817f8a4..0a63f735e6 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -24,34 +24,34 @@ /* ── the fixture ─────────────────────────────────────────────────── */ /* docs/guide.md, line by line (the line numbers are asserted below). */ -static const char - MD_GUIDE[] = "Intro names `pkg/config.go` before any heading.\n" /* 1 */ - "\n" /* 2 */ - "# Guide\n" /* 3 */ - "\n" /* 4 */ - "See [the config](../pkg/config.go) and [routing][r].\n" /* 5 */ - "\n" /* 6 */ - "[r]: ../app/routing.py\n" /* 7 */ - "\n" /* 8 */ - "## Paths\n" /* 9 */ - "\n" /* 10 */ - "Range `app/routing.py#L4-L5`, span `app/routing.py:1-9`, bare " - "app/applications.py.\n" /* 11 */ - "Dir `app/middleware/`, route `/docs`, ![x](img/a.png) and " - "https://example.org/a/b.py.\n" /* 12 */ - "\n" /* 13 */ - "## Names\n" /* 14 */ - "\n" /* 15 */ - "Use `app.routing.APIRouter.add`, `app.middleware`, `pkg.Config.Name`, " - "`self.Name`.\n" /* 16 */ - "Also `os.path.join`, `app.Nope`, `test_routing.only_in_tests`, `main.py`, " - "`../app/gone.py`.\n" /* 17 */ - "\n" /* 18 */ - "```\n" /* 19 */ - "`app/fenced.py` inside a fence\n" /* 20 */ - "```\n" /* 21 */ - "\n" /* 22 */ - "app/html.py in HTML, `cfg.Name` through an instance.\n"; /* 23 */ +static const char MD_GUIDE[] = + "Intro names `pkg/config.go` before any heading.\n" /* 1 */ + "\n" /* 2 */ + "# Guide\n" /* 3 */ + "\n" /* 4 */ + "See [the config](../pkg/config.go) and [routing][r].\n" /* 5 */ + "\n" /* 6 */ + "[r]: ../app/routing.py\n" /* 7 */ + "\n" /* 8 */ + "## Paths\n" /* 9 */ + "\n" /* 10 */ + "Range `app/routing.py#L4-L5`, span `app/routing.py:1-9`, bare " + "app/applications.py.\n" /* 11 */ + "Dir `app/middleware/`, route `/docs`, ![x](img/a.png) and " + "https://example.org/a/b.py.\n" /* 12 */ + "\n" /* 13 */ + "## Names\n" /* 14 */ + "\n" /* 15 */ + "Use `app.routing.APIRouter.add`, `app.middleware`, `pkg.Config.Name`, " + "`self.Name`.\n" /* 16 */ + "Also `os.path.join`, `app.Nope`, `test_routing.only_in_tests`, `main.py`, " + "`../app/gone.py`.\n" /* 17 */ + "\n" /* 18 */ + "```\n" /* 19 */ + "`app/fenced.py` inside a fence\n" /* 20 */ + "```\n" /* 21 */ + "\n" /* 22 */ + "app/html.py in HTML, `cfg.Name` through an instance.\n"; /* 23 */ static void md_write_fixture(const char *tmp) { th_write_file(TH_PATH(tmp, "docs/guide.md"), MD_GUIDE); @@ -400,6 +400,20 @@ TEST(doc_links_md_incremental) { snprintf(guide, sizeof(guide), "%s\nSee also [helpers](../app/routing.py#L8-L9).\n", MD_GUIDE); th_write_file(TH_PATH(repo, "docs/guide.md"), guide); ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* a body edit that moves lines: the document bound by lines (#L8-L9, the + * helper until now) re-resolves, though no name changed */ + th_write_file(TH_PATH(repo, "app/routing.py"), "# moved\n" + "# down\n" + "class APIRouter:\n" + " prefix = ''\n" + "\n" + " def add(self, p):\n" + " return p + p\n" + "\n" + "\n" + "def helper():\n" + " return 2\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); /* the name a row calls missing is added: the run goes FULL and the row * becomes an edge, exactly as a full index has it */ th_write_file(TH_PATH(repo, "app/nope.py"), "def Nope():\n" @@ -433,24 +447,23 @@ static void md_write_adrs(const char *tmp, const char *first_status) { "Superseded by [ADR-0002](0002-use-y.md).\n", first_status); th_write_file(TH_PATH(tmp, "docs/adr/0001-use-x.md"), first); - th_write_file(TH_PATH(tmp, "docs/adr/0002-use-y.md"), - "# 2. Use Y instead\n" - "\n" - "Date: 12 May 2024\n" - "\n" - "## Status\n" - "\n" - "Accepted\n" - "\n" - "Supersedes [ADR-0001](0001-use-x.md).\n" - "\n" - "## Context\n" - "\n" - "X was slow.\n" - "\n" - "## Decision\n" - "\n" - "We use Y.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0002-use-y.md"), "# 2. Use Y instead\n" + "\n" + "Date: 12 May 2024\n" + "\n" + "## Status\n" + "\n" + "Accepted\n" + "\n" + "Supersedes [ADR-0001](0001-use-x.md).\n" + "\n" + "## Context\n" + "\n" + "X was slow.\n" + "\n" + "## Decision\n" + "\n" + "We use Y.\n"); th_write_file(TH_PATH(tmp, "docs/adr/0003-cache.md"), "# Cache reads\n" "\n" diff --git a/tests/test_doc_links_rst.c b/tests/test_doc_links_rst.c index 160dd55812..98afb9a959 100644 --- a/tests/test_doc_links_rst.c +++ b/tests/test_doc_links_rst.c @@ -413,6 +413,20 @@ TEST(rst_links_incremental) { snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); rst_write_repo(repo, INIT_PY); ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* a body edit moves the lines a literalinclude names: the document + * re-resolves with the code file (its names did not change) */ + th_write_file(TH_PATH(repo, "pkg/base.py"), "# moved\n" + "# down\n" + "class Model:\n" + " def save(self):\n" + " return None\n" + "\n" + "\n" + "def top():\n" + " return 1\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_FALSE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); /* the document changes alone: it re-resolves through the stored re-exports */ th_write_file(TH_PATH(repo, "docs/api.rst"), "Models\n" "======\n" From 92f5edfe32aa913a6898f8f4f102ad1fdb1ce33a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 01:19:28 +0200 Subject: [PATCH 06/25] doc-links(asciidoc): AsciiDoc sections linked to the code they include and name AsciiDoc (.adoc, .asciidoc) is now indexed, without a grammar like PDF: a Section per heading (`=` .. `======`, or `#`), the text up to the next heading as its docstring, delimited blocks respected (listing, literal, comment, passthrough, example, sidebar, quote, table, fences). Its references become MENTIONS edges, via "asciidoc", from the section they are written in: include include::target[...] -- Asciidoctor's preprocessor directive, read in every block but a comment block; the page's own attributes, then the Antora component's and the playbook's, substituted attribute an {attribute} whose value is an Antora javadoc link ({javadoc-root}///.html[#member]) binds the Java or Kotlin type, or its one member, that the repository's main sources declare under that path code_path, monospace spans naming a path or a qualified name (the code_name Markdown code-span rules) The rules are the field test's (H8 Asciidoctor/Antora harvester, audited ~100 % precise): an Antora resource ID ([component:][module:]family$path) names /modules///, or the file a collector scan contributes there (a scan of build output is generated: no row); any other target is relative to the including file. `lines=` and `tag=`/`tags=` regions bind by H8's rule: edge blank and comment lines trimmed, the innermost enclosing definition, unless that is a type (or there is none) and the definitions inside cover every other line (an annotation right above one counts) -- then that definition; tag regions come from the included file's own `tag::x[]` / `end::x[]` comments, read from that indexed file; a tag the file lacks binds the file. One reference makes one edge: a region holding several top-level definitions binds the file. Antora settings travel as YAML scope blobs (antora.yml: component name, asciidoc attributes, collector scans; antora-playbook*.yml: attributes), read as text, so an incremental run resolves as a full one; a change to them is GLOBAL. AsciiDoc joins the documents bound by lines: a body edit of an included file re-resolves its pages. Also: the Markdown resolver's module index no longer takes AsciiDoc or PDF files for code modules. Proof on junit-team/junit-framework at 6c8ca116 (H8's corpus and commit), C against H8's recorded links: javadoc attributes 362/362 identical; includes 292 identical, 60 bind the type enclosing the several definitions H8 names, 7 regions with several top-level definitions bind the file. 317 sections, 668 edges, no include rows (9 generated partials, 1 file type the indexer does not read); monospace spans 30/30 correct by hand. 8.3 s for the whole index. Tests: suite doc_links_adoc (structure and tokens, the Antora blob, the pipeline: resource IDs through a collector scan, tag regions, a region with its imports binding its class, lines=, javadoc attributes, spans; incremental == full: a moved tag region, a changed component attribute). Revert check: without H8's region rule and without AsciiDoc among the line-bound documents exactly their assertions fail (observed). Signed-off-by: Martin Vogel --- Makefile.cbm | 4 +- internal/cbm/cbm.c | 19 + internal/cbm/cbm.h | 1 + internal/cbm/doc_adoc.c | 944 +++++++++++++++++++++++++ internal/cbm/doclink.c | 5 + internal/cbm/doclink.h | 21 + internal/cbm/lang_specs.c | 3 + src/discover/language.c | 4 + src/discover/userconfig.c | 1 + src/pipeline/doc_links.c | 8 +- src/pipeline/doc_links.h | 15 +- src/pipeline/doc_links_adoc.c | 919 ++++++++++++++++++++++++ src/pipeline/doc_links_md.c | 228 +++++- tests/repro/repro_call_node_manifest.c | 5 +- tests/repro/repro_language_registry.c | 6 +- tests/test_doc_links_adoc.c | 287 ++++++++ tests/test_main.c | 2 + 17 files changed, 2459 insertions(+), 13 deletions(-) create mode 100644 internal/cbm/doc_adoc.c create mode 100644 src/pipeline/doc_links_adoc.c create mode 100644 tests/test_doc_links_adoc.c diff --git a/Makefile.cbm b/Makefile.cbm index 9e1a7ac94c..cd5eb41240 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -380,6 +380,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink_py.c \ $(CBM_DIR)/doc_adr.c \ $(CBM_DIR)/doc_pdf.c \ + $(CBM_DIR)/doc_adoc.c \ $(CBM_DIR)/pdf/pdf_obj.c \ $(CBM_DIR)/pdf/pdf_font.c \ $(CBM_DIR)/pdf/pdf_text.c \ @@ -510,6 +511,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_md.c \ src/pipeline/doc_links_pdf.c \ src/pipeline/doc_links_rst.c \ + src/pipeline/doc_links_adoc.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -772,7 +774,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index c740953f60..b9e3237b28 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2821,6 +2821,25 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_index_mark_done(rel_path); return result; } + /* AsciiDoc has no grammar either: its headings, includes, attribute + * references and monospace spans are read from the text (doc_adoc.c). */ + if (language == CBM_LANG_ASCIIDOC) { + result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language); + CBMExtractCtx adoc_ctx = { + .arena = a, + .scratch = scratch, + .result = result, + .source = source, + .source_len = source_len, + .language = language, + .project = project, + .rel_path = rel_path, + .module_qn = result->module_qn, + }; + cbm_adoc_extract_document(&adoc_ctx); + cbm_index_mark_done(rel_path); + return result; + } // Get language spec const CBMLangSpec *spec = cbm_lang_spec(language); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 0cb9c323cf..b1868baec4 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -210,6 +210,7 @@ typedef enum { CBM_LANG_PLSQL, // Oracle PL/SQL CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language) CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c) + CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c) CBM_LANG_COUNT } CBMLanguage; diff --git a/internal/cbm/doc_adoc.c b/internal/cbm/doc_adoc.c new file mode 100644 index 0000000000..bfe70b69ae --- /dev/null +++ b/internal/cbm/doc_adoc.c @@ -0,0 +1,944 @@ +/* + * doc_adoc.c — AsciiDoc documents in the graph (the extraction half; + * src/pipeline/doc_links_adoc.c resolves), and the Antora scope blob. + * + * AsciiDoc has no grammar here: like a PDF, the file is read directly. Each + * heading (`= Title` .. `====== Title`, or `#` .. `######`) outside a + * delimited block becomes a Section (name the title, the text up to the next + * heading as its docstring, its lines up to the next heading); text before + * the first heading belongs to the file. Delimited blocks (`----` listing, + * `....` literal, `////` comment, `++++` passthrough, `====` example, `****` + * sidebar, `____` quote, `|===` table, ``` fences) open and close on the same + * delimiter line (the field test's H8 rules). + * + * References become tokens whose `raw` is the reference as written, then + * TAB-separated fields: + * include include::target[attrs] (Asciidoctor's preprocessor directive: + * read in every block but a comment block; `\include::` is + * escaped): written, target with the page's attributes + * substituted (references the page does not define stay as + * `{name}` for the component's), tags (`tag=` / `tags=`), + * lines (`lines=`) + * attribute an attribute reference `{name}` in the text: written, name, + * the page's value as written and as the page expands it (both + * "" when the page does not define it). + * Antora components define their API links as attributes + * (`{Assertions}` -> a javadoc page): the resolver reads them. + * code_path, a monospace span (`x`, ``x``, `+x+`) naming a path or a + * code_name qualified name (the Markdown code-span classifier); raw is + * the span's text + * Page attributes (`:name: value`, `:name!:` / `:!name:` to unset) apply from + * their line on, in document order. + * + * The Antora scope (cbm_doclink_antora_scan_scope): an antora.yml's component + * name, its asciidoc attributes and its collector scans; an + * antora-playbook*.yml's asciidoc attributes. Read as text, never run. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "helpers.h" + +#include +#include +#include +#include +#include + +enum { + AD_HEAD_MAX = 6, /* `======` */ + AD_DELIM_MIN = 4, /* `----` and its kin */ + AD_TABLE_MIN = 3, /* `|===` */ + AD_FENCE_MIN = 3, /* ``` */ + AD_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */ + AD_SPAN_MAX = 150, /* a span the classifier reads (MD_SPAN_MAX) */ + AD_ATTRS_INIT = 16, /* first capacity of the page attribute list */ + AD_EXPAND_DEPTH = 10, /* nested attribute references expanded (H8) */ + AD_FIELD_MAX = 512, /* an attribute value or target longer than this is skipped */ +}; + +typedef struct { + const char *s; + int len; +} ad_line_t; + +typedef struct { + char *name; + char *value; +} ad_attr_t; + +typedef struct { + int line; /* 0-based */ + const char *title; + const char *qn; + bool shared_qn; +} ad_head_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *scratch; + ad_line_t *L; + int n; + unsigned char *in_block; /* 1: inside a delimited block; 2: inside a comment block */ + ad_head_t *heads; + int nheads; + ad_attr_t *attrs; + int nattrs; + int cap_attrs; + char *span_buf; + bool failed; +} ad_doc_t; + +static bool ad_blank(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +static bool ad_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The line without trailing whitespace: its length. */ +static int ad_rlen(const ad_line_t *l) { + int n = l->len; + while (n > 0 && ad_blank(l->s[n - 1])) { + n--; + } + return n; +} + +/* ADOC_DELIM_RE: the delimiter's length, or 0. */ +static int ad_delim(const ad_line_t *l) { + int n = ad_rlen(l); + if (n == 0) { + return 0; + } + const char *s = l->s; + if (s[0] == '|') { + int k = 1; + while (k < n && s[k] == '=') { + k++; + } + return (k == n && n - 1 >= AD_TABLE_MIN) ? n : 0; + } + if (!strchr("-.=*_+/`", s[0])) { + return 0; + } + for (int k = 1; k < n; k++) { + if (s[k] != s[0]) { + return 0; + } + } + int min = s[0] == '`' ? AD_FENCE_MIN : AD_DELIM_MIN; + return n >= min ? n : 0; +} + +/* ADOC_HEAD_RE ^(={1,6}|#{1,6})\s+(\S.*?)\s*$: the level and the title span. */ +static int ad_heading(const ad_line_t *l, int *ta, int *tb) { + const char *s = l->s; + int n = ad_rlen(l); + if (n == 0 || (s[0] != '=' && s[0] != '#')) { + return 0; + } + int k = 0; + while (k < n && s[k] == s[0]) { + k++; + } + if (k > AD_HEAD_MAX || k >= n || !ad_blank(s[k])) { + return 0; + } + int a = k; + while (a < n && ad_blank(s[a])) { + a++; + } + if (a >= n) { + return 0; + } + *ta = a; + *tb = n; + return k; +} + +static bool ad_lines(ad_doc_t *d, const char *src, int len) { + int count = 1; + for (int i = 0; i < len; i++) { + count += src[i] == '\n'; + } + d->L = (ad_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L)); + d->in_block = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count); + if (!d->L || !d->in_block) { + return false; + } + memset(d->in_block, 0, (size_t)count); + int pos = 0; + for (int i = 0; i < count; i++) { + const char *nl = memchr(src + pos, '\n', (size_t)(len - pos)); + int end = nl ? (int)(nl - src) : len; + d->L[i] = (ad_line_t){src + pos, end - pos}; + pos = end + 1; + } + d->n = count; + return true; +} + +/* Delimited blocks, then the headings outside them. */ +static void ad_structure(ad_doc_t *d) { + int open = -1; + for (int i = 0; i < d->n; i++) { + int dl = ad_delim(&d->L[i]); + if (open < 0) { + if (dl) { + open = i; + d->in_block[i] = d->L[i].s[0] == '/' ? 2 : 1; + } + continue; + } + const ad_line_t *o = &d->L[open]; + d->in_block[i] = d->L[open].s[0] == '/' ? 2 : 1; + if (dl && dl == ad_rlen(o) && memcmp(d->L[i].s, o->s, (size_t)dl) == 0) { + open = -1; + } + } + d->heads = (ad_head_t *)cbm_arena_alloc(d->scratch, (size_t)d->n * sizeof(*d->heads)); + if (!d->heads) { + d->failed = true; + return; + } + for (int i = 0; i < d->n; i++) { + int ta; + int tb; + if (d->in_block[i] || ad_heading(&d->L[i], &ta, &tb) == 0) { + continue; + } + char *title = cbm_arena_strndup(d->ctx->arena, d->L[i].s + ta, (size_t)(tb - ta)); + if (!title) { + d->failed = true; + return; + } + d->heads[d->nheads++] = (ad_head_t){.line = i, .title = title}; + } +} + +/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */ +static const char *ad_qn_segment(CBMArena *a, const char *name) { + char *out = cbm_arena_strdup(a, name); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = false; + for (const char *p = name; *p; p++) { + if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = '-'; + } + in_ws = false; + out[w++] = *p; + } + out[w] = '\0'; + return out; +} + +/* The section's own text, whitespace collapsed, at most AD_BODY_MAX bytes and + * never a cut UTF-8 character. */ +static const char *ad_body(ad_doc_t *d, int from, int to) { + char *out = (char *)cbm_arena_alloc(d->ctx->arena, AD_BODY_MAX + 1); + if (!out) { + return NULL; + } + int w = 0; + for (int k = from; k < to && w < AD_BODY_MAX; k++) { + const char *s = d->L[k].s; + int len = d->L[k].len; + for (int i = 0; i < len && w < AD_BODY_MAX; i++) { + char c = s[i]; + if (ad_blank(c)) { + if (w > 0 && out[w - 1] != ' ') { + out[w++] = ' '; + } + continue; + } + out[w++] = c; + } + if (w > 0 && w < AD_BODY_MAX && out[w - 1] != ' ') { + out[w++] = ' '; + } + } + while (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0x80) { + w--; /* never end inside a character */ + } + if (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0xC0) { + w--; + } + while (w > 0 && out[w - 1] == ' ') { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +static int ad_head_qn_cmp(const void *a, const void *b) { + const ad_head_t *x = *(const ad_head_t *const *)a; + const ad_head_t *y = *(const ad_head_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static void ad_sections(ad_doc_t *d) { + CBMExtractCtx *ctx = d->ctx; + CBMArena *a = ctx->arena; + for (int h = 0; h < d->nheads; h++) { + const char *seg = ad_qn_segment(a, d->heads[h].title); + d->heads[h].qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL; + if (!d->heads[h].qn) { + d->failed = true; + return; + } + } + if (d->nheads > 1) { + ad_head_t **by = (ad_head_t **)cbm_arena_alloc(d->scratch, (size_t)d->nheads * sizeof(*by)); + if (!by) { + d->failed = true; + return; + } + for (int h = 0; h < d->nheads; h++) { + by[h] = &d->heads[h]; + } + qsort(by, (size_t)d->nheads, sizeof(*by), ad_head_qn_cmp); + for (int h = 1; h < d->nheads; h++) { + if (strcmp(by[h]->qn, by[h - 1]->qn) == 0) { + by[h]->shared_qn = true; + by[h - 1]->shared_qn = true; + } + } + } + for (int h = 0; h < d->nheads; h++) { + const ad_head_t *hd = &d->heads[h]; + int end = h + 1 < d->nheads ? d->heads[h + 1].line : d->n; + int last = end - 1; + while (last > hd->line && ad_rlen(&d->L[last]) == 0) { + last--; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = hd->title; + def.qualified_name = hd->qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = (uint32_t)hd->line + 1; + def.end_line = (uint32_t)last + 1; + def.is_exported = true; + def.docstring = ad_body(d, hd->line + 1, end); + cbm_defs_push(&ctx->result->defs, a, def); + } +} + +/* ── Page attributes ─────────────────────────────────────────────── */ + +static ad_attr_t *ad_attr_find(ad_doc_t *d, const char *name, size_t n) { + for (int i = 0; i < d->nattrs; i++) { + if (strlen(d->attrs[i].name) == n && memcmp(d->attrs[i].name, name, n) == 0) { + return &d->attrs[i]; + } + } + return NULL; +} + +/* ATTR_ENTRY_RE ^:(!?)([\w][\w\-]*)(!?):(?:\s+(.*?))?\s*$ -- true when the + * line is an attribute entry (set or unset). */ +static bool ad_attr_entry(ad_doc_t *d, const ad_line_t *l) { + const char *s = l->s; + int n = ad_rlen(l); + if (n < 3 || s[0] != ':') { + return false; + } + int k = 1; + bool unset = false; + if (s[k] == '!') { + unset = true; + k++; + } + int na = k; + if (k >= n || !ad_word(s[k])) { + return false; + } + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + int nb = k; + if (k < n && s[k] == '!') { + unset = true; + k++; + } + if (k >= n || s[k] != ':') { + return false; + } + k++; + if (k < n && !ad_blank(s[k])) { + return false; + } + while (k < n && ad_blank(s[k])) { + k++; + } + ad_attr_t *a = ad_attr_find(d, s + na, (size_t)(nb - na)); + if (unset) { + if (a) { + *a = d->attrs[--d->nattrs]; + } + return true; + } + char *value = cbm_arena_strndup(d->scratch, s + k, (size_t)(n - k)); + if (!value) { + d->failed = true; + return true; + } + if (a) { + a->value = value; + return true; + } + if (d->nattrs >= d->cap_attrs) { + int ncap = d->cap_attrs ? d->cap_attrs * 2 : AD_ATTRS_INIT; + ad_attr_t *grown = (ad_attr_t *)cbm_arena_alloc(d->scratch, (size_t)ncap * sizeof(*grown)); + if (!grown) { + d->failed = true; + return true; + } + if (d->nattrs) { + memcpy(grown, d->attrs, (size_t)d->nattrs * sizeof(*grown)); + } + d->attrs = grown; + d->cap_attrs = ncap; + } + char *name = cbm_arena_strndup(d->scratch, s + na, (size_t)(nb - na)); + if (!name) { + d->failed = true; + return true; + } + d->attrs[d->nattrs++] = (ad_attr_t){name, value}; + return true; +} + +/* A `{name}` reference at s[i] (not escaped `\{`): the name's span. */ +static bool ad_attr_ref(const char *s, int n, int i, int *a, int *b) { + if (s[i] != '{' || (i > 0 && s[i - 1] == '\\') || i + 2 >= n || !ad_word(s[i + 1])) { + return false; + } + int k = i + 1; + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + if (k >= n || s[k] != '}') { + return false; + } + *a = i + 1; + *b = k; + return true; +} + +/* The text with the page's attribute references substituted, pass after pass + * (a value may reference others; at most AD_EXPAND_DEPTH passes, as H8's + * depth); references the page does not define stay as written. NULL when it + * grows past AD_FIELD_MAX or memory ran out. */ +static char *ad_expand(ad_doc_t *d, const char *s, int n) { + char cur[AD_FIELD_MAX + 1]; + char out[AD_FIELD_MAX + 1]; + if (n > AD_FIELD_MAX) { + return NULL; + } + memcpy(cur, s, (size_t)n); + int cl = n; + for (int pass = 0; pass < AD_EXPAND_DEPTH; pass++) { + int w = 0; + bool changed = false; + for (int i = 0; i < cl; i++) { + int a; + int b; + const ad_attr_t *at = NULL; + if (ad_attr_ref(cur, cl, i, &a, &b)) { + at = ad_attr_find(d, cur + a, (size_t)(b - a)); + } + if (at) { + int vl = (int)strlen(at->value); + if (w + vl > AD_FIELD_MAX) { + return NULL; + } + memcpy(out + w, at->value, (size_t)vl); + w += vl; + i = b; + changed = true; + continue; + } + if (w + 1 > AD_FIELD_MAX) { + return NULL; + } + out[w++] = cur[i]; + } + if (w > 0) { + memcpy(cur, out, (size_t)w); + } + cl = w; + if (!changed) { + break; + } + } + return cbm_arena_strndup(d->scratch, cur, (size_t)cl); +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +static void ad_push(ad_doc_t *d, int line, int syntax, const char *raw) { + if (!raw) { + d->failed = true; + return; + } + CBMExtractCtx *ctx = d->ctx; + int lo = 0; + int hi = d->nheads; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (d->heads[mid].line <= line) { + lo = mid + 1; + } else { + hi = mid; + } + } + const ad_head_t *sec = lo > 0 ? &d->heads[lo - 1] : NULL; + if (sec && sec->shared_qn) { + sec = NULL; + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""), + .raw = raw, + .line = (uint32_t)line + 1, + .def_line = sec ? (uint32_t)sec->line + 1 : 1, + .syntax = (uint16_t)syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link); +} + +/* A value of the include's attribute list (`tag=x`, `tags="a;b"`, `lines=1..5`). */ +static void ad_attrlist_value(const char *al, int n, const char *key, char *out, size_t cap) { + out[0] = '\0'; + size_t kl = strlen(key); + for (int i = 0; i + (int)kl < n; i++) { + if ((i > 0 && ad_word(al[i - 1])) || memcmp(al + i, key, kl) != 0) { + continue; + } + int k = i + (int)kl; + while (k < n && ad_blank(al[k])) { + k++; + } + if (k >= n || al[k] != '=') { + continue; + } + k++; + while (k < n && ad_blank(al[k])) { + k++; + } + int a = k; + int b; + if (k < n && al[k] == '"') { + a = k + 1; + const char *q = memchr(al + a, '"', (size_t)(n - a)); + b = q ? (int)(q - al) : n; + } else { + b = a; + while (b < n && al[b] != ',') { + b++; + } + } + while (b > a && ad_blank(al[b - 1])) { + b--; + } + snprintf(out, cap, "%.*s", b - a, al + a); + return; + } +} + +/* INCLUDE_RE ^(\\?)include::([^\[\s][^\[]*)\[(.*)\]\s*$ */ +static void ad_include(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = ad_rlen(l); + static const char kw[] = "include::"; + int kl = (int)strlen(kw); + if (n < kl + 2 || memcmp(s, kw, (size_t)kl) != 0 || s[kl] == '[' || ad_blank(s[kl]) || + s[n - 1] != ']') { + return; + } + const char *open = memchr(s + kl, '[', (size_t)(n - kl)); + if (!open) { + return; + } + int ta = kl; + int tb = (int)(open - s); + int aa = tb + 1; + int ab = n - 1; + char *target = ad_expand(d, s + ta, tb - ta); + if (!target || strchr(target, '\t')) { + return; + } + char tags[AD_FIELD_MAX]; + char tag1[AD_FIELD_MAX]; + char lines[AD_FIELD_MAX]; + ad_attrlist_value(s + aa, ab - aa, "tags", tags, sizeof(tags)); + ad_attrlist_value(s + aa, ab - aa, "tag", tag1, sizeof(tag1)); + ad_attrlist_value(s + aa, ab - aa, "lines", lines, sizeof(lines)); + char *written = cbm_arena_strndup(d->ctx->arena, s, (size_t)n); + ad_push(d, i, CBM_DOCLINK_ADOC_INCLUDE, + written ? cbm_arena_sprintf(d->ctx->arena, "%s\t%s\t%s\t%s", written, target, + tags[0] ? tags : tag1, lines) + : NULL); +} + +/* Built-in attributes: characters and document settings, never a link. */ +static bool ad_builtin(const char *s, size_t n) { + static const char *const names[] = {"nbsp", "sp", + "empty", "blank", + "zwsp", "wj", + "apos", "quot", + "lsquo", "rsquo", + "ldquo", "rdquo", + "deg", "plus", + "brvbar", "vbar", + "amp", "lt", + "gt", "startsb", + "endsb", "caret", + "asterisk", "tilde", + "backslash", "backtick", + "two-colons", "two-semicolons", + "cpp", "cxx", + "pp", "docname", + "docdir", "docfile", + "doctitle", "toc", + "version", "revnumber", + "revdate", "author", + "email", "page-component-version", + NULL}; + for (int i = 0; names[i]; i++) { + if (strlen(names[i]) == n && memcmp(names[i], s, n) == 0) { + return true; + } + } + return false; +} + +static void ad_attr_refs(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = l->len; + for (int k = 0; k < n; k++) { + int a; + int b; + if (!ad_attr_ref(s, n, k, &a, &b) || ad_builtin(s + a, (size_t)(b - a))) { + continue; + } + const ad_attr_t *at = ad_attr_find(d, s + a, (size_t)(b - a)); + char *value = at ? ad_expand(d, at->value, (int)strlen(at->value)) : NULL; + if (at && (!value || strchr(value, '\t') || strchr(at->value, '\t'))) { + k = b; + continue; + } + ad_push(d, i, CBM_DOCLINK_ADOC_ATTRIBUTE, + cbm_arena_sprintf(d->ctx->arena, "{%.*s}\t%.*s\t%s\t%s", b - a, s + a, b - a, s + a, + at ? at->value : "", value ? value : "")); + k = b; + } +} + +/* Monospace spans of a prose line: `x` and ``x``, a `+x+` passthrough's inner + * text; each a code_path / code_name when the classifier names one. */ +static void ad_spans(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = l->len; + int k = 0; + while (k < n) { + if (s[k] != '`') { + k++; + continue; + } + int run = 1; + while (k + run < n && s[k + run] == '`') { + run++; + } + if (run > 2) { + k += run; + continue; + } + int a = k + run; + int e = a; + while (e + run <= n && !(memcmp(s + e, run == 2 ? "``" : "`", (size_t)run) == 0)) { + e++; + } + if (e + run > n || e == a) { + k = a; + continue; + } + int b = e; + if (b - a >= 2 && s[a] == '+' && s[b - 1] == '+') { + a++; + b--; + } + CBMDocLinkMdPath p; + if (b > a && b - a <= AD_SPAN_MAX && + cbm_doclink_md_classify_span(s + a, (size_t)(b - a), d->span_buf, AD_SPAN_MAX + 1, + &p)) { + ad_push(d, i, + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_ADOC_CODE_NAME + : CBM_DOCLINK_ADOC_CODE_PATH, + cbm_arena_strndup(d->ctx->arena, s + a, (size_t)(b - a))); + } + k = e + run; + } +} + +static void ad_tokens(ad_doc_t *d) { + for (int i = 0; i < d->n && !d->failed; i++) { + if (d->in_block[i] == 2) { + continue; /* a comment block: nothing in it is read */ + } + const ad_line_t *l = &d->L[i]; + if (l->len >= 2 && l->s[0] == '/' && l->s[1] == '/' && !d->in_block[i]) { + continue; /* a line comment */ + } + if (!d->in_block[i] && ad_attr_entry(d, l)) { + continue; + } + if (l->len > 0 && l->s[0] == 'i') { + ad_include(d, i); + } + ad_attr_refs(d, i); + if (!d->in_block[i]) { + ad_spans(d, i); + } + } +} + +void cbm_adoc_extract_document(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + ad_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena}; + d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(AD_SPAN_MAX + 1) * 2); + if (!d.span_buf || !ad_lines(&d, ctx->source, ctx->source_len)) { + ctx->result->doc_links.failed = true; + return; + } + ad_structure(&d); + if (!d.failed) { + ad_sections(&d); + } + if (!d.failed) { + ad_tokens(&d); + } + if (d.failed) { + ctx->result->doc_links.failed = true; + } +} + +/* ── The Antora scope (antora.yml, antora-playbook*.yml) ─────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} ad_sb_t; + +static void ad_put(ad_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + 1 > sb->cap) { + size_t ncap = sb->cap ? sb->cap : AD_FIELD_MAX; + while (ncap < sb->len + n + 1) { + ncap *= 2; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void ad_rec(ad_sb_t *sb, const char *kind, const char *a, size_t an, const char *b, + size_t bn) { + for (size_t i = 0; i < an; i++) { + if (a[i] == '\t' || a[i] == '\n') { + return; + } + } + for (size_t i = 0; b && i < bn; i++) { + if (b[i] == '\t' || b[i] == '\n') { + return; + } + } + ad_put(sb, kind, strlen(kind)); + ad_put(sb, "\t", 1); + ad_put(sb, a, an); + if (b) { + ad_put(sb, "\t", 1); + ad_put(sb, b, bn); + } + ad_put(sb, "\n", 1); +} + +static int ad_indent(const char *s, int n) { + int k = 0; + while (k < n && s[k] == ' ') { + k++; + } + return k; +} + +/* `key: value` at s (after the indent): the key and value spans (value + * unquoted, comments not stripped: the field test's reading). */ +static bool ad_yaml_kv(const char *s, int n, int *ka, int *kb, int *va, int *vb) { + int k = 0; + if (k >= n || !(ad_word(s[k]))) { + return false; + } + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + int e = k; + while (k < n && ad_blank(s[k])) { + k++; + } + if (k >= n || s[k] != ':') { + return false; + } + k++; + while (k < n && ad_blank(s[k])) { + k++; + } + *ka = 0; + *kb = e; + *va = k; + *vb = n; /* the caller's line has no trailing whitespace */ + return true; +} + +/* A value in matching quotes loses them. */ +static void ad_unquote(const char *s, int *va, int *vb) { + if (*vb - *va >= 2 && s[*va] == s[*vb - 1] && (s[*va] == '\'' || s[*va] == '"')) { + (*va)++; + (*vb)--; + } +} + +/* H8 parse_antora_yml: `name:`, `asciidoc: attributes:` and + * `ext: collector: scan: - dir: / into:`. */ +static void ad_antora(const char *src, int len, bool component, ad_sb_t *sb) { + bool in_asciidoc = false; + bool in_attrs = false; + int attrs_ind = -1; + bool in_scan = false; + int pos = 0; + while (pos <= len) { + const char *nl = memchr(src + pos, '\n', (size_t)(len - pos)); + int end = nl ? (int)(nl - src) : len; + const char *s = src + pos; + int n = end - pos; + while (n > 0 && ad_blank(s[n - 1])) { + n--; + } + int ind = ad_indent(s, n); + int ka; + int kb; + int va; + int vb; + bool kv = ad_yaml_kv(s + ind, n - ind, &ka, &kb, &va, &vb); + if (kv) { + ad_unquote(s + ind, &va, &vb); + } + if (ind == 0 && n > 0) { + in_asciidoc = kv && kb == (int)strlen("asciidoc") && memcmp(s, "asciidoc", 8) == 0; + in_attrs = false; + in_scan = false; + if (component && kv && kb == (int)strlen("name") && memcmp(s, "name", 4) == 0 && + vb > va) { + ad_rec(sb, "N", s + va, (size_t)(vb - va), NULL, 0); + } + } else if (in_asciidoc && kv && va == vb && kb == (int)strlen("attributes") && + memcmp(s + ind, "attributes", 10) == 0) { + in_attrs = true; + attrs_ind = -1; + } else if (in_attrs && n > ind && s[ind] != '#') { + if (attrs_ind < 0) { + attrs_ind = ind; + } + if (ind < attrs_ind) { + in_attrs = false; + } else if (ind == attrs_ind && kv) { + ad_rec(sb, "A", s + ind, (size_t)kb, s + ind + va, (size_t)(vb - va)); + } + } + if (component && kv && kb == (int)strlen("scan") && memcmp(s + ind, "scan", 4) == 0) { + in_scan = true; + } else if (component && in_scan) { + const char *t = s + ind; + int tn = n - ind; + if (tn > 1 && t[0] == '-' && ad_blank(t[1])) { + int sk = 1; + while (sk < tn && ad_blank(t[sk])) { + sk++; + } + t += sk; + tn -= sk; + } + int a; + int b; + int c; + int e; + bool tkv = ad_yaml_kv(t, tn, &a, &b, &c, &e); + if (tkv) { + ad_unquote(t, &c, &e); + } + if (tkv && e > c) { + if (b == 3 && memcmp(t, "dir", 3) == 0) { + ad_rec(sb, "D", t + c, (size_t)(e - c), NULL, 0); + } else if (b == 4 && memcmp(t, "into", 4) == 0) { + ad_rec(sb, "I", t + c, (size_t)(e - c), NULL, 0); + } + } + } + pos = end + 1; + } +} + +const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->rel_path || !ctx->source || ctx->source_len <= 0) { + return NULL; + } + const char *base = strrchr(ctx->rel_path, '/'); + base = base ? base + 1 : ctx->rel_path; + bool component = strcmp(base, "antora.yml") == 0; + bool playbook = strncmp(base, "antora-playbook", strlen("antora-playbook")) == 0; + if (!component && !playbook) { + return NULL; + } + ad_sb_t sb = {.a = ctx->arena}; + ad_put(&sb, CBM_DOCLINK_ADOC_SCOPE_TAG "\n", strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n")); + ad_put(&sb, component ? "C\n" : "P\n", 2); + ad_antora(ctx->source, ctx->source_len, component, &sb); + if (sb.failed) { + ctx->result->doc_links.failed = true; + return NULL; + } + return sb.buf; +} diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index 1a3d06b3c4..09ee36c25f 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -175,6 +175,11 @@ static const doclink_lang_t DOCLINK_LANGS[] = { {.lang = CBM_LANG_PYTHON, .scan_scope = cbm_doclink_py_scan_scope, .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG}, + /* YAML: the Antora component and playbook settings the AsciiDoc resolver + * reads */ + {.lang = CBM_LANG_YAML, + .scan_scope = cbm_doclink_antora_scan_scope, + .scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 741b95aaaa..e97aaf4886 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -92,6 +92,11 @@ X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, true, "MENTIONS") \ X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \ X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \ + /* asciidoc (doc_adoc.c): includes, Antora API attributes, monospace spans */ \ + X(CBM_DOCLINK_ADOC_INCLUDE, CBM_LANG_ASCIIDOC, "include", false, true, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_LANG_ASCIIDOC, "attribute", false, true, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_CODE_PATH, CBM_LANG_ASCIIDOC, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_CODE_NAME, CBM_LANG_ASCIIDOC, "code_name", false, true, "MENTIONS") \ /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") @@ -333,6 +338,22 @@ const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx); /* The Python scope blob starts with this tag line. */ #define CBM_DOCLINK_PY_SCOPE_TAG "py1" +/* ── AsciiDoc (doc_adoc.c) ─────────────────────────────────────────── */ + +/* AsciiDoc documents: a Section per heading and the references to code + * (include::, Antora API attributes, monospace spans) as doc-link tokens. + * Runs instead of a grammar for CBM_LANG_ASCIIDOC. */ +void cbm_adoc_extract_document(CBMExtractCtx *ctx); + +/* The scope blob of a YAML file, which the AsciiDoc resolver reads: an + * antora.yml's component name, asciidoc attributes and collector scans, an + * antora-playbook*.yml's attributes (read as text). NULL for every other + * file. */ +const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx); + +/* The Antora scope blob starts with this tag line. */ +#define CBM_DOCLINK_ADOC_SCOPE_TAG "ad1" + /* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's * text as docstring, property page) and the pages' structural code mentions * as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */ diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index a37c3e52ce..6672011d07 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -2718,6 +2718,9 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { // looked up, so this language never reaches cbm_lang_spec()/cbm_ts_language(). // Left as a zero spec. + // CBM_LANG_ASCIIDOC — AsciiDoc documents. No grammar row, as PDF: extraction + // reads the file itself (cbm_adoc_extract_document, doc_adoc.c). Zero spec. + // CBM_LANG_OBJECTSCRIPT_EXPORT — Studio Export XML. No grammar row: the // pipeline transcodes Export XML to UDL (iris_export_xml.c) and re-extracts // each class as CBM_LANG_OBJECTSCRIPT_UDL, so this language never reaches diff --git a/src/discover/language.c b/src/discover/language.c index c15a1d43d8..b1cd7d9388 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -619,6 +619,9 @@ static const ext_entry_t EXT_TABLE[] = { {".clib", CBM_LANG_CHIALISP}, {".clinc", CBM_LANG_CHIALISP}, {".pdf", CBM_LANG_PDF}, + /* AsciiDoc; `.asc` is left out: it is as often an ASCII-armored signature */ + {".adoc", CBM_LANG_ASCIIDOC}, + {".asciidoc", CBM_LANG_ASCIIDOC}, /* Slang */ {".slang", CBM_LANG_SLANG}, @@ -831,6 +834,7 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_SCHEME] = "Scheme", [CBM_LANG_CHIALISP] = "Chialisp", [CBM_LANG_PDF] = "PDF", + [CBM_LANG_ASCIIDOC] = "AsciiDoc", [CBM_LANG_FENNEL] = "Fennel", [CBM_LANG_FISH] = "Fish", [CBM_LANG_AWK] = "AWK", diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index 7b1f3d2b59..a0d306008c 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -161,6 +161,7 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = { {"plsql", CBM_LANG_PLSQL}, {"chialisp", CBM_LANG_CHIALISP}, {"pdf", CBM_LANG_PDF}, + {"asciidoc", CBM_LANG_ASCIIDOC}, {"lean", CBM_LANG_LEAN}, {"form", CBM_LANG_FORM}, {"magma", CBM_LANG_MAGMA}, diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 71159220d6..c5023658d0 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -45,10 +45,8 @@ const char *cbm_doclink_reason_name(int reason) { /* One pointer per language with a resolver: the one line a language leg adds * to this file. */ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { - &cbm_doclink_cs_resolver, - &cbm_doclink_md_resolver, - &cbm_doclink_pdf_resolver, - &cbm_doclink_rst_resolver, + &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, + &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; @@ -93,7 +91,7 @@ static const cbm_doclink_resolver_t *resolver_of_scope(const char *scope) { /* ── Incremental scope rules ─────────────────────────────────────── */ bool cbm_doclinks_binds_lines(CBMLanguage lang) { - return lang == CBM_LANG_MARKDOWN || lang == CBM_LANG_RST; + return lang == CBM_LANG_MARKDOWN || lang == CBM_LANG_RST || lang == CBM_LANG_ASCIIDOC; } bool cbm_doclinks_is_scope_input(const char *rel_path) { diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index ec1416f274..77a5dfb67b 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -245,6 +245,9 @@ extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver; /* reStructuredText documents and the Python scope they read (doc_links_rst.c): * via "rst". */ extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver; +/* AsciiDoc documents and the Antora YAML scope they read (doc_links_adoc.c): + * via "asciidoc". */ +extern const cbm_doclink_resolver_t cbm_doclink_adoc_resolver; /* Over a Markdown resolver index (cbm_doclink_md_resolver.build), for the * other document resolvers: the innermost code definition of `path` holding @@ -253,6 +256,15 @@ extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver; const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, uint32_t first, uint32_t last, const char *member); const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path); +/* The definition a region [first, last] of `path` (its text in `text`) binds, + * by the field test's rule (H8 bind_range): edge blank and comment lines + * trimmed; the innermost enclosing definition, unless it is a type (or there + * is none) and the definitions inside cover every other line of the region + * (an annotation right above one counts) -- then that one definition (the + * type for several). NULL: the file. */ +const cbm_gbuf_node_t *cbm_doclink_md_region(const void *md_index, const char *path, + const char *text, size_t text_len, uint32_t first, + uint32_t last); bool cbm_doclink_md_test_path(const char *path); #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS @@ -278,7 +290,8 @@ void cbm_doclink_cs_test_unit_memo(bool on); #endif /* True for a document language whose references may bind the LINES of a file - * (Markdown `file#L3-L9`, reST :lines:): a body edit of that file keeps its + * (Markdown `file#L3-L9`, reST :lines:, AsciiDoc tags and lines=): a body + * edit of that file keeps its * names but can move what those lines hold, so an incremental run * re-resolves such documents with the edited file. */ bool cbm_doclinks_binds_lines(CBMLanguage lang); diff --git a/src/pipeline/doc_links_adoc.c b/src/pipeline/doc_links_adoc.c new file mode 100644 index 0000000000..863462e41e --- /dev/null +++ b/src/pipeline/doc_links_adoc.c @@ -0,0 +1,919 @@ +/* + * doc_links_adoc.c — AsciiDoc references -> MENTIONS edges, via "asciidoc" + * (the resolving half of internal/cbm/doc_adoc.c), with the Antora rules the + * field test's harvester measured (H8: includes 97.6-100 % resolved, ~100 % + * precise; javadoc attributes 320/320). + * + * Antora: a component is the directory of an antora.yml; its pages live under + * /modules//pages. Its attributes and collector scans, and + * a playbook's attributes, come from the YAML scope blobs (doc_adoc.c), so an + * incremental run resolves as a full one. + * + * include::target[...]: references the page left are expanded from the + * component's attributes, then the playbook's. A resource ID + * ([[version@]component:]module:]family$path) names + * /modules///, or a file a collector scan + * contributes there; any other target is relative to the including file. Its + * `lines=` bind the innermost definition holding them; its `tag=` / `tags=` + * regions -- the included file's own `tag::x[]` .. `end::x[]` comments, read + * from that indexed file -- likewise; a tag the file does not have binds the + * file (the document stays a dependent of that file, so it re-resolves when + * the file changes, on an incremental run as on a full one). + * + * {attribute} references whose value is an Antora javadoc link + * ({javadoc-root}///.html[#member]) bind the + * Java or Kotlin type (or its member) that the repository's main sources + * declare under that package path. + * + * Monospace spans: the Markdown resolver's code-span rules (doc_links_md.c). + */ +#include "pipeline/doc_links.h" + +#include "discover/discover.h" /* cbm_language_for_filename */ +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/compat_fs.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + RA_PATH_CAP = 1024, + RA_FIELDS = 6, + RA_EXPAND_PASSES = 10, /* nested attribute references (H8's depth) */ + RA_TAG_FILE_MAX = 8 * 1024 * 1024, /* an included file read for its tags */ + RA_TAGS = 16, /* tags of one include */ + RA_TAG_LEN = 128, /* a tag name */ + RA_INIT = 8, +}; + +typedef struct { + const char *key; + const char *value; +} ra_attr_t; + +typedef struct { + const char *dir; + const char *into; +} ra_scan_t; + +typedef struct { + const char *root; /* the antora.yml's directory ("" for the repository root) */ + const char *name; + ra_attr_t *attrs; + int nattrs; + ra_scan_t *scans; + int nscans; +} ra_comp_t; + +typedef struct { + const char *project; + const char *repo; /* the repository on disk: included files are read for their tags */ + void *md; + const char **run_paths; + int run_count; + CBMArena arena; + ra_comp_t *comps; + int ncomps; + ra_attr_t *pattrs; /* playbook attributes, the first definition of a key wins */ + int npattrs; +} ra_index_t; + +static void ra_destroy(void *index); + +static const char *ra_base(const char *path) { + const char *s = strrchr(path, '/'); + return s ? s + SKIP_ONE : path; +} + +static bool ra_ends(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n >= k && strcmp(s + n - k, suffix) == 0; +} + +static bool ra_grow(CBMArena *a, void **items, int count, int *cap, size_t size) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : RA_INIT; + void *grown = cbm_arena_alloc(a, (size_t)ncap * size); + if (!grown) { + return false; + } + if (count > 0) { + memcpy(grown, *items, (size_t)count * size); + } + *items = grown; + *cap = ncap; + return true; +} + +/* One YAML scope blob: a component (C) or a playbook (P). */ +static bool ra_read_scope(ra_index_t *x, const char *path, const char *scope, int *comp_cap, + int *pattr_cap) { + char *copy = cbm_arena_strdup(&x->arena, scope); + if (!copy) { + return false; + } + char *save = NULL; + (void)strtok_r(copy, "\n", &save); /* the tag line */ + char *line = strtok_r(NULL, "\n", &save); + if (!line || (strcmp(line, "C") != 0 && strcmp(line, "P") != 0)) { + return true; + } + bool component = line[0] == 'C'; + ra_comp_t comp = {0}; + int acap = 0; + int scap = 0; + if (component) { + const char *b = ra_base(path); + comp.root = + cbm_arena_strndup(&x->arena, path, b > path ? (size_t)(b - path - SKIP_ONE) : 0); + comp.name = ""; + if (!comp.root) { + return false; + } + } + for (line = strtok_r(NULL, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { + char *tab = strchr(line, '\t'); + if (!tab) { + continue; + } + *tab = '\0'; + char *v = tab + SKIP_ONE; + if (strcmp(line, "N") == 0 && component) { + comp.name = v; + } else if (strcmp(line, "A") == 0) { + char *t2 = strchr(v, '\t'); + if (!t2) { + continue; + } + *t2 = '\0'; + ra_attr_t at = {v, t2 + SKIP_ONE}; + if (component) { + if (!ra_grow(&x->arena, (void **)&comp.attrs, comp.nattrs, &acap, sizeof(at))) { + return false; + } + comp.attrs[comp.nattrs++] = at; + } else { + bool seen = false; + for (int i = 0; i < x->npattrs; i++) { + seen = seen || strcmp(x->pattrs[i].key, at.key) == 0; + } + if (!seen) { + if (!ra_grow(&x->arena, (void **)&x->pattrs, x->npattrs, pattr_cap, + sizeof(at))) { + return false; + } + x->pattrs[x->npattrs++] = at; + } + } + } else if (strcmp(line, "D") == 0 && component) { + if (!ra_grow(&x->arena, (void **)&comp.scans, comp.nscans, &scap, sizeof(ra_scan_t))) { + return false; + } + comp.scans[comp.nscans++] = (ra_scan_t){v, ""}; + } else if (strcmp(line, "I") == 0 && component && comp.nscans > 0) { + comp.scans[comp.nscans - 1].into = v; + } + } + if (component) { + if (!ra_grow(&x->arena, (void **)&x->comps, x->ncomps, comp_cap, sizeof(comp))) { + return false; + } + x->comps[x->ncomps++] = comp; + } + return true; +} + +static void *ra_build(const cbm_doclink_build_in_t *in) { + ra_index_t *x = (ra_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->repo = in->ctx ? in->ctx->repo_path : NULL; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->md = cbm_doclink_md_resolver.build(in); + if ((in->run_file_count > 0 && !x->run_paths) || !x->md || !x->project) { + ra_destroy(x); + return NULL; + } + int comp_cap = 0; + int pattr_cap = 0; + for (int i = 0; i < in->file_count; i++) { + const cbm_doclink_file_t *f = &in->files[i]; + if (f->run_file >= 0 && f->run_file < x->run_count) { + x->run_paths[f->run_file] = f->rel_path; /* valid until destroy */ + } + if (f->scope && + strncmp(f->scope, CBM_DOCLINK_ADOC_SCOPE_TAG "\n", + strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n")) == 0 && + !ra_read_scope(x, f->rel_path, f->scope, &comp_cap, &pattr_cap)) { + ra_destroy(x); + return NULL; + } + } + return x; +} + +static void ra_destroy(void *index) { + ra_index_t *x = (ra_index_t *)index; + if (!x) { + return; + } + if (x->md) { + cbm_doclink_md_resolver.destroy(x->md); + } + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* H8 component_of: the deepest component whose modules/ hold the document. */ +static const ra_comp_t *ra_component_of(const ra_index_t *x, const char *doc) { + const ra_comp_t *best = NULL; + for (int i = 0; i < x->ncomps; i++) { + const ra_comp_t *c = &x->comps[i]; + char pref[RA_PATH_CAP]; + snprintf(pref, sizeof(pref), "%s%smodules/", c->root, c->root[0] ? "/" : ""); + if (strncmp(doc, pref, strlen(pref)) == 0 && + (!best || strlen(c->root) > strlen(best->root))) { + best = c; + } + } + return best; +} + +static const char *ra_attr(const ra_index_t *x, const ra_comp_t *c, const char *key, size_t n) { + for (int i = 0; c && i < c->nattrs; i++) { + if (strlen(c->attrs[i].key) == n && memcmp(c->attrs[i].key, key, n) == 0) { + return c->attrs[i].value; + } + } + for (int i = 0; i < x->npattrs; i++) { + if (strlen(x->pattrs[i].key) == n && memcmp(x->pattrs[i].key, key, n) == 0) { + return x->pattrs[i].value; + } + } + return NULL; +} + +static bool ra_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The text with the component's and the playbook's attribute references + * substituted, pass after pass. False when one stays undefined or it does + * not fit. */ +static bool ra_expand(const ra_index_t *x, const ra_comp_t *c, const char *s, char *out, + size_t cap) { + char cur[RA_PATH_CAP]; + char nxt[RA_PATH_CAP]; + if (snprintf(cur, sizeof(cur), "%s", s) >= (int)sizeof(cur)) { + return false; + } + bool undefined = false; + for (int pass = 0; pass < RA_EXPAND_PASSES; pass++) { + size_t w = 0; + bool changed = false; + undefined = false; + for (size_t i = 0; cur[i]; i++) { + const char *v = NULL; + size_t e = i + SKIP_ONE; + if (cur[i] == '{' && (i == 0 || cur[i - SKIP_ONE] != '\\') && ra_word(cur[e])) { + while (cur[e] && (ra_word(cur[e]) || cur[e] == '-')) { + e++; + } + if (cur[e] == '}') { + v = ra_attr(x, c, cur + i + SKIP_ONE, e - i - SKIP_ONE); + undefined = undefined || !v; + } + } + if (v) { + size_t vl = strlen(v); + if (w + vl >= sizeof(nxt)) { + return false; + } + memcpy(nxt + w, v, vl); + w += vl; + i = e; + changed = true; + continue; + } + if (w + SKIP_ONE >= sizeof(nxt)) { + return false; + } + nxt[w++] = cur[i]; + } + nxt[w] = '\0'; + memcpy(cur, nxt, w + SKIP_ONE); + if (!changed) { + break; + } + } + if (undefined || snprintf(out, cap, "%s", cur) >= (int)cap) { + return false; + } + return true; +} + +/* os.path.normpath(base/p); false when it leaves the repository. */ +static bool ra_norm(const char *base, const char *p, char *out, size_t cap) { + char tmp[RA_PATH_CAP * PAIR_LEN]; + snprintf(tmp, sizeof(tmp), "%s%s%s", base, base[0] && p[0] ? "/" : "", p); + size_t w = 0; + out[0] = '\0'; + for (char *s = tmp; *s;) { + char *e = strchr(s, '/'); + size_t l = e ? (size_t)(e - s) : strlen(s); + if (l == PAIR_LEN && s[0] == '.' && s[1] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + if (w > 0) { + w--; + } + out[w] = '\0'; + } else if (l > 0 && !(l == SKIP_ONE && s[0] == '.')) { + if (w + l + PAIR_LEN > cap) { + return false; + } + if (w > 0) { + out[w++] = '/'; + } + memcpy(out + w, s, l); + w += l; + out[w] = '\0'; + } + if (!e) { + break; + } + s = e + SKIP_ONE; + } + return true; +} + +static void ra_unresolved(cbm_doclink_outcome_t *out, int reason) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = reason; +} + +static void ra_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *n, uint32_t a, uint32_t b) { + out->kind = CBM_DOCLINK_EDGE; + out->target = n; + out->exact = true; + out->target_first = a; + out->target_last = b; +} + +/* A path of a type the indexer does not read (a service file, an image): no + * reference to code, decided by its name as the Markdown resolver does -- never + * by the disk, which an incremental run does not watch for such files. */ +static bool ra_unindexable(const char *path) { + return cbm_language_for_filename(ra_base(path)) == CBM_LANG_COUNT; +} + +/* A collector scan of build output (`./build/...`): what it contributes is + * generated, not a file of the repository (H8). */ +static bool ra_build_dir(const char *dir) { + while (dir[0] == '.' && dir[1] == '/') { + dir += PAIR_LEN; + } + return strcmp(dir, "build") == 0 || strncmp(dir, "build/", strlen("build/")) == 0 || + strstr(dir, "/build/") != NULL; +} + +static const char *const RA_FAMILIES[][2] = {{"example", "examples"}, + {"partial", "partials"}, + {"page", "pages"}, + {"image", "images"}, + {"attachment", "attachments"}}; + +/* H8 resolve_target: an Antora resource ID or a path relative to the including + * file. Returns the repository path in `out`, or false with `out->kind` set. */ +static bool ra_target(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_comp_t *comp, const char *target, char *path, size_t cap, + cbm_doclink_outcome_t *out) { + out->kind = CBM_DOCLINK_LOCAL; + if (strstr(target, "://")) { + return false; /* a URL include */ + } + const char *dollar = strchr(target, '$'); + if (dollar && comp) { + char pre[RA_PATH_CAP]; + snprintf(pre, sizeof(pre), "%.*s", (int)(dollar - target), target); + const char *at = strrchr(pre, '@'); + char *segs = at ? pre + (at - pre) + SKIP_ONE : pre; + /* family is the last segment; module and component before it */ + char *fam = strrchr(segs, ':'); + const char *module = NULL; + if (fam) { + *fam = '\0'; + fam++; + char *comp_sep = strchr(segs, ':'); + if (comp_sep) { + *comp_sep = '\0'; + if (strcmp(segs, comp->name) != 0) { + return false; /* another component's resource */ + } + module = comp_sep + SKIP_ONE; + } else { + module = segs; + } + } else { + fam = segs; + } + char doc_module[RA_PATH_CAP] = "ROOT"; + if (!module) { + /* the including page's own module: modules//... */ + const char *rel = doc + strlen(comp->root) + (comp->root[0] ? SKIP_ONE : 0); + if (strncmp(rel, "modules/", strlen("modules/")) == 0) { + const char *m = rel + strlen("modules/"); + const char *slash = strchr(m, '/'); + if (slash) { + snprintf(doc_module, sizeof(doc_module), "%.*s", (int)(slash - m), m); + } + } + module = doc_module; + } + const char *famdir = NULL; + for (size_t i = 0; i < sizeof(RA_FAMILIES) / sizeof(RA_FAMILIES[0]); i++) { + if (strcmp(fam, RA_FAMILIES[i][0]) == 0) { + famdir = RA_FAMILIES[i][1]; + } + } + if (!famdir) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return false; + } + char virt[RA_PATH_CAP]; + snprintf(virt, sizeof(virt), "modules/%s/%s/%s", module, famdir, dollar + SKIP_ONE); + if (ra_norm(comp->root, virt, path, cap) && + cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + /* a file a collector scan contributes into the component */ + for (int i = 0; i < comp->nscans; i++) { + const char *into = comp->scans[i].into; + while (into[0] == '.' && into[1] == '/') { + into += PAIR_LEN; + } + size_t il = strlen(into); + while (il > 0 && into[il - SKIP_ONE] == '/') { + il--; + } + if (il == 0 || strncmp(virt, into, il) != 0 || virt[il] != '/') { + continue; + } + char sub[RA_PATH_CAP]; + snprintf(sub, sizeof(sub), "%s/%s", comp->scans[i].dir, virt + il + SKIP_ONE); + if (ra_norm(comp->root, sub, path, cap) && + cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + if (ra_build_dir(comp->scans[i].dir)) { + return false; /* build output (H8 generated): no file of the repository */ + } + } + if (ra_unindexable(virt)) { + return false; + } + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return false; + } + char dir[RA_PATH_CAP]; + const char *b = ra_base(doc); + snprintf(dir, sizeof(dir), "%.*s", b > doc ? (int)(b - doc - SKIP_ONE) : 0, doc); + if (!ra_norm(dir, target, path, cap)) { + return false; /* above the repository root */ + } + if (cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + char parent[RA_PATH_CAP]; + const char *pb = ra_base(path); + snprintf(parent, sizeof(parent), "%.*s", pb > path ? (int)(pb - path - SKIP_ONE) : 0, path); + if (!ra_unindexable(path) && parent[0] && cbm_doclink_md_folder(x->md, parent)) { + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } + return false; +} + +/* Asciidoctor's TagDirectiveRx \b(?:tag|(e)nd)::(\S+?)\[\](?=$|[ \r]) over the + * file: the 1-based lines from the first region's first content line to the + * last region's last one (directive lines excluded). False when a tag has no + * region. */ +static bool ra_tag_span(const char *src, size_t n, const char *const *tags, int ntags, + uint32_t *first, uint32_t *last) { + *first = 0; + *last = 0; + for (int t = 0; t < ntags; t++) { + size_t tl = strlen(tags[t]); + uint32_t line = 1; + uint32_t start = 0; + bool found = false; + size_t pos = 0; + while (pos < n) { + const char *nl = memchr(src + pos, '\n', n - pos); + size_t end = nl ? (size_t)(nl - src) : n; + for (size_t i = pos; i + PAIR_LEN < end; i++) { + bool is_end = false; + size_t k; + if (i + strlen("tag::") <= end && memcmp(src + i, "tag::", strlen("tag::")) == 0) { + k = i + strlen("tag::"); + } else if (i + strlen("end::") <= end && + memcmp(src + i, "end::", strlen("end::")) == 0) { + k = i + strlen("end::"); + is_end = true; + } else { + continue; + } + if (i > pos && ra_word(src[i - SKIP_ONE])) { + continue; /* \b */ + } + if (k + tl + PAIR_LEN > end || memcmp(src + k, tags[t], tl) != 0 || + src[k + tl] != '[' || src[k + tl + SKIP_ONE] != ']') { + continue; + } + size_t after = k + tl + PAIR_LEN; + if (after != end && src[after] != ' ' && src[after] != '\r') { + continue; + } + if (!is_end && start == 0) { + start = line + SKIP_ONE; + } else if (is_end && start != 0) { + if (start <= line - SKIP_ONE) { + *first = (*first == 0 || start < *first) ? start : *first; + *last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last; + found = true; + } + start = 0; + } + break; + } + line++; + pos = end + SKIP_ONE; + } + if (start != 0 && start <= line - SKIP_ONE) { + *first = (*first == 0 || start < *first) ? start : *first; + *last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last; + found = true; + } + if (!found) { + return false; + } + } + return *first > 0; +} + +/* The included file's text, read from the indexed file on disk (at most + * RA_TAG_FILE_MAX bytes): its tag regions and the lines of a region. NULL + * when it cannot be read; the caller frees it (cbm_free, OTHER). */ +static char *ra_read(const ra_index_t *x, const char *path, size_t *n) { + *n = 0; + if (!x->repo) { + return NULL; + } + char abs[RA_PATH_CAP * PAIR_LEN]; + snprintf(abs, sizeof(abs), "%s/%s", x->repo, path); + FILE *f = cbm_fopen(abs, "rb"); + if (!f) { + return NULL; + } + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, RA_TAG_FILE_MAX); + *n = buf ? fread(buf, SKIP_ONE, RA_TAG_FILE_MAX, f) : 0; + (void)fclose(f); + return buf; +} + +/* `lines=a..b;c;d..-1`: the span of all ranges (-1: the file's end). */ +static bool ra_lines(const char *v, uint32_t *first, uint32_t *last) { + *first = 0; + *last = 0; + const char *p = v; + while (*p) { + while (*p == ';' || *p == ',' || *p == ' ' || *p == '"') { + p++; + } + if (!*p) { + break; + } + if (*p < '0' || *p > '9') { + return false; + } + uint32_t a = (uint32_t)strtoul(p, (char **)&p, 10); + uint32_t b = a; + if (p[0] == '.' && p[1] == '.') { + p += PAIR_LEN; + if (p[0] == '-' && p[1] == '1') { + b = UINT32_MAX; + p += PAIR_LEN; + } else if (*p >= '0' && *p <= '9') { + b = (uint32_t)strtoul(p, (char **)&p, 10); + } else { + return false; + } + } + if (a == 0 || b < a) { + return false; + } + *first = (*first == 0 || a < *first) ? a : *first; + *last = b > *last ? b : *last; + } + return *first > 0; +} + +typedef struct { + char buf[RA_PATH_CAP * PAIR_LEN]; + char *f[RA_FIELDS]; + int n; +} ra_rec_t; + +static bool ra_record(const char *raw, ra_rec_t *r) { + if (snprintf(r->buf, sizeof(r->buf), "%s", raw) >= (int)sizeof(r->buf)) { + return false; + } + r->n = 0; + char *p = r->buf; + while (r->n < RA_FIELDS) { + r->f[r->n++] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + p = tab + SKIP_ONE; + } + for (int i = r->n; i < RA_FIELDS; i++) { + r->f[i] = ""; + } + return true; +} + +static void ra_include(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_rec_t *r, cbm_doclink_outcome_t *out) { + const ra_comp_t *comp = ra_component_of(x, doc); + char target[RA_PATH_CAP]; + if (strchr(r->f[1], '{')) { + if (!comp || !ra_expand(x, comp, r->f[1], target, sizeof(target))) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); /* an undefined attribute */ + return; + } + } else { + snprintf(target, sizeof(target), "%s", r->f[1]); + } + char path[RA_PATH_CAP]; + if (!ra_target(x, graph, doc, comp, target, path, sizeof(path), out)) { + return; + } + const cbm_gbuf_node_t *file = cbm_pipeline_file_node(graph, x->project, path); + if (!file) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + uint32_t first = 0; + uint32_t last = 0; + const char *tagv = r->f[2]; + if (!tagv[0] && r->f[3][0] && !ra_lines(r->f[3], &first, &last)) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + size_t tn = 0; + char *text = (tagv[0] || first > 0) ? ra_read(x, path, &tn) : NULL; + if (tagv[0]) { + /* H8: negated and wildcard tags leave the positive ones; none left + * includes the whole file */ + char tags[RA_TAGS][RA_TAG_LEN]; + const char *tagp[RA_TAGS]; + int nt = 0; + for (const char *p = tagv; *p && nt < RA_TAGS;) { + size_t l = strcspn(p, ";,"); + if (l > 0 && l < RA_TAG_LEN && p[0] != '!' && !memchr(p, '*', l)) { + snprintf(tags[nt], RA_TAG_LEN, "%.*s", (int)l, p); + tagp[nt] = tags[nt]; + nt++; + } + p += l + (p[l] ? SKIP_ONE : 0); + } + if (nt == 0 || !text || !ra_tag_span(text, tn, tagp, nt, &first, &last)) { + first = 0; /* wildcards only, or a tag the file lacks: the file */ + last = 0; + } + } + if (first > 0) { + const cbm_gbuf_node_t *seg = text + ? cbm_doclink_md_region(x->md, path, text, tn, first, last) + : cbm_doclink_md_segment(x->md, path, first, last, NULL); + cbm_free(CBM_MEM_CLASS_OTHER, text); + ra_edge(out, seg ? seg : file, first, last == UINT32_MAX ? 0 : last); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, text); + ra_edge(out, file, 0, 0); +} + +static bool ra_class_label(const char *label) { + static const char *const labels[] = {"Class", "Interface", "Enum", "Type", "Trait", NULL}; + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +static bool ra_member_label(const char *label) { + static const char *const labels[] = {"Method", "Field", "Variable", "Function", NULL}; + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* H8 javadoc: xref:attachment$api///.html[#member]. */ +static void ra_javadoc(const ra_index_t *x, const cbm_gbuf_t *graph, const char *val, + cbm_doclink_outcome_t *out) { + static const char key[] = "xref:attachment$api/"; + const char *p = strstr(val, key); + if (!p) { + return; + } + p += strlen(key); + const char *slash = strchr(p, '/'); /* after the JPMS module */ + if (!slash) { + return; + } + const char *path = slash + SKIP_ONE; + const char *html = strstr(path, ".html"); + if (!html || memchr(path, '[', (size_t)(html - path))) { + return; + } + char rel[RA_PATH_CAP]; + snprintf(rel, sizeof(rel), "%.*s", (int)(html - path), path); + const char *tname = ra_base(rel); + if (strcmp(tname, "package-summary") == 0 || strcmp(tname, "module-summary") == 0 || + strcmp(rel, "index") == 0) { + return; /* a package or module page: no code node */ + } + char anchor[RA_PATH_CAP] = ""; + if (html[strlen(".html")] == '#') { + const char *a = html + strlen(".html") + SKIP_ONE; + size_t al = strcspn(a, "[(-"); + snprintf(anchor, sizeof(anchor), "%.*s", (int)al, a); + } + char pkgdir[RA_PATH_CAP]; + snprintf(pkgdir, sizeof(pkgdir), "%.*s", (int)(tname - rel), rel); /* "org/x/" */ + char outer[RA_PATH_CAP]; + snprintf(outer, sizeof(outer), "%.*s", (int)strcspn(tname, "."), tname); + const char *want = strrchr(tname, '.') ? strrchr(tname, '.') + SKIP_ONE : tname; + /* the source files declaring /.java|.kt, main sources first */ + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, outer, &nodes, &count) != 0) { + return; + } + const char *files[2] = {NULL, NULL}; + int nfiles = 0; + for (int pass = 0; pass < PAIR_LEN && nfiles == 0; pass++) { + bool main_only = pass == 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *fp = n->file_path; + if (!ra_class_label(n->label) || !fp) { + continue; + } + char tail[RA_PATH_CAP * PAIR_LEN]; + bool java = ra_ends(fp, ".java"); + snprintf(tail, sizeof(tail), "%s%s%s", pkgdir, outer, java ? ".java" : ".kt"); + size_t fl = strlen(fp); + size_t tl = strlen(tail); + if (!(java || ra_ends(fp, ".kt")) || fl < tl || strcmp(fp + fl - tl, tail) != 0 || + (fl > tl && fp[fl - tl - SKIP_ONE] != '/') || + (main_only && !strstr(fp, "/src/main/"))) { + continue; + } + bool seen = false; + for (int k = 0; k < nfiles; k++) { + seen = seen || strcmp(files[k], fp) == 0; + } + if (!seen && nfiles < PAIR_LEN) { + files[nfiles++] = fp; + } else if (!seen) { + nfiles++; + } + } + } + if (nfiles == 0) { + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + if (nfiles > SKIP_ONE) { + ra_unresolved(out, CBM_DOCLINK_REASON_AMBIGUOUS); + return; + } + /* the type in that file (the first by line for a repeated name) */ + const cbm_gbuf_node_t *type = NULL; + if (cbm_gbuf_find_by_name(graph, want, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (ra_class_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 && + (!type || n->start_line < type->start_line || + (n->start_line == type->start_line && n->id < type->id))) { + type = n; + } + } + } + if (!type) { + ra_unresolved(out, CBM_DOCLINK_REASON_GRAPH_GAP); + return; + } + const cbm_gbuf_node_t *member = NULL; + int members = 0; + if (anchor[0] && cbm_gbuf_find_by_name(graph, anchor, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (ra_member_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 && + n->start_line >= type->start_line && n->start_line <= type->end_line) { + member = n; + members++; + } + } + } + /* an overloaded member: the type that declares it */ + ra_edge(out, members == SKIP_ONE ? member : type, 0, 0); +} + +static void ra_attribute(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_rec_t *r, cbm_doclink_outcome_t *out) { + const ra_comp_t *comp = ra_component_of(x, doc); + if (!comp) { + return; /* outside an Antora component: no API attributes (H8) */ + } + const char *raw = r->f[2][0] ? r->f[2] : ra_attr(x, comp, r->f[1], strlen(r->f[1])); + if (!raw || !strstr(raw, "{javadoc-root}")) { + return; + } + char val[RA_PATH_CAP]; + if (!ra_expand(x, comp, r->f[2][0] ? r->f[3] : raw, val, sizeof(val))) { + return; + } + ra_javadoc(x, graph, val, out); +} + +static void ra_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const ra_index_t *x = (const ra_index_t *)index; + out->kind = CBM_DOCLINK_LOCAL; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (link->syntax == CBM_DOCLINK_ADOC_CODE_PATH || link->syntax == CBM_DOCLINK_ADOC_CODE_NAME) { + CBMDocLink md = *link; + md.syntax = link->syntax == CBM_DOCLINK_ADOC_CODE_PATH ? CBM_DOCLINK_MD_CODE_PATH + : CBM_DOCLINK_MD_CODE_NAME; + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, &md, graph, out); + return; + } + ra_rec_t *r = (ra_rec_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*r)); + if (!r || !ra_record(link->raw, r)) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + cbm_free(CBM_MEM_CLASS_OTHER, r); + return; + } + if (link->syntax == CBM_DOCLINK_ADOC_INCLUDE) { + ra_include(x, graph, doc, r, out); + } else if (link->syntax == CBM_DOCLINK_ADOC_ATTRIBUTE) { + ra_attribute(x, graph, doc, r, out); + } + cbm_free(CBM_MEM_CLASS_OTHER, r); +} + +const cbm_doclink_resolver_t cbm_doclink_adoc_resolver = { + .langs = {CBM_LANG_ASCIIDOC, CBM_LANG_YAML}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG, + .build = ra_build, + .destroy = ra_destroy, + .resolve = ra_resolve, + .via = "asciidoc", +}; diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index b2a76594e7..667ea724aa 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -50,6 +50,8 @@ enum { MDR_HEX_BASE = 16, MDR_PCT_LEN = 3, /* `%XX` */ MDR_ADR_DIGITS = 5, /* an ADR number has at most this many digits */ + MDR_PRIO_FIELD = 3, /* H8's label priorities: types 0, Type 1, callables 2 */ + MDR_PRIO_OTHER = 4, }; /* Labels whose nodes are code a range or a member can name. */ @@ -108,8 +110,8 @@ static const char *mdr_base(const char *path) { static bool mdr_code_file(const char *path) { CBMLanguage lang = cbm_language_for_filename(mdr_base(path)); return lang != CBM_LANG_COUNT && lang != CBM_LANG_MARKDOWN && lang != CBM_LANG_RST && - lang != CBM_LANG_HTML && lang != CBM_LANG_CSS && lang != CBM_LANG_SCSS && - !cbm_has_config_extension(path); + lang != CBM_LANG_ASCIIDOC && lang != CBM_LANG_PDF && lang != CBM_LANG_HTML && + lang != CBM_LANG_CSS && lang != CBM_LANG_SCSS && !cbm_has_config_extension(path); } /* The stems of a package directory's index file. */ @@ -1205,6 +1207,228 @@ const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char * return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; } +/* ── A region of a file's text (the field test's H8 bind_range) ─────── */ + +/* A blank or comment-only line (H8 _skippable). */ +static bool mdr_skippable(const char *s, size_t n) { + size_t i = 0; + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r')) { + i++; + } + if (i == n) { + return true; + } + const char *t = s + i; + size_t r = n - i; + if ((r >= PAIR_LEN && t[0] == '/' && (t[1] == '/' || t[1] == '*')) || t[0] == '*' || + t[0] == ';' || t[0] == '%' || (r >= PAIR_LEN && t[0] == '\'' && t[1] == ' ') || + (r >= PAIR_LEN && t[0] == '-' && t[1] == '-' && r > PAIR_LEN && + (t[PAIR_LEN] == ' ' || t[PAIR_LEN] == '\t')) || + (r >= strlen("