diff --git a/Makefile.cbm b/Makefile.cbm index 3c34559142..b55cd9e50b 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -352,6 +352,17 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_dbt.c \ $(CBM_DIR)/doclink.c \ $(CBM_DIR)/doclink_cs.c \ + $(CBM_DIR)/doclink_md.c \ + $(CBM_DIR)/doclink_rst.c \ + $(CBM_DIR)/doclink_py.c \ + $(CBM_DIR)/doc_adr.c \ + $(CBM_DIR)/doc_pdf.c \ + $(CBM_DIR)/doc_adoc.c \ + $(CBM_DIR)/pdf/pdf_obj.c \ + $(CBM_DIR)/pdf/pdf_font.c \ + $(CBM_DIR)/pdf/pdf_text.c \ + $(CBM_DIR)/pdf/pdf_tables.c \ + $(CBM_DIR)/pdf/pdf_unicode.c \ $(CBM_DIR)/sql_values.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/result_compact.c \ @@ -458,6 +469,10 @@ PIPELINE_SRCS = \ src/pipeline/pass_doclinks.c \ src/pipeline/doc_links.c \ src/pipeline/doc_links_cs.c \ + src/pipeline/doc_links_md.c \ + src/pipeline/doc_links_pdf.c \ + src/pipeline/doc_links_rst.c \ + src/pipeline/doc_links_adoc.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -686,7 +701,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index c23f0ec4df..be8e2fea12 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -138,6 +138,109 @@ Semantic vector search uses static token embeddings derived from the See `vendored/nomic/NOTICE` for the exact derivation procedure (per-token inference + int8 quantization via `scripts/extract_nomic_vectors.py`). +## Embedded Character Data (PDF text layer) + +The PDF text-layer extractor (`internal/cbm/pdf/`) carries static tables +derived from the following sources. No upstream source code is included. + +- **Adobe Glyph List** — the glyph-name subset in `internal/cbm/pdf/pdf_tables.c` + (the names of the standard Latin encodings and common symbols, mapped to + Unicode) follows [adobe-type-tools/agl-aglfn](https://github.com/adobe-type-tools/agl-aglfn) + `glyphlist.txt`. + - **License:** BSD-3-Clause + - **Copyright:** Copyright 2002-2019 Adobe (http://www.adobe.com/). +- **Adobe core-14 font metrics** — the advance widths of the printable ASCII + range of Helvetica and Times-Roman in `pdf_tables.c` (used only for a + standard-14 font that omits its `/Widths`) are the values of Adobe's + core-14 AFM files. The AFM files themselves are not distributed. + - **Copyright:** Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems + Incorporated. All Rights Reserved. +- **Unicode Character Database** — `internal/cbm/pdf/pdf_unicode.c` (NFKC of + each code point, canonical compositions, character classes) is generated + by `scripts/gen-pdf-unicode.py` from CPython's `unicodedata` (Unicode + 16.0.0). + - **License:** Unicode License v3 (text below) + - **Copyright:** Copyright © 1991-2026 Unicode, Inc. +- The base encodings (Standard, WinAnsi, MacRoman, Symbol) are those of + ISO 32000-1, Annex D. + +Adobe Glyph List license (BSD-3-Clause): + +``` +Copyright 2002-2019 Adobe (http://www.adobe.com/). + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +Neither the name of Adobe nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +Unicode License v3: + +``` +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. +``` + ## Hybrid LSP — Reference Language Servers The Hybrid LSP layer (`internal/cbm/lsp/`) is an original C implementation diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index b351e5f2ed..25d8a2e210 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2617,6 +2617,45 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_test_fault_inject(rel_path); #endif + /* A PDF has no grammar: its text layer is read directly, a Section per + * page plus the pages' code mentions (doc_pdf.c). */ + if (language == CBM_LANG_PDF) { + result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language); + CBMExtractCtx pdf_ctx = { + .arena = a, + .scratch = scratch, + .result = result, + .source = source, + .source_len = source_len, + .language = language, + .project = project, + .rel_path = rel_path, + .module_qn = result->module_qn, + }; + cbm_pdf_extract_document(&pdf_ctx); + cbm_index_mark_done(rel_path); + return result; + } + /* AsciiDoc has no grammar either: its headings, includes, attribute + * references and monospace spans are read from the text (doc_adoc.c). */ + if (language == CBM_LANG_ASCIIDOC) { + result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language); + CBMExtractCtx adoc_ctx = { + .arena = a, + .scratch = scratch, + .result = result, + .source = source, + .source_len = source_len, + .language = language, + .project = project, + .rel_path = rel_path, + .module_qn = result->module_qn, + }; + cbm_adoc_extract_document(&adoc_ctx); + cbm_index_mark_done(rel_path); + return result; + } + // Get language spec const CBMLangSpec *spec = cbm_lang_spec(language); if (!spec) { diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 65f3554176..5bdef72604 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -188,6 +188,8 @@ typedef enum { CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI) CBM_LANG_PLSQL, // Oracle PL/SQL CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language) + CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c) + CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c) CBM_LANG_COUNT } CBMLanguage; @@ -260,6 +262,12 @@ typedef struct { * qualified_name (base QN = the first qn_sig_off bytes); 0 = no suffix. * Always 0 until a language enables its callable_identity mode. */ uint32_t qn_sig_off; + /* Documents only: facts a document definition adds to its node's + * properties, as a NULL-terminated key, value, key, value, ... list (an + * ADR's id, status, date, deciders, title). Keys are fixed literals; the + * values are escaped where the properties are written. Tail field: + * zero-init stays valid. */ + const char **extra_props; } CBMDefinition; /* Argument captured from a call expression */ diff --git a/internal/cbm/doc_adoc.c b/internal/cbm/doc_adoc.c new file mode 100644 index 0000000000..bfe70b69ae --- /dev/null +++ b/internal/cbm/doc_adoc.c @@ -0,0 +1,944 @@ +/* + * doc_adoc.c — AsciiDoc documents in the graph (the extraction half; + * src/pipeline/doc_links_adoc.c resolves), and the Antora scope blob. + * + * AsciiDoc has no grammar here: like a PDF, the file is read directly. Each + * heading (`= Title` .. `====== Title`, or `#` .. `######`) outside a + * delimited block becomes a Section (name the title, the text up to the next + * heading as its docstring, its lines up to the next heading); text before + * the first heading belongs to the file. Delimited blocks (`----` listing, + * `....` literal, `////` comment, `++++` passthrough, `====` example, `****` + * sidebar, `____` quote, `|===` table, ``` fences) open and close on the same + * delimiter line (the field test's H8 rules). + * + * References become tokens whose `raw` is the reference as written, then + * TAB-separated fields: + * include include::target[attrs] (Asciidoctor's preprocessor directive: + * read in every block but a comment block; `\include::` is + * escaped): written, target with the page's attributes + * substituted (references the page does not define stay as + * `{name}` for the component's), tags (`tag=` / `tags=`), + * lines (`lines=`) + * attribute an attribute reference `{name}` in the text: written, name, + * the page's value as written and as the page expands it (both + * "" when the page does not define it). + * Antora components define their API links as attributes + * (`{Assertions}` -> a javadoc page): the resolver reads them. + * code_path, a monospace span (`x`, ``x``, `+x+`) naming a path or a + * code_name qualified name (the Markdown code-span classifier); raw is + * the span's text + * Page attributes (`:name: value`, `:name!:` / `:!name:` to unset) apply from + * their line on, in document order. + * + * The Antora scope (cbm_doclink_antora_scan_scope): an antora.yml's component + * name, its asciidoc attributes and its collector scans; an + * antora-playbook*.yml's asciidoc attributes. Read as text, never run. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "helpers.h" + +#include +#include +#include +#include +#include + +enum { + AD_HEAD_MAX = 6, /* `======` */ + AD_DELIM_MIN = 4, /* `----` and its kin */ + AD_TABLE_MIN = 3, /* `|===` */ + AD_FENCE_MIN = 3, /* ``` */ + AD_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */ + AD_SPAN_MAX = 150, /* a span the classifier reads (MD_SPAN_MAX) */ + AD_ATTRS_INIT = 16, /* first capacity of the page attribute list */ + AD_EXPAND_DEPTH = 10, /* nested attribute references expanded (H8) */ + AD_FIELD_MAX = 512, /* an attribute value or target longer than this is skipped */ +}; + +typedef struct { + const char *s; + int len; +} ad_line_t; + +typedef struct { + char *name; + char *value; +} ad_attr_t; + +typedef struct { + int line; /* 0-based */ + const char *title; + const char *qn; + bool shared_qn; +} ad_head_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *scratch; + ad_line_t *L; + int n; + unsigned char *in_block; /* 1: inside a delimited block; 2: inside a comment block */ + ad_head_t *heads; + int nheads; + ad_attr_t *attrs; + int nattrs; + int cap_attrs; + char *span_buf; + bool failed; +} ad_doc_t; + +static bool ad_blank(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +static bool ad_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The line without trailing whitespace: its length. */ +static int ad_rlen(const ad_line_t *l) { + int n = l->len; + while (n > 0 && ad_blank(l->s[n - 1])) { + n--; + } + return n; +} + +/* ADOC_DELIM_RE: the delimiter's length, or 0. */ +static int ad_delim(const ad_line_t *l) { + int n = ad_rlen(l); + if (n == 0) { + return 0; + } + const char *s = l->s; + if (s[0] == '|') { + int k = 1; + while (k < n && s[k] == '=') { + k++; + } + return (k == n && n - 1 >= AD_TABLE_MIN) ? n : 0; + } + if (!strchr("-.=*_+/`", s[0])) { + return 0; + } + for (int k = 1; k < n; k++) { + if (s[k] != s[0]) { + return 0; + } + } + int min = s[0] == '`' ? AD_FENCE_MIN : AD_DELIM_MIN; + return n >= min ? n : 0; +} + +/* ADOC_HEAD_RE ^(={1,6}|#{1,6})\s+(\S.*?)\s*$: the level and the title span. */ +static int ad_heading(const ad_line_t *l, int *ta, int *tb) { + const char *s = l->s; + int n = ad_rlen(l); + if (n == 0 || (s[0] != '=' && s[0] != '#')) { + return 0; + } + int k = 0; + while (k < n && s[k] == s[0]) { + k++; + } + if (k > AD_HEAD_MAX || k >= n || !ad_blank(s[k])) { + return 0; + } + int a = k; + while (a < n && ad_blank(s[a])) { + a++; + } + if (a >= n) { + return 0; + } + *ta = a; + *tb = n; + return k; +} + +static bool ad_lines(ad_doc_t *d, const char *src, int len) { + int count = 1; + for (int i = 0; i < len; i++) { + count += src[i] == '\n'; + } + d->L = (ad_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L)); + d->in_block = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count); + if (!d->L || !d->in_block) { + return false; + } + memset(d->in_block, 0, (size_t)count); + int pos = 0; + for (int i = 0; i < count; i++) { + const char *nl = memchr(src + pos, '\n', (size_t)(len - pos)); + int end = nl ? (int)(nl - src) : len; + d->L[i] = (ad_line_t){src + pos, end - pos}; + pos = end + 1; + } + d->n = count; + return true; +} + +/* Delimited blocks, then the headings outside them. */ +static void ad_structure(ad_doc_t *d) { + int open = -1; + for (int i = 0; i < d->n; i++) { + int dl = ad_delim(&d->L[i]); + if (open < 0) { + if (dl) { + open = i; + d->in_block[i] = d->L[i].s[0] == '/' ? 2 : 1; + } + continue; + } + const ad_line_t *o = &d->L[open]; + d->in_block[i] = d->L[open].s[0] == '/' ? 2 : 1; + if (dl && dl == ad_rlen(o) && memcmp(d->L[i].s, o->s, (size_t)dl) == 0) { + open = -1; + } + } + d->heads = (ad_head_t *)cbm_arena_alloc(d->scratch, (size_t)d->n * sizeof(*d->heads)); + if (!d->heads) { + d->failed = true; + return; + } + for (int i = 0; i < d->n; i++) { + int ta; + int tb; + if (d->in_block[i] || ad_heading(&d->L[i], &ta, &tb) == 0) { + continue; + } + char *title = cbm_arena_strndup(d->ctx->arena, d->L[i].s + ta, (size_t)(tb - ta)); + if (!title) { + d->failed = true; + return; + } + d->heads[d->nheads++] = (ad_head_t){.line = i, .title = title}; + } +} + +/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */ +static const char *ad_qn_segment(CBMArena *a, const char *name) { + char *out = cbm_arena_strdup(a, name); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = false; + for (const char *p = name; *p; p++) { + if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = '-'; + } + in_ws = false; + out[w++] = *p; + } + out[w] = '\0'; + return out; +} + +/* The section's own text, whitespace collapsed, at most AD_BODY_MAX bytes and + * never a cut UTF-8 character. */ +static const char *ad_body(ad_doc_t *d, int from, int to) { + char *out = (char *)cbm_arena_alloc(d->ctx->arena, AD_BODY_MAX + 1); + if (!out) { + return NULL; + } + int w = 0; + for (int k = from; k < to && w < AD_BODY_MAX; k++) { + const char *s = d->L[k].s; + int len = d->L[k].len; + for (int i = 0; i < len && w < AD_BODY_MAX; i++) { + char c = s[i]; + if (ad_blank(c)) { + if (w > 0 && out[w - 1] != ' ') { + out[w++] = ' '; + } + continue; + } + out[w++] = c; + } + if (w > 0 && w < AD_BODY_MAX && out[w - 1] != ' ') { + out[w++] = ' '; + } + } + while (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0x80) { + w--; /* never end inside a character */ + } + if (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0xC0) { + w--; + } + while (w > 0 && out[w - 1] == ' ') { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +static int ad_head_qn_cmp(const void *a, const void *b) { + const ad_head_t *x = *(const ad_head_t *const *)a; + const ad_head_t *y = *(const ad_head_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static void ad_sections(ad_doc_t *d) { + CBMExtractCtx *ctx = d->ctx; + CBMArena *a = ctx->arena; + for (int h = 0; h < d->nheads; h++) { + const char *seg = ad_qn_segment(a, d->heads[h].title); + d->heads[h].qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL; + if (!d->heads[h].qn) { + d->failed = true; + return; + } + } + if (d->nheads > 1) { + ad_head_t **by = (ad_head_t **)cbm_arena_alloc(d->scratch, (size_t)d->nheads * sizeof(*by)); + if (!by) { + d->failed = true; + return; + } + for (int h = 0; h < d->nheads; h++) { + by[h] = &d->heads[h]; + } + qsort(by, (size_t)d->nheads, sizeof(*by), ad_head_qn_cmp); + for (int h = 1; h < d->nheads; h++) { + if (strcmp(by[h]->qn, by[h - 1]->qn) == 0) { + by[h]->shared_qn = true; + by[h - 1]->shared_qn = true; + } + } + } + for (int h = 0; h < d->nheads; h++) { + const ad_head_t *hd = &d->heads[h]; + int end = h + 1 < d->nheads ? d->heads[h + 1].line : d->n; + int last = end - 1; + while (last > hd->line && ad_rlen(&d->L[last]) == 0) { + last--; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = hd->title; + def.qualified_name = hd->qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = (uint32_t)hd->line + 1; + def.end_line = (uint32_t)last + 1; + def.is_exported = true; + def.docstring = ad_body(d, hd->line + 1, end); + cbm_defs_push(&ctx->result->defs, a, def); + } +} + +/* ── Page attributes ─────────────────────────────────────────────── */ + +static ad_attr_t *ad_attr_find(ad_doc_t *d, const char *name, size_t n) { + for (int i = 0; i < d->nattrs; i++) { + if (strlen(d->attrs[i].name) == n && memcmp(d->attrs[i].name, name, n) == 0) { + return &d->attrs[i]; + } + } + return NULL; +} + +/* ATTR_ENTRY_RE ^:(!?)([\w][\w\-]*)(!?):(?:\s+(.*?))?\s*$ -- true when the + * line is an attribute entry (set or unset). */ +static bool ad_attr_entry(ad_doc_t *d, const ad_line_t *l) { + const char *s = l->s; + int n = ad_rlen(l); + if (n < 3 || s[0] != ':') { + return false; + } + int k = 1; + bool unset = false; + if (s[k] == '!') { + unset = true; + k++; + } + int na = k; + if (k >= n || !ad_word(s[k])) { + return false; + } + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + int nb = k; + if (k < n && s[k] == '!') { + unset = true; + k++; + } + if (k >= n || s[k] != ':') { + return false; + } + k++; + if (k < n && !ad_blank(s[k])) { + return false; + } + while (k < n && ad_blank(s[k])) { + k++; + } + ad_attr_t *a = ad_attr_find(d, s + na, (size_t)(nb - na)); + if (unset) { + if (a) { + *a = d->attrs[--d->nattrs]; + } + return true; + } + char *value = cbm_arena_strndup(d->scratch, s + k, (size_t)(n - k)); + if (!value) { + d->failed = true; + return true; + } + if (a) { + a->value = value; + return true; + } + if (d->nattrs >= d->cap_attrs) { + int ncap = d->cap_attrs ? d->cap_attrs * 2 : AD_ATTRS_INIT; + ad_attr_t *grown = (ad_attr_t *)cbm_arena_alloc(d->scratch, (size_t)ncap * sizeof(*grown)); + if (!grown) { + d->failed = true; + return true; + } + if (d->nattrs) { + memcpy(grown, d->attrs, (size_t)d->nattrs * sizeof(*grown)); + } + d->attrs = grown; + d->cap_attrs = ncap; + } + char *name = cbm_arena_strndup(d->scratch, s + na, (size_t)(nb - na)); + if (!name) { + d->failed = true; + return true; + } + d->attrs[d->nattrs++] = (ad_attr_t){name, value}; + return true; +} + +/* A `{name}` reference at s[i] (not escaped `\{`): the name's span. */ +static bool ad_attr_ref(const char *s, int n, int i, int *a, int *b) { + if (s[i] != '{' || (i > 0 && s[i - 1] == '\\') || i + 2 >= n || !ad_word(s[i + 1])) { + return false; + } + int k = i + 1; + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + if (k >= n || s[k] != '}') { + return false; + } + *a = i + 1; + *b = k; + return true; +} + +/* The text with the page's attribute references substituted, pass after pass + * (a value may reference others; at most AD_EXPAND_DEPTH passes, as H8's + * depth); references the page does not define stay as written. NULL when it + * grows past AD_FIELD_MAX or memory ran out. */ +static char *ad_expand(ad_doc_t *d, const char *s, int n) { + char cur[AD_FIELD_MAX + 1]; + char out[AD_FIELD_MAX + 1]; + if (n > AD_FIELD_MAX) { + return NULL; + } + memcpy(cur, s, (size_t)n); + int cl = n; + for (int pass = 0; pass < AD_EXPAND_DEPTH; pass++) { + int w = 0; + bool changed = false; + for (int i = 0; i < cl; i++) { + int a; + int b; + const ad_attr_t *at = NULL; + if (ad_attr_ref(cur, cl, i, &a, &b)) { + at = ad_attr_find(d, cur + a, (size_t)(b - a)); + } + if (at) { + int vl = (int)strlen(at->value); + if (w + vl > AD_FIELD_MAX) { + return NULL; + } + memcpy(out + w, at->value, (size_t)vl); + w += vl; + i = b; + changed = true; + continue; + } + if (w + 1 > AD_FIELD_MAX) { + return NULL; + } + out[w++] = cur[i]; + } + if (w > 0) { + memcpy(cur, out, (size_t)w); + } + cl = w; + if (!changed) { + break; + } + } + return cbm_arena_strndup(d->scratch, cur, (size_t)cl); +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +static void ad_push(ad_doc_t *d, int line, int syntax, const char *raw) { + if (!raw) { + d->failed = true; + return; + } + CBMExtractCtx *ctx = d->ctx; + int lo = 0; + int hi = d->nheads; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (d->heads[mid].line <= line) { + lo = mid + 1; + } else { + hi = mid; + } + } + const ad_head_t *sec = lo > 0 ? &d->heads[lo - 1] : NULL; + if (sec && sec->shared_qn) { + sec = NULL; + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""), + .raw = raw, + .line = (uint32_t)line + 1, + .def_line = sec ? (uint32_t)sec->line + 1 : 1, + .syntax = (uint16_t)syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link); +} + +/* A value of the include's attribute list (`tag=x`, `tags="a;b"`, `lines=1..5`). */ +static void ad_attrlist_value(const char *al, int n, const char *key, char *out, size_t cap) { + out[0] = '\0'; + size_t kl = strlen(key); + for (int i = 0; i + (int)kl < n; i++) { + if ((i > 0 && ad_word(al[i - 1])) || memcmp(al + i, key, kl) != 0) { + continue; + } + int k = i + (int)kl; + while (k < n && ad_blank(al[k])) { + k++; + } + if (k >= n || al[k] != '=') { + continue; + } + k++; + while (k < n && ad_blank(al[k])) { + k++; + } + int a = k; + int b; + if (k < n && al[k] == '"') { + a = k + 1; + const char *q = memchr(al + a, '"', (size_t)(n - a)); + b = q ? (int)(q - al) : n; + } else { + b = a; + while (b < n && al[b] != ',') { + b++; + } + } + while (b > a && ad_blank(al[b - 1])) { + b--; + } + snprintf(out, cap, "%.*s", b - a, al + a); + return; + } +} + +/* INCLUDE_RE ^(\\?)include::([^\[\s][^\[]*)\[(.*)\]\s*$ */ +static void ad_include(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = ad_rlen(l); + static const char kw[] = "include::"; + int kl = (int)strlen(kw); + if (n < kl + 2 || memcmp(s, kw, (size_t)kl) != 0 || s[kl] == '[' || ad_blank(s[kl]) || + s[n - 1] != ']') { + return; + } + const char *open = memchr(s + kl, '[', (size_t)(n - kl)); + if (!open) { + return; + } + int ta = kl; + int tb = (int)(open - s); + int aa = tb + 1; + int ab = n - 1; + char *target = ad_expand(d, s + ta, tb - ta); + if (!target || strchr(target, '\t')) { + return; + } + char tags[AD_FIELD_MAX]; + char tag1[AD_FIELD_MAX]; + char lines[AD_FIELD_MAX]; + ad_attrlist_value(s + aa, ab - aa, "tags", tags, sizeof(tags)); + ad_attrlist_value(s + aa, ab - aa, "tag", tag1, sizeof(tag1)); + ad_attrlist_value(s + aa, ab - aa, "lines", lines, sizeof(lines)); + char *written = cbm_arena_strndup(d->ctx->arena, s, (size_t)n); + ad_push(d, i, CBM_DOCLINK_ADOC_INCLUDE, + written ? cbm_arena_sprintf(d->ctx->arena, "%s\t%s\t%s\t%s", written, target, + tags[0] ? tags : tag1, lines) + : NULL); +} + +/* Built-in attributes: characters and document settings, never a link. */ +static bool ad_builtin(const char *s, size_t n) { + static const char *const names[] = {"nbsp", "sp", + "empty", "blank", + "zwsp", "wj", + "apos", "quot", + "lsquo", "rsquo", + "ldquo", "rdquo", + "deg", "plus", + "brvbar", "vbar", + "amp", "lt", + "gt", "startsb", + "endsb", "caret", + "asterisk", "tilde", + "backslash", "backtick", + "two-colons", "two-semicolons", + "cpp", "cxx", + "pp", "docname", + "docdir", "docfile", + "doctitle", "toc", + "version", "revnumber", + "revdate", "author", + "email", "page-component-version", + NULL}; + for (int i = 0; names[i]; i++) { + if (strlen(names[i]) == n && memcmp(names[i], s, n) == 0) { + return true; + } + } + return false; +} + +static void ad_attr_refs(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = l->len; + for (int k = 0; k < n; k++) { + int a; + int b; + if (!ad_attr_ref(s, n, k, &a, &b) || ad_builtin(s + a, (size_t)(b - a))) { + continue; + } + const ad_attr_t *at = ad_attr_find(d, s + a, (size_t)(b - a)); + char *value = at ? ad_expand(d, at->value, (int)strlen(at->value)) : NULL; + if (at && (!value || strchr(value, '\t') || strchr(at->value, '\t'))) { + k = b; + continue; + } + ad_push(d, i, CBM_DOCLINK_ADOC_ATTRIBUTE, + cbm_arena_sprintf(d->ctx->arena, "{%.*s}\t%.*s\t%s\t%s", b - a, s + a, b - a, s + a, + at ? at->value : "", value ? value : "")); + k = b; + } +} + +/* Monospace spans of a prose line: `x` and ``x``, a `+x+` passthrough's inner + * text; each a code_path / code_name when the classifier names one. */ +static void ad_spans(ad_doc_t *d, int i) { + const ad_line_t *l = &d->L[i]; + const char *s = l->s; + int n = l->len; + int k = 0; + while (k < n) { + if (s[k] != '`') { + k++; + continue; + } + int run = 1; + while (k + run < n && s[k + run] == '`') { + run++; + } + if (run > 2) { + k += run; + continue; + } + int a = k + run; + int e = a; + while (e + run <= n && !(memcmp(s + e, run == 2 ? "``" : "`", (size_t)run) == 0)) { + e++; + } + if (e + run > n || e == a) { + k = a; + continue; + } + int b = e; + if (b - a >= 2 && s[a] == '+' && s[b - 1] == '+') { + a++; + b--; + } + CBMDocLinkMdPath p; + if (b > a && b - a <= AD_SPAN_MAX && + cbm_doclink_md_classify_span(s + a, (size_t)(b - a), d->span_buf, AD_SPAN_MAX + 1, + &p)) { + ad_push(d, i, + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_ADOC_CODE_NAME + : CBM_DOCLINK_ADOC_CODE_PATH, + cbm_arena_strndup(d->ctx->arena, s + a, (size_t)(b - a))); + } + k = e + run; + } +} + +static void ad_tokens(ad_doc_t *d) { + for (int i = 0; i < d->n && !d->failed; i++) { + if (d->in_block[i] == 2) { + continue; /* a comment block: nothing in it is read */ + } + const ad_line_t *l = &d->L[i]; + if (l->len >= 2 && l->s[0] == '/' && l->s[1] == '/' && !d->in_block[i]) { + continue; /* a line comment */ + } + if (!d->in_block[i] && ad_attr_entry(d, l)) { + continue; + } + if (l->len > 0 && l->s[0] == 'i') { + ad_include(d, i); + } + ad_attr_refs(d, i); + if (!d->in_block[i]) { + ad_spans(d, i); + } + } +} + +void cbm_adoc_extract_document(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + ad_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena}; + d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(AD_SPAN_MAX + 1) * 2); + if (!d.span_buf || !ad_lines(&d, ctx->source, ctx->source_len)) { + ctx->result->doc_links.failed = true; + return; + } + ad_structure(&d); + if (!d.failed) { + ad_sections(&d); + } + if (!d.failed) { + ad_tokens(&d); + } + if (d.failed) { + ctx->result->doc_links.failed = true; + } +} + +/* ── The Antora scope (antora.yml, antora-playbook*.yml) ─────────── */ + +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} ad_sb_t; + +static void ad_put(ad_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + 1 > sb->cap) { + size_t ncap = sb->cap ? sb->cap : AD_FIELD_MAX; + while (ncap < sb->len + n + 1) { + ncap *= 2; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +static void ad_rec(ad_sb_t *sb, const char *kind, const char *a, size_t an, const char *b, + size_t bn) { + for (size_t i = 0; i < an; i++) { + if (a[i] == '\t' || a[i] == '\n') { + return; + } + } + for (size_t i = 0; b && i < bn; i++) { + if (b[i] == '\t' || b[i] == '\n') { + return; + } + } + ad_put(sb, kind, strlen(kind)); + ad_put(sb, "\t", 1); + ad_put(sb, a, an); + if (b) { + ad_put(sb, "\t", 1); + ad_put(sb, b, bn); + } + ad_put(sb, "\n", 1); +} + +static int ad_indent(const char *s, int n) { + int k = 0; + while (k < n && s[k] == ' ') { + k++; + } + return k; +} + +/* `key: value` at s (after the indent): the key and value spans (value + * unquoted, comments not stripped: the field test's reading). */ +static bool ad_yaml_kv(const char *s, int n, int *ka, int *kb, int *va, int *vb) { + int k = 0; + if (k >= n || !(ad_word(s[k]))) { + return false; + } + while (k < n && (ad_word(s[k]) || s[k] == '-')) { + k++; + } + int e = k; + while (k < n && ad_blank(s[k])) { + k++; + } + if (k >= n || s[k] != ':') { + return false; + } + k++; + while (k < n && ad_blank(s[k])) { + k++; + } + *ka = 0; + *kb = e; + *va = k; + *vb = n; /* the caller's line has no trailing whitespace */ + return true; +} + +/* A value in matching quotes loses them. */ +static void ad_unquote(const char *s, int *va, int *vb) { + if (*vb - *va >= 2 && s[*va] == s[*vb - 1] && (s[*va] == '\'' || s[*va] == '"')) { + (*va)++; + (*vb)--; + } +} + +/* H8 parse_antora_yml: `name:`, `asciidoc: attributes:` and + * `ext: collector: scan: - dir: / into:`. */ +static void ad_antora(const char *src, int len, bool component, ad_sb_t *sb) { + bool in_asciidoc = false; + bool in_attrs = false; + int attrs_ind = -1; + bool in_scan = false; + int pos = 0; + while (pos <= len) { + const char *nl = memchr(src + pos, '\n', (size_t)(len - pos)); + int end = nl ? (int)(nl - src) : len; + const char *s = src + pos; + int n = end - pos; + while (n > 0 && ad_blank(s[n - 1])) { + n--; + } + int ind = ad_indent(s, n); + int ka; + int kb; + int va; + int vb; + bool kv = ad_yaml_kv(s + ind, n - ind, &ka, &kb, &va, &vb); + if (kv) { + ad_unquote(s + ind, &va, &vb); + } + if (ind == 0 && n > 0) { + in_asciidoc = kv && kb == (int)strlen("asciidoc") && memcmp(s, "asciidoc", 8) == 0; + in_attrs = false; + in_scan = false; + if (component && kv && kb == (int)strlen("name") && memcmp(s, "name", 4) == 0 && + vb > va) { + ad_rec(sb, "N", s + va, (size_t)(vb - va), NULL, 0); + } + } else if (in_asciidoc && kv && va == vb && kb == (int)strlen("attributes") && + memcmp(s + ind, "attributes", 10) == 0) { + in_attrs = true; + attrs_ind = -1; + } else if (in_attrs && n > ind && s[ind] != '#') { + if (attrs_ind < 0) { + attrs_ind = ind; + } + if (ind < attrs_ind) { + in_attrs = false; + } else if (ind == attrs_ind && kv) { + ad_rec(sb, "A", s + ind, (size_t)kb, s + ind + va, (size_t)(vb - va)); + } + } + if (component && kv && kb == (int)strlen("scan") && memcmp(s + ind, "scan", 4) == 0) { + in_scan = true; + } else if (component && in_scan) { + const char *t = s + ind; + int tn = n - ind; + if (tn > 1 && t[0] == '-' && ad_blank(t[1])) { + int sk = 1; + while (sk < tn && ad_blank(t[sk])) { + sk++; + } + t += sk; + tn -= sk; + } + int a; + int b; + int c; + int e; + bool tkv = ad_yaml_kv(t, tn, &a, &b, &c, &e); + if (tkv) { + ad_unquote(t, &c, &e); + } + if (tkv && e > c) { + if (b == 3 && memcmp(t, "dir", 3) == 0) { + ad_rec(sb, "D", t + c, (size_t)(e - c), NULL, 0); + } else if (b == 4 && memcmp(t, "into", 4) == 0) { + ad_rec(sb, "I", t + c, (size_t)(e - c), NULL, 0); + } + } + } + pos = end + 1; + } +} + +const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->rel_path || !ctx->source || ctx->source_len <= 0) { + return NULL; + } + const char *base = strrchr(ctx->rel_path, '/'); + base = base ? base + 1 : ctx->rel_path; + bool component = strcmp(base, "antora.yml") == 0; + bool playbook = strncmp(base, "antora-playbook", strlen("antora-playbook")) == 0; + if (!component && !playbook) { + return NULL; + } + ad_sb_t sb = {.a = ctx->arena}; + ad_put(&sb, CBM_DOCLINK_ADOC_SCOPE_TAG "\n", strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n")); + ad_put(&sb, component ? "C\n" : "P\n", 2); + ad_antora(ctx->source, ctx->source_len, component, &sb); + if (sb.failed) { + ctx->result->doc_links.failed = true; + return NULL; + } + return sb.buf; +} diff --git a/internal/cbm/doc_adr.c b/internal/cbm/doc_adr.c new file mode 100644 index 0000000000..27fc42efa0 --- /dev/null +++ b/internal/cbm/doc_adr.c @@ -0,0 +1,1535 @@ +/* + * doc_adr.c — architecture decision records: detection, facts, supersedes. + * + * A Markdown file that is an ADR gets one more definition: label "ADR", name + * its canonical id ("ADR-12"; another prefix as written, "DEC-5"; a dated + * record without a number keeps its file stem), qualified name + * ".__adr__", and the record's facts as node properties (adr_id, + * title, status, date, deciders, superseded_by). Its File DEFINES it like any + * definition, its sections link to code like any document's (doclink_md.c), + * and its own "supersedes X" / "replaces X" statements become tokens of the + * `supersedes` family, whose source is the ADR node: the resolver turns them + * into SUPERSEDES edges between ADR nodes. + * + * "Superseded by Y" is written in the OLD record, about another file: an edge + * from Y's node would be owned by a file whose text does not hold it, so it + * stays a fact of this record (status superseded, superseded_by) and the link + * itself is a MENTIONS edge of the status section. + * + * Detection (the field tests' rules, 99.2 % on 379 files): a file in a + * conventional ADR directory (adr, adrs, decisions, decision-records, ...) + * whose name carries an id or whose text has ADR structure; any `adr-NNN-*` + * file; a numbered or dated file with ADR structure. Never README, index or + * template files without an id, never test fixtures. Structure: a status, and + * a context, decision, consequences, decision outcome or considered options + * section. ADR-log configuration files (.adr-dir, log4brains) are not read: + * per-file detection, a pure function of the path and the text. + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "helpers.h" /* cbm_memmem */ + +#include +#include +#include +#include +#include + +enum { + ADR_ID_DIGITS = 5, /* an ADR number has at most this many digits */ + ADR_PADDED_DIGITS = 3, /* a bare number names a record when padded: `0259`, `012` */ + ADR_FIELD_KEY_MAX = 32, /* a field line key: `Status`, `Decision makers` */ + ADR_PREFIX_MIN = 2, /* `DEC-5`: a prefix of 2 ... */ + ADR_PREFIX_MAX = 12, /* ... to 12 letters */ + ADR_DOC_MAX = 500, /* the node's docstring: the decision, collapsed */ + ADR_FIELD_MAX = 256, /* a fact value is cut at a character boundary */ + ADR_FM_SCAN = 400, /* front matter ends within this many lines */ + ADR_HEAD_SCAN = 40, /* header fields within this many lines when there is no heading */ + ADR_NAME_MAX = 64, /* canonical id buffer */ + ADR_YEAR_MIN = 1900, + ADR_YEAR_MAX = 2099, + ADR_MONTHS = 12, + ADR_DAYS = 31, + ADR_YEAR_DIGITS = 4, + ADR_MD_DIGITS = 2, + ADR_DATE8 = 8, + ADR_ISO_LEN = 10, /* yyyy-mm-dd */ + ADR_MONTH_ABBR = 3, + ADR_BOM_LEN = 3, /* EF BB BF */ + ADR_RELATION_SPAN = 400, /* a relation's targets follow it in this many bytes */ +}; + +/* ── Text helpers ────────────────────────────────────────────────── */ + +static bool adr_blank(char c) { + return c == ' ' || c == '\t' || c == '\r'; +} + +static bool adr_digit(char c) { + return c >= '0' && c <= '9'; +} + +static bool adr_alpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static char adr_lower(char c) { + return (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; +} + +/* s[0..n) starts with `word`, ignoring case. */ +static bool adr_starts_ci(const char *s, size_t n, const char *word) { + size_t w = strlen(word); + if (n < w) { + return false; + } + for (size_t i = 0; i < w; i++) { + if (adr_lower(s[i]) != word[i]) { + return false; + } + } + return true; +} + +static bool adr_eq_ci(const char *s, size_t n, const char *word) { + return n == strlen(word) && adr_starts_ci(s, n, word); +} + +/* Trim blanks and Markdown emphasis/code marks around a value. */ +static void adr_trim(const char **s, size_t *n) { + const char *p = *s; + size_t len = *n; + while (len > 0 && (adr_blank(*p) || *p == '*' || *p == '_' || *p == '`' || *p == '"')) { + p++; + len--; + } + while (len > 0 && + (adr_blank(p[len - SKIP_ONE]) || p[len - SKIP_ONE] == '*' || p[len - SKIP_ONE] == '_' || + p[len - SKIP_ONE] == '`' || p[len - SKIP_ONE] == '"')) { + len--; + } + *s = p; + *n = len; +} + +/* A copy of s[0..n) with Markdown links reduced to their text, `**`, `__` + * and backticks dropped, blanks collapsed, cut at `cap` bytes on a UTF-8 + * boundary. NULL when nothing is left. */ +static char *adr_plain(CBMArena *a, const char *s, size_t n, size_t cap) { + char *out = (char *)cbm_arena_alloc(a, (n < cap ? n : cap) + SKIP_ONE); + if (!out) { + return NULL; + } + size_t w = 0; + bool space = false; + for (size_t i = 0; i < n && w < cap; i++) { + char c = s[i]; + if (c == '[') { + continue; /* [text](target): keep the text */ + } + if (c == ']' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '(') { + const char *close = memchr(s + i, ')', n - i); + if (close) { + i = (size_t)(close - s); + continue; + } + } + if (c == '*' || c == '`' || (c == '_' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '_')) { + continue; + } + if (adr_blank(c) || c == '\n') { + space = w > 0; + continue; + } + if (space && w < cap) { + out[w++] = ' '; + } + space = false; + if (w < cap) { + out[w++] = c; + } + } + while (w > 0 && ((unsigned char)out[w - SKIP_ONE] & 0xC0) == 0x80) { + w--; /* never end inside a character */ + } + if (w > 0 && ((unsigned char)out[w - SKIP_ONE] & 0xC0) == 0xC0) { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +typedef struct { + const char *s; /* the line, not NUL-terminated */ + int n; + bool text; /* not front matter, not fenced code, not an HTML comment */ +} adr_line_t; + +typedef struct { + adr_line_t *v; + int count; + int fm_end; /* the first line after the front matter (0: none) */ +} adr_lines_t; + +static bool adr_trim_eq(const char *s, int n, const char *word) { + const char *p = s; + size_t len = (size_t)n; + while (len > 0 && adr_blank(*p)) { + p++; + len--; + } + while (len > 0 && adr_blank(p[len - SKIP_ONE])) { + len--; + } + return len == strlen(word) && memcmp(p, word, len) == 0; +} + +static bool adr_split_lines(CBMArena *a, const char *src, int len, adr_lines_t *out) { + int count = 0; + for (int i = 0; i < len; i++) { + count += src[i] == '\n'; + } + count++; + out->v = (adr_line_t *)cbm_arena_alloc(a, (size_t)count * sizeof(*out->v)); + if (!out->v) { + return false; + } + out->count = 0; + out->fm_end = 0; + int pos = 0; + if (len >= ADR_BOM_LEN && memcmp(src, "\xEF\xBB\xBF", ADR_BOM_LEN) == 0) { + pos = ADR_BOM_LEN; /* a byte-order mark is no part of the first line */ + } + while (pos <= len && out->count < count) { + const char *nl = pos < len ? memchr(src + pos, '\n', (size_t)(len - pos)) : NULL; + int end = nl ? (int)(nl - src) : len; + int n = end - pos; + if (n > 0 && src[pos + n - SKIP_ONE] == '\r') { + n--; + } + out->v[out->count++] = (adr_line_t){.s = src + pos, .n = n, .text = true}; + pos = end + SKIP_ONE; + } + /* front matter */ + if (out->count > 0 && adr_trim_eq(out->v[0].s, out->v[0].n, "---")) { + for (int k = 1; k < out->count && k < ADR_FM_SCAN; k++) { + if (adr_trim_eq(out->v[k].s, out->v[k].n, "---") || + adr_trim_eq(out->v[k].s, out->v[k].n, "...")) { + out->fm_end = k + SKIP_ONE; + break; + } + } + } + for (int k = 0; k < out->fm_end; k++) { + out->v[k].text = false; + } + /* fences and HTML comments */ + char fence = 0; + int fence_len = 0; + bool comment = false; + for (int k = out->fm_end; k < out->count; k++) { + adr_line_t *l = &out->v[k]; + int i = 0; + while (i < l->n && adr_blank(l->s[i])) { + i++; + } + int run = 0; + while (i + run < l->n && (l->s[i + run] == '`' || l->s[i + run] == '~') && + l->s[i + run] == l->s[i]) { + run++; + } + if (fence) { + l->text = false; + if (run >= fence_len && l->s[i] == fence) { + fence = 0; + } + continue; + } + if (run >= PAIR_LEN + SKIP_ONE) { + fence = l->s[i]; + fence_len = run; + l->text = false; + continue; + } + if (comment) { + l->text = false; + if (cbm_memmem(l->s, (size_t)l->n, "-->", PAIR_LEN + SKIP_ONE)) { + comment = false; + } + continue; + } + if (l->n - i >= 4 && memcmp(l->s + i, "", PAIR_LEN + SKIP_ONE)) { + comment = true; + l->text = false; + } + } + return true; +} + +/* ── Front matter (top-level scalars and lists) ──────────────────── */ + +/* The value of front-matter key `key` (a list joined with ", "), or NULL. */ +static char *adr_fm_value(CBMArena *a, const adr_lines_t *L, const char *key) { + size_t kl = strlen(key); + for (int k = SKIP_ONE; k < L->fm_end - SKIP_ONE; k++) { + const adr_line_t *l = &L->v[k]; + if ((size_t)l->n <= kl || l->s[0] == ' ' || l->s[0] == '\t' || + !adr_starts_ci(l->s, (size_t)l->n, key) || l->s[kl] != ':') { + continue; + } + const char *v = l->s + kl + SKIP_ONE; + size_t vn = (size_t)l->n - kl - SKIP_ONE; + adr_trim(&v, &vn); + if (vn > 0 && v[0] == '[' && v[vn - SKIP_ONE] == ']') { + v++; + vn -= PAIR_LEN; + } + if (vn > 0) { + return adr_plain(a, v, vn, ADR_FIELD_MAX); + } + /* a block list on the following lines */ + char buf[ADR_FIELD_MAX + SKIP_ONE]; + size_t w = 0; + for (int j = k + SKIP_ONE; j < L->fm_end - SKIP_ONE; j++) { + const adr_line_t *it = &L->v[j]; + int i = 0; + while (i < it->n && adr_blank(it->s[i])) { + i++; + } + if (i == 0 || i >= it->n || it->s[i] != '-') { + break; + } + const char *iv = it->s + i + SKIP_ONE; + size_t ivn = (size_t)(it->n - i - SKIP_ONE); + adr_trim(&iv, &ivn); + if (ivn > 0 && w + ivn + PAIR_LEN < sizeof(buf)) { + if (w > 0) { + buf[w++] = ','; + buf[w++] = ' '; + } + memcpy(buf + w, iv, ivn); + w += ivn; + } + } + return w > 0 ? adr_plain(a, buf, w, ADR_FIELD_MAX) : NULL; + } + return NULL; +} + +/* ── Headings (the file's Section definitions) ───────────────────── */ + +typedef struct { + int line; /* 0-based index into the lines */ + int level; /* 1..6 */ + const char *title; + size_t title_n; +} adr_head_t; + +static int adr_head_cmp(const void *a, const void *b) { + const adr_head_t *x = (const adr_head_t *)a; + const adr_head_t *y = (const adr_head_t *)b; + return (x->line > y->line) - (x->line < y->line); +} + +/* The level of the heading on line k: `#` count, or 1/2 for a setext + * underline of `=`/`-` on the next line; 0 when the line is no heading. */ +static int adr_heading_level(const adr_lines_t *L, int k, const char **title, size_t *title_n) { + const adr_line_t *l = &L->v[k]; + int i = 0; + while (i < l->n && i < (PAIR_LEN + SKIP_ONE) && l->s[i] == ' ') { + i++; + } + int hashes = 0; + while (i + hashes < l->n && l->s[i + hashes] == '#') { + hashes++; + } + if (hashes > 0 && hashes <= (PAIR_LEN * PAIR_LEN + PAIR_LEN) && + (i + hashes == l->n || adr_blank(l->s[i + hashes]))) { + const char *t = l->s + i + hashes; + size_t tn = (size_t)(l->n - i - hashes); + while (tn > 0 && (t[tn - SKIP_ONE] == '#' || adr_blank(t[tn - SKIP_ONE]))) { + tn--; + } + adr_trim(&t, &tn); + *title = t; + *title_n = tn; + return hashes; + } + if (k + SKIP_ONE < L->count) { + const adr_line_t *u = &L->v[k + SKIP_ONE]; + int j = 0; + while (j < u->n && u->s[j] == ' ') { + j++; + } + char c = j < u->n ? u->s[j] : 0; + int run = 0; + while (j + run < u->n && u->s[j + run] == c) { + run++; + } + bool rest_blank = true; + for (int r = j + run; r < u->n; r++) { + rest_blank = rest_blank && adr_blank(u->s[r]); + } + if ((c == '=' || c == '-') && run >= SKIP_ONE && rest_blank) { + const char *t = l->s; + size_t tn = (size_t)l->n; + adr_trim(&t, &tn); + *title = t; + *title_n = tn; + return c == '=' ? SKIP_ONE : PAIR_LEN; + } + } + return 0; +} + +/* Section kinds the facts and the detection read. */ +typedef enum { + ADR_SEC_OTHER = 0, + ADR_SEC_STATUS, + ADR_SEC_CONTEXT, + ADR_SEC_DECISION, + ADR_SEC_OUTCOME, + ADR_SEC_OPTIONS, + ADR_SEC_CONSEQUENCES, +} adr_sec_t; + +static adr_sec_t adr_section_kind(const char *t, size_t n) { + /* a leading "1." / "2.3 " numbering is no part of the title */ + size_t i = 0; + while (i < n && (adr_digit(t[i]) || t[i] == '.')) { + i++; + } + if (i > 0 && i < n && t[i] == ' ') { + t += i + SKIP_ONE; + n -= i + SKIP_ONE; + } + while (n > 0 && (t[n - SKIP_ONE] == ':' || adr_blank(t[n - SKIP_ONE]))) { + n--; + } + if (adr_eq_ci(t, n, "status") || adr_starts_ci(t, n, "status:")) { + return ADR_SEC_STATUS; + } + if (adr_eq_ci(t, n, "context") || adr_starts_ci(t, n, "context and problem") || + adr_eq_ci(t, n, "problem") || adr_eq_ci(t, n, "problem statement") || + adr_eq_ci(t, n, "background") || adr_eq_ci(t, n, "motivation")) { + return ADR_SEC_CONTEXT; + } + if (adr_eq_ci(t, n, "decision outcome")) { + return ADR_SEC_OUTCOME; + } + if (adr_eq_ci(t, n, "decision") || adr_eq_ci(t, n, "decisions") || + adr_eq_ci(t, n, "the decision")) { + return ADR_SEC_DECISION; + } + if (adr_eq_ci(t, n, "considered options") || adr_eq_ci(t, n, "options considered") || + adr_eq_ci(t, n, "alternatives") || adr_eq_ci(t, n, "alternatives considered")) { + return ADR_SEC_OPTIONS; + } + if (adr_eq_ci(t, n, "consequences") || adr_starts_ci(t, n, "positive consequences") || + adr_starts_ci(t, n, "negative consequences")) { + return ADR_SEC_CONSEQUENCES; + } + return ADR_SEC_OTHER; +} + +/* ── File names ──────────────────────────────────────────────────── */ + +typedef enum { + ADR_ID_NONE = 0, + ADR_ID_ADR, /* adr-012-title.md */ + ADR_ID_NUMBER, /* 0012-title.md */ + ADR_ID_PREFIX, /* DEC-005-title.md */ + ADR_ID_DATE, /* 2023-04-05-title.md, 20230405-title.md */ +} adr_id_kind_t; + +typedef struct { + adr_id_kind_t kind; + unsigned number; + char prefix[ADR_PREFIX_MAX + SKIP_ONE]; + char date[ADR_ISO_LEN + SKIP_ONE]; +} adr_id_t; + +/* Digits at s, at most ADR_ID_DIGITS, followed by a separator or the end. */ +static bool adr_number_at(const char *s, size_t n, size_t *used, unsigned *value) { + size_t i = 0; + unsigned v = 0; + while (i < n && adr_digit(s[i])) { + if (i >= ADR_ID_DIGITS + ADR_ID_DIGITS) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (unsigned)(s[i] - '0'); + i++; + } + size_t significant = i; + for (size_t z = 0; z < i && s[z] == '0' && significant > SKIP_ONE; z++) { + significant--; + } + if (i == 0 || significant > ADR_ID_DIGITS) { + return false; + } + if (i < n && s[i] != '-' && s[i] != '_' && s[i] != ' ' && s[i] != '.') { + return false; + } + *used = i; + *value = v; + return true; +} + +static bool adr_date_prefix(const char *s, size_t n, char *iso) { + /* yyyy-mm-dd or yyyymmdd, then '-' or '_' */ + if (n > ADR_ISO_LEN && adr_digit(s[0]) && s[4] == '-' && s[7] == '-' && + (s[ADR_ISO_LEN] == '-' || s[ADR_ISO_LEN] == '_')) { + for (int i = 0; i < ADR_ISO_LEN; i++) { + if (i != 4 && i != 7 && !adr_digit(s[i])) { + return false; + } + } + memcpy(iso, s, ADR_ISO_LEN); + iso[ADR_ISO_LEN] = '\0'; + return (s[0] == '1' && s[1] == '9') || (s[0] == '2' && s[1] == '0'); + } + if (n > ADR_DATE8 && (s[ADR_DATE8] == '-' || s[ADR_DATE8] == '_')) { + for (int i = 0; i < ADR_DATE8; i++) { + if (!adr_digit(s[i])) { + return false; + } + } + snprintf(iso, ADR_ISO_LEN + SKIP_ONE, "%.4s-%.2s-%.2s", s, s + 4, s + 6); + return (s[0] == '1' && s[1] == '9') || (s[0] == '2' && s[1] == '0'); + } + return false; +} + +static void adr_filename_id(const char *base, adr_id_t *id) { + memset(id, 0, sizeof(*id)); + size_t n = strlen(base); + size_t used = 0; + unsigned v = 0; + if (adr_starts_ci(base, n, "adr")) { + size_t i = PAIR_LEN + SKIP_ONE; + if (i < n && (base[i] == '-' || base[i] == '_' || base[i] == ' ')) { + i++; + } + if (adr_number_at(base + i, n - i, &used, &v)) { + id->kind = ADR_ID_ADR; + id->number = v; + return; + } + } + if (adr_date_prefix(base, n, id->date)) { + id->kind = ADR_ID_DATE; + return; + } + if (adr_number_at(base, n, &used, &v) && used < n) { + id->kind = ADR_ID_NUMBER; + id->number = v; + return; + } + size_t p = 0; + while (p < n && adr_alpha(base[p])) { + p++; + } + if (p >= ADR_PREFIX_MIN && p <= ADR_PREFIX_MAX && p < n && (base[p] == '-' || base[p] == '_') && + adr_number_at(base + p + SKIP_ONE, n - p - SKIP_ONE, &used, &v) && + p + SKIP_ONE + used < n) { + id->kind = ADR_ID_PREFIX; + id->number = v; + for (size_t i = 0; i < p; i++) { + id->prefix[i] = + (base[i] >= 'a' && base[i] <= 'z') ? (char)(base[i] - 'a' + 'A') : base[i]; + } + id->prefix[p] = '\0'; + } +} + +static const char *const ADR_CONV_DIRS[] = {"adr", + "adrs", + "decisions", + "decision-records", + "decision_records", + "architecture-decisions", + "architecture_decisions", + "architectural-decisions", + "architecture-decision-records", + "decision-log", + NULL}; + +static const char *const ADR_FIXTURE_DIRS[] = { + "test", "tests", "__tests__", "integration-tests", "integration_tests", + "e2e-tests", "e2e_tests", "testdata", "test-data", "test_data", + "fixture", "fixtures", "__fixtures__", "__mocks__", "mock", + "mocks", NULL}; + +static const char *const ADR_NON_ADR_STEMS[] = {"readme", "index", "_index", "toc", + "summary", "contributing", "process", "changelog", + "glossary", "license", NULL}; + +static bool adr_in(const char *s, size_t n, const char *const *list) { + for (size_t i = 0; list[i]; i++) { + if (adr_eq_ci(s, n, list[i])) { + return true; + } + } + return false; +} + +/* Does a directory of `rel` match the list? */ +static bool adr_dir_in(const char *rel, const char *const *list) { + const char *s = rel; + for (const char *slash = strchr(s, '/'); slash; slash = strchr(s, '/')) { + if (adr_in(s, (size_t)(slash - s), list)) { + return true; + } + s = slash + SKIP_ONE; + } + return false; +} + +/* README, index, template ... (only for a name without an id). */ +static bool adr_non_adr_name(const char *base) { + size_t n = strlen(base); + const char *dot = strchr(base, '.'); + size_t stem = dot ? (size_t)(dot - base) : n; + if (adr_in(base, stem, ADR_NON_ADR_STEMS)) { + return true; + } + for (size_t i = 0; i + strlen("template") <= n; i++) { + bool start = i == 0 || base[i - SKIP_ONE] == '-' || base[i - SKIP_ONE] == '_' || + base[i - SKIP_ONE] == '.'; + size_t e = i + strlen("template"); + bool end = e == n || base[e] == '-' || base[e] == '_' || base[e] == '.'; + if (start && end && adr_starts_ci(base + i, n - i, "template")) { + return true; + } + } + return false; +} + +/* ── Status, dates, people ───────────────────────────────────────── */ + +static const struct { + const char *status; + const char *words[8]; +} ADR_STATUS_WORDS[] = { + {"superseded", {"supersed", "replaced by", NULL}}, + {"deprecated", {"deprecat", "obsolete", NULL}}, + {"rejected", {"reject", "declined", NULL}}, + {"abandoned", {"abandon", "withdrawn", "cancelled", "canceled", NULL}}, + {"implemented", {"implemented", "done", "completed", "complete", NULL}}, + {"accepted", {"accept", "approved", "adopted", "agreed", "decided", "active", "final", NULL}}, + {"proposed", + {"propos", "in review", "under review", "pending", "rfc", "under consideration", NULL}}, + {"draft", {"draft", "wip", "work in progress", "in progress", NULL}}, +}; + +/* The normalized status of a status text: the first status word (at a word + * start, not after "not ", "non-" or "un"), "unknown" when none. */ +static const char *adr_norm_status(const char *s) { + size_t n = strlen(s); + for (size_t i = 0; i < n; i++) { + if (i > 0 && (adr_alpha(s[i - SKIP_ONE]) || adr_digit(s[i - SKIP_ONE]))) { + continue; + } + bool negated = (i >= 4 && adr_starts_ci(s + i - 4, 4, "not ")) || + (i >= 4 && adr_starts_ci(s + i - 4, 4, "non-")); + for (size_t k = 0; k < sizeof(ADR_STATUS_WORDS) / sizeof(ADR_STATUS_WORDS[0]); k++) { + for (size_t w = 0; ADR_STATUS_WORDS[k].words[w]; w++) { + /* a whole word, or a stem ("accept" in "accepted") */ + if (adr_starts_ci(s + i, n - i, ADR_STATUS_WORDS[k].words[w]) && !negated && + !(i >= PAIR_LEN && adr_starts_ci(s + i - PAIR_LEN, PAIR_LEN, "un"))) { + return ADR_STATUS_WORDS[k].status; + } + } + } + } + return "unknown"; +} + +static const char *const ADR_MONTHS_ABBR[ADR_MONTHS] = {"jan", "feb", "mar", "apr", "may", "jun", + "jul", "aug", "sep", "oct", "nov", "dec"}; + +static int adr_month(const char *s, size_t n) { + if (n < ADR_MONTH_ABBR) { + return 0; + } + for (int m = 0; m < ADR_MONTHS; m++) { + if (adr_starts_ci(s, n, ADR_MONTHS_ABBR[m])) { + return m + SKIP_ONE; + } + } + return 0; +} + +static int adr_num(const char *s, size_t n, size_t *used) { + size_t i = 0; + int v = 0; + while (i < n && adr_digit(s[i]) && i < ADR_YEAR_DIGITS + SKIP_ONE) { + v = (v * CBM_DECIMAL_BASE) + (s[i] - '0'); + i++; + } + *used = i; + return v; +} + +/* An explicit date in s: yyyy-mm-dd (or / .), yyyymmdd, "d Month yyyy", + * "Month d, yyyy", or "Month yyyy" (month precision). Ambiguous numeric + * day/month orders give nothing. */ +static bool adr_parse_date(const char *s, char *out, size_t cap) { + size_t n = strlen(s); + for (size_t i = 0; i < n; i++) { + if (i > 0 && (adr_digit(s[i - SKIP_ONE]) || adr_alpha(s[i - SKIP_ONE]))) { + continue; + } + size_t u = 0; + int y = adr_num(s + i, n - i, &u); + if (u == ADR_YEAR_DIGITS && y >= ADR_YEAR_MIN && y <= ADR_YEAR_MAX && i + u < n && + (s[i + u] == '-' || s[i + u] == '/' || s[i + u] == '.')) { + size_t u2 = 0; + int mo = adr_num(s + i + u + SKIP_ONE, n - i - u - SKIP_ONE, &u2); + size_t at = i + u + SKIP_ONE + u2; + if (u2 >= SKIP_ONE && u2 <= ADR_MD_DIGITS && at < n && s[at] == s[i + u]) { + size_t u3 = 0; + int d = adr_num(s + at + SKIP_ONE, n - at - SKIP_ONE, &u3); + if (u3 >= SKIP_ONE && u3 <= ADR_MD_DIGITS && mo >= 1 && mo <= ADR_MONTHS && + d >= 1 && d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", y, mo, d); + return true; + } + } + } + /* yyyy Month d: "2019 Jul 31" */ + if (u == ADR_YEAR_DIGITS && y >= ADR_YEAR_MIN && y <= ADR_YEAR_MAX && i + u < n && + (s[i + u] == ' ' || s[i + u] == ',')) { + const char *p = s + i + u; + while (p < s + n && (*p == ' ' || *p == ',')) { + p++; + } + int mo = adr_month(p, (size_t)(s + n - p)); + if (mo) { + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ' ' || *p == ',')) { + p++; + } + size_t ud = 0; + int d = adr_num(p, (size_t)(s + n - p), &ud); + if (ud >= SKIP_ONE && ud <= ADR_MD_DIGITS && d >= 1 && d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", y, mo, d); + return true; + } + } + } + /* a/b/yyyy, a.b.yyyy: only when one of a, b is above 12 */ + if (u >= SKIP_ONE && u <= ADR_MD_DIGITS && i + u < n && + (s[i + u] == '/' || s[i + u] == '.')) { + char sep = s[i + u]; + size_t u2 = 0; + int b = adr_num(s + i + u + SKIP_ONE, n - i - u - SKIP_ONE, &u2); + size_t at = i + u + SKIP_ONE + u2; + size_t u3 = 0; + int yy = (u2 >= SKIP_ONE && u2 <= ADR_MD_DIGITS && at < n && s[at] == sep) + ? adr_num(s + at + SKIP_ONE, n - at - SKIP_ONE, &u3) + : 0; + if (u3 == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX) { + int mo = 0; + int d = 0; + if (y > ADR_MONTHS && b >= 1 && b <= ADR_MONTHS && y <= ADR_DAYS) { + d = y; + mo = b; + } else if (b > ADR_MONTHS && y >= 1 && y <= ADR_MONTHS && b <= ADR_DAYS) { + mo = y; + d = b; + } + if (mo) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } + } + if (u == ADR_DATE8) { + int yy = y / 10000; + int mo = (y / 100) % 100; + int d = y % 100; + if (yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && mo >= 1 && mo <= ADR_MONTHS && d >= 1 && + d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } + /* d Month yyyy */ + if (u >= SKIP_ONE && u <= ADR_MD_DIGITS && i + u < n && s[i + u] == ' ') { + const char *m = s + i + u + SKIP_ONE; + int mo = adr_month(m, n - (size_t)(m - s)); + if (mo) { + const char *p = m; + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ',' || *p == ' ')) { + p++; + } + size_t uy = 0; + int yy = adr_num(p, (size_t)(s + n - p), &uy); + if (uy == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && y >= 1 && + y <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, y); + return true; + } + } + } + /* Month d, yyyy / Month yyyy */ + int mo = u == 0 ? adr_month(s + i, n - i) : 0; + if (mo) { + const char *p = s + i; + while (p < s + n && adr_alpha(*p)) { + p++; + } + while (p < s + n && (*p == '.' || *p == ' ')) { + p++; + } + size_t ud = 0; + int d = adr_num(p, (size_t)(s + n - p), &ud); + if (ud >= SKIP_ONE && ud <= ADR_MD_DIGITS) { + const char *q = p + ud; + while (q < s + n && (adr_alpha(*q) || *q == ',' || *q == ' ')) { + q++; /* "5th, " */ + } + size_t uy = 0; + int yy = adr_num(q, (size_t)(s + n - q), &uy); + if (uy == ADR_YEAR_DIGITS && yy >= ADR_YEAR_MIN && yy <= ADR_YEAR_MAX && d >= 1 && + d <= ADR_DAYS) { + snprintf(out, cap, "%04d-%02d-%02d", yy, mo, d); + return true; + } + } else if (ud == ADR_YEAR_DIGITS && d >= ADR_YEAR_MIN && d <= ADR_YEAR_MAX) { + snprintf(out, cap, "%04d-%02d", d, mo); + return true; + } + } + } + return false; +} + +/* ── Header fields ───────────────────────────────────────────────── */ + +/* `Status: Accepted`, `* Status: Accepted`, `**Status:** Accepted`, + * `| Status | Accepted |`: the value of `key` on line l, or NULL. */ +static char *adr_field(CBMArena *a, const adr_line_t *l, const char *key) { + char buf[ADR_FIELD_MAX * PAIR_LEN]; + size_t w = 0; + for (int i = 0; i < l->n && w + SKIP_ONE < sizeof(buf); i++) { + if (l->s[i] == '*' || (l->s[i] == '_' && i + SKIP_ONE < l->n && l->s[i + 1] == '_')) { + continue; /* emphasis around the key */ + } + buf[w++] = l->s[i]; + } + buf[w] = '\0'; + const char *p = buf; + while (*p && adr_blank(*p)) { + p++; + } + bool table = *p == '|'; + if (table) { + p++; + while (*p && adr_blank(*p)) { + p++; + } + } else if ((*p == '-' || *p == '+') && adr_blank(p[SKIP_ONE])) { + p += PAIR_LEN; + while (*p && adr_blank(*p)) { + p++; + } + } + size_t kl = strlen(key); + if (!adr_starts_ci(p, strlen(p), key)) { + return NULL; + } + p += kl; + while (*p && adr_blank(*p)) { + p++; + } + if (*p != (table ? '|' : ':')) { + return NULL; + } + p++; + const char *end = table ? strchr(p, '|') : NULL; + size_t vn = end ? (size_t)(end - p) : strlen(p); + adr_trim(&p, &vn); + return vn > 0 ? adr_plain(a, p, vn, ADR_FIELD_MAX) : NULL; +} + +/* ── The record ──────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + adr_lines_t L; + adr_head_t *heads; + int nheads; + int title_head; /* index into heads, or -1 */ + bool numbered; /* the file name gives the record a number: */ + unsigned number; /* this one */ +} adr_doc_t; + +/* The paragraph after heading h (its first non-blank text lines). */ +static char *adr_first_paragraph(CBMArena *a, const adr_doc_t *d, int h, size_t cap) { + int start = d->heads[h].line + SKIP_ONE; + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + int k = start; + while (k < end && + (!d->L.v[k].text || d->L.v[k].n == 0 || adr_trim_eq(d->L.v[k].s, d->L.v[k].n, "") || + d->L.v[k].s[0] == '=' || d->L.v[k].s[0] == '-')) { + k++; + } + if (k >= end) { + return NULL; + } + const char *s = d->L.v[k].s; + int last = k; + while (last + SKIP_ONE < end && d->L.v[last + SKIP_ONE].text && + !adr_trim_eq(d->L.v[last + SKIP_ONE].s, d->L.v[last + SKIP_ONE].n, "")) { + last++; + } + const char *e = d->L.v[last].s + d->L.v[last].n; + return adr_plain(a, s, (size_t)(e - s), cap); +} + +/* The text of section h, as one collapsed string. */ +static char *adr_section_text(CBMArena *a, const adr_doc_t *d, int h, size_t cap) { + int start = d->heads[h].line + SKIP_ONE; + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + if (start >= end) { + return NULL; + } + const char *s = d->L.v[start].s; + const char *e = d->L.v[end - SKIP_ONE].s + d->L.v[end - SKIP_ONE].n; + return e > s ? adr_plain(a, s, (size_t)(e - s), cap) : NULL; +} + +static int adr_find_section(const adr_doc_t *d, adr_sec_t kind) { + for (int h = 0; h < d->nheads; h++) { + if (adr_section_kind(d->heads[h].title, d->heads[h].title_n) == kind) { + return h; + } + } + return CBM_NOT_FOUND; +} + +/* The value of header field `key` between the title and the first body + * heading (the first lines when there is no heading). */ +static char *adr_header_field(CBMArena *a, const adr_doc_t *d, const char *key) { + int from = d->title_head >= 0 ? d->heads[d->title_head].line + SKIP_ONE : d->L.fm_end; + int to = d->L.count; + for (int h = 0; h < d->nheads; h++) { + if (h != d->title_head && d->heads[h].line >= from) { + to = d->heads[h].line; + break; + } + } + if (to == d->L.count && to - from > ADR_HEAD_SCAN) { + to = from + ADR_HEAD_SCAN; + } + for (int k = from; k < to; k++) { + if (!d->L.v[k].text) { + continue; + } + char *v = adr_field(a, &d->L.v[k], key); + if (v) { + return v; + } + } + return NULL; +} + +/* ── Relations ───────────────────────────────────────────────────── */ + +/* `ADR-12`, `ADR 012`, `adr_7`: the number, and the reference's length. */ +static bool adr_id_ref(const char *s, size_t n, unsigned *number, size_t *len) { + if (n < PAIR_LEN * PAIR_LEN || !adr_starts_ci(s, n, "adr") || + (s[3] != '-' && s[3] != '_' && s[3] != ' ')) { + return false; + } + size_t used = 0; + unsigned v = 0; + size_t i = PAIR_LEN * PAIR_LEN; + size_t d = 0; + while (i + d < n && adr_digit(s[i + d])) { + d++; + } + if (d == 0 || (i + d < n && (adr_alpha(s[i + d]) || s[i + d] == '_'))) { + return false; + } + if (!adr_number_at(s + i, d, &used, &v)) { + return false; + } + *number = v; + *len = i + d; + return true; +} + +static void adr_push_relation(adr_doc_t *d, const char *adr_qn, const char *raw, size_t n, + uint32_t line, uint16_t syntax) { + CBMArena *a = d->ctx->arena; + char *text = cbm_arena_strndup(a, raw, n); + if (!text) { + d->ctx->result->doc_links.failed = true; + return; + } + CBMDocLink link = {.source_qn = adr_qn, + .raw = text, + .line = line, + .def_line = SKIP_ONE, + .syntax = syntax, + .flags = 0}; + cbm_doclinks_push(&d->ctx->result->doc_links, a, link); +} + +/* The targets after a relation phrase, up to the end of its sentence: link + * destinations and ADR ids. Forward relations become tokens of family + * `syntax`; for a backward one ("superseded by") the targets are joined into + * `*joined`. */ +static void adr_targets(adr_doc_t *d, const char *adr_qn, const char *s, size_t n, uint32_t line, + bool forward, uint16_t syntax, char *joined, size_t jcap) { + size_t end = n < ADR_RELATION_SPAN ? n : ADR_RELATION_SPAN; + for (size_t i = 0; i + SKIP_ONE < end; i++) { + /* a link is one reference whatever its text holds (adr-tools writes + * `Supersedes [3. Show links](0003-show-links.md)`) */ + const char *rb = s[i] == '[' ? memchr(s + i, ']', end - i) : NULL; + const char *close = rb && rb + SKIP_ONE < s + end && rb[SKIP_ONE] == '(' + ? memchr(rb, ')', (size_t)(s + end - rb)) + : NULL; + if (close) { + i = (size_t)(close - s); + continue; + } + /* a full stop after a number ends the sentence too (`supersedes 0092 + * and 0094. ADR 0202 is ...`: in a held-out audit the next sentence's + * record was taken for a target) */ + if ((s[i] == '.' && (i + SKIP_ONE == end || adr_blank(s[i + SKIP_ONE]))) || s[i] == ';') { + end = i; + break; + } + } + for (size_t i = 0; i < end; i++) { + const char *target = NULL; + size_t tn = 0; + const char *rb = s[i] == '[' ? memchr(s + i, ']', end - i) : NULL; + if (rb && rb + SKIP_ONE < s + end && rb[SKIP_ONE] == '(') { + /* [text](destination): one reference, the destination; ids in + * the link text name the same record */ + const char *dest = rb + PAIR_LEN; + const char *close = memchr(dest, ')', (size_t)(s + end - dest)); + if (close) { + target = dest; + tn = (size_t)(close - dest); + i = (size_t)(close - s); + } + } else { + unsigned num = 0; + size_t len = 0; + if ((i == 0 || !adr_alpha(s[i - SKIP_ONE])) && adr_id_ref(s + i, end - i, &num, &len)) { + target = s + i; + tn = len; + i += len - SKIP_ONE; + } + } + if (!target || tn == 0) { + continue; + } + if (forward) { + adr_push_relation(d, adr_qn, target, tn, line, syntax); + } else if (joined) { + size_t w = strlen(joined); + bool seen = false; + for (const char *p = joined; *p && !seen; p++) { + seen = strncmp(p, target, tn) == 0 && (p[tn] == '\0' || p[tn] == ',') && + (p == joined || p[-SKIP_ONE] == ' '); + } + if (!seen && w + tn + PAIR_LEN < jcap) { + if (w > 0) { + joined[w++] = ','; + joined[w++] = ' '; + } + memcpy(joined + w, target, tn); + joined[w + tn] = '\0'; + } + } + } +} + +/* Only list markers, quote marks, table bars and emphasis before position i + * of the line: the phrase opens the line's statement. */ +static bool adr_line_head(const adr_line_t *l, int i) { + for (int k = 0; k < i; k++) { + char c = l->s[k]; + if (!adr_blank(c) && c != '-' && c != '*' && c != '+' && c != '>' && c != '_' && c != '|') { + return false; + } + } + return true; +} + +/* A relation word joined to another by a slash, `Supersedes / Depends on:`: + * a template's combined field label, which states neither relation (in a + * held-out audit the record depended on the one it named). */ +static bool adr_label_choice(const adr_line_t *l, int a, int b) { + while (b < l->n && (l->s[b] == ' ' || l->s[b] == '\t')) { + b++; + } + while (a > 0 && (l->s[a - SKIP_ONE] == ' ' || l->s[a - SKIP_ONE] == '\t')) { + a--; + } + return (b < l->n && l->s[b] == '/') || (a > 0 && l->s[a - SKIP_ONE] == '/'); +} + +/* The field's value is "none" or "n/a" (`Supersedes: none (it extends ADR + * 0031)`): the record states no relation, and the records it goes on to name + * are extended or refined (a held-out audit's most frequent error). */ +static bool adr_none_value(const char *s, size_t n) { + size_t i = 0; + while (i < n && (adr_blank(s[i]) || s[i] == ':' || s[i] == '*' || s[i] == '_')) { + i++; + } + static const char *const none[] = {"none", "n/a", NULL}; + for (size_t k = 0; none[k]; k++) { + size_t w = strlen(none[k]); + if (adr_starts_ci(s + i, n - i, none[k]) && (i + w == n || !adr_alpha(s[i + w]))) { + return true; + } + } + return false; +} + +/* Another record is the phrase's subject (`ADR 0239 supersedes ADR 0203`, a + * table cell `0259 (supersedes ADR 0114's ...)`, a link to a record before + * it): this record reports that record's relation and states none of its + * own. Its own number as the subject is its own statement. */ +static bool adr_reported(const adr_doc_t *d, const adr_line_t *l, int i) { + int k = i; + while (k > 0 && + (adr_blank(l->s[k - SKIP_ONE]) || l->s[k - SKIP_ONE] == '(' || + l->s[k - SKIP_ONE] == '*' || l->s[k - SKIP_ONE] == '_' || l->s[k - SKIP_ONE] == '`')) { + k--; + } + if (k >= PAIR_LEN && l->s[k - SKIP_ONE] == ')') { + for (int o = k - PAIR_LEN; o > 0; o--) { /* `[text](destination)` */ + if (l->s[o] == '(') { + return l->s[o - SKIP_ONE] == ']'; + } + if (l->s[o] == ')' || adr_blank(l->s[o])) { + break; + } + } + return false; + } + int e = k; + while (k > 0 && adr_digit(l->s[k - SKIP_ONE])) { + k--; + } + if (e == k || (k > 0 && adr_alpha(l->s[k - SKIP_ONE]))) { + return false; + } + size_t used = 0; + unsigned v = 0; + if (!adr_number_at(l->s + k, (size_t)(e - k), &used, &v)) { + return false; + } + unsigned ref = 0; + size_t len = 0; + bool id = k >= PAIR_LEN * PAIR_LEN && + adr_id_ref(l->s + k - PAIR_LEN * PAIR_LEN, (size_t)(e - k) + PAIR_LEN * PAIR_LEN, + &ref, &len) && + len == (size_t)(e - k) + PAIR_LEN * PAIR_LEN; + /* a bare number is a record's only when padded like one (`0259`) */ + if (!id && e - k < ADR_PADDED_DIGITS) { + return false; + } + return !(d->numbered && v == d->number); +} + +/* A record reference right after the phrase (past a colon, a table bar and + * emphasis): a link or an ADR id, `Supersedes: [ADR-3](0003-x.md)`, + * `| Supersedes | ADR-0003 |`. */ +/* A colon right after the phrase (past emphasis): `Supersedes: ADR-3`, a + * field, whatever surrounds it. */ +static bool adr_field_value(const char *s, size_t n) { + size_t i = 0; + while (i < n && (adr_blank(s[i]) || s[i] == '*' || s[i] == '_')) { + i++; + } + return i < n && s[i] == ':'; +} + +/* `Status: Accepted`, `- **Date**: 2024-01-02`: a field line. */ +static bool adr_field_line(const adr_line_t *l) { + int i = 0; + while (i < l->n && (adr_blank(l->s[i]) || l->s[i] == '-' || l->s[i] == '*' || l->s[i] == '+' || + l->s[i] == '_' || l->s[i] == '|')) { + i++; + } + int w = i; + while (i < l->n && (adr_alpha(l->s[i]) || l->s[i] == ' ' || l->s[i] == '-')) { + i++; + } + if (i == w || i - w > ADR_FIELD_KEY_MAX) { + return false; + } + while (i < l->n && (l->s[i] == '*' || l->s[i] == '_' || l->s[i] == ' ')) { + i++; + } + return i < l->n && (l->s[i] == ':' || l->s[i] == '|'); +} + +/* Line k continues a paragraph: no list, quote or table mark opens it and the + * line before is running text, not blank, a heading or a field + * (`[ADR-0240](0240-x.md)` / `supersedes ADR-0134 and ...`: in a held-out + * census the subject was the record on the line before). */ +static bool adr_continuation(const adr_doc_t *d, int k) { + const adr_line_t *l = &d->L.v[k]; + int j = 0; + while (j < l->n && adr_blank(l->s[j])) { + j++; + } + if (j < l->n && (l->s[j] == '-' || l->s[j] == '*' || l->s[j] == '+' || l->s[j] == '>' || + l->s[j] == '|' || adr_digit(l->s[j]))) { + return false; /* its own list item, quote or table row */ + } + if (k <= d->L.fm_end || k == 0) { + return false; + } + const adr_line_t *p = &d->L.v[k - SKIP_ONE]; + int a = 0; + while (a < p->n && adr_blank(p->s[a])) { + a++; + } + return p->text && a < p->n && p->s[a] != '#' && !adr_field_line(p); +} + +static bool adr_direct_ref(const char *s, size_t n) { + size_t i = 0; + while (i < n && (adr_blank(s[i]) || s[i] == ':' || s[i] == '|' || s[i] == '*' || s[i] == '_')) { + i++; + } + unsigned number = 0; + size_t len = 0; + return i < n && (s[i] == '[' || adr_id_ref(s + i, n - i, &number, &len)); +} + +/* "supersedes" and "superseded by" are ADR words wherever they stand; + * "replaces" and "replaced by" are ordinary verbs of technical prose ("this + * middleware replaces the recovery described in ADR-22") and state a relation + * only where they open a line or list item ("Replaces ADR-3"). */ +static void adr_relations(adr_doc_t *d, const char *adr_qn, char *superseded_by, size_t cap) { + static const struct { + const char *phrase; + bool forward; + bool line_head; + } phrases[] = {{"superseded by", false, false}, + {"replaced by", false, true}, + {"supersedes", true, false}, + {"replaces", true, true}}; + for (int k = d->L.fm_end; k < d->L.count; k++) { + const adr_line_t *l = &d->L.v[k]; + if (!l->text) { + continue; + } + for (int i = 0; i < l->n; i++) { + if (i > 0 && adr_alpha(l->s[i - SKIP_ONE])) { + continue; + } + for (size_t p = 0; p < sizeof(phrases) / sizeof(phrases[0]); p++) { + size_t pl = strlen(phrases[p].phrase); + if (adr_starts_ci(l->s + i, (size_t)(l->n - i), phrases[p].phrase) && + (i + (int)pl == l->n || !adr_alpha(l->s[i + (int)pl])) && + (!phrases[p].line_head || adr_line_head(l, i)) && + !adr_label_choice(l, i, i + (int)pl)) { + if (!adr_none_value(l->s + i + pl, (size_t)(l->n - i - (int)pl)) && + !adr_reported(d, l, i)) { + /* a statement opens its line or field and names a + * record directly; the same words in running prose + * are their own family (held-out audits: prose 38 of + * 42 correct, statements 44 of 45) */ + const char *v = l->s + i + pl; + size_t vn = (size_t)(l->n - i - (int)pl); + bool statement = adr_line_head(l, i) && adr_direct_ref(v, vn) && + (adr_field_value(v, vn) || !adr_continuation(d, k)); + uint16_t syntax = statement ? (uint16_t)CBM_DOCLINK_MD_SUPERSEDES + : (uint16_t)CBM_DOCLINK_MD_SUPERSEDES_PROSE; + adr_targets(d, adr_qn, v, vn, (uint32_t)(k + SKIP_ONE), phrases[p].forward, + syntax, superseded_by, cap); + } + i += (int)pl - SKIP_ONE; + break; + } + } + } + } +} + +/* A template's choice list, `{proposed | accepted}` or `[a | b]`: the field + * was never filled in. A filled-in value that kept its braces (`{proposed}`) + * is a value. */ +static bool adr_placeholder(const char *s) { + for (const char *p = s; *p; p++) { + if (*p != '{' && *p != '[') { + continue; + } + char close = *p == '{' ? '}' : ']'; + for (const char *q = p + SKIP_ONE; *q && *q != close; q++) { + if (*q == '|') { + return true; + } + } + } + return false; +} + +/* The first date in the record's change-log section ("Changelog", "Change + * log", "History", "Revision history"): `* 2019-12-12: Initial version`. */ +static bool adr_changelog_date(const adr_doc_t *d, char *out, size_t cap) { + for (int h = 0; h < d->nheads; h++) { + const char *t = d->heads[h].title; + size_t n = d->heads[h].title_n; + while (n > 0 && (t[n - SKIP_ONE] == ':' || adr_blank(t[n - SKIP_ONE]))) { + n--; + } + if (!adr_eq_ci(t, n, "changelog") && !adr_eq_ci(t, n, "change log") && + !adr_eq_ci(t, n, "history") && !adr_eq_ci(t, n, "revision history")) { + continue; + } + int end = h + SKIP_ONE < d->nheads ? d->heads[h + SKIP_ONE].line : d->L.count; + for (int k = d->heads[h].line + SKIP_ONE; k < end; k++) { + char line[ADR_FIELD_MAX]; + const adr_line_t *l = &d->L.v[k]; + if (!l->text || l->n == 0) { + continue; + } + snprintf(line, sizeof(line), "%.*s", l->n, l->s); + if (adr_parse_date(line, out, cap)) { + return true; + } + } + return false; + } + return false; +} + +/* ── The driver ──────────────────────────────────────────────────── */ + +static bool adr_ends_ci(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n > k && adr_eq_ci(s + n - k, k, suffix); +} + +/* A Markdown or reStructuredText record (reST titles underlined with `=` or + * `-` read like setext headings). */ +static bool adr_document_path(const char *rel) { + return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown") || + adr_ends_ci(rel, ".rst"); +} + +void cbm_adr_extract(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0 || !ctx->rel_path || + !adr_document_path(ctx->rel_path)) { + return; + } + const char *rel = ctx->rel_path; + const char *base = strrchr(rel, '/'); + base = base ? base + SKIP_ONE : rel; + adr_id_t id; + adr_filename_id(base, &id); + bool conv = adr_dir_in(rel, ADR_CONV_DIRS); + if ((id.kind == ADR_ID_NONE && adr_non_adr_name(base)) || adr_dir_in(rel, ADR_FIXTURE_DIRS)) { + return; + } + if (!conv && id.kind == ADR_ID_NONE) { + return; /* structure alone never makes a record */ + } + CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena; + CBMArena *a = ctx->arena; + adr_doc_t d = {.ctx = ctx, + .title_head = CBM_NOT_FOUND, + .numbered = id.kind == ADR_ID_ADR || id.kind == ADR_ID_NUMBER || + id.kind == ADR_ID_PREFIX, + .number = id.number}; + if (!adr_split_lines(scratch, ctx->source, ctx->source_len, &d.L)) { + ctx->result->doc_links.failed = true; + return; + } + /* headings: the Section definitions, read back at their lines */ + int nsec = 0; + for (int i = 0; i < ctx->result->defs.count; i++) { + const CBMDefinition *def = &ctx->result->defs.items[i]; + nsec += def->label && strcmp(def->label, "Section") == 0; + } + d.heads = + nsec > 0 ? (adr_head_t *)cbm_arena_alloc(scratch, (size_t)nsec * sizeof(*d.heads)) : NULL; + if (nsec > 0 && !d.heads) { + ctx->result->doc_links.failed = true; + return; + } + for (int i = 0; i < ctx->result->defs.count; i++) { + const CBMDefinition *def = &ctx->result->defs.items[i]; + if (!def->label || strcmp(def->label, "Section") != 0 || def->start_line < SKIP_ONE || + (int)def->start_line > d.L.count) { + continue; + } + int k = (int)def->start_line - SKIP_ONE; + const char *t = NULL; + size_t tn = 0; + int level = adr_heading_level(&d.L, k, &t, &tn); + if (level > 0) { + d.heads[d.nheads++] = + (adr_head_t){.line = k, .level = level, .title = t, .title_n = tn}; + } + } + qsort(d.heads, (size_t)d.nheads, sizeof(*d.heads), adr_head_cmp); + for (int h = 0; h < d.nheads; h++) { + if (d.heads[h].level == SKIP_ONE) { + d.title_head = h; + break; + } + } + /* status: front matter, header field, the Status section */ + char *status_raw = adr_fm_value(a, &d.L, "status"); + if (!status_raw) { + status_raw = adr_header_field(a, &d, "status"); + } + int status_sec = adr_find_section(&d, ADR_SEC_STATUS); + if (!status_raw && status_sec >= 0) { + const adr_head_t *sh = &d.heads[status_sec]; + const char *colon = memchr(sh->title, ':', sh->title_n); + if (colon && colon + SKIP_ONE < sh->title + sh->title_n) { + status_raw = + adr_plain(a, colon + SKIP_ONE, (size_t)(sh->title + sh->title_n - colon - SKIP_ONE), + ADR_FIELD_MAX); + } else { + status_raw = adr_first_paragraph(a, &d, status_sec, ADR_FIELD_MAX); + } + } + bool placeholder = status_raw && adr_placeholder(status_raw); + bool structure = status_raw && (adr_find_section(&d, ADR_SEC_CONTEXT) >= 0 || + adr_find_section(&d, ADR_SEC_DECISION) >= 0 || + adr_find_section(&d, ADR_SEC_CONSEQUENCES) >= 0 || + adr_find_section(&d, ADR_SEC_OUTCOME) >= 0 || + adr_find_section(&d, ADR_SEC_OPTIONS) >= 0); + bool is_adr = (conv && (id.kind != ADR_ID_NONE || structure)) || id.kind == ADR_ID_ADR || + ((id.kind == ADR_ID_NUMBER || id.kind == ADR_ID_DATE) && structure); + if (!is_adr) { + return; + } + /* the record's facts */ + char name[ADR_NAME_MAX]; + if (id.kind == ADR_ID_ADR || id.kind == ADR_ID_NUMBER) { + snprintf(name, sizeof(name), "ADR-%u", id.number); + } else if (id.kind == ADR_ID_PREFIX) { + snprintf(name, sizeof(name), "%s-%u", id.prefix, id.number); + } else { + const char *dot = strrchr(base, '.'); + size_t stem = dot ? (size_t)(dot - base) : strlen(base); + snprintf(name, sizeof(name), "%.*s", + (int)(stem < sizeof(name) - SKIP_ONE ? stem : sizeof(name) - SKIP_ONE), base); + } + char *title = NULL; + if (d.title_head >= 0) { + title = + adr_plain(a, d.heads[d.title_head].title, d.heads[d.title_head].title_n, ADR_FIELD_MAX); + } + if (!title) { + title = adr_fm_value(a, &d.L, "title"); + } + char date[ADR_ISO_LEN + SKIP_ONE] = ""; + /* date: front matter, header field, "last updated", the change log's + * first date, the file name */ + char *date_raw = adr_fm_value(a, &d.L, "date"); + if (!date_raw) { + date_raw = adr_header_field(a, &d, "date"); + } + if (!date_raw) { + date_raw = adr_header_field(a, &d, "last updated"); + } + if (!(date_raw && adr_parse_date(date_raw, date, sizeof(date))) && + !(!date_raw && adr_changelog_date(&d, date, sizeof(date))) && id.date[0]) { + memcpy(date, id.date, sizeof(id.date)); + } + static const char *const people_keys[] = {"deciders", "decision-makers", "decision_makers", + "authors", "author", NULL}; + char *deciders = NULL; + for (size_t k = 0; !deciders && people_keys[k]; k++) { + deciders = adr_fm_value(a, &d.L, people_keys[k]); + } + for (size_t k = 0; !deciders && people_keys[k]; k++) { + deciders = adr_header_field(a, &d, people_keys[k]); + } + if (deciders && adr_placeholder(deciders)) { + deciders = NULL; /* a template placeholder */ + } + int decision = adr_find_section(&d, ADR_SEC_DECISION); + if (decision < 0) { + decision = adr_find_section(&d, ADR_SEC_OUTCOME); + } + if (decision < 0) { + decision = adr_find_section(&d, ADR_SEC_CONTEXT); + } + char *doc = decision >= 0 ? adr_section_text(a, &d, decision, ADR_DOC_MAX) : NULL; + char *qn = cbm_fqn_compute(a, ctx->project, rel, CBM_ADR_QN_NAME); + char *adr_name = cbm_arena_strdup(a, name); + char *adr_date = date[0] ? cbm_arena_strdup(a, date) : NULL; + char superseded_by[ADR_FIELD_MAX] = ""; + char *fm_by = adr_fm_value(a, &d.L, "superseded-by"); + if (!fm_by) { + fm_by = adr_fm_value(a, &d.L, "superseded_by"); + } + if (fm_by) { + snprintf(superseded_by, sizeof(superseded_by), "%s", fm_by); + } + if (!qn || !adr_name) { + ctx->result->doc_links.failed = true; + return; + } + adr_relations(&d, qn, superseded_by, sizeof(superseded_by)); + const char *status = placeholder || !status_raw ? NULL : adr_norm_status(status_raw); + const char *facts[][PAIR_LEN] = { + {"adr_id", adr_name}, + {"title", title}, + {"status", status}, + {"status_text", placeholder ? NULL : status_raw}, + {"date", adr_date}, + {"deciders", deciders}, + {"superseded_by", superseded_by[0] ? cbm_arena_strdup(a, superseded_by) : NULL}, + }; + /* sized by the facts themselves: room for every pair and the terminator */ + size_t nfacts = sizeof(facts) / sizeof(facts[0]); + const char **kv = + (const char **)cbm_arena_alloc(a, ((nfacts * PAIR_LEN) + SKIP_ONE) * sizeof(char *)); + if (!kv) { + ctx->result->doc_links.failed = true; + return; + } + size_t w = 0; + for (size_t f = 0; f < nfacts; f++) { + if (facts[f][SKIP_ONE] && facts[f][SKIP_ONE][0]) { + kv[w++] = facts[f][0]; + kv[w++] = facts[f][SKIP_ONE]; + } + } + kv[w] = NULL; + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = adr_name; + def.qualified_name = qn; + def.label = "ADR"; + def.file_path = rel; + def.start_line = SKIP_ONE; + def.end_line = (uint32_t)d.L.count; + def.is_exported = true; + def.docstring = doc; + def.extra_props = kv; + cbm_defs_push(&ctx->result->defs, a, def); +} diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.c new file mode 100644 index 0000000000..b3ca4a4f34 --- /dev/null +++ b/internal/cbm/doc_pdf.c @@ -0,0 +1,1392 @@ +/* + * doc_pdf.c — PDF documents in the graph. + * + * A PDF's text layer (pdf/) becomes one Section per page: name "page N", the + * page's text as its docstring, so search finds what a page says. The code a + * page names becomes doc-link tokens for the resolver (doc_links_pdf.c): + * STRUCTURAL mentions only -- paths ("src/auth/handler.go", "./x/y", + * "Pool.sol") and qualified names ("pkg.Type.method", "a::b", "x->y"). Bare + * identifiers never become tokens (field test E6c: bare names stay + * suggestions). + * + * The scanner is the field-tested one (E6 idents.py), ported with CPython's + * semantics: NFKC per code point plus canonical composition, the re module's + * leftmost-first matching, str predicates from the same Unicode tables + * (pdf/pdf_unicode.c). A mention split across two lines ("src/auth/" / + * "handler.go", "Handler::" / "run") is proposed as a JOIN token that lists + * the line-local tokens it would replace; the resolver keeps the join only + * when it resolves EXACT or UNIQUE, and the fragments stand otherwise. + * + * Hygiene from the field test, applied here where the text decides it: R1 a + * PDF under a test or fixture directory gives no tokens; R3 a mention written + * as a keyword argument ("(x, foo.bar=1)") is no reference. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "foundation/mem_core.h" +#include "helpers.h" +#include "pdf/pdf.h" +#include "pdf/pdf_unicode.h" + +#include +#include +#include +#include + +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +/* candidates a join compared, for the cost test */ +static atomic_size_t g_join_steps; +#endif + +enum { + PDF_R3_WINDOW = 400, /* the field test looked this far back for "(" or "," */ + PDF_PAGE_NAME = 32, + PDF_LIST_MIN = 64, + PDF_U8_MAX = 4, + PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */ +}; + +#define PDF_JOIN_SEP '\x1f' /* between a join's forms */ +#define PDF_JOIN_FRAG '\x1e' /* before a join's fragment token indices */ + +/* ── Word lists (the field test's, verbatim) ─────────────────────── */ + +static const char *const CODE_EXT[] = { + "py", "pyi", "pyx", "pxd", "c", "h", "cc", "cpp", "cxx", "hpp", + "hh", "hxx", "cs", "csx", "java", "kt", "kts", "scala", "go", "rs", + "rb", "php", "js", "jsx", "mjs", "cjs", "ts", "tsx", "swift", "m", + "mm", "sol", "sh", "bash", "zsh", "ps1", "psm1", "lua", "r", "jl", + "dart", "ex", "exs", "erl", "hs", "ml", "fs", "fsx", "vb", "sql", + "proto", "graphql", "gql", "thrift", "yaml", "yml", "json", "toml", "ini", "cfg", + "conf", "xml", "html", "htm", "css", "scss", "less", "md", "rst", "txt", + "adoc", "ipynb", "gradle", "csproj", "sln", "props", "targets", "cmake", "mk", "tf", + "hcl", "bzl", "lock", "env", "pdf", "vue", "svelte", "tex", "bib", "jsonl", + "csv", "tsv", "pom", "jar", "war", "dll", "so", "wasm", "zig", "nim", + "v", NULL}; + +static const char *const TLDS[] = {"com", "org", "net", "io", "dev", "edu", "gov", "de", "uk", + "ai", "co", "info", "app", "eu", "fr", "ch", "us", "ca", + "au", "jp", "cn", "ru", "nl", "se", "no", "fi", "it", + "es", "at", "be", "biz", "me", "tv", "ly", "gg", "sh", + "xyz", "cloud", "tech", "blog", NULL}; + +static const char *const SLASH_STOP[] = { + "and/or", "or/and", "input/output", "i/o", "read/write", "yes/no", "on/off", + "true/false", "he/she", "his/her", "s/he", "w/o", "n/a", "km/h", + "m/s", "tcp/ip", "client/server", "him/her", "either/or", NULL}; + +/* the field test's FIX: directories whose PDFs are fixtures (R1) */ +static const char *const FIXTURE_DIRS[] = {"test", "tests", "cypress", "baseline_images", + "resources", "samples", "e2e", "fixtures", + "testdata", "test_data", "data", "examples", + NULL}; + +/* ── Code-point buffers ──────────────────────────────────────────── */ + +typedef struct { + uint32_t *v; + int n; + int cap; + bool fail; +} u32v_t; + +static bool u32_push(u32v_t *b, uint32_t c) { + if (b->fail) { + return false; + } + if (b->n == b->cap) { + int ncap = b->cap ? b->cap * 2 : PDF_LIST_MIN; + uint32_t *g = + (uint32_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->v, (size_t)ncap * sizeof(*g)); + if (!g) { + b->fail = true; + return false; + } + b->v = g; + b->cap = ncap; + } + b->v[b->n++] = c; + return true; +} + +static void u32_free(u32v_t *b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b->v); + memset(b, 0, sizeof(*b)); +} + +/* ── normalise(): NFKC, zero-width characters dropped, hyphens unified ── */ + +static const pdf_nfkc_t *nfkc_find(uint32_t cp) { + int lo = 0; + int hi = PDF_NFKC_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (PDF_NFKC[mid].cp < cp) { + lo = mid + 1; + } else if (PDF_NFKC[mid].cp > cp) { + hi = mid - 1; + } else { + return &PDF_NFKC[mid]; + } + } + return NULL; +} + +static bool ucomp(uint32_t a, uint32_t b, uint32_t *out) { + /* Hangul, algorithmically */ + enum { + SBASE = 0xAC00, + LBASE = 0x1100, + VBASE = 0x1161, + TBASE = 0x11A7, + LCOUNT = 19, + VCOUNT = 21, + TCOUNT = 28, + NCOUNT = VCOUNT * TCOUNT, + SCOUNT = LCOUNT * NCOUNT + }; + if (a >= LBASE && a < LBASE + LCOUNT && b >= VBASE && b < VBASE + VCOUNT) { + *out = SBASE + ((a - LBASE) * VCOUNT + (b - VBASE)) * TCOUNT; + return true; + } + if (a >= SBASE && a < SBASE + SCOUNT && (a - SBASE) % TCOUNT == 0 && b > TBASE && + b < TBASE + TCOUNT) { + *out = a + (b - TBASE); + return true; + } + int lo = 0; + int hi = PDF_UCOMP_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + const pdf_ucomp_t *e = &PDF_UCOMP[mid]; + if (e->first < a || (e->first == a && e->second < b)) { + lo = mid + 1; + } else if (e->first > a || e->second > b) { + hi = mid - 1; + } else { + *out = e->composite; + return true; + } + } + return false; +} + +static bool normalise(const char *s, size_t n, u32v_t *out) { + size_t i = 0; + while (i < n) { + unsigned char c = (unsigned char)s[i]; + uint32_t cp = c; + size_t k = 1; + if (c >= 0xF0 && i + 3 < n) { + cp = ((uint32_t)(c & 0x07) << 18) | ((uint32_t)(s[i + 1] & 0x3F) << 12) | + ((uint32_t)(s[i + 2] & 0x3F) << 6) | (uint32_t)(s[i + 3] & 0x3F); + k = 4; + } else if (c >= 0xE0 && i + 2 < n) { + cp = ((uint32_t)(c & 0x0F) << 12) | ((uint32_t)(s[i + 1] & 0x3F) << 6) | + (uint32_t)(s[i + 2] & 0x3F); + k = 3; + } else if (c >= 0xC0 && i + 1 < n) { + cp = ((uint32_t)(c & 0x1F) << 6) | (uint32_t)(s[i + 1] & 0x3F); + k = 2; + } + i += k; + const pdf_nfkc_t *m = nfkc_find(cp); + int cnt = m ? m->len : 1; + for (int j = 0; j < cnt; j++) { + uint32_t x = m ? PDF_NFKC_SEQ[m->off + j] : cp; + uint32_t comp; + if (out->n > 0 && ucomp(out->v[out->n - 1], x, &comp)) { + out->v[out->n - 1] = comp; + } else if (!u32_push(out, x)) { + return false; + } + } + } + /* zero-width characters out; U+2010/U+2011 -> '-' */ + int k = 0; + for (int j = 0; j < out->n; j++) { + uint32_t x = out->v[j]; + if (x == 0x200B || x == 0x200C || x == 0x200D || x == 0x2060 || x == 0xFEFF) { + continue; + } + if (x == 0x2010 || x == 0x2011) { + x = '-'; + } + out->v[k++] = x; + } + out->n = k; + return true; +} + +/* ── Character classes of the field test's regexes ───────────────── */ + +static bool is_alnum_ascii(uint32_t c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +static bool is_alpha_ascii(uint32_t c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +/* [A-Za-z0-9_.$@\-] */ +static bool path_seg(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '.' || c == '$' || c == '@' || c == '-'; +} + +/* [A-Za-z0-9_.\-] */ +static bool path_tail(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '.' || c == '-'; +} + +/* [A-Za-z0-9_] */ +static bool path_end(uint32_t c) { + return is_alnum_ascii(c) || c == '_'; +} + +/* [A-Za-z_$] */ +static bool ident_start(uint32_t c) { + return is_alpha_ascii(c) || c == '_' || c == '$'; +} + +/* [\w$] */ +static bool ident_cont(uint32_t c) { + return pdf_u_word(c) || c == '$'; +} + +/* [A-Za-z0-9_$] (IDC) */ +static bool idc(uint32_t c) { + return is_alnum_ascii(c) || c == '_' || c == '$'; +} + +/* ── Candidates ──────────────────────────────────────────────────── */ + +typedef enum { + MK_PATH = 0, + MK_PATH1, + MK_FILE, + MK_QUAL, + MK_SNAKE, + MK_CAMEL, + MK_CALL, +} mention_kind_t; + +typedef struct { + int start; + int end; + mention_kind_t kind; + int norm_start; /* norm = a rewrite of the surface, kept in a pool */ + int norm_len; +} cand_t; + +typedef struct { + cand_t *v; + int n; + int cap; + bool fail; + u32v_t pool; /* norms */ +} candv_t; + +static bool cand_push(candv_t *cv, cand_t c) { + if (cv->fail) { + return false; + } + if (cv->n == cv->cap) { + int ncap = cv->cap ? cv->cap * 2 : PDF_LIST_MIN; + cand_t *g = (cand_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, cv->v, (size_t)ncap * sizeof(*g)); + if (!g) { + cv->fail = true; + return false; + } + cv->v = g; + cv->cap = ncap; + } + cv->v[cv->n++] = c; + return true; +} + +static void candv_free(candv_t *cv) { + cbm_free(CBM_MEM_CLASS_EXTRACT, cv->v); + u32_free(&cv->pool); + memset(cv, 0, sizeof(*cv)); +} + +static bool span_free(const uint8_t *taken, int a, int b) { + for (int i = a; i < b; i++) { + if (taken[i]) { + return false; + } + } + return true; +} + +static void span_take(uint8_t *taken, int a, int b) { + memset(taken + a, 1, (size_t)(b - a)); +} + +/* URL_RE: (?:https?|ftp|file)://\S+ | www\.\S+ | \S+@\S+\.[A-Za-z]{2,} */ +typedef struct { + int run_start; /* the non-space run the email alternative last looked at */ + int run_end; + int at; /* its last '@' with a domain after it, or -1 */ + int at_end; /* where that match ends */ +} url_cache_t; + +static bool lit_at(const uint32_t *t, int n, int p, const char *lit) { + for (int i = 0; lit[i]; i++) { + if (p + i >= n || t[p + i] != (uint32_t)(unsigned char)lit[i]) { + return false; + } + } + return true; +} + +static int run_end_from(const uint32_t *t, int n, int p) { + while (p < n && !pdf_u_space(t[p])) { + p++; + } + return p; +} + +static void url_cache_run(const uint32_t *t, int n, int p, url_cache_t *uc) { + if (p >= uc->run_start && p < uc->run_end) { + return; /* the run is known: validity of an '@' does not depend on p */ + } + /* the scan enters a run at its start (every earlier position of a run + * filled this cache); scanning back is only for a caller that did not */ + int rs = p; + while (rs > 0 && !pdf_u_space(t[rs - 1])) { + rs--; + } + uc->run_start = rs; + uc->run_end = run_end_from(t, n, p); + uc->at = -1; + uc->at_end = -1; + int re = uc->run_end; + /* the largest '.' followed by two ASCII letters, scanning from the right; + * for each '@' (right to left) the domain is the largest such '.' >= at+2 */ + int best_dot = -1; + for (int a = re - 1; a > rs; a--) { + if (a + 2 < re && t[a] == '.' && is_alpha_ascii(t[a + 1]) && is_alpha_ascii(t[a + 2])) { + if (best_dot < 0) { + best_dot = a; + } + } + if (t[a] == '@' && best_dot >= a + 2) { + int e = best_dot + 1; + while (e < re && is_alpha_ascii(t[e])) { + e++; + } + uc->at = a; + uc->at_end = e; + return; + } + } +} + +static int url_at(const uint32_t *t, int n, int p, url_cache_t *uc) { + if (lit_at(t, n, p, "https://") || lit_at(t, n, p, "http://") || lit_at(t, n, p, "ftp://") || + lit_at(t, n, p, "file://")) { + int q = p; + while (t[q] != '/') { + q++; + } + q += 2; + if (q < n && !pdf_u_space(t[q])) { + return run_end_from(t, n, q); + } + } + if (lit_at(t, n, p, "www.") && p + 4 < n && !pdf_u_space(t[p + 4])) { + return run_end_from(t, n, p + 4); + } + if (pdf_u_space(t[p])) { + return -1; + } + url_cache_run(t, n, p, uc); + if (uc->at > p) { + return uc->at_end; + } + return -1; +} + +/* PATH_RE's body after the optional prefix: (?:[seg]+/)+[tail]*[end] */ +static int path_body(const uint32_t *t, int n, int q) { + int p = q; + int last_ts = -1; + for (;;) { + int r = p; + while (r < n && path_seg(t[r])) { + r++; + } + if (r > p && r < n && t[r] == '/') { + last_ts = r + 1; + p = r + 1; + continue; + } + break; + } + if (last_ts < 0) { + return -1; + } + /* tail starts, from the last segment back to the first */ + int ts = last_ts; + for (;;) { + int r = ts; + int last_e = -1; + while (r < n && path_tail(t[r])) { + if (path_end(t[r])) { + last_e = r; + } + r++; + } + if (last_e >= 0) { + return last_e + 1; + } + /* give back one segment: the previous '/' before ts-1 */ + int s = ts - 2; + while (s >= q && t[s] != '/') { + s--; + } + if (s < q) { + return -1; + } + ts = s + 1; + } +} + +static int path_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '/' || b == '.' || b == '-' || b == '~' || b == '$' || b == '@') { + return -1; + } + } + /* (?:\.{1,2}/|~/|/)? tried in order, then without */ + int e; + if (p + 2 < n && t[p] == '.' && t[p + 1] == '.' && t[p + 2] == '/' && + (e = path_body(t, n, p + 3)) >= 0) { + return e; + } + if (p + 1 < n && t[p] == '.' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) { + return e; + } + if (p + 1 < n && t[p] == '~' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) { + return e; + } + if (t[p] == '/' && (e = path_body(t, n, p + 1)) >= 0) { + return e; + } + return path_body(t, n, p); +} + +/* QUAL_RE: [A-Za-z_$][\w$]*(?:(?:\.|::|->|#)[A-Za-z_$][\w$]*)+(?:\(\))? */ +static int qual_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') { + return -1; + } + } + if (!ident_start(t[p])) { + return -1; + } + int q = p + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + int groups = 0; + for (;;) { + int sep = 0; + if (q < n && t[q] == '.') { + sep = 1; + } else if (q + 1 < n && t[q] == ':' && t[q + 1] == ':') { + sep = 2; + } else if (q + 1 < n && t[q] == '-' && t[q + 1] == '>') { + sep = 2; + } else if (q < n && t[q] == '#') { + sep = 1; + } + if (!sep || q + sep >= n || !ident_start(t[q + sep])) { + break; + } + q += sep + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + groups++; + } + if (!groups) { + return -1; + } + if (q + 1 < n && t[q] == '(' && t[q + 1] == ')') { + q += 2; + } + return q; +} + +/* TOK_RE: [A-Za-z_$][\w$]* (whole identifier) */ +static int tok_at(const uint32_t *t, int n, int p) { + if (p > 0) { + uint32_t b = t[p - 1]; + if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') { + return -1; + } + } + if (!ident_start(t[p])) { + return -1; + } + int q = p + 1; + while (q < n && ident_cont(t[q])) { + q++; + } + return q; +} + +static uint32_t ascii_lower(uint32_t c) { + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static bool in_list_lower(const uint32_t *s, int n, const char *const *list) { + for (int i = 0; list[i]; i++) { + const char *w = list[i]; + int k = 0; + while (k < n && w[k] && ascii_lower(s[k]) == (uint32_t)(unsigned char)w[k]) { + k++; + } + if (k == n && !w[k]) { + return true; + } + } + return false; +} + +/* str.islower / str.isupper of a whole string */ +static bool str_islower(const uint32_t *s, int n) { + bool cased = false; + for (int i = 0; i < n; i++) { + if (pdf_u_upper(s[i]) || pdf_u_title(s[i])) { + return false; + } + cased = cased || pdf_u_lower(s[i]); + } + return cased; +} + +static bool str_isupper(const uint32_t *s, int n) { + bool cased = false; + for (int i = 0; i < n; i++) { + if (pdf_u_lower(s[i]) || pdf_u_title(s[i])) { + return false; + } + cased = cased || pdf_u_upper(s[i]); + } + return cased; +} + +static bool is_camel(const uint32_t *t, int n) { + if (n < 3 || str_isupper(t, n) || str_islower(t, n)) { + return false; + } + /* [A-Z]{2,}s: PDFs, APIs, IDs */ + if (t[n - 1] == 's') { + bool caps = true; + for (int i = 0; i < n - 1; i++) { + caps = caps && t[i] >= 'A' && t[i] <= 'Z'; + } + if (caps) { + return false; + } + } + for (int i = 0; i + 1 < n; i++) { + uint32_t a = t[i]; + uint32_t b = t[i + 1]; + if (pdf_u_lower(a) && pdf_u_upper(b)) { + return true; + } + if (i >= 1 && pdf_u_upper(t[i - 1]) && pdf_u_upper(a) && pdf_u_lower(b)) { + return true; + } + } + return false; +} + +/* The extension after the last '.' of a segment, when it is a code extension. */ +static bool has_code_ext(const uint32_t *s, int n) { + int dot = -1; + for (int i = 0; i < n; i++) { + if (s[i] == '.') { + dot = i; + } + } + return dot >= 0 && in_list_lower(s + dot + 1, n - dot - 1, CODE_EXT); +} + +static void path_candidate(const uint32_t *t, int a, int b, candv_t *cv) { + /* s.strip("./~"), split on '/', drop empties */ + int sa = a; + int sb = b; + while (sa < sb && (t[sa] == '.' || t[sa] == '/' || t[sa] == '~')) { + sa++; + } + while (sb > sa && (t[sb - 1] == '.' || t[sb - 1] == '/' || t[sb - 1] == '~')) { + sb--; + } + bool any_seg = false; + bool all_numeric = true; + int last_s = -1; + int last_e = -1; + for (int i = sa; i <= sb;) { + int j = i; + while (j < sb && t[j] != '/') { + j++; + } + if (j > i) { + any_seg = true; + for (int k = i; k < j; k++) { + if (!((t[k] >= '0' && t[k] <= '9') || t[k] == '.' || t[k] == '-')) { + all_numeric = false; + } + } + last_s = i; + last_e = j; + } + i = j + 1; + } + if (!any_seg || all_numeric) { + return; /* numeric-path */ + } + if (in_list_lower(t + a, b - a, SLASH_STOP)) { + return; + } + int slashes = 0; + for (int i = a; i < b; i++) { + slashes += t[i] == '/'; + } + bool has_ext = has_code_ext(t + last_s, last_e - last_s); + bool prefixed = + lit_at(t, b, a, "./") || lit_at(t, b, a, "../") || lit_at(t, b, a, "~/") || t[a] == '/'; + cand_t c = {a, b, (slashes >= 2 || has_ext || prefixed) ? MK_PATH : MK_PATH1, 0, 0}; + /* norm: leading ../ ./ ~/ / removed (repeatedly), trailing '/' removed */ + int na = a; + for (;;) { + if (lit_at(t, b, na, "../")) { + na += 3; + } else if (lit_at(t, b, na, "./") || lit_at(t, b, na, "~/")) { + na += 2; + } else if (na < b && t[na] == '/') { + na += 1; + } else { + break; + } + } + int nb = b; + while (nb > na && t[nb - 1] == '/') { + nb--; + } + c.norm_start = cv->pool.n; + for (int i = na; i < nb; i++) { + u32_push(&cv->pool, t[i]); + } + c.norm_len = nb - na; + cand_push(cv, c); +} + +static void qual_candidate(const uint32_t *t, int a, int b, candv_t *cv) { + int ce = b; + if (b - a >= 2 && t[b - 2] == '(' && t[b - 1] == ')') { + ce = b - 2; + } + bool colons = false; + bool arrow = false; + int ns = cv->pool.n; + for (int i = a; i < ce; i++) { + if (i + 1 < ce && t[i] == ':' && t[i + 1] == ':') { + colons = true; + u32_push(&cv->pool, '.'); + i++; + } else if (i + 1 < ce && t[i] == '-' && t[i + 1] == '>') { + arrow = true; + u32_push(&cv->pool, '.'); + i++; + } else if (t[i] == '#') { + u32_push(&cv->pool, '.'); + } else { + u32_push(&cv->pool, t[i]); + } + } + if (cv->pool.fail) { + cv->fail = true; + return; + } + const uint32_t *norm = cv->pool.v + ns; + int nl = cv->pool.n - ns; + bool short_segs = true; + bool has_us = false; + int seg_s = 0; + int last_s = 0; + for (int i = 0; i <= nl; i++) { + if (i == nl || norm[i] == '.') { + if (i - seg_s > 2) { + short_segs = false; + } + last_s = seg_s; + seg_s = i + 1; + } else if (norm[i] == '_') { + has_us = true; + } + } + bool lower = str_islower(norm, nl); + if ((short_segs && lower) || + (in_list_lower(norm + last_s, nl - last_s, TLDS) && lower && !has_us)) { + cv->pool.n = ns; /* abbreviation or domain: taken, no candidate */ + return; + } + bool file = in_list_lower(norm + last_s, nl - last_s, CODE_EXT) && !colons && !arrow; + cand_t c = {a, b, file ? MK_FILE : MK_QUAL, ns, nl}; + cand_push(cv, c); +} + +static void tok_candidate(const uint32_t *t, int n, int a, int b, uint8_t *taken, candv_t *cv) { + int len = b - a; + /* "_" in t.strip("_") */ + int sa = a; + int sb = b; + while (sa < sb && t[sa] == '_') { + sa++; + } + while (sb > sa && t[sb - 1] == '_') { + sb--; + } + bool inner_us = false; + for (int i = sa; i < sb; i++) { + inner_us = inner_us || t[i] == '_'; + } + bool dunder = len > 4 && t[a] == '_' && t[a + 1] == '_' && t[b - 1] == '_' && t[b - 2] == '_'; + mention_kind_t kind; + if (inner_us || dunder) { + kind = MK_SNAKE; + } else if (is_camel(t + a, len)) { + kind = MK_CAMEL; + } else if (b < n && t[b] == '(' && len >= 2 && !str_isupper(t + a, len)) { + kind = MK_CALL; + } else { + return; + } + span_take(taken, a, b); + cand_t c = {a, b, kind, cv->pool.n, len}; + for (int i = a; i < b; i++) { + u32_push(&cv->pool, t[i]); + } + cand_push(cv, c); +} + +static int cand_cmp(const void *x, const void *y) { + const cand_t *a = (const cand_t *)x; + const cand_t *b = (const cand_t *)y; + return a->start < b->start ? -1 : a->start > b->start ? 1 : 0; +} + +/* candidates(): line-local candidates of text, sorted by start. */ +static bool candidates(const uint32_t *t, int n, bool with_tok, candv_t *cv) { + uint8_t *taken = (uint8_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1)); + if (!taken) { + return false; + } + url_cache_t uc = {-1, -1, -1, -1}; + for (int p = 0; p < n;) { + int e = url_at(t, n, p, &uc); + if (e > p) { + span_take(taken, p, e); + p = e; + } else { + p++; + } + } + for (int p = 0; p < n;) { + int e = path_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + span_take(taken, p, e); + path_candidate(t, p, e, cv); + } + p = e; + } + for (int p = 0; p < n;) { + int e = qual_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + span_take(taken, p, e); + qual_candidate(t, p, e, cv); + } + p = e; + } + if (with_tok) { + for (int p = 0; p < n;) { + int e = tok_at(t, n, p); + if (e <= p) { + p++; + continue; + } + if (span_free(taken, p, e)) { + tok_candidate(t, n, p, e, taken, cv); + } + p = e; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, taken); + if (cv->n > 1) { + qsort(cv->v, (size_t)cv->n, sizeof(cand_t), cand_cmp); + } + return !cv->fail && !cv->pool.fail; +} + +/* ── Joins across a line break ───────────────────────────────────── */ + +typedef enum { JK_NONE = 0, JK_SOFT, JK_HYPHEN, JK_CODE } join_kind_t; + +static join_kind_t join_kind(const uint32_t *prev, int pn, const uint32_t *nxt, int nn) { + if (!pn || !nn) { + return JK_NONE; + } + if (prev[pn - 1] == 0x00AD && pdf_u_alpha(nxt[0])) { + return JK_SOFT; + } + if (prev[pn - 1] == '-' && pn >= 2 && pdf_u_alpha(prev[pn - 2]) && pdf_u_lower(nxt[0])) { + return JK_HYPHEN; + } + static const char *const sufs[] = {"::", "->", "_", "/"}; + for (int k = 0; k < 4; k++) { + int sl = (int)strlen(sufs[k]); + if (pn > sl && lit_at(prev, pn, pn - sl, sufs[k]) && idc(prev[pn - sl - 1]) && + idc(nxt[0])) { + return JK_CODE; + } + } + if (idc(prev[pn - 1])) { + /* (?:\.|::|->|_)[A-Za-z_] | \(\) at the start of nxt */ + int s = 0; + if (nxt[0] == '.' || nxt[0] == '_') { + s = 1; + } else if (nn >= 2 && + ((nxt[0] == ':' && nxt[1] == ':') || (nxt[0] == '-' && nxt[1] == '>'))) { + s = 2; + } + if (s && s < nn && (is_alpha_ascii(nxt[s]) || nxt[s] == '_')) { + return JK_CODE; + } + if (nn >= 2 && nxt[0] == '(' && nxt[1] == ')') { + return JK_CODE; + } + } + return JK_NONE; +} + +/* [A-Za-z0-9_$@.:/\-#>­] and [A-Za-z0-9_$@.:/\-#>()] */ +static bool left_run_char(uint32_t c) { + return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' || + c == '>' || c == 0x00AD; +} + +static bool right_run_char(uint32_t c) { + return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' || + c == '>' || c == '(' || c == ')'; +} + +/* ── Emission ────────────────────────────────────────────────────── */ + +typedef struct { + CBMExtractCtx *ctx; + const char *page_qn; + uint32_t page; + const uint32_t *flat; + int n; + const int *line_starts; + int nlines; + bool failed; +} pdf_emit_t; + +static uint32_t line_of(const pdf_emit_t *e, int off) { + int lo = 0; + int hi = e->nlines - 1; + while (lo < hi) { + int mid = lo + (hi - lo + 1) / 2; + if (e->line_starts[mid] <= off) { + lo = mid; + } else { + hi = mid - 1; + } + } + return (uint32_t)lo + 1; +} + +/* R3: "foo.bar=" right after "(" or "," -- a keyword argument, not a reference */ +static bool kwarg_at(const pdf_emit_t *e, int start, int end) { + if (end >= e->n || e->flat[end] != '=' || (end + 1 < e->n && e->flat[end + 1] == '=')) { + return false; + } + int lim = start - PDF_R3_WINDOW < 0 ? 0 : start - PDF_R3_WINDOW; + for (int i = start - 1; i >= lim; i--) { + if (!pdf_u_space(e->flat[i])) { + return e->flat[i] == '(' || e->flat[i] == ','; + } + } + return false; +} + +static int u8_put(uint32_t cp, char *o) { + if (cp < 0x80) { + o[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + o[0] = (char)(0xC0 | (cp >> 6)); + o[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + if (cp < 0x10000) { + o[0] = (char)(0xE0 | (cp >> 12)); + o[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + o[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + o[0] = (char)(0xF0 | (cp >> 18)); + o[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + o[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + o[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +static int kind_syntax(mention_kind_t k) { + switch (k) { + case MK_PATH: + case MK_PATH1: + return CBM_DOCLINK_PDF_PATH; + case MK_FILE: + return CBM_DOCLINK_PDF_FILE; + case MK_QUAL: + return CBM_DOCLINK_PDF_QN; + default: + return CBM_DOCLINK_PDF_NAME; + } +} + +static bool structural(mention_kind_t k) { + return k == MK_PATH || k == MK_PATH1 || k == MK_FILE || k == MK_QUAL; +} + +/* Push a token; returns its index in the file's token array, or -1. */ +static int emit(pdf_emit_t *e, int syntax, const char *raw, uint32_t line, uint16_t flags) { + CBMArena *a = e->ctx->arena; + char *r = cbm_arena_strdup(a, raw); + if (!r) { + e->failed = true; + return -1; + } + CBMDocLink link = { + .source_qn = e->page_qn, + .raw = r, + .line = line, + .def_line = e->page, + .syntax = (uint16_t)syntax, + .flags = flags, + }; + int idx = e->ctx->result->doc_links.count; + cbm_doclinks_push(&e->ctx->result->doc_links, a, link); + if (e->ctx->result->doc_links.count != idx + 1) { + e->failed = true; + return -1; + } + return idx; +} + +typedef struct { + char *p; + size_t n; + size_t cap; + bool fail; +} sbuf_t; + +static void sb_put(sbuf_t *b, const char *s, size_t n) { + if (b->fail) { + return; + } + if (b->n + n + 1 > b->cap) { + size_t ncap = b->cap ? b->cap * 2 : 256; + while (ncap < b->n + n + 1) { + ncap *= 2; + } + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->p, ncap); + if (!g) { + b->fail = true; + return; + } + b->p = g; + b->cap = ncap; + } + memcpy(b->p + b->n, s, n); + b->n += n; + b->p[b->n] = '\0'; +} + +static void sb_cps(sbuf_t *b, const uint32_t *s, int n) { + char u[PDF_U8_MAX]; + for (int i = 0; i < n; i++) { + int k = u8_put(s[i], u); + sb_put(b, u, (size_t)k); + } +} + +/* One page: its line-local structural tokens, then its join proposals. */ +static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn) { + u32v_t norm = {0}; + if (!normalise(text, len, &norm)) { + u32_free(&norm); + ctx->result->doc_links.failed = true; + return; + } + /* lines, each stripped; flat = lines joined by '\n' */ + u32v_t flat = {0}; + int *starts = NULL; + int nlines = 0; + int cap = 0; + int *lens = NULL; + for (int i = 0; i <= norm.n;) { + int j = i; + while (j < norm.n && norm.v[j] != '\n') { + j++; + } + int a = i; + int b = j; + while (a < b && pdf_u_space(norm.v[a])) { + a++; + } + while (b > a && pdf_u_space(norm.v[b - 1])) { + b--; + } + if (nlines == cap) { + cap = cap ? cap * 2 : PDF_LIST_MIN; + int *g1 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, starts, (size_t)cap * sizeof(int)); + if (g1) { + starts = g1; + } + int *g2 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, lens, (size_t)cap * sizeof(int)); + if (g2) { + lens = g2; + } + if (!g1 || !g2) { + flat.fail = true; + break; + } + } + if (nlines) { + u32_push(&flat, '\n'); + } + starts[nlines] = flat.n; + lens[nlines] = b - a; + nlines++; + for (int k = a; k < b; k++) { + u32_push(&flat, norm.v[k]); + } + if (j >= norm.n) { + break; + } + i = j + 1; + } + u32_free(&norm); + candv_t cv = {0}; + if (flat.fail || !candidates(flat.v, flat.n, false, &cv)) { + ctx->result->doc_links.failed = true; + goto done; + } + pdf_emit_t e = {ctx, page_qn, page, flat.v, flat.n, starts, nlines, false}; + /* the token index of every structural candidate (-1: not emitted) */ + int *tok = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(cv.n ? cv.n : 1) * sizeof(int)); + if (!tok) { + ctx->result->doc_links.failed = true; + goto done; + } + sbuf_t sb = {0}; + for (int i = 0; i < cv.n; i++) { + const cand_t *c = &cv.v[i]; + tok[i] = -1; + if (!structural(c->kind) || kwarg_at(&e, c->start, c->end)) { + continue; + } + sb.n = 0; + sb_cps(&sb, cv.pool.v + c->norm_start, c->norm_len); + if (sb.fail) { + break; + } + tok[i] = emit(&e, kind_syntax(c->kind), sb.p ? sb.p : "", line_of(&e, c->start), 0); + } + /* The emitted candidates of each line, ascending (CSR: lc[lc_off[l] .. + * lc_off[l + 1])). A join's span covers the end of one line and the start of + * the next, so only their candidates can overlap it; reading every candidate + * of the page per join cost (joins x candidates), quadratic in the text. */ + int *lc_off = (int *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(nlines + 2) * sizeof(int)); + int *lc = NULL; + if (!lc_off) { + e.failed = true; + } else { + for (int f = 0; f < cv.n; f++) { + if (tok[f] >= 0) { + int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start; + for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) { + lc_off[l]++; /* line_of is 1-based: line l - 1 counts at l */ + } + } + } + for (int l = 0; l < nlines; l++) { + lc_off[l + 1] += lc_off[l]; + } + lc = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, + (size_t)(lc_off[nlines] ? lc_off[nlines] : 1) * sizeof(int)); + if (!lc) { + e.failed = true; + } else { + /* lc_off[l] runs as line l's cursor, then shifts back to its start */ + for (int f = 0; f < cv.n; f++) { + if (tok[f] >= 0) { + int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start; + for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) { + lc[lc_off[l - 1]++] = f; + } + } + } + for (int l = nlines; l > 0; l--) { + lc_off[l] = lc_off[l - 1]; + } + lc_off[0] = 0; + } + } + /* joins */ + for (int li = 0; li + 1 < nlines && !sb.fail && !e.failed; li++) { + const uint32_t *prev = flat.v + starts[li]; + const uint32_t *nxt = flat.v + starts[li + 1]; + int pn = lens[li]; + int nn = lens[li + 1]; + join_kind_t jk = join_kind(prev, pn, nxt, nn); + if (jk == JK_NONE) { + continue; + } + int ls = pn; + while (ls > 0 && left_run_char(prev[ls - 1])) { + ls--; + } + int re = 0; + while (re < nn && right_run_char(nxt[re])) { + re++; + } + if (ls == pn || re == 0) { + continue; + } + int left_len = pn - ls; + int dropped = (jk == JK_HYPHEN || jk == JK_SOFT) ? 1 : 0; + int junction = left_len - dropped; + u32v_t joined = {0}; + for (int k = ls; k < pn - dropped; k++) { + u32_push(&joined, prev[k]); + } + for (int k = 0; k < re; k++) { + u32_push(&joined, nxt[k]); + } + candv_t jc = {0}; + if (joined.fail || !candidates(joined.v, joined.n, true, &jc)) { + u32_free(&joined); + candv_free(&jc); + ctx->result->doc_links.failed = true; + break; + } + int a0 = starts[li] + ls; + int b0 = starts[li + 1]; + for (int k = 0; k < jc.n; k++) { + const cand_t *c = &jc.v[k]; + if (!(c->start < junction && junction < c->end)) { + continue; + } + int s_flat = a0 + c->start; + int e_flat = b0 + (c->end - junction); + if (structural(c->kind) && kwarg_at(&e, s_flat, e_flat)) { + continue; + } + sb.n = 0; + const uint32_t *nm = jc.pool.v + c->norm_start; + sb_cps(&sb, nm, c->norm_len); + if (dropped) { + /* alts: the norm with the hyphen kept, at the junction */ + /* Python slices clamp: past the end, the hyphen goes last */ + int at = junction - c->start; + at = at > c->norm_len ? c->norm_len : at; + char sep = PDF_JOIN_SEP; + sb_put(&sb, &sep, 1); + sb_cps(&sb, nm, at); + sb_put(&sb, "-", 1); + sb_cps(&sb, nm + at, c->norm_len - at); + } + char fs = PDF_JOIN_FRAG; + sb_put(&sb, &fs, 1); + bool first = true; + int ia = lc_off[li]; + int ea = lc_off[li + 1]; + int ib = lc_off[li + 1]; + int eb = lc_off[li + 2]; + while (ia < ea || ib < eb) { + int f; + if (ib >= eb || (ia < ea && lc[ia] < lc[ib])) { + f = lc[ia++]; + } else if (ia >= ea || lc[ib] < lc[ia]) { + f = lc[ib++]; + } else { + f = lc[ia++]; /* on both lines */ + ib++; + } +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&g_join_steps, 1, memory_order_relaxed); +#endif + const cand_t *x = &cv.v[f]; + if (x->start < e_flat && x->end > s_flat) { + char num[16]; + int nlen = snprintf(num, sizeof(num), first ? "%d" : ",%d", tok[f]); + sb_put(&sb, num, (size_t)nlen); + first = false; + } + } + if (sb.fail) { + break; + } + emit(&e, kind_syntax(c->kind), sb.p, line_of(&e, s_flat), CBM_DOCLINK_FLAG_JOIN); + } + u32_free(&joined); + candv_free(&jc); + } + if (sb.fail || e.failed) { + ctx->result->doc_links.failed = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, sb.p); + cbm_free(CBM_MEM_CLASS_EXTRACT, tok); + cbm_free(CBM_MEM_CLASS_EXTRACT, lc_off); + cbm_free(CBM_MEM_CLASS_EXTRACT, lc); +done: + candv_free(&cv); + u32_free(&flat); + cbm_free(CBM_MEM_CLASS_EXTRACT, starts); + cbm_free(CBM_MEM_CLASS_EXTRACT, lens); +} + +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn) { + scan_page(ctx, text, len, page, page_qn); +} + +size_t cbm_pdf_test_join_steps(void) { + return atomic_load_explicit(&g_join_steps, memory_order_relaxed); +} +#endif + +/* ── The document ────────────────────────────────────────────────── */ + +/* R1: the field test's category(): a PDF under a test or fixture directory. */ +static bool fixture_pdf(const char *rel_path) { + const char *s = rel_path; + for (;;) { + const char *slash = strchr(s, '/'); + if (!slash) { + return false; + } + size_t n = (size_t)(slash - s); + char comp[256]; + if (n < sizeof(comp)) { + for (size_t i = 0; i < n; i++) { + comp[i] = (char)(s[i] >= 'A' && s[i] <= 'Z' ? s[i] + ('a' - 'A') : s[i]); + } + comp[n] = '\0'; + for (int i = 0; FIXTURE_DIRS[i]; i++) { + if (strcmp(comp, FIXTURE_DIRS[i]) == 0) { + return true; + } + } + if (strstr(comp, "test") && strcmp(comp, "testing") != 0) { + return true; + } + } + s = slash + 1; + } +} + +/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes + * cut on a character boundary and then said so. A page holds a few kilobytes; + * only a crafted text layer reaches the bound, and the mentions are still read + * from the whole text. */ +static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) { + if (pg->len <= PDF_PAGE_DOC_MAX) { + return pg->text; + } + size_t cut = PDF_PAGE_DOC_MAX; + while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) { + cut--; /* not inside a UTF-8 sequence */ + } + const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut, + pg->text, cut, pg->len); + return doc ? doc : pg->text; +} + +void cbm_pdf_extract_document(CBMExtractCtx *ctx) { + CBMFileResult *result = ctx->result; + CBMArena *a = ctx->arena; + cbm_pdf_result_t pr; + if (!cbm_pdf_extract((const unsigned char *)ctx->source, (size_t)ctx->source_len, a, &pr)) { + result->has_error = true; + result->error_msg = cbm_arena_strdup(a, "pdf: out of memory"); + return; + } + if (pr.status != CBM_PDF_OK) { + static const char *const why[] = {"ok", "pdf: not a PDF", "pdf: no pages", "pdf: encrypted", + "pdf: out of memory"}; + result->has_error = pr.status != CBM_PDF_ENCRYPTED; + result->error_msg = cbm_arena_strdup(a, why[pr.status]); + return; + } + bool fixture = fixture_pdf(ctx->rel_path ? ctx->rel_path : ""); + for (int i = 0; i < pr.npages; i++) { + uint32_t page = (uint32_t)i + 1; + char name[PDF_PAGE_NAME]; + char seg[PDF_PAGE_NAME]; + snprintf(name, sizeof(name), "page %u", page); + snprintf(seg, sizeof(seg), "page_%u", page); + char *nm = cbm_arena_strdup(a, name); + char *qn = cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg); + char *pv = cbm_arena_sprintf(a, "%u", page); + const char **kv = (const char **)cbm_arena_alloc(a, 3 * sizeof(char *)); + if (!nm || !qn || !pv || !kv) { + result->has_error = true; + return; + } + kv[0] = "page"; + kv[1] = pv; + kv[2] = NULL; + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = nm; + def.qualified_name = qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = page; + def.end_line = page; + def.is_exported = true; + def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL; + def.extra_props = kv; + cbm_defs_push(&result->defs, a, def); + if (!fixture && pr.pages[i].len) { + scan_page(ctx, pr.pages[i].text, pr.pages[i].len, page, qn); + } + } +} diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index d7d9b8b419..09ee36c25f 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -69,7 +69,8 @@ void cbm_doclink_test_note_doc_work(uint64_t copied, uint64_t parse_input, uint6 * doclink.h (language, name, external, and the ship gate). */ static const CBMDocLinkFamily DOCLINK_FAMILIES[CBM_DOCLINK_SYNTAX_COUNT] = { -#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships) [id] = {lang, name, external, ships}, +#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships, edge) \ + [id] = {lang, name, external, ships, edge}, CBM_DOCLINK_FAMILY_LIST(DOCLINK_FAMILY_ROW) #undef DOCLINK_FAMILY_ROW }; @@ -87,6 +88,11 @@ const char *cbm_doclink_syntax_name(int syntax) { return f ? f->name : ""; } +const char *cbm_doclink_syntax_edge(int syntax) { + const CBMDocLinkFamily *f = cbm_doclink_family(syntax); + return f && f->edge ? f->edge : "MENTIONS"; +} + bool cbm_doclink_syntax_is_external(int syntax) { const CBMDocLinkFamily *f = cbm_doclink_family(syntax); return f && f->external; @@ -143,6 +149,9 @@ typedef struct { * the same doc): only the first label carries the doc's references. */ const char *twin_label; const char *twin_of; + /* A document language: the whole file is the documentation, so one scan + * of its bytes replaces parse_doc, the file-level doc and the scope. */ + void (*scan_file)(CBMExtractCtx *ctx); } doclink_lang_t; static const doclink_lang_t DOCLINK_LANGS[] = { @@ -159,6 +168,18 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scan_scope = cbm_doclink_cs_project_scan_scope, .scope_tag = CBM_DOCLINK_CS_SCOPE_TAG, .portable_scope = cbm_doclink_cs_portable_scope}, + {.lang = CBM_LANG_MARKDOWN, .scan_file = cbm_doclink_md_scan_file}, + {.lang = CBM_LANG_RST, .scan_file = cbm_doclink_rst_scan_file}, + /* Python: no doc references yet, only the scope the reST resolver reads + * (package re-exports, Sphinx conf.py settings) */ + {.lang = CBM_LANG_PYTHON, + .scan_scope = cbm_doclink_py_scan_scope, + .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG}, + /* YAML: the Antora component and playbook settings the AsciiDoc resolver + * reads */ + {.lang = CBM_LANG_YAML, + .scan_scope = cbm_doclink_antora_scan_scope, + .scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { @@ -171,7 +192,8 @@ static const doclink_lang_t *doclink_lang(CBMLanguage lang) { } bool cbm_doclink_lang_supported(CBMLanguage lang) { - return doclink_lang(lang) != NULL; + const doclink_lang_t *L = doclink_lang(lang); + return L && (L->parse_doc || L->scan_file); } /* ── Doc-line side map ───────────────────────────────────────────── */ @@ -388,6 +410,14 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) { if (!L) { return; } + if (L->scan_file) { + L->scan_file(ctx); + return; + } + if (!L->parse_doc) { /* a scope, and no references (Python) */ + ctx->result->doc_scope = L->scan_scope ? L->scan_scope(ctx) : NULL; + return; + } int twin_count = 0; doclink_twin_key_t *twins = L->twin_label ? twin_keys(ctx, L->twin_of, &twin_count) : NULL; CBMDefArray *defs = &ctx->result->defs; diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index 7b85ad45a3..b33812dec2 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -22,6 +22,10 @@ * identical on full and incremental runs. Optional. * scope_tag, the blob's tag line and its persisted form (line * portable_scope numbers dropped). Optional. + * scan_file instead of parse_doc, for a DOCUMENT language whose + * text is the documentation (Markdown): one pass over + * the file's bytes pushes every reference, with the + * section it is written in as its source. Optional. * State a language's hooks need between their calls for one file lives in * ctx->doclink_state (cbm.h), never in a static or thread-local. * A language without a row produces no tokens and no scope. The resolving @@ -39,9 +43,9 @@ /* Every link family, across languages: one line per (language, reference * form), and the ONE place a language leg adds its families -- * - * X(enum value, language, name, external, ships) + * X(enum value, language, name, external, ships, edge) * - * name the "syntax" MENTIONS-edge property and doc_link_unresolved + * name the "syntax" edge property and doc_link_unresolved * column, so it is public surface; with the language it is the * tier a release audit judges. Names repeat across languages * (`see` is C#'s, Java's, Kotlin's ...): the value, not the name, @@ -51,22 +55,73 @@ * edge. false: the tier is below the audit's bar; its references * still resolve, but a resolved one stays a doc_link_unresolved * row with reason below_bar_tier (an unresolved one keeps its own - * reason). Nothing else about the family changes. + * reason). Nothing else about the family changes. A document + * family ships once a held-out audit passed it: below the bar + * (AsciiDoc code spans naming paths, AsciiDoc code names, reST + * object directives; Markdown code spans naming a file or one + * directory alone, split off the Markdown code-path tier after + * its held-out failure; ADR supersedes words in running prose, + * split off after three held-out censuses failed on them), or + * not yet audited for want of held-out occurrences (reST + * include, kernel_doc; AsciiDoc attribute) means false. + * Markdown code paths without the bare part, reST + * literalinclude and code paths, PDF qualified names after the + * owner rule, and ADR supersedes statements (the phrase opens + * its line or field and names a record: 80 of 81 on a fresh + * census) passed supplementary held-out audits. + * edge the type of the edge a resolved reference becomes: MENTIONS + * (documentation names code), SUPERSEDES (an ADR states that it + * supersedes another). The edge's source is always the + * definition the reference is written in, so a file owns the + * edges its text produces, whatever their type. * * The enum below and the family table (doclink.c) are both generated from this * list, so a value cannot exist without its name and its gate. */ -#define CBM_DOCLINK_FAMILY_LIST(X) \ - /* csharp */ \ - X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true) \ - X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true) \ - X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true) \ - X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true) \ - /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ - X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false) +#define CBM_DOCLINK_FAMILY_LIST(X) \ + /* csharp */ \ + X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ + /* markdown (doclink_md.c): what a document's text names explicitly */ \ + X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ + /* a code span naming a file or directory alone (`config.yaml`, `doc/`) */ \ + X(CBM_DOCLINK_MD_BARE_PATH, CBM_LANG_MARKDOWN, "bare_path", false, false, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ + /* an ADR's own "supersedes" / "replaces" statement, opening its line or field */ \ + /* and naming a record directly (doc_adr.c): ADR -> ADR */ \ + X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ + /* the same words in running prose ("this record supersedes ADR-3") */ \ + X(CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_LANG_MARKDOWN, "supersedes_prose", false, false, \ + "SUPERSEDES") \ + /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ + /* names) only decides which line fragments a cross-line join replaces */ \ + X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ + /* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \ + X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, false, "MENTIONS") \ + X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, false, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \ + /* asciidoc (doc_adoc.c): includes, Antora API attributes, monospace spans */ \ + X(CBM_DOCLINK_ADOC_INCLUDE, CBM_LANG_ASCIIDOC, "include", false, true, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_LANG_ASCIIDOC, "attribute", false, false, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_CODE_PATH, CBM_LANG_ASCIIDOC, "code_path", false, false, "MENTIONS") \ + X(CBM_DOCLINK_ADOC_CODE_NAME, CBM_LANG_ASCIIDOC, "code_name", false, false, "MENTIONS") \ + /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ + X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") typedef enum { CBM_DOCLINK_NONE = 0, -#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships) id, +#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships, edge) id, CBM_DOCLINK_FAMILY_LIST(CBM_DOCLINK_FAMILY_VALUE) #undef CBM_DOCLINK_FAMILY_VALUE CBM_DOCLINK_SYNTAX_COUNT @@ -79,8 +134,13 @@ enum { * edge source is the file's File node, which the resolving half looks up * from the file it is resolving; `source_qn` is not the source then. Set * by the driver (cbm_doclinks_extract) on what a parser pushes for the - * file-level doc -- a parser never sets it. */ + * file-level doc -- a parser never sets it. A scan_file hook sets it on a + * reference written before the document's first section. */ CBM_DOCLINK_FLAG_FILE = 1, + /* A PDF mention that runs across a line break, proposed as one reference. + * `raw` holds its forms (separated by 0x1F) and, after 0x1E, the indices + * of the line-local tokens it replaces when it resolves (doc_pdf.c). */ + CBM_DOCLINK_FLAG_JOIN = 2, }; /* One link family: a line of CBM_DOCLINK_FAMILY_LIST. */ @@ -89,8 +149,12 @@ typedef struct { const char *name; bool external; bool ships; + const char *edge; /* the type of the edge a resolved reference becomes */ } CBMDocLinkFamily; +/* "MENTIONS" or "SUPERSEDES"; "MENTIONS" for an unknown value. */ +const char *cbm_doclink_syntax_edge(int syntax); + /* The family of a syntax value; NULL for a value that names none. */ const CBMDocLinkFamily *cbm_doclink_family(int syntax); @@ -221,4 +285,105 @@ size_t cbm_doclink_cs_norm_type(const char *in, size_t len, char *out, size_t ca /* The C# scope blob starts with this tag line. */ #define CBM_DOCLINK_CS_SCOPE_TAG "cs1" +/* ── Markdown (doclink_md.c) ───────────────────────────────────────── */ + +/* The scan_file hook: every explicit reference of a Markdown or MDX file (a + * link destination, a bare path, a path or a qualified name in a code span), + * each with the Section definition it is written under as its source. */ +void cbm_doclink_md_scan_file(CBMExtractCtx *ctx); + +/* What a code span's text names. */ +typedef enum { + CBM_DOCLINK_MD_NONE = 0, /* nothing linkable (a bare name, a command, a snippet, ...) */ + CBM_DOCLINK_MD_FILE, /* a path with a directory part, or a dot file */ + CBM_DOCLINK_MD_FILENAME, /* a file name alone: it names a file of the document's + * own directory, never one found elsewhere */ + CBM_DOCLINK_MD_DIR, /* a directory */ + CBM_DOCLINK_MD_QUALIFIED, /* a qualified name of two or more identifiers + * (`a.b`, `a::b`, `A#b`, `A\B`, `mod:attr`) */ +} CBMDocLinkMdShape; + +typedef struct { + CBMDocLinkMdShape shape; + const char *path; /* NUL-terminated, in the caller's buffer; for QUALIFIED the + * identifiers joined by '.' */ + const char *member; /* the text after a `path::`, NUL-terminated in the same + * buffer: the member the path's file declares; NULL: none */ + uint32_t first_line; /* a line range written with the path (`#L3-L9`, `:3-9`, `:3`); */ + uint32_t last_line; /* 0 and 0 when there is none */ + bool instance; /* QUALIFIED: written through an instance (`self.x`, `this.x`): + * the qualifier is no type, so it never names a field */ + bool colon; /* QUALIFIED: written `module:attr` (Python's entry-point form) */ +} CBMDocLinkMdPath; + +/* Classify the text of a code span. Returns false, with shape NONE, when it + * names no path and no qualified name, or when `buf` (cap bytes) cannot hold + * it: a name is never cut. Pure: the same text always gives the same answer. */ +bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_t cap, + CBMDocLinkMdPath *out); + +/* Parse a `#L3`, `#L3-L9` or `#L3C2-L9C5` fragment (without the `#`): true + * and the lines, or false. */ +bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last); + +/* ── ADRs (doc_adr.c) ──────────────────────────────────────────────── */ + +/* When the Markdown file being extracted is an architecture decision record: + * push its ADR definition (name its canonical id, qualified name + * ".__adr__", facts as extra properties) and a `supersedes` token for + * each of its own supersedes / replaces statements. Called by the Markdown + * scan, after the Section definitions exist. */ +void cbm_adr_extract(CBMExtractCtx *ctx); + +/* The qualified-name tail of an ADR node: "" + this. */ +#define CBM_ADR_QN_NAME "__adr__" + +/* ── reStructuredText (doclink_rst.c) and Python's scope (doclink_py.c) ── */ + +/* The scan_file hook of a reStructuredText document: a Section per title + * (the text up to the next title as its docstring) and its references to + * code -- Sphinx roles and object directives, autodoc, include / + * literalinclude / kernel-doc paths, inline literals naming a path or a + * qualified name -- each with the section it is written in as its source. */ +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx); + +/* The scope blob of a Python file, which the reST resolver reads: a package + * `__init__.py`'s top-level explicit `from x import y [as z]` (what the + * package re-exports), and a Sphinx `conf.py`'s primary domain, extlink roles + * to repository paths and intersphinx names (read as text, never run). NULL + * for every other file. */ +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx); + +/* The Python scope blob starts with this tag line. */ +#define CBM_DOCLINK_PY_SCOPE_TAG "py1" + +/* ── AsciiDoc (doc_adoc.c) ─────────────────────────────────────────── */ + +/* AsciiDoc documents: a Section per heading and the references to code + * (include::, Antora API attributes, monospace spans) as doc-link tokens. + * Runs instead of a grammar for CBM_LANG_ASCIIDOC. */ +void cbm_adoc_extract_document(CBMExtractCtx *ctx); + +/* The scope blob of a YAML file, which the AsciiDoc resolver reads: an + * antora.yml's component name, asciidoc attributes and collector scans, an + * antora-playbook*.yml's attributes (read as text). NULL for every other + * file. */ +const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx); + +/* The Antora scope blob starts with this tag line. */ +#define CBM_DOCLINK_ADOC_SCOPE_TAG "ad1" + +/* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's + * text as docstring, property page) and the pages' structural code mentions + * as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */ +void cbm_pdf_extract_document(CBMExtractCtx *ctx); +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +/* Test seam: scan one page's text as cbm_pdf_extract_document does (tokens + * into ctx->result->doc_links), without a PDF around it. */ +void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page, + const char *page_qn); +/* candidates the scanner's joins have compared so far (a cost counter) */ +size_t cbm_pdf_test_join_steps(void); +#endif + #endif /* CBM_DOCLINK_H */ diff --git a/internal/cbm/doclink_md.c b/internal/cbm/doclink_md.c new file mode 100644 index 0000000000..0468f72756 --- /dev/null +++ b/internal/cbm/doclink_md.c @@ -0,0 +1,1812 @@ +/* + * doclink_md.c — Markdown references to code: the extraction half. + * + * In a Markdown document the text IS the documentation, so its references are + * not taken from a definition's doc comment: one pass over the file's bytes + * finds the explicit ones and gives each to the section it is written in -- + * the Section definition (a heading) at or above its line, or the file itself + * for text before the first heading (CBM_DOCLINK_FLAG_FILE). Families + * (doclink.h): + * + * link the destination of an inline link `[text](dest)` or of a link + * reference definition `[label]: dest`, when it is no URL, no + * in-page anchor and no image + * path a bare path in prose: a slash and a file extension + * code_path a code span, or an HTML span, whose text is a path, a + * file name or a directory, with an optional line range + * (`#L3-L9`, `:3-9`) or member (`path::Name`) + * + * Never scanned: YAML front matter, fenced code blocks, HTML comments, the + * import/export lines of MDX. Never a reference here: a URL, an in-page + * anchor, an image, a name without a path (a bare name is at most a + * suggestion, never an edge). + * + * Cost. Each scan of a line is linear in the line, or n log n where it sorts. + * The searches that could otherwise rescan a stretch of the line for every + * candidate (the end of a link destination, the closing parenthesis, the + * closing backticks of a code span) are answered from position lists of the + * few bytes they look for, by binary search. The lists grow with the number + * of those bytes, not with the line, and are released at the end of the + * file. A list that cannot grow marks the file's doc links failed, so the + * resolving half fails the layer instead of publishing a graph that silently + * misses references. + * + * The rules are those the documentation field tests measured (H0): explicit + * paths and links bind EXACTLY, at 99 % precision on 53 repositories. + */ +#include "doclink.h" + +#include "arena.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" + +#include +#include +#include +#include + +#define MD_SLEN(s) (sizeof(s) - SKIP_ONE) + +enum { + MD_MASK = 0x01, /* a byte the later scans of the line must not see */ + MD_FENCE_MIN = 3, /* ``` or ~~~ */ + MD_INDENT_MAX = 3, /* a link reference definition may be indented this far */ + MD_SPAN_MAX = 150, /* longer code-span text is a snippet, not a name */ + MD_EXT_MAX = 12, /* a file extension: a letter and at most 11 more */ + MD_TAG_MAX = 200, /* an HTML tag inside a span is at most this long */ + MD_CALL_ARGS = 60, /* `name(args)`: the argument text of a call form */ + MD_CALL_ARGS_WIDE = 80, /* the same with spaces or commas in the arguments */ + MD_TINY_DOTFILE = 3, /* `.a` is no file name, `.ab` is */ + MD_LIST_INIT = 16, /* first capacity of a position list */ + MD_NUM_DIGITS = 9, /* a line number has at most this many digits */ + MD_PAIR = 2, /* ints per pair entry */ + MD_URL_SCHEMES = 3, /* http, https, ftp */ + MD_BOM_LEN = 3, /* EF BB BF */ +}; + +/* UTF-8 well-formedness (RFC 3629, table 3-7 of the Unicode standard). */ +enum { + UTF8_CTRL_END = 0x20, + UTF8_DEL = 0x7f, + UTF8_ASCII_END = 0x80, + UTF8_CONT_MIN = 0x80, + UTF8_CONT_MAX = 0xBF, + UTF8_LEAD2_MIN = 0xC2, + UTF8_LEAD2_MAX = 0xDF, + UTF8_LEAD3_MIN = 0xE0, + UTF8_LEAD3_MAX = 0xEF, + UTF8_E0_SECOND_MIN = 0xA0, + UTF8_SURROGATE_LEAD = 0xED, + UTF8_ED_SECOND_MAX = 0x9F, + UTF8_LEAD4_MIN = 0xF0, + UTF8_LEAD4_MAX = 0xF4, + UTF8_F0_SECOND_MIN = 0x90, + UTF8_F4_SECOND_MAX = 0x8F, +}; + +/* ── Small character classes (ASCII; bytes >= 0x80 are word bytes) ── */ + +static bool md_blank(unsigned char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f' || c == '\v'; +} + +static bool md_sp_tab(unsigned char c) { + return c == ' ' || c == '\t'; +} + +static bool md_alpha(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool md_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static bool md_alnum(unsigned char c) { + return md_alpha(c) || md_digit(c); +} + +/* Python's \w on text: letters, digits, '_' and any non-ASCII byte. */ +static bool md_word(unsigned char c) { + return md_alnum(c) || c == '_' || c >= 0x80; +} + +static unsigned char md_lower(unsigned char c) { + return (c >= 'A' && c <= 'Z') ? (unsigned char)(c - 'A' + 'a') : c; +} + +static bool md_has(const char *s, size_t n, char c) { + return n > 0 && memchr(s, c, n) != NULL; +} + +static bool md_eq_ci(const char *s, size_t n, const char *word) { + size_t w = strlen(word); + if (n != w) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (md_lower((unsigned char)s[i]) != (unsigned char)word[i]) { + return false; + } + } + return true; +} + +static bool md_in_list(const char *s, size_t n, const char *const *list, bool ci) { + for (size_t i = 0; list[i]; i++) { + if (ci ? md_eq_ci(s, n, list[i]) : (strlen(list[i]) == n && memcmp(s, list[i], n) == 0)) { + return true; + } + } + return false; +} + +/* File extensions that make a dotted token a FILE NAME. Strict: none of them + * is a common member name (`.log`, `.get` and `.map` are not here). */ +static const char *const MD_FILE_EXT[] = { + "py", "pyi", "js", "jsx", "mjs", "cjs", "ts", "tsx", "mts", "cts", "go", "rs", "c", "h", "cc", + "cpp", "cxx", "hpp", "hh", "java", "kt", "kts", "scala", "groovy", "cs", "php", "rb", "sh", + "bash", "zsh", "ps1", "md", "mdx", "rst", "adoc", "txt", "json", "jsonc", "yaml", "yml", "toml", + "ini", "cfg", "xml", "html", "htm", "css", "scss", "sass", "less", "vue", "svelte", "proto", + "sql", "lock", "gradle", "csproj", "sln", "fsproj", "props", "targets", "cmake", "mk", "tf", + "tfvars", "hcl", "png", "jpg", "jpeg", "gif", "svg", "pdf", "ipynb", "dockerfile", "swift", + "dart", "lua", "pl", "pm", "ex", "exs", "erl", "clj", "hs", "ml", "zig", "sol", "graphql", + "gql", "bzl", "nix", "plist", "env", "conf", "properties", "editorconfig", "gitignore", "npmrc", + "nvmrc", "j2", "jinja", "tmpl", "tpl", "erb", "hbs", "mustache", "patch", "diff", "whl", "jar", + "zip", "gz", "tgz", "so", "dll", "dylib", "wasm", "service", "socket", "timer", "mod", "sum", + /* built and shipped files, never a member's name (`OliveTin.exe` took the + * module stem of OliveTin.exe.manifest in a held-out audit) */ + "exe", "msi", "deb", "rpm", "dmg", "tar", "xz", "bz2", "7z", "apk", "ipa", "manifest", "csv", + "tsv", NULL}; + +/* File names that are files without an extension. */ +static const char *const MD_SPECIAL_FILES[] = { + "Makefile", "Dockerfile", "Containerfile", "Jenkinsfile", "Vagrantfile", "Gemfile", "Rakefile", + "Procfile", "Kconfig", "CODEOWNERS", "MAINTAINERS", "LICENSE", "NOTICE", "go.mod", + "go.sum", "Cargo.lock", "Justfile", "BUILD", "WORKSPACE", NULL}; + +/* Host names written as code: a site, not a file. */ +static const char *const MD_DOMAINS[] = {"docs.rs", "crates.io", "pkg.go.dev", + "npmjs.com", "pypi.org", "github.com", + "gitlab.com", "readthedocs.io", "rubygems.org", + "nuget.org", "go.dev", "golang.org", + "godoc.org", "packagist.org", "hex.pm", + "mvnrepository.com", "docs.python.org", NULL}; + +/* `.ext`, a letter and up to 11 of [A-Za-z0-9_+-], at the end of `s`. */ +static bool md_ext_ok(const char *s, size_t n) { + const char *dot = NULL; + for (size_t i = n; i > 0; i--) { + if (s[i - SKIP_ONE] == '.') { + dot = s + i - SKIP_ONE; + break; + } + } + if (!dot) { + return false; + } + size_t ext_len = n - (size_t)(dot - s) - SKIP_ONE; + if (ext_len == 0 || ext_len > MD_EXT_MAX || !md_alpha((unsigned char)dot[SKIP_ONE])) { + return false; + } + for (size_t i = 2; i <= ext_len; i++) { + unsigned char c = (unsigned char)dot[i]; + if (!md_alnum(c) && c != '_' && c != '+' && c != '-') { + return false; + } + } + return true; +} + +/* `^[A-Za-z][A-Za-z0-9+.-]*://`, `mailto:` or `www.`. */ +static bool md_is_url(const char *s, size_t n) { + if (n >= MD_SLEN("mailto:") && md_eq_ci(s, MD_SLEN("mailto:"), "mailto:")) { + return true; + } + if (n >= MD_SLEN("www.") && md_eq_ci(s, MD_SLEN("www."), "www.")) { + return true; + } + if (n == 0 || !md_alpha((unsigned char)s[0])) { + return false; + } + size_t i = SKIP_ONE; + while (i < n && (md_alnum((unsigned char)s[i]) || s[i] == '+' || s[i] == '.' || s[i] == '-')) { + i++; + } + return i + MD_SLEN("://") <= n && memcmp(s + i, "://", MD_SLEN("://")) == 0; +} + +/* `^[A-Za-z_$][\w$]*$` */ +static bool md_ident(const char *s, size_t n) { + if (n == 0) { + return false; + } + unsigned char c0 = (unsigned char)s[0]; + if (!(md_alpha(c0) || c0 == '_' || c0 == '$' || c0 >= 0x80)) { + return false; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!md_word((unsigned char)s[i]) && s[i] != '$') { + return false; + } + } + return true; +} + +/* Every byte of s[0..n) is a word byte or one of `extra`. */ +static bool md_all_of(const char *s, size_t n, const char *extra) { + if (n == 0) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!md_word((unsigned char)s[i]) && !strchr(extra, s[i])) { + return false; + } + } + return true; +} + +/* ── Line numbers written with a path ────────────────────────────── */ + +/* The decimal number s[0..n): false when it is empty, not all digits or too + * long to be a line. */ +static bool md_number(const char *s, size_t n, uint32_t *out) { + if (n == 0 || n > MD_NUM_DIGITS) { + return false; + } + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + if (!md_digit((unsigned char)s[i])) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (uint32_t)(s[i] - '0'); + } + *out = v; + return v > 0; +} + +/* Digits ending at s[end): their start. */ +static size_t md_digits_back(const char *s, size_t end) { + size_t i = end; + while (i > 0 && md_digit((unsigned char)s[i - SKIP_ONE])) { + i--; + } + return i; +} + +bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last) { + /* L[C][-[L][C]] */ + if (!frag || frag[0] != 'L') { + return false; + } + const char *p = frag + SKIP_ONE; + const char *q = p; + while (md_digit((unsigned char)*q)) { + q++; + } + uint32_t a = 0; + if (!md_number(p, (size_t)(q - p), &a)) { + return false; + } + if (*q == 'C') { + q++; + while (md_digit((unsigned char)*q)) { + q++; + } + } + uint32_t b = a; + if (*q == '-') { + q++; + if (*q == 'L') { + q++; + } + p = q; + while (md_digit((unsigned char)*q)) { + q++; + } + if (!md_number(p, (size_t)(q - p), &b)) { + return false; + } + if (*q == 'C') { + q++; + while (md_digit((unsigned char)*q)) { + q++; + } + } + } + if (*q != '\0') { + return false; + } + *first = a; + *last = b < a ? a : b; + return true; +} + +/* A line range at the end of p[0..*n): `#L[-[L]]`, `:`, `:-` + * or `::`. On a match the range is cut off (*n shrinks). */ +static bool md_line_suffix(const char *p, size_t *n, uint32_t *first, uint32_t *last) { + size_t len = *n; + /* `#L` form: the last '#' */ + for (size_t i = len; i > 0; i--) { + if (p[i - SKIP_ONE] != '#') { + continue; + } + size_t h = i - SKIP_ONE; + if (h + SKIP_ONE < len && p[h + SKIP_ONE] == 'L') { + size_t a0 = h + PAIR_LEN; + size_t a1 = a0; + while (a1 < len && md_digit((unsigned char)p[a1])) { + a1++; + } + uint32_t a = 0; + if (md_number(p + a0, a1 - a0, &a)) { + uint32_t b = a; + size_t k = a1; + bool ok = k == len; + if (!ok && p[k] == '-') { + k++; + if (k < len && p[k] == 'L') { + k++; + } + ok = md_number(p + k, len - k, &b); + } + if (ok) { + *first = a; + *last = b < a ? a : b; + *n = h; + return true; + } + } + } + break; + } + /* `:` form: digits at the end */ + size_t d2 = md_digits_back(p, len); + uint32_t b = 0; + if (d2 == len || d2 == 0 || !md_number(p + d2, len - d2, &b)) { + return false; + } + char sep = p[d2 - SKIP_ONE]; + if (sep == '-' || sep == ':') { + size_t d1 = md_digits_back(p, d2 - SKIP_ONE); + uint32_t a = 0; + if (d1 > 0 && d1 < d2 - SKIP_ONE && p[d1 - SKIP_ONE] == ':' && + md_number(p + d1, d2 - SKIP_ONE - d1, &a)) { + *first = a; + *last = b < a ? a : b; + *n = d1 - SKIP_ONE; + return true; + } + } + if (sep == ':') { + *first = b; + *last = b; + *n = d2 - SKIP_ONE; + return true; + } + return false; +} + +/* ── Code-span classification ────────────────────────────────────── */ + +/* Drop one trailing balanced `<...>` or `[...]` (type arguments). */ +static size_t md_strip_generics(const char *s, size_t n) { + static const char pairs[][PAIR_LEN] = {{'<', '>'}, {'[', ']'}}; + for (size_t p = 0; p < sizeof(pairs) / sizeof(pairs[0]); p++) { + char o = pairs[p][0]; + char c = pairs[p][SKIP_ONE]; + if (n == 0 || s[n - SKIP_ONE] != c || !md_has(s, n, o)) { + continue; + } + int depth = 0; + for (size_t k = n; k > 0; k--) { + char ch = s[k - SKIP_ONE]; + if (ch == c) { + depth++; + } else if (ch == o) { + depth--; + if (depth == 0) { + if (k - SKIP_ONE > 0) { + return k - SKIP_ONE; + } + break; + } + } + } + return n; + } + return n; +} + +/* `name(args)` with a short argument list that holds no parenthesis: the + * callee's length, or 0. `wide` is the first form of the field test (args + * with a space or comma, up to 80 bytes, callee [\w$.:#\\]); otherwise the + * second (args up to 60 bytes, callee [\w$.:#\\<>\[\]!~&*-]). */ +static size_t md_callee(const char *s, size_t n, bool wide) { + if (n < PAIR_LEN || s[n - SKIP_ONE] != ')') { + return 0; + } + size_t open = n; + for (size_t i = n - SKIP_ONE; i > 0; i--) { + char c = s[i - SKIP_ONE]; + if (c == ')') { + return 0; + } + if (c == '(') { + open = i - SKIP_ONE; + break; + } + } + if (open == n || open == 0) { + return 0; + } + size_t args = n - open - PAIR_LEN; + if (args > (wide ? (size_t)MD_CALL_ARGS_WIDE : (size_t)MD_CALL_ARGS)) { + return 0; + } + if (wide && !md_has(s + open + SKIP_ONE, args, ' ') && + !md_has(s + open + SKIP_ONE, args, ',')) { + return 0; + } + return md_all_of(s, open, wide ? "$.:#\\" : "$.:#\\<>[]!~&*-") ? open : 0; +} + +static void md_shape(CBMDocLinkMdPath *out, CBMDocLinkMdShape shape, const char *path, + const char *member, uint32_t first, uint32_t last) { + out->shape = shape; + out->path = path; + out->member = member; + out->first_line = first; + out->last_line = last; + out->instance = false; + out->colon = false; +} + +/* The path branch: `s` has a slash (or a Windows path with an extension). */ +static bool md_classify_path(char *s, size_t n, CBMDocLinkMdPath *out) { + for (size_t i = 0; i < n; i++) { + if (s[i] == '\\') { + s[i] = '/'; + } + } + if (n >= PAIR_LEN && s[0] == '/' && s[SKIP_ONE] == '/') { + return false; + } + uint32_t first = 0; + uint32_t last = 0; + (void)md_line_suffix(s, &n, &first, &last); + s[n] = '\0'; + const char *member = NULL; + char *sep = strstr(s, "::"); + if (sep) { + *sep = '\0'; + member = sep + PAIR_LEN; + n = (size_t)(sep - s); + if (!member[0]) { + member = NULL; + } + } + if (n == 0) { + return false; + } + bool named = false; + for (size_t i = 0; i < n && !named; i++) { + named = md_word((unsigned char)s[i]); + } + if (!named) { + return false; /* `../`, `./`, `/`: a traversal or a root, naming no directory */ + } + /* an import path (`github.com/x/y`, `k8s.io/client-go`): a host-like first + * segment -- dot-separated word groups -- and a last segment (as written, + * a trailing slash leaves it empty) without a file extension */ + size_t host = 0; + while (host < n && s[host] != '/') { + host++; + } + bool hostlike = host > 0 && host < n && md_has(s, host, '.') && md_all_of(s, host, ".-") && + s[0] != '.' && s[host - SKIP_ONE] != '.'; + for (size_t i = 0; hostlike && i + SKIP_ONE < host; i++) { + hostlike = !(s[i] == '.' && s[i + SKIP_ONE] == '.'); + } + size_t tail = n; + while (tail > 0 && s[tail - SKIP_ONE] != '/') { + tail--; + } + if (hostlike && !md_ext_ok(s + tail, n - tail)) { + return false; + } + /* the base name, after any trailing slashes */ + size_t end = n; + while (end > 0 && s[end - SKIP_ONE] == '/') { + end--; + } + size_t base = end; + while (base > 0 && s[base - SKIP_ONE] != '/') { + base--; + } + const char *last_seg = s + base; + size_t last_len = end - base; + if (!md_all_of(s, n, "./@+~-")) { + return false; + } + if (s[n - SKIP_ONE] == '/') { + md_shape(out, CBM_DOCLINK_MD_DIR, s, NULL, 0, 0); + return true; + } + const char *dot = last_len > 0 ? memchr(last_seg, '.', last_len) : NULL; + if (dot && last_seg[0] != '.') { + const char *ext = strrchr(last_seg, '.') + SKIP_ONE; + size_t ext_len = (size_t)(last_seg + last_len - ext); + if (ext_len > 0 && ext[0] >= 'A' && ext[0] <= 'Z' && md_ident(ext, ext_len) && + !md_in_list(ext, ext_len, MD_FILE_EXT, true)) { + return false; /* `pkg/Type.Member`: a name, not a file (a later rung) */ + } + if (md_ext_ok(last_seg, last_len)) { + md_shape(out, CBM_DOCLINK_MD_FILE, s, member, first, last); + return true; + } + } + if (last_len > SKIP_ONE && last_seg[0] == '.') { + md_shape(out, CBM_DOCLINK_MD_FILE, s, NULL, first, last); + return true; + } + md_shape(out, CBM_DOCLINK_MD_DIR, s, NULL, 0, 0); + return true; +} + +static const char *const MD_RUST_ROOTS[] = {"crate", "self", "super", "Self", NULL}; +static const char *const MD_INSTANCE_ROOTS[] = {"self", "this", "cls", "$this", NULL}; + +/* Split s[0..n) at every `sep`, check that each piece is an identifier (type + * arguments dropped), and join the pieces with '.' in place. A first piece in + * `skip_first` (Rust's `crate`, an instance's `self`) is dropped and *skipped + * says so. False when a piece is no identifier or fewer than two remain. */ +static bool md_join(char *s, size_t n, const char *sep, const char *const *skip_first, + bool *skipped, CBMDocLinkMdPath *out) { + size_t sl = strlen(sep); + size_t pieces = 0; + size_t w = 0; + size_t i = 0; + *skipped = false; + for (;;) { + size_t e = i; + while (e + sl <= n && memcmp(s + e, sep, sl) != 0) { + e++; + } + bool last_piece = e + sl > n; + if (last_piece) { + e = n; + } + size_t len = md_strip_generics(s + i, e - i); + if (pieces == 0 && !*skipped && skip_first && !last_piece && + md_in_list(s + i, len, skip_first, false)) { + *skipped = true; + } else { + if (!md_ident(s + i, len)) { + return false; + } + if (w > 0) { + s[w++] = '.'; + } + memmove(s + w, s + i, len); + w += len; + pieces++; + } + if (last_piece) { + break; + } + i = e + sl; + } + s[w] = '\0'; + if (pieces < PAIR_LEN) { + return false; + } + md_shape(out, CBM_DOCLINK_MD_QUALIFIED, s, NULL, 0, 0); + return true; +} + +/* `main.rs::run`: a file name with a member (an identifier). */ +static bool md_filename_member(char *s, char *sep, uint32_t first, uint32_t last, + CBMDocLinkMdPath *out) { + const char *head = s; + size_t head_len = (size_t)(sep - s); + const char *tail = sep + PAIR_LEN; + const char *tail_last = tail; + for (const char *t = strstr(tail, "::"); t; t = strstr(t + PAIR_LEN, "::")) { + tail_last = t + PAIR_LEN; + } + const char *hd = head_len ? memchr(head, '.', head_len) : NULL; + if (!hd || !md_ident(tail_last, strlen(tail_last)) || !md_all_of(head, head_len, "./\\-")) { + return false; + } + const char *ext = head; + for (size_t i = head_len; i > 0; i--) { + if (head[i - SKIP_ONE] == '.') { + ext = head + i; + break; + } + } + size_t ext_len = head_len - (size_t)(ext - head); + for (size_t i = 0; i < ext_len; i++) { + if (ext[i] < 'a' || ext[i] > 'z') { + return false; + } + } + if (ext_len == 0) { + return false; + } + *sep = '\0'; + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, tail[0] ? tail : NULL, first, last); + return true; +} + +/* One `mark` between a head `[A-Za-z_][\w.]*` and a tail `[A-Za-z_]\w*` + * (`dotted_tail`: `[A-Za-z_][\w.]*`): the shapes of `Type#member` and of + * `module:attr`. */ +static bool md_head_mark_tail(const char *s, size_t n, char mark, bool dotted_tail) { + const char *m = memchr(s, mark, n); + if (!m || m == s || memchr(m + SKIP_ONE, mark, n - (size_t)(m - s) - SKIP_ONE)) { + return false; + } + unsigned char c0 = (unsigned char)s[0]; + size_t head = (size_t)(m - s); + size_t tail = n - head - SKIP_ONE; + if (tail == 0) { + return false; + } + unsigned char t0 = (unsigned char)m[SKIP_ONE]; + return (md_alpha(c0) || c0 == '_') && md_all_of(s, head, ".") && (md_alpha(t0) || t0 == '_') && + md_all_of(m + SKIP_ONE, tail, dotted_tail ? "." : ""); +} + +/* No slash: a file name, a special file, `file.ext::member`, or a qualified + * name. A bare name is neither: at most a suggestion, never a link. */ +static bool md_classify_name(char *s, size_t n, CBMDocLinkMdPath *out) { + if (md_in_list(s, n, MD_SPECIAL_FILES, false)) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, 0, 0); + return true; + } + uint32_t first = 0; + uint32_t last = 0; + size_t cut = n; + if (md_line_suffix(s, &cut, &first, &last)) { + const char *dot = NULL; + for (size_t i = cut; i > 0; i--) { + if (s[i - SKIP_ONE] == '.') { + dot = s + i - SKIP_ONE; + break; + } + } + size_t ext_len = dot ? cut - (size_t)(dot - s) - SKIP_ONE : 0; + if (dot && md_in_list(dot + SKIP_ONE, ext_len, MD_FILE_EXT, true)) { + n = cut; + s[n] = '\0'; + } else { + first = 0; + last = 0; + } + } + bool skipped = false; + if (strstr(s, "::")) { + /* identifiers only: a qualified name; otherwise perhaps `file.ext::member` */ + char copy[MD_SPAN_MAX + SKIP_ONE]; + memcpy(copy, s, n + SKIP_ONE); + if (md_join(s, n, "::", MD_RUST_ROOTS, &skipped, out)) { + return true; + } + memcpy(s, copy, n + SKIP_ONE); + return md_filename_member(s, strstr(s, "::"), first, last, out); + } + if (md_has(s, n, '#')) { + if (!md_head_mark_tail(s, n, '#', false)) { + return false; + } + s[strcspn(s, "#")] = '.'; + return md_join(s, n, ".", NULL, &skipped, out); + } + if (md_has(s, n, '\\')) { + while (n > 0 && s[n - SKIP_ONE] == '\\') { + n--; + } + size_t lead = 0; + while (lead < n && s[lead] == '\\') { + lead++; + } + return md_join(s + lead, n - lead, "\\", NULL, &skipped, out); + } + if (md_has(s, n, ':')) { + if (!md_head_mark_tail(s, n, ':', true)) { + return false; + } + s[strcspn(s, ":")] = '.'; + if (!md_join(s, n, ".", NULL, &skipped, out)) { + return false; + } + out->colon = true; + return true; + } + if (!md_has(s, n, '.')) { + return false; + } + if (s[0] == '.') { + const char *rest = s + SKIP_ONE; + size_t rest_len = n - SKIP_ONE; + if (md_ident(rest, rest_len) && !md_in_list(rest, rest_len, MD_FILE_EXT, true)) { + return false; /* `.member` */ + } + if (md_all_of(rest, rest_len, ".-")) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, 0, 0); + return true; + } + return false; + } + const char *dot = strrchr(s, '.'); + const char *ext = dot + SKIP_ONE; + size_t ext_len = n - (size_t)(ext - s); + if (md_in_list(ext, ext_len, MD_FILE_EXT, true) && md_all_of(s, n, ".+-")) { + size_t stem_len = (size_t)(dot - s); + bool one_dot = memchr(s, '.', stem_len) == NULL; + if (one_dot && s[0] >= 'A' && s[0] <= 'Z' && md_eq_ci(ext, ext_len, "js")) { + return false; /* `Node.js`: a product */ + } + bool mod_sum = md_eq_ci(ext, ext_len, "mod") || md_eq_ci(ext, ext_len, "sum"); + if (!mod_sum || (stem_len == PAIR_LEN && memcmp(s, "go", PAIR_LEN) == 0)) { + md_shape(out, CBM_DOCLINK_MD_FILENAME, s, NULL, first, last); + return true; + } + } + if (!md_join(s, n, ".", MD_INSTANCE_ROOTS, &skipped, out)) { + return false; + } + out->instance = skipped; /* `self.x`: the qualifier is an instance, not a type */ + return true; +} + +bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_t cap, + CBMDocLinkMdPath *out) { + md_shape(out, CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0); + if (!text || !buf) { + return false; + } + while (len > 0 && md_blank((unsigned char)text[0])) { + text++; + len--; + } + while (len > 0 && md_blank((unsigned char)text[len - SKIP_ONE])) { + len--; + } + if (len == 0 || len > MD_SPAN_MAX || len + SKIP_ONE > cap || md_has(text, len, '\n')) { + return false; + } + memcpy(buf, text, len); + buf[len] = '\0'; + char *s = buf; + size_t n = len; + if (md_in_list(s, n, MD_DOMAINS, true) || md_is_url(s, n)) { + return false; + } + bool only_slashes = true; + for (size_t i = 0; i < n; i++) { + only_slashes = only_slashes && s[i] == '/'; + } + if (only_slashes) { + return false; + } + if (n > SKIP_ONE && strchr("$%>#", s[0]) && s[SKIP_ONE] == ' ') { + return false; /* a shell prompt */ + } + if (n > SKIP_ONE && s[0] == '-' && + (md_alpha((unsigned char)s[SKIP_ONE]) || s[SKIP_ONE] == '-')) { + return false; /* an option */ + } + if (s[0] == '$' || + (n >= PAIR_LEN && (s[0] == '"' || s[0] == '\'') && s[n - SKIP_ONE] == s[0]) || + (s[0] == '<' && s[n - SKIP_ONE] == '>')) { + return false; /* a variable, a string, markup */ + } + size_t eq = 0; + while (eq < n && (md_alnum((unsigned char)s[eq]) || s[eq] == '_')) { + eq++; + } + if (eq > 0 && eq < n && s[eq] == '=' && !md_digit((unsigned char)s[0]) && !md_has(s, n, ' ')) { + return false; /* NAME=value */ + } + size_t callee = md_callee(s, n, true); + if (callee) { + n = callee; + s[n] = '\0'; + } + if (md_has(s, n, ' ') || md_has(s, n, '\t')) { + return false; /* a declaration, a command line, a snippet */ + } + if (s[0] == '@') { + size_t slash = 0; + while (slash < n && s[slash] != '/') { + slash++; + } + if (slash < n && slash > SKIP_ONE) { + return false; /* `@scope/package`: an import path (a later rung) */ + } + s++; + n--; + if (n == 0) { + return false; + } + } + callee = md_callee(s, n, false); + if (callee) { + n = callee; + s[n] = '\0'; + } + while (n > 0 && strchr(";,:", s[n - SKIP_ONE])) { + n--; + } + if (n > SKIP_ONE && s[n - SKIP_ONE] == '!') { + n--; + } + while (n > 0 && s[n - SKIP_ONE] == '?') { + n--; + } + n = md_strip_generics(s, n); + while (n > 0 && strchr("&*~", s[0])) { + s++; + n--; + } + if (n >= PAIR_LEN && s[0] == ':' && s[SKIP_ONE] == ':') { + s += PAIR_LEN; + n -= PAIR_LEN; + } + if (n == 0) { + return false; + } + s[n] = '\0'; + size_t first_slash = 0; + while (first_slash < n && s[first_slash] != '/') { + first_slash++; + } + bool slash_path = first_slash < n && !strstr(s, "::"); + if (first_slash < n && !slash_path) { + /* `a::b/c`: a "::" after the first slash still makes a path */ + char *dc = strstr(s, "::"); + slash_path = dc && (size_t)(dc - s) > first_slash; + } + bool windows_path = false; + if (!slash_path && md_has(s, n, '\\')) { + const char *bs = strrchr(s, '\\'); + size_t tail = n - (size_t)(bs - s) - SKIP_ONE; + windows_path = tail > 0 && md_all_of(bs + SKIP_ONE, tail, ".-") && md_ext_ok(bs, tail + 1); + } + if (slash_path || windows_path) { + return md_classify_path(s, n, out); + } + return md_classify_name(s, n, out); +} + +/* ── Position lists ──────────────────────────────────────────────── */ + +typedef struct { + int32_t *v; + int n; + int cap; +} md_list_t; + +static bool md_push_int(md_list_t *l, int32_t x) { + if (l->n == l->cap) { + if (l->cap > INT32_MAX / PAIR_LEN) { + return false; + } + int ncap = l->cap ? l->cap * PAIR_LEN : MD_LIST_INIT; + int32_t *grown = + (int32_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*grown)); + if (!grown) { + return false; + } + l->v = grown; + l->cap = ncap; + } + l->v[l->n++] = x; + return true; +} + +static bool md_push_pair(md_list_t *l, int32_t a, int32_t b) { + return md_push_int(l, a) && md_push_int(l, b); +} + +static void md_list_free(md_list_t *l) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + l->v = NULL; + l->n = 0; + l->cap = 0; +} + +/* Index of the first value >= x in an ascending list. */ +static int md_lower_bound(const md_list_t *l, int32_t x) { + int lo = 0; + int hi = l->n; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (l->v[mid] < x) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +/* The first value >= x, or -1. */ +static int32_t md_next_at(const md_list_t *l, int32_t x) { + int i = md_lower_bound(l, x); + return i < l->n ? l->v[i] : CBM_NOT_FOUND; +} + +/* ── The scan ────────────────────────────────────────────────────── */ + +typedef struct { + uint32_t line; + const char *qn; + /* Another heading of the file has the same qualified name: the graph keeps + * one node for both, which owns only one of the headings, so this + * section's references belong to the file. */ + bool shared_qn; +} md_section_t; + +typedef struct { + bool in_front; + bool in_fence; + bool in_comment; + char fence_char; + int fence_len; +} md_state_t; + +typedef struct { + CBMExtractCtx *ctx; + md_section_t *secs; + int nsecs; + char *buf; /* the current line, masked in place */ + int len; + uint32_t line; + char *span_buf; /* classification scratch, MD_SPAN_MAX + 1 bytes */ + md_list_t runs; /* backtick runs: start, length */ + md_list_t by_len; /* the runs as (length, start), sorted */ + md_list_t ranges; /* ranges to mask: start, end */ + md_list_t stack; /* open '[' while matching */ + md_list_t cands; /* link candidates: '[', ']' */ + md_list_t opens; /* '(' */ + md_list_t closes; /* ')' */ + md_list_t bal; /* each ')' as (depth before it, position), sorted */ + md_list_t gts; /* '>' */ + md_list_t blanks; /* runs of spaces and tabs: start, end */ + bool failed; +} md_scan_t; + +static int md_section_cmp(const void *a, const void *b) { + const md_section_t *x = (const md_section_t *)a; + const md_section_t *y = (const md_section_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + return strcmp(x->qn, y->qn); +} + +static int md_section_ptr_qn_cmp(const void *a, const void *b) { + const md_section_t *x = *(const md_section_t *const *)a; + const md_section_t *y = *(const md_section_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static int md_pair_cmp(const void *a, const void *b) { + const int32_t *x = (const int32_t *)a; + const int32_t *y = (const int32_t *)b; + if (x[0] != y[0]) { + return x[0] < y[0] ? -1 : 1; + } + return (x[SKIP_ONE] > y[SKIP_ONE]) - (x[SKIP_ONE] < y[SKIP_ONE]); +} + +static void md_fail(md_scan_t *s) { + s->failed = true; + s->ctx->result->doc_links.failed = true; +} + +/* Text that may be a reference: well-formed UTF-8 (no overlong form, no + * surrogate, nothing above U+10FFFF) without control bytes. */ +static bool md_clean_text(const char *t, int n) { + int i = 0; + while (i < n) { + unsigned char c = (unsigned char)t[i]; + if (c < UTF8_CTRL_END || c == UTF8_DEL) { + return false; + } + int need; + unsigned char lo = UTF8_CONT_MIN; + unsigned char hi = UTF8_CONT_MAX; + if (c < UTF8_ASCII_END) { + need = 0; + } else if (c >= UTF8_LEAD2_MIN && c <= UTF8_LEAD2_MAX) { + need = SKIP_ONE; + } else if (c >= UTF8_LEAD3_MIN && c <= UTF8_LEAD3_MAX) { + need = PAIR_LEN; + lo = c == UTF8_LEAD3_MIN ? UTF8_E0_SECOND_MIN : lo; /* no overlong form */ + hi = c == UTF8_SURROGATE_LEAD ? UTF8_ED_SECOND_MAX : hi; /* no surrogate */ + } else if (c >= UTF8_LEAD4_MIN && c <= UTF8_LEAD4_MAX) { + need = PAIR_LEN + SKIP_ONE; + lo = c == UTF8_LEAD4_MIN ? UTF8_F0_SECOND_MIN : lo; + hi = c == UTF8_LEAD4_MAX ? UTF8_F4_SECOND_MAX : hi; + } else { + return false; + } + if (i + need >= n + (need == 0)) { + return false; + } + for (int k = SKIP_ONE; k <= need; k++) { + unsigned char cc = (unsigned char)t[i + k]; + unsigned char kl = k == SKIP_ONE ? lo : UTF8_CONT_MIN; + unsigned char kh = k == SKIP_ONE ? hi : UTF8_CONT_MAX; + if (cc < kl || cc > kh) { + return false; + } + } + i += need + SKIP_ONE; + } + return true; +} + +static void md_emit(md_scan_t *s, int syntax, const char *text, int n) { + while (n > 0 && md_blank((unsigned char)text[0])) { + text++; + n--; + } + while (n > 0 && md_blank((unsigned char)text[n - SKIP_ONE])) { + n--; + } + if (n <= 0 || memchr(text, MD_MASK, (size_t)n) || !md_clean_text(text, n)) { + return; + } + int lo = 0; + int hi = s->nsecs; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (s->secs[mid].line <= s->line) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + const md_section_t *sec = lo > 0 ? &s->secs[lo - SKIP_ONE] : NULL; + if (sec && sec->shared_qn) { + sec = NULL; /* the file, not the node another heading owns */ + } + char *raw = cbm_arena_strndup(s->ctx->arena, text, (size_t)n); + if (!raw) { + md_fail(s); + return; + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (s->ctx->module_qn ? s->ctx->module_qn : ""), + .raw = raw, + .line = s->line, + .def_line = sec ? sec->line : SKIP_ONE, + .syntax = (uint16_t)syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&s->ctx->result->doc_links, s->ctx->arena, link); +} + +/* A directory written as one name (`doc/`, `/docs`): no parent says whose it is. */ +static bool md_one_dir(const CBMDocLinkMdPath *p) { + if (p->shape != CBM_DOCLINK_MD_DIR || !p->path) { + return false; + } + const char *s = p->path + (p->path[0] == '/'); + const char *slash = strchr(s, '/'); + return !slash || !slash[SKIP_ONE]; +} + +/* A code span's text: a reference when it names a path or a qualified name. + * A name alone -- a file (`config.yaml`) or a directory (`doc/`) -- is its own + * family: in the held-out audits it was the reader's own file or another + * project's directory as often as this repository's. */ +static void md_emit_span(md_scan_t *s, const char *text, int n) { + CBMDocLinkMdPath p; + if (n > 0 && + cbm_doclink_md_classify_span(text, (size_t)n, s->span_buf, MD_SPAN_MAX + SKIP_ONE, &p)) { + bool bare = p.shape == CBM_DOCLINK_MD_FILENAME || md_one_dir(&p); + md_emit(s, + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_MD_CODE_NAME + : bare ? CBM_DOCLINK_MD_BARE_PATH + : CBM_DOCLINK_MD_CODE_PATH, + text, n); + } +} + +static void md_mask(md_scan_t *s, int a, int b) { + if (a < 0) { + a = 0; + } + if (b > s->len) { + b = s->len; + } + if (a < b) { + memset(s->buf + a, MD_MASK, (size_t)(b - a)); + } +} + +static int md_find(const char *buf, int len, int from, const char *needle, bool ci) { + int k = (int)strlen(needle); + for (int i = from; i + k <= len; i++) { + int j = 0; + while (j < k && (ci ? md_lower((unsigned char)buf[i + j]) : (unsigned char)buf[i + j]) == + (unsigned char)needle[j]) { + j++; + } + if (j == k) { + return i; + } + } + return CBM_NOT_FOUND; +} + +/* HTML spans (tables in READMEs write code that way): their text, + * without inner tags and with the common entities decoded. */ +static void md_html_code(md_scan_t *s) { + int i = 0; + while (i < s->len) { + int open = md_find(s->buf, s->len, i, "len && s->buf[after] == '>') { + body = after + SKIP_ONE; + } else if (after < s->len && md_blank((unsigned char)s->buf[after])) { + const char *gt = memchr(s->buf + after, '>', (size_t)(s->len - after)); + if (!gt) { + return; + } + body = (int)(gt - s->buf) + SKIP_ONE; + } else { + i = open + SKIP_ONE; + continue; + } + int close = md_find(s->buf, s->len, body, "", true); + if (close < 0) { + return; + } + /* text without tags, entities decoded; never longer than the body */ + char *text = s->span_buf + MD_SPAN_MAX + SKIP_ONE; /* second half of the scratch */ + int w = 0; + int k = body; + while (k < close && w < MD_SPAN_MAX + SKIP_ONE) { + char c = s->buf[k]; + if (c == '<') { + int lim = k + MD_TAG_MAX + PAIR_LEN; /* `<`, up to 200 bytes, `>` */ + const char *gt = memchr(s->buf + k + SKIP_ONE, '>', + (size_t)((lim < close ? lim : close) - k - SKIP_ONE)); + if (gt) { + k = (int)(gt - s->buf) + SKIP_ONE; + continue; + } + } else if (c == '&') { + static const struct { + const char *ent; + char ch; + } ents[] = {{"<", '<'}, {">", '>'}, {"&", '&'}, {""", '"'}, + {"'", '\''}, {"'", '\''}, {"'", '\''}, {" ", ' '}}; + bool hit = false; + for (size_t e = 0; e < sizeof(ents) / sizeof(ents[0]); e++) { + int el = (int)strlen(ents[e].ent); + if (k + el <= close && memcmp(s->buf + k, ents[e].ent, (size_t)el) == 0) { + text[w++] = ents[e].ch; + k += el; + hit = true; + break; + } + } + if (hit) { + continue; + } + } + text[w++] = c; + k++; + } + if (k >= close) { + md_emit_span(s, text, w); /* a text longer than a span's limit is no name */ + } + md_mask(s, open, close + (int)MD_SLEN("")); + i = close + (int)MD_SLEN(""); + } +} + +/* Code spans: a run of N backticks up to the next run of exactly N. An + * escaped first backtick (an odd run of backslashes before it) does not open + * a span. Closers come from the runs sorted by (length, start). */ +static void md_code_spans(md_scan_t *s) { + s->runs.n = 0; + s->by_len.n = 0; + s->ranges.n = 0; + for (int i = 0; i < s->len;) { + if (s->buf[i] != '`') { + i++; + continue; + } + int j = i; + while (j < s->len && s->buf[j] == '`') { + j++; + } + if (!md_push_pair(&s->runs, i, j - i) || !md_push_pair(&s->by_len, j - i, i)) { + md_fail(s); + return; + } + i = j; + } + int nruns = s->runs.n / PAIR_LEN; + if (nruns < PAIR_LEN) { + return; + } + qsort(s->by_len.v, (size_t)nruns, sizeof(int32_t) * PAIR_LEN, md_pair_cmp); + int pos = 0; + for (int r = 0; r < nruns; r++) { + int start = s->runs.v[r * PAIR_LEN]; + int len = s->runs.v[(r * PAIR_LEN) + SKIP_ONE]; + if (start < pos) { + continue; + } + int bs = 0; + while (start - bs > 0 && s->buf[start - bs - SKIP_ONE] == '\\') { + bs++; + } + if (bs % PAIR_LEN) { + start++; + len--; + if (len == 0) { + continue; + } + } + /* the first run of exactly `len` starting at or after start + len */ + int lo = 0; + int hi = nruns; + int32_t key[PAIR_LEN] = {len, start + len}; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (md_pair_cmp(&s->by_len.v[mid * PAIR_LEN], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < nruns && s->by_len.v[lo * PAIR_LEN] == len) { + int close = s->by_len.v[(lo * PAIR_LEN) + SKIP_ONE]; + int a = start + len; + int b = close; + if (b - a >= PAIR_LEN && s->buf[a] == ' ' && s->buf[b - SKIP_ONE] == ' ') { + bool all_space = true; + for (int k = a; k < b && all_space; k++) { + all_space = s->buf[k] == ' '; + } + if (!all_space) { + a++; + b--; + } + } + md_emit_span(s, s->buf + a, b - a); + if (!md_push_pair(&s->ranges, start, close + len)) { + md_fail(s); + return; + } + pos = close + len; + } else { + pos = start + len; + } + } + for (int i = 0; i < s->ranges.n; i += PAIR_LEN) { + md_mask(s, s->ranges.v[i], s->ranges.v[i + SKIP_ONE]); + } +} + +/* `blanks` holds the space/tab runs as start, end, start, end, ... -- one + * strictly ascending list. The first value above i: an END when i lies in a + * run (its index is odd), a START (or nothing) when it does not. */ +static int md_blank_after(const md_scan_t *s, int i) { + return md_lower_bound(&s->blanks, i + SKIP_ONE); +} + +/* The first space or tab at or after i (the line's end when there is none). */ +static int md_next_blank(const md_scan_t *s, int i) { + int k = md_blank_after(s, i); + if (k % PAIR_LEN == SKIP_ONE) { + return i; + } + return k < s->blanks.n ? s->blanks.v[k] : s->len; +} + +/* The first byte at or after i that is no space or tab. */ +static int md_skip_blank(const md_scan_t *s, int i) { + int k = md_blank_after(s, i); + return k % PAIR_LEN == SKIP_ONE ? s->blanks.v[k] : i; +} + +/* Where a link destination that starts at t ends: at the first space or tab, + * or at the first ')' at the destination's own parenthesis depth 0. */ +static int md_dest_end(const md_scan_t *s, int t) { + int depth = md_lower_bound(&s->opens, t) - md_lower_bound(&s->closes, t); + int end = md_next_blank(s, t); + /* the first ')' at or after t with the depth t had */ + int nb = s->bal.n / PAIR_LEN; + int lo = 0; + int hi = nb; + int32_t key[PAIR_LEN] = {depth, t}; + while (lo < hi) { + int mid = lo + ((hi - lo) / PAIR_LEN); + if (md_pair_cmp(&s->bal.v[mid * PAIR_LEN], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + if (lo < nb && s->bal.v[lo * PAIR_LEN] == depth) { + int p = s->bal.v[(lo * PAIR_LEN) + SKIP_ONE]; + if (p < end) { + end = p; + } + } + return end; +} + +static bool md_line_lists(md_scan_t *s) { + s->stack.n = 0; + s->cands.n = 0; + s->opens.n = 0; + s->closes.n = 0; + s->bal.n = 0; + s->gts.n = 0; + s->blanks.n = 0; + int depth = 0; + for (int i = 0; i < s->len; i++) { + char c = s->buf[i]; + bool ok = true; + switch (c) { + case '[': + ok = md_push_int(&s->stack, i); + break; + case ']': + if (s->stack.n > 0) { + int o = s->stack.v[--s->stack.n]; + if (i + SKIP_ONE < s->len && s->buf[i + SKIP_ONE] == '(') { + ok = md_push_pair(&s->cands, o, i); + } + } + break; + case '(': + ok = md_push_int(&s->opens, i); + depth++; + break; + case ')': + ok = md_push_int(&s->closes, i) && md_push_pair(&s->bal, depth, i); + depth--; + break; + case '>': + ok = md_push_int(&s->gts, i); + break; + case ' ': + case '\t': { + int j = i; + while (j < s->len && md_sp_tab((unsigned char)s->buf[j])) { + j++; + } + ok = md_push_pair(&s->blanks, i, j); + i = j - SKIP_ONE; + break; + } + default: + break; + } + if (!ok) { + return false; + } + } + if (s->bal.n > PAIR_LEN) { + qsort(s->bal.v, (size_t)(s->bal.n / PAIR_LEN), sizeof(int32_t) * PAIR_LEN, md_pair_cmp); + } + return true; +} + +static void md_link_target(md_scan_t *s, int a, int b, bool image) { + while (a < b && md_blank((unsigned char)s->buf[a])) { + a++; + } + while (b > a && md_blank((unsigned char)s->buf[b - SKIP_ONE])) { + b--; + } + if (a >= b || image || s->buf[a] == '#' || memchr(s->buf + a, MD_MASK, (size_t)(b - a)) || + md_is_url(s->buf + a, (size_t)(b - a))) { + return; /* empty, an image, an in-page anchor, masked text, a URL */ + } + md_emit(s, CBM_DOCLINK_MD_LINK, s->buf + a, b - a); +} + +/* Inline links and images `[text](dest "title")`. */ +static void md_links(md_scan_t *s) { + if (!md_line_lists(s)) { + md_fail(s); + return; + } + s->ranges.n = 0; + int pos = 0; + int floor = 0; + for (int c = 0; c < s->cands.n; c += PAIR_LEN) { + int o = s->cands.v[c]; + int k = s->cands.v[c + SKIP_ONE]; + if (k < pos || o < floor) { + continue; + } + int t = k + PAIR_LEN; + while (t < s->len && s->buf[t] == ' ') { + t++; + } + int ta; + int tb; + int after; + if (t < s->len && s->buf[t] == '<') { + int e = md_next_at(&s->gts, t); + if (e < 0) { + pos = k + PAIR_LEN; + continue; + } + ta = t + SKIP_ONE; + tb = e; + after = e + SKIP_ONE; + } else { + int e = md_dest_end(s, t); + ta = t; + tb = e; + after = e; + } + int close = md_next_at(&s->closes, after); + if (close < 0) { + pos = k + PAIR_LEN; + continue; + } + int q = md_skip_blank(s, after); + if (q < close && !strchr("\"'(", s->buf[q])) { + pos = k + PAIR_LEN; + continue; + } + bool image = o > 0 && s->buf[o - SKIP_ONE] == '!'; + md_link_target(s, ta, tb, image); + if (!md_push_pair(&s->ranges, o - (image ? SKIP_ONE : 0), close + SKIP_ONE)) { + md_fail(s); + return; + } + pos = close + SKIP_ONE; + floor = pos; + } + for (int i = 0; i < s->ranges.n; i += PAIR_LEN) { + md_mask(s, s->ranges.v[i], s->ranges.v[i + SKIP_ONE]); + } +} + +/* A link reference definition `[label]: dest` (the whole line). */ +static void md_refdef(md_scan_t *s) { + int i = 0; + while (i < s->len && i < MD_INDENT_MAX && md_blank((unsigned char)s->buf[i])) { + i++; + } + if (i >= s->len || s->buf[i] != '[') { + return; + } + const char *rb = memchr(s->buf + i + SKIP_ONE, ']', (size_t)(s->len - i - SKIP_ONE)); + if (!rb) { + return; + } + int j = (int)(rb - s->buf); + if (j == i + SKIP_ONE || j + SKIP_ONE >= s->len || s->buf[j + SKIP_ONE] != ':') { + return; + } + int k = j + PAIR_LEN; + while (k < s->len && md_blank((unsigned char)s->buf[k])) { + k++; + } + if (k < s->len && s->buf[k] == '<') { + k++; + } + int e = k; + while (e < s->len && !md_blank((unsigned char)s->buf[e]) && s->buf[e] != '>') { + e++; + } + if (e > k) { + md_link_target(s, k, e, false); + } + md_mask(s, 0, s->len); +} + +static bool md_url_stop(unsigned char c) { + return md_blank(c) || c == '<' || c == '>' || c == '"' || c == '\'' || c == '`' || + c == MD_MASK || c == ')' || c == ']' || c == '|'; +} + +static bool md_path_stop(unsigned char c) { + return md_blank(c) || strchr("()[]{}<>\"',;:`*|", (int)c) != NULL || c == MD_MASK; +} + +/* Bare URLs are masked (they are no repository paths); then every token with + * a slash and a file extension is a bare path. */ +static void md_urls_and_paths(md_scan_t *s) { + static const char *const schemes[MD_URL_SCHEMES] = {"http://", "https://", "ftp://"}; + for (int i = 0; i < s->len; i++) { + char c = s->buf[i]; + if (c != 'h' && c != 'f') { + continue; + } + for (int k = 0; k < MD_URL_SCHEMES; k++) { + int sl = (int)strlen(schemes[k]); + if (i + sl < s->len && memcmp(s->buf + i, schemes[k], (size_t)sl) == 0 && + !md_url_stop((unsigned char)s->buf[i + sl])) { + int j = i + sl; + while (j < s->len && !md_url_stop((unsigned char)s->buf[j])) { + j++; + } + int e = j; + while (e > i && strchr(".,;:!?", s->buf[e - SKIP_ONE])) { + e--; + } + md_mask(s, i, e); + i = j - SKIP_ONE; + break; + } + } + } + for (int i = 0; i < s->len;) { + if (md_path_stop((unsigned char)s->buf[i])) { + i++; + continue; + } + int j = i; + while (j < s->len && !md_path_stop((unsigned char)s->buf[j])) { + j++; + } + int e = j; + while (e > i && s->buf[e - SKIP_ONE] == '.') { + e--; + } + if (e == i) { + i = j; /* only dots */ + continue; + } + const char *tok = s->buf + i; + size_t n = (size_t)(e - i); + bool path = md_has(tok, n, '/') && !(n >= PAIR_LEN && tok[0] == '/' && tok[1] == '/'); + for (size_t k = 0; path && k < n; k++) { + unsigned char c = (unsigned char)tok[k]; + path = md_alnum(c) || strchr("_-+@~./", (int)c) != NULL; + } + if (path) { + size_t end = n; + while (end > 0 && tok[end - SKIP_ONE] == '/') { + end--; + } + size_t base = end; + while (base > 0 && tok[base - SKIP_ONE] != '/') { + base--; + } + const char *last = tok + base; + size_t last_len = end - base; + size_t dots = 0; + for (size_t k = 0; k < last_len; k++) { + dots += last[k] == '.'; + } + bool tiny_dotfile = last_len > 0 && last[0] == '.' && dots == SKIP_ONE && + last_len < (size_t)MD_TINY_DOTFILE; + if (md_ext_ok(last, last_len) && !tiny_dotfile) { + md_emit(s, CBM_DOCLINK_MD_PATH, tok, (int)n); + } + } + i = j; + } +} + +static void md_inline(md_scan_t *s) { + md_html_code(s); + if (!s->failed) { + md_code_spans(s); + } + if (!s->failed) { + md_links(s); + } + if (!s->failed) { + md_refdef(s); + md_urls_and_paths(s); + } +} + +/* Leading blanks skipped: the line's content. */ +static const char *md_lstrip(const char *p, int len, int *rest) { + int i = 0; + while (i < len && md_blank((unsigned char)p[i])) { + i++; + } + *rest = len - i; + return p + i; +} + +static bool md_trim_eq(const char *p, int len, const char *word) { + int n = 0; + const char *t = md_lstrip(p, len, &n); + while (n > 0 && md_blank((unsigned char)t[n - SKIP_ONE])) { + n--; + } + return (size_t)n == strlen(word) && memcmp(t, word, (size_t)n) == 0; +} + +/* One line of the document. `raw` is not NUL-terminated. */ +static void md_line(md_scan_t *s, md_state_t *st, const char *raw, int len, bool mdx) { + if (s->line == SKIP_ONE && md_trim_eq(raw, len, "---")) { + st->in_front = true; + return; + } + if (st->in_front) { + if (md_trim_eq(raw, len, "---") || md_trim_eq(raw, len, "...")) { + st->in_front = false; + } + return; + } + int rest = 0; + const char *t = md_lstrip(raw, len, &rest); + if (st->in_fence) { + int k = 0; + while (k < rest && t[k] == st->fence_char) { + k++; + } + int tail = k; + while (tail < rest && md_blank((unsigned char)t[tail])) { + tail++; + } + if (k >= st->fence_len && tail == rest) { + st->in_fence = false; + } + return; + } + if (rest >= MD_FENCE_MIN && (t[0] == '`' || t[0] == '~')) { + int k = 0; + while (k < rest && t[k] == t[0]) { + k++; + } + if (k >= MD_FENCE_MIN && !md_has(t + k, (size_t)(rest - k), '`')) { + st->in_fence = true; + st->fence_char = t[0]; + st->fence_len = k; + return; + } + } + memcpy(s->buf, raw, (size_t)len); + s->buf[len] = '\0'; + s->len = len; + int from = 0; + if (st->in_comment) { + int end = md_find(s->buf, s->len, 0, "-->", false); + if (end < 0) { + return; + } + st->in_comment = false; + memset(s->buf, ' ', (size_t)end + MD_SLEN("-->")); + from = end + (int)MD_SLEN("-->"); + } + for (;;) { + int a = md_find(s->buf, s->len, from, "", false); + if (b < 0) { + s->len = a; + s->buf[a] = '\0'; + st->in_comment = true; + break; + } + memset(s->buf + a, ' ', (size_t)(b + (int)MD_SLEN("-->") - a)); + from = b + (int)MD_SLEN("-->"); + } + if (mdx && + (strncmp(s->buf, "import", MD_SLEN("import")) == 0 || + strncmp(s->buf, "export", MD_SLEN("export")) == 0) && + s->len > (int)MD_SLEN("import") && md_blank((unsigned char)s->buf[MD_SLEN("import")])) { + return; + } + md_inline(s); +} + +/* The file's sections: its Section definitions by start line. False only when + * memory ran out (a file without headings has none, and that is no failure). */ +static bool md_sections(CBMExtractCtx *ctx, CBMArena *a, md_section_t **out, int *count) { + const CBMDefArray *defs = &ctx->result->defs; + int n = 0; + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + n += d->label && d->qualified_name && strcmp(d->label, "Section") == 0; + } + *out = NULL; + *count = 0; + if (n == 0) { + return true; + } + md_section_t *secs = (md_section_t *)cbm_arena_alloc(a, (size_t)n * sizeof(*secs)); + if (!secs) { + return false; + } + for (int i = 0; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->label && d->qualified_name && strcmp(d->label, "Section") == 0) { + secs[(*count)++] = + (md_section_t){.line = d->start_line, .qn = d->qualified_name, .shared_qn = false}; + } + } + /* repeated qualified names, found through a copy sorted by name */ + const md_section_t **by_qn = + (const md_section_t **)cbm_arena_alloc(a, (size_t)n * sizeof(*by_qn)); + if (!by_qn) { + return false; + } + for (int i = 0; i < *count; i++) { + by_qn[i] = &secs[i]; + } + qsort(by_qn, (size_t)*count, sizeof(*by_qn), md_section_ptr_qn_cmp); + for (int i = SKIP_ONE; i < *count; i++) { + if (strcmp(by_qn[i]->qn, by_qn[i - SKIP_ONE]->qn) == 0) { + ((md_section_t *)by_qn[i])->shared_qn = true; + ((md_section_t *)by_qn[i - SKIP_ONE])->shared_qn = true; + } + } + qsort(secs, (size_t)*count, sizeof(*secs), md_section_cmp); + *out = secs; + return true; +} + +void cbm_doclink_md_scan_file(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena; + md_scan_t s = {.ctx = ctx}; + bool have_secs = md_sections(ctx, scratch, &s.secs, &s.nsecs); + s.buf = (char *)cbm_arena_alloc(scratch, (size_t)ctx->source_len + SKIP_ONE); + s.span_buf = (char *)cbm_arena_alloc(scratch, (size_t)(MD_SPAN_MAX + SKIP_ONE) * PAIR_LEN); + if (!s.buf || !s.span_buf || !have_secs) { + ctx->result->doc_links.failed = true; + return; + } + const char *rel = ctx->rel_path ? ctx->rel_path : ""; + size_t rel_len = strlen(rel); + bool mdx = rel_len >= MD_SLEN(".mdx") && + md_eq_ci(rel + rel_len - MD_SLEN(".mdx"), MD_SLEN(".mdx"), ".mdx"); + md_state_t st = {0}; + const char *src = ctx->source; + int n = ctx->source_len; + int pos = 0; + if (n >= MD_BOM_LEN && memcmp(src, "\xEF\xBB\xBF", MD_BOM_LEN) == 0) { + pos = MD_BOM_LEN; /* a byte-order mark is no part of the first line */ + } + while (pos < n && !s.failed) { + const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); + int end = nl ? (int)(nl - src) : n; + int len = end - pos; + if (len > 0 && src[pos + len - SKIP_ONE] == '\r') { + len--; + } + s.line++; + md_line(&s, &st, src + pos, len, mdx); + pos = end + SKIP_ONE; + } + md_list_free(&s.runs); + md_list_free(&s.by_len); + md_list_free(&s.ranges); + md_list_free(&s.stack); + md_list_free(&s.cands); + md_list_free(&s.opens); + md_list_free(&s.closes); + md_list_free(&s.bal); + md_list_free(&s.gts); + md_list_free(&s.blanks); + /* an architecture decision record: its node, facts and supersedes */ + cbm_adr_extract(ctx); +} diff --git a/internal/cbm/doclink_py.c b/internal/cbm/doclink_py.c new file mode 100644 index 0000000000..d762ab2e35 --- /dev/null +++ b/internal/cbm/doclink_py.c @@ -0,0 +1,419 @@ +/* + * doclink_py.c — the doc-link scope of a Python file: what the reST resolver + * (src/pipeline/doc_links_rst.c) needs from Python sources to resolve Sphinx + * references the way Sphinx does. + * + * I \t level \t module \t name \t alias a package `__init__.py`'s top-level + * `from module import name [as alias]`: + * the package re-exports `alias` + * (`django.db.models.Model` is + * django/db/models/base.py's Model). + * Star imports are not read (the field + * test's resolver did not follow them). + * C the file is a `conf.py`: documents in + * and below its directory are one + * Sphinx documentation set + * D \t domain its primary_domain + * X \t role an extlink role whose URL is a + * repository blob/tree path + * (:source:`django/db/x.py`) + * S \t name an intersphinx name (`python:` in + * :class:`python:dict` is external) + * + * conf.py is read as text, with the field test's patterns, and never run. + * Every other Python file has no scope (NULL), so a change to it never + * changes another file's resolution through this blob. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" + +#include +#include +#include +#include + +enum { + PY_SCOPE_FIELD_MAX = 256, /* a module, name or setting longer than this is skipped */ + PY_SB_INIT = 256, + PY_INT_DIGITS = 16, +}; + +/* A small arena string builder (the blob lives in the result arena). */ +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} py_sb_t; + +static void py_put(py_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + 1 > sb->cap) { + size_t ncap = sb->cap ? sb->cap : PY_SB_INIT; + while (ncap < sb->len + n + 1) { + ncap *= 2; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +/* One record: its kind, then each field after a TAB, then a newline. */ +static void py_rec(py_sb_t *sb, const char *kind, int nfields, const char *const *f, + const size_t *fl) { + py_put(sb, kind, strlen(kind)); + for (int i = 0; i < nfields; i++) { + py_put(sb, "\t", 1); + py_put(sb, f[i], fl[i]); + } + py_put(sb, "\n", 1); +} + +static const char *py_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + 1 : path; +} + +static bool py_field_ok(const char *s, size_t n) { + if (n == 0 || n > PY_SCOPE_FIELD_MAX) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '\t' || c == '\n' || c == '\r' || c < ' ') { + return false; + } + } + return true; +} + +static void py_text(const CBMExtractCtx *ctx, TSNode n, const char **s, size_t *len) { + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (b > (uint32_t)ctx->source_len || a > b) { + *s = ""; + *len = 0; + return; + } + *s = ctx->source + a; + *len = b - a; +} + +/* One `from module import names` statement of the package's top level. */ +static void py_from_import(const CBMExtractCtx *ctx, TSNode stmt, py_sb_t *sb) { + TSNode mod = ts_node_child_by_field_name(stmt, "module_name", (uint32_t)strlen("module_name")); + if (ts_node_is_null(mod)) { + return; + } + int level = 0; + const char *ms = ""; + size_t ml = 0; + if (strcmp(ts_node_type(mod), "relative_import") == 0) { + uint32_t nc = ts_node_child_count(mod); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(mod, i); + const char *k = ts_node_type(c); + if (strcmp(k, "import_prefix") == 0) { + const char *ps; + size_t pl; + py_text(ctx, c, &ps, &pl); + for (size_t j = 0; j < pl; j++) { + level += ps[j] == '.'; + } + } else if (strcmp(k, "dotted_name") == 0) { + py_text(ctx, c, &ms, &ml); + } + } + } else { + py_text(ctx, mod, &ms, &ml); + } + if (ml > 0 && !py_field_ok(ms, ml)) { + return; + } + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + const char *field = ts_node_field_name_for_child(stmt, i); + if (!field || strcmp(field, "name") != 0) { + continue; + } + TSNode c = ts_node_child(stmt, i); + TSNode name = c; + TSNode alias = c; + if (strcmp(ts_node_type(c), "aliased_import") == 0) { + name = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name")); + alias = ts_node_child_by_field_name(c, "alias", (uint32_t)strlen("alias")); + if (ts_node_is_null(name) || ts_node_is_null(alias)) { + continue; + } + } + const char *ns; + size_t nl; + const char *as; + size_t al; + py_text(ctx, name, &ns, &nl); + py_text(ctx, alias, &as, &al); + if (!py_field_ok(ns, nl) || !py_field_ok(as, al)) { + continue; + } + char lv[PY_INT_DIGITS]; + int lvn = snprintf(lv, sizeof(lv), "%d", level); + const char *f[] = {lv, ms, ns, as}; + const size_t fl[] = {lvn > 0 ? (size_t)lvn : 0, ml, nl, al}; + py_rec(sb, "I", 4, f, fl); + } +} + +/* ── conf.py, read as text (the field test's patterns) ───────────── */ + +static bool py_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool py_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +static size_t py_skip_ws(const char *s, size_t n, size_t i) { + while (i < n && py_ws(s[i])) { + i++; + } + return i; +} + +/* `^name\s*=\s*` at a line start: the index after it, or 0. */ +static size_t py_assign_at(const char *s, size_t n, size_t line, const char *name) { + size_t nl = strlen(name); + if (line + nl > n || memcmp(s + line, name, nl) != 0) { + return 0; + } + size_t i = py_skip_ws(s, n, line + nl); + if (i >= n || s[i] != '=') { + return 0; + } + return py_skip_ws(s, n, i + 1); +} + +/* conf_scalar: ^name\s*=\s*["']([^"']*)["'] (first match). */ +static bool py_conf_scalar(const char *s, size_t n, const char *name, const char **v, size_t *vl) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && (s[i] == '"' || s[i] == '\'')) { + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j < n) { + *v = s + i + 1; + *vl = j - i - 1; + return true; + } + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* conf_dict_block: ^name\s*=\s*\{(.*?)^\} -- the text between, or false. */ +static bool py_conf_block(const char *s, size_t n, const char *name, size_t *a, size_t *b) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && s[i] == '{') { + size_t k = i + 1; + while (k < n) { + if (s[k] == '}' && (k == 0 || s[k - 1] == '\n')) { + *a = i + 1; + *b = k; + return true; + } + k++; + } + return false; + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* ["']([\w.-]+)["'] at s[i] (extra: also '.'): its end after the closing quote, or 0. */ +static size_t py_quoted_key(const char *s, size_t n, size_t i, bool dots, size_t *ka, size_t *kb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return 0; + } + size_t j = i + 1; + while (j < n && (py_word(s[j]) || s[j] == '-' || (dots && s[j] == '.'))) { + j++; + } + if (j == i + 1 || j >= n || (s[j] != '"' && s[j] != '\'')) { + return 0; + } + *ka = i + 1; + *kb = j; + return j + 1; +} + +/* ["']([^"']+)["'] at s[i]: the value span, or false. */ +static bool py_quoted_value(const char *s, size_t n, size_t i, size_t *va, size_t *vb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return false; + } + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j == i + 1 || j >= n) { + return false; + } + *va = i + 1; + *vb = j; + return true; +} + +/* /(blob|tree)/[^/]+/%s in the URL: a repository path pattern. */ +static bool py_repo_url(const char *u, size_t n) { + static const char *const kinds[] = {"/blob/", "/tree/", NULL}; + for (int k = 0; kinds[k]; k++) { + size_t kl = strlen(kinds[k]); + for (size_t i = 0; i + kl <= n; i++) { + if (memcmp(u + i, kinds[k], kl) != 0) { + continue; + } + size_t j = i + kl; + size_t ref = j; + while (j < n && u[j] != '/') { + j++; + } + if (j > ref && j + 3 <= n && memcmp(u + j, "/%s", 3) == 0) { + return true; + } + } + } + return false; +} + +static void py_extlink(py_sb_t *sb, const char *s, size_t ka, size_t kb, size_t va, size_t vb) { + if (py_repo_url(s + va, vb - va) && py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "X", 1, f, fl); + } +} + +static void py_conf(const char *s, size_t n, py_sb_t *sb) { + py_rec(sb, "C", 0, NULL, NULL); + const char *v; + size_t vl; + if (py_conf_scalar(s, n, "primary_domain", &v, &vl) && py_field_ok(v, vl)) { + const char *f[] = {v}; + const size_t fl[] = {vl}; + py_rec(sb, "D", 1, f, fl); + } + size_t a; + size_t b; + if (py_conf_block(s, n, "intersphinx_mapping", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, true, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "S", 1, f, fl); + } + i = e - 1; + } + } + /* extlinks = {'role': ('url%s', ...)} and extlinks['role'] = ('url%s', ...) */ + if (py_conf_block(s, n, "extlinks", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, false, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + size_t va; + size_t vb; + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_quoted_value(s, b, py_skip_ws(s, b, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + i = e - 1; + } + } + static const char key_open[] = "extlinks["; + size_t kol = strlen(key_open); + for (size_t at = 0; at + kol <= n; at++) { + if (memcmp(s + at, key_open, kol) != 0) { + continue; + } + size_t i = at + kol; + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, n, i, false, &ka, &kb); + if (e == 0 || e >= n || s[e] != ']') { + continue; + } + size_t c = py_skip_ws(s, n, e + 1); + size_t va; + size_t vb; + if (c < n && s[c] == '=' && (c = py_skip_ws(s, n, c + 1)) < n && s[c] == '(' && + py_quoted_value(s, n, py_skip_ws(s, n, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + } +} + +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->rel_path || !ctx->source) { + return NULL; + } + const char *base = py_base(ctx->rel_path); + bool init = strcmp(base, "__init__.py") == 0; + bool conf = strcmp(base, "conf.py") == 0; + if (!init && !conf) { + return NULL; + } + py_sb_t sb = {.a = ctx->arena}; + py_put(&sb, CBM_DOCLINK_PY_SCOPE_TAG "\n", strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n")); + if (conf) { + py_conf(ctx->source, (size_t)ctx->source_len, &sb); + } + if (init && !ts_node_is_null(ctx->root)) { + uint32_t nc = ts_node_child_count(ctx->root); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(ctx->root, i); + if (strcmp(ts_node_type(c), "import_from_statement") == 0) { + py_from_import(ctx, c, &sb); + } + } + } + if (sb.failed) { + ctx->result->doc_links.failed = true; + return NULL; + } + return sb.buf; +} diff --git a/internal/cbm/doclink_rst.c b/internal/cbm/doclink_rst.c new file mode 100644 index 0000000000..a50498f144 --- /dev/null +++ b/internal/cbm/doclink_rst.c @@ -0,0 +1,1791 @@ +/* + * doclink_rst.c — reStructuredText documents: their sections, and the code + * their text names (the extraction half; src/pipeline/doc_links_rst.c + * resolves). + * + * Structure: the field test's line model (H3 RstDoc) with CPython's str + * semantics -- tabs expanded to 8 columns, str.strip / str.split whitespace + * and re's \s and \w from the Unicode tables the PDF scanner uses. Every line + * gets a kind: title, adornment, prose, directive head / argument / option, + * literal block (after `::` or in a code directive), comment, hyperlink + * target. Titles are found by adornment style (over+under or under only; the + * level is the style's first appearance). Each title becomes a Section + * definition: name the title, docstring the text up to the next title, lines + * from its title to the line before the next one. Text before the first title + * belongs to the file. + * + * References become tokens whose `raw` is the reference AS WRITTEN followed by + * TAB-separated fields the resolver reads (a doc_link_unresolved row shows the + * written form first; whitespace inside a field is collapsed, so no field + * holds a TAB): + * role :py:class:`x`, :func:`~a.b`, :c:type:`t`, :source:`p` + * written, role, content, module, class, C namespace + * object .. py:class:: sig, .. function:: sig (one per signature) + * written, directive, signature, module, class, C namespace, + * the Python full name in its class ("" when Sphinx's + * signature grammar does not read it) + * autodoc .. autoclass:: a.b + * written, directive, argument, module, class + * include .. include:: / .. kernel-include:: of a code file + * written, directive, path + * literalinclude .. literalinclude:: path + * written, path, :lines:, :pyobject:, "1" when a + * :start-after: / :end-before: / :start-at: / :end-at: + * bounds it by text (the resolver binds the file then) + * kernel_doc .. kernel-doc:: path, and each name of :identifiers: / + * :functions: -- written, path, name ("" for the file) + * code_path, an inline literal ``...`` that names a path or a qualified + * code_name name (the Markdown code-span classifier); raw is its text + * A role or directive without a domain is taken for Python's when this file + * sets contexts (`module`, `currentmodule`, class directives: Sphinx's + * default domain); which domain it belongs to is the resolver's call (the + * documentation set's conf.py). Roles of no code domain are tokens only when + * their content is a path (an extlink such as :source:`django/db/x.py`). + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "helpers.h" +#include "pdf/pdf_unicode.h" + +#include +#include +#include +#include + +enum { + RST_TAB = 8, + RST_SHORT_ADORN = 4, /* an underline this long fits any title; a transition */ + RST_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */ + RST_SPAN_MAX = 150, /* an inline literal the span classifier reads (MD_SPAN_MAX) */ + RST_INIT_CAP = 16, + RST_REPLACEMENT = 0xFFFD, + RST_ASCII_END = 0x80, + RST_LEAD2_MIN = 0xC2, /* UTF-8 lead bytes (no overlong two-byte form) */ + RST_LEAD3_MIN = 0xE0, + RST_LEAD3_MAX = 0xEF, + RST_LEAD4_MIN = 0xF0, + RST_LEAD4_MAX = 0xF4, + RST_NEED3 = 2, + RST_NEED4 = 3, + RST_BULLET = 0x2022, + RST_SIG_ARROW = 2, /* "->" */ +}; + +#define RST_SEP '\t' + +typedef enum { + K_BLANK = 0, + K_TITLE, + K_ADORN, + K_DHEAD, + K_DHEAD_PROSE, + K_DARG, + K_DARG_PROSE, + K_DOPT, + K_LITERAL, + K_PLIT, + K_SKIP, + K_TARGET, + K_COMMENT, + K_PROSE, +} rst_kind_t; + +typedef struct { + char *s; /* the line, tabs expanded, NUL-terminated */ + int len; /* bytes */ + int indent; /* leading spaces (H3 indent_of) */ + bool blank; /* nothing but whitespace */ +} rst_line_t; + +typedef struct { + const char *key; + const char *val; +} rst_opt_t; + +typedef struct { + int line; + int indent; + const char *name; /* as written, lower-cased, with its domain */ + const char *dom; /* "py", "c", "std" or the written domain */ + const char *dname; /* without the domain */ + const char **args; + int nargs; + rst_opt_t *opts; + int nopts; + int cs; /* content lines [cs, ce) */ + int ce; +} rst_dir_t; + +typedef struct { + int first; /* the title block's first line (an overline, or the title) */ + int idx; /* the title text's line */ + int next; /* the line after the title block */ + const char *text; + const char *qn; + bool shared_qn; /* another title of the file has the same QN: the file is the source */ +} rst_title_t; + +typedef struct { + int line; /* 0-based */ + int col; /* byte offset in its line; directives use their indent */ + int syntax; + const char *raw; +} rst_tok_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *scratch; + rst_line_t *L; + int n; + unsigned char *K; + rst_dir_t *dirs; + int ndirs; + int cap_dirs; + rst_title_t *titles; + int ntitles; + int cap_titles; + char styles[2 * RST_INIT_CAP * RST_INIT_CAP]; /* (char, overline) pairs in first-seen order */ + int nstyles; + const char **mod_at; + const char **cls_at; + const char **ns_at; + rst_tok_t *toks; + int ntoks; + int cap_toks; + char *span_buf; + bool failed; +} rst_doc_t; + +/* ── Unicode, as CPython's str and re see it ─────────────────────── */ + +/* The code point at s[i] and its length; a malformed byte is U+FFFD of one + * byte (decode(errors='replace')). */ +static uint32_t rst_cp(const char *s, int len, int i, int *n) { + unsigned char c = (unsigned char)s[i]; + if (c < RST_ASCII_END) { + *n = 1; + return c; + } + int need = 0; /* continuation bytes of a well-formed lead byte */ + if (c >= RST_LEAD4_MIN && c <= RST_LEAD4_MAX) { + need = RST_NEED4; + } else if (c >= RST_LEAD3_MIN && c <= RST_LEAD3_MAX) { + need = RST_NEED3; + } else if (c >= RST_LEAD2_MIN && c < RST_LEAD3_MIN) { + need = 1; + } + if (need == 0 || i + need >= len) { + *n = 1; + return RST_REPLACEMENT; + } + uint32_t cp = c & (0x3F >> need); + for (int k = 1; k <= need; k++) { + unsigned char cc = (unsigned char)s[i + k]; + if ((cc & 0xC0) != 0x80) { + *n = 1; + return RST_REPLACEMENT; + } + cp = (cp << 6) | (cc & 0x3F); + } + static const uint32_t min_cp[] = {0, 0x80, 0x800, 0x10000}; + if (cp < min_cp[need] || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + *n = 1; + return RST_REPLACEMENT; + } + *n = need + 1; + return cp; +} + +static bool rst_space_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return cp == ' ' || (cp >= '\t' && cp <= '\r') || (cp >= 0x1C && cp <= 0x1F); + } + return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, cp); +} + +static bool rst_word_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z') || (cp >= '0' && cp <= '9') || + cp == '_'; + } + return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, cp); +} + +static bool rst_alpha_ascii(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool rst_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +/* Whitespace code points from s[i]: the index after them. */ +static int rst_skip_ws(const char *s, int len, int i) { + while (i < len) { + int n; + if (!rst_space_cp(rst_cp(s, len, i, &n))) { + break; + } + i += n; + } + return i; +} + +/* str.strip(): [*a, *b) without leading and trailing whitespace. */ +static void rst_strip(const char *s, int len, int *a, int *b) { + int i = rst_skip_ws(s, len, 0); + int end = i; + int k = i; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + k += n; + if (!rst_space_cp(cp)) { + end = k; + } + } + *a = i; + *b = end; +} + +/* Code points in s[a, b). */ +static int rst_cps(const char *s, int a, int b) { + int count = 0; + while (a < b) { + int n; + (void)rst_cp(s, b, a, &n); + a += n; + count++; + } + return count; +} + +/* ' '.join(s.split()), into the arena. */ +static char *rst_collapse(CBMArena *a, const char *s, int len) { + char *out = (char *)cbm_arena_alloc(a, (size_t)len + 1); + if (!out) { + return NULL; + } + int w = 0; + int i = 0; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + out[w] = '\0'; + return out; +} + +/* ── Growing arrays in the scratch arena ─────────────────────────── */ + +static void *rst_grow(rst_doc_t *d, void *items, int count, int *cap, size_t size) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * 2 : RST_INIT_CAP; + void *neu = cbm_arena_alloc(d->scratch, (size_t)ncap * size); + if (!neu) { + d->failed = true; + return NULL; + } + if (items && count) { + memcpy(neu, items, (size_t)count * size); + } + *cap = ncap; + return neu; +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +/* text.expandtabs(8).split('\n'): the column restarts after '\r' and '\n'. */ +static bool rst_lines(rst_doc_t *d, const char *src, int n) { + int count = 1; + for (int i = 0; i < n; i++) { + count += src[i] == '\n'; + } + d->L = (rst_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L)); + d->K = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count); + if (!d->L || !d->K) { + return false; + } + memset(d->K, 0, (size_t)count); + int pos = 0; + for (int li = 0; li < count; li++) { + const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); + int end = nl ? (int)(nl - src) : n; + int tabs = 0; + for (int i = pos; i < end; i++) { + tabs += src[i] == '\t'; + } + int cap = (end - pos) + tabs * RST_TAB + 1; + char *out = (char *)cbm_arena_alloc(d->scratch, (size_t)cap); + if (!out) { + return false; + } + int w = 0; + int col = 0; + int i = pos; + while (i < end) { + int cn; + uint32_t cp = rst_cp(src, end, i, &cn); + if (cp == '\t') { + int fill = RST_TAB - (col % RST_TAB); + memset(out + w, ' ', (size_t)fill); + w += fill; + col += fill; + } else { + memcpy(out + w, src + i, (size_t)cn); + w += cn; + col = cp == '\r' ? 0 : col + 1; + } + i += cn; + } + out[w] = '\0'; + rst_line_t *ln = &d->L[li]; + ln->s = out; + ln->len = w; + int sp = 0; + while (sp < w && out[sp] == ' ') { + sp++; + } + ln->indent = sp; + int a; + int b; + rst_strip(out, w, &a, &b); + ln->blank = a >= b; + pos = end + 1; + } + d->n = count; + return true; +} + +/* ── Line patterns (H3's regexes) ────────────────────────────────── */ + +static bool rst_punct(unsigned char c) { + return (c >= '!' && c <= '/') || (c >= ':' && c <= '@') || (c >= '[' && c <= '`') || + (c >= '{' && c <= '~'); +} + +/* ADORN_RE ^([!-/:-@\[-`{-~])\1*\s*$ */ +static bool rst_adorn(const char *s, int len) { + if (len == 0 || !rst_punct((unsigned char)s[0])) { + return false; + } + int i = 1; + while (i < len && s[i] == s[0]) { + i++; + } + return rst_skip_ws(s, len, i) == len; +} + +/* s.rstrip() length. */ +static int rst_rlen(const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + return a >= b ? 0 : b; +} + +static bool rst_style_index(rst_doc_t *d, char ch, bool over, int *level) { + for (int i = 0; i < d->nstyles; i++) { + if (d->styles[2 * i] == ch && d->styles[(2 * i) + 1] == (char)over) { + *level = i; + return true; + } + } + if ((size_t)(2 * (d->nstyles + 1)) > sizeof(d->styles)) { + return false; + } + d->styles[2 * d->nstyles] = ch; + d->styles[(2 * d->nstyles) + 1] = (char)over; + *level = d->nstyles++; + return true; +} + +/* H3 _title_at: the title's line, its style and the line after the block. */ +static bool rst_title_at(const rst_doc_t *d, int i, int *tidx, char *ch, bool *over, int *next) { + const rst_line_t *L = d->L; + int n = d->n; + int sl = rst_rlen(L[i].s, L[i].len); + int sa; + int sb; + rst_strip(L[i].s, sl, &sa, &sb); + if (rst_adorn(L[i].s, sl)) { + if (sb - sa == 2 && L[i].s[sa] == ':' && L[i].s[sa + 1] == ':') { + return false; + } + if (i + 2 < n) { + const rst_line_t *t = &L[i + 1]; + int ul = rst_rlen(L[i + 2].s, L[i + 2].len); + const char *u = L[i + 2].s; + if (!t->blank && !rst_adorn(t->s, t->len) && rst_adorn(u, ul) && u[0] == L[i].s[sa]) { + *tidx = i + 1; + *ch = L[i].s[sa]; + *over = true; + *next = i + 3; + return true; + } + } + return false; + } + if (i + 1 < n && !(sl >= 2 && L[i].s[0] == '.' && L[i].s[1] == '.')) { + const char *u = L[i + 1].s; + int ul = rst_rlen(u, L[i + 1].len); + if (ul > 0 && !rst_space_cp((unsigned char)u[0]) && rst_adorn(u, ul)) { + int ua; + int ub; + rst_strip(u, ul, &ua, &ub); + bool dcolon = ub - ua == 2 && u[ua] == ':' && u[ua + 1] == ':'; + int ulen = ub - ua; /* ASCII punctuation: bytes are code points */ + if (!dcolon && (ulen >= rst_cps(L[i].s, sa, sb) || ulen >= RST_SHORT_ADORN)) { + *tidx = i; + *ch = u[ua]; + *over = false; + *next = i + 2; + return true; + } + } + } + return false; +} + +static void rst_mark(rst_doc_t *d, int a, int b, rst_kind_t kind) { + for (int k = a; k < b; k++) { + d->K[k] = (unsigned char)(d->L[k].blank ? K_BLANK : kind); + } +} + +static int rst_block_end(const rst_doc_t *d, int j, int ind) { + int k = j; + while (k < d->n && (d->L[k].blank || d->L[k].indent > ind)) { + k++; + } + return k; +} + +static bool rst_name_char(const char *s, int len, int i, int *n) { + uint32_t cp = rst_cp(s, len, i, n); + return cp == ':' || cp == '.' || cp == '+' || cp == '-' || rst_word_cp(cp); +} + +/* DIRECTIVE_RE: ^(\s*)\.\.\s+(?:\|[^|]+\|\s+)?([A-Za-z0-9][\w:.+-]*?)::(?:\s+(.*?))?\s*$ + * Fills the indent (code points), the name span and the first argument + * span (stripped). */ +static bool rst_directive_match(const char *s, int len, int *ind, int *na, int *nb, int *aa, + int *ab) { + int p = rst_skip_ws(s, len, 0); + *ind = rst_cps(s, 0, p); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + int q = rst_skip_ws(s, len, p + 2); + if (q == p + 2) { + return false; + } + p = q; + if (p < len && s[p] == '|') { + const char *bar = memchr(s + p + 1, '|', (size_t)(len - p - 1)); + if (bar && bar > s + p + 1) { + int after = (int)(bar - s) + 1; + int w = rst_skip_ws(s, len, after); + if (w > after) { + p = w; + } + } + } + if (p >= len || !(rst_alpha_ascii((unsigned char)s[p]) || rst_digit((unsigned char)s[p]))) { + return false; + } + int e = p + 1; + for (;;) { + if (e + 2 <= len && s[e] == ':' && s[e + 1] == ':') { + int t = e + 2; + int w = rst_skip_ws(s, len, t); + if (t == len || w > t) { + *na = p; + *nb = e; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *aa = w + a; + *ab = w + (b > a ? b : a); + return true; + } + } + int cn; + if (e >= len || !rst_name_char(s, len, e, &cn)) { + return false; + } + e += cn; + } +} + +/* EXPLICIT_RE ^(\s*)\.\.(?:\s|$) */ +static bool rst_explicit_match(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + if (p + 2 == len) { + return true; + } + int n; + return rst_space_cp(rst_cp(s, len, p + 2, &n)); +} + +/* OPTION_LINE_RE ^(\s*):([A-Za-z][\w-]*):(?:\s+(.*?))?\s*$ */ +static bool rst_option_match(const char *s, int len, int *ka, int *kb, int *va, int *vb) { + int p = rst_skip_ws(s, len, 0); + if (p >= len || s[p] != ':' || p + 1 >= len || !rst_alpha_ascii((unsigned char)s[p + 1])) { + return false; + } + int k = p + 2; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + if (cp != '-' && !rst_word_cp(cp)) { + break; + } + k += n; + } + if (k >= len || s[k] != ':') { + return false; + } + int t = k + 1; + int w = rst_skip_ws(s, len, t); + if (t != len && w == t) { + return false; + } + *ka = p + 1; + *kb = k; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *va = w + a; + *vb = w + (b > a ? b : a); + return true; +} + +static bool rst_roman(unsigned char c) { + return strchr("ivxlcdmIVXLCDM", c) != NULL && c != '\0'; +} + +/* LIST_MARKER_RE: the match length in code points, or -1. */ +static int rst_list_marker(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p >= len) { + return -1; + } + int n; + uint32_t cp = rst_cp(s, len, p, &n); + int q = -1; + if (cp == '-' || cp == '*' || cp == '+' || cp == RST_BULLET) { + q = p + n; + } else { + int r = p + (s[p] == '('); + int e = -1; + if (r < len && rst_digit((unsigned char)s[r])) { + e = r; + while (e < len && rst_digit((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e < 0 && r + 1 < len && rst_alpha_ascii((unsigned char)s[r]) && + (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r + 1 < len && s[r] == '#' && (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r < len && rst_roman((unsigned char)s[r])) { + e = r; + while (e < len && rst_roman((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e >= 0) { + q = e + 1; + } + } + if (q < 0) { + return -1; + } + int w = rst_skip_ws(s, len, q); + if (w == q || w >= len) { + return -1; + } + return rst_cps(s, 0, w); +} + +/* ── Block structure (H3 _scan) ──────────────────────────────────── */ + +static const char *const PY_DOMAIN_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", "currentmodule", NULL}; +static const char *const LITERAL_CONTENT[] = { + "code-block", "code", "sourcecode", "literalinclude", "raw", "math", + "graphviz", "digraph", "graph", "productionlist", "console", "doctest", + "testcode", "testoutput", "testsetup", "testcleanup", "highlight", "kernel-render", + "prompt", "kernel-include", "include", "kernel-abi", "kernel-feat", "uml", + "ipython", "mermaid", NULL}; +static const char *const SKIP_CONTENT[] = {"toctree", "index", "tabularcolumns", NULL}; +static const char *const ARG_PROSE[] = {"note", "warning", "tip", + "hint", "important", "caution", + "attention", "danger", "error", + "seealso", "admonition", "versionadded", + "versionchanged", "deprecated", "versionremoved", + "rubric", "topic", "sidebar", + "replace", "centered", NULL}; +static const char *const PY_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", NULL}; +static const char *const C_OBJ[] = {"function", "macro", "type", "struct", "union", + "enum", "enumerator", "member", "var", NULL}; +static const char *const AUTODOC[] = { + "automodule", "autoclass", "autofunction", "automethod", "autoattribute", + "autoexception", "autodata", "autodecorator", "autoproperty", NULL}; +static const char *const PY_ROLES[] = {"func", "class", "meth", "attr", "mod", "exc", + "data", "obj", "const", "deco", NULL}; +static const char *const C_ROLES[] = {"func", "macro", "struct", "union", "enum", "enumerator", + "type", "member", "data", "var", NULL}; +/* include / kernel-include of these is code (H3 CODE_EXT) */ +static const char *const CODE_EXT[] = {".c", ".h", ".s", ".py", ".sh", ".rs", ".pl", + ".awk", ".cpp", ".cc", ".hpp", ".go", ".js", ".ts", + ".lds", ".dts", ".dtsi", ".yaml", ".yml", ".json", ".toml", + ".mk", ".cmake", ".bpf", ".tc", NULL}; + +static bool rst_in(const char *s, const char *const *list) { + for (int i = 0; s && list[i]; i++) { + if (strcmp(s, list[i]) == 0) { + return true; + } + } + return false; +} + +/* H3 split_domain with Python as the primary domain. */ +static void rst_split_domain(rst_doc_t *d, const char *name, const char *const *py_members, + const char **dom, const char **rest) { + const char *colon = strchr(name, ':'); + if (colon) { + *dom = cbm_arena_strndup(d->scratch, name, (size_t)(colon - name)); + *rest = colon + 1; + if (!*dom) { + d->failed = true; + *dom = ""; + } + return; + } + *dom = rst_in(name, py_members) ? "py" : "std"; + *rest = name; +} + +static char *rst_dup_stripped(rst_doc_t *d, const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + char *out = cbm_arena_strndup(d->scratch, s + a, (size_t)(b > a ? b - a : 0)); + if (!out) { + d->failed = true; + } + return out; +} + +static int rst_directive(rst_doc_t *d, int i, int ind, int na, int nb, int aa, int ab) { + rst_line_t *L = d->L; + int n = d->n; + char *name = cbm_arena_strndup(d->scratch, L[i].s + na, (size_t)(nb - na)); + if (!name) { + d->failed = true; + return n; + } + for (char *c = name; *c; c++) { + if (*c >= 'A' && *c <= 'Z') { + *c = (char)(*c + ('a' - 'A')); + } + } + rst_dir_t dir = {.line = i, .indent = ind, .name = name}; + rst_split_domain(d, name, PY_DOMAIN_OBJ, &dir.dom, &dir.dname); + bool prose_arg = rst_in(dir.dname, ARG_PROSE); + d->K[i] = prose_arg ? K_DHEAD_PROSE : K_DHEAD; + int cap_args = 0; + if (ab > aa) { + dir.args = (const char **)rst_grow(d, NULL, 0, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs] = rst_dup_stripped(d, L[i].s + aa, ab - aa); + dir.nargs++; + } + int j = i + 1; + int ka; + int kb; + int va; + int vb; + while (j < n && !L[j].blank && L[j].indent > ind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + dir.args = (const char **)rst_grow(d, dir.args, dir.nargs, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs++] = rst_dup_stripped(d, L[j].s, L[j].len); + d->K[j] = prose_arg ? K_DARG_PROSE : K_DARG; + j++; + } + int cap_opts = 0; + while (j < n && !L[j].blank && L[j].indent > ind && + rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *key = cbm_arena_strndup(d->scratch, L[j].s + ka, (size_t)(kb - ka)); + char *val = cbm_arena_strndup(d->scratch, L[j].s + va, (size_t)(vb - va)); + int oind = L[j].indent; + d->K[j] = K_DOPT; + j++; + while (j < n && !L[j].blank && L[j].indent > oind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *more = rst_dup_stripped(d, L[j].s, L[j].len); + if (val && more) { + val = cbm_arena_sprintf(d->scratch, "%s%s%s", val, val[0] && more[0] ? " " : "", + more); + } + d->K[j] = K_DOPT; + j++; + } + dir.opts = (rst_opt_t *)rst_grow(d, dir.opts, dir.nopts, &cap_opts, sizeof(rst_opt_t)); + if (!key || !val || !dir.opts) { + d->failed = true; + return n; + } + /* options[key] = val: a repeated key keeps its first place, the last value */ + bool found = false; + for (int k = 0; k < dir.nopts; k++) { + if (strcmp(dir.opts[k].key, key) == 0) { + dir.opts[k].val = val; + found = true; + } + } + if (!found) { + dir.opts[dir.nopts++] = (rst_opt_t){key, val}; + } + } + dir.cs = j; + dir.ce = rst_block_end(d, j, ind); + d->dirs = (rst_dir_t *)rst_grow(d, d->dirs, d->ndirs, &d->cap_dirs, sizeof(rst_dir_t)); + if (!d->dirs) { + return n; + } + d->dirs[d->ndirs++] = dir; + if (rst_in(dir.dname, LITERAL_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_LITERAL); + return dir.ce; + } + if (strcmp(dir.dname, "parsed-literal") == 0) { + rst_mark(d, dir.cs, dir.ce, K_PLIT); + return dir.ce; + } + if (rst_in(dir.dname, SKIP_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_SKIP); + return dir.ce; + } + return dir.cs; +} + +static int rst_explicit(rst_doc_t *d, int i, int ind) { + rst_line_t *L = d->L; + int a; + int b; + rst_strip(L[i].s, L[i].len, &a, &b); + const char *s = L[i].s + a; + int sl = b - a; + if ((sl >= 4 && strncmp(s, ".. _", 4) == 0) || (sl >= 3 && strncmp(s, "__ ", 3) == 0)) { + d->K[i] = K_TARGET; + int j = i + 1; + while (j < d->n && !L[j].blank && L[j].indent > ind) { + d->K[j] = K_TARGET; + j++; + } + return j; + } + /* \.\.\s+\[[^\]]+\] : a footnote or citation */ + int p = rst_skip_ws(s, sl, 2); + if (p > 2 && p < sl && s[p] == '[') { + const char *close = memchr(s + p + 1, ']', (size_t)(sl - p - 1)); + if (close && close > s + p + 1) { + d->K[i] = K_PROSE; + return i + 1; + } + } + if (sl == 2 && (i + 1 >= d->n || L[i + 1].blank)) { + d->K[i] = K_COMMENT; + return i + 1; + } + int j = rst_block_end(d, i + 1, ind); + rst_mark(d, i, j, K_COMMENT); + return j; +} + +static void rst_scan(rst_doc_t *d) { + rst_line_t *L = d->L; + int n = d->n; + int i = 0; + int para_indent = 0; + int literal_pending = -1; + while (i < n && !d->failed) { + if (L[i].blank) { + d->K[i] = K_BLANK; + i++; + continue; + } + bool prev_blank = i == 0 || L[i - 1].blank; + int ind = L[i].indent; + if (literal_pending >= 0) { + int lp = literal_pending; + literal_pending = -1; + if (prev_blank && ind > lp) { + int j = rst_block_end(d, i, lp); + rst_mark(d, i, j, K_LITERAL); + i = j; + continue; + } + } + if (ind == 0 && prev_blank) { + int tidx; + char ch; + bool over; + int next; + if (rst_title_at(d, i, &tidx, &ch, &over, &next)) { + int level; + d->titles = (rst_title_t *)rst_grow(d, d->titles, d->ntitles, &d->cap_titles, + sizeof(rst_title_t)); + char *text = rst_collapse(d->scratch, L[tidx].s, L[tidx].len); + if (!d->titles || !text || !rst_style_index(d, ch, over, &level)) { + d->failed = true; + return; + } + (void)level; + d->titles[d->ntitles++] = + (rst_title_t){.first = i, .idx = tidx, .next = next, .text = text}; + rst_mark(d, i, next, K_ADORN); + d->K[tidx] = K_TITLE; + i = next; + continue; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (rst_adorn(L[i].s, L[i].len) && rst_cps(L[i].s, sa, sb) >= RST_SHORT_ADORN && + (i + 1 >= n || L[i + 1].blank)) { + d->K[i] = K_ADORN; + i++; + continue; + } + } + int dind; + int na; + int nb; + int aa; + int ab; + if (rst_directive_match(L[i].s, L[i].len, &dind, &na, &nb, &aa, &ab)) { + i = rst_directive(d, i, dind, na, nb, aa, ab); + continue; + } + if (rst_explicit_match(L[i].s, L[i].len)) { + i = rst_explicit(d, i, ind); + continue; + } + d->K[i] = K_PROSE; + if (prev_blank) { + int mm = rst_list_marker(L[i].s, L[i].len); + para_indent = mm >= 0 ? mm : ind; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (sb - sa >= 2 && L[i].s[sb - 1] == ':' && L[i].s[sb - 2] == ':') { + literal_pending = para_indent; + } + i++; + } +} + +/* ── Python signatures (Sphinx py_sig_re, H3 join_sig_lines / py_fullname) ── */ + +/* One signature per argument line; a trailing backslash continues it. */ +static int rst_join_sigs(rst_doc_t *d, const rst_dir_t *dir, const char ***out) { + const char **sigs = + (const char **)cbm_arena_alloc(d->scratch, (size_t)(dir->nargs + 1) * sizeof(char *)); + if (!sigs) { + d->failed = true; + return 0; + } + int ns = 0; + const char *cur = ""; + bool open = false; + for (int k = 0; k < dir->nargs; k++) { + const char *a = dir->args[k]; + size_t al = strlen(a); + if (al > 0 && a[al - 1] == '\\') { + cur = cbm_arena_sprintf(d->scratch, "%s%.*s ", cur, (int)(al - 1), a); + open = true; + } else { + sigs[ns++] = cbm_arena_sprintf(d->scratch, "%s%s", cur, a); + cur = ""; + open = false; + } + if (!cur || (ns > 0 && !sigs[ns - 1])) { + d->failed = true; + return 0; + } + } + if (open) { + sigs[ns++] = cur; + } + *out = sigs; + return ns; +} + +/* After the name: \s* (\[ .* \])? \s* ( \( .* \) (\s* -> .*)? )? $ */ +static bool rst_sig_paren_ok(const char *s, int len, int i) { + if (i == len) { + return true; + } + if (s[i] != '(') { + return false; + } + for (int q = len - 1; q > i; q--) { + if (s[q] != ')') { + continue; + } + if (q + 1 == len) { + return true; + } + int w = rst_skip_ws(s, len, q + 1); + if (w + RST_SIG_ARROW <= len && s[w] == '-' && s[w + 1] == '>') { + return true; + } + } + return false; +} + +static bool rst_sig_rest_ok(const char *s, int len, int i) { + i = rst_skip_ws(s, len, i); + if (i == len) { + return true; + } + if (s[i] == '[') { + for (int q = len - 1; q > i; q--) { + if (s[q] == ']' && rst_sig_paren_ok(s, len, rst_skip_ws(s, len, q + 1))) { + return true; + } + } + return false; + } + return rst_sig_paren_ok(s, len, i); +} + +/* PY_SIG_RE on sig.strip(): the prefix ("a.b." or "") and the name. */ +static bool rst_py_sig(rst_doc_t *d, const char *sig, const char **prefix, const char **name) { + int a; + int b; + rst_strip(sig, (int)strlen(sig), &a, &b); + const char *s = sig + a; + int len = b > a ? b - a : 0; + int r = 0; + int last_dot = -1; + while (r < len) { + int n; + uint32_t cp = rst_cp(s, len, r, &n); + if (cp == '.') { + last_dot = r; + } else if (!rst_word_cp(cp)) { + break; + } + r += n; + } + int ns = last_dot + 1; + if (ns >= r || !rst_sig_rest_ok(s, len, r)) { + return false; + } + *prefix = cbm_arena_strndup(d->scratch, s, (size_t)ns); + *name = cbm_arena_strndup(d->scratch, s + ns, (size_t)(r - ns)); + if (!*prefix || !*name) { + d->failed = true; + return false; + } + return true; +} + +/* Sphinx PyObject.handle_signature's fullname, and the remaining prefix. */ +static void rst_py_fullname(rst_doc_t *d, const char *prefix, const char *name, const char *cls, + const char **full, const char **rem) { + if (cls && cls[0]) { + size_t cl = strlen(cls); + if (prefix[0] && + (strcmp(prefix, cls) == 0 || (strncmp(prefix, cls, cl) == 0 && prefix[cl] == '.'))) { + const char *r = prefix + cl; + while (*r == '.') { + r++; + } + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = r; + } else if (prefix[0]) { + *full = cbm_arena_sprintf(d->scratch, "%s.%s%s", cls, prefix, name); + *rem = prefix; + } else { + *full = cbm_arena_sprintf(d->scratch, "%s.%s", cls, name); + *rem = ""; + } + } else { + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = prefix; + } + if (!*full) { + d->failed = true; + *full = ""; + } +} + +static const char *rst_opt(const rst_dir_t *dir, const char *key) { + for (int k = 0; k < dir->nopts; k++) { + if (strcmp(dir->opts[k].key, key) == 0) { + return dir->opts[k].val; + } + } + return NULL; +} + +/* H3 _contexts: the module, C namespace and class in force on every line. */ +static void rst_contexts(rst_doc_t *d) { + int n = d->n; + d->mod_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->ns_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->cls_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + if (!d->mod_at || !d->ns_at || !d->cls_at) { + d->failed = true; + return; + } + memset(d->cls_at, 0, (size_t)n * sizeof(char *)); + const char *cm = NULL; + const char *cn = NULL; + int k = 0; + /* directives are in line order (the scan appends them so) */ + for (int di = 0; di <= d->ndirs; di++) { + int upto = di < d->ndirs ? d->dirs[di].line : n; + for (; k < upto; k++) { + d->mod_at[k] = cm; + d->ns_at[k] = cn; + } + if (di == d->ndirs) { + break; + } + const rst_dir_t *dir = &d->dirs[di]; + const char *v = dir->nargs ? dir->args[0] : NULL; + if (strcmp(dir->dom, "py") == 0 && + (strcmp(dir->dname, "module") == 0 || strcmp(dir->dname, "currentmodule") == 0)) { + cm = (!v || !v[0] || strcmp(v, "None") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && + (strcmp(dir->dname, "namespace") == 0 || strcmp(dir->dname, "namespace-push") == 0)) { + cn = (!v || !v[0] || strcmp(v, "NULL") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && strcmp(dir->dname, "namespace-pop") == 0) { + cn = NULL; + } + } + for (int di = 0; di < d->ndirs && !d->failed; di++) { + const rst_dir_t *dir = &d->dirs[di]; + if (strcmp(dir->dom, "py") != 0 || !rst_in(dir->dname, PY_OBJ) || + strcmp(dir->dname, "module") == 0 || dir->cs >= dir->ce) { + continue; + } + const char *outer = d->cls_at[dir->line]; + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + const char *content_cls = outer; + const char *prefix; + const char *name; + if (ns > 0 && rst_py_sig(d, sigs[ns - 1], &prefix, &name)) { + const char *full; + const char *rem; + rst_py_fullname(d, prefix, name, outer, &full, &rem); + if (strcmp(dir->dname, "class") == 0 || strcmp(dir->dname, "exception") == 0) { + content_cls = full; + } else if (rem[0]) { + size_t rl = strlen(rem); + const char *r = rem; + while (*r == '.') { + r++; + rl--; + } + while (rl > 0 && r[rl - 1] == '.') { + rl--; + } + content_cls = cbm_arena_strndup(d->scratch, r, rl); + } + } + const char *mod = rst_opt(dir, "module"); + if (mod) { + for (int j = dir->cs; j < dir->ce; j++) { + d->mod_at[j] = mod; + } + } + for (int j = dir->cs; j < dir->ce; j++) { + d->cls_at[j] = content_cls; + } + } +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +static void rst_tok(rst_doc_t *d, int line, int col, int syntax, const char *raw) { + if (!raw) { + d->failed = true; + return; + } + d->toks = (rst_tok_t *)rst_grow(d, d->toks, d->ntoks, &d->cap_toks, sizeof(rst_tok_t)); + if (!d->toks) { + return; + } + d->toks[d->ntoks++] = (rst_tok_t){.line = line, .col = col, .syntax = syntax, .raw = raw}; +} + +static const char *rst_s(const char *s) { + return s ? s : ""; +} + +/* The written form of a directive: its head line, whitespace collapsed. */ +static const char *rst_written_dir(rst_doc_t *d, const rst_dir_t *dir) { + const rst_line_t *ln = &d->L[dir->line]; + return rst_collapse(d->ctx->arena, ln->s, ln->len); +} + +static bool rst_code_ext(const char *path) { + const char *base = strrchr(path, '/'); + base = base ? base + 1 : path; + if (strncmp(base, "Makefile", strlen("Makefile")) == 0 || + strncmp(base, "Kconfig", strlen("Kconfig")) == 0) { + return true; + } + const char *dot = strrchr(base, '.'); + if (!dot) { + return false; + } + char ext[16]; + size_t el = strlen(dot); + if (el >= sizeof(ext)) { + return false; + } + for (size_t i = 0; i <= el; i++) { + char c = dot[i]; + ext[i] = (char)(c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c); + } + return rst_in(ext, CODE_EXT); +} + +static void rst_directive_tokens(rst_doc_t *d, const rst_dir_t *dir) { + CBMArena *a = d->ctx->arena; + const char *dn = dir->dname; + const char *dom = dir->dom; + bool py_obj = (strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(dn, PY_OBJ); + bool c_obj = (strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0 || strcmp(dom, "py") == 0) && + rst_in(dn, C_OBJ); + if (strcmp(dom, "py") != 0 && strcmp(dom, "c") != 0 && strcmp(dom, "std") != 0) { + py_obj = false; + c_obj = false; + } + if (strcmp(dom, "c") == 0) { + py_obj = false; + } + int line = dir->line; + const char *written = NULL; + if (py_obj || c_obj) { + written = rst_written_dir(d, dir); + const char *mod = rst_opt(dir, "module"); + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + if (ns == 0 && !d->failed) { + sigs = (const char **)cbm_arena_alloc(d->scratch, sizeof(char *)); + if (!sigs) { + d->failed = true; + return; + } + sigs[0] = ""; + ns = 1; + } + bool attr_like = + strcmp(dn, "attribute") == 0 || strcmp(dn, "data") == 0 || strcmp(dn, "property") == 0; + for (int k = 0; k < ns && !d->failed; k++) { + char *sig = rst_collapse(d->scratch, sigs[k], (int)strlen(sigs[k])); + if (!sig) { + d->failed = true; + return; + } + /* Python's reading (H3 py_directive): an attribute's type or + * value is no part of its name; the full name in the class */ + char *py_sig = sig; + if (attr_like) { + size_t cut = strcspn(sig, ":="); + while (cut > 0 && sig[cut - 1] == ' ') { + cut--; + } + py_sig = cbm_arena_strndup(d->scratch, sig, cut); + } + const char *prefix; + const char *name; + const char *full = ""; + const char *rem; + if (py_sig && strcmp(dn, "module") != 0 && rst_py_sig(d, py_sig, &prefix, &name)) { + rst_py_fullname(d, prefix, name, d->cls_at[line], &full, &rem); + } + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_OBJECT, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s\t%s", rst_s(written), dir->name, + sig, rst_s(mod ? mod : d->mod_at[line]), + rst_s(d->cls_at[line]), rst_s(d->ns_at[line]), full)); + } + return; + } + const char *arg0 = dir->nargs ? dir->args[0] : ""; + if (rst_in(dn, AUTODOC)) { + written = rst_written_dir(d, dir); + char *arg = rst_collapse(d->scratch, arg0, (int)strlen(arg0)); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_AUTODOC, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), dn, rst_s(arg), + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]))); + return; + } + if (strcmp(dn, "literalinclude") == 0) { + written = rst_written_dir(d, dir); + const char *lines = rst_opt(dir, "lines"); + const char *pyobject = rst_opt(dir, "pyobject"); + bool bounded = rst_opt(dir, "start-after") || rst_opt(dir, "end-before") || + rst_opt(dir, "start-at") || rst_opt(dir, "end-at"); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_LITERALINCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), arg0, rst_s(lines), + rst_s(pyobject), bounded ? "1" : "")); + return; + } + if (strcmp(dn, "include") == 0 || strcmp(dn, "kernel-include") == 0) { + if (!arg0[0] || arg0[0] == '<' || !rst_code_ext(arg0)) { + return; /* a document or text include: not code */ + } + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_INCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s", rst_s(written), dn, arg0)); + return; + } + if (strcmp(dn, "kernel-doc") == 0) { + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t", rst_s(written), arg0)); + const char *ids = rst_opt(dir, "identifiers"); + if (!ids) { + ids = rst_opt(dir, "functions"); + } + if (!ids || rst_opt(dir, "export") || rst_opt(dir, "internal") || rst_opt(dir, "doc")) { + return; /* the names come from the source file's own kernel-doc comments */ + } + /* each name, a trailing backslash dropped (kerneldoc.py) */ + const char *p = ids; + while (*p && !d->failed) { + while (*p == ' ') { + p++; + } + const char *e = p; + while (*e && *e != ' ') { + e++; + } + size_t nl = (size_t)(e - p); + while (nl > 0 && p[nl - 1] == '\\') { + nl--; + } + if (nl > 0) { + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t%.*s", rst_s(written), arg0, (int)nl, p)); + } + p = e; + } + } +} + +/* ── Inline text: roles and inline literals ──────────────────────── */ + +static bool rst_inline_kind(unsigned char k) { + return k == K_PROSE || k == K_TITLE || k == K_DHEAD_PROSE || k == K_DARG_PROSE || k == K_PLIT; +} + +typedef struct { + char *text; /* the run's lines joined by '\n' */ + char *mask; /* inline literals blanked (H3 mask) */ + int len; + int a; /* first line */ + int *offs; + int nlines; +} rst_run_t; + +static int rst_run_line(const rst_run_t *r, int pos) { + int lo = 0; + int hi = r->nlines; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (r->offs[mid] <= pos) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo - 1; +} + +/* Is the code point before byte i a role's left neighbour that blocks it + * ((? 0 && ((unsigned char)s[k] & 0xC0) == 0x80) { + k--; + } + int n; + uint32_t cp = rst_cp(s, i, k, &n); + return cp == '`' || cp == '\\' || rst_word_cp(cp); +} + +/* [A-Za-z][\w.+-]* at s[i]: its end, or -1. */ +static int rst_role_name(const char *s, int len, int i) { + if (i >= len || !rst_alpha_ascii((unsigned char)s[i])) { + return -1; + } + i++; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (cp != '.' && cp != '+' && cp != '-' && !rst_word_cp(cp)) { + break; + } + i += n; + } + return i; +} + +static bool rst_role_end_ok(const char *s, int len, int i) { + if (i >= len) { + return true; + } + int n; + uint32_t cp = rst_cp(s, len, i, &n); + return cp != '`' && !rst_word_cp(cp); +} + +static void rst_emit_role(rst_doc_t *d, const rst_run_t *r, int start, int role_a, int role_b, + int ca, int cb) { + CBMArena *a = d->ctx->arena; + char *role = cbm_arena_strndup(d->scratch, r->text + role_a, (size_t)(role_b - role_a)); + if (!role) { + d->failed = true; + return; + } + const char *dom; + const char *rname; + rst_split_domain(d, role, PY_ROLES, &dom, &rname); + bool code = ((strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, PY_ROLES)) || + ((strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, C_ROLES)); + char *content = rst_collapse(d->scratch, r->text + ca, cb - ca); + if (!content) { + d->failed = true; + return; + } + if (!code) { + /* an extlink to a repository path (:source:`django/db/x.py`, or with a + * title, :source:`x `): only a role without a domain + * whose target is a path */ + const char *target = content; + size_t tl = strlen(content); + const char *lt = strrchr(content, '<'); + if (tl > 0 && content[tl - 1] == '>' && lt && lt > content) { + target = lt + 1; + tl = (size_t)(content + tl - 1 - target); + } + CBMDocLinkMdPath p; + if (strchr(role, ':') || + !cbm_doclink_md_classify_span(target, tl, d->span_buf, RST_SPAN_MAX + 1, &p) || + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + return; + } + } + int li = rst_run_line(r, start); + int line = r->a + li; + char *written = rst_collapse(a, r->text + start, cb + 1 - start); + rst_tok(d, line, start - r->offs[li], CBM_DOCLINK_RST_ROLE, + written ? cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s", written, role, content, + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]), + rst_s(d->ns_at[line])) + : NULL); +} + +/* ROLE_RE over the masked text. */ +static void rst_roles(rst_doc_t *d, const rst_run_t *r) { + const char *m = r->mask; + int len = r->len; + int i = 0; + while (i < len && !d->failed) { + const char *colon = memchr(m + i, ':', (size_t)(len - i)); + if (!colon) { + break; + } + i = (int)(colon - m); + if (rst_role_blocked(m, i)) { + i++; + continue; + } + int e1 = rst_role_name(m, len, i + 1); + int role_end = -1; + if (e1 > 0 && e1 < len && m[e1] == ':') { + int e2 = rst_role_name(m, len, e1 + 1); + if (e2 > 0 && e2 + 1 < len && m[e2] == ':' && m[e2 + 1] == '`') { + role_end = e2; + } else if (e1 + 1 < len && m[e1 + 1] == '`') { + role_end = e1; + } + } + if (role_end < 0) { + i++; + continue; + } + int open = role_end + 1; + const char *close = memchr(m + open + 1, '`', (size_t)(len - open - 1)); + if (!close || close == m + open + 1 || !rst_role_end_ok(m, len, (int)(close - m) + 1)) { + i++; + continue; + } + int cb = (int)(close - m); + rst_emit_role(d, r, i, i + 1, role_end, open + 1, cb); + i = cb + 1; + } +} + +static void rst_literal_token(rst_doc_t *d, const rst_run_t *r, int i, int a, int b) { + if (b <= a || b - a > RST_SPAN_MAX || memchr(r->text + a, '\n', (size_t)(b - a))) { + return; + } + CBMDocLinkMdPath p; + if (!cbm_doclink_md_classify_span(r->text + a, (size_t)(b - a), d->span_buf, RST_SPAN_MAX + 1, + &p)) { + return; + } + int li = rst_run_line(r, i); + rst_tok(d, r->a + li, i - r->offs[li], + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_RST_CODE_NAME + : CBM_DOCLINK_RST_CODE_PATH, + cbm_arena_strndup(d->ctx->arena, r->text + a, (size_t)(b - a))); +} + +/* INLINE_LIT_RE ``.+?`` (DOTALL): blank them in the mask, and take each as a + * possible code_path / code_name. */ +static void rst_literals(rst_doc_t *d, rst_run_t *r) { + const char *t = r->text; + int len = r->len; + int i = 0; + while (i + 1 < len && !d->failed) { + if (t[i] != '`' || t[i + 1] != '`') { + i++; + continue; + } + int j = i + 3; + while (j + 1 < len && !(t[j] == '`' && t[j + 1] == '`')) { + j++; + } + if (j + 1 >= len) { + i++; + continue; + } + for (int k = i; k < j + 2; k++) { + if (r->mask[k] != '\n') { + r->mask[k] = ' '; + } + } + rst_literal_token(d, r, i, i + 2, j); + i = j + 2; + } +} + +static void rst_inline(rst_doc_t *d) { + int i = 0; + while (i < d->n && !d->failed) { + if (!rst_inline_kind(d->K[i])) { + i++; + continue; + } + int j = i; + int len = 0; + while (j < d->n && rst_inline_kind(d->K[j])) { + len += d->L[j].len + 1; + j++; + } + rst_run_t r = {.a = i, .nlines = j - i}; + r.text = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.mask = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.offs = (int *)cbm_arena_alloc(d->scratch, (size_t)(j - i) * sizeof(int)); + if (!r.text || !r.mask || !r.offs) { + d->failed = true; + return; + } + int pos = 0; + for (int k = i; k < j; k++) { + r.offs[k - i] = pos; + memcpy(r.text + pos, d->L[k].s, (size_t)d->L[k].len); + pos += d->L[k].len; + if (k + 1 < j) { + r.text[pos++] = '\n'; + } + } + r.text[pos] = '\0'; + r.len = pos; + memcpy(r.mask, r.text, (size_t)pos + 1); + rst_literals(d, &r); + rst_roles(d, &r); + i = j; + } +} + +/* ── Sections ────────────────────────────────────────────────────── */ + +/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */ +static const char *rst_qn_segment(CBMArena *a, const char *name) { + char *out = cbm_arena_strdup(a, name); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = false; + for (const char *p = name; *p; p++) { + if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = '-'; + } + in_ws = false; + out[w++] = *p; + } + out[w] = '\0'; + return out; +} + +/* The section's own text (title block to the next title), whitespace + * collapsed, at most RST_BODY_MAX bytes and never a cut character. */ +static const char *rst_body(rst_doc_t *d, int from, int to) { + char *out = (char *)cbm_arena_alloc(d->ctx->arena, RST_BODY_MAX + 1); + if (!out) { + return NULL; + } + int w = 0; + for (int k = from; k < to && w < RST_BODY_MAX; k++) { + const char *s = d->L[k].s; + int len = d->L[k].len; + int i = 0; + while (i < len && w < RST_BODY_MAX) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + if (w + n > RST_BODY_MAX) { + w = RST_BODY_MAX + 1; /* stop: the next character does not fit */ + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + } + if (w > RST_BODY_MAX) { + w = RST_BODY_MAX; + while (w > 0 && ((unsigned char)out[w] & 0xC0) == 0x80) { + w--; + } + } + while (w > 0 && out[w - 1] == ' ') { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +static int rst_title_qn_cmp(const void *a, const void *b) { + const rst_title_t *x = *(const rst_title_t *const *)a; + const rst_title_t *y = *(const rst_title_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static void rst_sections(rst_doc_t *d) { + CBMExtractCtx *ctx = d->ctx; + CBMArena *a = ctx->arena; + for (int t = 0; t < d->ntitles; t++) { + rst_title_t *ti = &d->titles[t]; + const char *seg = rst_qn_segment(a, ti->text); + ti->qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL; + if (!ti->qn) { + d->failed = true; + return; + } + } + if (d->ntitles > 1) { + rst_title_t **by_qn = + (rst_title_t **)cbm_arena_alloc(d->scratch, (size_t)d->ntitles * sizeof(*by_qn)); + if (!by_qn) { + d->failed = true; + return; + } + for (int t = 0; t < d->ntitles; t++) { + by_qn[t] = &d->titles[t]; + } + qsort(by_qn, (size_t)d->ntitles, sizeof(*by_qn), rst_title_qn_cmp); + for (int t = 1; t < d->ntitles; t++) { + if (strcmp(by_qn[t]->qn, by_qn[t - 1]->qn) == 0) { + by_qn[t]->shared_qn = true; + by_qn[t - 1]->shared_qn = true; + } + } + } + for (int t = 0; t < d->ntitles; t++) { + const rst_title_t *ti = &d->titles[t]; + int end = t + 1 < d->ntitles ? d->titles[t + 1].first : d->n; + int last = end - 1; + while (last > ti->idx && d->L[last].blank) { + last--; + } + char *name = cbm_arena_strdup(a, ti->text); + if (!name) { + d->failed = true; + return; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = name; + def.qualified_name = ti->qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = (uint32_t)ti->idx + 1; + def.end_line = (uint32_t)last + 1; + def.is_exported = true; + def.docstring = rst_body(d, ti->next, end); + cbm_defs_push(&ctx->result->defs, a, def); + } +} + +/* ── Driver ──────────────────────────────────────────────────────── */ + +static int rst_tok_cmp(const void *a, const void *b) { + const rst_tok_t *x = (const rst_tok_t *)a; + const rst_tok_t *y = (const rst_tok_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + if (x->col != y->col) { + return x->col < y->col ? -1 : 1; + } + if (x->syntax != y->syntax) { + return x->syntax < y->syntax ? -1 : 1; + } + return strcmp(x->raw, y->raw); +} + +/* The section a 0-based line is written in (by its title block's first line). */ +static const rst_title_t *rst_section_of(const rst_doc_t *d, int line) { + int lo = 0; + int hi = d->ntitles; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (d->titles[mid].first <= line) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo > 0 ? &d->titles[lo - 1] : NULL; +} + +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + rst_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena}; + d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(RST_SPAN_MAX + 1) * 2); + if (!d.span_buf || !rst_lines(&d, ctx->source, ctx->source_len)) { + ctx->result->doc_links.failed = true; + return; + } + rst_scan(&d); + if (!d.failed) { + rst_contexts(&d); + } + if (!d.failed) { + rst_sections(&d); + } + for (int i = 0; i < d.ndirs && !d.failed; i++) { + rst_directive_tokens(&d, &d.dirs[i]); + } + if (!d.failed) { + rst_inline(&d); + } + if (d.failed) { + ctx->result->doc_links.failed = true; + return; + } + qsort(d.toks, (size_t)d.ntoks, sizeof(*d.toks), rst_tok_cmp); + for (int i = 0; i < d.ntoks; i++) { + const rst_tok_t *t = &d.toks[i]; + const rst_title_t *sec = rst_section_of(&d, t->line); + if (sec && sec->shared_qn) { + sec = NULL; /* the file, not the node another title owns */ + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""), + .raw = t->raw, + .line = (uint32_t)t->line + 1, + .def_line = sec ? (uint32_t)sec->idx + 1 : 1, + .syntax = (uint16_t)t->syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link); + } + /* an architecture decision record in reST: its node, facts and supersedes */ + cbm_adr_extract(ctx); +} diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 0621382d7e..3b86f2ac60 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -216,7 +216,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec); static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn, - const CBMLangSpec *spec); + const CBMLangSpec *spec, int fields_from); +static void fence_fields_named_like_methods(CBMExtractCtx *ctx, int methods_from, int fields_from); static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); static void extract_class_methods(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn, const CBMLangSpec *spec); @@ -6649,13 +6650,16 @@ static void emit_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *s } // Extract methods inside the class + int methods_from = ctx->result->defs.count; extract_class_methods(ctx, node, class_qn, spec); // Extract typed struct/class fields (for cross-file LSP type resolution) + int fields_from = ctx->result->defs.count; extract_class_fields(ctx, node, class_qn, spec); + fence_fields_named_like_methods(ctx, methods_from, fields_from); - // Extract class-level variables (field declarations) - extract_class_variables(ctx, node, class_qn, spec); + // Extract class-level variables (field declarations not already Fields) + extract_class_variables(ctx, node, class_qn, spec, fields_from); if (ctx->language == CBM_LANG_PYTHON) { extract_py_field_types(ctx, node, class_qn); @@ -9012,6 +9016,15 @@ static TSNode resolve_field_name_node(TSNode child) { return null_node; } const char *nk = ts_node_type(name_node); + /* Java `private static final String PROP = "v";`: the declarator holds the + * initializer too, so its text named the Field `PROP = "v"`; its name is + * the declarator's own name field */ + if (strcmp(nk, "variable_declarator") == 0) { + TSNode inner = ts_node_child_by_field_name(name_node, TS_FIELD("name")); + if (!ts_node_is_null(inner)) { + return inner; + } + } if (strcmp(nk, "pointer_declarator") == 0 || strcmp(nk, "array_declarator") == 0) { TSNode inner = ts_node_child_by_field_name(name_node, TS_FIELD("declarator")); if (!ts_node_is_null(inner)) { @@ -9404,9 +9417,59 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch } } -// Extract class-level variables (field declarations inside class bodies) +static int cmp_cstr_ptr(const void *pa, const void *pb) { + return strcmp(*(const char *const *)pa, *(const char *const *)pb); +} + +// A field and a method of one class body may share a name (Java `count` and +// `count()`), and the graph keeps one node per qualified name, so one of them +// was lost. The field then takes the qualified name `.#field` +// (the `base#suffix` fence of `#macro` and the Rust cfg twins; the registry +// still files it under its plain name). Only a collision in the same body +// renames, so every other field keeps its name. The body's methods are the +// defs [methods_from, fields_from), its fields [fields_from, count); a sorted +// copy of the method names keeps a large class linear-logarithmic. +static void fence_fields_named_like_methods(CBMExtractCtx *ctx, int methods_from, int fields_from) { + CBMDefinition *defs = ctx->result->defs.items; + int fields_to = ctx->result->defs.count; + if (fields_from >= fields_to || methods_from >= fields_from) { + return; + } + const char **qns = (const char **)cbm_arena_alloc( + ctx->arena, (size_t)(fields_from - methods_from) * sizeof(const char *)); + if (!qns) { + return; + } + size_t n = 0; + for (int i = methods_from; i < fields_from; i++) { + if (defs[i].qualified_name && strcmp(defs[i].label, "Method") == 0) { + qns[n++] = defs[i].qualified_name; + } + } + if (n == 0) { + return; + } + qsort(qns, n, sizeof(qns[0]), cmp_cstr_ptr); + for (int i = fields_from; i < fields_to; i++) { + const char *qn = defs[i].qualified_name; + if (qn && strcmp(defs[i].label, "Field") == 0 && + bsearch(&qn, qns, n, sizeof(qns[0]), cmp_cstr_ptr)) { + char *fenced = cbm_arena_sprintf(ctx->arena, "%s#field", qn); + if (fenced) { + defs[i].qualified_name = fenced; + } + } + } +} + +// Extract class-level variables (field declarations inside class bodies). +// A declaration that extract_class_fields already made a Field (the defs from +// `fields_from` on) is not minted again: Java, C# and Solidity list the same +// kind as field and variable, and the second copy was a Variable with the +// module's qualified name (`demo.query.filters` for field Query.filters), +// which collided with package paths. static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn, - const CBMLangSpec *spec) { + const CBMLangSpec *spec, int fields_from) { if (!spec->variable_node_types || !spec->variable_node_types[0]) { return; } @@ -9416,6 +9479,10 @@ static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const return; } + /* the Fields of this body, in body order: walked alongside the children */ + int fields_to = ctx->result->defs.count; + int fi = fields_from; + /* Record the declaring class on every variable minted from this body (see * push_var_def_qn); saved/restored so module-level minting stays bare. */ const char *saved_parent = ctx->var_parent_class; @@ -9423,9 +9490,19 @@ static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const uint32_t count = ts_node_named_child_count(body); for (uint32_t i = 0; i < count; i++) { TSNode child = ts_node_named_child(body, i); - if (cbm_kind_in_set(child, spec->variable_node_types)) { - extract_var_names(ctx, child, spec); + if (!cbm_kind_in_set(child, spec->variable_node_types)) { + continue; + } + int line = (int)ts_node_start_point(child).row + TS_LINE_OFFSET; + while (fi < fields_to && (strcmp(ctx->result->defs.items[fi].label, "Field") != 0 || + (int)ctx->result->defs.items[fi].start_line < line)) { + fi++; + } + if (fi < fields_to && (int)ctx->result->defs.items[fi].start_line == line && + spec->field_node_types && cbm_kind_in_set(child, spec->field_node_types)) { + continue; /* already a Field of this class */ } + extract_var_names(ctx, child, spec); } ctx->var_parent_class = saved_parent; } diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index cfd363c71e..abca2fdbfd 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -2713,6 +2713,14 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { empty_types, NULL, NULL, tree_sitter_objectscript_routine, NULL}, + // CBM_LANG_PDF — PDF documents. No grammar row: extraction reads the text + // layer itself (cbm_pdf_extract_document, doc_pdf.c) before any grammar is + // looked up, so this language never reaches cbm_lang_spec()/cbm_ts_language(). + // Left as a zero spec. + + // CBM_LANG_ASCIIDOC — AsciiDoc documents. No grammar row, as PDF: extraction + // reads the file itself (cbm_adoc_extract_document, doc_adoc.c). Zero spec. + // CBM_LANG_OBJECTSCRIPT_EXPORT — Studio Export XML. No grammar row: the // pipeline transcodes Export XML to UDL (iris_export_xml.c) and re-extracts // each class as CBM_LANG_OBJECTSCRIPT_UDL, so this language never reaches diff --git a/internal/cbm/pdf/pdf.h b/internal/cbm/pdf/pdf.h new file mode 100644 index 0000000000..599a7b7040 --- /dev/null +++ b/internal/cbm/pdf/pdf.h @@ -0,0 +1,63 @@ +/* + * pdf.h — the text layer of a PDF, page by page. + * + * Reads what a PDF says in text (content-stream text operators decoded through + * the document's fonts) and nothing else: no rendering, no images, no OCR, no + * JavaScript, actions or URIs, nothing fetched, nothing executed. Input is + * untrusted: every read is bounds-checked, every recursion is depth-limited, + * every decoder is guarded against decompression bombs. + * + * Not read: encrypted documents (the result says so). Scanned pages have no + * text layer and come back empty. + */ +#ifndef CBM_PDF_H +#define CBM_PDF_H + +#include "../arena.h" + +#include +#include + +typedef enum { + CBM_PDF_OK = 0, + CBM_PDF_NOT_PDF, /* no "%PDF-" within the first 1024 bytes */ + CBM_PDF_PARSE_FAILURE, /* no page could be found */ + CBM_PDF_ENCRYPTED, /* an Encrypt dictionary: the text is not read */ + CBM_PDF_NOMEM, +} cbm_pdf_status_t; + +typedef struct { + const char *text; /* UTF-8, NUL-terminated, no NUL inside */ + size_t len; + int glyphs; /* glyphs shown on the page */ + int unmapped; /* glyphs without a Unicode value (U+FFFD in the text) */ +} cbm_pdf_page_t; + +/* How the document was read: counts of the repairs and limits that applied. */ +typedef struct { + int xref_reconstructed; /* cross-reference rebuilt from a scan of the file */ + int xref_offset_fixed; /* an xref section found near a wrong offset */ + int stream_length_recovered; /* a stream's /Length was wrong */ + int objstm_nested; /* opening an object stream needed an object of one */ + int xref_streams; /* cross-reference streams decoded */ + int form_rereads_cut; /* a form not read again: re-readings past 256 MiB */ + int flate_recovered; /* a damaged Flate stream decoded in part */ + int stream_guard_hits; /* a decoded stream cut at 512 MiB */ + int filters_unsupported; /* a stream with a filter this reader does not decode */ + int op_errors; /* content operators that could not be applied */ + int page_errors; /* pages whose content stopped early */ +} cbm_pdf_counts_t; + +typedef struct { + cbm_pdf_status_t status; + cbm_pdf_page_t *pages; + int npages; + cbm_pdf_counts_t counts; +} cbm_pdf_result_t; + +/* Read the text layer of data[0..len). Everything the result points to is + * allocated from `out` and lives as long as it. Returns false only when memory + * ran out (status CBM_PDF_NOMEM). */ +bool cbm_pdf_extract(const unsigned char *data, size_t len, CBMArena *out, cbm_pdf_result_t *res); + +#endif /* CBM_PDF_H */ diff --git a/internal/cbm/pdf/pdf_font.c b/internal/cbm/pdf/pdf_font.c new file mode 100644 index 0000000000..d93896155c --- /dev/null +++ b/internal/cbm/pdf/pdf_font.c @@ -0,0 +1,1983 @@ +/* + * pdf_font.c — fonts of the PDF text-layer extractor: what a shown byte + * string says in Unicode, and how far each glyph advances. + * + * Simple fonts map one byte per glyph: ToUnicode first, then the encoding's + * glyph name (a base encoding, /Differences, a Type1 program's built-in + * table), then -- for fonts whose encoding cannot be read -- the byte itself + * when it is printable ASCII. Type0 fonts split bytes by their CMap's code + * space (or two bytes per code), map through ToUnicode, a UCS-2 CMap, or the + * embedded TrueType program's cmap read backwards. + * + * Ranges (CMap bfrange/cidrange, CID widths) are painted once into disjoint + * segments that keep the prototype's precedence (the first range written wins + * a code; the last /W entry wins a CID), so a lookup is a binary search. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" + +#include +#include +#include + +enum { + PDF_CODE_BYTES = 4, /* CMap codes longer than this are ignored */ + PDF_BIG_RANGE = 4096, /* bfranges up to this size are written out */ + PDF_GID_SPACE = 65536, + PDF_UTF8_MAX = 4, + PDF_INLINE_MAX = 15, /* pdf_glyph_out_t.inl capacity, minus the NUL */ + PDF_STD_FIRST = 0x20, + PDF_STD_LAST = 0x7E, + PDF_DEFAULT_WIDTH = 500, + PDF_COURIER_WIDTH = 600, + PDF_TYPE1_CLEAR = 65536, /* the prototype's look into a Type1 program without Length1 */ + PDF_FLAG_SYMBOLIC = 4, + PDF_LIST_MIN = 16, +}; + +#define PDF_SURR_LO 0xD800U +#define PDF_SURR_HI 0xDFFFU +#define PDF_MAX_CP 0x10FFFFU +#define PDF_REPL 0xFFFDU + +/* ── UTF-8 ───────────────────────────────────────────────────────── */ + +/* One code point as UTF-8. U+0000 is written C0 80 so a glyph that maps to it + * still counts as text in the layout (the prototype emitted it); the page + * text drops it at the end. Surrogates become U+FFFD. */ +static int utf8_put(uint32_t cp, char *out) { + if (cp == 0) { + out[0] = (char)0xC0; + out[1] = (char)0x80; + return 2; + } + if ((cp >= PDF_SURR_LO && cp <= PDF_SURR_HI) || cp > PDF_MAX_CP) { + cp = PDF_REPL; + } + if (cp < 0x80) { + out[0] = (char)cp; + return 1; + } + if (cp < 0x800) { + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + if (cp < 0x10000) { + out[0] = (char)(0xE0 | (cp >> 12)); + out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + out[0] = (char)(0xF0 | (cp >> 18)); + out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[3] = (char)(0x80 | (cp & 0x3F)); + return 4; +} + +static char *persist(pdf_doc_t *d, const char *s, size_t n) { + char *o = (char *)cbm_arena_alloc(d->a, n + 1); + if (!o) { + d->nomem = true; + return NULL; + } + memcpy(o, s, n); + o[n] = '\0'; + return o; +} + +/* UTF-16BE bytes as UTF-8 (the prototype's _u16be): one byte is a Latin-1 + * character; an odd tail byte is dropped; a lone surrogate is U+FFFD. Writes + * to buf when it fits, else to the arena. */ +static const char *u16be(pdf_doc_t *d, const unsigned char *b, size_t n, char *buf, size_t cap) { + char tmp[64]; + char *out = tmp; + size_t need = n * 2 + 4; + if (need > sizeof(tmp)) { + out = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, need); + if (!out) { + d->nomem = true; + return NULL; + } + } + size_t k = 0; + if (n == 1) { + k = (size_t)utf8_put(b[0], out); + } else { + n &= ~(size_t)1; + for (size_t i = 0; i < n; i += 2) { + uint32_t u = ((uint32_t)b[i] << 8) | b[i + 1]; + if (u >= PDF_SURR_LO && u <= 0xDBFFU && i + 3 < n) { + uint32_t lo = ((uint32_t)b[i + 2] << 8) | b[i + 3]; + if (lo >= 0xDC00U && lo <= PDF_SURR_HI) { + uint32_t cp = 0x10000U + ((u - PDF_SURR_LO) << 10) + (lo - 0xDC00U); + k += (size_t)utf8_put(cp, out + k); + i += 2; + continue; + } + } + k += (size_t)utf8_put(u, out + k); /* a lone surrogate becomes U+FFFD */ + } + } + const char *res; + if (buf && k < cap) { + memcpy(buf, out, k); + buf[k] = '\0'; + res = buf; + } else { + res = persist(d, out, k); + } + if (out != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, out); + } + return res; +} + +/* ── Glyph names ─────────────────────────────────────────────────── */ + +static const char *glyph_lookup(const unsigned char *name, size_t n) { + int lo = 0; + int hi = PDF_GLYPH_COUNT - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + const char *g = PDF_GLYPHS[mid].name; + size_t gn = strlen(g); + size_t m = gn < n ? gn : n; + int c = m ? memcmp(g, name, m) : 0; + if (!c) { + c = gn < n ? -1 : gn > n ? 1 : 0; + } + if (!c) { + return PDF_GLYPHS[mid].utf8; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return NULL; +} + +static bool all_hex(const unsigned char *s, size_t n) { + for (size_t i = 0; i < n; i++) { + unsigned char c = s[i]; + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) { + return false; + } + } + return true; +} + +static uint32_t hex_num(const unsigned char *s, size_t n) { + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = s[i]; + uint32_t x = c <= '9' ? (uint32_t)(c - '0') : (uint32_t)((c | 0x20) - 'a' + 10); + v = v * 16 + x; + } + return v; +} + +static const char *g2u(pdf_doc_t *d, const unsigned char *name, size_t n, int depth) { + const char *r = glyph_lookup(name, n); + if (r) { + return r; + } + const unsigned char *base = name; + size_t bn = n; + const unsigned char *dot = n > 1 ? (const unsigned char *)memchr(name + 1, '.', n - 1) : NULL; + if (dot) { + const unsigned char *first = (const unsigned char *)memchr(name, '.', n); + bn = (size_t)(first - name); + r = glyph_lookup(base, bn); + if (r) { + return r; + } + if (!bn) { + return NULL; + } + } + if (memchr(base, '_', bn)) { + if (depth > 0) { + return NULL; + } + char tmp[256]; + size_t k = 0; + char *big = NULL; + size_t cap = sizeof(tmp); + char *out = tmp; + size_t i = 0; + for (;;) { + const unsigned char *us = (const unsigned char *)memchr(base + i, '_', bn - i); + size_t pe = us ? (size_t)(us - base) : bn; + const char *part = g2u(d, base + i, pe - i, depth + 1); + if (!part) { + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + return NULL; + } + size_t pl = strlen(part); + if (k + pl + 1 > cap) { + size_t ncap = (k + pl + 1) * 2; + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, big, ncap); + if (!g) { + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + d->nomem = true; + return NULL; + } + if (!big) { + memcpy(g, tmp, k); + } + big = g; + out = big; + cap = ncap; + } + memcpy(out + k, part, pl); + k += pl; + if (!us) { + break; + } + i = pe + 1; + } + const char *res = persist(d, out, k); + cbm_free(CBM_MEM_CLASS_EXTRACT, big); + return res; + } + if (bn >= 7 && memcmp(base, "uni", 3) == 0 && (bn - 3) % 4 == 0 && all_hex(base + 3, bn - 3)) { + size_t units = (bn - 3) / 4; + char *o = (char *)cbm_arena_alloc(d->a, units * PDF_UTF8_MAX + 1); + if (!o) { + d->nomem = true; + return NULL; + } + size_t k = 0; + for (size_t i = 3; i < bn; i += 4) { + uint32_t v = hex_num(base + i, 4); + if (v >= PDF_SURR_LO && v <= PDF_SURR_HI) { + return NULL; + } + k += (size_t)utf8_put(v, o + k); + } + o[k] = '\0'; + return o; + } + if (bn >= 5 && bn <= 7 && base[0] == 'u' && all_hex(base + 1, bn - 1)) { + uint32_t v = hex_num(base + 1, bn - 1); + if (v <= PDF_MAX_CP && !(v >= PDF_SURR_LO && v <= PDF_SURR_HI)) { + char b[PDF_UTF8_MAX]; + int k = utf8_put(v, b); + return persist(d, b, (size_t)k); + } + } + if (bn == 1) { + char b[PDF_UTF8_MAX]; + int k = utf8_put(base[0], b); + return persist(d, b, (size_t)k); + } + return NULL; +} + +const char *pdf_glyph_to_utf8(pdf_doc_t *d, const unsigned char *name, size_t n) { + return name ? g2u(d, name, n, 0) : NULL; +} + +/* ── Interval painting ───────────────────────────────────────────── */ + +typedef struct { + uint64_t lo; + uint64_t hi; + int idx; /* written order */ +} pdf_iv_t; + +typedef struct { + uint64_t lo; + uint64_t hi; + int idx; +} pdf_seg_t; + +typedef struct { + pdf_iv_t *v; + int n; + int cap; +} iv_list_t; + +static bool ivl_push(pdf_doc_t *d, iv_list_t *l, uint64_t lo, uint64_t hi, int idx) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_LIST_MIN; + pdf_iv_t *g = + (pdf_iv_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].lo = lo; + l->v[l->n].hi = hi; + l->v[l->n].idx = idx; + l->n++; + return true; +} + +static int iv_lo_cmp(const void *x, const void *y) { + const pdf_iv_t *a = (const pdf_iv_t *)x; + const pdf_iv_t *b = (const pdf_iv_t *)y; + if (a->lo != b->lo) { + return a->lo < b->lo ? -1 : 1; + } + return a->idx < b->idx ? -1 : a->idx > b->idx ? 1 : 0; +} + +static int u64_cmp(const void *x, const void *y) { + uint64_t a = *(const uint64_t *)x; + uint64_t b = *(const uint64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +/* A binary heap of interval positions, ordered by written order (smallest + * first for first-wins, largest first for last-wins). */ +typedef struct { + int *h; + int n; + const pdf_iv_t *iv; + bool first_wins; +} iv_heap_t; + +static bool heap_before(const iv_heap_t *hp, int a, int b) { + return hp->first_wins ? hp->iv[a].idx < hp->iv[b].idx : hp->iv[a].idx > hp->iv[b].idx; +} + +static void heap_push(iv_heap_t *hp, int x) { + int i = hp->n++; + hp->h[i] = x; + while (i > 0) { + int p = (i - 1) / 2; + if (!heap_before(hp, hp->h[i], hp->h[p])) { + break; + } + int t = hp->h[i]; + hp->h[i] = hp->h[p]; + hp->h[p] = t; + i = p; + } +} + +static void heap_pop(iv_heap_t *hp) { + hp->h[0] = hp->h[--hp->n]; + int i = 0; + for (;;) { + int l = 2 * i + 1; + int r = l + 1; + int m = i; + if (l < hp->n && heap_before(hp, hp->h[l], hp->h[m])) { + m = l; + } + if (r < hp->n && heap_before(hp, hp->h[r], hp->h[m])) { + m = r; + } + if (m == i) { + break; + } + int t = hp->h[i]; + hp->h[i] = hp->h[m]; + hp->h[m] = t; + i = m; + } +} + +/* Disjoint segments, ascending, each owned by the winning interval. The list + * is consumed (sorted). Intervals must have hi < UINT64_MAX. */ +static pdf_seg_t *paint(pdf_doc_t *d, iv_list_t *l, bool first_wins, int *nout) { + *nout = 0; + if (!l->n) { + return NULL; + } + int n = l->n; + qsort(l->v, (size_t)n, sizeof(*l->v), iv_lo_cmp); + uint64_t *pts = (uint64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)n * 2 * sizeof(uint64_t)); + int *heap = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)n * sizeof(int)); + pdf_seg_t *seg = (pdf_seg_t *)pdf_big(d, (size_t)n * 2 * sizeof(pdf_seg_t)); + if (!pts || !heap || !seg) { + cbm_free(CBM_MEM_CLASS_EXTRACT, pts); + cbm_free(CBM_MEM_CLASS_EXTRACT, heap); + d->nomem = true; + return NULL; + } + int np = 0; + for (int i = 0; i < n; i++) { + pts[np++] = l->v[i].lo; + pts[np++] = l->v[i].hi + 1; + } + qsort(pts, (size_t)np, sizeof(uint64_t), u64_cmp); + iv_heap_t hp = {heap, 0, l->v, first_wins}; + int next = 0; + int ns = 0; + for (int k = 0; k < np; k++) { + if (k > 0 && pts[k] == pts[k - 1]) { + continue; + } + uint64_t x = pts[k]; + while (next < n && l->v[next].lo <= x) { + heap_push(&hp, next++); + } + while (hp.n && l->v[hp.h[0]].hi < x) { + heap_pop(&hp); + } + if (!hp.n) { + continue; + } + int k2 = k + 1; + while (k2 < np && pts[k2] == x) { + k2++; + } + if (k2 >= np) { + break; + } + uint64_t end = pts[k2] - 1; + int w = l->v[hp.h[0]].idx; + if (ns && seg[ns - 1].idx == w && seg[ns - 1].hi + 1 == x) { + seg[ns - 1].hi = end; + } else { + seg[ns].lo = x; + seg[ns].hi = end; + seg[ns].idx = w; + ns++; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, pts); + cbm_free(CBM_MEM_CLASS_EXTRACT, heap); + *nout = ns; + return seg; +} + +static int seg_find(const pdf_seg_t *s, int n, uint64_t key) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (s[mid].hi < key) { + lo = mid + 1; + } else if (s[mid].lo > key) { + hi = mid - 1; + } else { + return s[mid].idx; + } + } + return -1; +} + +/* ── CMaps ───────────────────────────────────────────────────────── */ + +typedef struct { + uint64_t key; /* bytes << 32 | code */ + const char *utf8; + int64_t cid; + int seq; +} cm_ent_t; + +typedef struct { + uint64_t lo; + uint64_t hi; + const unsigned char *prefix; + size_t plen; + uint32_t last; +} cm_urange_t; + +typedef struct { + uint64_t lo; + int64_t cid0; +} cm_crange_t; + +typedef struct { + cm_ent_t *uni; + int nuni; + cm_urange_t *ur; /* big bfranges in written order */ + pdf_seg_t *urseg; + int nurseg; + cm_ent_t *cid; + int ncid; + cm_crange_t *cr; /* cidranges in written order */ + pdf_seg_t *crseg; + int ncrseg; + pdf_seg_t *space[PDF_CODE_BYTES + 1]; /* code space per byte count (merged) */ + int nspace[PDF_CODE_BYTES + 1]; + bool has_space; +} pdf_cmap_t; + +typedef struct { + cm_ent_t *v; + int n; + int cap; +} ent_list_t; + +static bool el_push(pdf_doc_t *d, ent_list_t *l, cm_ent_t e) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_LIST_MIN; + cm_ent_t *g = + (cm_ent_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + e.seq = l->n; + l->v[l->n++] = e; + return true; +} + +static int ent_cmp(const void *x, const void *y) { + const cm_ent_t *a = (const cm_ent_t *)x; + const cm_ent_t *b = (const cm_ent_t *)y; + if (a->key != b->key) { + return a->key < b->key ? -1 : 1; + } + return a->seq < b->seq ? -1 : a->seq > b->seq ? 1 : 0; +} + +/* Sorted, one entry per key: the last written. */ +static cm_ent_t *el_finish(pdf_doc_t *d, ent_list_t *l, int *nout) { + *nout = 0; + if (!l->n) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return NULL; + } + qsort(l->v, (size_t)l->n, sizeof(*l->v), ent_cmp); + cm_ent_t *o = (cm_ent_t *)pdf_big(d, (size_t)l->n * sizeof(*o)); + int k = 0; + if (o) { + for (int i = 0; i < l->n; i++) { + if (i + 1 < l->n && l->v[i + 1].key == l->v[i].key) { + continue; + } + o[k++] = l->v[i]; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + *nout = k; + return o; +} + +static const cm_ent_t *ent_find(const cm_ent_t *v, int n, uint64_t key) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (v[mid].key < key) { + lo = mid + 1; + } else if (v[mid].key > key) { + hi = mid - 1; + } else { + return &v[mid]; + } + } + return NULL; +} + +static uint32_t be_code(const unsigned char *s, size_t n) { + uint32_t v = 0; + for (size_t i = 0; i < n; i++) { + v = (v << 8) | s[i]; + } + return v; +} + +static uint64_t cm_key(size_t n, uint32_t code) { + return ((uint64_t)n << 32) | code; +} + +typedef struct { + ent_list_t uni; + ent_list_t cid; + cm_urange_t *ur; + int nur; + int cur_cap; + iv_list_t ur_iv; + cm_crange_t *cr; + int ncr; + int ccap; + iv_list_t cr_iv; + iv_list_t space[PDF_CODE_BYTES + 1]; + bool any_space; /* the prototype tested its code-space list for being non-empty */ +} cm_build_t; + +static bool is_code_str(const pdf_val_t *v) { + return v && v->kind == PV_STR && v->n >= 1 && v->n <= PDF_CODE_BYTES; +} + +static bool is_int_val(const pdf_val_t *v) { + return v && v->kind == PV_NUM && v->is_int; +} + +static void cm_flush(pdf_doc_t *d, cm_build_t *b, const char *state, pdf_val_t **buf, int n) { + if (strcmp(state, "codespacerange") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + if (lo && lo->kind == PV_STR && lo->n && hi && hi->kind == PV_STR) { + b->any_space = true; + } + if (is_code_str(lo) && hi && hi->kind == PV_STR) { + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (z >= a) { + ivl_push(d, &b->space[m], a, z, b->space[m].n); + } + } + } + } else if (strcmp(state, "bfchar") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *s = buf[i]; + const pdf_val_t *dst = buf[i + 1]; + if (!is_code_str(s)) { + continue; + } + cm_ent_t e = {cm_key(s->n, be_code(s->s, s->n)), NULL, 0, 0}; + if (dst && dst->kind == PV_STR) { + e.utf8 = u16be(d, dst->s, dst->n, NULL, 0); + } else if (dst && dst->kind == PV_NAME) { + e.utf8 = g2u(d, dst->s, dst->n, 0); + if (!e.utf8) { + continue; + } + } else if (dst && dst->kind == PV_NUM && dst->is_int) { + char tmp[PDF_UTF8_MAX]; + uint32_t cp = (dst->inum >= 0 && dst->inum <= (int64_t)PDF_MAX_CP) + ? (uint32_t)dst->inum + : PDF_REPL; + int k = utf8_put(cp, tmp); + e.utf8 = persist(d, tmp, (size_t)k); + } else { + continue; + } + if (e.utf8) { + el_push(d, &b->uni, e); + } + } + } else if (strcmp(state, "bfrange") == 0) { + for (int i = 0; i + 2 < n; i += 3) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + const pdf_val_t *dst = buf[i + 2]; + if (!is_code_str(lo) || !hi || hi->kind != PV_STR) { + continue; + } + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (z < a) { + continue; + } + if (dst && dst->kind == PV_ARR) { + uint64_t cnt = z - a + 1; + for (int j = 0; j < dst->count && (uint64_t)j < cnt; j++) { + const pdf_val_t *x = dst->items[j]; + if (x && x->kind == PV_STR) { + cm_ent_t e = {cm_key(m, (uint32_t)(a + (uint64_t)j)), NULL, 0, 0}; + e.utf8 = u16be(d, x->s, x->n, NULL, 0); + if (e.utf8) { + el_push(d, &b->uni, e); + } + } + } + } else if (dst && dst->kind == PV_STR && dst->n) { + unsigned char two[2] = {0, dst->s[0]}; + const unsigned char *db = dst->s; + size_t dn = dst->n; + if (dn == 1) { + db = two; + dn = 2; + } + uint32_t last = ((uint32_t)db[dn - 2] << 8) | db[dn - 1]; + size_t plen = dn - 2; + if (z - a <= PDF_BIG_RANGE) { + unsigned char tmp[64]; + unsigned char *w = + plen + 2 <= sizeof(tmp) + ? tmp + : (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, plen + 2); + if (!w) { + d->nomem = true; + return; + } + memcpy(w, db, plen); + for (uint64_t j = 0; j <= z - a; j++) { + uint32_t u = (last + (uint32_t)j) & 0xFFFFU; + w[plen] = (unsigned char)(u >> 8); + w[plen + 1] = (unsigned char)u; + cm_ent_t e = {cm_key(m, (uint32_t)(a + j)), NULL, 0, 0}; + e.utf8 = u16be(d, w, plen + 2, NULL, 0); + if (!e.utf8 || !el_push(d, &b->uni, e)) { + break; + } + } + if (w != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, w); + } + } else { + if (b->nur == b->cur_cap) { + int ncap = b->cur_cap ? b->cur_cap * 2 : PDF_LIST_MIN; + cm_urange_t *g = (cm_urange_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->ur, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + b->ur = g; + b->cur_cap = ncap; + } + cm_urange_t *r = &b->ur[b->nur]; + r->lo = cm_key(m, (uint32_t)a); + r->hi = cm_key(m, (uint32_t)z); + /* the token lives in the parse arena: keep a copy */ + r->prefix = (const unsigned char *)persist(d, (const char *)db, plen); + if (!r->prefix) { + return; + } + r->plen = plen; + r->last = last; + ivl_push(d, &b->ur_iv, r->lo, r->hi, b->nur); + b->nur++; + } + } + } + } else if (strcmp(state, "cidrange") == 0 || strcmp(state, "notdefrange") == 0) { + for (int i = 0; i + 2 < n; i += 3) { + const pdf_val_t *lo = buf[i]; + const pdf_val_t *hi = buf[i + 1]; + const pdf_val_t *c = buf[i + 2]; + if (!is_code_str(lo) || !hi || hi->kind != PV_STR || !is_int_val(c)) { + continue; + } + size_t m = lo->n; + uint64_t a = be_code(lo->s, m); + uint64_t z = hi->n <= PDF_CODE_BYTES ? be_code(hi->s, hi->n) : UINT32_MAX; + if (b->ncr == b->ccap) { + int ncap = b->ccap ? b->ccap * 2 : PDF_LIST_MIN; + cm_crange_t *g = (cm_crange_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->cr, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + b->cr = g; + b->ccap = ncap; + } + b->cr[b->ncr].lo = cm_key(m, (uint32_t)a); + b->cr[b->ncr].cid0 = c->inum; + if (z >= a) { + ivl_push(d, &b->cr_iv, cm_key(m, (uint32_t)a), cm_key(m, (uint32_t)z), b->ncr); + } + b->ncr++; + } + } else if (strcmp(state, "cidchar") == 0) { + for (int i = 0; i + 1 < n; i += 2) { + const pdf_val_t *s = buf[i]; + const pdf_val_t *c = buf[i + 1]; + if (is_code_str(s) && is_int_val(c)) { + cm_ent_t e = {cm_key(s->n, be_code(s->s, s->n)), NULL, c->inum, 0}; + el_push(d, &b->cid, e); + } + } + } +} + +static const char *const CM_BEGIN[] = {"begincodespacerange", "beginbfchar", "beginbfrange", + "begincidrange", "begincidchar", "beginnotdefrange"}; + +static pdf_cmap_t *cmap_parse(pdf_doc_t *d, const unsigned char *data, size_t len) { + if (!d->scratch_arena_live) { + cbm_arena_init_lazy(&d->scratch_arena, CBM_ARENA_DEFAULT_BLOCK_SIZE); + d->scratch_arena_live = true; + } + CBMArena *tmp = &d->scratch_arena; + CBMArena *saved = d->cur; + d->cur = tmp; + cm_build_t b; + memset(&b, 0, sizeof(b)); + pdf_val_t **buf = NULL; + int nbuf = 0; + int cap = 0; + char state[32] = ""; + bool in_state = false; + bool failed = false; + size_t pos = 0; + for (;;) { + pdf_tok_t t; + pdf_lex(d, data, len, &pos, &t); + if (t.kind == PT_EOF || d->nomem) { + break; + } + if (t.kind == PT_KW) { + bool begun = false; + for (size_t i = 0; i < sizeof(CM_BEGIN) / sizeof(CM_BEGIN[0]); i++) { + if (pdf_kw_is(&t, CM_BEGIN[i])) { + snprintf(state, sizeof(state), "%s", CM_BEGIN[i] + 5); + in_state = true; + nbuf = 0; + begun = true; + break; + } + } + if (!begun && in_state && t.n >= 3 && memcmp(t.s, "end", 3) == 0) { + d->cur = saved; + cm_flush(d, &b, state, buf, nbuf); + d->cur = tmp; + in_state = false; + nbuf = 0; + cbm_arena_rewind(tmp); + } + continue; + } + if (!in_state) { + continue; + } + pdf_val_t *v = NULL; + if (t.kind == PT_STR || t.kind == PT_NUM || t.kind == PT_NAME) { + v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = t.kind == PT_STR ? PV_STR : t.kind == PT_NUM ? PV_NUM : PV_NAME; + v->is_int = t.is_int; + v->num = t.num; + v->inum = t.inum; + v->s = t.s; + v->n = t.n; + } + } else if (t.kind == PT_AS) { + v = pdf_parse_content_array(d, data, len, &pos, 0); + if (!v) { + failed = true; + break; + } + } else { + continue; + } + if (!v) { + break; + } + if (nbuf == cap) { + int ncap = cap ? cap * 2 : PDF_LIST_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)buf, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + break; + } + buf = g; + cap = ncap; + } + buf[nbuf++] = v; + } + d->cur = saved; + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)buf); + cbm_arena_rewind(tmp); + pdf_cmap_t *cm = NULL; + if (!failed && !d->nomem) { + cm = (pdf_cmap_t *)cbm_arena_calloc(d->a, sizeof(*cm)); + } + if (cm) { + cm->uni = el_finish(d, &b.uni, &cm->nuni); + cm->cid = el_finish(d, &b.cid, &cm->ncid); + b.uni.v = NULL; + b.cid.v = NULL; + if (b.nur) { + cm->ur = (cm_urange_t *)pdf_big(d, (size_t)b.nur * sizeof(*cm->ur)); + if (cm->ur) { + memcpy(cm->ur, b.ur, (size_t)b.nur * sizeof(*cm->ur)); + } + cm->urseg = paint(d, &b.ur_iv, true, &cm->nurseg); + } + if (b.ncr) { + cm->cr = (cm_crange_t *)pdf_big(d, (size_t)b.ncr * sizeof(*cm->cr)); + if (cm->cr) { + memcpy(cm->cr, b.cr, (size_t)b.ncr * sizeof(*cm->cr)); + } + cm->crseg = paint(d, &b.cr_iv, true, &cm->ncrseg); + } + for (int m = 1; m <= PDF_CODE_BYTES; m++) { + if (b.space[m].n) { + cm->space[m] = paint(d, &b.space[m], true, &cm->nspace[m]); + } + } + cm->has_space = b.any_space; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, b.uni.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cid.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.ur); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.ur_iv.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cr); + cbm_free(CBM_MEM_CLASS_EXTRACT, b.cr_iv.v); + for (int m = 0; m <= PDF_CODE_BYTES; m++) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b.space[m].v); + } + return d->nomem ? NULL : cm; +} + +/* ToUnicode: an exact entry, else the first big range holding the code. */ +static const char *cmap_lookup(pdf_doc_t *d, const pdf_cmap_t *cm, size_t n, uint32_t code, + char *buf, size_t cap) { + uint64_t key = cm_key(n, code); + const cm_ent_t *e = ent_find(cm->uni, cm->nuni, key); + if (e) { + return e->utf8; + } + int r = seg_find(cm->urseg, cm->nurseg, key); + if (r < 0) { + return NULL; + } + const cm_urange_t *u = &cm->ur[r]; + uint32_t v = (u->last + (uint32_t)(key - u->lo)) & 0xFFFFU; + unsigned char tmp[64]; + unsigned char *w = u->plen + 2 <= sizeof(tmp) + ? tmp + : (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, u->plen + 2); + if (!w) { + d->nomem = true; + return NULL; + } + memcpy(w, u->prefix, u->plen); + w[u->plen] = (unsigned char)(v >> 8); + w[u->plen + 1] = (unsigned char)v; + const char *res = u16be(d, w, u->plen + 2, buf, cap); + if (w != tmp) { + cbm_free(CBM_MEM_CLASS_EXTRACT, w); + } + return res; +} + +static bool cmap_to_cid(const pdf_cmap_t *cm, size_t n, uint32_t code, int64_t *cid) { + uint64_t key = cm_key(n, code); + const cm_ent_t *e = ent_find(cm->cid, cm->ncid, key); + if (e) { + *cid = e->cid; + return true; + } + int r = seg_find(cm->crseg, cm->ncrseg, key); + if (r < 0) { + return false; + } + *cid = pdf_sat_add(cm->cr[r].cid0, (int64_t)(key - cm->cr[r].lo)); + return true; +} + +/* ── TrueType cmap, read backwards ───────────────────────────────── */ + +static bool rd16(const unsigned char *p, size_t len, size_t at, uint32_t *v) { + if (at > len || len - at < 2) { + return false; + } + *v = ((uint32_t)p[at] << 8) | p[at + 1]; + return true; +} + +static bool rd32(const unsigned char *p, size_t len, size_t at, uint32_t *v) { + if (at > len || len - at < 4) { + return false; + } + *v = ((uint32_t)p[at] << 24) | ((uint32_t)p[at + 1] << 16) | ((uint32_t)p[at + 2] << 8) | + p[at + 3]; + return true; +} + +/* "next gid not yet assigned, from x" with path halving; nxt has 65537 slots */ +static uint32_t dsu_next(uint32_t *nxt, uint32_t x) { + while (nxt[x] != x) { + nxt[x] = nxt[nxt[x]]; + x = nxt[x]; + } + return x; +} + +static void dsu_take(uint32_t *nxt, uint32_t x) { + nxt[x] = x + 1; +} + +typedef struct { + uint32_t pe; + uint64_t off; + int rank; + int order; +} ttf_sub_t; + +static int ttf_sub_cmp(const void *x, const void *y) { + const ttf_sub_t *a = (const ttf_sub_t *)x; + const ttf_sub_t *b = (const ttf_sub_t *)y; + if (a->rank != b->rank) { + return a->rank < b->rank ? -1 : 1; + } + return a->order < b->order ? -1 : 1; +} + +/* Fill rev[gid] = code point + 1 from one subtable; false: a read past the end + * (the prototype then had no map at all). */ +static bool ttf_fmt4(const unsigned char *p, size_t len, size_t so, uint32_t *rev, uint32_t *nxt, + bool *any) { + uint32_t segx2; + if (!rd16(p, len, so + 6, &segx2)) { + return false; + } + uint32_t seg = segx2 / 2; + size_t ends = so + 14; + size_t starts = so + 16 + segx2; + size_t deltas = so + 16 + 2 * (size_t)segx2; + size_t ro_pos = so + 16 + 3 * (size_t)segx2; + /* the prototype unpacked each array from a slice of segx2 bytes: an odd + * count or a short slice failed the whole map */ + if (seg && ((segx2 & 1U) || ro_pos + segx2 > len)) { + return false; + } + for (uint32_t s = 0; s < seg; s++) { + uint32_t end; + uint32_t start; + uint32_t delta; + uint32_t ro; + /* in range by the check above; a failed read fails the map like the + * prototype's short slice (and gcc -O2 sees every value set) */ + if (!rd16(p, len, ends + 2 * (size_t)s, &end) || + !rd16(p, len, starts + 2 * (size_t)s, &start) || + !rd16(p, len, deltas + 2 * (size_t)s, &delta) || + !rd16(p, len, ro_pos + 2 * (size_t)s, &ro)) { + return false; + } + uint32_t last = end < 0xFFFEU ? end : 0xFFFEU; + if (start > last) { + continue; + } + if (ro == 0) { + /* gid = (c + delta) & 0xFFFF walks gids upward; assign only those + * still free (the first assignment wins) */ + uint32_t g0 = (start + delta) & 0xFFFFU; + uint32_t span = last - start + 1; + uint32_t done = 0; + while (done < span) { + uint32_t g = (g0 + done) & 0xFFFFU; + uint32_t piece = + (PDF_GID_SPACE - g) < (span - done) ? (PDF_GID_SPACE - g) : (span - done); + uint32_t x = dsu_next(nxt, g); + while (x < g + piece) { + if (x != 0) { + rev[x] = start + done + (x - g) + 1; + *any = true; + } + dsu_take(nxt, x); + x = dsu_next(nxt, x); + } + done += piece; + } + } else { + for (uint32_t c = start; c <= last; c++) { + size_t a = ro_pos + 2 * (size_t)s + ro + 2 * (size_t)(c - start); + uint32_t g = 0; + if (a + 2 <= len) { + g = ((uint32_t)p[a] << 8) | p[a + 1]; + } + if (g) { + g = (g + delta) & 0xFFFFU; + } + if (g && !rev[g] && nxt[g] == g) { + rev[g] = c + 1; + dsu_take(nxt, g); + *any = true; + } + } + } + } + return true; +} + +static bool ttf_fmt12(const unsigned char *p, size_t len, size_t so, uint32_t *rev, uint32_t *nxt, + bool *any) { + uint32_t ng; + if (!rd32(p, len, so + 12, &ng)) { + return false; + } + if ((uint64_t)so + 16 + (uint64_t)ng * 12 > len) { + return false; + } + for (uint32_t i = 0; i < ng; i++) { + uint32_t a; + uint32_t b; + uint32_t g0; + size_t at = so + 16 + 12 * (size_t)i; + /* in range by the check above (gcc -O2 sees every value set) */ + if (!rd32(p, len, at, &a) || !rd32(p, len, at + 4, &b) || !rd32(p, len, at + 8, &g0)) { + return false; + } + uint64_t cmax = (uint64_t)a + PDF_GID_SPACE; + if ((uint64_t)b < cmax) { + cmax = b; + } + if (cmax > PDF_MAX_CP) { + cmax = PDF_MAX_CP; + } + if ((uint64_t)a > cmax || g0 >= PDF_GID_SPACE) { + continue; + } + uint64_t glast = (uint64_t)g0 + (cmax - a); + if (glast >= PDF_GID_SPACE) { + glast = PDF_GID_SPACE - 1; + } + uint32_t x = dsu_next(nxt, g0); + while (x <= glast) { + rev[x] = a + (x - g0) + 1; + *any = true; + dsu_take(nxt, x); + x = dsu_next(nxt, x); + } + } + return true; +} + +/* The prototype's ttf_reverse_cmap: gid -> code point + 1 (0: none), or NULL. */ +static uint32_t *ttf_reverse(pdf_doc_t *d, const unsigned char *p, size_t len) { + uint32_t num; + if (!rd16(p, len, 4, &num)) { + return NULL; + } + size_t off = 0; + bool found = false; + for (uint32_t i = 0; i < num; i++) { + size_t r = 12 + 16 * (size_t)i; + if (r + 4 <= len && memcmp(p + r, "cmap", 4) == 0) { + uint32_t o; + if (!rd32(p, len, r + 8, &o)) { + return NULL; + } + off = o; + found = true; + break; + } + if (r >= len) { + break; + } + } + if (!found) { + return NULL; + } + uint32_t n; + if (!rd16(p, len, off + 2, &n)) { + return NULL; + } + ttf_sub_t *subs = + (ttf_sub_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1) * sizeof(*subs)); + uint32_t *rev = (uint32_t *)pdf_big(d, PDF_GID_SPACE * sizeof(uint32_t)); + uint32_t *nxt = + (uint32_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (PDF_GID_SPACE + 1) * sizeof(uint32_t)); + if (!subs || !rev || !nxt) { + cbm_free(CBM_MEM_CLASS_EXTRACT, subs); + cbm_free(CBM_MEM_CLASS_EXTRACT, nxt); + d->nomem = true; + return NULL; + } + bool bad = false; + for (uint32_t i = 0; i < n; i++) { + uint32_t pl; + uint32_t en; + uint32_t o; + size_t at = off + 4 + 8 * (size_t)i; + if (!rd16(p, len, at, &pl) || !rd16(p, len, at + 2, &en) || !rd32(p, len, at + 4, &o)) { + bad = true; + break; + } + subs[i].pe = (pl << 16) | en; + subs[i].off = (uint64_t)off + o; + subs[i].order = (int)i; + uint32_t pe = subs[i].pe; + subs[i].rank = pe == 0x0003000AU ? 0 + : pe == 0x00030001U ? 1 + : pe == 0x00000004U ? 2 + : pe == 0x00000003U ? 3 + : 9; + } + uint32_t *res = NULL; + if (!bad) { + qsort(subs, n, sizeof(*subs), ttf_sub_cmp); + for (uint32_t i = 0; i < n && !res; i++) { + uint32_t pe = subs[i].pe; + if (!(pe == 0x0003000AU || pe == 0x00030001U || pe == 0x00000004U || + pe == 0x00000003U || pe == 0x00000006U || pe == 0x00000000U || + pe == 0x00000001U)) { + continue; + } + if (subs[i].off > len) { + break; + } + size_t so = (size_t)subs[i].off; + uint32_t fmt; + if (!rd16(p, len, so, &fmt)) { + break; + } + memset(rev, 0, PDF_GID_SPACE * sizeof(uint32_t)); + for (uint32_t g = 0; g <= PDF_GID_SPACE; g++) { + nxt[g] = g; + } + bool any = false; + bool ok = true; + if (fmt == 4) { + ok = ttf_fmt4(p, len, so, rev, nxt, &any); + } else if (fmt == 12) { + ok = ttf_fmt12(p, len, so, rev, nxt, &any); + } + if (!ok) { + break; + } + if (any) { + res = rev; + } + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, subs); + cbm_free(CBM_MEM_CLASS_EXTRACT, nxt); + return res; +} + +/* ── Fonts ───────────────────────────────────────────────────────── */ + +typedef enum { + BASE_NONE = 0, + BASE_TABLE, /* a base encoding, code -> UTF-8 */ + BASE_NAMES, /* a Type1 program's built-in table, code -> glyph name */ +} pdf_base_kind_t; + +struct pdf_font { + bool multibyte; + bool type3; + double em; + double wscale; + pdf_cmap_t *tounicode; + /* simple fonts */ + const char *map1[256]; + double widths[256]; + bool has_width[256]; + double missing; + bool has_missing; + bool std14; + const unsigned short *std14_tab; /* NULL with std14: Courier */ + /* Type0 */ + pdf_cmap_t *enc; + bool ucs2; + bool cid_type2; + double dw; + pdf_seg_t *wseg; + int nwseg; + double *wval; + pdf_val_t *fdesc; + pdf_val_t *c2g_val; + bool rev_done; + uint32_t *rev; + const unsigned char *c2g; + size_t c2g_len; +}; + +static const char ASCII1[128][2] = { +#define A(c) {(char)(c), 0} + A(0), A(1), A(2), A(3), A(4), A(5), A(6), A(7), A(8), A(9), A(10), A(11), + A(12), A(13), A(14), A(15), A(16), A(17), A(18), A(19), A(20), A(21), A(22), A(23), + A(24), A(25), A(26), A(27), A(28), A(29), A(30), A(31), A(32), A(33), A(34), A(35), + A(36), A(37), A(38), A(39), A(40), A(41), A(42), A(43), A(44), A(45), A(46), A(47), + A(48), A(49), A(50), A(51), A(52), A(53), A(54), A(55), A(56), A(57), A(58), A(59), + A(60), A(61), A(62), A(63), A(64), A(65), A(66), A(67), A(68), A(69), A(70), A(71), + A(72), A(73), A(74), A(75), A(76), A(77), A(78), A(79), A(80), A(81), A(82), A(83), + A(84), A(85), A(86), A(87), A(88), A(89), A(90), A(91), A(92), A(93), A(94), A(95), + A(96), A(97), A(98), A(99), A(100), A(101), A(102), A(103), A(104), A(105), A(106), A(107), + A(108), A(109), A(110), A(111), A(112), A(113), A(114), A(115), A(116), A(117), A(118), A(119), + A(120), A(121), A(122), A(123), A(124), A(125), A(126), A(127), +#undef A +}; + +static bool name_eq(const pdf_val_t *v, const char *s) { + return pdf_is_name(v, s); +} + +static bool bytes_starts(const unsigned char *s, size_t n, const char *pre) { + size_t m = strlen(pre); + return n >= m && memcmp(s, pre, m) == 0; +} + +static bool std14_name(const unsigned char *s, size_t n) { + for (int i = 0; PDF_STD14[i]; i++) { + size_t m = strlen(PDF_STD14[i]); + if (m == n && memcmp(PDF_STD14[i], s, n) == 0) { + return true; + } + } + return false; +} + +static bool py_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool py_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +/* A name character of the prototype's "dup N /name put" pattern. */ +static bool t1_name_char(unsigned char c) { + return !py_space(c) && c != '/' && c != '[' && c != ']' && c != '{' && c != '}' && c != '(' && + c != ')' && c != '<' && c != '>'; +} + +/* The built-in encoding of a Type1 program: names per code (NULL: none), or + * STANDARD (*standard), or neither (false). */ +static bool type1_builtin(pdf_doc_t *d, pdf_val_t *ff, const unsigned char **names, size_t *nlen, + bool *standard) { + *standard = false; + if (!ff || ff->kind != PV_STREAM) { + return false; + } + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, ff->stream, &len); + if (!data || !len) { + return false; + } + pdf_val_t *l1 = pdf_resolve(d, pdf_dget_raw(ff->stream->dict, "Length1")); + size_t clear = len < PDF_TYPE1_CLEAR ? len : PDF_TYPE1_CLEAR; + if (l1 && l1->kind == PV_NUM && l1->is_int && l1->inum > 0 && (uint64_t)l1->inum <= len) { + clear = (size_t)l1->inum; + } + /* /Encoding\s+StandardEncoding\s+def */ + for (size_t i = 0; i + 9 <= clear; i++) { + if (data[i] != '/' || memcmp(data + i, "/Encoding", 9) != 0) { + continue; + } + size_t p = i + 9; + size_t w = p; + while (p < clear && py_space(data[p])) { + p++; + } + if (p == w || clear - p < 16 || memcmp(data + p, "StandardEncoding", 16) != 0) { + continue; + } + p += 16; + w = p; + while (p < clear && py_space(data[p])) { + p++; + } + if (p == w || clear - p < 3 || memcmp(data + p, "def", 3) != 0) { + continue; + } + *standard = true; + return true; + } + /* dup\s+(\d+)\s*\/([^\s/\[\]{}()<>]+)\s+put */ + bool hit = false; + size_t i = 0; + while (i + 3 <= clear) { + if (data[i] != 'd' || memcmp(data + i, "dup", 3) != 0) { + i++; + continue; + } + size_t p = i + 3; + size_t w = p; + while (p < clear && py_space(data[p])) { + p++; + } + size_t ds = p; + while (p < clear && py_digit(data[p])) { + p++; + } + bool ok = ds > w && p > ds; + size_t de = p; + while (ok && p < clear && py_space(data[p])) { + p++; + } + ok = ok && p < clear && data[p] == '/'; + size_t ns = p + 1; + if (ok) { + p = ns; + while (p < clear && t1_name_char(data[p])) { + p++; + } + ok = p > ns; + } + size_t ne = p; + if (ok) { + size_t w2 = p; + while (p < clear && py_space(data[p])) { + p++; + } + ok = p > w2 && clear - p >= 3 && memcmp(data + p, "put", 3) == 0; + } + if (!ok) { + i++; + continue; + } + uint64_t c = 0; + for (size_t k = ds; k < de && c < 256; k++) { + c = c * 10 + (uint64_t)(data[k] - '0'); + } + if (c < 256) { + names[c] = data + ns; + nlen[c] = ne - ns; + hit = true; + } + i = p + 3; + } + return hit; +} + +static const char *const *base_table(const pdf_val_t *name) { + if (name_eq(name, "StandardEncoding")) { + return PDF_ENC_STANDARD; + } + if (name_eq(name, "WinAnsiEncoding")) { + return PDF_ENC_WINANSI; + } + if (name_eq(name, "MacRomanEncoding")) { + return PDF_ENC_MACROMAN; + } + if (name_eq(name, "PDFDocEncoding")) { + return PDF_ENC_PDFDOC; + } + return NULL; +} + +static int dbl_cmp(const void *x, const void *y) { + double a = *(const double *)x; + double b = *(const double *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +static pdf_val_t *font_desc(pdf_doc_t *d, pdf_val_t *fd) { + pdf_val_t *v = pdf_dget(d, fd, "FontDescriptor"); + return (v && v->kind == PV_DICT) ? v : NULL; +} + +/* Which of FontFile, FontFile2, FontFile3 is present (the last listed wins). */ +static const char *font_prog(const pdf_val_t *fdesc) { + const char *prog = NULL; + static const char *const keys[] = {"FontFile", "FontFile2", "FontFile3"}; + for (int i = 0; i < 3; i++) { + if (pdf_dget_raw(fdesc, keys[i])) { + prog = keys[i]; + } + } + return prog; +} + +static void font_init_simple(pdf_doc_t *d, pdf_font_t *f, pdf_val_t *fd, const pdf_val_t *subtype, + const unsigned char *base, size_t base_n) { + pdf_val_t *fdesc = font_desc(d, fd); + pdf_val_t *flags = pdf_dget(d, fdesc, "Flags"); + int64_t fl = pdf_is_num(flags) ? (flags->is_int ? flags->inum : pdf_d2i(flags->num)) : 0; + bool symbolic = (fl & PDF_FLAG_SYMBOLIC) != 0; + pdf_val_t *fcv = pdf_dget(d, fd, "FirstChar"); + int64_t fc = pdf_is_num(fcv) ? (fcv->is_int ? fcv->inum : pdf_d2i(fcv->num)) : 0; + pdf_val_t *ws = pdf_dget(d, fd, "Widths"); + bool any_width = false; + double *all = NULL; + int nall = 0; + if (ws && ws->kind == PV_ARR) { + all = (double *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, + (size_t)(ws->count ? ws->count : 1) * sizeof(double)); + for (int i = 0; i < ws->count; i++) { + pdf_val_t *w = pdf_resolve(d, ws->items[i]); + if (!pdf_is_num(w)) { + continue; + } + any_width = true; + int64_t code = pdf_sat_add(fc, i); + if (code >= 0 && code < 256) { + f->widths[code] = w->num; + f->has_width[code] = true; + } + if (all) { + all[nall++] = w->num; + } + } + } + pdf_val_t *mw = pdf_dget(d, fdesc, "MissingWidth"); + if (pdf_is_num(mw) && mw->num > 0) { + f->missing = mw->num; + f->has_missing = true; + } + const unsigned char *bare = base; + size_t bare_n = base_n; + const unsigned char *plus = base_n ? (const unsigned char *)memchr(base, '+', base_n) : NULL; + if (plus) { + bare = plus + 1; + bare_n = base_n - (size_t)(bare - base); + } + f->std14 = std14_name(bare, bare_n) && !any_width; + if (f->std14) { + /* std14_width reads the name after the LAST '+' */ + const unsigned char *b = base; + size_t bn = base_n; + for (size_t i = 0; i < base_n; i++) { + if (base[i] == '+') { + b = base + i + 1; + bn = base_n - i - 1; + } + } + f->std14_tab = bytes_starts(b, bn, "Courier") ? NULL + : bytes_starts(b, bn, "Times") ? PDF_WIDTH_TIMES + : PDF_WIDTH_HELVETICA; + } + const char *prog = font_prog(fdesc); + bool is_t1 = name_eq(subtype, "Type1") || name_eq(subtype, "MMType1"); + bool is_tt = name_eq(subtype, "TrueType"); + if (f->type3) { + pdf_val_t *fm = pdf_dget(d, fd, "FontMatrix"); + double m0 = 0.001; + if (fm && fm->kind == PV_ARR && fm->count == 6) { + pdf_val_t *x = pdf_resolve(d, fm->items[0]); + m0 = pdf_is_num(x) ? x->num : 0.0; + } + f->wscale = m0 != 0.0 ? m0 : 0.001; + double sc = f->wscale < 0 ? -f->wscale : f->wscale; + int npos = 0; + for (int i = 0; i < nall; i++) { + if (all[i] > 0) { + all[npos++] = all[i] * sc; + } + } + qsort(all, (size_t)npos, sizeof(double), dbl_cmp); + double med = npos ? all[npos / 2] : 0.5; + if (med >= 0.2 && med <= 1.5) { + f->em = 1.0; + } else { + double e = 2.0 * med; + f->em = e < 0.05 ? 0.05 : e > 20.0 ? 20.0 : e; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, all); + /* the encoding */ + pdf_val_t *enc = pdf_dget(d, fd, "Encoding"); + const char *const *table = NULL; + pdf_val_t *diffs = NULL; + if (enc && enc->kind == PV_NAME) { + table = base_table(enc); + } else if (enc && enc->kind == PV_DICT) { + pdf_val_t *be = pdf_dget(d, enc, "BaseEncoding"); + if (be) { + table = base_table(be); + } + diffs = pdf_dget(d, enc, "Differences"); + } + pdf_base_kind_t kind = table ? BASE_TABLE : BASE_NONE; + bool fallback_identity = false; + const unsigned char *names[256]; + size_t nlen[256]; + memset((void *)names, 0, sizeof(names)); + memset(nlen, 0, sizeof(nlen)); + if (kind == BASE_NONE) { + bool got = false; + if (prog && strcmp(prog, "FontFile") == 0 && is_t1) { + bool standard; + if (type1_builtin(d, pdf_dget(d, fdesc, "FontFile"), names, nlen, &standard)) { + got = true; + if (standard) { + table = PDF_ENC_STANDARD; + kind = BASE_TABLE; + } else { + kind = BASE_NAMES; + } + } + } + if (!got && bytes_starts(bare, bare_n, "Symbol") && !prog) { + table = PDF_ENC_SYMBOL; + kind = BASE_TABLE; + got = true; + } + if (!got) { + if ((is_t1 || is_tt) && !symbolic) { + table = PDF_ENC_STANDARD; + kind = BASE_TABLE; + } else if (!f->type3) { + fallback_identity = true; + } + } + } + bool named[256]; + memset(named, 0, sizeof(named)); + if (kind == BASE_NAMES) { + for (int c = 0; c < 256; c++) { + named[c] = names[c] != NULL; + } + } + if (diffs && diffs->kind == PV_ARR) { + int64_t code = 0; + for (int i = 0; i < diffs->count; i++) { + pdf_val_t *x = pdf_resolve(d, diffs->items[i]); + if (pdf_is_num(x)) { + code = x->is_int ? x->inum : pdf_d2i(x->num); + } else if (x && x->kind == PV_NAME) { + if (code >= 0 && code < 256) { + names[code] = x->s; + nlen[code] = x->n; + named[code] = true; + } + code = pdf_sat_add(code, 1); + } + } + } + for (int c = 0; c < 256; c++) { + const char *u = NULL; + if (f->tounicode) { + u = cmap_lookup(d, f->tounicode, 1, (uint32_t)c, NULL, 0); + if (!u) { + u = cmap_lookup(d, f->tounicode, 2, (uint32_t)c, NULL, 0); + } + } + if (!u) { + if (named[c]) { + u = g2u(d, names[c], nlen[c], 0); + } else if (kind == BASE_TABLE) { + u = table[c]; + } + } + bool entry_none = !named[c] && kind == BASE_NONE; + if (!u && (fallback_identity || entry_none) && c >= PDF_STD_FIRST && c <= PDF_STD_LAST && + !f->type3) { + u = ASCII1[c]; + } + f->map1[c] = u; + } +} + +static void font_init_type0(pdf_doc_t *d, pdf_font_t *f, pdf_val_t *fd) { + f->multibyte = true; + pdf_val_t *enc = pdf_dget(d, fd, "Encoding"); + if (enc && enc->kind == PV_NAME) { + if (!name_eq(enc, "Identity-H") && !name_eq(enc, "Identity-V")) { + for (size_t i = 0; i + 4 <= enc->n; i++) { + if (memcmp(enc->s + i, "UCS2", 4) == 0) { + f->ucs2 = true; + } + } + for (size_t i = 0; i + 5 <= enc->n; i++) { + if (memcmp(enc->s + i, "UTF16", 5) == 0) { + f->ucs2 = true; + } + } + } + } else if (enc && enc->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, enc->stream, &len); + f->enc = cmap_parse(d, data ? data : (const unsigned char *)"", data ? len : 0); + } + pdf_val_t *desc = pdf_dget(d, fd, "DescendantFonts"); + pdf_val_t *d0 = NULL; + if (desc && desc->kind == PV_ARR && desc->count) { + d0 = pdf_resolve(d, desc->items[0]); + } + if (!d0 || d0->kind != PV_DICT) { + d0 = NULL; + } + f->cid_type2 = name_eq(pdf_dget(d, d0, "Subtype"), "CIDFontType2"); + pdf_val_t *dw = pdf_dget(d, d0, "DW"); + f->dw = pdf_is_num(dw) ? dw->num : 1000.0; + pdf_val_t *W = pdf_dget(d, d0, "W"); + if (W && W->kind == PV_ARR) { + iv_list_t iv = {0}; + double *vals = NULL; + int nv = 0; + int vcap = 0; + int i = 0; + while (i < W->count && !d->nomem) { + pdf_val_t *first = pdf_resolve(d, W->items[i]); + pdf_val_t *nxt = i + 1 < W->count ? pdf_resolve(d, W->items[i + 1]) : NULL; + if (pdf_is_num(first) && nxt && nxt->kind == PV_ARR) { + int64_t c0 = first->is_int ? first->inum : pdf_d2i(first->num); + for (int j = 0; j < nxt->count; j++) { + pdf_val_t *w = pdf_resolve(d, nxt->items[j]); + int64_t c = pdf_sat_add(c0, j); + if (!pdf_is_num(w) || c < 0 || c > (int64_t)UINT32_MAX) { + continue; + } + if (nv == vcap) { + vcap = vcap ? vcap * 2 : PDF_LIST_MIN; + double *g = (double *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, vals, + (size_t)vcap * sizeof(double)); + if (!g) { + d->nomem = true; + break; + } + vals = g; + } + vals[nv] = w->num; + ivl_push(d, &iv, (uint64_t)c, (uint64_t)c, nv); + nv++; + } + i += 2; + } else if (pdf_is_num(first) && pdf_is_num(nxt) && i + 2 < W->count) { + pdf_val_t *w = pdf_resolve(d, W->items[i + 2]); + int64_t a = first->is_int ? first->inum : pdf_d2i(first->num); + double span = nxt->num - first->num; + if (pdf_is_num(w) && span >= 0 && span <= 65535) { + int64_t z = nxt->is_int ? nxt->inum : pdf_d2i(nxt->num); + if (z >= 0 && a <= (int64_t)UINT32_MAX) { + uint64_t lo = a < 0 ? 0 : (uint64_t)a; + if (nv == vcap) { + vcap = vcap ? vcap * 2 : PDF_LIST_MIN; + double *g = (double *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, vals, + (size_t)vcap * sizeof(double)); + if (!g) { + d->nomem = true; + break; + } + vals = g; + } + vals[nv] = w->num; + if ((uint64_t)z >= lo) { + ivl_push(d, &iv, lo, (uint64_t)z, nv); + } + nv++; + } + } + i += 3; + } else { + i += 1; + } + } + f->wseg = paint(d, &iv, false, &f->nwseg); + cbm_free(CBM_MEM_CLASS_EXTRACT, iv.v); + if (nv) { + f->wval = (double *)pdf_big(d, (size_t)nv * sizeof(double)); + if (f->wval) { + memcpy(f->wval, vals, (size_t)nv * sizeof(double)); + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, vals); + } + f->fdesc = font_desc(d, d0); + f->c2g_val = pdf_dget(d, d0, "CIDToGIDMap"); +} + +static pdf_font_t *font_new(pdf_doc_t *d, pdf_val_t *fd) { + pdf_font_t *f = (pdf_font_t *)cbm_arena_calloc(d->a, sizeof(*f)); + if (!f) { + d->nomem = true; + return NULL; + } + f->em = 1.0; + f->wscale = 0.001; + pdf_val_t *subtype = pdf_dget(d, fd, "Subtype"); + pdf_val_t *basev = pdf_dget(d, fd, "BaseFont"); + const unsigned char *base = + (basev && basev->kind == PV_NAME) ? basev->s : (const unsigned char *)""; + size_t base_n = (basev && basev->kind == PV_NAME) ? basev->n : 0; + f->type3 = name_eq(subtype, "Type3"); + pdf_val_t *tu = pdf_dget(d, fd, "ToUnicode"); + if (tu && tu->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, tu->stream, &len); + if (data && len) { + f->tounicode = cmap_parse(d, data, len); + } + } + if (name_eq(subtype, "Type0")) { + font_init_type0(d, f, fd); + } else { + font_init_simple(d, f, fd, subtype, base, base_n); + } + return d->nomem ? NULL : f; +} + +pdf_font_t *pdf_font_for(pdf_doc_t *d, pdf_val_t *resources, const pdf_val_t *name) { + if (!resources || resources->kind != PV_DICT || !name || name->kind != PV_NAME) { + return NULL; + } + pdf_val_t *fdict = pdf_dget(d, resources, "Font"); + if (!fdict || fdict->kind != PV_DICT) { + return NULL; + } + pdf_val_t *ref = NULL; + for (int lo = 0, hi = fdict->nkv - 1; lo <= hi;) { + int mid = lo + (hi - lo) / 2; + size_t kn = fdict->kv[mid].klen; + size_t m = kn < name->n ? kn : name->n; + int c = m ? memcmp(fdict->kv[mid].key, name->s, m) : 0; + if (!c) { + c = kn < name->n ? -1 : kn > name->n ? 1 : 0; + } + if (!c) { + ref = fdict->kv[mid].val; + break; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + if (ref && ref->kind == PV_NULL) { + ref = NULL; + } + uint64_t key = 0; + bool keyed = false; + if (ref && ref->kind == PV_REF) { + key = (1ULL << 63) | (((uint64_t)ref->ref_num & 0xFFFFFFFFFFULL) << 22) | + ((uint64_t)ref->ref_gen & 0x3FFFFFULL); + keyed = true; + int64_t i = pdf_map_get(&d->fonts, key); + if (i >= 0) { + return d->font_list[i]; + } + } + pdf_val_t *fd = pdf_resolve(d, ref); + if (!fd || fd->kind != PV_DICT) { + return NULL; + } + if (!keyed) { + key = (uint64_t)(uintptr_t)fd & ~(1ULL << 63); + int64_t i = pdf_map_get(&d->fonts, key); + if (i >= 0) { + return d->font_list[i]; + } + } + pdf_font_t *f = font_new(d, fd); + if (d->nfonts == d->font_cap) { + int ncap = d->font_cap ? d->font_cap * 2 : PDF_LIST_MIN; + pdf_font_t **g = (pdf_font_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)d->font_list, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return f; + } + d->font_list = g; + d->font_cap = ncap; + } + d->font_list[d->nfonts] = f; + if (!pdf_map_put(&d->fonts, key, d->nfonts)) { + d->nomem = true; + } + d->nfonts++; + return f; +} + +double pdf_font_em(const pdf_font_t *f) { + return f->em; +} + +static const uint32_t *font_rev(pdf_doc_t *d, pdf_font_t *f) { + if (f->rev_done) { + return f->rev; + } + f->rev_done = true; + pdf_val_t *ff = pdf_dget(d, f->fdesc, "FontFile2"); + if (ff && ff->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *data = pdf_stream_decode(d, ff->stream, &len); + if (data && len) { + f->rev = ttf_reverse(d, data, len); + } + } + if (f->c2g_val && f->c2g_val->kind == PV_STREAM) { + size_t len = 0; + const unsigned char *m = pdf_stream_decode(d, f->c2g_val->stream, &len); + f->c2g = m; + f->c2g_len = m ? len : 0; + } + return f->rev; +} + +void pdf_font_decode(pdf_doc_t *d, pdf_font_t *f, const unsigned char *s, size_t len, + pdf_glyph_out_t *out, size_t *n) { + size_t k = 0; + if (!f->multibyte) { + for (size_t i = 0; i < len; i++) { + unsigned c = s[i]; + double w; + if (f->has_width[c]) { + w = f->widths[c]; + } else if (f->std14) { + if (!f->std14_tab) { + w = PDF_COURIER_WIDTH; + } else { + w = (c >= PDF_STD_FIRST && c <= PDF_STD_LAST) ? f->std14_tab[c - PDF_STD_FIRST] + : PDF_DEFAULT_WIDTH; + } + } else if (f->has_missing) { + w = f->missing; + } else { + w = f->type3 ? 0.0 : PDF_DEFAULT_WIDTH; + } + out[k].utf8 = f->map1[c]; + out[k].width = w * f->wscale; + out[k].space = c == ' '; + k++; + } + *n = k; + return; + } + size_t i = 0; + while (i < len) { + size_t nb = 0; + uint32_t code = 0; + if (f->enc && f->enc->has_space) { + for (int m = 1; m <= PDF_CODE_BYTES && !nb; m++) { + if (!f->enc->nspace[m] || i + (size_t)m > len) { + continue; + } + uint32_t c = be_code(s + i, (size_t)m); + if (seg_find(f->enc->space[m], f->enc->nspace[m], c) >= 0) { + nb = (size_t)m; + code = c; + } + } + if (!nb) { + nb = 1; + code = s[i]; + } + } else if (i + 1 < len) { + nb = 2; + code = ((uint32_t)s[i] << 8) | s[i + 1]; + } else { + nb = 1; + code = s[i]; + } + i += nb; + pdf_glyph_out_t *g = &out[k++]; + g->utf8 = NULL; + g->inl[0] = '\0'; + if (f->tounicode) { + g->utf8 = cmap_lookup(d, f->tounicode, nb, code, g->inl, sizeof(g->inl)); + } + int64_t cid = code; + if (f->enc) { + int64_t c2; + if (cmap_to_cid(f->enc, nb, code, &c2)) { + cid = c2; + } + } + if (!g->utf8 && f->ucs2 && !(code >= PDF_SURR_LO && code <= PDF_SURR_HI)) { + int m = utf8_put(code, g->inl); + g->inl[m] = '\0'; + g->utf8 = g->inl; + } + if (!g->utf8 && f->cid_type2) { + const uint32_t *rev = font_rev(d, f); + if (rev) { + int64_t gid = cid; + if (f->c2g && f->c2g_len) { + gid = 0; + if (cid >= 0 && (uint64_t)cid * 2 + 2 <= f->c2g_len) { + gid = ((int64_t)f->c2g[2 * cid] << 8) | f->c2g[2 * cid + 1]; + } + } + if (gid >= 0 && gid < PDF_GID_SPACE && rev[gid]) { + int m = utf8_put(rev[gid] - 1, g->inl); + g->inl[m] = '\0'; + g->utf8 = g->inl; + } + } + } + double w = f->dw; + if (cid >= 0 && f->nwseg) { + int r = seg_find(f->wseg, f->nwseg, (uint64_t)cid); + if (r >= 0 && f->wval) { + w = f->wval[r]; + } + } + g->width = w * 0.001; + g->space = nb == 1 && code == ' '; + } + *n = k; +} diff --git a/internal/cbm/pdf/pdf_internal.h b/internal/cbm/pdf/pdf_internal.h new file mode 100644 index 0000000000..95b94c4f6f --- /dev/null +++ b/internal/cbm/pdf/pdf_internal.h @@ -0,0 +1,330 @@ +/* + * pdf_internal.h — the PDF text-layer extractor's internals (pdf_*.c). + * + * Stages, each in its own file: + * pdf_obj.c lexer, values, the document (xref tables and streams, /Prev + * chains, reconstruction when offsets are broken), indirect + * objects, object streams, stream filters + * pdf_font.c glyph names, ToUnicode and encoding CMaps, simple fonts + * (base encodings, /Differences, built-in Type1 encodings, + * standard-14 widths, Type3), Type0/CID fonts, the TrueType + * cmap fallback + * pdf_text.c content streams (graphics and text state, text operators, + * Form XObjects, inline images skipped), layout (word and line + * breaks from glyph geometry), pages, the driver + * pdf_tables.c generated static tables + * + * The reader is a port of the field-tested prototype (field test E6): the + * same tokens, repairs, font rules and layout thresholds, so its text is the + * text that was measured. Where the prototype's cost could grow faster than + * the input (a search per object, a table walk per glyph), this port answers + * from an index built once, with the same result. + */ +#ifndef CBM_PDF_INTERNAL_H +#define CBM_PDF_INTERNAL_H + +#include "pdf.h" + +#include +#include +#include + +/* Layout geometry must come out bit for bit the same on every platform (and + * the same as the measured prototype): no fused multiply-add. GCC in ISO C + * mode never contracts; clang does unless told. */ +#if defined(__clang__) +#pragma STDC FP_CONTRACT OFF +#endif + +/* ── Tables (pdf_tables.c) ───────────────────────────────────────── */ + +typedef struct { + const char *name; + const char *utf8; +} pdf_glyph_t; + +enum { PDF_STD_WIDTHS = 95 }; /* codes 0x20..0x7E */ + +extern const pdf_glyph_t PDF_GLYPHS[]; +extern const int PDF_GLYPH_COUNT; +extern const char *const PDF_ENC_STANDARD[256]; +extern const char *const PDF_ENC_WINANSI[256]; +extern const char *const PDF_ENC_MACROMAN[256]; +extern const char *const PDF_ENC_PDFDOC[256]; +extern const char *const PDF_ENC_SYMBOL[256]; +extern const unsigned short PDF_WIDTH_HELVETICA[PDF_STD_WIDTHS]; +extern const unsigned short PDF_WIDTH_TIMES[PDF_STD_WIDTHS]; +extern const char *const PDF_STD14[]; + +/* ── Limits: structure, never content ────────────────────────────── */ + +enum { + PDF_MAX_NESTING = 256, /* arrays/dicts inside one value */ + PDF_MAX_RESOLVE = 32, /* reference chains */ + PDF_MAX_FORM_DEPTH = 16, /* Form XObjects inside Form XObjects */ + PDF_MAX_PAGE_DEPTH = 64, /* page-tree depth */ + PDF_MAX_PREV = 4096, /* xref sections along /Prev */ + PDF_HEADER_WINDOW = 1024, /* "%PDF-" must start within this many bytes */ +}; + +/* A decoded stream above this size is cut and counted: a decompression bomb is + * no text layer, and real documents stay far below it. */ +#define PDF_MAX_STREAM_OUT ((size_t)512 << 20) + +/* A Form XObject's first reading in a document is always whole. Reading one + * again (the same content drawn elsewhere: a logo, a header) is counted, and + * beyond this many decoded bytes of such re-readings the document reads no + * form again (counted): forms drawing the next one many times, level under + * level, asked for (draws ^ depth) readings from a few kilobytes. */ +#define PDF_MAX_FORM_REREAD ((size_t)256 << 20) + +/* ── Integers from the file: saturated, never undefined ──────────── */ + +static inline int64_t pdf_sat_add(int64_t a, int64_t b) { + int64_t r; + if (__builtin_add_overflow(a, b, &r)) { + return b > 0 ? INT64_MAX : INT64_MIN; + } + return r; +} + +/* Python's int() of a float: truncated toward zero; saturated here, NaN 0. */ +static inline int64_t pdf_d2i(double x) { + if (x != x) { + return 0; + } + if (x >= 9223372036854775807.0) { + return INT64_MAX; + } + if (x <= -9223372036854775808.0) { + return INT64_MIN; + } + return (int64_t)x; +} + +/* ── Values ──────────────────────────────────────────────────────── */ + +typedef enum { + PV_NULL = 0, + PV_BOOL, + PV_NUM, + PV_NAME, + PV_STR, + PV_ARR, + PV_DICT, + PV_REF, + PV_STREAM, + PV_KW, +} pdf_kind_t; + +typedef struct pdf_val pdf_val_t; +typedef struct pdf_stream pdf_stream_t; + +typedef struct { + const unsigned char *key; /* a name's bytes, NUL-terminated */ + size_t klen; + pdf_val_t *val; +} pdf_kv_t; + +struct pdf_val { + pdf_kind_t kind; + bool b; + bool is_int; + double num; + int64_t inum; + const unsigned char *s; /* PV_NAME, PV_STR, PV_KW (NUL-terminated copies) */ + size_t n; + pdf_val_t **items; /* PV_ARR */ + int count; + pdf_kv_t *kv; /* PV_DICT: sorted by key, one entry per key (the last written) */ + int nkv; + int64_t ref_num; /* PV_REF */ + int64_t ref_gen; + pdf_stream_t *stream; /* PV_STREAM */ +}; + +struct pdf_stream { + pdf_val_t *dict; + const unsigned char *raw; + size_t raw_len; + bool done; + const unsigned char *decoded; /* NULL: nothing decodable (image, unsupported) */ + size_t dec_len; +}; + +/* ── Lexer ───────────────────────────────────────────────────────── */ + +typedef enum { + PT_EOF = 0, + PT_NUM, + PT_NAME, + PT_STR, + PT_DS, /* << */ + PT_DE, /* >> */ + PT_AS, /* [ */ + PT_AE, /* ] */ + PT_BS, /* { */ + PT_BE, /* } */ + PT_KW, +} pdf_tok_kind_t; + +typedef struct { + pdf_tok_kind_t kind; + bool is_int; + double num; + int64_t inum; + const unsigned char *s; /* NAME, STR: arena copies; KW: points into the data */ + size_t n; +} pdf_tok_t; + +typedef struct pdf_doc pdf_doc_t; + +/* The next token of data[*pos..len); *pos moves past it. */ +void pdf_lex(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, pdf_tok_t *t); +/* The value that starts with token t. NULL: out of memory or nested too deep. */ +pdf_val_t *pdf_parse_value(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + const pdf_tok_t *t, int depth); +/* An array operand of a content stream: no references, an operator ends it. */ +pdf_val_t *pdf_parse_content_array(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + int depth); +bool pdf_kw_is(const pdf_tok_t *t, const char *kw); + +/* ── Integer maps (open addressing, owned by the document) ───────── */ + +typedef struct { + uint64_t *keys; + int64_t *vals; + uint32_t cap; /* a power of two, or 0 */ + uint32_t n; +} pdf_map_t; + +/* -1 when absent. */ +int64_t pdf_map_get(const pdf_map_t *m, uint64_t key); +bool pdf_map_put(pdf_map_t *m, uint64_t key, int64_t val); +void pdf_map_free(pdf_map_t *m); + +/* ── Document ────────────────────────────────────────────────────── */ + +typedef struct pdf_objstm pdf_objstm_t; + +typedef struct { + int64_t num; + uint8_t type; /* 0 free, 1 at an offset, 2 inside an object stream */ + int64_t a; /* type 1: offset; type 2: the object stream's number */ + int64_t b; /* type 1: generation; type 2: index in the stream */ + pdf_val_t *obj; + bool cached; + bool loading; + bool not_length; /* read as a stream's /Length, it was a stream: no length */ + pdf_objstm_t *stm; /* this object as a decoded object stream */ +} pdf_xent_t; + +typedef struct { + pdf_xent_t *v; + int n; + int cap; + pdf_map_t by_num; /* object number -> index in v */ +} pdf_xref_t; + +/* One "N G obj" header found by scanning the whole file. */ +typedef struct { + int64_t num; + int64_t gen; + size_t off; /* where N starts */ + uint8_t num_digits; + uint8_t gen_digits; + bool canonical; /* both written without leading zeros */ + bool reg_after; /* "obj" runs on into a regular character */ +} pdf_hdr_t; + +typedef struct { + int64_t num; + int idx; +} pdf_numidx_t; + +typedef struct pdf_font pdf_font_t; + +struct pdf_doc { + const unsigned char *data; + size_t len; + size_t hdr; /* offset of "%PDF-" */ + CBMArena *a; /* the document's values, fonts, CMaps */ + CBMArena *cur; /* where the lexer and the value parser allocate (a, or a + * content stream's operand arena) */ + bool peek; /* lex without copying names or strings */ + bool nomem; /* an allocation failed: the result is CBM_PDF_NOMEM */ + pdf_xref_t xref; + pdf_val_t *trailer; /* a dict */ + bool reconstructed; + int objstm_opening; /* object streams being opened, one inside the other */ + /* every "N G obj" in file order, built on first need */ + pdf_hdr_t *hdrs; + int nhdrs; + bool hdrs_built; + pdf_numidx_t *hdr_sorted; /* (num, index) sorted, for the per-object fallback */ + /* blocks from the memory core, freed with the document */ + void **owned; + int nowned; + int owned_cap; + /* an arena for token values that end with one CMap's parse */ + CBMArena scratch_arena; + bool scratch_arena_live; + /* a scratch buffer for string and stream assembly */ + unsigned char *scratch; + size_t scratch_cap; + /* fonts by reference (num << 20 | gen) or by inline dict address */ + pdf_map_t fonts; + pdf_font_t **font_list; + int nfonts; + int font_cap; + cbm_pdf_counts_t *counts; +}; + +pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num); +pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v); +/* dict[key] as written (a reference stays a reference); NULL when absent, null, + * or the value is no dict. */ +pdf_val_t *pdf_dget_raw(const pdf_val_t *dict, const char *key); +/* Whether dict has key at all (a null value counts). */ +bool pdf_dhas(const pdf_val_t *dict, const char *key); +/* dict[key] as written, a null value included; NULL only when absent. */ +pdf_val_t *pdf_dfind(const pdf_val_t *dict, const char *key); +pdf_val_t *pdf_dfind_n(const pdf_val_t *dict, const unsigned char *key, size_t n); +/* dict[key] resolved; NULL when absent or null. */ +pdf_val_t *pdf_dget(pdf_doc_t *d, const pdf_val_t *dict, const char *key); +bool pdf_is_name(const pdf_val_t *v, const char *name); +bool pdf_is_num(const pdf_val_t *v); +/* A number's value; 0 for anything else. */ +double pdf_num(const pdf_val_t *v); +/* The decoded data of a stream; NULL when it has none. */ +const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len); +bool pdf_doc_open(pdf_doc_t *d); +void pdf_doc_close(pdf_doc_t *d); + +/* Memory: arena values, and big blocks the document owns. */ +void *pdf_alloc(pdf_doc_t *d, size_t n); +void *pdf_calloc(pdf_doc_t *d, size_t n); +void *pdf_big(pdf_doc_t *d, size_t n); /* owned, freed with the document */ +bool pdf_own(pdf_doc_t *d, void *block); /* hand a memory-core block to the document */ +bool pdf_scratch(pdf_doc_t *d, size_t need); /* d->scratch holds at least `need` bytes */ + +/* ── Fonts (pdf_font.c) ──────────────────────────────────────────── */ + +typedef struct { + const char *utf8; /* NULL: unmapped; "" maps to nothing; may point into inl */ + double width; /* advance in text-space units per unit of font size */ + bool space; /* the single-byte code 32 */ + char inl[16]; /* a short mapping made for this glyph */ +} pdf_glyph_out_t; + +/* The font resources[/Font][name] names; NULL when there is none. Cached. */ +pdf_font_t *pdf_font_for(pdf_doc_t *d, pdf_val_t *resources, const pdf_val_t *name); +/* Decode shown bytes into glyphs, out[0..*n) (out holds at least len entries). */ +void pdf_font_decode(pdf_doc_t *d, pdf_font_t *f, const unsigned char *s, size_t len, + pdf_glyph_out_t *out, size_t *n); +/* The text-space size of one em per unit of font size. */ +double pdf_font_em(const pdf_font_t *f); +/* A glyph name's Unicode value as UTF-8 (arena or static); NULL when unknown. */ +const char *pdf_glyph_to_utf8(pdf_doc_t *d, const unsigned char *name, size_t n); + +#endif /* CBM_PDF_INTERNAL_H */ diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.c new file mode 100644 index 0000000000..d3c728db65 --- /dev/null +++ b/internal/cbm/pdf/pdf_obj.c @@ -0,0 +1,2771 @@ +/* + * pdf_obj.c — lexer, values, the cross-reference, indirect objects, object + * streams and stream filters of the PDF text-layer extractor. + * + * Repairs follow the field-tested prototype: an xref section is looked for + * near a wrong offset; a stream whose /Length does not end at "endstream" is + * cut at the next "endstream"; an object missing at its offset is looked up + * by its header; when the cross-reference cannot be read at all it is rebuilt + * from every "N G obj" header in the file. Every header the repairs need comes + * from one scan of the file, built on first need. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" +#include "helpers.h" + +#include +#include +#include +#include +#include + +enum { + PDF_NUM_BUF = 64, /* a number's text for strtod */ + PDF_XREF_NEAR = 1024, /* how far around a wrong xref offset to look */ + PDF_TAIL_WINDOW = 32, /* where "endstream" must follow /Length */ + PDF_INFLATE_CHUNK = 4096, /* the prototype's recovery chunk */ + PDF_INFLATE_GROW = 65536, /* output growth step */ + PDF_ZLIB_WBITS = 15, + PDF_MAX_FIELD = 8, /* bytes of one xref-stream field */ + PDF_A85_GROUP = 5, + PDF_A85_BASE = 85, + PDF_LZW_CLEAR = 256, + PDF_LZW_EOD = 257, + PDF_LZW_FIRST = 258, + PDF_LZW_MAX_BITS = 12, + PDF_LZW_MIN_BITS = 9, + PDF_LZW_TABLE = 4096, + PDF_RLD_EOD = 128, + PDF_PNG_PRED = 10, + PDF_TIFF_PRED = 2, + PDF_BITS_PER_BYTE = 8, + PDF_MAP_MIN = 16, + PDF_OWN_MIN = 16, + PDF_SCRATCH_MIN = 256, + PDF_HEX_BASE = 16, + PDF_OCT_DIGITS = 3, +}; + +#define PDF_SAT_LIMIT ((INT64_MAX - 9) / 10) + +static pdf_val_t PDF_NULL_VALUE = {.kind = PV_NULL}; + +/* ── Memory ──────────────────────────────────────────────────────── */ + +void *pdf_alloc(pdf_doc_t *d, size_t n) { + void *p = cbm_arena_alloc(d->cur, n ? n : 1); + if (!p) { + d->nomem = true; + } + return p; +} + +void *pdf_calloc(pdf_doc_t *d, size_t n) { + void *p = cbm_arena_calloc(d->cur, n ? n : 1); + if (!p) { + d->nomem = true; + } + return p; +} + +bool pdf_own(pdf_doc_t *d, void *block) { + if (!block) { + return false; + } + if (d->nowned == d->owned_cap) { + int ncap = d->owned_cap ? d->owned_cap * 2 : PDF_OWN_MIN; + void **g = (void **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)d->owned, + (size_t)ncap * sizeof(*g)); + if (!g) { + cbm_free(CBM_MEM_CLASS_EXTRACT, block); + d->nomem = true; + return false; + } + d->owned = g; + d->owned_cap = ncap; + } + d->owned[d->nowned++] = block; + return true; +} + +void *pdf_big(pdf_doc_t *d, size_t n) { + void *p = cbm_alloc(CBM_MEM_CLASS_EXTRACT, n ? n : 1); + if (!p) { + d->nomem = true; + return NULL; + } + return pdf_own(d, p) ? p : NULL; +} + +bool pdf_scratch(pdf_doc_t *d, size_t need) { + if (need <= d->scratch_cap) { + return true; + } + size_t ncap = d->scratch_cap ? d->scratch_cap : PDF_SCRATCH_MIN; + while (ncap < need) { + if (ncap > SIZE_MAX / 2) { + ncap = need; + break; + } + ncap *= 2; + } + unsigned char *g = (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, d->scratch, ncap); + if (!g) { + d->nomem = true; + return false; + } + d->scratch = g; + d->scratch_cap = ncap; + return true; +} + +/* A growable byte buffer from the memory core (decoder output). */ +typedef struct { + unsigned char *p; + size_t n; + size_t cap; + bool fail; +} pdf_buf_t; + +static bool buf_reserve(pdf_buf_t *b, size_t add) { + if (b->fail) { + return false; + } + if (add > SIZE_MAX - b->n) { + b->fail = true; + return false; + } + size_t need = b->n + add; + if (need <= b->cap) { + return true; + } + size_t ncap = b->cap ? b->cap : PDF_INFLATE_GROW; + while (ncap < need) { + if (ncap > SIZE_MAX / 2) { + ncap = need; + break; + } + ncap *= 2; + } + unsigned char *g = (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->p, ncap); + if (!g) { + b->fail = true; + return false; + } + b->p = g; + b->cap = ncap; + return true; +} + +static bool buf_put(pdf_buf_t *b, const unsigned char *src, size_t n) { + if (!n) { + return !b->fail; + } + if (!buf_reserve(b, n)) { + return false; + } + memcpy(b->p + b->n, src, n); + b->n += n; + return true; +} + +static bool buf_byte(pdf_buf_t *b, unsigned char c) { + if (!buf_reserve(b, 1)) { + return false; + } + b->p[b->n++] = c; + return true; +} + +static void buf_free(pdf_buf_t *b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, b->p); + memset(b, 0, sizeof(*b)); +} + +/* ── Integer maps ────────────────────────────────────────────────── */ + +static uint64_t map_mix(uint64_t x) { + /* splitmix64's finalizer; the seed keeps crafted object numbers from + * clustering (it changes speed only, never a result). */ + static const uint64_t seed = 0x9e3779b97f4a7c15ULL; + x += seed; + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9ULL; + x ^= x >> 27; + x *= 0x94d049bb133111ebULL; + x ^= x >> 31; + return x; +} + +int64_t pdf_map_get(const pdf_map_t *m, uint64_t key) { + if (!m->cap) { + return -1; + } + uint32_t mask = m->cap - 1; + uint32_t i = (uint32_t)map_mix(key) & mask; + while (m->vals[i] >= 0) { + if (m->keys[i] == key) { + return m->vals[i]; + } + i = (i + 1) & mask; + } + return -1; +} + +static bool map_grow(pdf_map_t *m) { + uint32_t ncap = m->cap ? m->cap * 2 : PDF_MAP_MIN; + if (ncap < m->cap) { + return false; + } + uint64_t *keys = (uint64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)ncap * sizeof(*keys)); + int64_t *vals = (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)ncap * sizeof(*vals)); + if (!keys || !vals) { + cbm_free(CBM_MEM_CLASS_EXTRACT, keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, vals); + return false; + } + for (uint32_t i = 0; i < ncap; i++) { + vals[i] = -1; + } + uint32_t mask = ncap - 1; + for (uint32_t i = 0; i < m->cap; i++) { + if (m->vals[i] < 0) { + continue; + } + uint32_t j = (uint32_t)map_mix(m->keys[i]) & mask; + while (vals[j] >= 0) { + j = (j + 1) & mask; + } + keys[j] = m->keys[i]; + vals[j] = m->vals[i]; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, m->keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, m->vals); + m->keys = keys; + m->vals = vals; + m->cap = ncap; + return true; +} + +bool pdf_map_put(pdf_map_t *m, uint64_t key, int64_t val) { + if ((m->n + 1) * 2 > m->cap && !map_grow(m)) { + return false; + } + uint32_t mask = m->cap - 1; + uint32_t i = (uint32_t)map_mix(key) & mask; + while (m->vals[i] >= 0) { + if (m->keys[i] == key) { + m->vals[i] = val; + return true; + } + i = (i + 1) & mask; + } + m->keys[i] = key; + m->vals[i] = val; + m->n++; + return true; +} + +void pdf_map_free(pdf_map_t *m) { + cbm_free(CBM_MEM_CLASS_EXTRACT, m->keys); + cbm_free(CBM_MEM_CLASS_EXTRACT, m->vals); + memset(m, 0, sizeof(*m)); +} + +/* ── Characters ──────────────────────────────────────────────────── */ + +static inline bool is_ws(unsigned char c) { + return c == 0 || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' '; +} + +static inline bool is_delim(unsigned char c) { + return c == '(' || c == ')' || c == '<' || c == '>' || c == '[' || c == ']' || c == '{' || + c == '}' || c == '/' || c == '%'; +} + +static inline bool is_reg(unsigned char c) { + return !is_ws(c) && !is_delim(c); +} + +static inline bool is_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static int hex_val(unsigned char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +/* A digit run as a saturated integer. */ +static int64_t sat_digits(const unsigned char *s, size_t n) { + int64_t v = 0; + for (size_t i = 0; i < n; i++) { + if (v > PDF_SAT_LIMIT) { + return INT64_MAX; + } + v = v * 10 + (s[i] - '0'); + } + return v; +} + +/* ── Lexer ───────────────────────────────────────────────────────── */ + +bool pdf_kw_is(const pdf_tok_t *t, const char *kw) { + size_t n = strlen(kw); + return t->kind == PT_KW && t->n == n && memcmp(t->s, kw, n) == 0; +} + +/* [+-]?(\d+\.?\d*|\.\d+) not followed by a regular character. */ +static bool lex_num(const unsigned char *data, size_t len, size_t p, size_t *end, pdf_tok_t *t) { + size_t q = p; + bool neg = false; + if (data[q] == '+' || data[q] == '-') { + neg = data[q] == '-'; + q++; + } + size_t ds = q; + bool dot = false; + if (q < len && is_digit(data[q])) { + while (q < len && is_digit(data[q])) { + q++; + } + if (q < len && data[q] == '.') { + dot = true; + q++; + while (q < len && is_digit(data[q])) { + q++; + } + } + } else if (q + 1 < len && data[q] == '.' && is_digit(data[q + 1])) { + dot = true; + q++; + while (q < len && is_digit(data[q])) { + q++; + } + } else { + return false; + } + if (q < len && is_reg(data[q])) { + return false; + } + t->kind = PT_NUM; + t->is_int = !dot; + if (!dot) { + int64_t v = sat_digits(data + ds, q - ds); + t->inum = neg ? -v : v; + t->num = (double)t->inum; + } else { + char buf[PDF_NUM_BUF]; + size_t n = q - p; + if (n < sizeof(buf)) { + memcpy(buf, data + p, n); + buf[n] = '\0'; + t->num = strtod(buf, NULL); + } else { + /* a number this long is noise; read its leading digits */ + memcpy(buf, data + p, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + t->num = strtod(buf, NULL); + } + t->inum = (int64_t)0; + } + *end = q; + return true; +} + +static void lex_name(pdf_doc_t *d, const unsigned char *s, size_t n, pdf_tok_t *t) { + t->kind = PT_NAME; + if (d->peek) { + t->s = s; + t->n = n; + return; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, n + 1); + if (!o) { + t->s = (const unsigned char *)""; + t->n = 0; + return; + } + size_t k = 0; + for (size_t i = 0; i < n;) { + if (s[i] == '#' && i + 2 < n && hex_val(s[i + 1]) >= 0 && hex_val(s[i + 2]) >= 0) { + o[k++] = (unsigned char)(hex_val(s[i + 1]) * PDF_HEX_BASE + hex_val(s[i + 2])); + i += 3; + } else { + o[k++] = s[i++]; + } + } + o[k] = '\0'; + t->s = o; + t->n = k; +} + +static bool lstr_special(unsigned char c) { + return c == '(' || c == ')' || c == '\\' || c == '\r'; +} + +/* A literal string from data[i..] (after its "("); returns the position after it. */ +static size_t lex_lstr(pdf_doc_t *d, const unsigned char *data, size_t len, size_t i, + pdf_tok_t *t) { + int depth = 1; + size_t k = 0; + bool keep = !d->peek; + size_t end = len; + t->kind = PT_STR; + for (;;) { + size_t j = i; + while (j < len && !lstr_special(data[j])) { + j++; + } + if (keep && j > i) { + if (!pdf_scratch(d, k + (j - i) + PDF_OCT_DIGITS + 1)) { + keep = false; + } else { + memcpy(d->scratch + k, data + i, j - i); + k += j - i; + } + } + if (j >= len) { + end = len; + break; + } + if (keep && !pdf_scratch(d, k + 2)) { + keep = false; + } + unsigned char c = data[j]; + int out = -1; + if (c == '\\') { + if (j + 1 >= len) { + end = len; + break; + } + unsigned char nx = data[j + 1]; + static const char esc_from[] = "nrtbf()\\"; + static const char esc_to[] = "\n\r\t\b\f()\\"; + const char *e = (nx != 0) ? strchr(esc_from, nx) : NULL; + if (e) { + out = (unsigned char)esc_to[e - esc_from]; + i = j + 2; + } else if (nx >= '0' && nx <= '7') { + size_t q = j + 1; + unsigned v = 0; + while (q < len && q < j + 1 + PDF_OCT_DIGITS && data[q] >= '0' && data[q] <= '7') { + v = v * PDF_BITS_PER_BYTE + (unsigned)(data[q] - '0'); + q++; + } + out = (int)(v & 0xFFU); + i = q; + } else if (nx == '\r') { + i = (j + 2 < len && data[j + 2] == '\n') ? j + 3 : j + 2; + } else if (nx == '\n') { + i = j + 2; + } else { + out = nx; + i = j + 2; + } + } else if (c == '\r') { + out = '\n'; + i = (j + 1 < len && data[j + 1] == '\n') ? j + 2 : j + 1; + } else if (c == '(') { + depth++; + out = c; + i = j + 1; + } else { + depth--; + if (depth == 0) { + end = j + 1; + break; + } + out = c; + i = j + 1; + } + if (keep && out >= 0) { + d->scratch[k++] = (unsigned char)out; + } + } + if (d->peek) { + t->s = NULL; + t->n = 0; + return end; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, k + 1); + if (o) { + if (k) { + memcpy(o, d->scratch, k); + } + o[k] = '\0'; + } + t->s = o ? o : (const unsigned char *)""; + t->n = o ? k : 0; + return end; +} + +/* A hex string from data[i..] (after its "<"). */ +static size_t lex_hstr(pdf_doc_t *d, const unsigned char *data, size_t len, size_t i, + pdf_tok_t *t) { + const unsigned char *gt = (const unsigned char *)memchr(data + i, '>', len - i); + size_t e = gt ? (size_t)(gt - data) : len; + t->kind = PT_STR; + size_t end = e < len ? e + 1 : len; + if (d->peek) { + t->s = NULL; + t->n = 0; + return end; + } + size_t nhex = 0; + for (size_t q = i; q < e; q++) { + nhex += hex_val(data[q]) >= 0; + } + unsigned char *o = (unsigned char *)pdf_alloc(d, (nhex + 1) / 2 + 1); + if (!o) { + t->s = (const unsigned char *)""; + t->n = 0; + return end; + } + size_t k = 0; + int hi = -1; + for (size_t q = i; q < e; q++) { + int v = hex_val(data[q]); + if (v < 0) { + continue; + } + if (hi < 0) { + hi = v; + } else { + o[k++] = (unsigned char)(hi * PDF_HEX_BASE + v); + hi = -1; + } + } + if (hi >= 0) { + o[k++] = (unsigned char)(hi * PDF_HEX_BASE); + } + o[k] = '\0'; + t->s = o; + t->n = k; + return end; +} + +void pdf_lex(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, pdf_tok_t *t) { + size_t p = *pos; + memset(t, 0, sizeof(*t)); + while (p < len) { + unsigned char c = data[p]; + if (is_ws(c)) { + p++; + continue; + } + if (c == '%') { + while (p < len && data[p] != '\r' && data[p] != '\n') { + p++; + } + continue; + } + if (is_digit(c) || c == '+' || c == '-' || c == '.') { + size_t e; + if (lex_num(data, len, p, &e, t)) { + *pos = e; + return; + } + } + switch (c) { + case '/': { + size_t q = p + 1; + while (q < len && is_reg(data[q])) { + q++; + } + lex_name(d, data + p + 1, q - p - 1, t); + *pos = q; + return; + } + case '(': + *pos = lex_lstr(d, data, len, p + 1, t); + return; + case '<': + if (p + 1 < len && data[p + 1] == '<') { + t->kind = PT_DS; + *pos = p + 2; + return; + } + *pos = lex_hstr(d, data, len, p + 1, t); + return; + case '>': + if (p + 1 < len && data[p + 1] == '>') { + t->kind = PT_DE; + *pos = p + 2; + return; + } + p++; + continue; + case ')': + p++; + continue; + case '[': + t->kind = PT_AS; + *pos = p + 1; + return; + case ']': + t->kind = PT_AE; + *pos = p + 1; + return; + case '{': + t->kind = PT_BS; + *pos = p + 1; + return; + case '}': + t->kind = PT_BE; + *pos = p + 1; + return; + default: { + size_t q = p; + while (q < len && is_reg(data[q])) { + q++; + } + t->kind = PT_KW; + t->s = data + p; + t->n = q - p; + *pos = q; + return; + } + } + } + t->kind = PT_EOF; + *pos = len; +} + +/* The next token without copying names or strings. */ +static void lex_peek(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + pdf_tok_t *t) { + bool was = d->peek; + d->peek = true; + pdf_lex(d, data, len, pos, t); + d->peek = was; +} + +/* ── Values ──────────────────────────────────────────────────────── */ + +static pdf_val_t *new_val(pdf_doc_t *d, pdf_kind_t kind) { + pdf_val_t *v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = kind; + } + return v; +} + +static pdf_val_t *val_from_tok(pdf_doc_t *d, const pdf_tok_t *t) { + pdf_val_t *v = new_val(d, t->kind == PT_NUM ? PV_NUM : t->kind == PT_NAME ? PV_NAME : PV_STR); + if (!v) { + return NULL; + } + v->is_int = t->is_int; + v->num = t->num; + v->inum = t->inum; + v->s = t->s; + v->n = t->n; + return v; +} + +typedef struct { + pdf_val_t **v; + int n; + int cap; + bool fail; +} val_list_t; + +static void vl_push(val_list_t *l, pdf_val_t *x) { + if (l->fail) { + return; + } + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_MAP_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)l->v, + (size_t)ncap * sizeof(*g)); + if (!g) { + l->fail = true; + return; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n++] = x; +} + +static pdf_val_t *vl_finish_array(pdf_doc_t *d, val_list_t *l) { + pdf_val_t *arr = NULL; + if (!l->fail) { + arr = new_val(d, PV_ARR); + if (arr && l->n) { + arr->items = (pdf_val_t **)pdf_alloc(d, (size_t)l->n * sizeof(*arr->items)); + if (arr->items) { + memcpy((void *)arr->items, (void *)l->v, (size_t)l->n * sizeof(*arr->items)); + arr->count = l->n; + } else { + arr = NULL; + } + } + } else { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l->v); + return arr; +} + +typedef struct { + pdf_kv_t kv; + int seq; +} kv_seq_t; + +static int kv_key_cmp(const unsigned char *a, size_t an, const unsigned char *b, size_t bn) { + size_t m = an < bn ? an : bn; + int c = m ? memcmp(a, b, m) : 0; + if (c) { + return c; + } + return an < bn ? -1 : an > bn ? 1 : 0; +} + +static int kv_seq_cmp(const void *x, const void *y) { + const kv_seq_t *a = (const kv_seq_t *)x; + const kv_seq_t *b = (const kv_seq_t *)y; + int c = kv_key_cmp(a->kv.key, a->kv.klen, b->kv.key, b->kv.klen); + if (c) { + return c; + } + return a->seq < b->seq ? -1 : a->seq > b->seq ? 1 : 0; +} + +typedef struct { + kv_seq_t *v; + int n; + int cap; + bool fail; +} kv_list_t; + +static void kl_push(kv_list_t *l, const unsigned char *key, size_t klen, pdf_val_t *val) { + if (l->fail) { + return; + } + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_MAP_MIN; + kv_seq_t *g = + (kv_seq_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + l->fail = true; + return; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].kv.key = key; + l->v[l->n].kv.klen = klen; + l->v[l->n].kv.val = val; + l->v[l->n].seq = l->n; + l->n++; +} + +/* A dict from key/value pairs in written order: one entry per key, the last + * written (keep_last) or the first. */ +static pdf_val_t *kl_finish_dict(pdf_doc_t *d, kv_list_t *l, bool keep_last) { + pdf_val_t *dict = NULL; + if (l->fail) { + d->nomem = true; + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return NULL; + } + if (l->n > 1) { + qsort(l->v, (size_t)l->n, sizeof(*l->v), kv_seq_cmp); + } + dict = new_val(d, PV_DICT); + if (dict && l->n) { + dict->kv = (pdf_kv_t *)pdf_alloc(d, (size_t)l->n * sizeof(*dict->kv)); + if (!dict->kv) { + dict = NULL; + } else { + int k = 0; + for (int i = 0; i < l->n;) { + int j = i; + while (j + 1 < l->n && kv_key_cmp(l->v[j + 1].kv.key, l->v[j + 1].kv.klen, + l->v[i].kv.key, l->v[i].kv.klen) == 0) { + j++; + } + dict->kv[k++] = keep_last ? l->v[j].kv : l->v[i].kv; + i = j + 1; + } + dict->nkv = k; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, l->v); + return dict; +} + +static bool kw_ends_value(const pdf_tok_t *t) { + return pdf_kw_is(t, "endobj") || pdf_kw_is(t, "stream") || pdf_kw_is(t, "endstream"); +} + +pdf_val_t *pdf_parse_value(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + const pdf_tok_t *t, int depth) { + if (depth > PDF_MAX_NESTING || d->nomem) { + return NULL; + } + switch (t->kind) { + case PT_NUM: + if (t->is_int) { + size_t p2 = *pos; + pdf_tok_t t2; + lex_peek(d, data, len, &p2, &t2); + if (t2.kind == PT_NUM && t2.is_int) { + pdf_tok_t t3; + lex_peek(d, data, len, &p2, &t3); + if (pdf_kw_is(&t3, "R")) { + pdf_val_t *r = new_val(d, PV_REF); + if (r) { + r->ref_num = t->inum; + r->ref_gen = t2.inum; + *pos = p2; + } + return r; + } + } + } + return val_from_tok(d, t); + case PT_NAME: + case PT_STR: + return val_from_tok(d, t); + case PT_AS: { + val_list_t l = {0}; + for (;;) { + pdf_tok_t a; + pdf_lex(d, data, len, pos, &a); + if (a.kind == PT_AE || a.kind == PT_EOF) { + break; + } + if (a.kind == PT_DE || a.kind == PT_BE || a.kind == PT_BS) { + continue; + } + if (a.kind == PT_KW && kw_ends_value(&a)) { + break; + } + pdf_val_t *x = pdf_parse_value(d, data, len, pos, &a, depth + 1); + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l.v); + return NULL; + } + vl_push(&l, x); + } + return vl_finish_array(d, &l); + } + case PT_DS: { + kv_list_t l = {0}; + for (;;) { + size_t save = *pos; + pdf_tok_t k; + pdf_lex(d, data, len, pos, &k); + if (k.kind == PT_DE || k.kind == PT_EOF) { + break; + } + if (k.kind == PT_KW && (kw_ends_value(&k) || pdf_kw_is(&k, "obj"))) { + *pos = save; + break; + } + if (k.kind != PT_NAME) { + continue; + } + pdf_tok_t k2; + pdf_lex(d, data, len, pos, &k2); + if (k2.kind == PT_DE) { + kl_push(&l, k.s, k.n, &PDF_NULL_VALUE); + break; + } + pdf_val_t *x = pdf_parse_value(d, data, len, pos, &k2, depth + 1); + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, l.v); + return NULL; + } + kl_push(&l, k.s, k.n, x); + } + return kl_finish_dict(d, &l, true); + } + case PT_KW: + if (pdf_kw_is(t, "true") || pdf_kw_is(t, "false")) { + pdf_val_t *b = new_val(d, PV_BOOL); + if (b) { + b->b = t->s[0] == 't'; + } + return b; + } + if (pdf_kw_is(t, "null")) { + return &PDF_NULL_VALUE; + } + { + pdf_val_t *k = new_val(d, PV_KW); + if (k) { + k->s = t->s; + k->n = t->n; + } + return k; + } + default: + return &PDF_NULL_VALUE; + } +} + +pdf_val_t *pdf_parse_content_array(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, + int depth) { + if (depth > PDF_MAX_NESTING) { + return NULL; + } + val_list_t l = {0}; + for (;;) { + pdf_tok_t a; + pdf_lex(d, data, len, pos, &a); + if (a.kind == PT_AE || a.kind == PT_EOF) { + break; + } + pdf_val_t *x = NULL; + if (a.kind == PT_NUM || a.kind == PT_STR || a.kind == PT_NAME) { + x = val_from_tok(d, &a); + } else if (a.kind == PT_AS) { + x = pdf_parse_content_array(d, data, len, pos, depth + 1); + } else if (a.kind == PT_DS) { + x = pdf_parse_value(d, data, len, pos, &a, depth + 1); + } else if (a.kind == PT_KW) { + *pos = (size_t)(a.s - data); /* the operator is the caller's */ + break; + } else { + continue; + } + if (!x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)l.v); + return NULL; + } + vl_push(&l, x); + } + return vl_finish_array(d, &l); +} + +/* ── Dict access ─────────────────────────────────────────────────── */ + +pdf_val_t *pdf_dfind_n(const pdf_val_t *dict, const unsigned char *key, size_t kn) { + if (!dict || dict->kind != PV_DICT) { + return NULL; + } + int lo = 0; + int hi = dict->nkv - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int c = kv_key_cmp(dict->kv[mid].key, dict->kv[mid].klen, key, kn); + if (c == 0) { + return dict->kv[mid].val; + } + if (c < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return NULL; +} + +static pdf_val_t *dict_find(const pdf_val_t *dict, const char *key) { + return pdf_dfind_n(dict, (const unsigned char *)key, strlen(key)); +} + +pdf_val_t *pdf_dfind(const pdf_val_t *dict, const char *key) { + return dict_find(dict, key); +} + +bool pdf_dhas(const pdf_val_t *dict, const char *key) { + return dict_find(dict, key) != NULL; +} + +pdf_val_t *pdf_dget_raw(const pdf_val_t *dict, const char *key) { + pdf_val_t *v = dict_find(dict, key); + return (v && v->kind != PV_NULL) ? v : NULL; +} + +pdf_val_t *pdf_dget(pdf_doc_t *d, const pdf_val_t *dict, const char *key) { + pdf_val_t *v = pdf_resolve(d, pdf_dget_raw(dict, key)); + return (v && v->kind != PV_NULL) ? v : NULL; +} + +bool pdf_is_name(const pdf_val_t *v, const char *name) { + size_t n = strlen(name); + return v && v->kind == PV_NAME && v->n == n && memcmp(v->s, name, n) == 0; +} + +bool pdf_is_num(const pdf_val_t *v) { + return v && v->kind == PV_NUM; +} + +double pdf_num(const pdf_val_t *v) { + return pdf_is_num(v) ? v->num : 0.0; +} + +/* ── Objects ─────────────────────────────────────────────────────── */ + +/* "[ws]*(\d+)[ws]+(\d+)[ws]+obj" not followed by a regular character, at base. */ +static bool match_obj_hdr(const pdf_doc_t *d, size_t base, int64_t *num, size_t *end) { + const unsigned char *s = d->data; + size_t len = d->len; + size_t p = base; + while (p < len && is_ws(s[p])) { + p++; + } + size_t a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a) { + return false; + } + *num = sat_digits(s + a, p - a); + size_t w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w) { + return false; + } + a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a) { + return false; + } + w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w || p + 3 > len || memcmp(s + p, "obj", 3) != 0) { + return false; + } + p += 3; + if (p < len && is_reg(s[p])) { + return false; + } + *end = p; + return true; +} + +static pdf_xent_t *xent(pdf_doc_t *d, int64_t num) { + int64_t i = pdf_map_get(&d->xref.by_num, (uint64_t)num); + return i < 0 ? NULL : &d->xref.v[i]; +} + +/* "endstream" follows q, after whitespace, within the 32 bytes after it. */ +static bool endstream_follows(const pdf_doc_t *d, size_t q) { + size_t lim = d->len - q < PDF_TAIL_WINDOW ? d->len : q + PDF_TAIL_WINDOW; + for (size_t p = q; p < lim; p++) { + if (!is_ws(d->data[p])) { + return lim - p >= 9 && memcmp(d->data + p, "endstream", 9) == 0; + } + } + return false; +} + +static bool build_hdrs(pdf_doc_t *d); + +/* Where the first object header at or after `at` starts (d->len when none): + * an object's value is read up to there. The objects of a file do not overlap; + * a token that ran on into the next ones (a string without its end) made every + * object before it read the rest of the file again, n objects times the file. + * The bytes of a stream are not bounded by this: its /Length places them. */ +static size_t object_end(pdf_doc_t *d, size_t at) { + if (!build_hdrs(d)) { + return d->len; + } + int lo = 0; + int hi = d->nhdrs; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (d->hdrs[mid].off >= at) { + hi = mid; + } else { + lo = mid + 1; + } + } + return lo < d->nhdrs ? d->hdrs[lo].off : d->len; +} + +/* The value of the object at off (or off + hdr), without its stream; *stream_at + * is where a "stream" keyword after the value starts, 0 when none follows. NULL: + * no matching header there. */ +static pdf_val_t *indirect_value_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect, + bool *ok, size_t *stream_at) { + *ok = false; + *stream_at = 0; + size_t at = 0; + bool found = false; + int64_t num = 0; + for (int k = 0; k < 2 && !found; k++) { + int64_t base = pdf_sat_add(off, k ? (int64_t)d->hdr : 0); + if (base < 0 || (uint64_t)base > d->len) { + continue; + } + size_t end; + if (match_obj_hdr(d, (size_t)base, &num, &end) && (!has_expect || num == expect)) { + at = end; + found = true; + } + } + if (!found) { + return NULL; + } + const unsigned char *s = d->data; + size_t len = object_end(d, at); + size_t p = at; + pdf_tok_t t; + pdf_lex(d, s, len, &p, &t); + pdf_val_t *val = pdf_parse_value(d, s, len, &p, &t, 0); + if (!val) { + return NULL; + } + *ok = true; + size_t p2 = p; + pdf_tok_t t2; + lex_peek(d, s, len, &p2, &t2); + if (pdf_kw_is(&t2, "stream") && val->kind == PV_DICT) { + *stream_at = p2; + } + return val; +} + +static int64_t hdr_first_exact(pdf_doc_t *d, int64_t num, int64_t gen); + +/* An object a stream's /Length names, read as a value only and cached as + * pdf_get caches it. An object that is a stream there gives no length and is + * marked: loading it would read ITS /Length first, one nested call per stream + * of a chain of such streams. */ +static pdf_val_t *length_object(pdf_doc_t *d, int64_t num) { + pdf_xent_t *e = xent(d, num); + bool ok; + size_t stream_at; + pdf_val_t *v = indirect_value_at(d, e->a, true, num, &ok, &stream_at); + if (!ok && !d->reconstructed) { + int64_t off = hdr_first_exact(d, num, e->b); + if (off >= 0) { + v = indirect_value_at(d, off, true, num, &ok, &stream_at); + } + } + e = xent(d, num); + if (ok && stream_at) { + e->not_length = true; + return NULL; + } + e->cached = true; + e->obj = ok ? v : NULL; + return e->obj; +} + +/* A stream's /Length through its references (at most PDF_MAX_RESOLVE, as + * pdf_resolve). */ +static pdf_val_t *length_value(pdf_doc_t *d, pdf_val_t *v) { + for (int depth = 0; v && v->kind == PV_REF; depth++) { + pdf_xent_t *e = xent(d, v->ref_num); + if (depth >= PDF_MAX_RESOLVE || !e || e->loading || e->not_length) { + return NULL; + } + if (e->cached) { + v = e->obj; + } else if (e->type == 1) { + v = length_object(d, v->ref_num); + } else { + v = pdf_get(d, v->ref_num); + } + } + return v; +} + +/* The object at off (or off + hdr), a stream with its raw bytes when one + * follows. NULL: no matching header there. */ +static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect, + bool *ok) { + size_t st; + pdf_val_t *val = indirect_value_at(d, off, has_expect, expect, ok, &st); + if (!val || !st) { + return val; + } + const unsigned char *s = d->data; + size_t len = d->len; + if (st + 1 < len && s[st] == '\r' && s[st + 1] == '\n') { + st += 2; + } else if (st < len && (s[st] == '\n' || s[st] == '\r')) { + st += 1; + } + pdf_val_t *ln = length_value(d, pdf_dget_raw(val, "Length")); + const unsigned char *raw = NULL; + size_t raw_len = 0; + if (ln && ln->kind == PV_NUM && ln->is_int && ln->inum >= 0 && + (uint64_t)ln->inum <= (uint64_t)(len - st)) { + if (endstream_follows(d, st + (size_t)ln->inum)) { + raw = s + st; + raw_len = (size_t)ln->inum; + } + } + if (!raw) { + d->counts->stream_length_recovered++; + const unsigned char *e = cbm_memmem(s + st, len - st, "endstream", 9); + size_t end = e ? (size_t)(e - s) : len; + raw = s + st; + raw_len = end - st; + if (raw_len >= 2 && raw[raw_len - 2] == '\r' && raw[raw_len - 1] == '\n') { + raw_len -= 2; + } else if (raw_len >= 1 && (raw[raw_len - 1] == '\n' || raw[raw_len - 1] == '\r')) { + raw_len -= 1; + } + } + pdf_stream_t *sm = (pdf_stream_t *)pdf_calloc(d, sizeof(*sm)); + pdf_val_t *sv = new_val(d, PV_STREAM); + if (!sm || !sv) { + return NULL; + } + sm->dict = val; + sm->raw = raw; + sm->raw_len = raw_len; + sv->stream = sm; + return sv; +} + +/* (number, position) pairs, sorted to find the first of a number. */ +static int numidx_cmp(const void *x, const void *y) { + const pdf_numidx_t *a = (const pdf_numidx_t *)x; + const pdf_numidx_t *b = (const pdf_numidx_t *)y; + if (a->num != b->num) { + return a->num < b->num ? -1 : 1; + } + return a->idx < b->idx ? -1 : a->idx > b->idx ? 1 : 0; +} + +/* The first pair of `num` in a sorted array; n when none. */ +static int numidx_first(const pdf_numidx_t *v, int n, int64_t num) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (v[mid].num < num) { + lo = mid + 1; + } else { + hi = mid; + } + } + return (lo < n && v[lo].num == num) ? lo : n; +} + +/* Every "N G obj" header in file order (the reconstruction and the fallback + * for objects missing at their offset both read it). */ +static bool build_hdrs(pdf_doc_t *d) { + if (d->hdrs_built) { + return true; + } + d->hdrs_built = true; + const unsigned char *s = d->data; + size_t len = d->len; + int cap = 0; + size_t from = 0; + while (from + 3 <= len) { + const unsigned char *o = cbm_memmem(s + from, len - from, "obj", 3); + if (!o) { + break; + } + size_t q = (size_t)(o - s); + from = q + 3; + size_t p = q; + while (p > 0 && is_ws(s[p - 1])) { + p--; + } + if (p == q) { + continue; + } + size_t ge = p; + while (p > 0 && is_digit(s[p - 1])) { + p--; + } + size_t gs = p; + if (gs == ge) { + continue; + } + size_t w = p; + while (p > 0 && is_ws(s[p - 1])) { + p--; + } + if (p == w) { + continue; + } + size_t ne = p; + while (p > 0 && is_digit(s[p - 1])) { + p--; + } + size_t ns = p; + if (ns == ne) { + continue; + } + if (d->nhdrs == cap) { + int ncap = cap ? cap * 2 : PDF_MAP_MIN; + pdf_hdr_t *g = + (pdf_hdr_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, d->hdrs, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + d->hdrs = g; + cap = ncap; + } + pdf_hdr_t *h = &d->hdrs[d->nhdrs++]; + h->num = sat_digits(s + ns, ne - ns); + h->gen = sat_digits(s + gs, ge - gs); + h->off = ns; + h->num_digits = (uint8_t)((ne - ns) > UINT8_MAX ? UINT8_MAX : (ne - ns)); + h->gen_digits = (uint8_t)((ge - gs) > UINT8_MAX ? UINT8_MAX : (ge - gs)); + h->canonical = (s[ns] != '0' || ne - ns == 1) && (s[gs] != '0' || ge - gs == 1); + h->reg_after = q + 3 < len && is_reg(s[q + 3]); + } + return true; +} + +/* The first header written exactly "num gen obj" (the prototype's search). */ +static int64_t hdr_first_exact(pdf_doc_t *d, int64_t num, int64_t gen) { + if (!build_hdrs(d) || !d->nhdrs) { + return -1; + } + if (!d->hdr_sorted) { + d->hdr_sorted = (pdf_numidx_t *)pdf_big(d, (size_t)d->nhdrs * sizeof(pdf_numidx_t)); + if (!d->hdr_sorted) { + return -1; + } + for (int i = 0; i < d->nhdrs; i++) { + d->hdr_sorted[i].num = d->hdrs[i].num; + d->hdr_sorted[i].idx = i; + } + qsort(d->hdr_sorted, (size_t)d->nhdrs, sizeof(pdf_numidx_t), numidx_cmp); + } + for (int k = numidx_first(d->hdr_sorted, d->nhdrs, num); + k < d->nhdrs && d->hdr_sorted[k].num == num; k++) { + const pdf_hdr_t *h = &d->hdrs[d->hdr_sorted[k].idx]; + if (h->gen == gen && h->canonical) { + return (int64_t)h->off; + } + } + return -1; +} + +static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum); + +struct pdf_objstm { + const unsigned char *data; + size_t len; + int64_t first; + int64_t *onum; /* member numbers, in stream order */ + int64_t *ooff; + int n; + pdf_numidx_t *sorted; /* (onum, order), built on first need */ + int64_t *starts; /* first + each offset, ascending, built on first need */ +}; + +static int i64_cmp(const void *x, const void *y); + +/* Where the member after the one at `at` starts (sm->len when none): members + * do not overlap, as the objects of a file do not (object_end). */ +static size_t member_end(pdf_doc_t *d, pdf_objstm_t *sm, size_t at) { + if (!sm->starts) { + sm->starts = (int64_t *)pdf_big(d, (size_t)(sm->n ? sm->n : 1) * sizeof(int64_t)); + if (!sm->starts) { + return sm->len; + } + for (int i = 0; i < sm->n; i++) { + sm->starts[i] = pdf_sat_add(sm->first, sm->ooff[i]); + } + qsort(sm->starts, (size_t)sm->n, sizeof(int64_t), i64_cmp); + } + int lo = 0; + int hi = sm->n; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (sm->starts[mid] > (int64_t)at) { + hi = mid; + } else { + lo = mid + 1; + } + } + if (lo < sm->n && (uint64_t)sm->starts[lo] < (uint64_t)sm->len) { + return (size_t)sm->starts[lo]; + } + return sm->len; +} + +static int64_t objstm_find(pdf_doc_t *d, pdf_objstm_t *sm, int64_t num, int64_t idx) { + if (idx >= 0 && idx < sm->n && sm->onum[idx] == num) { + return sm->ooff[idx]; + } + if (!sm->sorted) { + sm->sorted = (pdf_numidx_t *)pdf_big(d, (size_t)(sm->n ? sm->n : 1) * sizeof(pdf_numidx_t)); + if (!sm->sorted) { + return INT64_MIN; + } + for (int i = 0; i < sm->n; i++) { + sm->sorted[i].num = sm->onum[i]; + sm->sorted[i].idx = i; + } + qsort(sm->sorted, (size_t)sm->n, sizeof(pdf_numidx_t), numidx_cmp); + } + int k = numidx_first(sm->sorted, sm->n, num); + return k < sm->n ? sm->ooff[sm->sorted[k].idx] : INT64_MIN; +} + +pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num) { + int64_t idx = pdf_map_get(&d->xref.by_num, (uint64_t)num); + if (idx < 0) { + return NULL; + } + pdf_xent_t *e = &d->xref.v[idx]; + if (e->cached) { + return e->obj; + } + if (e->loading) { + return NULL; + } + if (e->type == 2 && d->objstm_opening > 0) { + /* The objects that open an object stream (its /Length, its filters) are + * no objects of an object stream (ISO 32000-1, 7.5.7): one that is would + * open the next stream inside this one, one nested call per stream of a + * chain. */ + d->counts->objstm_nested++; + return NULL; + } + e->loading = true; + uint8_t type = e->type; + int64_t a = e->a; + int64_t b = e->b; + pdf_val_t *obj = NULL; + if (type == 1) { + bool ok; + obj = parse_indirect_at(d, a, true, num, &ok); + if (!ok && !d->reconstructed) { + int64_t off = hdr_first_exact(d, num, b); + if (off >= 0) { + obj = parse_indirect_at(d, off, true, num, &ok); + } + } + if (!ok) { + obj = NULL; + } + } else if (type == 2) { + pdf_objstm_t *sm = objstm(d, a); + if (sm) { + int64_t off = objstm_find(d, sm, num, b); + int64_t at = off == INT64_MIN ? -1 : pdf_sat_add(sm->first, off); + if (at >= 0 && (uint64_t)at <= (uint64_t)sm->len) { + size_t p = (size_t)at; + size_t end = member_end(d, sm, (size_t)at); + pdf_tok_t t; + pdf_lex(d, sm->data, end, &p, &t); + obj = pdf_parse_value(d, sm->data, end, &p, &t, 0); + } + } + } + e = &d->xref.v[idx]; + e->loading = false; + e->cached = true; + e->obj = obj; + return obj; +} + +pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v) { + int depth = 0; + while (v && v->kind == PV_REF) { + v = pdf_get(d, v->ref_num); + if (++depth > PDF_MAX_RESOLVE) { + return NULL; + } + } + return v; +} + +static pdf_objstm_t *objstm_open(pdf_doc_t *d, int64_t stmnum) { + pdf_xent_t *e = xent(d, stmnum); + if (e && e->stm) { + return e->stm; + } + pdf_val_t *st = pdf_get(d, stmnum); + if (!st || st->kind != PV_STREAM) { + return NULL; + } + size_t dlen = 0; + const unsigned char *data = pdf_stream_decode(d, st->stream, &dlen); + if (!data) { + data = (const unsigned char *)""; + dlen = 0; + } + pdf_val_t *nv = pdf_dget_raw(st->stream->dict, "N"); + pdf_val_t *fv = pdf_dget_raw(st->stream->dict, "First"); + int64_t n = (nv && nv->kind == PV_NUM && nv->is_int) ? nv->inum : 0; + pdf_objstm_t *sm = (pdf_objstm_t *)pdf_calloc(d, sizeof(*sm)); + if (!sm) { + return NULL; + } + sm->data = data; + sm->len = dlen; + sm->first = (fv && fv->kind == PV_NUM && fv->is_int) ? fv->inum : 0; + /* at most one pair per two tokens of the header: bounded by the data */ + int64_t cap = n < 0 ? 0 : n; + if (cap > (int64_t)(dlen / 2 + 1)) { + cap = (int64_t)(dlen / 2 + 1); + } + sm->onum = (int64_t *)pdf_big(d, (size_t)(cap ? cap : 1) * sizeof(int64_t)); + sm->ooff = (int64_t *)pdf_big(d, (size_t)(cap ? cap : 1) * sizeof(int64_t)); + if (!sm->onum || !sm->ooff) { + return NULL; + } + size_t p = 0; + for (int64_t i = 0; i < n && sm->n < cap; i++) { + pdf_tok_t t1; + pdf_tok_t t2; + lex_peek(d, data, dlen, &p, &t1); + lex_peek(d, data, dlen, &p, &t2); + if (t1.kind != PT_NUM || t2.kind != PT_NUM) { + break; + } + sm->onum[sm->n] = t1.is_int ? t1.inum : pdf_d2i(t1.num); + sm->ooff[sm->n] = t2.is_int ? t2.inum : pdf_d2i(t2.num); + sm->n++; + } + e = xent(d, stmnum); + if (e) { + e->stm = sm; + } + return sm; +} + +static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) { + d->objstm_opening++; + pdf_objstm_t *sm = objstm_open(d, stmnum); + d->objstm_opening--; + return sm; +} + +/* ── Cross-reference ─────────────────────────────────────────────── */ + +static void xref_free(pdf_xref_t *x) { + cbm_free(CBM_MEM_CLASS_EXTRACT, x->v); + pdf_map_free(&x->by_num); + memset(x, 0, sizeof(*x)); +} + +/* The entry for num; created (zeroed) when absent. *fresh says which. */ +static pdf_xent_t *xref_slot(pdf_doc_t *d, pdf_xref_t *x, int64_t num, bool *fresh) { + int64_t i = pdf_map_get(&x->by_num, (uint64_t)num); + if (i >= 0) { + *fresh = false; + return &x->v[i]; + } + if (x->n == x->cap) { + int ncap = x->cap ? x->cap * 2 : PDF_MAP_MIN; + pdf_xent_t *g = + (pdf_xent_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, x->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return NULL; + } + x->v = g; + x->cap = ncap; + } + if (!pdf_map_put(&x->by_num, (uint64_t)num, x->n)) { + d->nomem = true; + return NULL; + } + pdf_xent_t *e = &x->v[x->n++]; + memset(e, 0, sizeof(*e)); + e->num = num; + *fresh = true; + return e; +} + +/* "[ws]*(\d{1,10})[ws]+(\d{1,5})[ws]*([nf])" at p. */ +static bool match_xref_entry(const pdf_doc_t *d, size_t p, int64_t *off, int64_t *gen, char *kind, + size_t *end) { + const unsigned char *s = d->data; + size_t len = d->len; + while (p < len && is_ws(s[p])) { + p++; + } + size_t a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a || p - a > 10) { + return false; + } + *off = sat_digits(s + a, p - a); + size_t w = p; + while (p < len && is_ws(s[p])) { + p++; + } + if (p == w) { + return false; + } + a = p; + while (p < len && is_digit(s[p])) { + p++; + } + if (p == a || p - a > 5) { + return false; + } + *gen = sat_digits(s + a, p - a); + while (p < len && is_ws(s[p])) { + p++; + } + if (p >= len || (s[p] != 'n' && s[p] != 'f')) { + return false; + } + *kind = (char)s[p]; + *end = p + 1; + return true; +} + +static bool xref_table(pdf_doc_t *d, size_t p, pdf_xref_t *sec, pdf_val_t **tr) { + const unsigned char *s = d->data; + size_t len = d->len; + for (;;) { + pdf_tok_t t; + size_t p2 = p; + pdf_lex(d, s, len, &p2, &t); + if (pdf_kw_is(&t, "trailer")) { + pdf_tok_t t2; + pdf_lex(d, s, len, &p2, &t2); + pdf_val_t *v = pdf_parse_value(d, s, len, &p2, &t2, 0); + if (!v) { + return false; + } + *tr = v->kind == PV_DICT ? v : NULL; + return true; + } + if (t.kind != PT_NUM) { + return false; + } + pdf_tok_t tc; + pdf_lex(d, s, len, &p2, &tc); + if (tc.kind != PT_NUM || !tc.is_int) { + return false; + } + int64_t start = t.is_int ? t.inum : pdf_d2i(t.num); + bool use = t.is_int || t.num == (double)start; + p = p2; + for (int64_t i = 0; i < tc.inum; i++) { + int64_t off; + int64_t gen; + char kind; + size_t end; + if (!match_xref_entry(d, p, &off, &gen, &kind, &end)) { + return false; + } + p = end; + if (!use) { + continue; + } + bool fresh; + pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh); + if (!e) { + return false; + } + if (fresh && kind == 'n') { + e->type = 1; + e->a = off; + e->b = gen; + } + } + } +} + +static int64_t be_field(const unsigned char *p, int w) { + uint64_t v = 0; + for (int i = 0; i < w; i++) { + v = (v << PDF_BITS_PER_BYTE) | p[i]; + } + return (int64_t)v; +} + +static bool num_to_int(const pdf_val_t *v, int64_t *out) { + if (!v || v->kind != PV_NUM) { + return false; + } + *out = v->is_int ? v->inum : pdf_d2i(v->num); + return true; +} + +static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len, + pdf_buf_t *held_out); + +/* The rows of a cross-reference stream's decoded bytes into sec. */ +static bool xref_rows(pdf_doc_t *d, pdf_xref_t *sec, const unsigned char *raw, size_t rlen, + const int64_t w[3], pdf_val_t **items, int nitems, const int64_t pairs[2]) { + int64_t rec = w[0] + w[1] + w[2]; + if (rec == 0) { + return false; + } + size_t pos = 0; + for (int j = 0; j + 1 < nitems; j += 2) { + int64_t start; + int64_t count; + if (items) { + if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) { + return false; + } + } else { + start = pairs[0]; + count = pairs[1]; + } + for (int64_t i = 0; i < count; i++) { + if (pos + (size_t)rec > rlen) { + break; + } + const unsigned char *r = raw + pos; + int64_t t = w[0] ? be_field(r, (int)w[0]) : 1; + int64_t f2 = be_field(r + w[0], (int)w[1]); + int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0; + pos += (size_t)rec; + bool fresh; + pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh); + if (!e) { + return false; + } + if (!fresh) { + continue; + } + if (t == 1 || t == 2) { + e->type = (uint8_t)t; + e->a = f2; + e->b = f3; + } + } + } + return true; +} + +/* A cross-reference stream at off into sec. Its decoded bytes are freed once + * read: a stream parsed here is no cached object, and keeping them made every + * section that names one stream add a copy until the document closed. */ +static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_t **tr) { + bool ok; + pdf_val_t *v = parse_indirect_at(d, off, false, 0, &ok); + if (!v || v->kind != PV_STREAM) { + return false; + } + pdf_val_t *dict = v->stream->dict; + pdf_val_t *W = pdf_dget_raw(dict, "W"); + if (!W || W->kind != PV_ARR || W->count != 3) { + return false; + } + int64_t w[3]; + for (int i = 0; i < 3; i++) { + if (!num_to_int(W->items[i], &w[i]) || w[i] < 0 || w[i] > PDF_MAX_FIELD) { + return false; + } + } + pdf_val_t *size = pdf_dget_raw(dict, "Size"); + pdf_val_t *index = pdf_dget_raw(dict, "Index"); + bool dflt = !index || (index->kind == PV_ARR && index->count == 0) || + (index->kind == PV_NUM && index->num == 0.0) || + (index->kind == PV_BOOL && !index->b); + int64_t pairs[2] = {0, 0}; /* read only on the default /Index (no items) */ + pdf_val_t **items = NULL; + int nitems = 0; + if (dflt) { + if (size && !num_to_int(size, &pairs[1])) { + return false; + } + nitems = 2; + } else if (index->kind == PV_ARR) { + items = index->items; + nitems = index->count; + } else { + return false; + } + size_t rlen = 0; + pdf_buf_t held = {0}; + const unsigned char *raw = decode_filters(d, v->stream, &rlen, &held); + if (!raw) { + return false; + } + d->counts->xref_streams++; + bool rows_ok = xref_rows(d, sec, raw, rlen, w, items, nitems, pairs); + buf_free(&held); + if (!rows_ok) { + return false; + } + *tr = dict; + return true; +} + +static size_t skip_ws(const pdf_doc_t *d, size_t p) { + while (p < d->len && is_ws(d->data[p])) { + p++; + } + return p; +} + +static bool starts_with(const pdf_doc_t *d, size_t p, const char *lit) { + size_t n = strlen(lit); + return p <= d->len && d->len - p >= n && memcmp(d->data + p, lit, n) == 0; +} + +static bool find_xref_at(pdf_doc_t *d, int64_t off, size_t *out) { + for (int k = 0; k < 2; k++) { + int64_t base = pdf_sat_add(off, k ? (int64_t)d->hdr : 0); + if (base < 0 || (uint64_t)base > d->len) { + continue; + } + size_t p = skip_ws(d, (size_t)base); + if (starts_with(d, p, "xref")) { + *out = p; + return true; + } + int64_t num; + size_t end; + if (match_obj_hdr(d, (size_t)base, &num, &end)) { + *out = (size_t)base; + return true; + } + } + int64_t lo = pdf_sat_add(off, -PDF_XREF_NEAR); + lo = lo < 0 ? 0 : lo; + int64_t hi = pdf_sat_add(off, PDF_XREF_NEAR); + if ((uint64_t)hi > d->len) { + hi = (int64_t)d->len; + } + if (hi - lo >= 4 && (uint64_t)lo < d->len) { + const unsigned char *x = cbm_memmem(d->data + lo, (size_t)(hi - lo), "xref", 4); + if (x) { + d->counts->xref_offset_fixed++; + *out = (size_t)(x - d->data); + return true; + } + } + return false; +} + +static bool trailer_key_structural(const pdf_kv_t *kv) { + static const char *const keys[] = {"Prev", "XRefStm", "Type", "W", "Index", + "Filter", "DecodeParms", "Length", "Size"}; + for (size_t i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + size_t n = strlen(keys[i]); + if (kv->klen == n && memcmp(kv->key, keys[i], n) == 0) { + return true; + } + } + return false; +} + +static bool load_xref(pdf_doc_t *d) { + const unsigned char *s = d->data; + size_t len = d->len; + /* the last "startxref" */ + size_t i = len < 9 ? 0 : len - 9 + 1; + bool found = false; + while (i-- > 0) { + if (s[i] == 's' && memcmp(s + i, "startxref", 9) == 0) { + found = true; + break; + } + } + if (!found) { + return false; + } + size_t p = i + 9; + pdf_tok_t t; + pdf_lex(d, s, len, &p, &t); + if (t.kind != PT_NUM) { + return false; + } + pdf_map_t seen = {0}; + /* /XRefStm targets already merged: a later (older) section naming one again + * adds nothing, since every number it holds is in d->xref already */ + pdf_map_t stm_seen = {0}; + kv_list_t trail = {0}; + bool first = true; + bool ok = true; + pdf_val_t *offv = new_val(d, PV_NUM); + if (!offv) { + return false; + } + offv->is_int = t.is_int; + offv->inum = t.inum; + pdf_val_t *off = offv; + while (off && off->kind == PV_NUM && off->is_int && + pdf_map_get(&seen, (uint64_t)off->inum) < 0 && seen.n < PDF_MAX_PREV) { + if (!pdf_map_put(&seen, (uint64_t)off->inum, 1)) { + d->nomem = true; + ok = false; + break; + } + size_t at; + if (!find_xref_at(d, off->inum, &at)) { + ok = false; + break; + } + pdf_xref_t sec = {0}; + pdf_val_t *tr = NULL; + if (starts_with(d, at, "xref")) { + if (!xref_table(d, at + 4, &sec, &tr)) { + xref_free(&sec); + ok = false; + break; + } + pdf_val_t *xs = pdf_dget_raw(tr, "XRefStm"); + if (xs && xs->kind == PV_NUM && xs->is_int && + pdf_map_get(&stm_seen, (uint64_t)xs->inum) < 0) { + if (!pdf_map_put(&stm_seen, (uint64_t)xs->inum, 1)) { + d->nomem = true; + xref_free(&sec); + ok = false; + break; + } + pdf_xref_t sec2 = {0}; + pdf_val_t *tr2 = NULL; + if (xref_stream_at(d, xs->inum, &sec2, &tr2)) { + for (int k = 0; k < sec2.n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &sec, sec2.v[k].num, &fresh); + if (e && (fresh || e->type == 0)) { + e->type = sec2.v[k].type; + e->a = sec2.v[k].a; + e->b = sec2.v[k].b; + } + } + } + xref_free(&sec2); + } + } else if (!xref_stream_at(d, (int64_t)at, &sec, &tr)) { + xref_free(&sec); + ok = false; + break; + } + for (int k = 0; k < sec.n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, sec.v[k].num, &fresh); + if (e && fresh) { + e->type = sec.v[k].type; + e->a = sec.v[k].a; + e->b = sec.v[k].b; + } + } + xref_free(&sec); + for (int k = 0; tr && k < tr->nkv; k++) { + if (first || !trailer_key_structural(&tr->kv[k])) { + kl_push(&trail, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + } + first = false; + off = pdf_dget_raw(tr, "Prev"); + } + pdf_map_free(&seen); + pdf_map_free(&stm_seen); + d->trailer = kl_finish_dict(d, &trail, false); + if (!ok || d->nomem) { + return false; + } + return d->xref.n > 0; +} + +static int i64_cmp(const void *x, const void *y) { + int64_t a = *(const int64_t *)x; + int64_t b = *(const int64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +/* The xref's object numbers, ascending (a snapshot). */ +static int64_t *xref_sorted_nums(pdf_doc_t *d, int *n) { + *n = d->xref.n; + int64_t *v = (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(*n ? *n : 1) * sizeof(*v)); + if (!v) { + d->nomem = true; + *n = 0; + return NULL; + } + for (int i = 0; i < *n; i++) { + v[i] = d->xref.v[i].num; + } + qsort(v, (size_t)*n, sizeof(*v), i64_cmp); + return v; +} + +static void cache_clear(pdf_doc_t *d) { + for (int i = 0; i < d->xref.n; i++) { + d->xref.v[i].cached = false; + d->xref.v[i].obj = NULL; + } +} + +static void reconstruct(pdf_doc_t *d) { + d->reconstructed = true; + d->counts->xref_reconstructed++; + xref_free(&d->xref); + if (!build_hdrs(d)) { + return; + } + for (int i = 0; i < d->nhdrs; i++) { + const pdf_hdr_t *h = &d->hdrs[i]; + if (h->num_digits > 10 || h->gen_digits > 5 || h->reg_after) { + continue; + } + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, h->num, &fresh); + if (!e) { + return; + } + e->type = 1; + e->a = (int64_t)h->off; + e->b = h->gen; + } + /* every "trailer[ws]*<<" dict, the later overriding */ + const unsigned char *s = d->data; + size_t len = d->len; + kv_list_t trail = {0}; + size_t from = 0; + while (from + 7 <= len) { + const unsigned char *x = cbm_memmem(s + from, len - from, "trailer", 7); + if (!x) { + break; + } + size_t q = (size_t)(x - s) + 7; + from = q; + q = skip_ws(d, q); + if (!starts_with(d, q, "<<")) { + continue; + } + size_t p = q; + pdf_tok_t t; + /* read up to the next "trailer": scanned trailers do not overlap either */ + const unsigned char *nx = cbm_memmem(s + q, len - q, "trailer", 7); + size_t tlen = nx ? (size_t)(nx - s) : len; + pdf_lex(d, s, tlen, &p, &t); + pdf_val_t *v = pdf_parse_value(d, s, tlen, &p, &t, 0); + if (v && v->kind == PV_DICT) { + for (int k = 0; k < v->nkv; k++) { + kl_push(&trail, v->kv[k].key, v->kv[k].klen, v->kv[k].val); + } + } + from = q + 2 > from ? q + 2 : from; + } + pdf_val_t *tr = kl_finish_dict(d, &trail, true); + static const char *const xkeys[] = {"Root", "Info", "ID", "Encrypt"}; + bool taken[4] = {false, false, false, false}; + kv_list_t extra = {0}; + int n = 0; + int64_t *nums = xref_sorted_nums(d, &n); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (!o || o->kind != PV_STREAM) { + continue; + } + pdf_val_t *typ = pdf_dget_raw(o->stream->dict, "Type"); + if (pdf_is_name(typ, "ObjStm")) { + pdf_objstm_t *sm = objstm(d, nums[i]); + for (int k = 0; sm && k < sm->n; k++) { + bool fresh; + pdf_xent_t *e = xref_slot(d, &d->xref, sm->onum[k], &fresh); + if (e && fresh) { + e->type = 2; + e->a = nums[i]; + e->b = k; + } + } + } else if (pdf_is_name(typ, "XRef")) { + for (size_t k = 0; k < sizeof(xkeys) / sizeof(xkeys[0]); k++) { + pdf_val_t *v = dict_find(o->stream->dict, xkeys[k]); + if (v && !taken[k] && !dict_find(tr, xkeys[k])) { + kl_push(&extra, (const unsigned char *)xkeys[k], strlen(xkeys[k]), v); + taken[k] = true; + } + } + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); + if (extra.n && tr) { + for (int k = 0; k < tr->nkv; k++) { + kl_push(&extra, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + /* the scanned trailers come after: keep_last lets them win, and the + * stream keys were only added where the trailers had none */ + tr = kl_finish_dict(d, &extra, true); + } else { + cbm_free(CBM_MEM_CLASS_EXTRACT, extra.v); + } + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(tr, "Root")); + if (!root || root->kind != PV_DICT) { + nums = xref_sorted_nums(d, &n); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (o && o->kind == PV_DICT && pdf_is_name(pdf_dget_raw(o, "Type"), "Catalog")) { + pdf_val_t *r = new_val(d, PV_REF); + if (r) { + r->ref_num = nums[i]; + kv_list_t one = {0}; + for (int k = 0; tr && k < tr->nkv; k++) { + kl_push(&one, tr->kv[k].key, tr->kv[k].klen, tr->kv[k].val); + } + kl_push(&one, (const unsigned char *)"Root", 4, r); + tr = kl_finish_dict(d, &one, true); + } + break; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); + } + d->trailer = tr; + cache_clear(d); +} + +bool pdf_doc_open(pdf_doc_t *d) { + const unsigned char *h = cbm_memmem(d->data, d->len, "%PDF-", 5); + d->hdr = h ? (size_t)(h - d->data) : 0; + bool ok = load_xref(d); + if (ok) { + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(d->trailer, "Root")); + ok = root && root->kind == PV_DICT; + } + if (!ok && !d->nomem) { + reconstruct(d); + } + return !d->nomem; +} + +void pdf_doc_close(pdf_doc_t *d) { + if (d->scratch_arena_live) { + cbm_arena_destroy(&d->scratch_arena); + } + xref_free(&d->xref); + pdf_map_free(&d->fonts); + cbm_free(CBM_MEM_CLASS_EXTRACT, d->hdrs); + cbm_free(CBM_MEM_CLASS_EXTRACT, d->scratch); + for (int i = 0; i < d->nowned; i++) { + cbm_free(CBM_MEM_CLASS_EXTRACT, d->owned[i]); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)d->owned); + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)d->font_list); +} + +/* ── Filters ─────────────────────────────────────────────────────── */ + +/* Inflate src into out until the stream ends, an error, or the output passes + * the guard. chunk: feed the input in the prototype's 4096-byte steps, where + * an error drops the output of the step it happened in. */ +static void inflate_run(const unsigned char *src, size_t n, int wbits, bool chunk, pdf_buf_t *out, + bool *ended, bool *errored) { + *ended = false; + *errored = false; + z_stream z; + memset(&z, 0, sizeof(z)); + if (inflateInit2(&z, wbits) != Z_OK) { + *errored = true; + return; + } + size_t step = chunk ? PDF_INFLATE_CHUNK : (n ? n : 1); + for (size_t i = 0; i < n || (n == 0 && i == 0); i += step) { + size_t cl = (n - i < step) ? n - i : step; + size_t mark = out->n; + z.next_in = (Bytef *)(uintptr_t)(src + i); + z.avail_in = (uInt)cl; + bool bad = false; + for (;;) { + if (!buf_reserve(out, PDF_INFLATE_GROW)) { + bad = true; + break; + } + z.next_out = out->p + out->n; + z.avail_out = (uInt)(out->cap - out->n > UINT_MAX ? UINT_MAX : out->cap - out->n); + uInt before = z.avail_out; + int r = inflate(&z, Z_NO_FLUSH); + out->n += before - z.avail_out; + if (r == Z_STREAM_END) { + *ended = true; + break; + } + if (r == Z_BUF_ERROR) { + break; + } + if (r != Z_OK) { + bad = true; + break; + } + if (out->n > PDF_MAX_STREAM_OUT) { + break; + } + if (z.avail_in == 0 && z.avail_out != 0) { + break; + } + } + if (bad) { + out->n = mark; + *errored = true; + break; + } + if (*ended || out->n > PDF_MAX_STREAM_OUT || n == 0) { + break; + } + } + inflateEnd(&z); +} + +static bool f_flate(pdf_doc_t *d, const unsigned char *in, size_t n, pdf_buf_t *out) { + bool ended; + bool errored; + inflate_run(in, n, PDF_ZLIB_WBITS, false, out, &ended, &errored); + if (out->fail) { + return false; + } + if (ended || out->n > PDF_MAX_STREAM_OUT) { + return true; + } + d->counts->flate_recovered++; + for (int k = 0; k < 2; k++) { + out->n = 0; + const unsigned char *src = k ? in + (n >= 2 ? 2 : n) : in; + size_t sn = k ? (n >= 2 ? n - 2 : 0) : n; + inflate_run(src, sn, k ? -PDF_ZLIB_WBITS : PDF_ZLIB_WBITS, true, out, &ended, &errored); + if (out->fail) { + return false; + } + if (out->n) { + return true; + } + } + out->n = 0; + return true; +} + +/* PNG (>= 10) and TIFF (2) predictors. false: parameters it cannot apply. */ +static bool f_predict(pdf_doc_t *d, pdf_val_t *parms, pdf_buf_t *io) { + if (!parms || parms->kind != PV_DICT) { + return true; + } + pdf_val_t *pv = pdf_dget(d, parms, "Predictor"); + double pred = 1; + if (pv) { + if (pv->kind != PV_NUM) { + return false; + } + pred = pv->num == 0.0 ? 1 : pv->num; + } + if (pred == 1) { + return true; + } + static const char *const keys[] = {"Colors", "BitsPerComponent", "Columns"}; + static const int64_t dflt[] = {1, 8, 1}; + int64_t prm[3]; + bool ints = true; + for (int i = 0; i < 3; i++) { + pdf_val_t *v = pdf_dget(d, parms, keys[i]); + if (!v || (v->kind == PV_NUM && v->num == 0.0)) { + prm[i] = dflt[i]; + } else if (v->kind != PV_NUM) { + return false; + } else { + ints = ints && v->is_int; + prm[i] = v->is_int ? v->inum : pdf_d2i(v->num); + } + } + int64_t colors = prm[0]; + int64_t bpc = prm[1]; + int64_t cols = prm[2]; + if (pred < PDF_PNG_PRED && !(pred == PDF_TIFF_PRED && bpc == PDF_BITS_PER_BYTE)) { + return true; + } + if (!ints) { + return false; + } + /* products of hostile parameters: computed in double and bounded */ + double bits = (double)colors * (double)bpc; + double rowd = floor((bits * (double)cols + 7) / PDF_BITS_PER_BYTE); + double bppd = floor(bits / PDF_BITS_PER_BYTE); + if (rowd > (double)PDF_MAX_STREAM_OUT) { + return false; + } + int64_t rowlen = rowd < -2 ? -2 : (int64_t)rowd; + int64_t bpp = + bppd<1 ? 1 : bppd>(double) PDF_MAX_STREAM_OUT ? (int64_t)PDF_MAX_STREAM_OUT : (int64_t)bppd; + if (pred >= PDF_PNG_PRED) { + if (rowlen == -1) { + return false; + } + pdf_buf_t out = {0}; + if (rowlen < 0) { + buf_free(io); + *io = out; + return true; + } + size_t rl = (size_t)rowlen; + unsigned char *prev = (unsigned char *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, rl ? rl : 1); + if (!prev) { + d->nomem = true; + return false; + } + for (size_t i = 0; i < io->n; i += rl + 1) { + unsigned ft = io->p[i]; + if (!buf_reserve(&out, rl ? rl : 1)) { + break; + } + unsigned char *row = out.p + out.n; + size_t have = io->n - (i + 1) < rl ? io->n - (i + 1) : rl; + if (have) { + memcpy(row, io->p + i + 1, have); + } + memset(row + have, 0, rl - have); + size_t b = (size_t)bpp; + for (size_t j = 0; j < rl; j++) { + unsigned a = j >= b ? row[j - b] : 0; + unsigned up = prev[j]; + unsigned c = j >= b ? prev[j - b] : 0; + switch (ft) { + case 1: + if (j >= b) { + row[j] = (unsigned char)(row[j] + a); + } + break; + case 2: + row[j] = (unsigned char)(row[j] + up); + break; + case 3: + row[j] = (unsigned char)(row[j] + ((a + up) >> 1)); + break; + case 4: { + int p = (int)a + (int)up - (int)c; + int pa = abs(p - (int)a); + int pb = abs(p - (int)up); + int pc = abs(p - (int)c); + unsigned pr = (pa <= pb && pa <= pc) ? a : (pb <= pc ? up : c); + row[j] = (unsigned char)(row[j] + pr); + break; + } + default: + break; + } + } + if (rl) { + memcpy(prev, row, rl); + } + out.n += rl; + if (out.n > PDF_MAX_STREAM_OUT) { + break; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, prev); + if (out.fail) { + buf_free(&out); + d->nomem = true; + return false; + } + buf_free(io); + *io = out; + return true; + } + /* TIFF predictor 2, 8 bits per component */ + if (rowlen == 0) { + return false; + } + if (rowlen < 0) { + return true; + } + size_t rl = (size_t)rowlen; + size_t b = (size_t)bpp; + for (size_t r = 0; r < io->n; r += rl) { + size_t lim = r + rl < io->n ? r + rl : io->n; + for (size_t j = r + b; j < lim; j++) { + io->p[j] = (unsigned char)(io->p[j] + io->p[j - b]); + } + } + return true; +} + +static void f_lzw(const unsigned char *in, size_t n, double early, pdf_buf_t *out) { + /* table entries >= 258 are runs of the output: (position, length) */ + size_t *pos = (size_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, PDF_LZW_TABLE * sizeof(size_t)); + size_t *lenv = (size_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, PDF_LZW_TABLE * sizeof(size_t)); + if (!pos || !lenv) { + out->fail = true; + cbm_free(CBM_MEM_CLASS_EXTRACT, pos); + cbm_free(CBM_MEM_CLASS_EXTRACT, lenv); + return; + } + int bits = PDF_LZW_MIN_BITS; + uint32_t buf = 0; + int nbits = 0; + int64_t table = PDF_LZW_FIRST; /* the prototype's table length (it never shrinks below) */ + bool have_prev = false; + size_t prev_pos = 0; + size_t prev_len = 0; + for (size_t i = 0; i < n; i++) { + buf = (buf << PDF_BITS_PER_BYTE) | in[i]; + nbits += PDF_BITS_PER_BYTE; + while (nbits >= bits) { + nbits -= bits; + int64_t code = (int64_t)((buf >> nbits) & ((1U << bits) - 1)); + if (code == PDF_LZW_CLEAR) { + table = PDF_LZW_FIRST; + bits = PDF_LZW_MIN_BITS; + have_prev = false; + continue; + } + if (code == PDF_LZW_EOD) { + goto done; + } + size_t at = out->n; + if (!have_prev) { + if (code >= table) { + goto done; + } + if (!buf_byte(out, (unsigned char)code)) { + goto done; + } + } else if (code < table) { + if (code < PDF_LZW_CLEAR) { + if (!buf_byte(out, (unsigned char)code)) { + goto done; + } + } else { + size_t sp = pos[code]; + size_t sl = lenv[code]; + if (!buf_reserve(out, sl)) { + goto done; + } + memmove(out->p + out->n, out->p + sp, sl); + out->n += sl; + } + if (table < PDF_LZW_TABLE) { + pos[table] = prev_pos; + lenv[table] = prev_len + 1; + } + table++; + } else if (code == table) { + size_t sl = prev_len + 1; + if (!buf_reserve(out, sl)) { + goto done; + } + for (size_t k = 0; k < sl; k++) { + out->p[out->n + k] = out->p[prev_pos + k]; + } + out->n += sl; + if (table < PDF_LZW_TABLE) { + pos[table] = prev_pos; + lenv[table] = sl; + } + table++; + } else { + goto done; + } + have_prev = true; + prev_pos = at; + prev_len = out->n - at; + if ((double)table + early >= (double)(1U << bits) && bits < PDF_LZW_MAX_BITS) { + bits++; + } + if (out->n > PDF_MAX_STREAM_OUT) { + goto done; + } + } + } +done: + cbm_free(CBM_MEM_CLASS_EXTRACT, pos); + cbm_free(CBM_MEM_CLASS_EXTRACT, lenv); +} + +/* Python's base64.a85decode: false on a character or group it rejects. */ +static bool a85_strict(const unsigned char *in, size_t n, pdf_buf_t *out) { + static const unsigned char pad[4] = {'u', 'u', 'u', 'u'}; + unsigned char cur[PDF_A85_GROUP]; + int nc = 0; + out->n = 0; + for (size_t i = 0; i < n + 4; i++) { + unsigned char x = i < n ? in[i] : pad[i - n]; + if (x >= '!' && x <= 'u') { + cur[nc++] = x; + if (nc == PDF_A85_GROUP) { + uint64_t acc = 0; + for (int k = 0; k < PDF_A85_GROUP; k++) { + acc = acc * PDF_A85_BASE + (uint64_t)(cur[k] - '!'); + } + if (acc > 0xFFFFFFFFULL) { + return false; + } + unsigned char b4[4] = {(unsigned char)(acc >> 24), (unsigned char)(acc >> 16), + (unsigned char)(acc >> 8), (unsigned char)acc}; + if (!buf_put(out, b4, 4)) { + return false; + } + nc = 0; + } + } else if (x == 'z') { + if (nc) { + return false; + } + static const unsigned char zero[4] = {0, 0, 0, 0}; + if (!buf_put(out, zero, 4)) { + return false; + } + } else if (x == ' ' || x == '\t' || x == '\n' || x == '\r' || x == '\v') { + continue; + } else { + return false; + } + } + size_t padding = (size_t)(4 - nc); + out->n = padding >= out->n ? 0 : out->n - padding; + return true; +} + +static void f_a85(pdf_doc_t *d, const unsigned char *in, size_t n, pdf_buf_t *out) { + unsigned char *t = (unsigned char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n ? n : 1); + if (!t) { + d->nomem = true; + out->fail = true; + return; + } + size_t k = 0; + for (size_t i = 0; i < n; i++) { + if (!is_ws(in[i])) { + t[k++] = in[i]; + } + } + size_t s0 = 0; + if (k >= 2 && t[0] == '<' && t[1] == '~') { + s0 = 2; + } + const unsigned char *e = cbm_memmem(t + s0, k - s0, "~>", 2); + size_t m = e ? (size_t)(e - (t + s0)) : k - s0; + if (!a85_strict(t + s0, m, out) && !out->fail) { + if (!a85_strict(t + s0, m - m % PDF_A85_GROUP, out) && !out->fail) { + out->n = 0; + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, t); +} + +static void f_ahx(const unsigned char *in, size_t n, pdf_buf_t *out) { + const unsigned char *gt = (const unsigned char *)memchr(in, '>', n); + size_t e = gt ? (size_t)(gt - in) : n; + int hi = -1; + for (size_t i = 0; i < e; i++) { + int v = hex_val(in[i]); + if (v < 0) { + continue; + } + if (hi < 0) { + hi = v; + } else if (!buf_byte(out, (unsigned char)(hi * PDF_HEX_BASE + v))) { + return; + } else { + hi = -1; + } + } + if (hi >= 0) { + buf_byte(out, (unsigned char)(hi * PDF_HEX_BASE)); + } +} + +static void f_rld(const unsigned char *in, size_t n, pdf_buf_t *out) { + size_t i = 0; + while (i < n) { + unsigned L = in[i]; + if (L == PDF_RLD_EOD) { + break; + } + if (L < PDF_RLD_EOD) { + size_t take = (size_t)L + 1; + size_t have = n - (i + 1) < take ? n - (i + 1) : take; + if (!buf_put(out, in + i + 1, have)) { + return; + } + i += (size_t)L + 2; + } else { + if (i + 1 < n) { + size_t rep = 257 - (size_t)L; + if (!buf_reserve(out, rep)) { + return; + } + memset(out->p + out->n, in[i + 1], rep); + out->n += rep; + } + i += 2; + } + if (out->n > PDF_MAX_STREAM_OUT) { + return; + } + } +} + +typedef enum { + FL_FLATE, + FL_LZW, + FL_A85, + FL_AHX, + FL_RLD, + FL_CRYPT, + FL_IMAGE, + FL_UNKNOWN, +} pdf_filter_t; + +static pdf_filter_t filter_kind(const pdf_val_t *f) { + if (!f || f->kind != PV_NAME) { + return FL_UNKNOWN; + } + static const struct { + const char *name; + pdf_filter_t kind; + } names[] = { + {"FlateDecode", FL_FLATE}, + {"Fl", FL_FLATE}, + {"LZWDecode", FL_LZW}, + {"LZW", FL_LZW}, + {"ASCII85Decode", FL_A85}, + {"A85", FL_A85}, + {"ASCIIHexDecode", FL_AHX}, + {"AHx", FL_AHX}, + {"RunLengthDecode", FL_RLD}, + {"RL", FL_RLD}, + {"Crypt", FL_CRYPT}, + {"DCTDecode", FL_IMAGE}, + {"DCT", FL_IMAGE}, + {"JPXDecode", FL_IMAGE}, + {"CCITTFaxDecode", FL_IMAGE}, + {"CCF", FL_IMAGE}, + {"JBIG2Decode", FL_IMAGE}, + }; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + if (pdf_is_name(f, names[i].name)) { + return names[i].kind; + } + } + return FL_UNKNOWN; +} + +/* The stream's bytes through its filters: the raw bytes themselves, or a + * buffer in *held that the caller owns. NULL: a filter failed or is not read. */ +static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len, + pdf_buf_t *held_out) { + *len = 0; + pdf_val_t *filters = pdf_resolve(d, pdf_dget_raw(st->dict, "Filter")); + pdf_val_t *parms = pdf_resolve(d, pdf_dget_raw(st->dict, "DecodeParms")); + pdf_val_t *single[1] = {filters}; + pdf_val_t **fl = single; + int nf = 0; + if (filters && filters->kind == PV_ARR) { + fl = filters->items; + nf = filters->count; + } else if (filters && filters->kind != PV_NULL) { + nf = 1; + } + const unsigned char *cur = st->raw; + size_t cur_len = st->raw_len; + pdf_buf_t held = {0}; /* the buffer cur points into, when it is ours */ + for (int i = 0; i < nf; i++) { + pdf_filter_t kind = filter_kind(pdf_resolve(d, fl[i])); + pdf_val_t *p = NULL; + if (parms && parms->kind == PV_ARR) { + p = i < parms->count ? pdf_resolve(d, parms->items[i]) : NULL; + } else { + p = parms; + } + if (kind == FL_CRYPT) { + continue; + } + if (kind == FL_IMAGE || kind == FL_UNKNOWN) { + if (kind == FL_UNKNOWN) { + d->counts->filters_unsupported++; + } + buf_free(&held); + return NULL; + } + pdf_buf_t out = {0}; + bool ok = true; + switch (kind) { + case FL_FLATE: + ok = f_flate(d, cur, cur_len, &out) && f_predict(d, p, &out); + break; + case FL_LZW: { + double early = 1; + if (p && p->kind == PV_DICT) { + pdf_val_t *ev = pdf_dget(d, p, "EarlyChange"); + if (ev && ev->kind != PV_NUM) { + ok = false; + } else if (ev) { + early = ev->num; + } + } + if (ok) { + f_lzw(cur, cur_len, early, &out); + ok = f_predict(d, p, &out); + } + break; + } + case FL_A85: + f_a85(d, cur, cur_len, &out); + break; + case FL_AHX: + f_ahx(cur, cur_len, &out); + break; + case FL_RLD: + f_rld(cur, cur_len, &out); + break; + default: + break; + } + if (out.fail) { + d->nomem = true; + ok = false; + } + buf_free(&held); + if (!ok) { + buf_free(&out); + return NULL; + } + if (out.n > PDF_MAX_STREAM_OUT) { + d->counts->stream_guard_hits++; + out.n = PDF_MAX_STREAM_OUT; + } + held = out; + cur = held.p ? held.p : (const unsigned char *)""; + cur_len = held.n; + } + *held_out = held; + *len = cur_len; + return cur; +} + +const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) { + if (st->done) { + *len = st->dec_len; + return st->decoded; + } + st->done = true; + st->decoded = NULL; + st->dec_len = 0; + *len = 0; + pdf_buf_t held = {0}; + size_t cur_len; + const unsigned char *cur = decode_filters(d, st, &cur_len, &held); + if (!cur) { + return NULL; + } + if (held.p) { + unsigned char *fit = + (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, held.p, held.n ? held.n : 1); + if (fit) { + held.p = fit; + } + if (!pdf_own(d, held.p)) { + return NULL; + } + cur = held.p; + } + st->decoded = cur; + st->dec_len = cur_len; + *len = cur_len; + return cur; +} diff --git a/internal/cbm/pdf/pdf_tables.c b/internal/cbm/pdf/pdf_tables.c new file mode 100644 index 0000000000..3d4256052f --- /dev/null +++ b/internal/cbm/pdf/pdf_tables.c @@ -0,0 +1,1624 @@ +/* + * pdf_tables.c — static tables of the PDF text-layer extractor (generated). + * + * Glyph names (the Adobe Glyph List subset every standard Latin encoding + * uses, common TeX and symbol names, and the accented Latin letters), + * the base encodings of ISO 32000-1 Annex D as code -> UTF-8, and + * approximate standard-14 widths. Generated once from the field-tested + * prototype's tables; maintained data from here on (edit by hand, keep the + * sources). Sources and notices (THIRD_PARTY.md): the Adobe Glyph List, + * Copyright 2002-2019 Adobe, BSD-3-Clause; the printable-ASCII advance + * widths of Helvetica and Times-Roman from Adobe's core-14 font metrics, + * Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems Incorporated; + * ISO 32000-1 Annex D for the encodings. + */ +#include "pdf_internal.h" + +#include + +const pdf_glyph_t PDF_GLYPHS[] = { + {"A", "A"}, + {"AE", "\303\206"}, + {"Aacute", "\303\201"}, + {"Abreve", "\304\202"}, + {"Acircumflex", "\303\202"}, + {"Adieresis", "\303\204"}, + {"Agrave", "\303\200"}, + {"Alpha", "\316\221"}, + {"Amacron", "\304\200"}, + {"Aogonek", "\304\204"}, + {"Aring", "\303\205"}, + {"Atilde", "\303\203"}, + {"B", "B"}, + {"Beta", "\316\222"}, + {"C", "C"}, + {"Cacute", "\304\206"}, + {"Ccaron", "\304\214"}, + {"Ccedilla", "\303\207"}, + {"Ccircumflex", "\304\210"}, + {"Cdotaccent", "\304\212"}, + {"Chi", "\316\247"}, + {"D", "D"}, + {"Dcaron", "\304\216"}, + {"Dcroat", "\304\220"}, + {"Delta", "\316\224"}, + {"Dslash", "\304\220"}, + {"E", "E"}, + {"Eacute", "\303\211"}, + {"Ebreve", "\304\224"}, + {"Ecaron", "\304\232"}, + {"Ecircumflex", "\303\212"}, + {"Edieresis", "\303\213"}, + {"Edotaccent", "\304\226"}, + {"Egrave", "\303\210"}, + {"Emacron", "\304\222"}, + {"Eng", "\305\212"}, + {"Eogonek", "\304\230"}, + {"Epsilon", "\316\225"}, + {"Eta", "\316\227"}, + {"Eth", "\303\220"}, + {"Euro", "\342\202\254"}, + {"F", "F"}, + {"G", "G"}, + {"Gamma", "\316\223"}, + {"Gbreve", "\304\236"}, + {"Gcedilla", "\304\242"}, + {"Gcircumflex", "\304\234"}, + {"Gdotaccent", "\304\240"}, + {"H", "H"}, + {"Hbar", "\304\246"}, + {"Hcircumflex", "\304\244"}, + {"Hslash", "\304\246"}, + {"I", "I"}, + {"IJ", "\304\262"}, + {"Iacute", "\303\215"}, + {"Ibreve", "\304\254"}, + {"Icircumflex", "\303\216"}, + {"Idieresis", "\303\217"}, + {"Idotaccent", "\304\260"}, + {"Ifraktur", "\342\204\221"}, + {"Igrave", "\303\214"}, + {"Imacron", "\304\252"}, + {"Iogonek", "\304\256"}, + {"Iota", "\316\231"}, + {"Itilde", "\304\250"}, + {"J", "J"}, + {"Jcircumflex", "\304\264"}, + {"K", "K"}, + {"Kappa", "\316\232"}, + {"Kcedilla", "\304\266"}, + {"L", "L"}, + {"Lacute", "\304\271"}, + {"Lambda", "\316\233"}, + {"Lcaron", "\304\275"}, + {"Lcedilla", "\304\273"}, + {"Ldot", "\304\277"}, + {"Lslash", "\305\201"}, + {"M", "M"}, + {"Mu", "\316\234"}, + {"N", "N"}, + {"Nacute", "\305\203"}, + {"Ncaron", "\305\207"}, + {"Ncedilla", "\305\205"}, + {"Ntilde", "\303\221"}, + {"Nu", "\316\235"}, + {"O", "O"}, + {"OE", "\305\222"}, + {"Oacute", "\303\223"}, + {"Obreve", "\305\216"}, + {"Ocircumflex", "\303\224"}, + {"Odieresis", "\303\226"}, + {"Ograve", "\303\222"}, + {"Ohungarumlaut", "\305\220"}, + {"Omacron", "\305\214"}, + {"Omega", "\316\251"}, + {"Omicron", "\316\237"}, + {"Oslash", "\303\230"}, + {"Otilde", "\303\225"}, + {"P", "P"}, + {"Phi", "\316\246"}, + {"Pi", "\316\240"}, + {"Psi", "\316\250"}, + {"Q", "Q"}, + {"R", "R"}, + {"Racute", "\305\224"}, + {"Rcaron", "\305\230"}, + {"Rcedilla", "\305\226"}, + {"Rfraktur", "\342\204\234"}, + {"Rho", "\316\241"}, + {"S", "S"}, + {"Sacute", "\305\232"}, + {"Scaron", "\305\240"}, + {"Scedilla", "\305\236"}, + {"Scircumflex", "\305\234"}, + {"Scommaaccent", "\310\230"}, + {"Sigma", "\316\243"}, + {"T", "T"}, + {"Tau", "\316\244"}, + {"Tbar", "\305\246"}, + {"Tcaron", "\305\244"}, + {"Tcedilla", "\305\242"}, + {"Tcommaaccent", "\310\232"}, + {"Theta", "\316\230"}, + {"Thorn", "\303\236"}, + {"Tslash", "\305\246"}, + {"U", "U"}, + {"Uacute", "\303\232"}, + {"Ubreve", "\305\254"}, + {"Ucircumflex", "\303\233"}, + {"Udieresis", "\303\234"}, + {"Ugrave", "\303\231"}, + {"Uhungarumlaut", "\305\260"}, + {"Umacron", "\305\252"}, + {"Uogonek", "\305\262"}, + {"Upsilon", "\316\245"}, + {"Upsilon1", "\317\222"}, + {"Uring", "\305\256"}, + {"Utilde", "\305\250"}, + {"V", "V"}, + {"W", "W"}, + {"Wcircumflex", "\305\264"}, + {"X", "X"}, + {"Xi", "\316\236"}, + {"Y", "Y"}, + {"Yacute", "\303\235"}, + {"Ycircumflex", "\305\266"}, + {"Ydieresis", "\305\270"}, + {"Z", "Z"}, + {"Zacute", "\305\271"}, + {"Zcaron", "\305\275"}, + {"Zdotaccent", "\305\273"}, + {"Zeta", "\316\226"}, + {"a", "a"}, + {"aacute", "\303\241"}, + {"abreve", "\304\203"}, + {"acircumflex", "\303\242"}, + {"acute", "\302\264"}, + {"adieresis", "\303\244"}, + {"ae", "\303\246"}, + {"afii00208", "\342\200\225"}, + {"afii10017", "\320\220"}, + {"afii61289", "\342\204\223"}, + {"agrave", "\303\240"}, + {"aleph", "\342\204\265"}, + {"alpha", "\316\261"}, + {"amacron", "\304\201"}, + {"ampersand", "&"}, + {"angbracketleft", "\342\237\250"}, + {"angbracketright", "\342\237\251"}, + {"angle", "\342\210\240"}, + {"aogonek", "\304\205"}, + {"approxequal", "\342\211\210"}, + {"aring", "\303\245"}, + {"arrowboth", "\342\206\224"}, + {"arrowdblboth", "\342\207\224"}, + {"arrowdblleft", "\342\207\220"}, + {"arrowdblright", "\342\207\222"}, + {"arrowdown", "\342\206\223"}, + {"arrowleft", "\342\206\220"}, + {"arrowright", "\342\206\222"}, + {"arrowup", "\342\206\221"}, + {"asciicircum", "^"}, + {"asciitilde", "~"}, + {"asterisk", "*"}, + {"asteriskmath", "\342\210\227"}, + {"at", "@"}, + {"atilde", "\303\243"}, + {"b", "b"}, + {"backslash", "\134"}, + {"bar", "|"}, + {"beta", "\316\262"}, + {"braceleft", "{"}, + {"braceright", "}"}, + {"bracketleft", "["}, + {"bracketright", "]"}, + {"breve", "\313\230"}, + {"brokenbar", "\302\246"}, + {"bullet", "\342\200\242"}, + {"bulletmath", "\342\210\231"}, + {"c", "c"}, + {"cacute", "\304\207"}, + {"caron", "\313\207"}, + {"ccaron", "\304\215"}, + {"ccedilla", "\303\247"}, + {"ccircumflex", "\304\211"}, + {"cdotaccent", "\304\213"}, + {"cedilla", "\302\270"}, + {"cent", "\302\242"}, + {"checkmark", "\342\234\223"}, + {"chi", "\317\207"}, + {"circlemultiply", "\342\212\227"}, + {"circleplus", "\342\212\225"}, + {"circumflex", "\313\206"}, + {"club", "\342\231\243"}, + {"colon", ":"}, + {"comma", ","}, + {"commaaccent", "\314\246"}, + {"congruent", "\342\211\205"}, + {"copyright", "\302\251"}, + {"currency", "\302\244"}, + {"d", "d"}, + {"dagger", "\342\200\240"}, + {"daggerdbl", "\342\200\241"}, + {"dcaron", "\304\217"}, + {"dcroat", "\304\221"}, + {"degree", "\302\260"}, + {"delta", "\316\264"}, + {"diamond", "\342\231\246"}, + {"dieresis", "\302\250"}, + {"divide", "\303\267"}, + {"dollar", "$"}, + {"dotaccent", "\313\231"}, + {"dotlessi", "\304\261"}, + {"dotlessi.sc", "\304\261"}, + {"dotlessj", "\310\267"}, + {"dotmath", "\342\213\205"}, + {"dslash", "\304\221"}, + {"e", "e"}, + {"eacute", "\303\251"}, + {"ebreve", "\304\225"}, + {"ecaron", "\304\233"}, + {"ecircumflex", "\303\252"}, + {"edieresis", "\303\253"}, + {"edotaccent", "\304\227"}, + {"egrave", "\303\250"}, + {"eight", "8"}, + {"element", "\342\210\210"}, + {"ellipsis", "\342\200\246"}, + {"emacron", "\304\223"}, + {"emdash", "\342\200\224"}, + {"emptyset", "\342\210\205"}, + {"endash", "\342\200\223"}, + {"eng", "\305\213"}, + {"eogonek", "\304\231"}, + {"epsilon", "\316\265"}, + {"equal", "="}, + {"equivalence", "\342\211\241"}, + {"eta", "\316\267"}, + {"eth", "\303\260"}, + {"exclam", "!"}, + {"exclamdbl", "\342\200\274"}, + {"exclamdown", "\302\241"}, + {"existential", "\342\210\203"}, + {"f", "f"}, + {"ff", "ff"}, + {"ffi", "ffi"}, + {"ffl", "ffl"}, + {"fi", "fi"}, + {"figuredash", "\342\200\222"}, + {"five", "5"}, + {"fl", "fl"}, + {"florin", "\306\222"}, + {"four", "4"}, + {"fraction", "\342\201\204"}, + {"g", "g"}, + {"gamma", "\316\263"}, + {"gbreve", "\304\237"}, + {"gcedilla", "\304\243"}, + {"gcircumflex", "\304\235"}, + {"gdotaccent", "\304\241"}, + {"germandbls", "\303\237"}, + {"gradient", "\342\210\207"}, + {"grave", "`"}, + {"greater", ">"}, + {"greaterequal", "\342\211\245"}, + {"guillemotleft", "\302\253"}, + {"guillemotright", "\302\273"}, + {"guilsinglleft", "\342\200\271"}, + {"guilsinglright", "\342\200\272"}, + {"h", "h"}, + {"hbar", "\304\247"}, + {"hcircumflex", "\304\245"}, + {"heart", "\342\231\245"}, + {"horizontalbar", "\342\200\225"}, + {"hslash", "\304\247"}, + {"hungarumlaut", "\313\235"}, + {"hyphen", "-"}, + {"hyphen.cap", "-"}, + {"i", "i"}, + {"iacute", "\303\255"}, + {"ibreve", "\304\255"}, + {"icircumflex", "\303\256"}, + {"idieresis", "\303\257"}, + {"igrave", "\303\254"}, + {"ij", "\304\263"}, + {"imacron", "\304\253"}, + {"infinity", "\342\210\236"}, + {"integral", "\342\210\253"}, + {"intersection", "\342\210\251"}, + {"iogonek", "\304\257"}, + {"iota", "\316\271"}, + {"itilde", "\304\251"}, + {"j", "j"}, + {"jcircumflex", "\304\265"}, + {"k", "k"}, + {"kappa", "\316\272"}, + {"kcedilla", "\304\267"}, + {"kgreenlandic", "\304\270"}, + {"l", "l"}, + {"lacute", "\304\272"}, + {"lambda", "\316\273"}, + {"lcaron", "\304\276"}, + {"lcedilla", "\304\274"}, + {"ldot", "\305\200"}, + {"less", "<"}, + {"lessequal", "\342\211\244"}, + {"logicaland", "\342\210\247"}, + {"logicalnot", "\302\254"}, + {"logicalor", "\342\210\250"}, + {"longs", "\305\277"}, + {"lozenge", "\342\227\212"}, + {"lslash", "\305\202"}, + {"m", "m"}, + {"macron", "\302\257"}, + {"minus", "\342\210\222"}, + {"minute", "\342\200\262"}, + {"mu", "\302\265"}, + {"mu1", "\302\265"}, + {"multiply", "\303\227"}, + {"n", "n"}, + {"nacute", "\305\204"}, + {"napostrophe", "\305\211"}, + {"nbspace", "\302\240"}, + {"ncaron", "\305\210"}, + {"ncedilla", "\305\206"}, + {"nine", "9"}, + {"nobreakspace", "\302\240"}, + {"notelement", "\342\210\211"}, + {"notequal", "\342\211\240"}, + {"ntilde", "\303\261"}, + {"nu", "\316\275"}, + {"numbersign", "#"}, + {"o", "o"}, + {"oacute", "\303\263"}, + {"obreve", "\305\217"}, + {"ocircumflex", "\303\264"}, + {"odieresis", "\303\266"}, + {"oe", "\305\223"}, + {"ogonek", "\313\233"}, + {"ograve", "\303\262"}, + {"ohungarumlaut", "\305\221"}, + {"omacron", "\305\215"}, + {"omega", "\317\211"}, + {"omega1", "\317\226"}, + {"omicron", "\316\277"}, + {"one", "1"}, + {"onedotenleader", "\342\200\244"}, + {"onehalf", "\302\275"}, + {"onequarter", "\302\274"}, + {"onesuperior", "\302\271"}, + {"openbullet", "\342\227\246"}, + {"ordfeminine", "\302\252"}, + {"ordmasculine", "\302\272"}, + {"oslash", "\303\270"}, + {"otilde", "\303\265"}, + {"p", "p"}, + {"paragraph", "\302\266"}, + {"parenleft", "("}, + {"parenright", ")"}, + {"partialdiff", "\342\210\202"}, + {"percent", "%"}, + {"period", "."}, + {"periodcentered", "\302\267"}, + {"perpendicular", "\342\212\245"}, + {"perthousand", "\342\200\260"}, + {"phi", "\317\206"}, + {"phi1", "\317\225"}, + {"pi", "\317\200"}, + {"plus", "+"}, + {"plusminus", "\302\261"}, + {"prime", "\342\200\262"}, + {"product", "\342\210\217"}, + {"propersubset", "\342\212\202"}, + {"propersuperset", "\342\212\203"}, + {"proportional", "\342\210\235"}, + {"psi", "\317\210"}, + {"q", "q"}, + {"question", "\077"}, + {"questiondown", "\302\277"}, + {"quotedbl", "\042"}, + {"quotedblbase", "\342\200\236"}, + {"quotedblbase.cap", "\342\200\236"}, + {"quotedblleft", "\342\200\234"}, + {"quotedblright", "\342\200\235"}, + {"quoteleft", "\342\200\230"}, + {"quoteright", "\342\200\231"}, + {"quotesinglbase", "\342\200\232"}, + {"quotesingle", "'"}, + {"r", "r"}, + {"racute", "\305\225"}, + {"radical", "\342\210\232"}, + {"rcaron", "\305\231"}, + {"rcedilla", "\305\227"}, + {"reflexsubset", "\342\212\206"}, + {"reflexsuperset", "\342\212\207"}, + {"registered", "\302\256"}, + {"rho", "\317\201"}, + {"ring", "\313\232"}, + {"s", "s"}, + {"sacute", "\305\233"}, + {"scaron", "\305\241"}, + {"scedilla", "\305\237"}, + {"scircumflex", "\305\235"}, + {"scommaaccent", "\310\231"}, + {"second", "\342\200\263"}, + {"section", "\302\247"}, + {"semicolon", ";"}, + {"seven", "7"}, + {"sfthyphen", "\302\255"}, + {"sigma", "\317\203"}, + {"sigma1", "\317\202"}, + {"similar", "\342\210\274"}, + {"six", "6"}, + {"slash", "/"}, + {"space", " "}, + {"space.alt", " "}, + {"spade", "\342\231\240"}, + {"st", "st"}, + {"sterling", "\302\243"}, + {"summation", "\342\210\221"}, + {"suppress", ""}, + {"t", "t"}, + {"tau", "\317\204"}, + {"tbar", "\305\247"}, + {"tcaron", "\305\245"}, + {"tcedilla", "\305\243"}, + {"tcommaaccent", "\310\233"}, + {"therefore", "\342\210\264"}, + {"theta", "\316\270"}, + {"theta1", "\317\221"}, + {"thorn", "\303\276"}, + {"three", "3"}, + {"threequarters", "\302\276"}, + {"threesuperior", "\302\263"}, + {"tilde", "\313\234"}, + {"trademark", "\342\204\242"}, + {"tslash", "\305\247"}, + {"two", "2"}, + {"twodotenleader", "\342\200\245"}, + {"twosuperior", "\302\262"}, + {"u", "u"}, + {"uacute", "\303\272"}, + {"ubreve", "\305\255"}, + {"ucircumflex", "\303\273"}, + {"udieresis", "\303\274"}, + {"ugrave", "\303\271"}, + {"uhungarumlaut", "\305\261"}, + {"umacron", "\305\253"}, + {"underscore", "_"}, + {"underscoredbl", "\342\200\227"}, + {"uni00A0", "\302\240"}, + {"union", "\342\210\252"}, + {"universal", "\342\210\200"}, + {"uogonek", "\305\263"}, + {"upsilon", "\317\205"}, + {"uring", "\305\257"}, + {"utilde", "\305\251"}, + {"v", "v"}, + {"visiblespace", "\342\220\243"}, + {"w", "w"}, + {"wcircumflex", "\305\265"}, + {"weierstrass", "\342\204\230"}, + {"x", "x"}, + {"xi", "\316\276"}, + {"y", "y"}, + {"yacute", "\303\275"}, + {"ycircumflex", "\305\267"}, + {"ydieresis", "\303\277"}, + {"yen", "\302\245"}, + {"z", "z"}, + {"zacute", "\305\272"}, + {"zcaron", "\305\276"}, + {"zdotaccent", "\305\274"}, + {"zero", "0"}, + {"zeta", "\316\266"}, +}; +const int PDF_GLYPH_COUNT = (int)(sizeof(PDF_GLYPHS) / sizeof(PDF_GLYPHS[0])); + +const char *const PDF_ENC_STANDARD[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "\342\200\231", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "\342\200\230", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\302\241", + "\302\242", + "\302\243", + "\342\201\204", + "\302\245", + "\306\222", + "\302\247", + "\302\244", + "'", + "\342\200\234", + "\302\253", + "\342\200\271", + "\342\200\272", + "fi", + "fl", + NULL, + "\342\200\223", + "\342\200\240", + "\342\200\241", + "\302\267", + NULL, + "\302\266", + "\342\200\242", + "\342\200\232", + "\342\200\236", + "\342\200\235", + "\302\273", + "\342\200\246", + "\342\200\260", + NULL, + "\302\277", + NULL, + "`", + "\302\264", + "\313\206", + "\313\234", + "\302\257", + "\313\230", + "\313\231", + "\302\250", + NULL, + "\313\232", + "\302\270", + NULL, + "\313\235", + "\313\233", + "\313\207", + "\342\200\224", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\303\206", + NULL, + "\302\252", + NULL, + NULL, + NULL, + NULL, + "\305\201", + "\303\230", + "\305\222", + "\302\272", + NULL, + NULL, + NULL, + NULL, + NULL, + "\303\246", + NULL, + NULL, + NULL, + "\304\261", + NULL, + NULL, + "\305\202", + "\303\270", + "\305\223", + "\303\237", + NULL, + NULL, + NULL, + NULL, +}; + +const char *const PDF_ENC_WINANSI[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + "\342\202\254", + NULL, + "\342\200\232", + "\306\222", + "\342\200\236", + "\342\200\246", + "\342\200\240", + "\342\200\241", + "\313\206", + "\342\200\260", + "\305\240", + "\342\200\271", + "\305\222", + NULL, + "\305\275", + NULL, + NULL, + "\342\200\230", + "\342\200\231", + "\342\200\234", + "\342\200\235", + "\342\200\242", + "\342\200\223", + "\342\200\224", + "\313\234", + "\342\204\242", + "\305\241", + "\342\200\272", + "\305\223", + NULL, + "\305\276", + "\305\270", + " ", + "\302\241", + "\302\242", + "\302\243", + "\302\244", + "\302\245", + "\302\246", + "\302\247", + "\302\250", + "\302\251", + "\302\252", + "\302\253", + "\302\254", + "-", + "\302\256", + "\302\257", + "\302\260", + "\302\261", + "\302\262", + "\302\263", + "\302\264", + "\302\265", + "\302\266", + "\302\267", + "\302\270", + "\302\271", + "\302\272", + "\302\273", + "\302\274", + "\302\275", + "\302\276", + "\302\277", + "\303\200", + "\303\201", + "\303\202", + "\303\203", + "\303\204", + "\303\205", + "\303\206", + "\303\207", + "\303\210", + "\303\211", + "\303\212", + "\303\213", + "\303\214", + "\303\215", + "\303\216", + "\303\217", + "\303\220", + "\303\221", + "\303\222", + "\303\223", + "\303\224", + "\303\225", + "\303\226", + "\303\227", + "\303\230", + "\303\231", + "\303\232", + "\303\233", + "\303\234", + "\303\235", + "\303\236", + "\303\237", + "\303\240", + "\303\241", + "\303\242", + "\303\243", + "\303\244", + "\303\245", + "\303\246", + "\303\247", + "\303\250", + "\303\251", + "\303\252", + "\303\253", + "\303\254", + "\303\255", + "\303\256", + "\303\257", + "\303\260", + "\303\261", + "\303\262", + "\303\263", + "\303\264", + "\303\265", + "\303\266", + "\303\267", + "\303\270", + "\303\271", + "\303\272", + "\303\273", + "\303\274", + "\303\275", + "\303\276", + "\303\277", +}; + +const char *const PDF_ENC_MACROMAN[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\042", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\134", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + NULL, + "\303\204", + "\303\205", + "\303\207", + "\303\211", + "\303\221", + "\303\226", + "\303\234", + "\303\241", + "\303\240", + "\303\242", + "\303\244", + "\303\243", + "\303\245", + "\303\247", + "\303\251", + "\303\250", + "\303\252", + "\303\253", + "\303\255", + "\303\254", + "\303\256", + "\303\257", + "\303\261", + "\303\263", + "\303\262", + "\303\264", + "\303\266", + "\303\265", + "\303\272", + "\303\271", + "\303\273", + "\303\274", + "\342\200\240", + "\302\260", + "\302\242", + "\302\243", + "\302\247", + "\342\200\242", + "\302\266", + "\303\237", + "\302\256", + "\302\251", + "\342\204\242", + "\302\264", + "\302\250", + "\342\211\240", + "\303\206", + "\303\230", + "\342\210\236", + "\302\261", + "\342\211\244", + "\342\211\245", + "\302\245", + "\302\265", + "\342\210\202", + "\342\210\221", + "\342\210\217", + "\317\200", + "\342\210\253", + "\302\252", + "\302\272", + "\316\251", + "\303\246", + "\303\270", + "\302\277", + "\302\241", + "\302\254", + "\342\210\232", + "\306\222", + "\342\211\210", + "\342\210\206", + "\302\253", + "\302\273", + "\342\200\246", + "\302\240", + "\303\200", + "\303\203", + "\303\225", + "\305\222", + "\305\223", + "\342\200\223", + "\342\200\224", + "\342\200\234", + "\342\200\235", + "\342\200\230", + "\342\200\231", + "\303\267", + "\342\227\212", + "\303\277", + "\305\270", + "\342\201\204", + "\342\202\254", + "\342\200\271", + "\342\200\272", + "\357\254\201", + "\357\254\202", + "\342\200\241", + "\302\267", + "\342\200\232", + "\342\200\236", + "\342\200\260", + "\303\202", + "\303\212", + "\303\201", + "\303\213", + "\303\210", + "\303\215", + "\303\216", + "\303\217", + "\303\214", + "\303\223", + "\303\224", + "\357\243\277", + "\303\222", + "\303\232", + "\303\233", + "\303\231", + "\304\261", + "\313\206", + "\313\234", + "\302\257", + "\313\230", + "\313\231", + "\313\232", + "\302\270", + "\313\235", + "\313\233", + "\313\207", +}; + +const char *const PDF_ENC_PDFDOC[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + " ", "!", "\042", "#", "$", "%", "&", "'", + "(", ")", "*", "+", ",", "-", ".", "/", + "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", ":", ";", "<", "=", ">", "\077", + "@", "A", "B", "C", "D", "E", "F", "G", + "H", "I", "J", "K", "L", "M", "N", "O", + "P", "Q", "R", "S", "T", "U", "V", "W", + "X", "Y", "Z", "[", "\134", "]", "^", "_", + "`", "a", "b", "c", "d", "e", "f", "g", + "h", "i", "j", "k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", + "x", "y", "z", "{", "|", "}", "~", NULL, + "\302\200", "\302\201", "\302\202", "\302\203", "\302\204", "\302\205", "\302\206", "\302\207", + "\302\210", "\302\211", "\302\212", "\302\213", "\302\214", "\302\215", "\302\216", "\302\217", + "\302\220", "\302\221", "\302\222", "\302\223", "\302\224", "\302\225", "\302\226", "\302\227", + "\302\230", "\302\231", "\302\232", "\302\233", "\302\234", "\302\235", "\302\236", "\302\237", + "\302\240", "\302\241", "\302\242", "\302\243", "\302\244", "\302\245", "\302\246", "\302\247", + "\302\250", "\302\251", "\302\252", "\302\253", "\302\254", "\302\255", "\302\256", "\302\257", + "\302\260", "\302\261", "\302\262", "\302\263", "\302\264", "\302\265", "\302\266", "\302\267", + "\302\270", "\302\271", "\302\272", "\302\273", "\302\274", "\302\275", "\302\276", "\302\277", + "\303\200", "\303\201", "\303\202", "\303\203", "\303\204", "\303\205", "\303\206", "\303\207", + "\303\210", "\303\211", "\303\212", "\303\213", "\303\214", "\303\215", "\303\216", "\303\217", + "\303\220", "\303\221", "\303\222", "\303\223", "\303\224", "\303\225", "\303\226", "\303\227", + "\303\230", "\303\231", "\303\232", "\303\233", "\303\234", "\303\235", "\303\236", "\303\237", + "\303\240", "\303\241", "\303\242", "\303\243", "\303\244", "\303\245", "\303\246", "\303\247", + "\303\250", "\303\251", "\303\252", "\303\253", "\303\254", "\303\255", "\303\256", "\303\257", + "\303\260", "\303\261", "\303\262", "\303\263", "\303\264", "\303\265", "\303\266", "\303\267", + "\303\270", "\303\271", "\303\272", "\303\273", "\303\274", "\303\275", "\303\276", "\303\277", +}; + +const char *const PDF_ENC_SYMBOL[256] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + " ", + "!", + "\342\210\200", + "#", + "\342\210\203", + "%", + "&", + NULL, + "(", + ")", + "\342\210\227", + "+", + ",", + "\342\210\222", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "\077", + "\342\211\205", + "\316\221", + "\316\222", + "\316\247", + "\316\224", + "\316\225", + "\316\246", + "\316\223", + "\316\227", + "\316\231", + "\317\221", + "\316\232", + "\316\233", + "\316\234", + "\316\235", + "\316\237", + "\316\240", + "\316\230", + "\316\241", + "\316\243", + "\316\244", + "\316\245", + "\317\202", + "\316\251", + "\316\236", + "\316\250", + "\316\226", + "[", + "\342\210\264", + "]", + "\342\212\245", + "_", + NULL, + "\316\261", + "\316\262", + "\317\207", + "\316\264", + "\316\265", + "\317\206", + "\316\263", + "\316\267", + "\316\271", + "\317\225", + "\316\272", + "\316\273", + "\302\265", + "\316\275", + "\316\277", + "\317\200", + "\316\270", + "\317\201", + "\317\203", + "\317\204", + "\317\205", + "\317\226", + "\317\211", + "\316\276", + "\317\210", + "\316\266", + "{", + "|", + "}", + "\342\210\274", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\211\244", + NULL, + "\342\210\236", + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\206\224", + "\342\206\220", + "\342\206\221", + "\342\206\222", + "\342\206\223", + "\302\260", + "\302\261", + NULL, + "\342\211\245", + "\303\227", + "\342\210\235", + "\342\210\202", + "\342\200\242", + "\303\267", + "\342\211\240", + "\342\211\241", + "\342\211\210", + "\342\200\246", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\210", + NULL, + NULL, + "\342\210\207", + NULL, + NULL, + NULL, + NULL, + "\342\210\232", + "\342\213\205", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\221", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + "\342\210\253", + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, +}; + +const unsigned short PDF_WIDTH_HELVETICA[95] = { + 278, 278, 355, 556, 556, 889, 667, 191, 333, 333, 389, 584, 278, 333, 278, 278, 556, 556, 556, + 556, 556, 556, 556, 556, 556, 556, 278, 278, 584, 584, 584, 556, 1015, 667, 667, 722, 722, 667, + 611, 778, 722, 278, 500, 667, 556, 833, 722, 778, 667, 778, 722, 667, 611, 722, 667, 944, 667, + 667, 611, 278, 278, 278, 469, 556, 333, 556, 556, 500, 556, 556, 278, 556, 556, 222, 222, 500, + 222, 833, 556, 556, 556, 556, 333, 500, 278, 556, 500, 722, 500, 500, 500, 334, 260, 334, 584}; +const unsigned short PDF_WIDTH_TIMES[95] = { + 250, 333, 408, 500, 500, 833, 778, 180, 333, 333, 500, 564, 250, 333, 250, 278, 500, 500, 500, + 500, 500, 500, 500, 500, 500, 500, 278, 278, 564, 564, 564, 444, 921, 722, 667, 667, 722, 611, + 556, 722, 722, 333, 389, 722, 611, 889, 722, 722, 556, 722, 667, 556, 611, 722, 722, 944, 722, + 722, 611, 333, 278, 333, 469, 500, 333, 444, 500, 444, 500, 444, 333, 500, 500, 278, 278, 500, + 278, 778, 500, 500, 500, 500, 333, 389, 278, 500, 500, 722, 500, 500, 444, 480, 200, 480, 541}; + +const char *const PDF_STD14[] = {"Arial", + "Arial,Bold", + "Arial,BoldItalic", + "Arial,Italic", + "Courier", + "Courier-Bold", + "Courier-BoldOblique", + "Courier-Oblique", + "CourierNew", + "CourierNew,Bold", + "Helvetica", + "Helvetica-Bold", + "Helvetica-BoldOblique", + "Helvetica-Oblique", + "Symbol", + "Times-Bold", + "Times-BoldItalic", + "Times-Italic", + "Times-Roman", + "TimesNewRoman", + "TimesNewRoman,Bold", + "ZapfDingbats", + NULL}; diff --git a/internal/cbm/pdf/pdf_text.c b/internal/cbm/pdf/pdf_text.c new file mode 100644 index 0000000000..dcd361b095 --- /dev/null +++ b/internal/cbm/pdf/pdf_text.c @@ -0,0 +1,1054 @@ +/* + * pdf_text.c — content streams, layout, pages and the driver of the PDF + * text-layer extractor. + * + * Text comes out in content order. A word break is a gap of more than 0.15 em + * along the writing direction (or a step back of more than 0.6 em); a line + * break is a baseline shift of more than 0.5 em or a change of direction -- + * all measured on device-space glyph geometry. Runs of Hebrew/Arabic letters, + * drawn in visual order, are reversed into logical order. + */ +#include "pdf_internal.h" + +#include "foundation/mem_core.h" +#include "helpers.h" + +#include +#include +#include + +enum { + PDF_OPS_MIN = 16, + PDF_OPS_ARENA = 16 * 1024, + PDF_TEXT_MIN = 4096, +}; + +#define PDF_WORD_GAP 0.15 +#define PDF_LINE_SHIFT 0.5 +#define PDF_BACKSTEP 0.6 +#define PDF_SAME_DIR 0.95 + +static const double IDENT[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}; + +/* ── Arithmetic, exactly as the prototype ────────────────────────── */ + +static void mat_mul(const double *m, const double *n, double *o) { + double a = m[0]; + double b = m[1]; + double c = m[2]; + double d = m[3]; + double e = m[4]; + double f = m[5]; + double r[6]; + r[0] = a * n[0] + b * n[2]; + r[1] = a * n[1] + b * n[3]; + r[2] = c * n[0] + d * n[2]; + r[3] = c * n[1] + d * n[3]; + r[4] = e * n[0] + f * n[2] + n[4]; + r[5] = e * n[1] + f * n[3] + n[5]; + memcpy(o, r, sizeof(r)); +} + +typedef struct { + double hi; + double lo; +} dl_t; + +static dl_t dl_fast_sum(double a, double b) { + double x = a + b; + double z = x - a; + double y = b - z; + dl_t r = {x, y}; + return r; +} + +static dl_t dl_mul(double x, double y) { + double z = x * y; + double zz = fma(x, y, -z); + dl_t r = {z, zz}; + return r; +} + +/* CPython's math.hypot for two coordinates (vector_norm): the prototype's + * value to the last bit, on every platform. */ +static double vec_norm2(double x0, double x1, double max) { + int max_e; + frexp(max, &max_e); + if (max_e < -1023) { + return DBL_MIN * vec_norm2(x0 / DBL_MIN, x1 / DBL_MIN, max / DBL_MIN); + } + double scale = ldexp(1.0, -max_e); + double csum = 1.0; + double frac1 = 0.0; + double frac2 = 0.0; + double v[2] = {x0, x1}; + for (int i = 0; i < 2; i++) { + double x = v[i] * scale; + dl_t pr = dl_mul(x, x); + dl_t sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + } + double h = sqrt(csum - 1.0 + (frac1 + frac2)); + dl_t pr = dl_mul(-h, h); + dl_t sm = dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + double x = csum - 1.0 + (frac1 + frac2); + h += x / (2.0 * h); + return h / scale; +} + +static double py_hypot(double a, double b) { + double x0 = fabs(a); + double x1 = fabs(b); + double max = 0.0; + bool nan = false; + if (isnan(x0)) { + nan = true; + } else if (x0 > max) { + max = x0; + } + if (isnan(x1)) { + nan = true; + } else if (x1 > max) { + max = x1; + } + if (isinf(max)) { + return max; + } + if (nan) { + return NAN; + } + if (max == 0.0) { + return max; + } + return vec_norm2(x0, x1, max); +} + +/* ── Page text ───────────────────────────────────────────────────── */ + +typedef struct { + char *p; + size_t n; + size_t cap; + bool fail; + bool has_last; + double lx; + double ly; + double ldx; + double ldy; + double lsize; + int glyphs; + int unmapped; +} pdf_out_t; + +static void out_put(pdf_out_t *o, const char *s, size_t n) { + if (o->fail || !n) { + return; + } + if (o->n + n + 1 > o->cap) { + size_t ncap = o->cap ? o->cap : PDF_TEXT_MIN; + while (ncap < o->n + n + 1) { + ncap *= 2; + } + char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, o->p, ncap); + if (!g) { + o->fail = true; + return; + } + o->p = g; + o->cap = ncap; + } + memcpy(o->p + o->n, s, n); + o->n += n; +} + +static void emit(pdf_out_t *o, const char *u, double ox, double oy, double ex, double ey, double dx, + double dy, double size) { + if (o->has_last) { + if (dx * o->ldx + dy * o->ldy < PDF_SAME_DIR) { + out_put(o, "\n", 1); + } else { + double vx = ox - o->lx; + double vy = oy - o->ly; + double along = vx * o->ldx + vy * o->ldy; + double perp = vy * o->ldx - vx * o->ldy; + double L = size > o->lsize ? size : o->lsize; + if (L <= 0) { + L = 1e-6; + } + if (fabs(perp) > PDF_LINE_SHIFT * L) { + out_put(o, "\n", 1); + } else if (along > PDF_WORD_GAP * L || along < -PDF_BACKSTEP * L) { + out_put(o, " ", 1); + } + } + } + out_put(o, u, strlen(u)); + o->has_last = true; + o->lx = ex; + o->ly = ey; + o->ldx = dx; + o->ldy = dy; + o->lsize = size; +} + +/* One UTF-8 sequence at s[i] (C0 80 is the NUL marker): its code point and length. */ +static uint32_t u8_at(const char *s, size_t n, size_t i, size_t *len) { + unsigned char c = (unsigned char)s[i]; + if (c < 0x80) { + *len = 1; + return c; + } + size_t k = c >= 0xF0 ? 4 : c >= 0xE0 ? 3 : 2; + if (i + k > n) { + *len = 1; + return c; + } + uint32_t cp = c & (k == 2 ? 0x1FU : k == 3 ? 0x0FU : 0x07U); + for (size_t j = 1; j < k; j++) { + cp = (cp << 6) | ((unsigned char)s[i + j] & 0x3FU); + } + *len = k; + return cp; +} + +static bool is_rtl(uint32_t cp) { + return (cp >= 0x0590 && cp <= 0x05FF) || (cp >= 0x0600 && cp <= 0x06FF) || + (cp >= 0x0700 && cp <= 0x074F) || (cp >= 0x0750 && cp <= 0x077F) || + (cp >= 0x08A0 && cp <= 0x08FF) || (cp >= 0xFB1D && cp <= 0xFDFF) || + (cp >= 0xFE70 && cp <= 0xFEFF); +} + +/* The prototype's PageText.text(): [ \t ]+ -> " ", " ?\n ?" -> "\n", + * RTL runs reversed, spaces stripped at both ends; then the NUL markers go. */ +static char *page_text(pdf_doc_t *d, pdf_out_t *o, CBMArena *dst, size_t *out_len) { + const char *s = o->p ? o->p : ""; + size_t n = o->n; + char *a = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n + 1); + char *b = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, n + 1); + if (!a || !b) { + cbm_free(CBM_MEM_CLASS_EXTRACT, a); + cbm_free(CBM_MEM_CLASS_EXTRACT, b); + d->nomem = true; + return NULL; + } + /* 1: runs of space, tab, no-break space */ + size_t k = 0; + for (size_t i = 0; i < n;) { + bool sp = s[i] == ' ' || s[i] == '\t' || + (i + 1 < n && (unsigned char)s[i] == 0xC2 && (unsigned char)s[i + 1] == 0xA0); + if (!sp) { + a[k++] = s[i++]; + continue; + } + while (i < n && + (s[i] == ' ' || s[i] == '\t' || + (i + 1 < n && (unsigned char)s[i] == 0xC2 && (unsigned char)s[i + 1] == 0xA0))) { + i += s[i] == ' ' || s[i] == '\t' ? 1 : 2; + } + a[k++] = ' '; + } + /* 2: one space either side of a newline */ + size_t m = 0; + for (size_t i = 0; i < k;) { + if (a[i] == ' ' && i + 1 < k && a[i + 1] == '\n') { + b[m++] = '\n'; + i += 2; + if (i < k && a[i] == ' ') { + i++; + } + } else if (a[i] == '\n') { + b[m++] = '\n'; + i++; + if (i < k && a[i] == ' ') { + i++; + } + } else { + b[m++] = a[i++]; + } + } + /* 3: RTL runs ([RTL]+( [RTL]+)*) reversed by code point */ + k = 0; + for (size_t i = 0; i < m;) { + size_t l; + uint32_t cp = u8_at(b, m, i, &l); + if (!is_rtl(cp)) { + memcpy(a + k, b + i, l); + k += l; + i += l; + continue; + } + size_t j = i; + for (;;) { + size_t l2; + while (j < m && is_rtl(u8_at(b, m, j, &l2))) { + j += l2; + } + if (j + 1 < m && b[j] == ' ' && is_rtl(u8_at(b, m, j + 1, &l2))) { + j += 1; + continue; + } + break; + } + /* write the run's sequences in reverse order */ + size_t end = j; + size_t w = k + (end - i); + for (size_t q = i; q < end;) { + size_t l3; + u8_at(b, m, q, &l3); + w -= l3; + memcpy(a + w, b + q, l3); + q += l3; + } + k += end - i; + i = end; + } + /* 4: strip spaces; 5: drop NUL markers */ + size_t lo = 0; + size_t hi = k; + while (lo < hi && a[lo] == ' ') { + lo++; + } + while (hi > lo && a[hi - 1] == ' ') { + hi--; + } + char *res = (char *)cbm_arena_alloc(dst, hi - lo + 1); + size_t r = 0; + if (res) { + for (size_t i = lo; i < hi; i++) { + if ((unsigned char)a[i] == 0xC0 && i + 1 < hi && (unsigned char)a[i + 1] == 0x80) { + i++; + continue; + } + res[r++] = a[i]; + } + res[r] = '\0'; + } else { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, a); + cbm_free(CBM_MEM_CLASS_EXTRACT, b); + *out_len = r; + return res; +} + +/* ── Interpreter ─────────────────────────────────────────────────── */ + +typedef struct { + double ctm[6]; + pdf_font_t *font; + double fs; + double tc; + double tw; + double th; + double tl; + double rise; +} pdf_gs_t; + +typedef struct { + bool is_ref; + uint64_t key; +} pdf_formkey_t; + +typedef struct { + pdf_doc_t *d; + pdf_out_t out; /* the page being read */ + pdf_formkey_t forms[PDF_MAX_FORM_DEPTH + 2]; + int nforms; + pdf_map_t forms_read; /* the forms the document has read once */ + size_t reread; /* decoded bytes of forms read again */ + pdf_glyph_out_t *gbuf; + size_t gcap; +} pdf_interp_t; + +static double onum(const pdf_val_t *v) { + return (v && v->kind == PV_NUM) ? v->num : 0.0; +} + +static void show(pdf_interp_t *it, const pdf_gs_t *gs, double *tm, const pdf_val_t *str) { + pdf_font_t *font = gs->font; + if (!font) { + return; + } + pdf_doc_t *d = it->d; + pdf_out_t *out = &it->out; + if (str->n > it->gcap) { + size_t ncap = str->n * 2; + pdf_glyph_out_t *g = + (pdf_glyph_out_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, it->gbuf, ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return; + } + it->gbuf = g; + it->gcap = ncap; + } + size_t ng = 0; + pdf_font_decode(d, font, str->s, str->n, it->gbuf, &ng); + double fs = gs->fs; + double tc = gs->tc; + double tw = gs->tw; + double th = gs->th; + double rise = gs->rise; + const double *ctm = gs->ctm; + double m[6]; + mat_mul(tm, ctm, m); + double a = m[0]; + double b = m[1]; + double size = fabs(fs) * py_hypot(m[2], m[3]) * pdf_font_em(font); + double nrm = py_hypot(a, b); + double dx = 1.0; + double dy = 0.0; + if (nrm > 0) { + dx = a / nrm; + dy = b / nrm; + } + if (th < 0) { + dx = -dx; + dy = -dy; + } + double e = tm[4]; + double f = tm[5]; + double ta = tm[0]; + double tb = tm[1]; + for (size_t i = 0; i < ng; i++) { + const pdf_glyph_out_t *g = &it->gbuf[i]; + double adv = (g->width * fs + tc + (g->space ? tw : 0.0)) * th; + out->glyphs++; + const char *u = g->utf8; + if (!u) { + out->unmapped++; + u = "\xEF\xBF\xBD"; + } + if (u[0]) { + /* the glyph's origin (0, rise) in text space, in device space */ + double ux = rise * tm[2] + e; + double uy = rise * tm[3] + f; + double ox = ux * ctm[0] + uy * ctm[2] + ctm[4]; + double oy = ux * ctm[1] + uy * ctm[3] + ctm[5]; + double ex = ox + adv * a; /* where the glyph ends */ + double ey = oy + adv * b; + emit(out, u, ox, oy, ex, ey, dx, dy, size); + } + e += adv * ta; + f += adv * tb; + } + tm[4] = e; + tm[5] = f; +} + +static bool run(pdf_interp_t *it, const unsigned char *data, size_t len, pdf_val_t *resources, + const double *ctm, int depth); + +static bool form_seen(const pdf_interp_t *it, pdf_formkey_t k) { + for (int i = 0; i < it->nforms; i++) { + if (it->forms[i].is_ref == k.is_ref && it->forms[i].key == k.key) { + return true; + } + } + return false; +} + +/* false: the form's content could not be read to its end */ +static bool do_xobject(pdf_interp_t *it, pdf_val_t *res, const pdf_val_t *name, const pdf_gs_t *gs, + int depth) { + pdf_doc_t *d = it->d; + pdf_val_t *xd = pdf_dget(d, res, "XObject"); + if (!xd || xd->kind != PV_DICT) { + return true; + } + pdf_val_t *ref = pdf_dfind_n(xd, name->s, name->n); + if (ref && ref->kind == PV_NULL) { + ref = NULL; + } + pdf_val_t *xo = pdf_resolve(d, ref); + if (!xo || xo->kind != PV_STREAM) { + return true; + } + pdf_val_t *sub = pdf_dget_raw(xo->stream->dict, "Subtype"); + if (!pdf_is_name(sub, "Form")) { + return true; + } + pdf_formkey_t key = {false, (uint64_t)(uintptr_t)xo}; + if (ref && ref->kind == PV_REF) { + key.is_ref = true; + key.key = (uint64_t)ref->ref_num; + } + if (form_seen(it, key)) { + return true; + } + size_t dlen = 0; + const unsigned char *data = pdf_stream_decode(d, xo->stream, &dlen); + if (!data || !dlen) { + return true; + } + uint64_t once = key.key << 1 | (key.is_ref ? 1U : 0U); + if (pdf_map_get(&it->forms_read, once) >= 0) { + if (dlen > PDF_MAX_FORM_REREAD - it->reread) { + d->counts->form_rereads_cut++; + return true; + } + it->reread += dlen; + } else if (!pdf_map_put(&it->forms_read, once, 1)) { + d->nomem = true; + return false; + } + pdf_val_t *mat = pdf_dget(d, xo->stream->dict, "Matrix"); + double m[6]; + memcpy(m, IDENT, sizeof(m)); + if (mat && mat->kind == PV_ARR && mat->count == 6) { + for (int i = 0; i < 6; i++) { + m[i] = onum(pdf_resolve(d, mat->items[i])); + } + } + pdf_val_t *fres = pdf_dget_raw(xo->stream->dict, "Resources"); + if (!fres) { + fres = res; + } + double ctm[6]; + mat_mul(m, gs->ctm, ctm); + if (it->nforms >= (int)(sizeof(it->forms) / sizeof(it->forms[0]))) { + return true; + } + it->forms[it->nforms++] = key; + bool ok = run(it, data, dlen, fres, ctm, depth + 1); + it->nforms--; + return ok; +} + +static bool kw_is(const pdf_tok_t *t, const char *s) { + return pdf_kw_is(t, s); +} + +static bool path_paint(const pdf_tok_t *t) { + return kw_is(t, "S") || kw_is(t, "s") || kw_is(t, "f") || kw_is(t, "F") || kw_is(t, "f*") || + kw_is(t, "B") || kw_is(t, "B*") || kw_is(t, "b") || kw_is(t, "b*") || kw_is(t, "sh"); +} + +/* "[ws]EI" followed by whitespace or the end, from p: the position after EI. */ +static size_t skip_inline_image(const unsigned char *data, size_t len, size_t from) { + for (size_t i = from; i + 3 <= len; i++) { + unsigned char c = data[i]; + bool ws = c == 0 || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' '; + if (!ws || data[i + 1] != 'E' || data[i + 2] != 'I') { + continue; + } + size_t e = i + 3; + if (e == len) { + return e; + } + unsigned char x = data[e]; + if (x == 0 || x == '\t' || x == '\n' || x == '\f' || x == '\r' || x == ' ') { + return e; + } + } + return len; +} + +typedef struct { + pdf_val_t **v; + int n; + int cap; +} ops_t; + +static bool ops_push(pdf_doc_t *d, ops_t *o, pdf_val_t *x) { + if (o->n == o->cap) { + int ncap = o->cap ? o->cap * 2 : PDF_OPS_MIN; + pdf_val_t **g = (pdf_val_t **)cbm_realloc(CBM_MEM_CLASS_EXTRACT, (void *)o->v, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + o->v = g; + o->cap = ncap; + } + o->v[o->n++] = x; + return true; +} + +static void tlm_next_line(const pdf_gs_t *gs, double *tm, double *tlm) { + double e = -gs->tl * tlm[2] + tlm[4]; + double f = -gs->tl * tlm[3] + tlm[5]; + tlm[4] = e; + tlm[5] = f; + memcpy(tm, tlm, 6 * sizeof(double)); +} + +static bool run(pdf_interp_t *it, const unsigned char *data, size_t len, pdf_val_t *resources, + const double *ctm, int depth) { + if (depth > PDF_MAX_FORM_DEPTH) { + return true; + } + pdf_doc_t *d = it->d; + pdf_val_t *res = pdf_resolve(d, resources); + if (res && res->kind != PV_DICT) { + res = NULL; + } + pdf_gs_t gs; + memset(&gs, 0, sizeof(gs)); + memcpy(gs.ctm, ctm, sizeof(gs.ctm)); + gs.th = 1.0; + pdf_gs_t *stack = NULL; + int nstack = 0; + int stack_cap = 0; + double tm[6]; + double tlm[6]; + memcpy(tm, IDENT, sizeof(tm)); + memcpy(tlm, IDENT, sizeof(tlm)); + CBMArena opa; + cbm_arena_init_lazy(&opa, PDF_OPS_ARENA); + ops_t ops = {0}; + size_t pos = 0; + bool ok = true; + while (pos < len && !d->nomem) { + d->cur = &opa; + pdf_tok_t t; + pdf_lex(d, data, len, &pos, &t); + if (t.kind == PT_EOF) { + break; + } + pdf_val_t *v = NULL; + bool operand = true; + if (t.kind == PT_NUM || t.kind == PT_STR || t.kind == PT_NAME) { + v = (pdf_val_t *)pdf_calloc(d, sizeof(*v)); + if (v) { + v->kind = t.kind == PT_NUM ? PV_NUM : t.kind == PT_STR ? PV_STR : PV_NAME; + v->is_int = t.is_int; + v->num = t.num; + v->inum = t.inum; + v->s = t.s; + v->n = t.n; + } + } else if (t.kind == PT_AS) { + v = pdf_parse_content_array(d, data, len, &pos, 0); + if (!v) { + ok = false; + break; + } + } else if (t.kind == PT_DS) { + v = pdf_parse_value(d, data, len, &pos, &t, 0); + if (!v) { + ok = false; + break; + } + } else if (t.kind != PT_KW) { + continue; + } else { + operand = false; + } + d->cur = d->a; + if (operand) { + if (v) { + ops_push(d, &ops, v); + } + continue; + } + int n = ops.n; + pdf_val_t **o = ops.v; + if (kw_is(&t, "Tj") || kw_is(&t, "'") || kw_is(&t, "\"") || kw_is(&t, "TJ")) { + bool tj = kw_is(&t, "TJ"); + if (kw_is(&t, "\"") && n >= 3) { + gs.tw = onum(o[n - 3]); + gs.tc = onum(o[n - 2]); + } + if (!kw_is(&t, "Tj") && !tj) { + tlm_next_line(&gs, tm, tlm); + } + if (n == 0) { + /* nothing shown */ + } else if (tj) { + const pdf_val_t *arr = o[n - 1]; + for (int i = 0; arr && arr->kind == PV_ARR && i < arr->count; i++) { + const pdf_val_t *item = arr->items[i]; + if (item->kind == PV_STR) { + show(it, &gs, tm, item); + } else if (item->kind == PV_NUM) { + double tx = -item->num / 1000.0 * gs.fs * gs.th; + double e = tm[4] + tx * tm[0]; + double f = tm[5] + tx * tm[1]; + tm[4] = e; + tm[5] = f; + } + } + } else if (o[n - 1]->kind == PV_STR) { + show(it, &gs, tm, o[n - 1]); + } + } else if (kw_is(&t, "Td") || kw_is(&t, "TD")) { + if (n >= 2) { + double tx = onum(o[n - 2]); + double ty = onum(o[n - 1]); + if (kw_is(&t, "TD")) { + gs.tl = -ty; + } + double e = tx * tlm[0] + ty * tlm[2] + tlm[4]; + double f = tx * tlm[1] + ty * tlm[3] + tlm[5]; + tlm[4] = e; + tlm[5] = f; + memcpy(tm, tlm, sizeof(tm)); + } + } else if (kw_is(&t, "Tm")) { + if (n >= 6) { + for (int i = 0; i < 6; i++) { + tm[i] = onum(o[n - 6 + i]); + } + memcpy(tlm, tm, sizeof(tlm)); + } + } else if (kw_is(&t, "T*")) { + tlm_next_line(&gs, tm, tlm); + } else if (kw_is(&t, "Tf")) { + if (n >= 2) { + gs.fs = onum(o[n - 1]); + gs.font = o[n - 2]->kind == PV_NAME ? pdf_font_for(d, res, o[n - 2]) : NULL; + } + } else if (kw_is(&t, "BT")) { + memcpy(tm, IDENT, sizeof(tm)); + memcpy(tlm, IDENT, sizeof(tlm)); + } else if (kw_is(&t, "Tc")) { + if (n) { + gs.tc = onum(o[n - 1]); + } + } else if (kw_is(&t, "Tw")) { + if (n) { + gs.tw = onum(o[n - 1]); + } + } else if (kw_is(&t, "Tz")) { + if (n) { + gs.th = onum(o[n - 1]) / 100.0; + } + } else if (kw_is(&t, "TL")) { + if (n) { + gs.tl = onum(o[n - 1]); + } + } else if (kw_is(&t, "Ts")) { + if (n) { + gs.rise = onum(o[n - 1]); + } + } else if (kw_is(&t, "q")) { + if (nstack == stack_cap) { + int ncap = stack_cap ? stack_cap * 2 : PDF_OPS_MIN; + pdf_gs_t *g = (pdf_gs_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, stack, + (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + break; + } + stack = g; + stack_cap = ncap; + } + stack[nstack++] = gs; + } else if (kw_is(&t, "Q")) { + if (nstack) { + gs = stack[--nstack]; + } + } else if (kw_is(&t, "cm")) { + if (n >= 6) { + double m[6]; + for (int i = 0; i < 6; i++) { + m[i] = onum(o[n - 6 + i]); + } + mat_mul(m, gs.ctm, gs.ctm); + } + } else if (kw_is(&t, "Do")) { + if (n && o[n - 1]->kind == PV_NAME && !do_xobject(it, res, o[n - 1], &gs, depth)) { + d->counts->op_errors++; + } + } else if (kw_is(&t, "BI")) { + const unsigned char *id = + pos < len ? (const unsigned char *)cbm_memmem(data + pos, len - pos, "ID", 2) + : NULL; + if (!id) { + break; + } + pos = skip_inline_image(data, len, (size_t)(id - data) + 2); + } else if (path_paint(&t)) { + /* vector content: not text */ + } + ops.n = 0; + cbm_arena_rewind(&opa); + } + d->cur = d->a; + cbm_free(CBM_MEM_CLASS_EXTRACT, (void *)ops.v); + cbm_free(CBM_MEM_CLASS_EXTRACT, stack); + cbm_arena_destroy(&opa); + return ok; +} + +/* ── Pages ───────────────────────────────────────────────────────── */ + +typedef struct { + pdf_val_t *node; + pdf_val_t *res; + pdf_val_t *box; +} pdf_pageref_t; + +typedef struct { + pdf_val_t *ref; + pdf_val_t *inh; + pdf_val_t *box; + int depth; +} pdf_walk_t; + +typedef struct { + pdf_pageref_t *v; + int n; + int cap; +} pagelist_t; + +static bool pl_push(pdf_doc_t *d, pagelist_t *l, pdf_val_t *node, pdf_val_t *res, pdf_val_t *box) { + if (l->n == l->cap) { + int ncap = l->cap ? l->cap * 2 : PDF_OPS_MIN; + pdf_pageref_t *g = + (pdf_pageref_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, l->v, (size_t)ncap * sizeof(*g)); + if (!g) { + d->nomem = true; + return false; + } + l->v = g; + l->cap = ncap; + } + l->v[l->n].node = node; + l->v[l->n].res = res; + l->v[l->n].box = box; + l->n++; + return true; +} + +/* dict.get(key, dflt): a present key wins even when its value is null. */ +static pdf_val_t *dget_or(const pdf_val_t *dict, const char *key, pdf_val_t *dflt) { + return pdf_dhas(dict, key) ? pdf_dfind(dict, key) : dflt; +} + +static int i64cmp(const void *x, const void *y) { + int64_t a = *(const int64_t *)x; + int64_t b = *(const int64_t *)y; + return a < b ? -1 : a > b ? 1 : 0; +} + +static void collect_pages(pdf_doc_t *d, pagelist_t *pl) { + pdf_val_t *root = pdf_resolve(d, pdf_dget_raw(d->trailer, "Root")); + if (root && root->kind == PV_DICT) { + pdf_map_t seen_ref = {0}; + pdf_map_t seen_ptr = {0}; + pdf_walk_t *stack = NULL; + int ns = 0; + int cap = 0; +#define WALK_PUSH(R, I, B, D) \ + do { \ + if (ns == cap) { \ + int nc = cap ? cap * 2 : PDF_OPS_MIN; \ + pdf_walk_t *g = \ + (pdf_walk_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, stack, (size_t)nc * sizeof(*g)); \ + if (!g) { \ + d->nomem = true; \ + goto done; \ + } \ + stack = g; \ + cap = nc; \ + } \ + stack[ns].ref = (R); \ + stack[ns].inh = (I); \ + stack[ns].box = (B); \ + stack[ns].depth = (D); \ + ns++; \ + } while (0) + WALK_PUSH(pdf_dget_raw(root, "Pages"), NULL, NULL, 0); + while (ns && !d->nomem) { + pdf_walk_t w = stack[--ns]; + bool seen; + if (w.ref && w.ref->kind == PV_REF) { + seen = pdf_map_get(&seen_ref, (uint64_t)w.ref->ref_num) >= 0; + if (!seen && !pdf_map_put(&seen_ref, (uint64_t)w.ref->ref_num, 1)) { + d->nomem = true; + } + } else { + uint64_t key = (uint64_t)(uintptr_t)w.ref; + seen = pdf_map_get(&seen_ptr, key) >= 0; + if (!seen && !pdf_map_put(&seen_ptr, key, 1)) { + d->nomem = true; + } + } + if (seen || w.depth > PDF_MAX_PAGE_DEPTH) { + continue; + } + pdf_val_t *node = pdf_resolve(d, w.ref); + if (!node || node->kind != PV_DICT) { + continue; + } + pdf_val_t *res = dget_or(node, "Resources", w.inh); + pdf_val_t *box = dget_or(node, "CropBox", dget_or(node, "MediaBox", w.box)); + pdf_val_t *kids = pdf_resolve(d, pdf_dget_raw(node, "Kids")); + pdf_val_t *typ = pdf_dget_raw(node, "Type"); + bool is_arr = kids && kids->kind == PV_ARR; + if (pdf_is_name(typ, "Pages") || (!pdf_is_name(typ, "Page") && is_arr)) { + for (int i = is_arr ? kids->count - 1 : -1; i >= 0; i--) { + WALK_PUSH(kids->items[i], res, box, w.depth + 1); + } + } else { + pl_push(d, pl, node, res, box); + } + } +#undef WALK_PUSH + done: + cbm_free(CBM_MEM_CLASS_EXTRACT, stack); + pdf_map_free(&seen_ref); + pdf_map_free(&seen_ptr); + } + if (pl->n || d->nomem) { + return; + } + int n = d->xref.n; + int64_t *nums = + (int64_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1) * sizeof(int64_t)); + if (!nums) { + d->nomem = true; + return; + } + for (int i = 0; i < n; i++) { + nums[i] = d->xref.v[i].num; + } + qsort(nums, (size_t)n, sizeof(int64_t), i64cmp); + for (int i = 0; i < n; i++) { + pdf_val_t *o = pdf_get(d, nums[i]); + if (o && o->kind == PV_DICT && pdf_is_name(pdf_dget_raw(o, "Type"), "Page")) { + pl_push(d, pl, o, pdf_dget_raw(o, "Resources"), + dget_or(o, "CropBox", pdf_dget_raw(o, "MediaBox"))); + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, nums); +} + +static void page_extract(pdf_doc_t *d, pdf_interp_t *it, const pdf_pageref_t *pg, CBMArena *dst, + cbm_pdf_page_t *out) { + pdf_out_t *o = &it->out; + memset(o, 0, sizeof(*o)); + it->nforms = 0; + pdf_val_t *cont = pdf_dget(d, pg->node, "Contents"); + pdf_val_t *single[1] = {cont}; + pdf_val_t **parts = single; + int nparts = cont ? 1 : 0; + if (cont && cont->kind == PV_ARR) { + parts = cont->items; + nparts = cont->count; + } + /* the parts' decoded data joined by "\n" */ + const unsigned char *one = NULL; + size_t one_len = 0; + unsigned char *joined = NULL; + size_t jn = 0; + int nchunks = 0; + for (int i = 0; i < nparts && !d->nomem; i++) { + pdf_val_t *p = pdf_resolve(d, parts[i]); + if (!p || p->kind != PV_STREAM) { + continue; + } + size_t dl = 0; + const unsigned char *dd = pdf_stream_decode(d, p->stream, &dl); + if (!dd || !dl) { + continue; + } + if (nchunks == 0) { + one = dd; + one_len = dl; + } else { + size_t add = (nchunks == 1 ? one_len : 0) + 1 + dl; + unsigned char *g = + (unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, joined, jn + add); + if (!g) { + d->nomem = true; + break; + } + joined = g; + if (nchunks == 1) { + memcpy(joined, one, one_len); + jn = one_len; + } + joined[jn++] = '\n'; + memcpy(joined + jn, dd, dl); + jn += dl; + } + nchunks++; + } + const unsigned char *content = nchunks > 1 ? joined : one; + size_t clen = nchunks > 1 ? jn : one_len; + if (!run(it, content ? content : (const unsigned char *)"", content ? clen : 0, pg->res, IDENT, + 0)) { + d->counts->page_errors++; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, joined); + size_t tl = 0; + out->text = page_text(d, o, dst, &tl); + out->len = tl; + out->glyphs = o->glyphs; + out->unmapped = o->unmapped; + if (!out->text) { + out->text = ""; + out->len = 0; + } + if (o->fail) { + d->nomem = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, o->p); +} + +bool cbm_pdf_extract(const unsigned char *data, size_t len, CBMArena *out, cbm_pdf_result_t *res) { + memset(res, 0, sizeof(*res)); + const unsigned char *h = data ? (const unsigned char *)cbm_memmem(data, len, "%PDF-", 5) : NULL; + if (!h || (size_t)(h - data) > PDF_HEADER_WINDOW) { + res->status = CBM_PDF_NOT_PDF; + return true; + } + CBMArena arena; + cbm_arena_init(&arena); + pdf_doc_t d; + memset(&d, 0, sizeof(d)); + d.data = data; + d.len = len; + d.a = &arena; + d.cur = &arena; + d.counts = &res->counts; + pdf_doc_open(&d); + pagelist_t pl = {0}; + if (!d.nomem) { + pdf_val_t *enc = pdf_dget_raw(d.trailer, "Encrypt"); + pdf_val_t *encd = pdf_resolve(&d, enc); + if (encd && encd->kind == PV_DICT) { + res->status = CBM_PDF_ENCRYPTED; + pdf_doc_close(&d); + cbm_arena_destroy(&arena); + return true; + } + collect_pages(&d, &pl); + } + if (!d.nomem && pl.n) { + res->pages = (cbm_pdf_page_t *)cbm_arena_calloc(out, (size_t)pl.n * sizeof(*res->pages)); + if (!res->pages) { + d.nomem = true; + } + } + pdf_interp_t it; + memset(&it, 0, sizeof(it)); + it.d = &d; + for (int i = 0; i < pl.n && !d.nomem; i++) { + page_extract(&d, &it, &pl.v[i], out, &res->pages[i]); + res->npages = i + 1; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, it.gbuf); + pdf_map_free(&it.forms_read); + cbm_free(CBM_MEM_CLASS_EXTRACT, pl.v); + bool nomem = d.nomem; + pdf_doc_close(&d); + cbm_arena_destroy(&arena); + if (nomem) { + res->status = CBM_PDF_NOMEM; + res->pages = NULL; + res->npages = 0; + return false; + } + res->status = res->npages ? CBM_PDF_OK : CBM_PDF_PARSE_FAILURE; + return true; +} diff --git a/internal/cbm/pdf/pdf_unicode.c b/internal/cbm/pdf/pdf_unicode.c new file mode 100644 index 0000000000..88ccaacb78 --- /dev/null +++ b/internal/cbm/pdf/pdf_unicode.c @@ -0,0 +1,3126 @@ +/* + * pdf_unicode.c -- Unicode 16.0.0 facts for the PDF mention scanner (generated). + * + * The field test's scanner ran on CPython's str and re; these tables are what + * CPython 3.14.6 answers: NFKC of each code point, the canonical compositions, + * str.isspace, the re module's \w (str.isalnum or '_'), str.isalpha, and the + * one-character str.isupper / str.islower / titlecase. Generated by + * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode + * Character Database, (c) Unicode, Inc., Unicode License v3 + * (THIRD_PARTY.md). + */ +#include "pdf_unicode.h" + +const pdf_nfkc_t PDF_NFKC[] = { + {0xA0, 0, 1}, {0xA8, 1, 2}, {0xAA, 3, 1}, {0xAF, 4, 2}, + {0xB2, 6, 1}, {0xB3, 7, 1}, {0xB4, 8, 2}, {0xB5, 10, 1}, + {0xB8, 11, 2}, {0xB9, 13, 1}, {0xBA, 14, 1}, {0xBC, 15, 3}, + {0xBD, 18, 3}, {0xBE, 21, 3}, {0x132, 24, 2}, {0x133, 26, 2}, + {0x13F, 28, 2}, {0x140, 30, 2}, {0x149, 32, 2}, {0x17F, 34, 1}, + {0x1C4, 35, 2}, {0x1C5, 37, 2}, {0x1C6, 39, 2}, {0x1C7, 41, 2}, + {0x1C8, 43, 2}, {0x1C9, 45, 2}, {0x1CA, 47, 2}, {0x1CB, 49, 2}, + {0x1CC, 51, 2}, {0x1F1, 53, 2}, {0x1F2, 55, 2}, {0x1F3, 57, 2}, + {0x2B0, 59, 1}, {0x2B1, 60, 1}, {0x2B2, 61, 1}, {0x2B3, 62, 1}, + {0x2B4, 63, 1}, {0x2B5, 64, 1}, {0x2B6, 65, 1}, {0x2B7, 66, 1}, + {0x2B8, 67, 1}, {0x2D8, 68, 2}, {0x2D9, 70, 2}, {0x2DA, 72, 2}, + {0x2DB, 74, 2}, {0x2DC, 76, 2}, {0x2DD, 78, 2}, {0x2E0, 80, 1}, + {0x2E1, 81, 1}, {0x2E2, 82, 1}, {0x2E3, 83, 1}, {0x2E4, 84, 1}, + {0x340, 85, 1}, {0x341, 86, 1}, {0x343, 87, 1}, {0x344, 88, 2}, + {0x374, 90, 1}, {0x37A, 91, 2}, {0x37E, 93, 1}, {0x384, 94, 2}, + {0x385, 96, 3}, {0x387, 99, 1}, {0x3D0, 100, 1}, {0x3D1, 101, 1}, + {0x3D2, 102, 1}, {0x3D3, 103, 1}, {0x3D4, 104, 1}, {0x3D5, 105, 1}, + {0x3D6, 106, 1}, {0x3F0, 107, 1}, {0x3F1, 108, 1}, {0x3F2, 109, 1}, + {0x3F4, 110, 1}, {0x3F5, 111, 1}, {0x3F9, 112, 1}, {0x587, 113, 2}, + {0x675, 115, 2}, {0x676, 117, 2}, {0x677, 119, 2}, {0x678, 121, 2}, + {0x958, 123, 2}, {0x959, 125, 2}, {0x95A, 127, 2}, {0x95B, 129, 2}, + {0x95C, 131, 2}, {0x95D, 133, 2}, {0x95E, 135, 2}, {0x95F, 137, 2}, + {0x9DC, 139, 2}, {0x9DD, 141, 2}, {0x9DF, 143, 2}, {0xA33, 145, 2}, + {0xA36, 147, 2}, {0xA59, 149, 2}, {0xA5A, 151, 2}, {0xA5B, 153, 2}, + {0xA5E, 155, 2}, {0xB5C, 157, 2}, {0xB5D, 159, 2}, {0xE33, 161, 2}, + {0xEB3, 163, 2}, {0xEDC, 165, 2}, {0xEDD, 167, 2}, {0xF0C, 169, 1}, + {0xF43, 170, 2}, {0xF4D, 172, 2}, {0xF52, 174, 2}, {0xF57, 176, 2}, + {0xF5C, 178, 2}, {0xF69, 180, 2}, {0xF73, 182, 2}, {0xF75, 184, 2}, + {0xF76, 186, 2}, {0xF77, 188, 3}, {0xF78, 191, 2}, {0xF79, 193, 3}, + {0xF81, 196, 2}, {0xF93, 198, 2}, {0xF9D, 200, 2}, {0xFA2, 202, 2}, + {0xFA7, 204, 2}, {0xFAC, 206, 2}, {0xFB9, 208, 2}, {0x10FC, 210, 1}, + {0x1D2C, 211, 1}, {0x1D2D, 212, 1}, {0x1D2E, 213, 1}, {0x1D30, 214, 1}, + {0x1D31, 215, 1}, {0x1D32, 216, 1}, {0x1D33, 217, 1}, {0x1D34, 218, 1}, + {0x1D35, 219, 1}, {0x1D36, 220, 1}, {0x1D37, 221, 1}, {0x1D38, 222, 1}, + {0x1D39, 223, 1}, {0x1D3A, 224, 1}, {0x1D3C, 225, 1}, {0x1D3D, 226, 1}, + {0x1D3E, 227, 1}, {0x1D3F, 228, 1}, {0x1D40, 229, 1}, {0x1D41, 230, 1}, + {0x1D42, 231, 1}, {0x1D43, 232, 1}, {0x1D44, 233, 1}, {0x1D45, 234, 1}, + {0x1D46, 235, 1}, {0x1D47, 236, 1}, {0x1D48, 237, 1}, {0x1D49, 238, 1}, + {0x1D4A, 239, 1}, {0x1D4B, 240, 1}, {0x1D4C, 241, 1}, {0x1D4D, 242, 1}, + {0x1D4F, 243, 1}, {0x1D50, 244, 1}, {0x1D51, 245, 1}, {0x1D52, 246, 1}, + {0x1D53, 247, 1}, {0x1D54, 248, 1}, {0x1D55, 249, 1}, {0x1D56, 250, 1}, + {0x1D57, 251, 1}, {0x1D58, 252, 1}, {0x1D59, 253, 1}, {0x1D5A, 254, 1}, + {0x1D5B, 255, 1}, {0x1D5C, 256, 1}, {0x1D5D, 257, 1}, {0x1D5E, 258, 1}, + {0x1D5F, 259, 1}, {0x1D60, 260, 1}, {0x1D61, 261, 1}, {0x1D62, 262, 1}, + {0x1D63, 263, 1}, {0x1D64, 264, 1}, {0x1D65, 265, 1}, {0x1D66, 266, 1}, + {0x1D67, 267, 1}, {0x1D68, 268, 1}, {0x1D69, 269, 1}, {0x1D6A, 270, 1}, + {0x1D78, 271, 1}, {0x1D9B, 272, 1}, {0x1D9C, 273, 1}, {0x1D9D, 274, 1}, + {0x1D9E, 275, 1}, {0x1D9F, 276, 1}, {0x1DA0, 277, 1}, {0x1DA1, 278, 1}, + {0x1DA2, 279, 1}, {0x1DA3, 280, 1}, {0x1DA4, 281, 1}, {0x1DA5, 282, 1}, + {0x1DA6, 283, 1}, {0x1DA7, 284, 1}, {0x1DA8, 285, 1}, {0x1DA9, 286, 1}, + {0x1DAA, 287, 1}, {0x1DAB, 288, 1}, {0x1DAC, 289, 1}, {0x1DAD, 290, 1}, + {0x1DAE, 291, 1}, {0x1DAF, 292, 1}, {0x1DB0, 293, 1}, {0x1DB1, 294, 1}, + {0x1DB2, 295, 1}, {0x1DB3, 296, 1}, {0x1DB4, 297, 1}, {0x1DB5, 298, 1}, + {0x1DB6, 299, 1}, {0x1DB7, 300, 1}, {0x1DB8, 301, 1}, {0x1DB9, 302, 1}, + {0x1DBA, 303, 1}, {0x1DBB, 304, 1}, {0x1DBC, 305, 1}, {0x1DBD, 306, 1}, + {0x1DBE, 307, 1}, {0x1DBF, 308, 1}, {0x1E9A, 309, 2}, {0x1E9B, 311, 1}, + {0x1F71, 312, 1}, {0x1F73, 313, 1}, {0x1F75, 314, 1}, {0x1F77, 315, 1}, + {0x1F79, 316, 1}, {0x1F7B, 317, 1}, {0x1F7D, 318, 1}, {0x1FBB, 319, 1}, + {0x1FBD, 320, 2}, {0x1FBE, 322, 1}, {0x1FBF, 323, 2}, {0x1FC0, 325, 2}, + {0x1FC1, 327, 3}, {0x1FC9, 330, 1}, {0x1FCB, 331, 1}, {0x1FCD, 332, 3}, + {0x1FCE, 335, 3}, {0x1FCF, 338, 3}, {0x1FD3, 341, 1}, {0x1FDB, 342, 1}, + {0x1FDD, 343, 3}, {0x1FDE, 346, 3}, {0x1FDF, 349, 3}, {0x1FE3, 352, 1}, + {0x1FEB, 353, 1}, {0x1FED, 354, 3}, {0x1FEE, 357, 3}, {0x1FEF, 360, 1}, + {0x1FF9, 361, 1}, {0x1FFB, 362, 1}, {0x1FFD, 363, 2}, {0x1FFE, 365, 2}, + {0x2000, 367, 1}, {0x2001, 368, 1}, {0x2002, 369, 1}, {0x2003, 370, 1}, + {0x2004, 371, 1}, {0x2005, 372, 1}, {0x2006, 373, 1}, {0x2007, 374, 1}, + {0x2008, 375, 1}, {0x2009, 376, 1}, {0x200A, 377, 1}, {0x2011, 378, 1}, + {0x2017, 379, 2}, {0x2024, 381, 1}, {0x2025, 382, 2}, {0x2026, 384, 3}, + {0x202F, 387, 1}, {0x2033, 388, 2}, {0x2034, 390, 3}, {0x2036, 393, 2}, + {0x2037, 395, 3}, {0x203C, 398, 2}, {0x203E, 400, 2}, {0x2047, 402, 2}, + {0x2048, 404, 2}, {0x2049, 406, 2}, {0x2057, 408, 4}, {0x205F, 412, 1}, + {0x2070, 413, 1}, {0x2071, 414, 1}, {0x2074, 415, 1}, {0x2075, 416, 1}, + {0x2076, 417, 1}, {0x2077, 418, 1}, {0x2078, 419, 1}, {0x2079, 420, 1}, + {0x207A, 421, 1}, {0x207B, 422, 1}, {0x207C, 423, 1}, {0x207D, 424, 1}, + {0x207E, 425, 1}, {0x207F, 426, 1}, {0x2080, 427, 1}, {0x2081, 428, 1}, + {0x2082, 429, 1}, {0x2083, 430, 1}, {0x2084, 431, 1}, {0x2085, 432, 1}, + {0x2086, 433, 1}, {0x2087, 434, 1}, {0x2088, 435, 1}, {0x2089, 436, 1}, + {0x208A, 437, 1}, {0x208B, 438, 1}, {0x208C, 439, 1}, {0x208D, 440, 1}, + {0x208E, 441, 1}, {0x2090, 442, 1}, {0x2091, 443, 1}, {0x2092, 444, 1}, + {0x2093, 445, 1}, {0x2094, 446, 1}, {0x2095, 447, 1}, {0x2096, 448, 1}, + {0x2097, 449, 1}, {0x2098, 450, 1}, {0x2099, 451, 1}, {0x209A, 452, 1}, + {0x209B, 453, 1}, {0x209C, 454, 1}, {0x20A8, 455, 2}, {0x2100, 457, 3}, + {0x2101, 460, 3}, {0x2102, 463, 1}, {0x2103, 464, 2}, {0x2105, 466, 3}, + {0x2106, 469, 3}, {0x2107, 472, 1}, {0x2109, 473, 2}, {0x210A, 475, 1}, + {0x210B, 476, 1}, {0x210C, 477, 1}, {0x210D, 478, 1}, {0x210E, 479, 1}, + {0x210F, 480, 1}, {0x2110, 481, 1}, {0x2111, 482, 1}, {0x2112, 483, 1}, + {0x2113, 484, 1}, {0x2115, 485, 1}, {0x2116, 486, 2}, {0x2119, 488, 1}, + {0x211A, 489, 1}, {0x211B, 490, 1}, {0x211C, 491, 1}, {0x211D, 492, 1}, + {0x2120, 493, 2}, {0x2121, 495, 3}, {0x2122, 498, 2}, {0x2124, 500, 1}, + {0x2126, 501, 1}, {0x2128, 502, 1}, {0x212A, 503, 1}, {0x212B, 504, 1}, + {0x212C, 505, 1}, {0x212D, 506, 1}, {0x212F, 507, 1}, {0x2130, 508, 1}, + {0x2131, 509, 1}, {0x2133, 510, 1}, {0x2134, 511, 1}, {0x2135, 512, 1}, + {0x2136, 513, 1}, {0x2137, 514, 1}, {0x2138, 515, 1}, {0x2139, 516, 1}, + {0x213B, 517, 3}, {0x213C, 520, 1}, {0x213D, 521, 1}, {0x213E, 522, 1}, + {0x213F, 523, 1}, {0x2140, 524, 1}, {0x2145, 525, 1}, {0x2146, 526, 1}, + {0x2147, 527, 1}, {0x2148, 528, 1}, {0x2149, 529, 1}, {0x2150, 530, 3}, + {0x2151, 533, 3}, {0x2152, 536, 4}, {0x2153, 540, 3}, {0x2154, 543, 3}, + {0x2155, 546, 3}, {0x2156, 549, 3}, {0x2157, 552, 3}, {0x2158, 555, 3}, + {0x2159, 558, 3}, {0x215A, 561, 3}, {0x215B, 564, 3}, {0x215C, 567, 3}, + {0x215D, 570, 3}, {0x215E, 573, 3}, {0x215F, 576, 2}, {0x2160, 578, 1}, + {0x2161, 579, 2}, {0x2162, 581, 3}, {0x2163, 584, 2}, {0x2164, 586, 1}, + {0x2165, 587, 2}, {0x2166, 589, 3}, {0x2167, 592, 4}, {0x2168, 596, 2}, + {0x2169, 598, 1}, {0x216A, 599, 2}, {0x216B, 601, 3}, {0x216C, 604, 1}, + {0x216D, 605, 1}, {0x216E, 606, 1}, {0x216F, 607, 1}, {0x2170, 608, 1}, + {0x2171, 609, 2}, {0x2172, 611, 3}, {0x2173, 614, 2}, {0x2174, 616, 1}, + {0x2175, 617, 2}, {0x2176, 619, 3}, {0x2177, 622, 4}, {0x2178, 626, 2}, + {0x2179, 628, 1}, {0x217A, 629, 2}, {0x217B, 631, 3}, {0x217C, 634, 1}, + {0x217D, 635, 1}, {0x217E, 636, 1}, {0x217F, 637, 1}, {0x2189, 638, 3}, + {0x222C, 641, 2}, {0x222D, 643, 3}, {0x222F, 646, 2}, {0x2230, 648, 3}, + {0x2329, 651, 1}, {0x232A, 652, 1}, {0x2460, 653, 1}, {0x2461, 654, 1}, + {0x2462, 655, 1}, {0x2463, 656, 1}, {0x2464, 657, 1}, {0x2465, 658, 1}, + {0x2466, 659, 1}, {0x2467, 660, 1}, {0x2468, 661, 1}, {0x2469, 662, 2}, + {0x246A, 664, 2}, {0x246B, 666, 2}, {0x246C, 668, 2}, {0x246D, 670, 2}, + {0x246E, 672, 2}, {0x246F, 674, 2}, {0x2470, 676, 2}, {0x2471, 678, 2}, + {0x2472, 680, 2}, {0x2473, 682, 2}, {0x2474, 684, 3}, {0x2475, 687, 3}, + {0x2476, 690, 3}, {0x2477, 693, 3}, {0x2478, 696, 3}, {0x2479, 699, 3}, + {0x247A, 702, 3}, {0x247B, 705, 3}, {0x247C, 708, 3}, {0x247D, 711, 4}, + {0x247E, 715, 4}, {0x247F, 719, 4}, {0x2480, 723, 4}, {0x2481, 727, 4}, + {0x2482, 731, 4}, {0x2483, 735, 4}, {0x2484, 739, 4}, {0x2485, 743, 4}, + {0x2486, 747, 4}, {0x2487, 751, 4}, {0x2488, 755, 2}, {0x2489, 757, 2}, + {0x248A, 759, 2}, {0x248B, 761, 2}, {0x248C, 763, 2}, {0x248D, 765, 2}, + {0x248E, 767, 2}, {0x248F, 769, 2}, {0x2490, 771, 2}, {0x2491, 773, 3}, + {0x2492, 776, 3}, {0x2493, 779, 3}, {0x2494, 782, 3}, {0x2495, 785, 3}, + {0x2496, 788, 3}, {0x2497, 791, 3}, {0x2498, 794, 3}, {0x2499, 797, 3}, + {0x249A, 800, 3}, {0x249B, 803, 3}, {0x249C, 806, 3}, {0x249D, 809, 3}, + {0x249E, 812, 3}, {0x249F, 815, 3}, {0x24A0, 818, 3}, {0x24A1, 821, 3}, + {0x24A2, 824, 3}, {0x24A3, 827, 3}, {0x24A4, 830, 3}, {0x24A5, 833, 3}, + {0x24A6, 836, 3}, {0x24A7, 839, 3}, {0x24A8, 842, 3}, {0x24A9, 845, 3}, + {0x24AA, 848, 3}, {0x24AB, 851, 3}, {0x24AC, 854, 3}, {0x24AD, 857, 3}, + {0x24AE, 860, 3}, {0x24AF, 863, 3}, {0x24B0, 866, 3}, {0x24B1, 869, 3}, + {0x24B2, 872, 3}, {0x24B3, 875, 3}, {0x24B4, 878, 3}, {0x24B5, 881, 3}, + {0x24B6, 884, 1}, {0x24B7, 885, 1}, {0x24B8, 886, 1}, {0x24B9, 887, 1}, + {0x24BA, 888, 1}, {0x24BB, 889, 1}, {0x24BC, 890, 1}, {0x24BD, 891, 1}, + {0x24BE, 892, 1}, {0x24BF, 893, 1}, {0x24C0, 894, 1}, {0x24C1, 895, 1}, + {0x24C2, 896, 1}, {0x24C3, 897, 1}, {0x24C4, 898, 1}, {0x24C5, 899, 1}, + {0x24C6, 900, 1}, {0x24C7, 901, 1}, {0x24C8, 902, 1}, {0x24C9, 903, 1}, + {0x24CA, 904, 1}, {0x24CB, 905, 1}, {0x24CC, 906, 1}, {0x24CD, 907, 1}, + {0x24CE, 908, 1}, {0x24CF, 909, 1}, {0x24D0, 910, 1}, {0x24D1, 911, 1}, + {0x24D2, 912, 1}, {0x24D3, 913, 1}, {0x24D4, 914, 1}, {0x24D5, 915, 1}, + {0x24D6, 916, 1}, {0x24D7, 917, 1}, {0x24D8, 918, 1}, {0x24D9, 919, 1}, + {0x24DA, 920, 1}, {0x24DB, 921, 1}, {0x24DC, 922, 1}, {0x24DD, 923, 1}, + {0x24DE, 924, 1}, {0x24DF, 925, 1}, {0x24E0, 926, 1}, {0x24E1, 927, 1}, + {0x24E2, 928, 1}, {0x24E3, 929, 1}, {0x24E4, 930, 1}, {0x24E5, 931, 1}, + {0x24E6, 932, 1}, {0x24E7, 933, 1}, {0x24E8, 934, 1}, {0x24E9, 935, 1}, + {0x24EA, 936, 1}, {0x2A0C, 937, 4}, {0x2A74, 941, 3}, {0x2A75, 944, 2}, + {0x2A76, 946, 3}, {0x2ADC, 949, 2}, {0x2C7C, 951, 1}, {0x2C7D, 952, 1}, + {0x2D6F, 953, 1}, {0x2E9F, 954, 1}, {0x2EF3, 955, 1}, {0x2F00, 956, 1}, + {0x2F01, 957, 1}, {0x2F02, 958, 1}, {0x2F03, 959, 1}, {0x2F04, 960, 1}, + {0x2F05, 961, 1}, {0x2F06, 962, 1}, {0x2F07, 963, 1}, {0x2F08, 964, 1}, + {0x2F09, 965, 1}, {0x2F0A, 966, 1}, {0x2F0B, 967, 1}, {0x2F0C, 968, 1}, + {0x2F0D, 969, 1}, {0x2F0E, 970, 1}, {0x2F0F, 971, 1}, {0x2F10, 972, 1}, + {0x2F11, 973, 1}, {0x2F12, 974, 1}, {0x2F13, 975, 1}, {0x2F14, 976, 1}, + {0x2F15, 977, 1}, {0x2F16, 978, 1}, {0x2F17, 979, 1}, {0x2F18, 980, 1}, + {0x2F19, 981, 1}, {0x2F1A, 982, 1}, {0x2F1B, 983, 1}, {0x2F1C, 984, 1}, + {0x2F1D, 985, 1}, {0x2F1E, 986, 1}, {0x2F1F, 987, 1}, {0x2F20, 988, 1}, + {0x2F21, 989, 1}, {0x2F22, 990, 1}, {0x2F23, 991, 1}, {0x2F24, 992, 1}, + {0x2F25, 993, 1}, {0x2F26, 994, 1}, {0x2F27, 995, 1}, {0x2F28, 996, 1}, + {0x2F29, 997, 1}, {0x2F2A, 998, 1}, {0x2F2B, 999, 1}, {0x2F2C, 1000, 1}, + {0x2F2D, 1001, 1}, {0x2F2E, 1002, 1}, {0x2F2F, 1003, 1}, {0x2F30, 1004, 1}, + {0x2F31, 1005, 1}, {0x2F32, 1006, 1}, {0x2F33, 1007, 1}, {0x2F34, 1008, 1}, + {0x2F35, 1009, 1}, {0x2F36, 1010, 1}, {0x2F37, 1011, 1}, {0x2F38, 1012, 1}, + {0x2F39, 1013, 1}, {0x2F3A, 1014, 1}, {0x2F3B, 1015, 1}, {0x2F3C, 1016, 1}, + {0x2F3D, 1017, 1}, {0x2F3E, 1018, 1}, {0x2F3F, 1019, 1}, {0x2F40, 1020, 1}, + {0x2F41, 1021, 1}, {0x2F42, 1022, 1}, {0x2F43, 1023, 1}, {0x2F44, 1024, 1}, + {0x2F45, 1025, 1}, {0x2F46, 1026, 1}, {0x2F47, 1027, 1}, {0x2F48, 1028, 1}, + {0x2F49, 1029, 1}, {0x2F4A, 1030, 1}, {0x2F4B, 1031, 1}, {0x2F4C, 1032, 1}, + {0x2F4D, 1033, 1}, {0x2F4E, 1034, 1}, {0x2F4F, 1035, 1}, {0x2F50, 1036, 1}, + {0x2F51, 1037, 1}, {0x2F52, 1038, 1}, {0x2F53, 1039, 1}, {0x2F54, 1040, 1}, + {0x2F55, 1041, 1}, {0x2F56, 1042, 1}, {0x2F57, 1043, 1}, {0x2F58, 1044, 1}, + {0x2F59, 1045, 1}, {0x2F5A, 1046, 1}, {0x2F5B, 1047, 1}, {0x2F5C, 1048, 1}, + {0x2F5D, 1049, 1}, {0x2F5E, 1050, 1}, {0x2F5F, 1051, 1}, {0x2F60, 1052, 1}, + {0x2F61, 1053, 1}, {0x2F62, 1054, 1}, {0x2F63, 1055, 1}, {0x2F64, 1056, 1}, + {0x2F65, 1057, 1}, {0x2F66, 1058, 1}, {0x2F67, 1059, 1}, {0x2F68, 1060, 1}, + {0x2F69, 1061, 1}, {0x2F6A, 1062, 1}, {0x2F6B, 1063, 1}, {0x2F6C, 1064, 1}, + {0x2F6D, 1065, 1}, {0x2F6E, 1066, 1}, {0x2F6F, 1067, 1}, {0x2F70, 1068, 1}, + {0x2F71, 1069, 1}, {0x2F72, 1070, 1}, {0x2F73, 1071, 1}, {0x2F74, 1072, 1}, + {0x2F75, 1073, 1}, {0x2F76, 1074, 1}, {0x2F77, 1075, 1}, {0x2F78, 1076, 1}, + {0x2F79, 1077, 1}, {0x2F7A, 1078, 1}, {0x2F7B, 1079, 1}, {0x2F7C, 1080, 1}, + {0x2F7D, 1081, 1}, {0x2F7E, 1082, 1}, {0x2F7F, 1083, 1}, {0x2F80, 1084, 1}, + {0x2F81, 1085, 1}, {0x2F82, 1086, 1}, {0x2F83, 1087, 1}, {0x2F84, 1088, 1}, + {0x2F85, 1089, 1}, {0x2F86, 1090, 1}, {0x2F87, 1091, 1}, {0x2F88, 1092, 1}, + {0x2F89, 1093, 1}, {0x2F8A, 1094, 1}, {0x2F8B, 1095, 1}, {0x2F8C, 1096, 1}, + {0x2F8D, 1097, 1}, {0x2F8E, 1098, 1}, {0x2F8F, 1099, 1}, {0x2F90, 1100, 1}, + {0x2F91, 1101, 1}, {0x2F92, 1102, 1}, {0x2F93, 1103, 1}, {0x2F94, 1104, 1}, + {0x2F95, 1105, 1}, {0x2F96, 1106, 1}, {0x2F97, 1107, 1}, {0x2F98, 1108, 1}, + {0x2F99, 1109, 1}, {0x2F9A, 1110, 1}, {0x2F9B, 1111, 1}, {0x2F9C, 1112, 1}, + {0x2F9D, 1113, 1}, {0x2F9E, 1114, 1}, {0x2F9F, 1115, 1}, {0x2FA0, 1116, 1}, + {0x2FA1, 1117, 1}, {0x2FA2, 1118, 1}, {0x2FA3, 1119, 1}, {0x2FA4, 1120, 1}, + {0x2FA5, 1121, 1}, {0x2FA6, 1122, 1}, {0x2FA7, 1123, 1}, {0x2FA8, 1124, 1}, + {0x2FA9, 1125, 1}, {0x2FAA, 1126, 1}, {0x2FAB, 1127, 1}, {0x2FAC, 1128, 1}, + {0x2FAD, 1129, 1}, {0x2FAE, 1130, 1}, {0x2FAF, 1131, 1}, {0x2FB0, 1132, 1}, + {0x2FB1, 1133, 1}, {0x2FB2, 1134, 1}, {0x2FB3, 1135, 1}, {0x2FB4, 1136, 1}, + {0x2FB5, 1137, 1}, {0x2FB6, 1138, 1}, {0x2FB7, 1139, 1}, {0x2FB8, 1140, 1}, + {0x2FB9, 1141, 1}, {0x2FBA, 1142, 1}, {0x2FBB, 1143, 1}, {0x2FBC, 1144, 1}, + {0x2FBD, 1145, 1}, {0x2FBE, 1146, 1}, {0x2FBF, 1147, 1}, {0x2FC0, 1148, 1}, + {0x2FC1, 1149, 1}, {0x2FC2, 1150, 1}, {0x2FC3, 1151, 1}, {0x2FC4, 1152, 1}, + {0x2FC5, 1153, 1}, {0x2FC6, 1154, 1}, {0x2FC7, 1155, 1}, {0x2FC8, 1156, 1}, + {0x2FC9, 1157, 1}, {0x2FCA, 1158, 1}, {0x2FCB, 1159, 1}, {0x2FCC, 1160, 1}, + {0x2FCD, 1161, 1}, {0x2FCE, 1162, 1}, {0x2FCF, 1163, 1}, {0x2FD0, 1164, 1}, + {0x2FD1, 1165, 1}, {0x2FD2, 1166, 1}, {0x2FD3, 1167, 1}, {0x2FD4, 1168, 1}, + {0x2FD5, 1169, 1}, {0x3000, 1170, 1}, {0x3036, 1171, 1}, {0x3038, 1172, 1}, + {0x3039, 1173, 1}, {0x303A, 1174, 1}, {0x309B, 1175, 2}, {0x309C, 1177, 2}, + {0x309F, 1179, 2}, {0x30FF, 1181, 2}, {0x3131, 1183, 1}, {0x3132, 1184, 1}, + {0x3133, 1185, 1}, {0x3134, 1186, 1}, {0x3135, 1187, 1}, {0x3136, 1188, 1}, + {0x3137, 1189, 1}, {0x3138, 1190, 1}, {0x3139, 1191, 1}, {0x313A, 1192, 1}, + {0x313B, 1193, 1}, {0x313C, 1194, 1}, {0x313D, 1195, 1}, {0x313E, 1196, 1}, + {0x313F, 1197, 1}, {0x3140, 1198, 1}, {0x3141, 1199, 1}, {0x3142, 1200, 1}, + {0x3143, 1201, 1}, {0x3144, 1202, 1}, {0x3145, 1203, 1}, {0x3146, 1204, 1}, + {0x3147, 1205, 1}, {0x3148, 1206, 1}, {0x3149, 1207, 1}, {0x314A, 1208, 1}, + {0x314B, 1209, 1}, {0x314C, 1210, 1}, {0x314D, 1211, 1}, {0x314E, 1212, 1}, + {0x314F, 1213, 1}, {0x3150, 1214, 1}, {0x3151, 1215, 1}, {0x3152, 1216, 1}, + {0x3153, 1217, 1}, {0x3154, 1218, 1}, {0x3155, 1219, 1}, {0x3156, 1220, 1}, + {0x3157, 1221, 1}, {0x3158, 1222, 1}, {0x3159, 1223, 1}, {0x315A, 1224, 1}, + {0x315B, 1225, 1}, {0x315C, 1226, 1}, {0x315D, 1227, 1}, {0x315E, 1228, 1}, + {0x315F, 1229, 1}, {0x3160, 1230, 1}, {0x3161, 1231, 1}, {0x3162, 1232, 1}, + {0x3163, 1233, 1}, {0x3164, 1234, 1}, {0x3165, 1235, 1}, {0x3166, 1236, 1}, + {0x3167, 1237, 1}, {0x3168, 1238, 1}, {0x3169, 1239, 1}, {0x316A, 1240, 1}, + {0x316B, 1241, 1}, {0x316C, 1242, 1}, {0x316D, 1243, 1}, {0x316E, 1244, 1}, + {0x316F, 1245, 1}, {0x3170, 1246, 1}, {0x3171, 1247, 1}, {0x3172, 1248, 1}, + {0x3173, 1249, 1}, {0x3174, 1250, 1}, {0x3175, 1251, 1}, {0x3176, 1252, 1}, + {0x3177, 1253, 1}, {0x3178, 1254, 1}, {0x3179, 1255, 1}, {0x317A, 1256, 1}, + {0x317B, 1257, 1}, {0x317C, 1258, 1}, {0x317D, 1259, 1}, {0x317E, 1260, 1}, + {0x317F, 1261, 1}, {0x3180, 1262, 1}, {0x3181, 1263, 1}, {0x3182, 1264, 1}, + {0x3183, 1265, 1}, {0x3184, 1266, 1}, {0x3185, 1267, 1}, {0x3186, 1268, 1}, + {0x3187, 1269, 1}, {0x3188, 1270, 1}, {0x3189, 1271, 1}, {0x318A, 1272, 1}, + {0x318B, 1273, 1}, {0x318C, 1274, 1}, {0x318D, 1275, 1}, {0x318E, 1276, 1}, + {0x3192, 1277, 1}, {0x3193, 1278, 1}, {0x3194, 1279, 1}, {0x3195, 1280, 1}, + {0x3196, 1281, 1}, {0x3197, 1282, 1}, {0x3198, 1283, 1}, {0x3199, 1284, 1}, + {0x319A, 1285, 1}, {0x319B, 1286, 1}, {0x319C, 1287, 1}, {0x319D, 1288, 1}, + {0x319E, 1289, 1}, {0x319F, 1290, 1}, {0x3200, 1291, 3}, {0x3201, 1294, 3}, + {0x3202, 1297, 3}, {0x3203, 1300, 3}, {0x3204, 1303, 3}, {0x3205, 1306, 3}, + {0x3206, 1309, 3}, {0x3207, 1312, 3}, {0x3208, 1315, 3}, {0x3209, 1318, 3}, + {0x320A, 1321, 3}, {0x320B, 1324, 3}, {0x320C, 1327, 3}, {0x320D, 1330, 3}, + {0x320E, 1333, 3}, {0x320F, 1336, 3}, {0x3210, 1339, 3}, {0x3211, 1342, 3}, + {0x3212, 1345, 3}, {0x3213, 1348, 3}, {0x3214, 1351, 3}, {0x3215, 1354, 3}, + {0x3216, 1357, 3}, {0x3217, 1360, 3}, {0x3218, 1363, 3}, {0x3219, 1366, 3}, + {0x321A, 1369, 3}, {0x321B, 1372, 3}, {0x321C, 1375, 3}, {0x321D, 1378, 4}, + {0x321E, 1382, 4}, {0x3220, 1386, 3}, {0x3221, 1389, 3}, {0x3222, 1392, 3}, + {0x3223, 1395, 3}, {0x3224, 1398, 3}, {0x3225, 1401, 3}, {0x3226, 1404, 3}, + {0x3227, 1407, 3}, {0x3228, 1410, 3}, {0x3229, 1413, 3}, {0x322A, 1416, 3}, + {0x322B, 1419, 3}, {0x322C, 1422, 3}, {0x322D, 1425, 3}, {0x322E, 1428, 3}, + {0x322F, 1431, 3}, {0x3230, 1434, 3}, {0x3231, 1437, 3}, {0x3232, 1440, 3}, + {0x3233, 1443, 3}, {0x3234, 1446, 3}, {0x3235, 1449, 3}, {0x3236, 1452, 3}, + {0x3237, 1455, 3}, {0x3238, 1458, 3}, {0x3239, 1461, 3}, {0x323A, 1464, 3}, + {0x323B, 1467, 3}, {0x323C, 1470, 3}, {0x323D, 1473, 3}, {0x323E, 1476, 3}, + {0x323F, 1479, 3}, {0x3240, 1482, 3}, {0x3241, 1485, 3}, {0x3242, 1488, 3}, + {0x3243, 1491, 3}, {0x3244, 1494, 1}, {0x3245, 1495, 1}, {0x3246, 1496, 1}, + {0x3247, 1497, 1}, {0x3250, 1498, 3}, {0x3251, 1501, 2}, {0x3252, 1503, 2}, + {0x3253, 1505, 2}, {0x3254, 1507, 2}, {0x3255, 1509, 2}, {0x3256, 1511, 2}, + {0x3257, 1513, 2}, {0x3258, 1515, 2}, {0x3259, 1517, 2}, {0x325A, 1519, 2}, + {0x325B, 1521, 2}, {0x325C, 1523, 2}, {0x325D, 1525, 2}, {0x325E, 1527, 2}, + {0x325F, 1529, 2}, {0x3260, 1531, 1}, {0x3261, 1532, 1}, {0x3262, 1533, 1}, + {0x3263, 1534, 1}, {0x3264, 1535, 1}, {0x3265, 1536, 1}, {0x3266, 1537, 1}, + {0x3267, 1538, 1}, {0x3268, 1539, 1}, {0x3269, 1540, 1}, {0x326A, 1541, 1}, + {0x326B, 1542, 1}, {0x326C, 1543, 1}, {0x326D, 1544, 1}, {0x326E, 1545, 1}, + {0x326F, 1546, 1}, {0x3270, 1547, 1}, {0x3271, 1548, 1}, {0x3272, 1549, 1}, + {0x3273, 1550, 1}, {0x3274, 1551, 1}, {0x3275, 1552, 1}, {0x3276, 1553, 1}, + {0x3277, 1554, 1}, {0x3278, 1555, 1}, {0x3279, 1556, 1}, {0x327A, 1557, 1}, + {0x327B, 1558, 1}, {0x327C, 1559, 2}, {0x327D, 1561, 2}, {0x327E, 1563, 1}, + {0x3280, 1564, 1}, {0x3281, 1565, 1}, {0x3282, 1566, 1}, {0x3283, 1567, 1}, + {0x3284, 1568, 1}, {0x3285, 1569, 1}, {0x3286, 1570, 1}, {0x3287, 1571, 1}, + {0x3288, 1572, 1}, {0x3289, 1573, 1}, {0x328A, 1574, 1}, {0x328B, 1575, 1}, + {0x328C, 1576, 1}, {0x328D, 1577, 1}, {0x328E, 1578, 1}, {0x328F, 1579, 1}, + {0x3290, 1580, 1}, {0x3291, 1581, 1}, {0x3292, 1582, 1}, {0x3293, 1583, 1}, + {0x3294, 1584, 1}, {0x3295, 1585, 1}, {0x3296, 1586, 1}, {0x3297, 1587, 1}, + {0x3298, 1588, 1}, {0x3299, 1589, 1}, {0x329A, 1590, 1}, {0x329B, 1591, 1}, + {0x329C, 1592, 1}, {0x329D, 1593, 1}, {0x329E, 1594, 1}, {0x329F, 1595, 1}, + {0x32A0, 1596, 1}, {0x32A1, 1597, 1}, {0x32A2, 1598, 1}, {0x32A3, 1599, 1}, + {0x32A4, 1600, 1}, {0x32A5, 1601, 1}, {0x32A6, 1602, 1}, {0x32A7, 1603, 1}, + {0x32A8, 1604, 1}, {0x32A9, 1605, 1}, {0x32AA, 1606, 1}, {0x32AB, 1607, 1}, + {0x32AC, 1608, 1}, {0x32AD, 1609, 1}, {0x32AE, 1610, 1}, {0x32AF, 1611, 1}, + {0x32B0, 1612, 1}, {0x32B1, 1613, 2}, {0x32B2, 1615, 2}, {0x32B3, 1617, 2}, + {0x32B4, 1619, 2}, {0x32B5, 1621, 2}, {0x32B6, 1623, 2}, {0x32B7, 1625, 2}, + {0x32B8, 1627, 2}, {0x32B9, 1629, 2}, {0x32BA, 1631, 2}, {0x32BB, 1633, 2}, + {0x32BC, 1635, 2}, {0x32BD, 1637, 2}, {0x32BE, 1639, 2}, {0x32BF, 1641, 2}, + {0x32C0, 1643, 2}, {0x32C1, 1645, 2}, {0x32C2, 1647, 2}, {0x32C3, 1649, 2}, + {0x32C4, 1651, 2}, {0x32C5, 1653, 2}, {0x32C6, 1655, 2}, {0x32C7, 1657, 2}, + {0x32C8, 1659, 2}, {0x32C9, 1661, 3}, {0x32CA, 1664, 3}, {0x32CB, 1667, 3}, + {0x32CC, 1670, 2}, {0x32CD, 1672, 3}, {0x32CE, 1675, 2}, {0x32CF, 1677, 3}, + {0x32D0, 1680, 1}, {0x32D1, 1681, 1}, {0x32D2, 1682, 1}, {0x32D3, 1683, 1}, + {0x32D4, 1684, 1}, {0x32D5, 1685, 1}, {0x32D6, 1686, 1}, {0x32D7, 1687, 1}, + {0x32D8, 1688, 1}, {0x32D9, 1689, 1}, {0x32DA, 1690, 1}, {0x32DB, 1691, 1}, + {0x32DC, 1692, 1}, {0x32DD, 1693, 1}, {0x32DE, 1694, 1}, {0x32DF, 1695, 1}, + {0x32E0, 1696, 1}, {0x32E1, 1697, 1}, {0x32E2, 1698, 1}, {0x32E3, 1699, 1}, + {0x32E4, 1700, 1}, {0x32E5, 1701, 1}, {0x32E6, 1702, 1}, {0x32E7, 1703, 1}, + {0x32E8, 1704, 1}, {0x32E9, 1705, 1}, {0x32EA, 1706, 1}, {0x32EB, 1707, 1}, + {0x32EC, 1708, 1}, {0x32ED, 1709, 1}, {0x32EE, 1710, 1}, {0x32EF, 1711, 1}, + {0x32F0, 1712, 1}, {0x32F1, 1713, 1}, {0x32F2, 1714, 1}, {0x32F3, 1715, 1}, + {0x32F4, 1716, 1}, {0x32F5, 1717, 1}, {0x32F6, 1718, 1}, {0x32F7, 1719, 1}, + {0x32F8, 1720, 1}, {0x32F9, 1721, 1}, {0x32FA, 1722, 1}, {0x32FB, 1723, 1}, + {0x32FC, 1724, 1}, {0x32FD, 1725, 1}, {0x32FE, 1726, 1}, {0x32FF, 1727, 2}, + {0x3300, 1729, 4}, {0x3301, 1733, 4}, {0x3302, 1737, 4}, {0x3303, 1741, 3}, + {0x3304, 1744, 4}, {0x3305, 1748, 3}, {0x3306, 1751, 3}, {0x3307, 1754, 5}, + {0x3308, 1759, 4}, {0x3309, 1763, 3}, {0x330A, 1766, 3}, {0x330B, 1769, 3}, + {0x330C, 1772, 4}, {0x330D, 1776, 4}, {0x330E, 1780, 3}, {0x330F, 1783, 3}, + {0x3310, 1786, 2}, {0x3311, 1788, 3}, {0x3312, 1791, 4}, {0x3313, 1795, 4}, + {0x3314, 1799, 2}, {0x3315, 1801, 5}, {0x3316, 1806, 6}, {0x3317, 1812, 5}, + {0x3318, 1817, 3}, {0x3319, 1820, 5}, {0x331A, 1825, 5}, {0x331B, 1830, 4}, + {0x331C, 1834, 3}, {0x331D, 1837, 3}, {0x331E, 1840, 3}, {0x331F, 1843, 4}, + {0x3320, 1847, 5}, {0x3321, 1852, 4}, {0x3322, 1856, 3}, {0x3323, 1859, 3}, + {0x3324, 1862, 3}, {0x3325, 1865, 2}, {0x3326, 1867, 2}, {0x3327, 1869, 2}, + {0x3328, 1871, 2}, {0x3329, 1873, 3}, {0x332A, 1876, 3}, {0x332B, 1879, 5}, + {0x332C, 1884, 3}, {0x332D, 1887, 4}, {0x332E, 1891, 5}, {0x332F, 1896, 3}, + {0x3330, 1899, 2}, {0x3331, 1901, 2}, {0x3332, 1903, 5}, {0x3333, 1908, 4}, + {0x3334, 1912, 5}, {0x3335, 1917, 3}, {0x3336, 1920, 5}, {0x3337, 1925, 2}, + {0x3338, 1927, 3}, {0x3339, 1930, 3}, {0x333A, 1933, 3}, {0x333B, 1936, 3}, + {0x333C, 1939, 3}, {0x333D, 1942, 4}, {0x333E, 1946, 3}, {0x333F, 1949, 2}, + {0x3340, 1951, 3}, {0x3341, 1954, 3}, {0x3342, 1957, 3}, {0x3343, 1960, 4}, + {0x3344, 1964, 3}, {0x3345, 1967, 3}, {0x3346, 1970, 3}, {0x3347, 1973, 5}, + {0x3348, 1978, 4}, {0x3349, 1982, 2}, {0x334A, 1984, 5}, {0x334B, 1989, 2}, + {0x334C, 1991, 4}, {0x334D, 1995, 4}, {0x334E, 1999, 3}, {0x334F, 2002, 3}, + {0x3350, 2005, 3}, {0x3351, 2008, 4}, {0x3352, 2012, 2}, {0x3353, 2014, 3}, + {0x3354, 2017, 4}, {0x3355, 2021, 2}, {0x3356, 2023, 5}, {0x3357, 2028, 3}, + {0x3358, 2031, 2}, {0x3359, 2033, 2}, {0x335A, 2035, 2}, {0x335B, 2037, 2}, + {0x335C, 2039, 2}, {0x335D, 2041, 2}, {0x335E, 2043, 2}, {0x335F, 2045, 2}, + {0x3360, 2047, 2}, {0x3361, 2049, 2}, {0x3362, 2051, 3}, {0x3363, 2054, 3}, + {0x3364, 2057, 3}, {0x3365, 2060, 3}, {0x3366, 2063, 3}, {0x3367, 2066, 3}, + {0x3368, 2069, 3}, {0x3369, 2072, 3}, {0x336A, 2075, 3}, {0x336B, 2078, 3}, + {0x336C, 2081, 3}, {0x336D, 2084, 3}, {0x336E, 2087, 3}, {0x336F, 2090, 3}, + {0x3370, 2093, 3}, {0x3371, 2096, 3}, {0x3372, 2099, 2}, {0x3373, 2101, 2}, + {0x3374, 2103, 3}, {0x3375, 2106, 2}, {0x3376, 2108, 2}, {0x3377, 2110, 2}, + {0x3378, 2112, 3}, {0x3379, 2115, 3}, {0x337A, 2118, 2}, {0x337B, 2120, 2}, + {0x337C, 2122, 2}, {0x337D, 2124, 2}, {0x337E, 2126, 2}, {0x337F, 2128, 4}, + {0x3380, 2132, 2}, {0x3381, 2134, 2}, {0x3382, 2136, 2}, {0x3383, 2138, 2}, + {0x3384, 2140, 2}, {0x3385, 2142, 2}, {0x3386, 2144, 2}, {0x3387, 2146, 2}, + {0x3388, 2148, 3}, {0x3389, 2151, 4}, {0x338A, 2155, 2}, {0x338B, 2157, 2}, + {0x338C, 2159, 2}, {0x338D, 2161, 2}, {0x338E, 2163, 2}, {0x338F, 2165, 2}, + {0x3390, 2167, 2}, {0x3391, 2169, 3}, {0x3392, 2172, 3}, {0x3393, 2175, 3}, + {0x3394, 2178, 3}, {0x3395, 2181, 2}, {0x3396, 2183, 2}, {0x3397, 2185, 2}, + {0x3398, 2187, 2}, {0x3399, 2189, 2}, {0x339A, 2191, 2}, {0x339B, 2193, 2}, + {0x339C, 2195, 2}, {0x339D, 2197, 2}, {0x339E, 2199, 2}, {0x339F, 2201, 3}, + {0x33A0, 2204, 3}, {0x33A1, 2207, 2}, {0x33A2, 2209, 3}, {0x33A3, 2212, 3}, + {0x33A4, 2215, 3}, {0x33A5, 2218, 2}, {0x33A6, 2220, 3}, {0x33A7, 2223, 3}, + {0x33A8, 2226, 4}, {0x33A9, 2230, 2}, {0x33AA, 2232, 3}, {0x33AB, 2235, 3}, + {0x33AC, 2238, 3}, {0x33AD, 2241, 3}, {0x33AE, 2244, 5}, {0x33AF, 2249, 6}, + {0x33B0, 2255, 2}, {0x33B1, 2257, 2}, {0x33B2, 2259, 2}, {0x33B3, 2261, 2}, + {0x33B4, 2263, 2}, {0x33B5, 2265, 2}, {0x33B6, 2267, 2}, {0x33B7, 2269, 2}, + {0x33B8, 2271, 2}, {0x33B9, 2273, 2}, {0x33BA, 2275, 2}, {0x33BB, 2277, 2}, + {0x33BC, 2279, 2}, {0x33BD, 2281, 2}, {0x33BE, 2283, 2}, {0x33BF, 2285, 2}, + {0x33C0, 2287, 2}, {0x33C1, 2289, 2}, {0x33C2, 2291, 4}, {0x33C3, 2295, 2}, + {0x33C4, 2297, 2}, {0x33C5, 2299, 2}, {0x33C6, 2301, 4}, {0x33C7, 2305, 3}, + {0x33C8, 2308, 2}, {0x33C9, 2310, 2}, {0x33CA, 2312, 2}, {0x33CB, 2314, 2}, + {0x33CC, 2316, 2}, {0x33CD, 2318, 2}, {0x33CE, 2320, 2}, {0x33CF, 2322, 2}, + {0x33D0, 2324, 2}, {0x33D1, 2326, 2}, {0x33D2, 2328, 3}, {0x33D3, 2331, 2}, + {0x33D4, 2333, 2}, {0x33D5, 2335, 3}, {0x33D6, 2338, 3}, {0x33D7, 2341, 2}, + {0x33D8, 2343, 4}, {0x33D9, 2347, 3}, {0x33DA, 2350, 2}, {0x33DB, 2352, 2}, + {0x33DC, 2354, 2}, {0x33DD, 2356, 2}, {0x33DE, 2358, 3}, {0x33DF, 2361, 3}, + {0x33E0, 2364, 2}, {0x33E1, 2366, 2}, {0x33E2, 2368, 2}, {0x33E3, 2370, 2}, + {0x33E4, 2372, 2}, {0x33E5, 2374, 2}, {0x33E6, 2376, 2}, {0x33E7, 2378, 2}, + {0x33E8, 2380, 2}, {0x33E9, 2382, 3}, {0x33EA, 2385, 3}, {0x33EB, 2388, 3}, + {0x33EC, 2391, 3}, {0x33ED, 2394, 3}, {0x33EE, 2397, 3}, {0x33EF, 2400, 3}, + {0x33F0, 2403, 3}, {0x33F1, 2406, 3}, {0x33F2, 2409, 3}, {0x33F3, 2412, 3}, + {0x33F4, 2415, 3}, {0x33F5, 2418, 3}, {0x33F6, 2421, 3}, {0x33F7, 2424, 3}, + {0x33F8, 2427, 3}, {0x33F9, 2430, 3}, {0x33FA, 2433, 3}, {0x33FB, 2436, 3}, + {0x33FC, 2439, 3}, {0x33FD, 2442, 3}, {0x33FE, 2445, 3}, {0x33FF, 2448, 3}, + {0xA69C, 2451, 1}, {0xA69D, 2452, 1}, {0xA770, 2453, 1}, {0xA7F2, 2454, 1}, + {0xA7F3, 2455, 1}, {0xA7F4, 2456, 1}, {0xA7F8, 2457, 1}, {0xA7F9, 2458, 1}, + {0xAB5C, 2459, 1}, {0xAB5D, 2460, 1}, {0xAB5E, 2461, 1}, {0xAB5F, 2462, 1}, + {0xAB69, 2463, 1}, {0xF900, 2464, 1}, {0xF901, 2465, 1}, {0xF902, 2466, 1}, + {0xF903, 2467, 1}, {0xF904, 2468, 1}, {0xF905, 2469, 1}, {0xF906, 2470, 1}, + {0xF907, 2471, 1}, {0xF908, 2472, 1}, {0xF909, 2473, 1}, {0xF90A, 2474, 1}, + {0xF90B, 2475, 1}, {0xF90C, 2476, 1}, {0xF90D, 2477, 1}, {0xF90E, 2478, 1}, + {0xF90F, 2479, 1}, {0xF910, 2480, 1}, {0xF911, 2481, 1}, {0xF912, 2482, 1}, + {0xF913, 2483, 1}, {0xF914, 2484, 1}, {0xF915, 2485, 1}, {0xF916, 2486, 1}, + {0xF917, 2487, 1}, {0xF918, 2488, 1}, {0xF919, 2489, 1}, {0xF91A, 2490, 1}, + {0xF91B, 2491, 1}, {0xF91C, 2492, 1}, {0xF91D, 2493, 1}, {0xF91E, 2494, 1}, + {0xF91F, 2495, 1}, {0xF920, 2496, 1}, {0xF921, 2497, 1}, {0xF922, 2498, 1}, + {0xF923, 2499, 1}, {0xF924, 2500, 1}, {0xF925, 2501, 1}, {0xF926, 2502, 1}, + {0xF927, 2503, 1}, {0xF928, 2504, 1}, {0xF929, 2505, 1}, {0xF92A, 2506, 1}, + {0xF92B, 2507, 1}, {0xF92C, 2508, 1}, {0xF92D, 2509, 1}, {0xF92E, 2510, 1}, + {0xF92F, 2511, 1}, {0xF930, 2512, 1}, {0xF931, 2513, 1}, {0xF932, 2514, 1}, + {0xF933, 2515, 1}, {0xF934, 2516, 1}, {0xF935, 2517, 1}, {0xF936, 2518, 1}, + {0xF937, 2519, 1}, {0xF938, 2520, 1}, {0xF939, 2521, 1}, {0xF93A, 2522, 1}, + {0xF93B, 2523, 1}, {0xF93C, 2524, 1}, {0xF93D, 2525, 1}, {0xF93E, 2526, 1}, + {0xF93F, 2527, 1}, {0xF940, 2528, 1}, {0xF941, 2529, 1}, {0xF942, 2530, 1}, + {0xF943, 2531, 1}, {0xF944, 2532, 1}, {0xF945, 2533, 1}, {0xF946, 2534, 1}, + {0xF947, 2535, 1}, {0xF948, 2536, 1}, {0xF949, 2537, 1}, {0xF94A, 2538, 1}, + {0xF94B, 2539, 1}, {0xF94C, 2540, 1}, {0xF94D, 2541, 1}, {0xF94E, 2542, 1}, + {0xF94F, 2543, 1}, {0xF950, 2544, 1}, {0xF951, 2545, 1}, {0xF952, 2546, 1}, + {0xF953, 2547, 1}, {0xF954, 2548, 1}, {0xF955, 2549, 1}, {0xF956, 2550, 1}, + {0xF957, 2551, 1}, {0xF958, 2552, 1}, {0xF959, 2553, 1}, {0xF95A, 2554, 1}, + {0xF95B, 2555, 1}, {0xF95C, 2556, 1}, {0xF95D, 2557, 1}, {0xF95E, 2558, 1}, + {0xF95F, 2559, 1}, {0xF960, 2560, 1}, {0xF961, 2561, 1}, {0xF962, 2562, 1}, + {0xF963, 2563, 1}, {0xF964, 2564, 1}, {0xF965, 2565, 1}, {0xF966, 2566, 1}, + {0xF967, 2567, 1}, {0xF968, 2568, 1}, {0xF969, 2569, 1}, {0xF96A, 2570, 1}, + {0xF96B, 2571, 1}, {0xF96C, 2572, 1}, {0xF96D, 2573, 1}, {0xF96E, 2574, 1}, + {0xF96F, 2575, 1}, {0xF970, 2576, 1}, {0xF971, 2577, 1}, {0xF972, 2578, 1}, + {0xF973, 2579, 1}, {0xF974, 2580, 1}, {0xF975, 2581, 1}, {0xF976, 2582, 1}, + {0xF977, 2583, 1}, {0xF978, 2584, 1}, {0xF979, 2585, 1}, {0xF97A, 2586, 1}, + {0xF97B, 2587, 1}, {0xF97C, 2588, 1}, {0xF97D, 2589, 1}, {0xF97E, 2590, 1}, + {0xF97F, 2591, 1}, {0xF980, 2592, 1}, {0xF981, 2593, 1}, {0xF982, 2594, 1}, + {0xF983, 2595, 1}, {0xF984, 2596, 1}, {0xF985, 2597, 1}, {0xF986, 2598, 1}, + {0xF987, 2599, 1}, {0xF988, 2600, 1}, {0xF989, 2601, 1}, {0xF98A, 2602, 1}, + {0xF98B, 2603, 1}, {0xF98C, 2604, 1}, {0xF98D, 2605, 1}, {0xF98E, 2606, 1}, + {0xF98F, 2607, 1}, {0xF990, 2608, 1}, {0xF991, 2609, 1}, {0xF992, 2610, 1}, + {0xF993, 2611, 1}, {0xF994, 2612, 1}, {0xF995, 2613, 1}, {0xF996, 2614, 1}, + {0xF997, 2615, 1}, {0xF998, 2616, 1}, {0xF999, 2617, 1}, {0xF99A, 2618, 1}, + {0xF99B, 2619, 1}, {0xF99C, 2620, 1}, {0xF99D, 2621, 1}, {0xF99E, 2622, 1}, + {0xF99F, 2623, 1}, {0xF9A0, 2624, 1}, {0xF9A1, 2625, 1}, {0xF9A2, 2626, 1}, + {0xF9A3, 2627, 1}, {0xF9A4, 2628, 1}, {0xF9A5, 2629, 1}, {0xF9A6, 2630, 1}, + {0xF9A7, 2631, 1}, {0xF9A8, 2632, 1}, {0xF9A9, 2633, 1}, {0xF9AA, 2634, 1}, + {0xF9AB, 2635, 1}, {0xF9AC, 2636, 1}, {0xF9AD, 2637, 1}, {0xF9AE, 2638, 1}, + {0xF9AF, 2639, 1}, {0xF9B0, 2640, 1}, {0xF9B1, 2641, 1}, {0xF9B2, 2642, 1}, + {0xF9B3, 2643, 1}, {0xF9B4, 2644, 1}, {0xF9B5, 2645, 1}, {0xF9B6, 2646, 1}, + {0xF9B7, 2647, 1}, {0xF9B8, 2648, 1}, {0xF9B9, 2649, 1}, {0xF9BA, 2650, 1}, + {0xF9BB, 2651, 1}, {0xF9BC, 2652, 1}, {0xF9BD, 2653, 1}, {0xF9BE, 2654, 1}, + {0xF9BF, 2655, 1}, {0xF9C0, 2656, 1}, {0xF9C1, 2657, 1}, {0xF9C2, 2658, 1}, + {0xF9C3, 2659, 1}, {0xF9C4, 2660, 1}, {0xF9C5, 2661, 1}, {0xF9C6, 2662, 1}, + {0xF9C7, 2663, 1}, {0xF9C8, 2664, 1}, {0xF9C9, 2665, 1}, {0xF9CA, 2666, 1}, + {0xF9CB, 2667, 1}, {0xF9CC, 2668, 1}, {0xF9CD, 2669, 1}, {0xF9CE, 2670, 1}, + {0xF9CF, 2671, 1}, {0xF9D0, 2672, 1}, {0xF9D1, 2673, 1}, {0xF9D2, 2674, 1}, + {0xF9D3, 2675, 1}, {0xF9D4, 2676, 1}, {0xF9D5, 2677, 1}, {0xF9D6, 2678, 1}, + {0xF9D7, 2679, 1}, {0xF9D8, 2680, 1}, {0xF9D9, 2681, 1}, {0xF9DA, 2682, 1}, + {0xF9DB, 2683, 1}, {0xF9DC, 2684, 1}, {0xF9DD, 2685, 1}, {0xF9DE, 2686, 1}, + {0xF9DF, 2687, 1}, {0xF9E0, 2688, 1}, {0xF9E1, 2689, 1}, {0xF9E2, 2690, 1}, + {0xF9E3, 2691, 1}, {0xF9E4, 2692, 1}, {0xF9E5, 2693, 1}, {0xF9E6, 2694, 1}, + {0xF9E7, 2695, 1}, {0xF9E8, 2696, 1}, {0xF9E9, 2697, 1}, {0xF9EA, 2698, 1}, + {0xF9EB, 2699, 1}, {0xF9EC, 2700, 1}, {0xF9ED, 2701, 1}, {0xF9EE, 2702, 1}, + {0xF9EF, 2703, 1}, {0xF9F0, 2704, 1}, {0xF9F1, 2705, 1}, {0xF9F2, 2706, 1}, + {0xF9F3, 2707, 1}, {0xF9F4, 2708, 1}, {0xF9F5, 2709, 1}, {0xF9F6, 2710, 1}, + {0xF9F7, 2711, 1}, {0xF9F8, 2712, 1}, {0xF9F9, 2713, 1}, {0xF9FA, 2714, 1}, + {0xF9FB, 2715, 1}, {0xF9FC, 2716, 1}, {0xF9FD, 2717, 1}, {0xF9FE, 2718, 1}, + {0xF9FF, 2719, 1}, {0xFA00, 2720, 1}, {0xFA01, 2721, 1}, {0xFA02, 2722, 1}, + {0xFA03, 2723, 1}, {0xFA04, 2724, 1}, {0xFA05, 2725, 1}, {0xFA06, 2726, 1}, + {0xFA07, 2727, 1}, {0xFA08, 2728, 1}, {0xFA09, 2729, 1}, {0xFA0A, 2730, 1}, + {0xFA0B, 2731, 1}, {0xFA0C, 2732, 1}, {0xFA0D, 2733, 1}, {0xFA10, 2734, 1}, + {0xFA12, 2735, 1}, {0xFA15, 2736, 1}, {0xFA16, 2737, 1}, {0xFA17, 2738, 1}, + {0xFA18, 2739, 1}, {0xFA19, 2740, 1}, {0xFA1A, 2741, 1}, {0xFA1B, 2742, 1}, + {0xFA1C, 2743, 1}, {0xFA1D, 2744, 1}, {0xFA1E, 2745, 1}, {0xFA20, 2746, 1}, + {0xFA22, 2747, 1}, {0xFA25, 2748, 1}, {0xFA26, 2749, 1}, {0xFA2A, 2750, 1}, + {0xFA2B, 2751, 1}, {0xFA2C, 2752, 1}, {0xFA2D, 2753, 1}, {0xFA2E, 2754, 1}, + {0xFA2F, 2755, 1}, {0xFA30, 2756, 1}, {0xFA31, 2757, 1}, {0xFA32, 2758, 1}, + {0xFA33, 2759, 1}, {0xFA34, 2760, 1}, {0xFA35, 2761, 1}, {0xFA36, 2762, 1}, + {0xFA37, 2763, 1}, {0xFA38, 2764, 1}, {0xFA39, 2765, 1}, {0xFA3A, 2766, 1}, + {0xFA3B, 2767, 1}, {0xFA3C, 2768, 1}, {0xFA3D, 2769, 1}, {0xFA3E, 2770, 1}, + {0xFA3F, 2771, 1}, {0xFA40, 2772, 1}, {0xFA41, 2773, 1}, {0xFA42, 2774, 1}, + {0xFA43, 2775, 1}, {0xFA44, 2776, 1}, {0xFA45, 2777, 1}, {0xFA46, 2778, 1}, + {0xFA47, 2779, 1}, {0xFA48, 2780, 1}, {0xFA49, 2781, 1}, {0xFA4A, 2782, 1}, + {0xFA4B, 2783, 1}, {0xFA4C, 2784, 1}, {0xFA4D, 2785, 1}, {0xFA4E, 2786, 1}, + {0xFA4F, 2787, 1}, {0xFA50, 2788, 1}, {0xFA51, 2789, 1}, {0xFA52, 2790, 1}, + {0xFA53, 2791, 1}, {0xFA54, 2792, 1}, {0xFA55, 2793, 1}, {0xFA56, 2794, 1}, + {0xFA57, 2795, 1}, {0xFA58, 2796, 1}, {0xFA59, 2797, 1}, {0xFA5A, 2798, 1}, + {0xFA5B, 2799, 1}, {0xFA5C, 2800, 1}, {0xFA5D, 2801, 1}, {0xFA5E, 2802, 1}, + {0xFA5F, 2803, 1}, {0xFA60, 2804, 1}, {0xFA61, 2805, 1}, {0xFA62, 2806, 1}, + {0xFA63, 2807, 1}, {0xFA64, 2808, 1}, {0xFA65, 2809, 1}, {0xFA66, 2810, 1}, + {0xFA67, 2811, 1}, {0xFA68, 2812, 1}, {0xFA69, 2813, 1}, {0xFA6A, 2814, 1}, + {0xFA6B, 2815, 1}, {0xFA6C, 2816, 1}, {0xFA6D, 2817, 1}, {0xFA70, 2818, 1}, + {0xFA71, 2819, 1}, {0xFA72, 2820, 1}, {0xFA73, 2821, 1}, {0xFA74, 2822, 1}, + {0xFA75, 2823, 1}, {0xFA76, 2824, 1}, {0xFA77, 2825, 1}, {0xFA78, 2826, 1}, + {0xFA79, 2827, 1}, {0xFA7A, 2828, 1}, {0xFA7B, 2829, 1}, {0xFA7C, 2830, 1}, + {0xFA7D, 2831, 1}, {0xFA7E, 2832, 1}, {0xFA7F, 2833, 1}, {0xFA80, 2834, 1}, + {0xFA81, 2835, 1}, {0xFA82, 2836, 1}, {0xFA83, 2837, 1}, {0xFA84, 2838, 1}, + {0xFA85, 2839, 1}, {0xFA86, 2840, 1}, {0xFA87, 2841, 1}, {0xFA88, 2842, 1}, + {0xFA89, 2843, 1}, {0xFA8A, 2844, 1}, {0xFA8B, 2845, 1}, {0xFA8C, 2846, 1}, + {0xFA8D, 2847, 1}, {0xFA8E, 2848, 1}, {0xFA8F, 2849, 1}, {0xFA90, 2850, 1}, + {0xFA91, 2851, 1}, {0xFA92, 2852, 1}, {0xFA93, 2853, 1}, {0xFA94, 2854, 1}, + {0xFA95, 2855, 1}, {0xFA96, 2856, 1}, {0xFA97, 2857, 1}, {0xFA98, 2858, 1}, + {0xFA99, 2859, 1}, {0xFA9A, 2860, 1}, {0xFA9B, 2861, 1}, {0xFA9C, 2862, 1}, + {0xFA9D, 2863, 1}, {0xFA9E, 2864, 1}, {0xFA9F, 2865, 1}, {0xFAA0, 2866, 1}, + {0xFAA1, 2867, 1}, {0xFAA2, 2868, 1}, {0xFAA3, 2869, 1}, {0xFAA4, 2870, 1}, + {0xFAA5, 2871, 1}, {0xFAA6, 2872, 1}, {0xFAA7, 2873, 1}, {0xFAA8, 2874, 1}, + {0xFAA9, 2875, 1}, {0xFAAA, 2876, 1}, {0xFAAB, 2877, 1}, {0xFAAC, 2878, 1}, + {0xFAAD, 2879, 1}, {0xFAAE, 2880, 1}, {0xFAAF, 2881, 1}, {0xFAB0, 2882, 1}, + {0xFAB1, 2883, 1}, {0xFAB2, 2884, 1}, {0xFAB3, 2885, 1}, {0xFAB4, 2886, 1}, + {0xFAB5, 2887, 1}, {0xFAB6, 2888, 1}, {0xFAB7, 2889, 1}, {0xFAB8, 2890, 1}, + {0xFAB9, 2891, 1}, {0xFABA, 2892, 1}, {0xFABB, 2893, 1}, {0xFABC, 2894, 1}, + {0xFABD, 2895, 1}, {0xFABE, 2896, 1}, {0xFABF, 2897, 1}, {0xFAC0, 2898, 1}, + {0xFAC1, 2899, 1}, {0xFAC2, 2900, 1}, {0xFAC3, 2901, 1}, {0xFAC4, 2902, 1}, + {0xFAC5, 2903, 1}, {0xFAC6, 2904, 1}, {0xFAC7, 2905, 1}, {0xFAC8, 2906, 1}, + {0xFAC9, 2907, 1}, {0xFACA, 2908, 1}, {0xFACB, 2909, 1}, {0xFACC, 2910, 1}, + {0xFACD, 2911, 1}, {0xFACE, 2912, 1}, {0xFACF, 2913, 1}, {0xFAD0, 2914, 1}, + {0xFAD1, 2915, 1}, {0xFAD2, 2916, 1}, {0xFAD3, 2917, 1}, {0xFAD4, 2918, 1}, + {0xFAD5, 2919, 1}, {0xFAD6, 2920, 1}, {0xFAD7, 2921, 1}, {0xFAD8, 2922, 1}, + {0xFAD9, 2923, 1}, {0xFB00, 2924, 2}, {0xFB01, 2926, 2}, {0xFB02, 2928, 2}, + {0xFB03, 2930, 3}, {0xFB04, 2933, 3}, {0xFB05, 2936, 2}, {0xFB06, 2938, 2}, + {0xFB13, 2940, 2}, {0xFB14, 2942, 2}, {0xFB15, 2944, 2}, {0xFB16, 2946, 2}, + {0xFB17, 2948, 2}, {0xFB1D, 2950, 2}, {0xFB1F, 2952, 2}, {0xFB20, 2954, 1}, + {0xFB21, 2955, 1}, {0xFB22, 2956, 1}, {0xFB23, 2957, 1}, {0xFB24, 2958, 1}, + {0xFB25, 2959, 1}, {0xFB26, 2960, 1}, {0xFB27, 2961, 1}, {0xFB28, 2962, 1}, + {0xFB29, 2963, 1}, {0xFB2A, 2964, 2}, {0xFB2B, 2966, 2}, {0xFB2C, 2968, 3}, + {0xFB2D, 2971, 3}, {0xFB2E, 2974, 2}, {0xFB2F, 2976, 2}, {0xFB30, 2978, 2}, + {0xFB31, 2980, 2}, {0xFB32, 2982, 2}, {0xFB33, 2984, 2}, {0xFB34, 2986, 2}, + {0xFB35, 2988, 2}, {0xFB36, 2990, 2}, {0xFB38, 2992, 2}, {0xFB39, 2994, 2}, + {0xFB3A, 2996, 2}, {0xFB3B, 2998, 2}, {0xFB3C, 3000, 2}, {0xFB3E, 3002, 2}, + {0xFB40, 3004, 2}, {0xFB41, 3006, 2}, {0xFB43, 3008, 2}, {0xFB44, 3010, 2}, + {0xFB46, 3012, 2}, {0xFB47, 3014, 2}, {0xFB48, 3016, 2}, {0xFB49, 3018, 2}, + {0xFB4A, 3020, 2}, {0xFB4B, 3022, 2}, {0xFB4C, 3024, 2}, {0xFB4D, 3026, 2}, + {0xFB4E, 3028, 2}, {0xFB4F, 3030, 2}, {0xFB50, 3032, 1}, {0xFB51, 3033, 1}, + {0xFB52, 3034, 1}, {0xFB53, 3035, 1}, {0xFB54, 3036, 1}, {0xFB55, 3037, 1}, + {0xFB56, 3038, 1}, {0xFB57, 3039, 1}, {0xFB58, 3040, 1}, {0xFB59, 3041, 1}, + {0xFB5A, 3042, 1}, {0xFB5B, 3043, 1}, {0xFB5C, 3044, 1}, {0xFB5D, 3045, 1}, + {0xFB5E, 3046, 1}, {0xFB5F, 3047, 1}, {0xFB60, 3048, 1}, {0xFB61, 3049, 1}, + {0xFB62, 3050, 1}, {0xFB63, 3051, 1}, {0xFB64, 3052, 1}, {0xFB65, 3053, 1}, + {0xFB66, 3054, 1}, {0xFB67, 3055, 1}, {0xFB68, 3056, 1}, {0xFB69, 3057, 1}, + {0xFB6A, 3058, 1}, {0xFB6B, 3059, 1}, {0xFB6C, 3060, 1}, {0xFB6D, 3061, 1}, + {0xFB6E, 3062, 1}, {0xFB6F, 3063, 1}, {0xFB70, 3064, 1}, {0xFB71, 3065, 1}, + {0xFB72, 3066, 1}, {0xFB73, 3067, 1}, {0xFB74, 3068, 1}, {0xFB75, 3069, 1}, + {0xFB76, 3070, 1}, {0xFB77, 3071, 1}, {0xFB78, 3072, 1}, {0xFB79, 3073, 1}, + {0xFB7A, 3074, 1}, {0xFB7B, 3075, 1}, {0xFB7C, 3076, 1}, {0xFB7D, 3077, 1}, + {0xFB7E, 3078, 1}, {0xFB7F, 3079, 1}, {0xFB80, 3080, 1}, {0xFB81, 3081, 1}, + {0xFB82, 3082, 1}, {0xFB83, 3083, 1}, {0xFB84, 3084, 1}, {0xFB85, 3085, 1}, + {0xFB86, 3086, 1}, {0xFB87, 3087, 1}, {0xFB88, 3088, 1}, {0xFB89, 3089, 1}, + {0xFB8A, 3090, 1}, {0xFB8B, 3091, 1}, {0xFB8C, 3092, 1}, {0xFB8D, 3093, 1}, + {0xFB8E, 3094, 1}, {0xFB8F, 3095, 1}, {0xFB90, 3096, 1}, {0xFB91, 3097, 1}, + {0xFB92, 3098, 1}, {0xFB93, 3099, 1}, {0xFB94, 3100, 1}, {0xFB95, 3101, 1}, + {0xFB96, 3102, 1}, {0xFB97, 3103, 1}, {0xFB98, 3104, 1}, {0xFB99, 3105, 1}, + {0xFB9A, 3106, 1}, {0xFB9B, 3107, 1}, {0xFB9C, 3108, 1}, {0xFB9D, 3109, 1}, + {0xFB9E, 3110, 1}, {0xFB9F, 3111, 1}, {0xFBA0, 3112, 1}, {0xFBA1, 3113, 1}, + {0xFBA2, 3114, 1}, {0xFBA3, 3115, 1}, {0xFBA4, 3116, 1}, {0xFBA5, 3117, 1}, + {0xFBA6, 3118, 1}, {0xFBA7, 3119, 1}, {0xFBA8, 3120, 1}, {0xFBA9, 3121, 1}, + {0xFBAA, 3122, 1}, {0xFBAB, 3123, 1}, {0xFBAC, 3124, 1}, {0xFBAD, 3125, 1}, + {0xFBAE, 3126, 1}, {0xFBAF, 3127, 1}, {0xFBB0, 3128, 1}, {0xFBB1, 3129, 1}, + {0xFBD3, 3130, 1}, {0xFBD4, 3131, 1}, {0xFBD5, 3132, 1}, {0xFBD6, 3133, 1}, + {0xFBD7, 3134, 1}, {0xFBD8, 3135, 1}, {0xFBD9, 3136, 1}, {0xFBDA, 3137, 1}, + {0xFBDB, 3138, 1}, {0xFBDC, 3139, 1}, {0xFBDD, 3140, 2}, {0xFBDE, 3142, 1}, + {0xFBDF, 3143, 1}, {0xFBE0, 3144, 1}, {0xFBE1, 3145, 1}, {0xFBE2, 3146, 1}, + {0xFBE3, 3147, 1}, {0xFBE4, 3148, 1}, {0xFBE5, 3149, 1}, {0xFBE6, 3150, 1}, + {0xFBE7, 3151, 1}, {0xFBE8, 3152, 1}, {0xFBE9, 3153, 1}, {0xFBEA, 3154, 2}, + {0xFBEB, 3156, 2}, {0xFBEC, 3158, 2}, {0xFBED, 3160, 2}, {0xFBEE, 3162, 2}, + {0xFBEF, 3164, 2}, {0xFBF0, 3166, 2}, {0xFBF1, 3168, 2}, {0xFBF2, 3170, 2}, + {0xFBF3, 3172, 2}, {0xFBF4, 3174, 2}, {0xFBF5, 3176, 2}, {0xFBF6, 3178, 2}, + {0xFBF7, 3180, 2}, {0xFBF8, 3182, 2}, {0xFBF9, 3184, 2}, {0xFBFA, 3186, 2}, + {0xFBFB, 3188, 2}, {0xFBFC, 3190, 1}, {0xFBFD, 3191, 1}, {0xFBFE, 3192, 1}, + {0xFBFF, 3193, 1}, {0xFC00, 3194, 2}, {0xFC01, 3196, 2}, {0xFC02, 3198, 2}, + {0xFC03, 3200, 2}, {0xFC04, 3202, 2}, {0xFC05, 3204, 2}, {0xFC06, 3206, 2}, + {0xFC07, 3208, 2}, {0xFC08, 3210, 2}, {0xFC09, 3212, 2}, {0xFC0A, 3214, 2}, + {0xFC0B, 3216, 2}, {0xFC0C, 3218, 2}, {0xFC0D, 3220, 2}, {0xFC0E, 3222, 2}, + {0xFC0F, 3224, 2}, {0xFC10, 3226, 2}, {0xFC11, 3228, 2}, {0xFC12, 3230, 2}, + {0xFC13, 3232, 2}, {0xFC14, 3234, 2}, {0xFC15, 3236, 2}, {0xFC16, 3238, 2}, + {0xFC17, 3240, 2}, {0xFC18, 3242, 2}, {0xFC19, 3244, 2}, {0xFC1A, 3246, 2}, + {0xFC1B, 3248, 2}, {0xFC1C, 3250, 2}, {0xFC1D, 3252, 2}, {0xFC1E, 3254, 2}, + {0xFC1F, 3256, 2}, {0xFC20, 3258, 2}, {0xFC21, 3260, 2}, {0xFC22, 3262, 2}, + {0xFC23, 3264, 2}, {0xFC24, 3266, 2}, {0xFC25, 3268, 2}, {0xFC26, 3270, 2}, + {0xFC27, 3272, 2}, {0xFC28, 3274, 2}, {0xFC29, 3276, 2}, {0xFC2A, 3278, 2}, + {0xFC2B, 3280, 2}, {0xFC2C, 3282, 2}, {0xFC2D, 3284, 2}, {0xFC2E, 3286, 2}, + {0xFC2F, 3288, 2}, {0xFC30, 3290, 2}, {0xFC31, 3292, 2}, {0xFC32, 3294, 2}, + {0xFC33, 3296, 2}, {0xFC34, 3298, 2}, {0xFC35, 3300, 2}, {0xFC36, 3302, 2}, + {0xFC37, 3304, 2}, {0xFC38, 3306, 2}, {0xFC39, 3308, 2}, {0xFC3A, 3310, 2}, + {0xFC3B, 3312, 2}, {0xFC3C, 3314, 2}, {0xFC3D, 3316, 2}, {0xFC3E, 3318, 2}, + {0xFC3F, 3320, 2}, {0xFC40, 3322, 2}, {0xFC41, 3324, 2}, {0xFC42, 3326, 2}, + {0xFC43, 3328, 2}, {0xFC44, 3330, 2}, {0xFC45, 3332, 2}, {0xFC46, 3334, 2}, + {0xFC47, 3336, 2}, {0xFC48, 3338, 2}, {0xFC49, 3340, 2}, {0xFC4A, 3342, 2}, + {0xFC4B, 3344, 2}, {0xFC4C, 3346, 2}, {0xFC4D, 3348, 2}, {0xFC4E, 3350, 2}, + {0xFC4F, 3352, 2}, {0xFC50, 3354, 2}, {0xFC51, 3356, 2}, {0xFC52, 3358, 2}, + {0xFC53, 3360, 2}, {0xFC54, 3362, 2}, {0xFC55, 3364, 2}, {0xFC56, 3366, 2}, + {0xFC57, 3368, 2}, {0xFC58, 3370, 2}, {0xFC59, 3372, 2}, {0xFC5A, 3374, 2}, + {0xFC5B, 3376, 2}, {0xFC5C, 3378, 2}, {0xFC5D, 3380, 2}, {0xFC5E, 3382, 3}, + {0xFC5F, 3385, 3}, {0xFC60, 3388, 3}, {0xFC61, 3391, 3}, {0xFC62, 3394, 3}, + {0xFC63, 3397, 3}, {0xFC64, 3400, 2}, {0xFC65, 3402, 2}, {0xFC66, 3404, 2}, + {0xFC67, 3406, 2}, {0xFC68, 3408, 2}, {0xFC69, 3410, 2}, {0xFC6A, 3412, 2}, + {0xFC6B, 3414, 2}, {0xFC6C, 3416, 2}, {0xFC6D, 3418, 2}, {0xFC6E, 3420, 2}, + {0xFC6F, 3422, 2}, {0xFC70, 3424, 2}, {0xFC71, 3426, 2}, {0xFC72, 3428, 2}, + {0xFC73, 3430, 2}, {0xFC74, 3432, 2}, {0xFC75, 3434, 2}, {0xFC76, 3436, 2}, + {0xFC77, 3438, 2}, {0xFC78, 3440, 2}, {0xFC79, 3442, 2}, {0xFC7A, 3444, 2}, + {0xFC7B, 3446, 2}, {0xFC7C, 3448, 2}, {0xFC7D, 3450, 2}, {0xFC7E, 3452, 2}, + {0xFC7F, 3454, 2}, {0xFC80, 3456, 2}, {0xFC81, 3458, 2}, {0xFC82, 3460, 2}, + {0xFC83, 3462, 2}, {0xFC84, 3464, 2}, {0xFC85, 3466, 2}, {0xFC86, 3468, 2}, + {0xFC87, 3470, 2}, {0xFC88, 3472, 2}, {0xFC89, 3474, 2}, {0xFC8A, 3476, 2}, + {0xFC8B, 3478, 2}, {0xFC8C, 3480, 2}, {0xFC8D, 3482, 2}, {0xFC8E, 3484, 2}, + {0xFC8F, 3486, 2}, {0xFC90, 3488, 2}, {0xFC91, 3490, 2}, {0xFC92, 3492, 2}, + {0xFC93, 3494, 2}, {0xFC94, 3496, 2}, {0xFC95, 3498, 2}, {0xFC96, 3500, 2}, + {0xFC97, 3502, 2}, {0xFC98, 3504, 2}, {0xFC99, 3506, 2}, {0xFC9A, 3508, 2}, + {0xFC9B, 3510, 2}, {0xFC9C, 3512, 2}, {0xFC9D, 3514, 2}, {0xFC9E, 3516, 2}, + {0xFC9F, 3518, 2}, {0xFCA0, 3520, 2}, {0xFCA1, 3522, 2}, {0xFCA2, 3524, 2}, + {0xFCA3, 3526, 2}, {0xFCA4, 3528, 2}, {0xFCA5, 3530, 2}, {0xFCA6, 3532, 2}, + {0xFCA7, 3534, 2}, {0xFCA8, 3536, 2}, {0xFCA9, 3538, 2}, {0xFCAA, 3540, 2}, + {0xFCAB, 3542, 2}, {0xFCAC, 3544, 2}, {0xFCAD, 3546, 2}, {0xFCAE, 3548, 2}, + {0xFCAF, 3550, 2}, {0xFCB0, 3552, 2}, {0xFCB1, 3554, 2}, {0xFCB2, 3556, 2}, + {0xFCB3, 3558, 2}, {0xFCB4, 3560, 2}, {0xFCB5, 3562, 2}, {0xFCB6, 3564, 2}, + {0xFCB7, 3566, 2}, {0xFCB8, 3568, 2}, {0xFCB9, 3570, 2}, {0xFCBA, 3572, 2}, + {0xFCBB, 3574, 2}, {0xFCBC, 3576, 2}, {0xFCBD, 3578, 2}, {0xFCBE, 3580, 2}, + {0xFCBF, 3582, 2}, {0xFCC0, 3584, 2}, {0xFCC1, 3586, 2}, {0xFCC2, 3588, 2}, + {0xFCC3, 3590, 2}, {0xFCC4, 3592, 2}, {0xFCC5, 3594, 2}, {0xFCC6, 3596, 2}, + {0xFCC7, 3598, 2}, {0xFCC8, 3600, 2}, {0xFCC9, 3602, 2}, {0xFCCA, 3604, 2}, + {0xFCCB, 3606, 2}, {0xFCCC, 3608, 2}, {0xFCCD, 3610, 2}, {0xFCCE, 3612, 2}, + {0xFCCF, 3614, 2}, {0xFCD0, 3616, 2}, {0xFCD1, 3618, 2}, {0xFCD2, 3620, 2}, + {0xFCD3, 3622, 2}, {0xFCD4, 3624, 2}, {0xFCD5, 3626, 2}, {0xFCD6, 3628, 2}, + {0xFCD7, 3630, 2}, {0xFCD8, 3632, 2}, {0xFCD9, 3634, 2}, {0xFCDA, 3636, 2}, + {0xFCDB, 3638, 2}, {0xFCDC, 3640, 2}, {0xFCDD, 3642, 2}, {0xFCDE, 3644, 2}, + {0xFCDF, 3646, 2}, {0xFCE0, 3648, 2}, {0xFCE1, 3650, 2}, {0xFCE2, 3652, 2}, + {0xFCE3, 3654, 2}, {0xFCE4, 3656, 2}, {0xFCE5, 3658, 2}, {0xFCE6, 3660, 2}, + {0xFCE7, 3662, 2}, {0xFCE8, 3664, 2}, {0xFCE9, 3666, 2}, {0xFCEA, 3668, 2}, + {0xFCEB, 3670, 2}, {0xFCEC, 3672, 2}, {0xFCED, 3674, 2}, {0xFCEE, 3676, 2}, + {0xFCEF, 3678, 2}, {0xFCF0, 3680, 2}, {0xFCF1, 3682, 2}, {0xFCF2, 3684, 3}, + {0xFCF3, 3687, 3}, {0xFCF4, 3690, 3}, {0xFCF5, 3693, 2}, {0xFCF6, 3695, 2}, + {0xFCF7, 3697, 2}, {0xFCF8, 3699, 2}, {0xFCF9, 3701, 2}, {0xFCFA, 3703, 2}, + {0xFCFB, 3705, 2}, {0xFCFC, 3707, 2}, {0xFCFD, 3709, 2}, {0xFCFE, 3711, 2}, + {0xFCFF, 3713, 2}, {0xFD00, 3715, 2}, {0xFD01, 3717, 2}, {0xFD02, 3719, 2}, + {0xFD03, 3721, 2}, {0xFD04, 3723, 2}, {0xFD05, 3725, 2}, {0xFD06, 3727, 2}, + {0xFD07, 3729, 2}, {0xFD08, 3731, 2}, {0xFD09, 3733, 2}, {0xFD0A, 3735, 2}, + {0xFD0B, 3737, 2}, {0xFD0C, 3739, 2}, {0xFD0D, 3741, 2}, {0xFD0E, 3743, 2}, + {0xFD0F, 3745, 2}, {0xFD10, 3747, 2}, {0xFD11, 3749, 2}, {0xFD12, 3751, 2}, + {0xFD13, 3753, 2}, {0xFD14, 3755, 2}, {0xFD15, 3757, 2}, {0xFD16, 3759, 2}, + {0xFD17, 3761, 2}, {0xFD18, 3763, 2}, {0xFD19, 3765, 2}, {0xFD1A, 3767, 2}, + {0xFD1B, 3769, 2}, {0xFD1C, 3771, 2}, {0xFD1D, 3773, 2}, {0xFD1E, 3775, 2}, + {0xFD1F, 3777, 2}, {0xFD20, 3779, 2}, {0xFD21, 3781, 2}, {0xFD22, 3783, 2}, + {0xFD23, 3785, 2}, {0xFD24, 3787, 2}, {0xFD25, 3789, 2}, {0xFD26, 3791, 2}, + {0xFD27, 3793, 2}, {0xFD28, 3795, 2}, {0xFD29, 3797, 2}, {0xFD2A, 3799, 2}, + {0xFD2B, 3801, 2}, {0xFD2C, 3803, 2}, {0xFD2D, 3805, 2}, {0xFD2E, 3807, 2}, + {0xFD2F, 3809, 2}, {0xFD30, 3811, 2}, {0xFD31, 3813, 2}, {0xFD32, 3815, 2}, + {0xFD33, 3817, 2}, {0xFD34, 3819, 2}, {0xFD35, 3821, 2}, {0xFD36, 3823, 2}, + {0xFD37, 3825, 2}, {0xFD38, 3827, 2}, {0xFD39, 3829, 2}, {0xFD3A, 3831, 2}, + {0xFD3B, 3833, 2}, {0xFD3C, 3835, 2}, {0xFD3D, 3837, 2}, {0xFD50, 3839, 3}, + {0xFD51, 3842, 3}, {0xFD52, 3845, 3}, {0xFD53, 3848, 3}, {0xFD54, 3851, 3}, + {0xFD55, 3854, 3}, {0xFD56, 3857, 3}, {0xFD57, 3860, 3}, {0xFD58, 3863, 3}, + {0xFD59, 3866, 3}, {0xFD5A, 3869, 3}, {0xFD5B, 3872, 3}, {0xFD5C, 3875, 3}, + {0xFD5D, 3878, 3}, {0xFD5E, 3881, 3}, {0xFD5F, 3884, 3}, {0xFD60, 3887, 3}, + {0xFD61, 3890, 3}, {0xFD62, 3893, 3}, {0xFD63, 3896, 3}, {0xFD64, 3899, 3}, + {0xFD65, 3902, 3}, {0xFD66, 3905, 3}, {0xFD67, 3908, 3}, {0xFD68, 3911, 3}, + {0xFD69, 3914, 3}, {0xFD6A, 3917, 3}, {0xFD6B, 3920, 3}, {0xFD6C, 3923, 3}, + {0xFD6D, 3926, 3}, {0xFD6E, 3929, 3}, {0xFD6F, 3932, 3}, {0xFD70, 3935, 3}, + {0xFD71, 3938, 3}, {0xFD72, 3941, 3}, {0xFD73, 3944, 3}, {0xFD74, 3947, 3}, + {0xFD75, 3950, 3}, {0xFD76, 3953, 3}, {0xFD77, 3956, 3}, {0xFD78, 3959, 3}, + {0xFD79, 3962, 3}, {0xFD7A, 3965, 3}, {0xFD7B, 3968, 3}, {0xFD7C, 3971, 3}, + {0xFD7D, 3974, 3}, {0xFD7E, 3977, 3}, {0xFD7F, 3980, 3}, {0xFD80, 3983, 3}, + {0xFD81, 3986, 3}, {0xFD82, 3989, 3}, {0xFD83, 3992, 3}, {0xFD84, 3995, 3}, + {0xFD85, 3998, 3}, {0xFD86, 4001, 3}, {0xFD87, 4004, 3}, {0xFD88, 4007, 3}, + {0xFD89, 4010, 3}, {0xFD8A, 4013, 3}, {0xFD8B, 4016, 3}, {0xFD8C, 4019, 3}, + {0xFD8D, 4022, 3}, {0xFD8E, 4025, 3}, {0xFD8F, 4028, 3}, {0xFD92, 4031, 3}, + {0xFD93, 4034, 3}, {0xFD94, 4037, 3}, {0xFD95, 4040, 3}, {0xFD96, 4043, 3}, + {0xFD97, 4046, 3}, {0xFD98, 4049, 3}, {0xFD99, 4052, 3}, {0xFD9A, 4055, 3}, + {0xFD9B, 4058, 3}, {0xFD9C, 4061, 3}, {0xFD9D, 4064, 3}, {0xFD9E, 4067, 3}, + {0xFD9F, 4070, 3}, {0xFDA0, 4073, 3}, {0xFDA1, 4076, 3}, {0xFDA2, 4079, 3}, + {0xFDA3, 4082, 3}, {0xFDA4, 4085, 3}, {0xFDA5, 4088, 3}, {0xFDA6, 4091, 3}, + {0xFDA7, 4094, 3}, {0xFDA8, 4097, 3}, {0xFDA9, 4100, 3}, {0xFDAA, 4103, 3}, + {0xFDAB, 4106, 3}, {0xFDAC, 4109, 3}, {0xFDAD, 4112, 3}, {0xFDAE, 4115, 3}, + {0xFDAF, 4118, 3}, {0xFDB0, 4121, 3}, {0xFDB1, 4124, 3}, {0xFDB2, 4127, 3}, + {0xFDB3, 4130, 3}, {0xFDB4, 4133, 3}, {0xFDB5, 4136, 3}, {0xFDB6, 4139, 3}, + {0xFDB7, 4142, 3}, {0xFDB8, 4145, 3}, {0xFDB9, 4148, 3}, {0xFDBA, 4151, 3}, + {0xFDBB, 4154, 3}, {0xFDBC, 4157, 3}, {0xFDBD, 4160, 3}, {0xFDBE, 4163, 3}, + {0xFDBF, 4166, 3}, {0xFDC0, 4169, 3}, {0xFDC1, 4172, 3}, {0xFDC2, 4175, 3}, + {0xFDC3, 4178, 3}, {0xFDC4, 4181, 3}, {0xFDC5, 4184, 3}, {0xFDC6, 4187, 3}, + {0xFDC7, 4190, 3}, {0xFDF0, 4193, 3}, {0xFDF1, 4196, 3}, {0xFDF2, 4199, 4}, + {0xFDF3, 4203, 4}, {0xFDF4, 4207, 4}, {0xFDF5, 4211, 4}, {0xFDF6, 4215, 4}, + {0xFDF7, 4219, 4}, {0xFDF8, 4223, 4}, {0xFDF9, 4227, 3}, {0xFDFA, 4230, 18}, + {0xFDFB, 4248, 8}, {0xFDFC, 4256, 4}, {0xFE10, 4260, 1}, {0xFE11, 4261, 1}, + {0xFE12, 4262, 1}, {0xFE13, 4263, 1}, {0xFE14, 4264, 1}, {0xFE15, 4265, 1}, + {0xFE16, 4266, 1}, {0xFE17, 4267, 1}, {0xFE18, 4268, 1}, {0xFE19, 4269, 3}, + {0xFE30, 4272, 2}, {0xFE31, 4274, 1}, {0xFE32, 4275, 1}, {0xFE33, 4276, 1}, + {0xFE34, 4277, 1}, {0xFE35, 4278, 1}, {0xFE36, 4279, 1}, {0xFE37, 4280, 1}, + {0xFE38, 4281, 1}, {0xFE39, 4282, 1}, {0xFE3A, 4283, 1}, {0xFE3B, 4284, 1}, + {0xFE3C, 4285, 1}, {0xFE3D, 4286, 1}, {0xFE3E, 4287, 1}, {0xFE3F, 4288, 1}, + {0xFE40, 4289, 1}, {0xFE41, 4290, 1}, {0xFE42, 4291, 1}, {0xFE43, 4292, 1}, + {0xFE44, 4293, 1}, {0xFE47, 4294, 1}, {0xFE48, 4295, 1}, {0xFE49, 4296, 2}, + {0xFE4A, 4298, 2}, {0xFE4B, 4300, 2}, {0xFE4C, 4302, 2}, {0xFE4D, 4304, 1}, + {0xFE4E, 4305, 1}, {0xFE4F, 4306, 1}, {0xFE50, 4307, 1}, {0xFE51, 4308, 1}, + {0xFE52, 4309, 1}, {0xFE54, 4310, 1}, {0xFE55, 4311, 1}, {0xFE56, 4312, 1}, + {0xFE57, 4313, 1}, {0xFE58, 4314, 1}, {0xFE59, 4315, 1}, {0xFE5A, 4316, 1}, + {0xFE5B, 4317, 1}, {0xFE5C, 4318, 1}, {0xFE5D, 4319, 1}, {0xFE5E, 4320, 1}, + {0xFE5F, 4321, 1}, {0xFE60, 4322, 1}, {0xFE61, 4323, 1}, {0xFE62, 4324, 1}, + {0xFE63, 4325, 1}, {0xFE64, 4326, 1}, {0xFE65, 4327, 1}, {0xFE66, 4328, 1}, + {0xFE68, 4329, 1}, {0xFE69, 4330, 1}, {0xFE6A, 4331, 1}, {0xFE6B, 4332, 1}, + {0xFE70, 4333, 2}, {0xFE71, 4335, 2}, {0xFE72, 4337, 2}, {0xFE74, 4339, 2}, + {0xFE76, 4341, 2}, {0xFE77, 4343, 2}, {0xFE78, 4345, 2}, {0xFE79, 4347, 2}, + {0xFE7A, 4349, 2}, {0xFE7B, 4351, 2}, {0xFE7C, 4353, 2}, {0xFE7D, 4355, 2}, + {0xFE7E, 4357, 2}, {0xFE7F, 4359, 2}, {0xFE80, 4361, 1}, {0xFE81, 4362, 1}, + {0xFE82, 4363, 1}, {0xFE83, 4364, 1}, {0xFE84, 4365, 1}, {0xFE85, 4366, 1}, + {0xFE86, 4367, 1}, {0xFE87, 4368, 1}, {0xFE88, 4369, 1}, {0xFE89, 4370, 1}, + {0xFE8A, 4371, 1}, {0xFE8B, 4372, 1}, {0xFE8C, 4373, 1}, {0xFE8D, 4374, 1}, + {0xFE8E, 4375, 1}, {0xFE8F, 4376, 1}, {0xFE90, 4377, 1}, {0xFE91, 4378, 1}, + {0xFE92, 4379, 1}, {0xFE93, 4380, 1}, {0xFE94, 4381, 1}, {0xFE95, 4382, 1}, + {0xFE96, 4383, 1}, {0xFE97, 4384, 1}, {0xFE98, 4385, 1}, {0xFE99, 4386, 1}, + {0xFE9A, 4387, 1}, {0xFE9B, 4388, 1}, {0xFE9C, 4389, 1}, {0xFE9D, 4390, 1}, + {0xFE9E, 4391, 1}, {0xFE9F, 4392, 1}, {0xFEA0, 4393, 1}, {0xFEA1, 4394, 1}, + {0xFEA2, 4395, 1}, {0xFEA3, 4396, 1}, {0xFEA4, 4397, 1}, {0xFEA5, 4398, 1}, + {0xFEA6, 4399, 1}, {0xFEA7, 4400, 1}, {0xFEA8, 4401, 1}, {0xFEA9, 4402, 1}, + {0xFEAA, 4403, 1}, {0xFEAB, 4404, 1}, {0xFEAC, 4405, 1}, {0xFEAD, 4406, 1}, + {0xFEAE, 4407, 1}, {0xFEAF, 4408, 1}, {0xFEB0, 4409, 1}, {0xFEB1, 4410, 1}, + {0xFEB2, 4411, 1}, {0xFEB3, 4412, 1}, {0xFEB4, 4413, 1}, {0xFEB5, 4414, 1}, + {0xFEB6, 4415, 1}, {0xFEB7, 4416, 1}, {0xFEB8, 4417, 1}, {0xFEB9, 4418, 1}, + {0xFEBA, 4419, 1}, {0xFEBB, 4420, 1}, {0xFEBC, 4421, 1}, {0xFEBD, 4422, 1}, + {0xFEBE, 4423, 1}, {0xFEBF, 4424, 1}, {0xFEC0, 4425, 1}, {0xFEC1, 4426, 1}, + {0xFEC2, 4427, 1}, {0xFEC3, 4428, 1}, {0xFEC4, 4429, 1}, {0xFEC5, 4430, 1}, + {0xFEC6, 4431, 1}, {0xFEC7, 4432, 1}, {0xFEC8, 4433, 1}, {0xFEC9, 4434, 1}, + {0xFECA, 4435, 1}, {0xFECB, 4436, 1}, {0xFECC, 4437, 1}, {0xFECD, 4438, 1}, + {0xFECE, 4439, 1}, {0xFECF, 4440, 1}, {0xFED0, 4441, 1}, {0xFED1, 4442, 1}, + {0xFED2, 4443, 1}, {0xFED3, 4444, 1}, {0xFED4, 4445, 1}, {0xFED5, 4446, 1}, + {0xFED6, 4447, 1}, {0xFED7, 4448, 1}, {0xFED8, 4449, 1}, {0xFED9, 4450, 1}, + {0xFEDA, 4451, 1}, {0xFEDB, 4452, 1}, {0xFEDC, 4453, 1}, {0xFEDD, 4454, 1}, + {0xFEDE, 4455, 1}, {0xFEDF, 4456, 1}, {0xFEE0, 4457, 1}, {0xFEE1, 4458, 1}, + {0xFEE2, 4459, 1}, {0xFEE3, 4460, 1}, {0xFEE4, 4461, 1}, {0xFEE5, 4462, 1}, + {0xFEE6, 4463, 1}, {0xFEE7, 4464, 1}, {0xFEE8, 4465, 1}, {0xFEE9, 4466, 1}, + {0xFEEA, 4467, 1}, {0xFEEB, 4468, 1}, {0xFEEC, 4469, 1}, {0xFEED, 4470, 1}, + {0xFEEE, 4471, 1}, {0xFEEF, 4472, 1}, {0xFEF0, 4473, 1}, {0xFEF1, 4474, 1}, + {0xFEF2, 4475, 1}, {0xFEF3, 4476, 1}, {0xFEF4, 4477, 1}, {0xFEF5, 4478, 2}, + {0xFEF6, 4480, 2}, {0xFEF7, 4482, 2}, {0xFEF8, 4484, 2}, {0xFEF9, 4486, 2}, + {0xFEFA, 4488, 2}, {0xFEFB, 4490, 2}, {0xFEFC, 4492, 2}, {0xFF01, 4494, 1}, + {0xFF02, 4495, 1}, {0xFF03, 4496, 1}, {0xFF04, 4497, 1}, {0xFF05, 4498, 1}, + {0xFF06, 4499, 1}, {0xFF07, 4500, 1}, {0xFF08, 4501, 1}, {0xFF09, 4502, 1}, + {0xFF0A, 4503, 1}, {0xFF0B, 4504, 1}, {0xFF0C, 4505, 1}, {0xFF0D, 4506, 1}, + {0xFF0E, 4507, 1}, {0xFF0F, 4508, 1}, {0xFF10, 4509, 1}, {0xFF11, 4510, 1}, + {0xFF12, 4511, 1}, {0xFF13, 4512, 1}, {0xFF14, 4513, 1}, {0xFF15, 4514, 1}, + {0xFF16, 4515, 1}, {0xFF17, 4516, 1}, {0xFF18, 4517, 1}, {0xFF19, 4518, 1}, + {0xFF1A, 4519, 1}, {0xFF1B, 4520, 1}, {0xFF1C, 4521, 1}, {0xFF1D, 4522, 1}, + {0xFF1E, 4523, 1}, {0xFF1F, 4524, 1}, {0xFF20, 4525, 1}, {0xFF21, 4526, 1}, + {0xFF22, 4527, 1}, {0xFF23, 4528, 1}, {0xFF24, 4529, 1}, {0xFF25, 4530, 1}, + {0xFF26, 4531, 1}, {0xFF27, 4532, 1}, {0xFF28, 4533, 1}, {0xFF29, 4534, 1}, + {0xFF2A, 4535, 1}, {0xFF2B, 4536, 1}, {0xFF2C, 4537, 1}, {0xFF2D, 4538, 1}, + {0xFF2E, 4539, 1}, {0xFF2F, 4540, 1}, {0xFF30, 4541, 1}, {0xFF31, 4542, 1}, + {0xFF32, 4543, 1}, {0xFF33, 4544, 1}, {0xFF34, 4545, 1}, {0xFF35, 4546, 1}, + {0xFF36, 4547, 1}, {0xFF37, 4548, 1}, {0xFF38, 4549, 1}, {0xFF39, 4550, 1}, + {0xFF3A, 4551, 1}, {0xFF3B, 4552, 1}, {0xFF3C, 4553, 1}, {0xFF3D, 4554, 1}, + {0xFF3E, 4555, 1}, {0xFF3F, 4556, 1}, {0xFF40, 4557, 1}, {0xFF41, 4558, 1}, + {0xFF42, 4559, 1}, {0xFF43, 4560, 1}, {0xFF44, 4561, 1}, {0xFF45, 4562, 1}, + {0xFF46, 4563, 1}, {0xFF47, 4564, 1}, {0xFF48, 4565, 1}, {0xFF49, 4566, 1}, + {0xFF4A, 4567, 1}, {0xFF4B, 4568, 1}, {0xFF4C, 4569, 1}, {0xFF4D, 4570, 1}, + {0xFF4E, 4571, 1}, {0xFF4F, 4572, 1}, {0xFF50, 4573, 1}, {0xFF51, 4574, 1}, + {0xFF52, 4575, 1}, {0xFF53, 4576, 1}, {0xFF54, 4577, 1}, {0xFF55, 4578, 1}, + {0xFF56, 4579, 1}, {0xFF57, 4580, 1}, {0xFF58, 4581, 1}, {0xFF59, 4582, 1}, + {0xFF5A, 4583, 1}, {0xFF5B, 4584, 1}, {0xFF5C, 4585, 1}, {0xFF5D, 4586, 1}, + {0xFF5E, 4587, 1}, {0xFF5F, 4588, 1}, {0xFF60, 4589, 1}, {0xFF61, 4590, 1}, + {0xFF62, 4591, 1}, {0xFF63, 4592, 1}, {0xFF64, 4593, 1}, {0xFF65, 4594, 1}, + {0xFF66, 4595, 1}, {0xFF67, 4596, 1}, {0xFF68, 4597, 1}, {0xFF69, 4598, 1}, + {0xFF6A, 4599, 1}, {0xFF6B, 4600, 1}, {0xFF6C, 4601, 1}, {0xFF6D, 4602, 1}, + {0xFF6E, 4603, 1}, {0xFF6F, 4604, 1}, {0xFF70, 4605, 1}, {0xFF71, 4606, 1}, + {0xFF72, 4607, 1}, {0xFF73, 4608, 1}, {0xFF74, 4609, 1}, {0xFF75, 4610, 1}, + {0xFF76, 4611, 1}, {0xFF77, 4612, 1}, {0xFF78, 4613, 1}, {0xFF79, 4614, 1}, + {0xFF7A, 4615, 1}, {0xFF7B, 4616, 1}, {0xFF7C, 4617, 1}, {0xFF7D, 4618, 1}, + {0xFF7E, 4619, 1}, {0xFF7F, 4620, 1}, {0xFF80, 4621, 1}, {0xFF81, 4622, 1}, + {0xFF82, 4623, 1}, {0xFF83, 4624, 1}, {0xFF84, 4625, 1}, {0xFF85, 4626, 1}, + {0xFF86, 4627, 1}, {0xFF87, 4628, 1}, {0xFF88, 4629, 1}, {0xFF89, 4630, 1}, + {0xFF8A, 4631, 1}, {0xFF8B, 4632, 1}, {0xFF8C, 4633, 1}, {0xFF8D, 4634, 1}, + {0xFF8E, 4635, 1}, {0xFF8F, 4636, 1}, {0xFF90, 4637, 1}, {0xFF91, 4638, 1}, + {0xFF92, 4639, 1}, {0xFF93, 4640, 1}, {0xFF94, 4641, 1}, {0xFF95, 4642, 1}, + {0xFF96, 4643, 1}, {0xFF97, 4644, 1}, {0xFF98, 4645, 1}, {0xFF99, 4646, 1}, + {0xFF9A, 4647, 1}, {0xFF9B, 4648, 1}, {0xFF9C, 4649, 1}, {0xFF9D, 4650, 1}, + {0xFF9E, 4651, 1}, {0xFF9F, 4652, 1}, {0xFFA0, 4653, 1}, {0xFFA1, 4654, 1}, + {0xFFA2, 4655, 1}, {0xFFA3, 4656, 1}, {0xFFA4, 4657, 1}, {0xFFA5, 4658, 1}, + {0xFFA6, 4659, 1}, {0xFFA7, 4660, 1}, {0xFFA8, 4661, 1}, {0xFFA9, 4662, 1}, + {0xFFAA, 4663, 1}, {0xFFAB, 4664, 1}, {0xFFAC, 4665, 1}, {0xFFAD, 4666, 1}, + {0xFFAE, 4667, 1}, {0xFFAF, 4668, 1}, {0xFFB0, 4669, 1}, {0xFFB1, 4670, 1}, + {0xFFB2, 4671, 1}, {0xFFB3, 4672, 1}, {0xFFB4, 4673, 1}, {0xFFB5, 4674, 1}, + {0xFFB6, 4675, 1}, {0xFFB7, 4676, 1}, {0xFFB8, 4677, 1}, {0xFFB9, 4678, 1}, + {0xFFBA, 4679, 1}, {0xFFBB, 4680, 1}, {0xFFBC, 4681, 1}, {0xFFBD, 4682, 1}, + {0xFFBE, 4683, 1}, {0xFFC2, 4684, 1}, {0xFFC3, 4685, 1}, {0xFFC4, 4686, 1}, + {0xFFC5, 4687, 1}, {0xFFC6, 4688, 1}, {0xFFC7, 4689, 1}, {0xFFCA, 4690, 1}, + {0xFFCB, 4691, 1}, {0xFFCC, 4692, 1}, {0xFFCD, 4693, 1}, {0xFFCE, 4694, 1}, + {0xFFCF, 4695, 1}, {0xFFD2, 4696, 1}, {0xFFD3, 4697, 1}, {0xFFD4, 4698, 1}, + {0xFFD5, 4699, 1}, {0xFFD6, 4700, 1}, {0xFFD7, 4701, 1}, {0xFFDA, 4702, 1}, + {0xFFDB, 4703, 1}, {0xFFDC, 4704, 1}, {0xFFE0, 4705, 1}, {0xFFE1, 4706, 1}, + {0xFFE2, 4707, 1}, {0xFFE3, 4708, 2}, {0xFFE4, 4710, 1}, {0xFFE5, 4711, 1}, + {0xFFE6, 4712, 1}, {0xFFE8, 4713, 1}, {0xFFE9, 4714, 1}, {0xFFEA, 4715, 1}, + {0xFFEB, 4716, 1}, {0xFFEC, 4717, 1}, {0xFFED, 4718, 1}, {0xFFEE, 4719, 1}, + {0x10781, 4720, 1}, {0x10782, 4721, 1}, {0x10783, 4722, 1}, {0x10784, 4723, 1}, + {0x10785, 4724, 1}, {0x10787, 4725, 1}, {0x10788, 4726, 1}, {0x10789, 4727, 1}, + {0x1078A, 4728, 1}, {0x1078B, 4729, 1}, {0x1078C, 4730, 1}, {0x1078D, 4731, 1}, + {0x1078E, 4732, 1}, {0x1078F, 4733, 1}, {0x10790, 4734, 1}, {0x10791, 4735, 1}, + {0x10792, 4736, 1}, {0x10793, 4737, 1}, {0x10794, 4738, 1}, {0x10795, 4739, 1}, + {0x10796, 4740, 1}, {0x10797, 4741, 1}, {0x10798, 4742, 1}, {0x10799, 4743, 1}, + {0x1079A, 4744, 1}, {0x1079B, 4745, 1}, {0x1079C, 4746, 1}, {0x1079D, 4747, 1}, + {0x1079E, 4748, 1}, {0x1079F, 4749, 1}, {0x107A0, 4750, 1}, {0x107A1, 4751, 1}, + {0x107A2, 4752, 1}, {0x107A3, 4753, 1}, {0x107A4, 4754, 1}, {0x107A5, 4755, 1}, + {0x107A6, 4756, 1}, {0x107A7, 4757, 1}, {0x107A8, 4758, 1}, {0x107A9, 4759, 1}, + {0x107AA, 4760, 1}, {0x107AB, 4761, 1}, {0x107AC, 4762, 1}, {0x107AD, 4763, 1}, + {0x107AE, 4764, 1}, {0x107AF, 4765, 1}, {0x107B0, 4766, 1}, {0x107B2, 4767, 1}, + {0x107B3, 4768, 1}, {0x107B4, 4769, 1}, {0x107B5, 4770, 1}, {0x107B6, 4771, 1}, + {0x107B7, 4772, 1}, {0x107B8, 4773, 1}, {0x107B9, 4774, 1}, {0x107BA, 4775, 1}, + {0x1CCD6, 4776, 1}, {0x1CCD7, 4777, 1}, {0x1CCD8, 4778, 1}, {0x1CCD9, 4779, 1}, + {0x1CCDA, 4780, 1}, {0x1CCDB, 4781, 1}, {0x1CCDC, 4782, 1}, {0x1CCDD, 4783, 1}, + {0x1CCDE, 4784, 1}, {0x1CCDF, 4785, 1}, {0x1CCE0, 4786, 1}, {0x1CCE1, 4787, 1}, + {0x1CCE2, 4788, 1}, {0x1CCE3, 4789, 1}, {0x1CCE4, 4790, 1}, {0x1CCE5, 4791, 1}, + {0x1CCE6, 4792, 1}, {0x1CCE7, 4793, 1}, {0x1CCE8, 4794, 1}, {0x1CCE9, 4795, 1}, + {0x1CCEA, 4796, 1}, {0x1CCEB, 4797, 1}, {0x1CCEC, 4798, 1}, {0x1CCED, 4799, 1}, + {0x1CCEE, 4800, 1}, {0x1CCEF, 4801, 1}, {0x1CCF0, 4802, 1}, {0x1CCF1, 4803, 1}, + {0x1CCF2, 4804, 1}, {0x1CCF3, 4805, 1}, {0x1CCF4, 4806, 1}, {0x1CCF5, 4807, 1}, + {0x1CCF6, 4808, 1}, {0x1CCF7, 4809, 1}, {0x1CCF8, 4810, 1}, {0x1CCF9, 4811, 1}, + {0x1D15E, 4812, 2}, {0x1D15F, 4814, 2}, {0x1D160, 4816, 3}, {0x1D161, 4819, 3}, + {0x1D162, 4822, 3}, {0x1D163, 4825, 3}, {0x1D164, 4828, 3}, {0x1D1BB, 4831, 2}, + {0x1D1BC, 4833, 2}, {0x1D1BD, 4835, 3}, {0x1D1BE, 4838, 3}, {0x1D1BF, 4841, 3}, + {0x1D1C0, 4844, 3}, {0x1D400, 4847, 1}, {0x1D401, 4848, 1}, {0x1D402, 4849, 1}, + {0x1D403, 4850, 1}, {0x1D404, 4851, 1}, {0x1D405, 4852, 1}, {0x1D406, 4853, 1}, + {0x1D407, 4854, 1}, {0x1D408, 4855, 1}, {0x1D409, 4856, 1}, {0x1D40A, 4857, 1}, + {0x1D40B, 4858, 1}, {0x1D40C, 4859, 1}, {0x1D40D, 4860, 1}, {0x1D40E, 4861, 1}, + {0x1D40F, 4862, 1}, {0x1D410, 4863, 1}, {0x1D411, 4864, 1}, {0x1D412, 4865, 1}, + {0x1D413, 4866, 1}, {0x1D414, 4867, 1}, {0x1D415, 4868, 1}, {0x1D416, 4869, 1}, + {0x1D417, 4870, 1}, {0x1D418, 4871, 1}, {0x1D419, 4872, 1}, {0x1D41A, 4873, 1}, + {0x1D41B, 4874, 1}, {0x1D41C, 4875, 1}, {0x1D41D, 4876, 1}, {0x1D41E, 4877, 1}, + {0x1D41F, 4878, 1}, {0x1D420, 4879, 1}, {0x1D421, 4880, 1}, {0x1D422, 4881, 1}, + {0x1D423, 4882, 1}, {0x1D424, 4883, 1}, {0x1D425, 4884, 1}, {0x1D426, 4885, 1}, + {0x1D427, 4886, 1}, {0x1D428, 4887, 1}, {0x1D429, 4888, 1}, {0x1D42A, 4889, 1}, + {0x1D42B, 4890, 1}, {0x1D42C, 4891, 1}, {0x1D42D, 4892, 1}, {0x1D42E, 4893, 1}, + {0x1D42F, 4894, 1}, {0x1D430, 4895, 1}, {0x1D431, 4896, 1}, {0x1D432, 4897, 1}, + {0x1D433, 4898, 1}, {0x1D434, 4899, 1}, {0x1D435, 4900, 1}, {0x1D436, 4901, 1}, + {0x1D437, 4902, 1}, {0x1D438, 4903, 1}, {0x1D439, 4904, 1}, {0x1D43A, 4905, 1}, + {0x1D43B, 4906, 1}, {0x1D43C, 4907, 1}, {0x1D43D, 4908, 1}, {0x1D43E, 4909, 1}, + {0x1D43F, 4910, 1}, {0x1D440, 4911, 1}, {0x1D441, 4912, 1}, {0x1D442, 4913, 1}, + {0x1D443, 4914, 1}, {0x1D444, 4915, 1}, {0x1D445, 4916, 1}, {0x1D446, 4917, 1}, + {0x1D447, 4918, 1}, {0x1D448, 4919, 1}, {0x1D449, 4920, 1}, {0x1D44A, 4921, 1}, + {0x1D44B, 4922, 1}, {0x1D44C, 4923, 1}, {0x1D44D, 4924, 1}, {0x1D44E, 4925, 1}, + {0x1D44F, 4926, 1}, {0x1D450, 4927, 1}, {0x1D451, 4928, 1}, {0x1D452, 4929, 1}, + {0x1D453, 4930, 1}, {0x1D454, 4931, 1}, {0x1D456, 4932, 1}, {0x1D457, 4933, 1}, + {0x1D458, 4934, 1}, {0x1D459, 4935, 1}, {0x1D45A, 4936, 1}, {0x1D45B, 4937, 1}, + {0x1D45C, 4938, 1}, {0x1D45D, 4939, 1}, {0x1D45E, 4940, 1}, {0x1D45F, 4941, 1}, + {0x1D460, 4942, 1}, {0x1D461, 4943, 1}, {0x1D462, 4944, 1}, {0x1D463, 4945, 1}, + {0x1D464, 4946, 1}, {0x1D465, 4947, 1}, {0x1D466, 4948, 1}, {0x1D467, 4949, 1}, + {0x1D468, 4950, 1}, {0x1D469, 4951, 1}, {0x1D46A, 4952, 1}, {0x1D46B, 4953, 1}, + {0x1D46C, 4954, 1}, {0x1D46D, 4955, 1}, {0x1D46E, 4956, 1}, {0x1D46F, 4957, 1}, + {0x1D470, 4958, 1}, {0x1D471, 4959, 1}, {0x1D472, 4960, 1}, {0x1D473, 4961, 1}, + {0x1D474, 4962, 1}, {0x1D475, 4963, 1}, {0x1D476, 4964, 1}, {0x1D477, 4965, 1}, + {0x1D478, 4966, 1}, {0x1D479, 4967, 1}, {0x1D47A, 4968, 1}, {0x1D47B, 4969, 1}, + {0x1D47C, 4970, 1}, {0x1D47D, 4971, 1}, {0x1D47E, 4972, 1}, {0x1D47F, 4973, 1}, + {0x1D480, 4974, 1}, {0x1D481, 4975, 1}, {0x1D482, 4976, 1}, {0x1D483, 4977, 1}, + {0x1D484, 4978, 1}, {0x1D485, 4979, 1}, {0x1D486, 4980, 1}, {0x1D487, 4981, 1}, + {0x1D488, 4982, 1}, {0x1D489, 4983, 1}, {0x1D48A, 4984, 1}, {0x1D48B, 4985, 1}, + {0x1D48C, 4986, 1}, {0x1D48D, 4987, 1}, {0x1D48E, 4988, 1}, {0x1D48F, 4989, 1}, + {0x1D490, 4990, 1}, {0x1D491, 4991, 1}, {0x1D492, 4992, 1}, {0x1D493, 4993, 1}, + {0x1D494, 4994, 1}, {0x1D495, 4995, 1}, {0x1D496, 4996, 1}, {0x1D497, 4997, 1}, + {0x1D498, 4998, 1}, {0x1D499, 4999, 1}, {0x1D49A, 5000, 1}, {0x1D49B, 5001, 1}, + {0x1D49C, 5002, 1}, {0x1D49E, 5003, 1}, {0x1D49F, 5004, 1}, {0x1D4A2, 5005, 1}, + {0x1D4A5, 5006, 1}, {0x1D4A6, 5007, 1}, {0x1D4A9, 5008, 1}, {0x1D4AA, 5009, 1}, + {0x1D4AB, 5010, 1}, {0x1D4AC, 5011, 1}, {0x1D4AE, 5012, 1}, {0x1D4AF, 5013, 1}, + {0x1D4B0, 5014, 1}, {0x1D4B1, 5015, 1}, {0x1D4B2, 5016, 1}, {0x1D4B3, 5017, 1}, + {0x1D4B4, 5018, 1}, {0x1D4B5, 5019, 1}, {0x1D4B6, 5020, 1}, {0x1D4B7, 5021, 1}, + {0x1D4B8, 5022, 1}, {0x1D4B9, 5023, 1}, {0x1D4BB, 5024, 1}, {0x1D4BD, 5025, 1}, + {0x1D4BE, 5026, 1}, {0x1D4BF, 5027, 1}, {0x1D4C0, 5028, 1}, {0x1D4C1, 5029, 1}, + {0x1D4C2, 5030, 1}, {0x1D4C3, 5031, 1}, {0x1D4C5, 5032, 1}, {0x1D4C6, 5033, 1}, + {0x1D4C7, 5034, 1}, {0x1D4C8, 5035, 1}, {0x1D4C9, 5036, 1}, {0x1D4CA, 5037, 1}, + {0x1D4CB, 5038, 1}, {0x1D4CC, 5039, 1}, {0x1D4CD, 5040, 1}, {0x1D4CE, 5041, 1}, + {0x1D4CF, 5042, 1}, {0x1D4D0, 5043, 1}, {0x1D4D1, 5044, 1}, {0x1D4D2, 5045, 1}, + {0x1D4D3, 5046, 1}, {0x1D4D4, 5047, 1}, {0x1D4D5, 5048, 1}, {0x1D4D6, 5049, 1}, + {0x1D4D7, 5050, 1}, {0x1D4D8, 5051, 1}, {0x1D4D9, 5052, 1}, {0x1D4DA, 5053, 1}, + {0x1D4DB, 5054, 1}, {0x1D4DC, 5055, 1}, {0x1D4DD, 5056, 1}, {0x1D4DE, 5057, 1}, + {0x1D4DF, 5058, 1}, {0x1D4E0, 5059, 1}, {0x1D4E1, 5060, 1}, {0x1D4E2, 5061, 1}, + {0x1D4E3, 5062, 1}, {0x1D4E4, 5063, 1}, {0x1D4E5, 5064, 1}, {0x1D4E6, 5065, 1}, + {0x1D4E7, 5066, 1}, {0x1D4E8, 5067, 1}, {0x1D4E9, 5068, 1}, {0x1D4EA, 5069, 1}, + {0x1D4EB, 5070, 1}, {0x1D4EC, 5071, 1}, {0x1D4ED, 5072, 1}, {0x1D4EE, 5073, 1}, + {0x1D4EF, 5074, 1}, {0x1D4F0, 5075, 1}, {0x1D4F1, 5076, 1}, {0x1D4F2, 5077, 1}, + {0x1D4F3, 5078, 1}, {0x1D4F4, 5079, 1}, {0x1D4F5, 5080, 1}, {0x1D4F6, 5081, 1}, + {0x1D4F7, 5082, 1}, {0x1D4F8, 5083, 1}, {0x1D4F9, 5084, 1}, {0x1D4FA, 5085, 1}, + {0x1D4FB, 5086, 1}, {0x1D4FC, 5087, 1}, {0x1D4FD, 5088, 1}, {0x1D4FE, 5089, 1}, + {0x1D4FF, 5090, 1}, {0x1D500, 5091, 1}, {0x1D501, 5092, 1}, {0x1D502, 5093, 1}, + {0x1D503, 5094, 1}, {0x1D504, 5095, 1}, {0x1D505, 5096, 1}, {0x1D507, 5097, 1}, + {0x1D508, 5098, 1}, {0x1D509, 5099, 1}, {0x1D50A, 5100, 1}, {0x1D50D, 5101, 1}, + {0x1D50E, 5102, 1}, {0x1D50F, 5103, 1}, {0x1D510, 5104, 1}, {0x1D511, 5105, 1}, + {0x1D512, 5106, 1}, {0x1D513, 5107, 1}, {0x1D514, 5108, 1}, {0x1D516, 5109, 1}, + {0x1D517, 5110, 1}, {0x1D518, 5111, 1}, {0x1D519, 5112, 1}, {0x1D51A, 5113, 1}, + {0x1D51B, 5114, 1}, {0x1D51C, 5115, 1}, {0x1D51E, 5116, 1}, {0x1D51F, 5117, 1}, + {0x1D520, 5118, 1}, {0x1D521, 5119, 1}, {0x1D522, 5120, 1}, {0x1D523, 5121, 1}, + {0x1D524, 5122, 1}, {0x1D525, 5123, 1}, {0x1D526, 5124, 1}, {0x1D527, 5125, 1}, + {0x1D528, 5126, 1}, {0x1D529, 5127, 1}, {0x1D52A, 5128, 1}, {0x1D52B, 5129, 1}, + {0x1D52C, 5130, 1}, {0x1D52D, 5131, 1}, {0x1D52E, 5132, 1}, {0x1D52F, 5133, 1}, + {0x1D530, 5134, 1}, {0x1D531, 5135, 1}, {0x1D532, 5136, 1}, {0x1D533, 5137, 1}, + {0x1D534, 5138, 1}, {0x1D535, 5139, 1}, {0x1D536, 5140, 1}, {0x1D537, 5141, 1}, + {0x1D538, 5142, 1}, {0x1D539, 5143, 1}, {0x1D53B, 5144, 1}, {0x1D53C, 5145, 1}, + {0x1D53D, 5146, 1}, {0x1D53E, 5147, 1}, {0x1D540, 5148, 1}, {0x1D541, 5149, 1}, + {0x1D542, 5150, 1}, {0x1D543, 5151, 1}, {0x1D544, 5152, 1}, {0x1D546, 5153, 1}, + {0x1D54A, 5154, 1}, {0x1D54B, 5155, 1}, {0x1D54C, 5156, 1}, {0x1D54D, 5157, 1}, + {0x1D54E, 5158, 1}, {0x1D54F, 5159, 1}, {0x1D550, 5160, 1}, {0x1D552, 5161, 1}, + {0x1D553, 5162, 1}, {0x1D554, 5163, 1}, {0x1D555, 5164, 1}, {0x1D556, 5165, 1}, + {0x1D557, 5166, 1}, {0x1D558, 5167, 1}, {0x1D559, 5168, 1}, {0x1D55A, 5169, 1}, + {0x1D55B, 5170, 1}, {0x1D55C, 5171, 1}, {0x1D55D, 5172, 1}, {0x1D55E, 5173, 1}, + {0x1D55F, 5174, 1}, {0x1D560, 5175, 1}, {0x1D561, 5176, 1}, {0x1D562, 5177, 1}, + {0x1D563, 5178, 1}, {0x1D564, 5179, 1}, {0x1D565, 5180, 1}, {0x1D566, 5181, 1}, + {0x1D567, 5182, 1}, {0x1D568, 5183, 1}, {0x1D569, 5184, 1}, {0x1D56A, 5185, 1}, + {0x1D56B, 5186, 1}, {0x1D56C, 5187, 1}, {0x1D56D, 5188, 1}, {0x1D56E, 5189, 1}, + {0x1D56F, 5190, 1}, {0x1D570, 5191, 1}, {0x1D571, 5192, 1}, {0x1D572, 5193, 1}, + {0x1D573, 5194, 1}, {0x1D574, 5195, 1}, {0x1D575, 5196, 1}, {0x1D576, 5197, 1}, + {0x1D577, 5198, 1}, {0x1D578, 5199, 1}, {0x1D579, 5200, 1}, {0x1D57A, 5201, 1}, + {0x1D57B, 5202, 1}, {0x1D57C, 5203, 1}, {0x1D57D, 5204, 1}, {0x1D57E, 5205, 1}, + {0x1D57F, 5206, 1}, {0x1D580, 5207, 1}, {0x1D581, 5208, 1}, {0x1D582, 5209, 1}, + {0x1D583, 5210, 1}, {0x1D584, 5211, 1}, {0x1D585, 5212, 1}, {0x1D586, 5213, 1}, + {0x1D587, 5214, 1}, {0x1D588, 5215, 1}, {0x1D589, 5216, 1}, {0x1D58A, 5217, 1}, + {0x1D58B, 5218, 1}, {0x1D58C, 5219, 1}, {0x1D58D, 5220, 1}, {0x1D58E, 5221, 1}, + {0x1D58F, 5222, 1}, {0x1D590, 5223, 1}, {0x1D591, 5224, 1}, {0x1D592, 5225, 1}, + {0x1D593, 5226, 1}, {0x1D594, 5227, 1}, {0x1D595, 5228, 1}, {0x1D596, 5229, 1}, + {0x1D597, 5230, 1}, {0x1D598, 5231, 1}, {0x1D599, 5232, 1}, {0x1D59A, 5233, 1}, + {0x1D59B, 5234, 1}, {0x1D59C, 5235, 1}, {0x1D59D, 5236, 1}, {0x1D59E, 5237, 1}, + {0x1D59F, 5238, 1}, {0x1D5A0, 5239, 1}, {0x1D5A1, 5240, 1}, {0x1D5A2, 5241, 1}, + {0x1D5A3, 5242, 1}, {0x1D5A4, 5243, 1}, {0x1D5A5, 5244, 1}, {0x1D5A6, 5245, 1}, + {0x1D5A7, 5246, 1}, {0x1D5A8, 5247, 1}, {0x1D5A9, 5248, 1}, {0x1D5AA, 5249, 1}, + {0x1D5AB, 5250, 1}, {0x1D5AC, 5251, 1}, {0x1D5AD, 5252, 1}, {0x1D5AE, 5253, 1}, + {0x1D5AF, 5254, 1}, {0x1D5B0, 5255, 1}, {0x1D5B1, 5256, 1}, {0x1D5B2, 5257, 1}, + {0x1D5B3, 5258, 1}, {0x1D5B4, 5259, 1}, {0x1D5B5, 5260, 1}, {0x1D5B6, 5261, 1}, + {0x1D5B7, 5262, 1}, {0x1D5B8, 5263, 1}, {0x1D5B9, 5264, 1}, {0x1D5BA, 5265, 1}, + {0x1D5BB, 5266, 1}, {0x1D5BC, 5267, 1}, {0x1D5BD, 5268, 1}, {0x1D5BE, 5269, 1}, + {0x1D5BF, 5270, 1}, {0x1D5C0, 5271, 1}, {0x1D5C1, 5272, 1}, {0x1D5C2, 5273, 1}, + {0x1D5C3, 5274, 1}, {0x1D5C4, 5275, 1}, {0x1D5C5, 5276, 1}, {0x1D5C6, 5277, 1}, + {0x1D5C7, 5278, 1}, {0x1D5C8, 5279, 1}, {0x1D5C9, 5280, 1}, {0x1D5CA, 5281, 1}, + {0x1D5CB, 5282, 1}, {0x1D5CC, 5283, 1}, {0x1D5CD, 5284, 1}, {0x1D5CE, 5285, 1}, + {0x1D5CF, 5286, 1}, {0x1D5D0, 5287, 1}, {0x1D5D1, 5288, 1}, {0x1D5D2, 5289, 1}, + {0x1D5D3, 5290, 1}, {0x1D5D4, 5291, 1}, {0x1D5D5, 5292, 1}, {0x1D5D6, 5293, 1}, + {0x1D5D7, 5294, 1}, {0x1D5D8, 5295, 1}, {0x1D5D9, 5296, 1}, {0x1D5DA, 5297, 1}, + {0x1D5DB, 5298, 1}, {0x1D5DC, 5299, 1}, {0x1D5DD, 5300, 1}, {0x1D5DE, 5301, 1}, + {0x1D5DF, 5302, 1}, {0x1D5E0, 5303, 1}, {0x1D5E1, 5304, 1}, {0x1D5E2, 5305, 1}, + {0x1D5E3, 5306, 1}, {0x1D5E4, 5307, 1}, {0x1D5E5, 5308, 1}, {0x1D5E6, 5309, 1}, + {0x1D5E7, 5310, 1}, {0x1D5E8, 5311, 1}, {0x1D5E9, 5312, 1}, {0x1D5EA, 5313, 1}, + {0x1D5EB, 5314, 1}, {0x1D5EC, 5315, 1}, {0x1D5ED, 5316, 1}, {0x1D5EE, 5317, 1}, + {0x1D5EF, 5318, 1}, {0x1D5F0, 5319, 1}, {0x1D5F1, 5320, 1}, {0x1D5F2, 5321, 1}, + {0x1D5F3, 5322, 1}, {0x1D5F4, 5323, 1}, {0x1D5F5, 5324, 1}, {0x1D5F6, 5325, 1}, + {0x1D5F7, 5326, 1}, {0x1D5F8, 5327, 1}, {0x1D5F9, 5328, 1}, {0x1D5FA, 5329, 1}, + {0x1D5FB, 5330, 1}, {0x1D5FC, 5331, 1}, {0x1D5FD, 5332, 1}, {0x1D5FE, 5333, 1}, + {0x1D5FF, 5334, 1}, {0x1D600, 5335, 1}, {0x1D601, 5336, 1}, {0x1D602, 5337, 1}, + {0x1D603, 5338, 1}, {0x1D604, 5339, 1}, {0x1D605, 5340, 1}, {0x1D606, 5341, 1}, + {0x1D607, 5342, 1}, {0x1D608, 5343, 1}, {0x1D609, 5344, 1}, {0x1D60A, 5345, 1}, + {0x1D60B, 5346, 1}, {0x1D60C, 5347, 1}, {0x1D60D, 5348, 1}, {0x1D60E, 5349, 1}, + {0x1D60F, 5350, 1}, {0x1D610, 5351, 1}, {0x1D611, 5352, 1}, {0x1D612, 5353, 1}, + {0x1D613, 5354, 1}, {0x1D614, 5355, 1}, {0x1D615, 5356, 1}, {0x1D616, 5357, 1}, + {0x1D617, 5358, 1}, {0x1D618, 5359, 1}, {0x1D619, 5360, 1}, {0x1D61A, 5361, 1}, + {0x1D61B, 5362, 1}, {0x1D61C, 5363, 1}, {0x1D61D, 5364, 1}, {0x1D61E, 5365, 1}, + {0x1D61F, 5366, 1}, {0x1D620, 5367, 1}, {0x1D621, 5368, 1}, {0x1D622, 5369, 1}, + {0x1D623, 5370, 1}, {0x1D624, 5371, 1}, {0x1D625, 5372, 1}, {0x1D626, 5373, 1}, + {0x1D627, 5374, 1}, {0x1D628, 5375, 1}, {0x1D629, 5376, 1}, {0x1D62A, 5377, 1}, + {0x1D62B, 5378, 1}, {0x1D62C, 5379, 1}, {0x1D62D, 5380, 1}, {0x1D62E, 5381, 1}, + {0x1D62F, 5382, 1}, {0x1D630, 5383, 1}, {0x1D631, 5384, 1}, {0x1D632, 5385, 1}, + {0x1D633, 5386, 1}, {0x1D634, 5387, 1}, {0x1D635, 5388, 1}, {0x1D636, 5389, 1}, + {0x1D637, 5390, 1}, {0x1D638, 5391, 1}, {0x1D639, 5392, 1}, {0x1D63A, 5393, 1}, + {0x1D63B, 5394, 1}, {0x1D63C, 5395, 1}, {0x1D63D, 5396, 1}, {0x1D63E, 5397, 1}, + {0x1D63F, 5398, 1}, {0x1D640, 5399, 1}, {0x1D641, 5400, 1}, {0x1D642, 5401, 1}, + {0x1D643, 5402, 1}, {0x1D644, 5403, 1}, {0x1D645, 5404, 1}, {0x1D646, 5405, 1}, + {0x1D647, 5406, 1}, {0x1D648, 5407, 1}, {0x1D649, 5408, 1}, {0x1D64A, 5409, 1}, + {0x1D64B, 5410, 1}, {0x1D64C, 5411, 1}, {0x1D64D, 5412, 1}, {0x1D64E, 5413, 1}, + {0x1D64F, 5414, 1}, {0x1D650, 5415, 1}, {0x1D651, 5416, 1}, {0x1D652, 5417, 1}, + {0x1D653, 5418, 1}, {0x1D654, 5419, 1}, {0x1D655, 5420, 1}, {0x1D656, 5421, 1}, + {0x1D657, 5422, 1}, {0x1D658, 5423, 1}, {0x1D659, 5424, 1}, {0x1D65A, 5425, 1}, + {0x1D65B, 5426, 1}, {0x1D65C, 5427, 1}, {0x1D65D, 5428, 1}, {0x1D65E, 5429, 1}, + {0x1D65F, 5430, 1}, {0x1D660, 5431, 1}, {0x1D661, 5432, 1}, {0x1D662, 5433, 1}, + {0x1D663, 5434, 1}, {0x1D664, 5435, 1}, {0x1D665, 5436, 1}, {0x1D666, 5437, 1}, + {0x1D667, 5438, 1}, {0x1D668, 5439, 1}, {0x1D669, 5440, 1}, {0x1D66A, 5441, 1}, + {0x1D66B, 5442, 1}, {0x1D66C, 5443, 1}, {0x1D66D, 5444, 1}, {0x1D66E, 5445, 1}, + {0x1D66F, 5446, 1}, {0x1D670, 5447, 1}, {0x1D671, 5448, 1}, {0x1D672, 5449, 1}, + {0x1D673, 5450, 1}, {0x1D674, 5451, 1}, {0x1D675, 5452, 1}, {0x1D676, 5453, 1}, + {0x1D677, 5454, 1}, {0x1D678, 5455, 1}, {0x1D679, 5456, 1}, {0x1D67A, 5457, 1}, + {0x1D67B, 5458, 1}, {0x1D67C, 5459, 1}, {0x1D67D, 5460, 1}, {0x1D67E, 5461, 1}, + {0x1D67F, 5462, 1}, {0x1D680, 5463, 1}, {0x1D681, 5464, 1}, {0x1D682, 5465, 1}, + {0x1D683, 5466, 1}, {0x1D684, 5467, 1}, {0x1D685, 5468, 1}, {0x1D686, 5469, 1}, + {0x1D687, 5470, 1}, {0x1D688, 5471, 1}, {0x1D689, 5472, 1}, {0x1D68A, 5473, 1}, + {0x1D68B, 5474, 1}, {0x1D68C, 5475, 1}, {0x1D68D, 5476, 1}, {0x1D68E, 5477, 1}, + {0x1D68F, 5478, 1}, {0x1D690, 5479, 1}, {0x1D691, 5480, 1}, {0x1D692, 5481, 1}, + {0x1D693, 5482, 1}, {0x1D694, 5483, 1}, {0x1D695, 5484, 1}, {0x1D696, 5485, 1}, + {0x1D697, 5486, 1}, {0x1D698, 5487, 1}, {0x1D699, 5488, 1}, {0x1D69A, 5489, 1}, + {0x1D69B, 5490, 1}, {0x1D69C, 5491, 1}, {0x1D69D, 5492, 1}, {0x1D69E, 5493, 1}, + {0x1D69F, 5494, 1}, {0x1D6A0, 5495, 1}, {0x1D6A1, 5496, 1}, {0x1D6A2, 5497, 1}, + {0x1D6A3, 5498, 1}, {0x1D6A4, 5499, 1}, {0x1D6A5, 5500, 1}, {0x1D6A8, 5501, 1}, + {0x1D6A9, 5502, 1}, {0x1D6AA, 5503, 1}, {0x1D6AB, 5504, 1}, {0x1D6AC, 5505, 1}, + {0x1D6AD, 5506, 1}, {0x1D6AE, 5507, 1}, {0x1D6AF, 5508, 1}, {0x1D6B0, 5509, 1}, + {0x1D6B1, 5510, 1}, {0x1D6B2, 5511, 1}, {0x1D6B3, 5512, 1}, {0x1D6B4, 5513, 1}, + {0x1D6B5, 5514, 1}, {0x1D6B6, 5515, 1}, {0x1D6B7, 5516, 1}, {0x1D6B8, 5517, 1}, + {0x1D6B9, 5518, 1}, {0x1D6BA, 5519, 1}, {0x1D6BB, 5520, 1}, {0x1D6BC, 5521, 1}, + {0x1D6BD, 5522, 1}, {0x1D6BE, 5523, 1}, {0x1D6BF, 5524, 1}, {0x1D6C0, 5525, 1}, + {0x1D6C1, 5526, 1}, {0x1D6C2, 5527, 1}, {0x1D6C3, 5528, 1}, {0x1D6C4, 5529, 1}, + {0x1D6C5, 5530, 1}, {0x1D6C6, 5531, 1}, {0x1D6C7, 5532, 1}, {0x1D6C8, 5533, 1}, + {0x1D6C9, 5534, 1}, {0x1D6CA, 5535, 1}, {0x1D6CB, 5536, 1}, {0x1D6CC, 5537, 1}, + {0x1D6CD, 5538, 1}, {0x1D6CE, 5539, 1}, {0x1D6CF, 5540, 1}, {0x1D6D0, 5541, 1}, + {0x1D6D1, 5542, 1}, {0x1D6D2, 5543, 1}, {0x1D6D3, 5544, 1}, {0x1D6D4, 5545, 1}, + {0x1D6D5, 5546, 1}, {0x1D6D6, 5547, 1}, {0x1D6D7, 5548, 1}, {0x1D6D8, 5549, 1}, + {0x1D6D9, 5550, 1}, {0x1D6DA, 5551, 1}, {0x1D6DB, 5552, 1}, {0x1D6DC, 5553, 1}, + {0x1D6DD, 5554, 1}, {0x1D6DE, 5555, 1}, {0x1D6DF, 5556, 1}, {0x1D6E0, 5557, 1}, + {0x1D6E1, 5558, 1}, {0x1D6E2, 5559, 1}, {0x1D6E3, 5560, 1}, {0x1D6E4, 5561, 1}, + {0x1D6E5, 5562, 1}, {0x1D6E6, 5563, 1}, {0x1D6E7, 5564, 1}, {0x1D6E8, 5565, 1}, + {0x1D6E9, 5566, 1}, {0x1D6EA, 5567, 1}, {0x1D6EB, 5568, 1}, {0x1D6EC, 5569, 1}, + {0x1D6ED, 5570, 1}, {0x1D6EE, 5571, 1}, {0x1D6EF, 5572, 1}, {0x1D6F0, 5573, 1}, + {0x1D6F1, 5574, 1}, {0x1D6F2, 5575, 1}, {0x1D6F3, 5576, 1}, {0x1D6F4, 5577, 1}, + {0x1D6F5, 5578, 1}, {0x1D6F6, 5579, 1}, {0x1D6F7, 5580, 1}, {0x1D6F8, 5581, 1}, + {0x1D6F9, 5582, 1}, {0x1D6FA, 5583, 1}, {0x1D6FB, 5584, 1}, {0x1D6FC, 5585, 1}, + {0x1D6FD, 5586, 1}, {0x1D6FE, 5587, 1}, {0x1D6FF, 5588, 1}, {0x1D700, 5589, 1}, + {0x1D701, 5590, 1}, {0x1D702, 5591, 1}, {0x1D703, 5592, 1}, {0x1D704, 5593, 1}, + {0x1D705, 5594, 1}, {0x1D706, 5595, 1}, {0x1D707, 5596, 1}, {0x1D708, 5597, 1}, + {0x1D709, 5598, 1}, {0x1D70A, 5599, 1}, {0x1D70B, 5600, 1}, {0x1D70C, 5601, 1}, + {0x1D70D, 5602, 1}, {0x1D70E, 5603, 1}, {0x1D70F, 5604, 1}, {0x1D710, 5605, 1}, + {0x1D711, 5606, 1}, {0x1D712, 5607, 1}, {0x1D713, 5608, 1}, {0x1D714, 5609, 1}, + {0x1D715, 5610, 1}, {0x1D716, 5611, 1}, {0x1D717, 5612, 1}, {0x1D718, 5613, 1}, + {0x1D719, 5614, 1}, {0x1D71A, 5615, 1}, {0x1D71B, 5616, 1}, {0x1D71C, 5617, 1}, + {0x1D71D, 5618, 1}, {0x1D71E, 5619, 1}, {0x1D71F, 5620, 1}, {0x1D720, 5621, 1}, + {0x1D721, 5622, 1}, {0x1D722, 5623, 1}, {0x1D723, 5624, 1}, {0x1D724, 5625, 1}, + {0x1D725, 5626, 1}, {0x1D726, 5627, 1}, {0x1D727, 5628, 1}, {0x1D728, 5629, 1}, + {0x1D729, 5630, 1}, {0x1D72A, 5631, 1}, {0x1D72B, 5632, 1}, {0x1D72C, 5633, 1}, + {0x1D72D, 5634, 1}, {0x1D72E, 5635, 1}, {0x1D72F, 5636, 1}, {0x1D730, 5637, 1}, + {0x1D731, 5638, 1}, {0x1D732, 5639, 1}, {0x1D733, 5640, 1}, {0x1D734, 5641, 1}, + {0x1D735, 5642, 1}, {0x1D736, 5643, 1}, {0x1D737, 5644, 1}, {0x1D738, 5645, 1}, + {0x1D739, 5646, 1}, {0x1D73A, 5647, 1}, {0x1D73B, 5648, 1}, {0x1D73C, 5649, 1}, + {0x1D73D, 5650, 1}, {0x1D73E, 5651, 1}, {0x1D73F, 5652, 1}, {0x1D740, 5653, 1}, + {0x1D741, 5654, 1}, {0x1D742, 5655, 1}, {0x1D743, 5656, 1}, {0x1D744, 5657, 1}, + {0x1D745, 5658, 1}, {0x1D746, 5659, 1}, {0x1D747, 5660, 1}, {0x1D748, 5661, 1}, + {0x1D749, 5662, 1}, {0x1D74A, 5663, 1}, {0x1D74B, 5664, 1}, {0x1D74C, 5665, 1}, + {0x1D74D, 5666, 1}, {0x1D74E, 5667, 1}, {0x1D74F, 5668, 1}, {0x1D750, 5669, 1}, + {0x1D751, 5670, 1}, {0x1D752, 5671, 1}, {0x1D753, 5672, 1}, {0x1D754, 5673, 1}, + {0x1D755, 5674, 1}, {0x1D756, 5675, 1}, {0x1D757, 5676, 1}, {0x1D758, 5677, 1}, + {0x1D759, 5678, 1}, {0x1D75A, 5679, 1}, {0x1D75B, 5680, 1}, {0x1D75C, 5681, 1}, + {0x1D75D, 5682, 1}, {0x1D75E, 5683, 1}, {0x1D75F, 5684, 1}, {0x1D760, 5685, 1}, + {0x1D761, 5686, 1}, {0x1D762, 5687, 1}, {0x1D763, 5688, 1}, {0x1D764, 5689, 1}, + {0x1D765, 5690, 1}, {0x1D766, 5691, 1}, {0x1D767, 5692, 1}, {0x1D768, 5693, 1}, + {0x1D769, 5694, 1}, {0x1D76A, 5695, 1}, {0x1D76B, 5696, 1}, {0x1D76C, 5697, 1}, + {0x1D76D, 5698, 1}, {0x1D76E, 5699, 1}, {0x1D76F, 5700, 1}, {0x1D770, 5701, 1}, + {0x1D771, 5702, 1}, {0x1D772, 5703, 1}, {0x1D773, 5704, 1}, {0x1D774, 5705, 1}, + {0x1D775, 5706, 1}, {0x1D776, 5707, 1}, {0x1D777, 5708, 1}, {0x1D778, 5709, 1}, + {0x1D779, 5710, 1}, {0x1D77A, 5711, 1}, {0x1D77B, 5712, 1}, {0x1D77C, 5713, 1}, + {0x1D77D, 5714, 1}, {0x1D77E, 5715, 1}, {0x1D77F, 5716, 1}, {0x1D780, 5717, 1}, + {0x1D781, 5718, 1}, {0x1D782, 5719, 1}, {0x1D783, 5720, 1}, {0x1D784, 5721, 1}, + {0x1D785, 5722, 1}, {0x1D786, 5723, 1}, {0x1D787, 5724, 1}, {0x1D788, 5725, 1}, + {0x1D789, 5726, 1}, {0x1D78A, 5727, 1}, {0x1D78B, 5728, 1}, {0x1D78C, 5729, 1}, + {0x1D78D, 5730, 1}, {0x1D78E, 5731, 1}, {0x1D78F, 5732, 1}, {0x1D790, 5733, 1}, + {0x1D791, 5734, 1}, {0x1D792, 5735, 1}, {0x1D793, 5736, 1}, {0x1D794, 5737, 1}, + {0x1D795, 5738, 1}, {0x1D796, 5739, 1}, {0x1D797, 5740, 1}, {0x1D798, 5741, 1}, + {0x1D799, 5742, 1}, {0x1D79A, 5743, 1}, {0x1D79B, 5744, 1}, {0x1D79C, 5745, 1}, + {0x1D79D, 5746, 1}, {0x1D79E, 5747, 1}, {0x1D79F, 5748, 1}, {0x1D7A0, 5749, 1}, + {0x1D7A1, 5750, 1}, {0x1D7A2, 5751, 1}, {0x1D7A3, 5752, 1}, {0x1D7A4, 5753, 1}, + {0x1D7A5, 5754, 1}, {0x1D7A6, 5755, 1}, {0x1D7A7, 5756, 1}, {0x1D7A8, 5757, 1}, + {0x1D7A9, 5758, 1}, {0x1D7AA, 5759, 1}, {0x1D7AB, 5760, 1}, {0x1D7AC, 5761, 1}, + {0x1D7AD, 5762, 1}, {0x1D7AE, 5763, 1}, {0x1D7AF, 5764, 1}, {0x1D7B0, 5765, 1}, + {0x1D7B1, 5766, 1}, {0x1D7B2, 5767, 1}, {0x1D7B3, 5768, 1}, {0x1D7B4, 5769, 1}, + {0x1D7B5, 5770, 1}, {0x1D7B6, 5771, 1}, {0x1D7B7, 5772, 1}, {0x1D7B8, 5773, 1}, + {0x1D7B9, 5774, 1}, {0x1D7BA, 5775, 1}, {0x1D7BB, 5776, 1}, {0x1D7BC, 5777, 1}, + {0x1D7BD, 5778, 1}, {0x1D7BE, 5779, 1}, {0x1D7BF, 5780, 1}, {0x1D7C0, 5781, 1}, + {0x1D7C1, 5782, 1}, {0x1D7C2, 5783, 1}, {0x1D7C3, 5784, 1}, {0x1D7C4, 5785, 1}, + {0x1D7C5, 5786, 1}, {0x1D7C6, 5787, 1}, {0x1D7C7, 5788, 1}, {0x1D7C8, 5789, 1}, + {0x1D7C9, 5790, 1}, {0x1D7CA, 5791, 1}, {0x1D7CB, 5792, 1}, {0x1D7CE, 5793, 1}, + {0x1D7CF, 5794, 1}, {0x1D7D0, 5795, 1}, {0x1D7D1, 5796, 1}, {0x1D7D2, 5797, 1}, + {0x1D7D3, 5798, 1}, {0x1D7D4, 5799, 1}, {0x1D7D5, 5800, 1}, {0x1D7D6, 5801, 1}, + {0x1D7D7, 5802, 1}, {0x1D7D8, 5803, 1}, {0x1D7D9, 5804, 1}, {0x1D7DA, 5805, 1}, + {0x1D7DB, 5806, 1}, {0x1D7DC, 5807, 1}, {0x1D7DD, 5808, 1}, {0x1D7DE, 5809, 1}, + {0x1D7DF, 5810, 1}, {0x1D7E0, 5811, 1}, {0x1D7E1, 5812, 1}, {0x1D7E2, 5813, 1}, + {0x1D7E3, 5814, 1}, {0x1D7E4, 5815, 1}, {0x1D7E5, 5816, 1}, {0x1D7E6, 5817, 1}, + {0x1D7E7, 5818, 1}, {0x1D7E8, 5819, 1}, {0x1D7E9, 5820, 1}, {0x1D7EA, 5821, 1}, + {0x1D7EB, 5822, 1}, {0x1D7EC, 5823, 1}, {0x1D7ED, 5824, 1}, {0x1D7EE, 5825, 1}, + {0x1D7EF, 5826, 1}, {0x1D7F0, 5827, 1}, {0x1D7F1, 5828, 1}, {0x1D7F2, 5829, 1}, + {0x1D7F3, 5830, 1}, {0x1D7F4, 5831, 1}, {0x1D7F5, 5832, 1}, {0x1D7F6, 5833, 1}, + {0x1D7F7, 5834, 1}, {0x1D7F8, 5835, 1}, {0x1D7F9, 5836, 1}, {0x1D7FA, 5837, 1}, + {0x1D7FB, 5838, 1}, {0x1D7FC, 5839, 1}, {0x1D7FD, 5840, 1}, {0x1D7FE, 5841, 1}, + {0x1D7FF, 5842, 1}, {0x1E030, 5843, 1}, {0x1E031, 5844, 1}, {0x1E032, 5845, 1}, + {0x1E033, 5846, 1}, {0x1E034, 5847, 1}, {0x1E035, 5848, 1}, {0x1E036, 5849, 1}, + {0x1E037, 5850, 1}, {0x1E038, 5851, 1}, {0x1E039, 5852, 1}, {0x1E03A, 5853, 1}, + {0x1E03B, 5854, 1}, {0x1E03C, 5855, 1}, {0x1E03D, 5856, 1}, {0x1E03E, 5857, 1}, + {0x1E03F, 5858, 1}, {0x1E040, 5859, 1}, {0x1E041, 5860, 1}, {0x1E042, 5861, 1}, + {0x1E043, 5862, 1}, {0x1E044, 5863, 1}, {0x1E045, 5864, 1}, {0x1E046, 5865, 1}, + {0x1E047, 5866, 1}, {0x1E048, 5867, 1}, {0x1E049, 5868, 1}, {0x1E04A, 5869, 1}, + {0x1E04B, 5870, 1}, {0x1E04C, 5871, 1}, {0x1E04D, 5872, 1}, {0x1E04E, 5873, 1}, + {0x1E04F, 5874, 1}, {0x1E050, 5875, 1}, {0x1E051, 5876, 1}, {0x1E052, 5877, 1}, + {0x1E053, 5878, 1}, {0x1E054, 5879, 1}, {0x1E055, 5880, 1}, {0x1E056, 5881, 1}, + {0x1E057, 5882, 1}, {0x1E058, 5883, 1}, {0x1E059, 5884, 1}, {0x1E05A, 5885, 1}, + {0x1E05B, 5886, 1}, {0x1E05C, 5887, 1}, {0x1E05D, 5888, 1}, {0x1E05E, 5889, 1}, + {0x1E05F, 5890, 1}, {0x1E060, 5891, 1}, {0x1E061, 5892, 1}, {0x1E062, 5893, 1}, + {0x1E063, 5894, 1}, {0x1E064, 5895, 1}, {0x1E065, 5896, 1}, {0x1E066, 5897, 1}, + {0x1E067, 5898, 1}, {0x1E068, 5899, 1}, {0x1E069, 5900, 1}, {0x1E06A, 5901, 1}, + {0x1E06B, 5902, 1}, {0x1E06C, 5903, 1}, {0x1E06D, 5904, 1}, {0x1EE00, 5905, 1}, + {0x1EE01, 5906, 1}, {0x1EE02, 5907, 1}, {0x1EE03, 5908, 1}, {0x1EE05, 5909, 1}, + {0x1EE06, 5910, 1}, {0x1EE07, 5911, 1}, {0x1EE08, 5912, 1}, {0x1EE09, 5913, 1}, + {0x1EE0A, 5914, 1}, {0x1EE0B, 5915, 1}, {0x1EE0C, 5916, 1}, {0x1EE0D, 5917, 1}, + {0x1EE0E, 5918, 1}, {0x1EE0F, 5919, 1}, {0x1EE10, 5920, 1}, {0x1EE11, 5921, 1}, + {0x1EE12, 5922, 1}, {0x1EE13, 5923, 1}, {0x1EE14, 5924, 1}, {0x1EE15, 5925, 1}, + {0x1EE16, 5926, 1}, {0x1EE17, 5927, 1}, {0x1EE18, 5928, 1}, {0x1EE19, 5929, 1}, + {0x1EE1A, 5930, 1}, {0x1EE1B, 5931, 1}, {0x1EE1C, 5932, 1}, {0x1EE1D, 5933, 1}, + {0x1EE1E, 5934, 1}, {0x1EE1F, 5935, 1}, {0x1EE21, 5936, 1}, {0x1EE22, 5937, 1}, + {0x1EE24, 5938, 1}, {0x1EE27, 5939, 1}, {0x1EE29, 5940, 1}, {0x1EE2A, 5941, 1}, + {0x1EE2B, 5942, 1}, {0x1EE2C, 5943, 1}, {0x1EE2D, 5944, 1}, {0x1EE2E, 5945, 1}, + {0x1EE2F, 5946, 1}, {0x1EE30, 5947, 1}, {0x1EE31, 5948, 1}, {0x1EE32, 5949, 1}, + {0x1EE34, 5950, 1}, {0x1EE35, 5951, 1}, {0x1EE36, 5952, 1}, {0x1EE37, 5953, 1}, + {0x1EE39, 5954, 1}, {0x1EE3B, 5955, 1}, {0x1EE42, 5956, 1}, {0x1EE47, 5957, 1}, + {0x1EE49, 5958, 1}, {0x1EE4B, 5959, 1}, {0x1EE4D, 5960, 1}, {0x1EE4E, 5961, 1}, + {0x1EE4F, 5962, 1}, {0x1EE51, 5963, 1}, {0x1EE52, 5964, 1}, {0x1EE54, 5965, 1}, + {0x1EE57, 5966, 1}, {0x1EE59, 5967, 1}, {0x1EE5B, 5968, 1}, {0x1EE5D, 5969, 1}, + {0x1EE5F, 5970, 1}, {0x1EE61, 5971, 1}, {0x1EE62, 5972, 1}, {0x1EE64, 5973, 1}, + {0x1EE67, 5974, 1}, {0x1EE68, 5975, 1}, {0x1EE69, 5976, 1}, {0x1EE6A, 5977, 1}, + {0x1EE6C, 5978, 1}, {0x1EE6D, 5979, 1}, {0x1EE6E, 5980, 1}, {0x1EE6F, 5981, 1}, + {0x1EE70, 5982, 1}, {0x1EE71, 5983, 1}, {0x1EE72, 5984, 1}, {0x1EE74, 5985, 1}, + {0x1EE75, 5986, 1}, {0x1EE76, 5987, 1}, {0x1EE77, 5988, 1}, {0x1EE79, 5989, 1}, + {0x1EE7A, 5990, 1}, {0x1EE7B, 5991, 1}, {0x1EE7C, 5992, 1}, {0x1EE7E, 5993, 1}, + {0x1EE80, 5994, 1}, {0x1EE81, 5995, 1}, {0x1EE82, 5996, 1}, {0x1EE83, 5997, 1}, + {0x1EE84, 5998, 1}, {0x1EE85, 5999, 1}, {0x1EE86, 6000, 1}, {0x1EE87, 6001, 1}, + {0x1EE88, 6002, 1}, {0x1EE89, 6003, 1}, {0x1EE8B, 6004, 1}, {0x1EE8C, 6005, 1}, + {0x1EE8D, 6006, 1}, {0x1EE8E, 6007, 1}, {0x1EE8F, 6008, 1}, {0x1EE90, 6009, 1}, + {0x1EE91, 6010, 1}, {0x1EE92, 6011, 1}, {0x1EE93, 6012, 1}, {0x1EE94, 6013, 1}, + {0x1EE95, 6014, 1}, {0x1EE96, 6015, 1}, {0x1EE97, 6016, 1}, {0x1EE98, 6017, 1}, + {0x1EE99, 6018, 1}, {0x1EE9A, 6019, 1}, {0x1EE9B, 6020, 1}, {0x1EEA1, 6021, 1}, + {0x1EEA2, 6022, 1}, {0x1EEA3, 6023, 1}, {0x1EEA5, 6024, 1}, {0x1EEA6, 6025, 1}, + {0x1EEA7, 6026, 1}, {0x1EEA8, 6027, 1}, {0x1EEA9, 6028, 1}, {0x1EEAB, 6029, 1}, + {0x1EEAC, 6030, 1}, {0x1EEAD, 6031, 1}, {0x1EEAE, 6032, 1}, {0x1EEAF, 6033, 1}, + {0x1EEB0, 6034, 1}, {0x1EEB1, 6035, 1}, {0x1EEB2, 6036, 1}, {0x1EEB3, 6037, 1}, + {0x1EEB4, 6038, 1}, {0x1EEB5, 6039, 1}, {0x1EEB6, 6040, 1}, {0x1EEB7, 6041, 1}, + {0x1EEB8, 6042, 1}, {0x1EEB9, 6043, 1}, {0x1EEBA, 6044, 1}, {0x1EEBB, 6045, 1}, + {0x1F100, 6046, 2}, {0x1F101, 6048, 2}, {0x1F102, 6050, 2}, {0x1F103, 6052, 2}, + {0x1F104, 6054, 2}, {0x1F105, 6056, 2}, {0x1F106, 6058, 2}, {0x1F107, 6060, 2}, + {0x1F108, 6062, 2}, {0x1F109, 6064, 2}, {0x1F10A, 6066, 2}, {0x1F110, 6068, 3}, + {0x1F111, 6071, 3}, {0x1F112, 6074, 3}, {0x1F113, 6077, 3}, {0x1F114, 6080, 3}, + {0x1F115, 6083, 3}, {0x1F116, 6086, 3}, {0x1F117, 6089, 3}, {0x1F118, 6092, 3}, + {0x1F119, 6095, 3}, {0x1F11A, 6098, 3}, {0x1F11B, 6101, 3}, {0x1F11C, 6104, 3}, + {0x1F11D, 6107, 3}, {0x1F11E, 6110, 3}, {0x1F11F, 6113, 3}, {0x1F120, 6116, 3}, + {0x1F121, 6119, 3}, {0x1F122, 6122, 3}, {0x1F123, 6125, 3}, {0x1F124, 6128, 3}, + {0x1F125, 6131, 3}, {0x1F126, 6134, 3}, {0x1F127, 6137, 3}, {0x1F128, 6140, 3}, + {0x1F129, 6143, 3}, {0x1F12A, 6146, 3}, {0x1F12B, 6149, 1}, {0x1F12C, 6150, 1}, + {0x1F12D, 6151, 2}, {0x1F12E, 6153, 2}, {0x1F130, 6155, 1}, {0x1F131, 6156, 1}, + {0x1F132, 6157, 1}, {0x1F133, 6158, 1}, {0x1F134, 6159, 1}, {0x1F135, 6160, 1}, + {0x1F136, 6161, 1}, {0x1F137, 6162, 1}, {0x1F138, 6163, 1}, {0x1F139, 6164, 1}, + {0x1F13A, 6165, 1}, {0x1F13B, 6166, 1}, {0x1F13C, 6167, 1}, {0x1F13D, 6168, 1}, + {0x1F13E, 6169, 1}, {0x1F13F, 6170, 1}, {0x1F140, 6171, 1}, {0x1F141, 6172, 1}, + {0x1F142, 6173, 1}, {0x1F143, 6174, 1}, {0x1F144, 6175, 1}, {0x1F145, 6176, 1}, + {0x1F146, 6177, 1}, {0x1F147, 6178, 1}, {0x1F148, 6179, 1}, {0x1F149, 6180, 1}, + {0x1F14A, 6181, 2}, {0x1F14B, 6183, 2}, {0x1F14C, 6185, 2}, {0x1F14D, 6187, 2}, + {0x1F14E, 6189, 3}, {0x1F14F, 6192, 2}, {0x1F16A, 6194, 2}, {0x1F16B, 6196, 2}, + {0x1F16C, 6198, 2}, {0x1F190, 6200, 2}, {0x1F200, 6202, 2}, {0x1F201, 6204, 2}, + {0x1F202, 6206, 1}, {0x1F210, 6207, 1}, {0x1F211, 6208, 1}, {0x1F212, 6209, 1}, + {0x1F213, 6210, 1}, {0x1F214, 6211, 1}, {0x1F215, 6212, 1}, {0x1F216, 6213, 1}, + {0x1F217, 6214, 1}, {0x1F218, 6215, 1}, {0x1F219, 6216, 1}, {0x1F21A, 6217, 1}, + {0x1F21B, 6218, 1}, {0x1F21C, 6219, 1}, {0x1F21D, 6220, 1}, {0x1F21E, 6221, 1}, + {0x1F21F, 6222, 1}, {0x1F220, 6223, 1}, {0x1F221, 6224, 1}, {0x1F222, 6225, 1}, + {0x1F223, 6226, 1}, {0x1F224, 6227, 1}, {0x1F225, 6228, 1}, {0x1F226, 6229, 1}, + {0x1F227, 6230, 1}, {0x1F228, 6231, 1}, {0x1F229, 6232, 1}, {0x1F22A, 6233, 1}, + {0x1F22B, 6234, 1}, {0x1F22C, 6235, 1}, {0x1F22D, 6236, 1}, {0x1F22E, 6237, 1}, + {0x1F22F, 6238, 1}, {0x1F230, 6239, 1}, {0x1F231, 6240, 1}, {0x1F232, 6241, 1}, + {0x1F233, 6242, 1}, {0x1F234, 6243, 1}, {0x1F235, 6244, 1}, {0x1F236, 6245, 1}, + {0x1F237, 6246, 1}, {0x1F238, 6247, 1}, {0x1F239, 6248, 1}, {0x1F23A, 6249, 1}, + {0x1F23B, 6250, 1}, {0x1F240, 6251, 3}, {0x1F241, 6254, 3}, {0x1F242, 6257, 3}, + {0x1F243, 6260, 3}, {0x1F244, 6263, 3}, {0x1F245, 6266, 3}, {0x1F246, 6269, 3}, + {0x1F247, 6272, 3}, {0x1F248, 6275, 3}, {0x1F250, 6278, 1}, {0x1F251, 6279, 1}, + {0x1FBF0, 6280, 1}, {0x1FBF1, 6281, 1}, {0x1FBF2, 6282, 1}, {0x1FBF3, 6283, 1}, + {0x1FBF4, 6284, 1}, {0x1FBF5, 6285, 1}, {0x1FBF6, 6286, 1}, {0x1FBF7, 6287, 1}, + {0x1FBF8, 6288, 1}, {0x1FBF9, 6289, 1}, {0x2F800, 6290, 1}, {0x2F801, 6291, 1}, + {0x2F802, 6292, 1}, {0x2F803, 6293, 1}, {0x2F804, 6294, 1}, {0x2F805, 6295, 1}, + {0x2F806, 6296, 1}, {0x2F807, 6297, 1}, {0x2F808, 6298, 1}, {0x2F809, 6299, 1}, + {0x2F80A, 6300, 1}, {0x2F80B, 6301, 1}, {0x2F80C, 6302, 1}, {0x2F80D, 6303, 1}, + {0x2F80E, 6304, 1}, {0x2F80F, 6305, 1}, {0x2F810, 6306, 1}, {0x2F811, 6307, 1}, + {0x2F812, 6308, 1}, {0x2F813, 6309, 1}, {0x2F814, 6310, 1}, {0x2F815, 6311, 1}, + {0x2F816, 6312, 1}, {0x2F817, 6313, 1}, {0x2F818, 6314, 1}, {0x2F819, 6315, 1}, + {0x2F81A, 6316, 1}, {0x2F81B, 6317, 1}, {0x2F81C, 6318, 1}, {0x2F81D, 6319, 1}, + {0x2F81E, 6320, 1}, {0x2F81F, 6321, 1}, {0x2F820, 6322, 1}, {0x2F821, 6323, 1}, + {0x2F822, 6324, 1}, {0x2F823, 6325, 1}, {0x2F824, 6326, 1}, {0x2F825, 6327, 1}, + {0x2F826, 6328, 1}, {0x2F827, 6329, 1}, {0x2F828, 6330, 1}, {0x2F829, 6331, 1}, + {0x2F82A, 6332, 1}, {0x2F82B, 6333, 1}, {0x2F82C, 6334, 1}, {0x2F82D, 6335, 1}, + {0x2F82E, 6336, 1}, {0x2F82F, 6337, 1}, {0x2F830, 6338, 1}, {0x2F831, 6339, 1}, + {0x2F832, 6340, 1}, {0x2F833, 6341, 1}, {0x2F834, 6342, 1}, {0x2F835, 6343, 1}, + {0x2F836, 6344, 1}, {0x2F837, 6345, 1}, {0x2F838, 6346, 1}, {0x2F839, 6347, 1}, + {0x2F83A, 6348, 1}, {0x2F83B, 6349, 1}, {0x2F83C, 6350, 1}, {0x2F83D, 6351, 1}, + {0x2F83E, 6352, 1}, {0x2F83F, 6353, 1}, {0x2F840, 6354, 1}, {0x2F841, 6355, 1}, + {0x2F842, 6356, 1}, {0x2F843, 6357, 1}, {0x2F844, 6358, 1}, {0x2F845, 6359, 1}, + {0x2F846, 6360, 1}, {0x2F847, 6361, 1}, {0x2F848, 6362, 1}, {0x2F849, 6363, 1}, + {0x2F84A, 6364, 1}, {0x2F84B, 6365, 1}, {0x2F84C, 6366, 1}, {0x2F84D, 6367, 1}, + {0x2F84E, 6368, 1}, {0x2F84F, 6369, 1}, {0x2F850, 6370, 1}, {0x2F851, 6371, 1}, + {0x2F852, 6372, 1}, {0x2F853, 6373, 1}, {0x2F854, 6374, 1}, {0x2F855, 6375, 1}, + {0x2F856, 6376, 1}, {0x2F857, 6377, 1}, {0x2F858, 6378, 1}, {0x2F859, 6379, 1}, + {0x2F85A, 6380, 1}, {0x2F85B, 6381, 1}, {0x2F85C, 6382, 1}, {0x2F85D, 6383, 1}, + {0x2F85E, 6384, 1}, {0x2F85F, 6385, 1}, {0x2F860, 6386, 1}, {0x2F861, 6387, 1}, + {0x2F862, 6388, 1}, {0x2F863, 6389, 1}, {0x2F864, 6390, 1}, {0x2F865, 6391, 1}, + {0x2F866, 6392, 1}, {0x2F867, 6393, 1}, {0x2F868, 6394, 1}, {0x2F869, 6395, 1}, + {0x2F86A, 6396, 1}, {0x2F86B, 6397, 1}, {0x2F86C, 6398, 1}, {0x2F86D, 6399, 1}, + {0x2F86E, 6400, 1}, {0x2F86F, 6401, 1}, {0x2F870, 6402, 1}, {0x2F871, 6403, 1}, + {0x2F872, 6404, 1}, {0x2F873, 6405, 1}, {0x2F874, 6406, 1}, {0x2F875, 6407, 1}, + {0x2F876, 6408, 1}, {0x2F877, 6409, 1}, {0x2F878, 6410, 1}, {0x2F879, 6411, 1}, + {0x2F87A, 6412, 1}, {0x2F87B, 6413, 1}, {0x2F87C, 6414, 1}, {0x2F87D, 6415, 1}, + {0x2F87E, 6416, 1}, {0x2F87F, 6417, 1}, {0x2F880, 6418, 1}, {0x2F881, 6419, 1}, + {0x2F882, 6420, 1}, {0x2F883, 6421, 1}, {0x2F884, 6422, 1}, {0x2F885, 6423, 1}, + {0x2F886, 6424, 1}, {0x2F887, 6425, 1}, {0x2F888, 6426, 1}, {0x2F889, 6427, 1}, + {0x2F88A, 6428, 1}, {0x2F88B, 6429, 1}, {0x2F88C, 6430, 1}, {0x2F88D, 6431, 1}, + {0x2F88E, 6432, 1}, {0x2F88F, 6433, 1}, {0x2F890, 6434, 1}, {0x2F891, 6435, 1}, + {0x2F892, 6436, 1}, {0x2F893, 6437, 1}, {0x2F894, 6438, 1}, {0x2F895, 6439, 1}, + {0x2F896, 6440, 1}, {0x2F897, 6441, 1}, {0x2F898, 6442, 1}, {0x2F899, 6443, 1}, + {0x2F89A, 6444, 1}, {0x2F89B, 6445, 1}, {0x2F89C, 6446, 1}, {0x2F89D, 6447, 1}, + {0x2F89E, 6448, 1}, {0x2F89F, 6449, 1}, {0x2F8A0, 6450, 1}, {0x2F8A1, 6451, 1}, + {0x2F8A2, 6452, 1}, {0x2F8A3, 6453, 1}, {0x2F8A4, 6454, 1}, {0x2F8A5, 6455, 1}, + {0x2F8A6, 6456, 1}, {0x2F8A7, 6457, 1}, {0x2F8A8, 6458, 1}, {0x2F8A9, 6459, 1}, + {0x2F8AA, 6460, 1}, {0x2F8AB, 6461, 1}, {0x2F8AC, 6462, 1}, {0x2F8AD, 6463, 1}, + {0x2F8AE, 6464, 1}, {0x2F8AF, 6465, 1}, {0x2F8B0, 6466, 1}, {0x2F8B1, 6467, 1}, + {0x2F8B2, 6468, 1}, {0x2F8B3, 6469, 1}, {0x2F8B4, 6470, 1}, {0x2F8B5, 6471, 1}, + {0x2F8B6, 6472, 1}, {0x2F8B7, 6473, 1}, {0x2F8B8, 6474, 1}, {0x2F8B9, 6475, 1}, + {0x2F8BA, 6476, 1}, {0x2F8BB, 6477, 1}, {0x2F8BC, 6478, 1}, {0x2F8BD, 6479, 1}, + {0x2F8BE, 6480, 1}, {0x2F8BF, 6481, 1}, {0x2F8C0, 6482, 1}, {0x2F8C1, 6483, 1}, + {0x2F8C2, 6484, 1}, {0x2F8C3, 6485, 1}, {0x2F8C4, 6486, 1}, {0x2F8C5, 6487, 1}, + {0x2F8C6, 6488, 1}, {0x2F8C7, 6489, 1}, {0x2F8C8, 6490, 1}, {0x2F8C9, 6491, 1}, + {0x2F8CA, 6492, 1}, {0x2F8CB, 6493, 1}, {0x2F8CC, 6494, 1}, {0x2F8CD, 6495, 1}, + {0x2F8CE, 6496, 1}, {0x2F8CF, 6497, 1}, {0x2F8D0, 6498, 1}, {0x2F8D1, 6499, 1}, + {0x2F8D2, 6500, 1}, {0x2F8D3, 6501, 1}, {0x2F8D4, 6502, 1}, {0x2F8D5, 6503, 1}, + {0x2F8D6, 6504, 1}, {0x2F8D7, 6505, 1}, {0x2F8D8, 6506, 1}, {0x2F8D9, 6507, 1}, + {0x2F8DA, 6508, 1}, {0x2F8DB, 6509, 1}, {0x2F8DC, 6510, 1}, {0x2F8DD, 6511, 1}, + {0x2F8DE, 6512, 1}, {0x2F8DF, 6513, 1}, {0x2F8E0, 6514, 1}, {0x2F8E1, 6515, 1}, + {0x2F8E2, 6516, 1}, {0x2F8E3, 6517, 1}, {0x2F8E4, 6518, 1}, {0x2F8E5, 6519, 1}, + {0x2F8E6, 6520, 1}, {0x2F8E7, 6521, 1}, {0x2F8E8, 6522, 1}, {0x2F8E9, 6523, 1}, + {0x2F8EA, 6524, 1}, {0x2F8EB, 6525, 1}, {0x2F8EC, 6526, 1}, {0x2F8ED, 6527, 1}, + {0x2F8EE, 6528, 1}, {0x2F8EF, 6529, 1}, {0x2F8F0, 6530, 1}, {0x2F8F1, 6531, 1}, + {0x2F8F2, 6532, 1}, {0x2F8F3, 6533, 1}, {0x2F8F4, 6534, 1}, {0x2F8F5, 6535, 1}, + {0x2F8F6, 6536, 1}, {0x2F8F7, 6537, 1}, {0x2F8F8, 6538, 1}, {0x2F8F9, 6539, 1}, + {0x2F8FA, 6540, 1}, {0x2F8FB, 6541, 1}, {0x2F8FC, 6542, 1}, {0x2F8FD, 6543, 1}, + {0x2F8FE, 6544, 1}, {0x2F8FF, 6545, 1}, {0x2F900, 6546, 1}, {0x2F901, 6547, 1}, + {0x2F902, 6548, 1}, {0x2F903, 6549, 1}, {0x2F904, 6550, 1}, {0x2F905, 6551, 1}, + {0x2F906, 6552, 1}, {0x2F907, 6553, 1}, {0x2F908, 6554, 1}, {0x2F909, 6555, 1}, + {0x2F90A, 6556, 1}, {0x2F90B, 6557, 1}, {0x2F90C, 6558, 1}, {0x2F90D, 6559, 1}, + {0x2F90E, 6560, 1}, {0x2F90F, 6561, 1}, {0x2F910, 6562, 1}, {0x2F911, 6563, 1}, + {0x2F912, 6564, 1}, {0x2F913, 6565, 1}, {0x2F914, 6566, 1}, {0x2F915, 6567, 1}, + {0x2F916, 6568, 1}, {0x2F917, 6569, 1}, {0x2F918, 6570, 1}, {0x2F919, 6571, 1}, + {0x2F91A, 6572, 1}, {0x2F91B, 6573, 1}, {0x2F91C, 6574, 1}, {0x2F91D, 6575, 1}, + {0x2F91E, 6576, 1}, {0x2F91F, 6577, 1}, {0x2F920, 6578, 1}, {0x2F921, 6579, 1}, + {0x2F922, 6580, 1}, {0x2F923, 6581, 1}, {0x2F924, 6582, 1}, {0x2F925, 6583, 1}, + {0x2F926, 6584, 1}, {0x2F927, 6585, 1}, {0x2F928, 6586, 1}, {0x2F929, 6587, 1}, + {0x2F92A, 6588, 1}, {0x2F92B, 6589, 1}, {0x2F92C, 6590, 1}, {0x2F92D, 6591, 1}, + {0x2F92E, 6592, 1}, {0x2F92F, 6593, 1}, {0x2F930, 6594, 1}, {0x2F931, 6595, 1}, + {0x2F932, 6596, 1}, {0x2F933, 6597, 1}, {0x2F934, 6598, 1}, {0x2F935, 6599, 1}, + {0x2F936, 6600, 1}, {0x2F937, 6601, 1}, {0x2F938, 6602, 1}, {0x2F939, 6603, 1}, + {0x2F93A, 6604, 1}, {0x2F93B, 6605, 1}, {0x2F93C, 6606, 1}, {0x2F93D, 6607, 1}, + {0x2F93E, 6608, 1}, {0x2F93F, 6609, 1}, {0x2F940, 6610, 1}, {0x2F941, 6611, 1}, + {0x2F942, 6612, 1}, {0x2F943, 6613, 1}, {0x2F944, 6614, 1}, {0x2F945, 6615, 1}, + {0x2F946, 6616, 1}, {0x2F947, 6617, 1}, {0x2F948, 6618, 1}, {0x2F949, 6619, 1}, + {0x2F94A, 6620, 1}, {0x2F94B, 6621, 1}, {0x2F94C, 6622, 1}, {0x2F94D, 6623, 1}, + {0x2F94E, 6624, 1}, {0x2F94F, 6625, 1}, {0x2F950, 6626, 1}, {0x2F951, 6627, 1}, + {0x2F952, 6628, 1}, {0x2F953, 6629, 1}, {0x2F954, 6630, 1}, {0x2F955, 6631, 1}, + {0x2F956, 6632, 1}, {0x2F957, 6633, 1}, {0x2F958, 6634, 1}, {0x2F959, 6635, 1}, + {0x2F95A, 6636, 1}, {0x2F95B, 6637, 1}, {0x2F95C, 6638, 1}, {0x2F95D, 6639, 1}, + {0x2F95E, 6640, 1}, {0x2F95F, 6641, 1}, {0x2F960, 6642, 1}, {0x2F961, 6643, 1}, + {0x2F962, 6644, 1}, {0x2F963, 6645, 1}, {0x2F964, 6646, 1}, {0x2F965, 6647, 1}, + {0x2F966, 6648, 1}, {0x2F967, 6649, 1}, {0x2F968, 6650, 1}, {0x2F969, 6651, 1}, + {0x2F96A, 6652, 1}, {0x2F96B, 6653, 1}, {0x2F96C, 6654, 1}, {0x2F96D, 6655, 1}, + {0x2F96E, 6656, 1}, {0x2F96F, 6657, 1}, {0x2F970, 6658, 1}, {0x2F971, 6659, 1}, + {0x2F972, 6660, 1}, {0x2F973, 6661, 1}, {0x2F974, 6662, 1}, {0x2F975, 6663, 1}, + {0x2F976, 6664, 1}, {0x2F977, 6665, 1}, {0x2F978, 6666, 1}, {0x2F979, 6667, 1}, + {0x2F97A, 6668, 1}, {0x2F97B, 6669, 1}, {0x2F97C, 6670, 1}, {0x2F97D, 6671, 1}, + {0x2F97E, 6672, 1}, {0x2F97F, 6673, 1}, {0x2F980, 6674, 1}, {0x2F981, 6675, 1}, + {0x2F982, 6676, 1}, {0x2F983, 6677, 1}, {0x2F984, 6678, 1}, {0x2F985, 6679, 1}, + {0x2F986, 6680, 1}, {0x2F987, 6681, 1}, {0x2F988, 6682, 1}, {0x2F989, 6683, 1}, + {0x2F98A, 6684, 1}, {0x2F98B, 6685, 1}, {0x2F98C, 6686, 1}, {0x2F98D, 6687, 1}, + {0x2F98E, 6688, 1}, {0x2F98F, 6689, 1}, {0x2F990, 6690, 1}, {0x2F991, 6691, 1}, + {0x2F992, 6692, 1}, {0x2F993, 6693, 1}, {0x2F994, 6694, 1}, {0x2F995, 6695, 1}, + {0x2F996, 6696, 1}, {0x2F997, 6697, 1}, {0x2F998, 6698, 1}, {0x2F999, 6699, 1}, + {0x2F99A, 6700, 1}, {0x2F99B, 6701, 1}, {0x2F99C, 6702, 1}, {0x2F99D, 6703, 1}, + {0x2F99E, 6704, 1}, {0x2F99F, 6705, 1}, {0x2F9A0, 6706, 1}, {0x2F9A1, 6707, 1}, + {0x2F9A2, 6708, 1}, {0x2F9A3, 6709, 1}, {0x2F9A4, 6710, 1}, {0x2F9A5, 6711, 1}, + {0x2F9A6, 6712, 1}, {0x2F9A7, 6713, 1}, {0x2F9A8, 6714, 1}, {0x2F9A9, 6715, 1}, + {0x2F9AA, 6716, 1}, {0x2F9AB, 6717, 1}, {0x2F9AC, 6718, 1}, {0x2F9AD, 6719, 1}, + {0x2F9AE, 6720, 1}, {0x2F9AF, 6721, 1}, {0x2F9B0, 6722, 1}, {0x2F9B1, 6723, 1}, + {0x2F9B2, 6724, 1}, {0x2F9B3, 6725, 1}, {0x2F9B4, 6726, 1}, {0x2F9B5, 6727, 1}, + {0x2F9B6, 6728, 1}, {0x2F9B7, 6729, 1}, {0x2F9B8, 6730, 1}, {0x2F9B9, 6731, 1}, + {0x2F9BA, 6732, 1}, {0x2F9BB, 6733, 1}, {0x2F9BC, 6734, 1}, {0x2F9BD, 6735, 1}, + {0x2F9BE, 6736, 1}, {0x2F9BF, 6737, 1}, {0x2F9C0, 6738, 1}, {0x2F9C1, 6739, 1}, + {0x2F9C2, 6740, 1}, {0x2F9C3, 6741, 1}, {0x2F9C4, 6742, 1}, {0x2F9C5, 6743, 1}, + {0x2F9C6, 6744, 1}, {0x2F9C7, 6745, 1}, {0x2F9C8, 6746, 1}, {0x2F9C9, 6747, 1}, + {0x2F9CA, 6748, 1}, {0x2F9CB, 6749, 1}, {0x2F9CC, 6750, 1}, {0x2F9CD, 6751, 1}, + {0x2F9CE, 6752, 1}, {0x2F9CF, 6753, 1}, {0x2F9D0, 6754, 1}, {0x2F9D1, 6755, 1}, + {0x2F9D2, 6756, 1}, {0x2F9D3, 6757, 1}, {0x2F9D4, 6758, 1}, {0x2F9D5, 6759, 1}, + {0x2F9D6, 6760, 1}, {0x2F9D7, 6761, 1}, {0x2F9D8, 6762, 1}, {0x2F9D9, 6763, 1}, + {0x2F9DA, 6764, 1}, {0x2F9DB, 6765, 1}, {0x2F9DC, 6766, 1}, {0x2F9DD, 6767, 1}, + {0x2F9DE, 6768, 1}, {0x2F9DF, 6769, 1}, {0x2F9E0, 6770, 1}, {0x2F9E1, 6771, 1}, + {0x2F9E2, 6772, 1}, {0x2F9E3, 6773, 1}, {0x2F9E4, 6774, 1}, {0x2F9E5, 6775, 1}, + {0x2F9E6, 6776, 1}, {0x2F9E7, 6777, 1}, {0x2F9E8, 6778, 1}, {0x2F9E9, 6779, 1}, + {0x2F9EA, 6780, 1}, {0x2F9EB, 6781, 1}, {0x2F9EC, 6782, 1}, {0x2F9ED, 6783, 1}, + {0x2F9EE, 6784, 1}, {0x2F9EF, 6785, 1}, {0x2F9F0, 6786, 1}, {0x2F9F1, 6787, 1}, + {0x2F9F2, 6788, 1}, {0x2F9F3, 6789, 1}, {0x2F9F4, 6790, 1}, {0x2F9F5, 6791, 1}, + {0x2F9F6, 6792, 1}, {0x2F9F7, 6793, 1}, {0x2F9F8, 6794, 1}, {0x2F9F9, 6795, 1}, + {0x2F9FA, 6796, 1}, {0x2F9FB, 6797, 1}, {0x2F9FC, 6798, 1}, {0x2F9FD, 6799, 1}, + {0x2F9FE, 6800, 1}, {0x2F9FF, 6801, 1}, {0x2FA00, 6802, 1}, {0x2FA01, 6803, 1}, + {0x2FA02, 6804, 1}, {0x2FA03, 6805, 1}, {0x2FA04, 6806, 1}, {0x2FA05, 6807, 1}, + {0x2FA06, 6808, 1}, {0x2FA07, 6809, 1}, {0x2FA08, 6810, 1}, {0x2FA09, 6811, 1}, + {0x2FA0A, 6812, 1}, {0x2FA0B, 6813, 1}, {0x2FA0C, 6814, 1}, {0x2FA0D, 6815, 1}, + {0x2FA0E, 6816, 1}, {0x2FA0F, 6817, 1}, {0x2FA10, 6818, 1}, {0x2FA11, 6819, 1}, + {0x2FA12, 6820, 1}, {0x2FA13, 6821, 1}, {0x2FA14, 6822, 1}, {0x2FA15, 6823, 1}, + {0x2FA16, 6824, 1}, {0x2FA17, 6825, 1}, {0x2FA18, 6826, 1}, {0x2FA19, 6827, 1}, + {0x2FA1A, 6828, 1}, {0x2FA1B, 6829, 1}, {0x2FA1C, 6830, 1}, {0x2FA1D, 6831, 1}, +}; +const int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0])); + +const uint32_t PDF_NFKC_SEQ[] = { + 0x20, 0x20, 0x308, 0x61, 0x20, 0x304, 0x32, 0x33, 0x20, 0x301, + 0x3BC, 0x20, 0x327, 0x31, 0x6F, 0x31, 0x2044, 0x34, 0x31, 0x2044, + 0x32, 0x33, 0x2044, 0x34, 0x49, 0x4A, 0x69, 0x6A, 0x4C, 0xB7, + 0x6C, 0xB7, 0x2BC, 0x6E, 0x73, 0x44, 0x17D, 0x44, 0x17E, 0x64, + 0x17E, 0x4C, 0x4A, 0x4C, 0x6A, 0x6C, 0x6A, 0x4E, 0x4A, 0x4E, + 0x6A, 0x6E, 0x6A, 0x44, 0x5A, 0x44, 0x7A, 0x64, 0x7A, 0x68, + 0x266, 0x6A, 0x72, 0x279, 0x27B, 0x281, 0x77, 0x79, 0x20, 0x306, + 0x20, 0x307, 0x20, 0x30A, 0x20, 0x328, 0x20, 0x303, 0x20, 0x30B, + 0x263, 0x6C, 0x73, 0x78, 0x295, 0x300, 0x301, 0x313, 0x308, 0x301, + 0x2B9, 0x20, 0x345, 0x3B, 0x20, 0x301, 0x20, 0x308, 0x301, 0xB7, + 0x3B2, 0x3B8, 0x3A5, 0x38E, 0x3AB, 0x3C6, 0x3C0, 0x3BA, 0x3C1, 0x3C2, + 0x398, 0x3B5, 0x3A3, 0x565, 0x582, 0x627, 0x674, 0x648, 0x674, 0x6C7, + 0x674, 0x64A, 0x674, 0x915, 0x93C, 0x916, 0x93C, 0x917, 0x93C, 0x91C, + 0x93C, 0x921, 0x93C, 0x922, 0x93C, 0x92B, 0x93C, 0x92F, 0x93C, 0x9A1, + 0x9BC, 0x9A2, 0x9BC, 0x9AF, 0x9BC, 0xA32, 0xA3C, 0xA38, 0xA3C, 0xA16, + 0xA3C, 0xA17, 0xA3C, 0xA1C, 0xA3C, 0xA2B, 0xA3C, 0xB21, 0xB3C, 0xB22, + 0xB3C, 0xE4D, 0xE32, 0xECD, 0xEB2, 0xEAB, 0xE99, 0xEAB, 0xEA1, 0xF0B, + 0xF42, 0xFB7, 0xF4C, 0xFB7, 0xF51, 0xFB7, 0xF56, 0xFB7, 0xF5B, 0xFB7, + 0xF40, 0xFB5, 0xF71, 0xF72, 0xF71, 0xF74, 0xFB2, 0xF80, 0xFB2, 0xF71, + 0xF80, 0xFB3, 0xF80, 0xFB3, 0xF71, 0xF80, 0xF71, 0xF80, 0xF92, 0xFB7, + 0xF9C, 0xFB7, 0xFA1, 0xFB7, 0xFA6, 0xFB7, 0xFAB, 0xFB7, 0xF90, 0xFB5, + 0x10DC, 0x41, 0xC6, 0x42, 0x44, 0x45, 0x18E, 0x47, 0x48, 0x49, + 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x222, 0x50, 0x52, 0x54, + 0x55, 0x57, 0x61, 0x250, 0x251, 0x1D02, 0x62, 0x64, 0x65, 0x259, + 0x25B, 0x25C, 0x67, 0x6B, 0x6D, 0x14B, 0x6F, 0x254, 0x1D16, 0x1D17, + 0x70, 0x74, 0x75, 0x1D1D, 0x26F, 0x76, 0x1D25, 0x3B2, 0x3B3, 0x3B4, + 0x3C6, 0x3C7, 0x69, 0x72, 0x75, 0x76, 0x3B2, 0x3B3, 0x3C1, 0x3C6, + 0x3C7, 0x43D, 0x252, 0x63, 0x255, 0xF0, 0x25C, 0x66, 0x25F, 0x261, + 0x265, 0x268, 0x269, 0x26A, 0x1D7B, 0x29D, 0x26D, 0x1D85, 0x29F, 0x271, + 0x270, 0x272, 0x273, 0x274, 0x275, 0x278, 0x282, 0x283, 0x1AB, 0x289, + 0x28A, 0x1D1C, 0x28B, 0x28C, 0x7A, 0x290, 0x291, 0x292, 0x3B8, 0x61, + 0x2BE, 0x1E61, 0x3AC, 0x3AD, 0x3AE, 0x3AF, 0x3CC, 0x3CD, 0x3CE, 0x386, + 0x20, 0x313, 0x3B9, 0x20, 0x313, 0x20, 0x342, 0x20, 0x308, 0x342, + 0x388, 0x389, 0x20, 0x313, 0x300, 0x20, 0x313, 0x301, 0x20, 0x313, + 0x342, 0x390, 0x38A, 0x20, 0x314, 0x300, 0x20, 0x314, 0x301, 0x20, + 0x314, 0x342, 0x3B0, 0x38E, 0x20, 0x308, 0x300, 0x20, 0x308, 0x301, + 0x60, 0x38C, 0x38F, 0x20, 0x301, 0x20, 0x314, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2010, 0x20, + 0x333, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x2E, 0x20, 0x2032, 0x2032, + 0x2032, 0x2032, 0x2032, 0x2035, 0x2035, 0x2035, 0x2035, 0x2035, 0x21, 0x21, + 0x20, 0x305, 0x3F, 0x3F, 0x3F, 0x21, 0x21, 0x3F, 0x2032, 0x2032, + 0x2032, 0x2032, 0x20, 0x30, 0x69, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x2B, 0x2212, 0x3D, 0x28, 0x29, 0x6E, 0x30, 0x31, 0x32, + 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2212, 0x3D, + 0x28, 0x29, 0x61, 0x65, 0x6F, 0x78, 0x259, 0x68, 0x6B, 0x6C, + 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x52, 0x73, 0x61, 0x2F, 0x63, + 0x61, 0x2F, 0x73, 0x43, 0xB0, 0x43, 0x63, 0x2F, 0x6F, 0x63, + 0x2F, 0x75, 0x190, 0xB0, 0x46, 0x67, 0x48, 0x48, 0x48, 0x68, + 0x127, 0x49, 0x49, 0x4C, 0x6C, 0x4E, 0x4E, 0x6F, 0x50, 0x51, + 0x52, 0x52, 0x52, 0x53, 0x4D, 0x54, 0x45, 0x4C, 0x54, 0x4D, + 0x5A, 0x3A9, 0x5A, 0x4B, 0xC5, 0x42, 0x43, 0x65, 0x45, 0x46, + 0x4D, 0x6F, 0x5D0, 0x5D1, 0x5D2, 0x5D3, 0x69, 0x46, 0x41, 0x58, + 0x3C0, 0x3B3, 0x393, 0x3A0, 0x2211, 0x44, 0x64, 0x65, 0x69, 0x6A, + 0x31, 0x2044, 0x37, 0x31, 0x2044, 0x39, 0x31, 0x2044, 0x31, 0x30, + 0x31, 0x2044, 0x33, 0x32, 0x2044, 0x33, 0x31, 0x2044, 0x35, 0x32, + 0x2044, 0x35, 0x33, 0x2044, 0x35, 0x34, 0x2044, 0x35, 0x31, 0x2044, + 0x36, 0x35, 0x2044, 0x36, 0x31, 0x2044, 0x38, 0x33, 0x2044, 0x38, + 0x35, 0x2044, 0x38, 0x37, 0x2044, 0x38, 0x31, 0x2044, 0x49, 0x49, + 0x49, 0x49, 0x49, 0x49, 0x49, 0x56, 0x56, 0x56, 0x49, 0x56, + 0x49, 0x49, 0x56, 0x49, 0x49, 0x49, 0x49, 0x58, 0x58, 0x58, + 0x49, 0x58, 0x49, 0x49, 0x4C, 0x43, 0x44, 0x4D, 0x69, 0x69, + 0x69, 0x69, 0x69, 0x69, 0x69, 0x76, 0x76, 0x76, 0x69, 0x76, + 0x69, 0x69, 0x76, 0x69, 0x69, 0x69, 0x69, 0x78, 0x78, 0x78, + 0x69, 0x78, 0x69, 0x69, 0x6C, 0x63, 0x64, 0x6D, 0x30, 0x2044, + 0x33, 0x222B, 0x222B, 0x222B, 0x222B, 0x222B, 0x222E, 0x222E, 0x222E, 0x222E, + 0x222E, 0x3008, 0x3009, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x31, 0x30, 0x31, 0x31, 0x31, 0x32, 0x31, 0x33, + 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, 0x31, 0x38, + 0x31, 0x39, 0x32, 0x30, 0x28, 0x31, 0x29, 0x28, 0x32, 0x29, + 0x28, 0x33, 0x29, 0x28, 0x34, 0x29, 0x28, 0x35, 0x29, 0x28, + 0x36, 0x29, 0x28, 0x37, 0x29, 0x28, 0x38, 0x29, 0x28, 0x39, + 0x29, 0x28, 0x31, 0x30, 0x29, 0x28, 0x31, 0x31, 0x29, 0x28, + 0x31, 0x32, 0x29, 0x28, 0x31, 0x33, 0x29, 0x28, 0x31, 0x34, + 0x29, 0x28, 0x31, 0x35, 0x29, 0x28, 0x31, 0x36, 0x29, 0x28, + 0x31, 0x37, 0x29, 0x28, 0x31, 0x38, 0x29, 0x28, 0x31, 0x39, + 0x29, 0x28, 0x32, 0x30, 0x29, 0x31, 0x2E, 0x32, 0x2E, 0x33, + 0x2E, 0x34, 0x2E, 0x35, 0x2E, 0x36, 0x2E, 0x37, 0x2E, 0x38, + 0x2E, 0x39, 0x2E, 0x31, 0x30, 0x2E, 0x31, 0x31, 0x2E, 0x31, + 0x32, 0x2E, 0x31, 0x33, 0x2E, 0x31, 0x34, 0x2E, 0x31, 0x35, + 0x2E, 0x31, 0x36, 0x2E, 0x31, 0x37, 0x2E, 0x31, 0x38, 0x2E, + 0x31, 0x39, 0x2E, 0x32, 0x30, 0x2E, 0x28, 0x61, 0x29, 0x28, + 0x62, 0x29, 0x28, 0x63, 0x29, 0x28, 0x64, 0x29, 0x28, 0x65, + 0x29, 0x28, 0x66, 0x29, 0x28, 0x67, 0x29, 0x28, 0x68, 0x29, + 0x28, 0x69, 0x29, 0x28, 0x6A, 0x29, 0x28, 0x6B, 0x29, 0x28, + 0x6C, 0x29, 0x28, 0x6D, 0x29, 0x28, 0x6E, 0x29, 0x28, 0x6F, + 0x29, 0x28, 0x70, 0x29, 0x28, 0x71, 0x29, 0x28, 0x72, 0x29, + 0x28, 0x73, 0x29, 0x28, 0x74, 0x29, 0x28, 0x75, 0x29, 0x28, + 0x76, 0x29, 0x28, 0x77, 0x29, 0x28, 0x78, 0x29, 0x28, 0x79, + 0x29, 0x28, 0x7A, 0x29, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, + 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x30, 0x222B, 0x222B, 0x222B, + 0x222B, 0x3A, 0x3A, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x2ADD, + 0x338, 0x6A, 0x56, 0x2D61, 0x6BCD, 0x9F9F, 0x4E00, 0x4E28, 0x4E36, 0x4E3F, + 0x4E59, 0x4E85, 0x4E8C, 0x4EA0, 0x4EBA, 0x513F, 0x5165, 0x516B, 0x5182, 0x5196, + 0x51AB, 0x51E0, 0x51F5, 0x5200, 0x529B, 0x52F9, 0x5315, 0x531A, 0x5338, 0x5341, + 0x535C, 0x5369, 0x5382, 0x53B6, 0x53C8, 0x53E3, 0x56D7, 0x571F, 0x58EB, 0x5902, + 0x590A, 0x5915, 0x5927, 0x5973, 0x5B50, 0x5B80, 0x5BF8, 0x5C0F, 0x5C22, 0x5C38, + 0x5C6E, 0x5C71, 0x5DDB, 0x5DE5, 0x5DF1, 0x5DFE, 0x5E72, 0x5E7A, 0x5E7F, 0x5EF4, + 0x5EFE, 0x5F0B, 0x5F13, 0x5F50, 0x5F61, 0x5F73, 0x5FC3, 0x6208, 0x6236, 0x624B, + 0x652F, 0x6534, 0x6587, 0x6597, 0x65A4, 0x65B9, 0x65E0, 0x65E5, 0x66F0, 0x6708, + 0x6728, 0x6B20, 0x6B62, 0x6B79, 0x6BB3, 0x6BCB, 0x6BD4, 0x6BDB, 0x6C0F, 0x6C14, + 0x6C34, 0x706B, 0x722A, 0x7236, 0x723B, 0x723F, 0x7247, 0x7259, 0x725B, 0x72AC, + 0x7384, 0x7389, 0x74DC, 0x74E6, 0x7518, 0x751F, 0x7528, 0x7530, 0x758B, 0x7592, + 0x7676, 0x767D, 0x76AE, 0x76BF, 0x76EE, 0x77DB, 0x77E2, 0x77F3, 0x793A, 0x79B8, + 0x79BE, 0x7A74, 0x7ACB, 0x7AF9, 0x7C73, 0x7CF8, 0x7F36, 0x7F51, 0x7F8A, 0x7FBD, + 0x8001, 0x800C, 0x8012, 0x8033, 0x807F, 0x8089, 0x81E3, 0x81EA, 0x81F3, 0x81FC, + 0x820C, 0x821B, 0x821F, 0x826E, 0x8272, 0x8278, 0x864D, 0x866B, 0x8840, 0x884C, + 0x8863, 0x897E, 0x898B, 0x89D2, 0x8A00, 0x8C37, 0x8C46, 0x8C55, 0x8C78, 0x8C9D, + 0x8D64, 0x8D70, 0x8DB3, 0x8EAB, 0x8ECA, 0x8F9B, 0x8FB0, 0x8FB5, 0x9091, 0x9149, + 0x91C6, 0x91CC, 0x91D1, 0x9577, 0x9580, 0x961C, 0x96B6, 0x96B9, 0x96E8, 0x9751, + 0x975E, 0x9762, 0x9769, 0x97CB, 0x97ED, 0x97F3, 0x9801, 0x98A8, 0x98DB, 0x98DF, + 0x9996, 0x9999, 0x99AC, 0x9AA8, 0x9AD8, 0x9ADF, 0x9B25, 0x9B2F, 0x9B32, 0x9B3C, + 0x9B5A, 0x9CE5, 0x9E75, 0x9E7F, 0x9EA5, 0x9EBB, 0x9EC3, 0x9ECD, 0x9ED1, 0x9EF9, + 0x9EFD, 0x9F0E, 0x9F13, 0x9F20, 0x9F3B, 0x9F4A, 0x9F52, 0x9F8D, 0x9F9C, 0x9FA0, + 0x20, 0x3012, 0x5341, 0x5344, 0x5345, 0x20, 0x3099, 0x20, 0x309A, 0x3088, + 0x308A, 0x30B3, 0x30C8, 0x1100, 0x1101, 0x11AA, 0x1102, 0x11AC, 0x11AD, 0x1103, + 0x1104, 0x1105, 0x11B0, 0x11B1, 0x11B2, 0x11B3, 0x11B4, 0x11B5, 0x111A, 0x1106, + 0x1107, 0x1108, 0x1121, 0x1109, 0x110A, 0x110B, 0x110C, 0x110D, 0x110E, 0x110F, + 0x1110, 0x1111, 0x1112, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, + 0x1168, 0x1169, 0x116A, 0x116B, 0x116C, 0x116D, 0x116E, 0x116F, 0x1170, 0x1171, + 0x1172, 0x1173, 0x1174, 0x1175, 0x1160, 0x1114, 0x1115, 0x11C7, 0x11C8, 0x11CC, + 0x11CE, 0x11D3, 0x11D7, 0x11D9, 0x111C, 0x11DD, 0x11DF, 0x111D, 0x111E, 0x1120, + 0x1122, 0x1123, 0x1127, 0x1129, 0x112B, 0x112C, 0x112D, 0x112E, 0x112F, 0x1132, + 0x1136, 0x1140, 0x1147, 0x114C, 0x11F1, 0x11F2, 0x1157, 0x1158, 0x1159, 0x1184, + 0x1185, 0x1188, 0x1191, 0x1192, 0x1194, 0x119E, 0x11A1, 0x4E00, 0x4E8C, 0x4E09, + 0x56DB, 0x4E0A, 0x4E2D, 0x4E0B, 0x7532, 0x4E59, 0x4E19, 0x4E01, 0x5929, 0x5730, + 0x4EBA, 0x28, 0x1100, 0x29, 0x28, 0x1102, 0x29, 0x28, 0x1103, 0x29, + 0x28, 0x1105, 0x29, 0x28, 0x1106, 0x29, 0x28, 0x1107, 0x29, 0x28, + 0x1109, 0x29, 0x28, 0x110B, 0x29, 0x28, 0x110C, 0x29, 0x28, 0x110E, + 0x29, 0x28, 0x110F, 0x29, 0x28, 0x1110, 0x29, 0x28, 0x1111, 0x29, + 0x28, 0x1112, 0x29, 0x28, 0xAC00, 0x29, 0x28, 0xB098, 0x29, 0x28, + 0xB2E4, 0x29, 0x28, 0xB77C, 0x29, 0x28, 0xB9C8, 0x29, 0x28, 0xBC14, + 0x29, 0x28, 0xC0AC, 0x29, 0x28, 0xC544, 0x29, 0x28, 0xC790, 0x29, + 0x28, 0xCC28, 0x29, 0x28, 0xCE74, 0x29, 0x28, 0xD0C0, 0x29, 0x28, + 0xD30C, 0x29, 0x28, 0xD558, 0x29, 0x28, 0xC8FC, 0x29, 0x28, 0xC624, + 0xC804, 0x29, 0x28, 0xC624, 0xD6C4, 0x29, 0x28, 0x4E00, 0x29, 0x28, + 0x4E8C, 0x29, 0x28, 0x4E09, 0x29, 0x28, 0x56DB, 0x29, 0x28, 0x4E94, + 0x29, 0x28, 0x516D, 0x29, 0x28, 0x4E03, 0x29, 0x28, 0x516B, 0x29, + 0x28, 0x4E5D, 0x29, 0x28, 0x5341, 0x29, 0x28, 0x6708, 0x29, 0x28, + 0x706B, 0x29, 0x28, 0x6C34, 0x29, 0x28, 0x6728, 0x29, 0x28, 0x91D1, + 0x29, 0x28, 0x571F, 0x29, 0x28, 0x65E5, 0x29, 0x28, 0x682A, 0x29, + 0x28, 0x6709, 0x29, 0x28, 0x793E, 0x29, 0x28, 0x540D, 0x29, 0x28, + 0x7279, 0x29, 0x28, 0x8CA1, 0x29, 0x28, 0x795D, 0x29, 0x28, 0x52B4, + 0x29, 0x28, 0x4EE3, 0x29, 0x28, 0x547C, 0x29, 0x28, 0x5B66, 0x29, + 0x28, 0x76E3, 0x29, 0x28, 0x4F01, 0x29, 0x28, 0x8CC7, 0x29, 0x28, + 0x5354, 0x29, 0x28, 0x796D, 0x29, 0x28, 0x4F11, 0x29, 0x28, 0x81EA, + 0x29, 0x28, 0x81F3, 0x29, 0x554F, 0x5E7C, 0x6587, 0x7B8F, 0x50, 0x54, + 0x45, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, 0x32, 0x34, 0x32, + 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, 0x33, + 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, + 0x35, 0x1100, 0x1102, 0x1103, 0x1105, 0x1106, 0x1107, 0x1109, 0x110B, 0x110C, + 0x110E, 0x110F, 0x1110, 0x1111, 0x1112, 0xAC00, 0xB098, 0xB2E4, 0xB77C, 0xB9C8, + 0xBC14, 0xC0AC, 0xC544, 0xC790, 0xCC28, 0xCE74, 0xD0C0, 0xD30C, 0xD558, 0xCC38, + 0xACE0, 0xC8FC, 0xC758, 0xC6B0, 0x4E00, 0x4E8C, 0x4E09, 0x56DB, 0x4E94, 0x516D, + 0x4E03, 0x516B, 0x4E5D, 0x5341, 0x6708, 0x706B, 0x6C34, 0x6728, 0x91D1, 0x571F, + 0x65E5, 0x682A, 0x6709, 0x793E, 0x540D, 0x7279, 0x8CA1, 0x795D, 0x52B4, 0x79D8, + 0x7537, 0x5973, 0x9069, 0x512A, 0x5370, 0x6CE8, 0x9805, 0x4F11, 0x5199, 0x6B63, + 0x4E0A, 0x4E2D, 0x4E0B, 0x5DE6, 0x53F3, 0x533B, 0x5B97, 0x5B66, 0x76E3, 0x4F01, + 0x8CC7, 0x5354, 0x591C, 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, + 0x39, 0x34, 0x30, 0x34, 0x31, 0x34, 0x32, 0x34, 0x33, 0x34, + 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, 0x34, 0x38, 0x34, + 0x39, 0x35, 0x30, 0x31, 0x6708, 0x32, 0x6708, 0x33, 0x6708, 0x34, + 0x6708, 0x35, 0x6708, 0x36, 0x6708, 0x37, 0x6708, 0x38, 0x6708, 0x39, + 0x6708, 0x31, 0x30, 0x6708, 0x31, 0x31, 0x6708, 0x31, 0x32, 0x6708, + 0x48, 0x67, 0x65, 0x72, 0x67, 0x65, 0x56, 0x4C, 0x54, 0x44, + 0x30A2, 0x30A4, 0x30A6, 0x30A8, 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3, + 0x30B5, 0x30B7, 0x30B9, 0x30BB, 0x30BD, 0x30BF, 0x30C1, 0x30C4, 0x30C6, 0x30C8, + 0x30CA, 0x30CB, 0x30CC, 0x30CD, 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, + 0x30DE, 0x30DF, 0x30E0, 0x30E1, 0x30E2, 0x30E4, 0x30E6, 0x30E8, 0x30E9, 0x30EA, + 0x30EB, 0x30EC, 0x30ED, 0x30EF, 0x30F0, 0x30F1, 0x30F2, 0x4EE4, 0x548C, 0x30A2, + 0x30D1, 0x30FC, 0x30C8, 0x30A2, 0x30EB, 0x30D5, 0x30A1, 0x30A2, 0x30F3, 0x30DA, + 0x30A2, 0x30A2, 0x30FC, 0x30EB, 0x30A4, 0x30CB, 0x30F3, 0x30B0, 0x30A4, 0x30F3, + 0x30C1, 0x30A6, 0x30A9, 0x30F3, 0x30A8, 0x30B9, 0x30AF, 0x30FC, 0x30C9, 0x30A8, + 0x30FC, 0x30AB, 0x30FC, 0x30AA, 0x30F3, 0x30B9, 0x30AA, 0x30FC, 0x30E0, 0x30AB, + 0x30A4, 0x30EA, 0x30AB, 0x30E9, 0x30C3, 0x30C8, 0x30AB, 0x30ED, 0x30EA, 0x30FC, + 0x30AC, 0x30ED, 0x30F3, 0x30AC, 0x30F3, 0x30DE, 0x30AE, 0x30AC, 0x30AE, 0x30CB, + 0x30FC, 0x30AD, 0x30E5, 0x30EA, 0x30FC, 0x30AE, 0x30EB, 0x30C0, 0x30FC, 0x30AD, + 0x30ED, 0x30AD, 0x30ED, 0x30B0, 0x30E9, 0x30E0, 0x30AD, 0x30ED, 0x30E1, 0x30FC, + 0x30C8, 0x30EB, 0x30AD, 0x30ED, 0x30EF, 0x30C3, 0x30C8, 0x30B0, 0x30E9, 0x30E0, + 0x30B0, 0x30E9, 0x30E0, 0x30C8, 0x30F3, 0x30AF, 0x30EB, 0x30BC, 0x30A4, 0x30ED, + 0x30AF, 0x30ED, 0x30FC, 0x30CD, 0x30B1, 0x30FC, 0x30B9, 0x30B3, 0x30EB, 0x30CA, + 0x30B3, 0x30FC, 0x30DD, 0x30B5, 0x30A4, 0x30AF, 0x30EB, 0x30B5, 0x30F3, 0x30C1, + 0x30FC, 0x30E0, 0x30B7, 0x30EA, 0x30F3, 0x30B0, 0x30BB, 0x30F3, 0x30C1, 0x30BB, + 0x30F3, 0x30C8, 0x30C0, 0x30FC, 0x30B9, 0x30C7, 0x30B7, 0x30C9, 0x30EB, 0x30C8, + 0x30F3, 0x30CA, 0x30CE, 0x30CE, 0x30C3, 0x30C8, 0x30CF, 0x30A4, 0x30C4, 0x30D1, + 0x30FC, 0x30BB, 0x30F3, 0x30C8, 0x30D1, 0x30FC, 0x30C4, 0x30D0, 0x30FC, 0x30EC, + 0x30EB, 0x30D4, 0x30A2, 0x30B9, 0x30C8, 0x30EB, 0x30D4, 0x30AF, 0x30EB, 0x30D4, + 0x30B3, 0x30D3, 0x30EB, 0x30D5, 0x30A1, 0x30E9, 0x30C3, 0x30C9, 0x30D5, 0x30A3, + 0x30FC, 0x30C8, 0x30D6, 0x30C3, 0x30B7, 0x30A7, 0x30EB, 0x30D5, 0x30E9, 0x30F3, + 0x30D8, 0x30AF, 0x30BF, 0x30FC, 0x30EB, 0x30DA, 0x30BD, 0x30DA, 0x30CB, 0x30D2, + 0x30D8, 0x30EB, 0x30C4, 0x30DA, 0x30F3, 0x30B9, 0x30DA, 0x30FC, 0x30B8, 0x30D9, + 0x30FC, 0x30BF, 0x30DD, 0x30A4, 0x30F3, 0x30C8, 0x30DC, 0x30EB, 0x30C8, 0x30DB, + 0x30F3, 0x30DD, 0x30F3, 0x30C9, 0x30DB, 0x30FC, 0x30EB, 0x30DB, 0x30FC, 0x30F3, + 0x30DE, 0x30A4, 0x30AF, 0x30ED, 0x30DE, 0x30A4, 0x30EB, 0x30DE, 0x30C3, 0x30CF, + 0x30DE, 0x30EB, 0x30AF, 0x30DE, 0x30F3, 0x30B7, 0x30E7, 0x30F3, 0x30DF, 0x30AF, + 0x30ED, 0x30F3, 0x30DF, 0x30EA, 0x30DF, 0x30EA, 0x30D0, 0x30FC, 0x30EB, 0x30E1, + 0x30AC, 0x30E1, 0x30AC, 0x30C8, 0x30F3, 0x30E1, 0x30FC, 0x30C8, 0x30EB, 0x30E4, + 0x30FC, 0x30C9, 0x30E4, 0x30FC, 0x30EB, 0x30E6, 0x30A2, 0x30F3, 0x30EA, 0x30C3, + 0x30C8, 0x30EB, 0x30EA, 0x30E9, 0x30EB, 0x30D4, 0x30FC, 0x30EB, 0x30FC, 0x30D6, + 0x30EB, 0x30EC, 0x30E0, 0x30EC, 0x30F3, 0x30C8, 0x30B2, 0x30F3, 0x30EF, 0x30C3, + 0x30C8, 0x30, 0x70B9, 0x31, 0x70B9, 0x32, 0x70B9, 0x33, 0x70B9, 0x34, + 0x70B9, 0x35, 0x70B9, 0x36, 0x70B9, 0x37, 0x70B9, 0x38, 0x70B9, 0x39, + 0x70B9, 0x31, 0x30, 0x70B9, 0x31, 0x31, 0x70B9, 0x31, 0x32, 0x70B9, + 0x31, 0x33, 0x70B9, 0x31, 0x34, 0x70B9, 0x31, 0x35, 0x70B9, 0x31, + 0x36, 0x70B9, 0x31, 0x37, 0x70B9, 0x31, 0x38, 0x70B9, 0x31, 0x39, + 0x70B9, 0x32, 0x30, 0x70B9, 0x32, 0x31, 0x70B9, 0x32, 0x32, 0x70B9, + 0x32, 0x33, 0x70B9, 0x32, 0x34, 0x70B9, 0x68, 0x50, 0x61, 0x64, + 0x61, 0x41, 0x55, 0x62, 0x61, 0x72, 0x6F, 0x56, 0x70, 0x63, + 0x64, 0x6D, 0x64, 0x6D, 0x32, 0x64, 0x6D, 0x33, 0x49, 0x55, + 0x5E73, 0x6210, 0x662D, 0x548C, 0x5927, 0x6B63, 0x660E, 0x6CBB, 0x682A, 0x5F0F, + 0x4F1A, 0x793E, 0x70, 0x41, 0x6E, 0x41, 0x3BC, 0x41, 0x6D, 0x41, + 0x6B, 0x41, 0x4B, 0x42, 0x4D, 0x42, 0x47, 0x42, 0x63, 0x61, + 0x6C, 0x6B, 0x63, 0x61, 0x6C, 0x70, 0x46, 0x6E, 0x46, 0x3BC, + 0x46, 0x3BC, 0x67, 0x6D, 0x67, 0x6B, 0x67, 0x48, 0x7A, 0x6B, + 0x48, 0x7A, 0x4D, 0x48, 0x7A, 0x47, 0x48, 0x7A, 0x54, 0x48, + 0x7A, 0x3BC, 0x6C, 0x6D, 0x6C, 0x64, 0x6C, 0x6B, 0x6C, 0x66, + 0x6D, 0x6E, 0x6D, 0x3BC, 0x6D, 0x6D, 0x6D, 0x63, 0x6D, 0x6B, + 0x6D, 0x6D, 0x6D, 0x32, 0x63, 0x6D, 0x32, 0x6D, 0x32, 0x6B, + 0x6D, 0x32, 0x6D, 0x6D, 0x33, 0x63, 0x6D, 0x33, 0x6D, 0x33, + 0x6B, 0x6D, 0x33, 0x6D, 0x2215, 0x73, 0x6D, 0x2215, 0x73, 0x32, + 0x50, 0x61, 0x6B, 0x50, 0x61, 0x4D, 0x50, 0x61, 0x47, 0x50, + 0x61, 0x72, 0x61, 0x64, 0x72, 0x61, 0x64, 0x2215, 0x73, 0x72, + 0x61, 0x64, 0x2215, 0x73, 0x32, 0x70, 0x73, 0x6E, 0x73, 0x3BC, + 0x73, 0x6D, 0x73, 0x70, 0x56, 0x6E, 0x56, 0x3BC, 0x56, 0x6D, + 0x56, 0x6B, 0x56, 0x4D, 0x56, 0x70, 0x57, 0x6E, 0x57, 0x3BC, + 0x57, 0x6D, 0x57, 0x6B, 0x57, 0x4D, 0x57, 0x6B, 0x3A9, 0x4D, + 0x3A9, 0x61, 0x2E, 0x6D, 0x2E, 0x42, 0x71, 0x63, 0x63, 0x63, + 0x64, 0x43, 0x2215, 0x6B, 0x67, 0x43, 0x6F, 0x2E, 0x64, 0x42, + 0x47, 0x79, 0x68, 0x61, 0x48, 0x50, 0x69, 0x6E, 0x4B, 0x4B, + 0x4B, 0x4D, 0x6B, 0x74, 0x6C, 0x6D, 0x6C, 0x6E, 0x6C, 0x6F, + 0x67, 0x6C, 0x78, 0x6D, 0x62, 0x6D, 0x69, 0x6C, 0x6D, 0x6F, + 0x6C, 0x50, 0x48, 0x70, 0x2E, 0x6D, 0x2E, 0x50, 0x50, 0x4D, + 0x50, 0x52, 0x73, 0x72, 0x53, 0x76, 0x57, 0x62, 0x56, 0x2215, + 0x6D, 0x41, 0x2215, 0x6D, 0x31, 0x65E5, 0x32, 0x65E5, 0x33, 0x65E5, + 0x34, 0x65E5, 0x35, 0x65E5, 0x36, 0x65E5, 0x37, 0x65E5, 0x38, 0x65E5, + 0x39, 0x65E5, 0x31, 0x30, 0x65E5, 0x31, 0x31, 0x65E5, 0x31, 0x32, + 0x65E5, 0x31, 0x33, 0x65E5, 0x31, 0x34, 0x65E5, 0x31, 0x35, 0x65E5, + 0x31, 0x36, 0x65E5, 0x31, 0x37, 0x65E5, 0x31, 0x38, 0x65E5, 0x31, + 0x39, 0x65E5, 0x32, 0x30, 0x65E5, 0x32, 0x31, 0x65E5, 0x32, 0x32, + 0x65E5, 0x32, 0x33, 0x65E5, 0x32, 0x34, 0x65E5, 0x32, 0x35, 0x65E5, + 0x32, 0x36, 0x65E5, 0x32, 0x37, 0x65E5, 0x32, 0x38, 0x65E5, 0x32, + 0x39, 0x65E5, 0x33, 0x30, 0x65E5, 0x33, 0x31, 0x65E5, 0x67, 0x61, + 0x6C, 0x44A, 0x44C, 0xA76F, 0x43, 0x46, 0x51, 0x126, 0x153, 0xA727, + 0xAB37, 0x26B, 0xAB52, 0x28D, 0x8C48, 0x66F4, 0x8ECA, 0x8CC8, 0x6ED1, 0x4E32, + 0x53E5, 0x9F9C, 0x9F9C, 0x5951, 0x91D1, 0x5587, 0x5948, 0x61F6, 0x7669, 0x7F85, + 0x863F, 0x87BA, 0x88F8, 0x908F, 0x6A02, 0x6D1B, 0x70D9, 0x73DE, 0x843D, 0x916A, + 0x99F1, 0x4E82, 0x5375, 0x6B04, 0x721B, 0x862D, 0x9E1E, 0x5D50, 0x6FEB, 0x85CD, + 0x8964, 0x62C9, 0x81D8, 0x881F, 0x5ECA, 0x6717, 0x6D6A, 0x72FC, 0x90CE, 0x4F86, + 0x51B7, 0x52DE, 0x64C4, 0x6AD3, 0x7210, 0x76E7, 0x8001, 0x8606, 0x865C, 0x8DEF, + 0x9732, 0x9B6F, 0x9DFA, 0x788C, 0x797F, 0x7DA0, 0x83C9, 0x9304, 0x9E7F, 0x8AD6, + 0x58DF, 0x5F04, 0x7C60, 0x807E, 0x7262, 0x78CA, 0x8CC2, 0x96F7, 0x58D8, 0x5C62, + 0x6A13, 0x6DDA, 0x6F0F, 0x7D2F, 0x7E37, 0x964B, 0x52D2, 0x808B, 0x51DC, 0x51CC, + 0x7A1C, 0x7DBE, 0x83F1, 0x9675, 0x8B80, 0x62CF, 0x6A02, 0x8AFE, 0x4E39, 0x5BE7, + 0x6012, 0x7387, 0x7570, 0x5317, 0x78FB, 0x4FBF, 0x5FA9, 0x4E0D, 0x6CCC, 0x6578, + 0x7D22, 0x53C3, 0x585E, 0x7701, 0x8449, 0x8AAA, 0x6BBA, 0x8FB0, 0x6C88, 0x62FE, + 0x82E5, 0x63A0, 0x7565, 0x4EAE, 0x5169, 0x51C9, 0x6881, 0x7CE7, 0x826F, 0x8AD2, + 0x91CF, 0x52F5, 0x5442, 0x5973, 0x5EEC, 0x65C5, 0x6FFE, 0x792A, 0x95AD, 0x9A6A, + 0x9E97, 0x9ECE, 0x529B, 0x66C6, 0x6B77, 0x8F62, 0x5E74, 0x6190, 0x6200, 0x649A, + 0x6F23, 0x7149, 0x7489, 0x79CA, 0x7DF4, 0x806F, 0x8F26, 0x84EE, 0x9023, 0x934A, + 0x5217, 0x52A3, 0x54BD, 0x70C8, 0x88C2, 0x8AAA, 0x5EC9, 0x5FF5, 0x637B, 0x6BAE, + 0x7C3E, 0x7375, 0x4EE4, 0x56F9, 0x5BE7, 0x5DBA, 0x601C, 0x73B2, 0x7469, 0x7F9A, + 0x8046, 0x9234, 0x96F6, 0x9748, 0x9818, 0x4F8B, 0x79AE, 0x91B4, 0x96B8, 0x60E1, + 0x4E86, 0x50DA, 0x5BEE, 0x5C3F, 0x6599, 0x6A02, 0x71CE, 0x7642, 0x84FC, 0x907C, + 0x9F8D, 0x6688, 0x962E, 0x5289, 0x677B, 0x67F3, 0x6D41, 0x6E9C, 0x7409, 0x7559, + 0x786B, 0x7D10, 0x985E, 0x516D, 0x622E, 0x9678, 0x502B, 0x5D19, 0x6DEA, 0x8F2A, + 0x5F8B, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540F, 0x5C65, 0x6613, 0x674E, + 0x68A8, 0x6CE5, 0x7406, 0x75E2, 0x7F79, 0x88CF, 0x88E1, 0x91CC, 0x96E2, 0x533F, + 0x6EBA, 0x541D, 0x71D0, 0x7498, 0x85FA, 0x96A3, 0x9C57, 0x9E9F, 0x6797, 0x6DCB, + 0x81E8, 0x7ACB, 0x7B20, 0x7C92, 0x72C0, 0x7099, 0x8B58, 0x4EC0, 0x8336, 0x523A, + 0x5207, 0x5EA6, 0x62D3, 0x7CD6, 0x5B85, 0x6D1E, 0x66B4, 0x8F3B, 0x884C, 0x964D, + 0x898B, 0x5ED3, 0x5140, 0x55C0, 0x585A, 0x6674, 0x51DE, 0x732A, 0x76CA, 0x793C, + 0x795E, 0x7965, 0x798F, 0x9756, 0x7CBE, 0x7FBD, 0x8612, 0x8AF8, 0x9038, 0x90FD, + 0x98EF, 0x98FC, 0x9928, 0x9DB4, 0x90DE, 0x96B7, 0x4FAE, 0x50E7, 0x514D, 0x52C9, + 0x52E4, 0x5351, 0x559D, 0x5606, 0x5668, 0x5840, 0x58A8, 0x5C64, 0x5C6E, 0x6094, + 0x6168, 0x618E, 0x61F2, 0x654F, 0x65E2, 0x6691, 0x6885, 0x6D77, 0x6E1A, 0x6F22, + 0x716E, 0x722B, 0x7422, 0x7891, 0x793E, 0x7949, 0x7948, 0x7950, 0x7956, 0x795D, + 0x798D, 0x798E, 0x7A40, 0x7A81, 0x7BC0, 0x7DF4, 0x7E09, 0x7E41, 0x7F72, 0x8005, + 0x81ED, 0x8279, 0x8279, 0x8457, 0x8910, 0x8996, 0x8B01, 0x8B39, 0x8CD3, 0x8D08, + 0x8FB6, 0x9038, 0x96E3, 0x97FF, 0x983B, 0x6075, 0x242EE, 0x8218, 0x4E26, 0x51B5, + 0x5168, 0x4F80, 0x5145, 0x5180, 0x52C7, 0x52FA, 0x559D, 0x5555, 0x5599, 0x55E2, + 0x585A, 0x58B3, 0x5944, 0x5954, 0x5A62, 0x5B28, 0x5ED2, 0x5ED9, 0x5F69, 0x5FAD, + 0x60D8, 0x614E, 0x6108, 0x618E, 0x6160, 0x61F2, 0x6234, 0x63C4, 0x641C, 0x6452, + 0x6556, 0x6674, 0x6717, 0x671B, 0x6756, 0x6B79, 0x6BBA, 0x6D41, 0x6EDB, 0x6ECB, + 0x6F22, 0x701E, 0x716E, 0x77A7, 0x7235, 0x72AF, 0x732A, 0x7471, 0x7506, 0x753B, + 0x761D, 0x761F, 0x76CA, 0x76DB, 0x76F4, 0x774A, 0x7740, 0x78CC, 0x7AB1, 0x7BC0, + 0x7C7B, 0x7D5B, 0x7DF4, 0x7F3E, 0x8005, 0x8352, 0x83EF, 0x8779, 0x8941, 0x8986, + 0x8996, 0x8ABF, 0x8AF8, 0x8ACB, 0x8B01, 0x8AFE, 0x8AED, 0x8B39, 0x8B8A, 0x8D08, + 0x8F38, 0x9072, 0x9199, 0x9276, 0x967C, 0x96E3, 0x9756, 0x97DB, 0x97FF, 0x980B, + 0x983B, 0x9B12, 0x9F9C, 0x2284A, 0x22844, 0x233D5, 0x3B9D, 0x4018, 0x4039, 0x25249, + 0x25CD0, 0x27ED3, 0x9F43, 0x9F8E, 0x66, 0x66, 0x66, 0x69, 0x66, 0x6C, + 0x66, 0x66, 0x69, 0x66, 0x66, 0x6C, 0x73, 0x74, 0x73, 0x74, + 0x574, 0x576, 0x574, 0x565, 0x574, 0x56B, 0x57E, 0x576, 0x574, 0x56D, + 0x5D9, 0x5B4, 0x5F2, 0x5B7, 0x5E2, 0x5D0, 0x5D3, 0x5D4, 0x5DB, 0x5DC, + 0x5DD, 0x5E8, 0x5EA, 0x2B, 0x5E9, 0x5C1, 0x5E9, 0x5C2, 0x5E9, 0x5BC, + 0x5C1, 0x5E9, 0x5BC, 0x5C2, 0x5D0, 0x5B7, 0x5D0, 0x5B8, 0x5D0, 0x5BC, + 0x5D1, 0x5BC, 0x5D2, 0x5BC, 0x5D3, 0x5BC, 0x5D4, 0x5BC, 0x5D5, 0x5BC, + 0x5D6, 0x5BC, 0x5D8, 0x5BC, 0x5D9, 0x5BC, 0x5DA, 0x5BC, 0x5DB, 0x5BC, + 0x5DC, 0x5BC, 0x5DE, 0x5BC, 0x5E0, 0x5BC, 0x5E1, 0x5BC, 0x5E3, 0x5BC, + 0x5E4, 0x5BC, 0x5E6, 0x5BC, 0x5E7, 0x5BC, 0x5E8, 0x5BC, 0x5E9, 0x5BC, + 0x5EA, 0x5BC, 0x5D5, 0x5B9, 0x5D1, 0x5BF, 0x5DB, 0x5BF, 0x5E4, 0x5BF, + 0x5D0, 0x5DC, 0x671, 0x671, 0x67B, 0x67B, 0x67B, 0x67B, 0x67E, 0x67E, + 0x67E, 0x67E, 0x680, 0x680, 0x680, 0x680, 0x67A, 0x67A, 0x67A, 0x67A, + 0x67F, 0x67F, 0x67F, 0x67F, 0x679, 0x679, 0x679, 0x679, 0x6A4, 0x6A4, + 0x6A4, 0x6A4, 0x6A6, 0x6A6, 0x6A6, 0x6A6, 0x684, 0x684, 0x684, 0x684, + 0x683, 0x683, 0x683, 0x683, 0x686, 0x686, 0x686, 0x686, 0x687, 0x687, + 0x687, 0x687, 0x68D, 0x68D, 0x68C, 0x68C, 0x68E, 0x68E, 0x688, 0x688, + 0x698, 0x698, 0x691, 0x691, 0x6A9, 0x6A9, 0x6A9, 0x6A9, 0x6AF, 0x6AF, + 0x6AF, 0x6AF, 0x6B3, 0x6B3, 0x6B3, 0x6B3, 0x6B1, 0x6B1, 0x6B1, 0x6B1, + 0x6BA, 0x6BA, 0x6BB, 0x6BB, 0x6BB, 0x6BB, 0x6C0, 0x6C0, 0x6C1, 0x6C1, + 0x6C1, 0x6C1, 0x6BE, 0x6BE, 0x6BE, 0x6BE, 0x6D2, 0x6D2, 0x6D3, 0x6D3, + 0x6AD, 0x6AD, 0x6AD, 0x6AD, 0x6C7, 0x6C7, 0x6C6, 0x6C6, 0x6C8, 0x6C8, + 0x6C7, 0x674, 0x6CB, 0x6CB, 0x6C5, 0x6C5, 0x6C9, 0x6C9, 0x6D0, 0x6D0, + 0x6D0, 0x6D0, 0x649, 0x649, 0x626, 0x627, 0x626, 0x627, 0x626, 0x6D5, + 0x626, 0x6D5, 0x626, 0x648, 0x626, 0x648, 0x626, 0x6C7, 0x626, 0x6C7, + 0x626, 0x6C6, 0x626, 0x6C6, 0x626, 0x6C8, 0x626, 0x6C8, 0x626, 0x6D0, + 0x626, 0x6D0, 0x626, 0x6D0, 0x626, 0x649, 0x626, 0x649, 0x626, 0x649, + 0x6CC, 0x6CC, 0x6CC, 0x6CC, 0x626, 0x62C, 0x626, 0x62D, 0x626, 0x645, + 0x626, 0x649, 0x626, 0x64A, 0x628, 0x62C, 0x628, 0x62D, 0x628, 0x62E, + 0x628, 0x645, 0x628, 0x649, 0x628, 0x64A, 0x62A, 0x62C, 0x62A, 0x62D, + 0x62A, 0x62E, 0x62A, 0x645, 0x62A, 0x649, 0x62A, 0x64A, 0x62B, 0x62C, + 0x62B, 0x645, 0x62B, 0x649, 0x62B, 0x64A, 0x62C, 0x62D, 0x62C, 0x645, + 0x62D, 0x62C, 0x62D, 0x645, 0x62E, 0x62C, 0x62E, 0x62D, 0x62E, 0x645, + 0x633, 0x62C, 0x633, 0x62D, 0x633, 0x62E, 0x633, 0x645, 0x635, 0x62D, + 0x635, 0x645, 0x636, 0x62C, 0x636, 0x62D, 0x636, 0x62E, 0x636, 0x645, + 0x637, 0x62D, 0x637, 0x645, 0x638, 0x645, 0x639, 0x62C, 0x639, 0x645, + 0x63A, 0x62C, 0x63A, 0x645, 0x641, 0x62C, 0x641, 0x62D, 0x641, 0x62E, + 0x641, 0x645, 0x641, 0x649, 0x641, 0x64A, 0x642, 0x62D, 0x642, 0x645, + 0x642, 0x649, 0x642, 0x64A, 0x643, 0x627, 0x643, 0x62C, 0x643, 0x62D, + 0x643, 0x62E, 0x643, 0x644, 0x643, 0x645, 0x643, 0x649, 0x643, 0x64A, + 0x644, 0x62C, 0x644, 0x62D, 0x644, 0x62E, 0x644, 0x645, 0x644, 0x649, + 0x644, 0x64A, 0x645, 0x62C, 0x645, 0x62D, 0x645, 0x62E, 0x645, 0x645, + 0x645, 0x649, 0x645, 0x64A, 0x646, 0x62C, 0x646, 0x62D, 0x646, 0x62E, + 0x646, 0x645, 0x646, 0x649, 0x646, 0x64A, 0x647, 0x62C, 0x647, 0x645, + 0x647, 0x649, 0x647, 0x64A, 0x64A, 0x62C, 0x64A, 0x62D, 0x64A, 0x62E, + 0x64A, 0x645, 0x64A, 0x649, 0x64A, 0x64A, 0x630, 0x670, 0x631, 0x670, + 0x649, 0x670, 0x20, 0x64C, 0x651, 0x20, 0x64D, 0x651, 0x20, 0x64E, + 0x651, 0x20, 0x64F, 0x651, 0x20, 0x650, 0x651, 0x20, 0x651, 0x670, + 0x626, 0x631, 0x626, 0x632, 0x626, 0x645, 0x626, 0x646, 0x626, 0x649, + 0x626, 0x64A, 0x628, 0x631, 0x628, 0x632, 0x628, 0x645, 0x628, 0x646, + 0x628, 0x649, 0x628, 0x64A, 0x62A, 0x631, 0x62A, 0x632, 0x62A, 0x645, + 0x62A, 0x646, 0x62A, 0x649, 0x62A, 0x64A, 0x62B, 0x631, 0x62B, 0x632, + 0x62B, 0x645, 0x62B, 0x646, 0x62B, 0x649, 0x62B, 0x64A, 0x641, 0x649, + 0x641, 0x64A, 0x642, 0x649, 0x642, 0x64A, 0x643, 0x627, 0x643, 0x644, + 0x643, 0x645, 0x643, 0x649, 0x643, 0x64A, 0x644, 0x645, 0x644, 0x649, + 0x644, 0x64A, 0x645, 0x627, 0x645, 0x645, 0x646, 0x631, 0x646, 0x632, + 0x646, 0x645, 0x646, 0x646, 0x646, 0x649, 0x646, 0x64A, 0x649, 0x670, + 0x64A, 0x631, 0x64A, 0x632, 0x64A, 0x645, 0x64A, 0x646, 0x64A, 0x649, + 0x64A, 0x64A, 0x626, 0x62C, 0x626, 0x62D, 0x626, 0x62E, 0x626, 0x645, + 0x626, 0x647, 0x628, 0x62C, 0x628, 0x62D, 0x628, 0x62E, 0x628, 0x645, + 0x628, 0x647, 0x62A, 0x62C, 0x62A, 0x62D, 0x62A, 0x62E, 0x62A, 0x645, + 0x62A, 0x647, 0x62B, 0x645, 0x62C, 0x62D, 0x62C, 0x645, 0x62D, 0x62C, + 0x62D, 0x645, 0x62E, 0x62C, 0x62E, 0x645, 0x633, 0x62C, 0x633, 0x62D, + 0x633, 0x62E, 0x633, 0x645, 0x635, 0x62D, 0x635, 0x62E, 0x635, 0x645, + 0x636, 0x62C, 0x636, 0x62D, 0x636, 0x62E, 0x636, 0x645, 0x637, 0x62D, + 0x638, 0x645, 0x639, 0x62C, 0x639, 0x645, 0x63A, 0x62C, 0x63A, 0x645, + 0x641, 0x62C, 0x641, 0x62D, 0x641, 0x62E, 0x641, 0x645, 0x642, 0x62D, + 0x642, 0x645, 0x643, 0x62C, 0x643, 0x62D, 0x643, 0x62E, 0x643, 0x644, + 0x643, 0x645, 0x644, 0x62C, 0x644, 0x62D, 0x644, 0x62E, 0x644, 0x645, + 0x644, 0x647, 0x645, 0x62C, 0x645, 0x62D, 0x645, 0x62E, 0x645, 0x645, + 0x646, 0x62C, 0x646, 0x62D, 0x646, 0x62E, 0x646, 0x645, 0x646, 0x647, + 0x647, 0x62C, 0x647, 0x645, 0x647, 0x670, 0x64A, 0x62C, 0x64A, 0x62D, + 0x64A, 0x62E, 0x64A, 0x645, 0x64A, 0x647, 0x626, 0x645, 0x626, 0x647, + 0x628, 0x645, 0x628, 0x647, 0x62A, 0x645, 0x62A, 0x647, 0x62B, 0x645, + 0x62B, 0x647, 0x633, 0x645, 0x633, 0x647, 0x634, 0x645, 0x634, 0x647, + 0x643, 0x644, 0x643, 0x645, 0x644, 0x645, 0x646, 0x645, 0x646, 0x647, + 0x64A, 0x645, 0x64A, 0x647, 0x640, 0x64E, 0x651, 0x640, 0x64F, 0x651, + 0x640, 0x650, 0x651, 0x637, 0x649, 0x637, 0x64A, 0x639, 0x649, 0x639, + 0x64A, 0x63A, 0x649, 0x63A, 0x64A, 0x633, 0x649, 0x633, 0x64A, 0x634, + 0x649, 0x634, 0x64A, 0x62D, 0x649, 0x62D, 0x64A, 0x62C, 0x649, 0x62C, + 0x64A, 0x62E, 0x649, 0x62E, 0x64A, 0x635, 0x649, 0x635, 0x64A, 0x636, + 0x649, 0x636, 0x64A, 0x634, 0x62C, 0x634, 0x62D, 0x634, 0x62E, 0x634, + 0x645, 0x634, 0x631, 0x633, 0x631, 0x635, 0x631, 0x636, 0x631, 0x637, + 0x649, 0x637, 0x64A, 0x639, 0x649, 0x639, 0x64A, 0x63A, 0x649, 0x63A, + 0x64A, 0x633, 0x649, 0x633, 0x64A, 0x634, 0x649, 0x634, 0x64A, 0x62D, + 0x649, 0x62D, 0x64A, 0x62C, 0x649, 0x62C, 0x64A, 0x62E, 0x649, 0x62E, + 0x64A, 0x635, 0x649, 0x635, 0x64A, 0x636, 0x649, 0x636, 0x64A, 0x634, + 0x62C, 0x634, 0x62D, 0x634, 0x62E, 0x634, 0x645, 0x634, 0x631, 0x633, + 0x631, 0x635, 0x631, 0x636, 0x631, 0x634, 0x62C, 0x634, 0x62D, 0x634, + 0x62E, 0x634, 0x645, 0x633, 0x647, 0x634, 0x647, 0x637, 0x645, 0x633, + 0x62C, 0x633, 0x62D, 0x633, 0x62E, 0x634, 0x62C, 0x634, 0x62D, 0x634, + 0x62E, 0x637, 0x645, 0x638, 0x645, 0x627, 0x64B, 0x627, 0x64B, 0x62A, + 0x62C, 0x645, 0x62A, 0x62D, 0x62C, 0x62A, 0x62D, 0x62C, 0x62A, 0x62D, + 0x645, 0x62A, 0x62E, 0x645, 0x62A, 0x645, 0x62C, 0x62A, 0x645, 0x62D, + 0x62A, 0x645, 0x62E, 0x62C, 0x645, 0x62D, 0x62C, 0x645, 0x62D, 0x62D, + 0x645, 0x64A, 0x62D, 0x645, 0x649, 0x633, 0x62D, 0x62C, 0x633, 0x62C, + 0x62D, 0x633, 0x62C, 0x649, 0x633, 0x645, 0x62D, 0x633, 0x645, 0x62D, + 0x633, 0x645, 0x62C, 0x633, 0x645, 0x645, 0x633, 0x645, 0x645, 0x635, + 0x62D, 0x62D, 0x635, 0x62D, 0x62D, 0x635, 0x645, 0x645, 0x634, 0x62D, + 0x645, 0x634, 0x62D, 0x645, 0x634, 0x62C, 0x64A, 0x634, 0x645, 0x62E, + 0x634, 0x645, 0x62E, 0x634, 0x645, 0x645, 0x634, 0x645, 0x645, 0x636, + 0x62D, 0x649, 0x636, 0x62E, 0x645, 0x636, 0x62E, 0x645, 0x637, 0x645, + 0x62D, 0x637, 0x645, 0x62D, 0x637, 0x645, 0x645, 0x637, 0x645, 0x64A, + 0x639, 0x62C, 0x645, 0x639, 0x645, 0x645, 0x639, 0x645, 0x645, 0x639, + 0x645, 0x649, 0x63A, 0x645, 0x645, 0x63A, 0x645, 0x64A, 0x63A, 0x645, + 0x649, 0x641, 0x62E, 0x645, 0x641, 0x62E, 0x645, 0x642, 0x645, 0x62D, + 0x642, 0x645, 0x645, 0x644, 0x62D, 0x645, 0x644, 0x62D, 0x64A, 0x644, + 0x62D, 0x649, 0x644, 0x62C, 0x62C, 0x644, 0x62C, 0x62C, 0x644, 0x62E, + 0x645, 0x644, 0x62E, 0x645, 0x644, 0x645, 0x62D, 0x644, 0x645, 0x62D, + 0x645, 0x62D, 0x62C, 0x645, 0x62D, 0x645, 0x645, 0x62D, 0x64A, 0x645, + 0x62C, 0x62D, 0x645, 0x62C, 0x645, 0x645, 0x62E, 0x62C, 0x645, 0x62E, + 0x645, 0x645, 0x62C, 0x62E, 0x647, 0x645, 0x62C, 0x647, 0x645, 0x645, + 0x646, 0x62D, 0x645, 0x646, 0x62D, 0x649, 0x646, 0x62C, 0x645, 0x646, + 0x62C, 0x645, 0x646, 0x62C, 0x649, 0x646, 0x645, 0x64A, 0x646, 0x645, + 0x649, 0x64A, 0x645, 0x645, 0x64A, 0x645, 0x645, 0x628, 0x62E, 0x64A, + 0x62A, 0x62C, 0x64A, 0x62A, 0x62C, 0x649, 0x62A, 0x62E, 0x64A, 0x62A, + 0x62E, 0x649, 0x62A, 0x645, 0x64A, 0x62A, 0x645, 0x649, 0x62C, 0x645, + 0x64A, 0x62C, 0x62D, 0x649, 0x62C, 0x645, 0x649, 0x633, 0x62E, 0x649, + 0x635, 0x62D, 0x64A, 0x634, 0x62D, 0x64A, 0x636, 0x62D, 0x64A, 0x644, + 0x62C, 0x64A, 0x644, 0x645, 0x64A, 0x64A, 0x62D, 0x64A, 0x64A, 0x62C, + 0x64A, 0x64A, 0x645, 0x64A, 0x645, 0x645, 0x64A, 0x642, 0x645, 0x64A, + 0x646, 0x62D, 0x64A, 0x642, 0x645, 0x62D, 0x644, 0x62D, 0x645, 0x639, + 0x645, 0x64A, 0x643, 0x645, 0x64A, 0x646, 0x62C, 0x62D, 0x645, 0x62E, + 0x64A, 0x644, 0x62C, 0x645, 0x643, 0x645, 0x645, 0x644, 0x62C, 0x645, + 0x646, 0x62C, 0x62D, 0x62C, 0x62D, 0x64A, 0x62D, 0x62C, 0x64A, 0x645, + 0x62C, 0x64A, 0x641, 0x645, 0x64A, 0x628, 0x62D, 0x64A, 0x643, 0x645, + 0x645, 0x639, 0x62C, 0x645, 0x635, 0x645, 0x645, 0x633, 0x62E, 0x64A, + 0x646, 0x62C, 0x64A, 0x635, 0x644, 0x6D2, 0x642, 0x644, 0x6D2, 0x627, + 0x644, 0x644, 0x647, 0x627, 0x643, 0x628, 0x631, 0x645, 0x62D, 0x645, + 0x62F, 0x635, 0x644, 0x639, 0x645, 0x631, 0x633, 0x648, 0x644, 0x639, + 0x644, 0x64A, 0x647, 0x648, 0x633, 0x644, 0x645, 0x635, 0x644, 0x649, + 0x635, 0x644, 0x649, 0x20, 0x627, 0x644, 0x644, 0x647, 0x20, 0x639, + 0x644, 0x64A, 0x647, 0x20, 0x648, 0x633, 0x644, 0x645, 0x62C, 0x644, + 0x20, 0x62C, 0x644, 0x627, 0x644, 0x647, 0x631, 0x6CC, 0x627, 0x644, + 0x2C, 0x3001, 0x3002, 0x3A, 0x3B, 0x21, 0x3F, 0x3016, 0x3017, 0x2E, + 0x2E, 0x2E, 0x2E, 0x2E, 0x2014, 0x2013, 0x5F, 0x5F, 0x28, 0x29, + 0x7B, 0x7D, 0x3014, 0x3015, 0x3010, 0x3011, 0x300A, 0x300B, 0x3008, 0x3009, + 0x300C, 0x300D, 0x300E, 0x300F, 0x5B, 0x5D, 0x20, 0x305, 0x20, 0x305, + 0x20, 0x305, 0x20, 0x305, 0x5F, 0x5F, 0x5F, 0x2C, 0x3001, 0x2E, + 0x3B, 0x3A, 0x3F, 0x21, 0x2014, 0x28, 0x29, 0x7B, 0x7D, 0x3014, + 0x3015, 0x23, 0x26, 0x2A, 0x2B, 0x2D, 0x3C, 0x3E, 0x3D, 0x5C, + 0x24, 0x25, 0x40, 0x20, 0x64B, 0x640, 0x64B, 0x20, 0x64C, 0x20, + 0x64D, 0x20, 0x64E, 0x640, 0x64E, 0x20, 0x64F, 0x640, 0x64F, 0x20, + 0x650, 0x640, 0x650, 0x20, 0x651, 0x640, 0x651, 0x20, 0x652, 0x640, + 0x652, 0x621, 0x622, 0x622, 0x623, 0x623, 0x624, 0x624, 0x625, 0x625, + 0x626, 0x626, 0x626, 0x626, 0x627, 0x627, 0x628, 0x628, 0x628, 0x628, + 0x629, 0x629, 0x62A, 0x62A, 0x62A, 0x62A, 0x62B, 0x62B, 0x62B, 0x62B, + 0x62C, 0x62C, 0x62C, 0x62C, 0x62D, 0x62D, 0x62D, 0x62D, 0x62E, 0x62E, + 0x62E, 0x62E, 0x62F, 0x62F, 0x630, 0x630, 0x631, 0x631, 0x632, 0x632, + 0x633, 0x633, 0x633, 0x633, 0x634, 0x634, 0x634, 0x634, 0x635, 0x635, + 0x635, 0x635, 0x636, 0x636, 0x636, 0x636, 0x637, 0x637, 0x637, 0x637, + 0x638, 0x638, 0x638, 0x638, 0x639, 0x639, 0x639, 0x639, 0x63A, 0x63A, + 0x63A, 0x63A, 0x641, 0x641, 0x641, 0x641, 0x642, 0x642, 0x642, 0x642, + 0x643, 0x643, 0x643, 0x643, 0x644, 0x644, 0x644, 0x644, 0x645, 0x645, + 0x645, 0x645, 0x646, 0x646, 0x646, 0x646, 0x647, 0x647, 0x647, 0x647, + 0x648, 0x648, 0x649, 0x649, 0x64A, 0x64A, 0x64A, 0x64A, 0x644, 0x622, + 0x644, 0x622, 0x644, 0x623, 0x644, 0x623, 0x644, 0x625, 0x644, 0x625, + 0x644, 0x627, 0x644, 0x627, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, + 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, + 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, + 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 0x60, 0x61, 0x62, + 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, + 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, + 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x2985, 0x2986, + 0x3002, 0x300C, 0x300D, 0x3001, 0x30FB, 0x30F2, 0x30A1, 0x30A3, 0x30A5, 0x30A7, + 0x30A9, 0x30E3, 0x30E5, 0x30E7, 0x30C3, 0x30FC, 0x30A2, 0x30A4, 0x30A6, 0x30A8, + 0x30AA, 0x30AB, 0x30AD, 0x30AF, 0x30B1, 0x30B3, 0x30B5, 0x30B7, 0x30B9, 0x30BB, + 0x30BD, 0x30BF, 0x30C1, 0x30C4, 0x30C6, 0x30C8, 0x30CA, 0x30CB, 0x30CC, 0x30CD, + 0x30CE, 0x30CF, 0x30D2, 0x30D5, 0x30D8, 0x30DB, 0x30DE, 0x30DF, 0x30E0, 0x30E1, + 0x30E2, 0x30E4, 0x30E6, 0x30E8, 0x30E9, 0x30EA, 0x30EB, 0x30EC, 0x30ED, 0x30EF, + 0x30F3, 0x3099, 0x309A, 0x1160, 0x1100, 0x1101, 0x11AA, 0x1102, 0x11AC, 0x11AD, + 0x1103, 0x1104, 0x1105, 0x11B0, 0x11B1, 0x11B2, 0x11B3, 0x11B4, 0x11B5, 0x111A, + 0x1106, 0x1107, 0x1108, 0x1121, 0x1109, 0x110A, 0x110B, 0x110C, 0x110D, 0x110E, + 0x110F, 0x1110, 0x1111, 0x1112, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, + 0x1167, 0x1168, 0x1169, 0x116A, 0x116B, 0x116C, 0x116D, 0x116E, 0x116F, 0x1170, + 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0xA2, 0xA3, 0xAC, 0x20, 0x304, + 0xA6, 0xA5, 0x20A9, 0x2502, 0x2190, 0x2191, 0x2192, 0x2193, 0x25A0, 0x25CB, + 0x2D0, 0x2D1, 0xE6, 0x299, 0x253, 0x2A3, 0xAB66, 0x2A5, 0x2A4, 0x256, + 0x257, 0x1D91, 0x258, 0x25E, 0x2A9, 0x264, 0x262, 0x260, 0x29B, 0x127, + 0x29C, 0x267, 0x284, 0x2AA, 0x2AB, 0x26C, 0x1DF04, 0xA78E, 0x26E, 0x1DF05, + 0x28E, 0x1DF06, 0xF8, 0x276, 0x277, 0x71, 0x27A, 0x1DF08, 0x27D, 0x27E, + 0x280, 0x2A8, 0x2A6, 0xAB67, 0x2A7, 0x288, 0x2C71, 0x28F, 0x2A1, 0x2A2, + 0x298, 0x1C0, 0x1C1, 0x1C2, 0x1DF0A, 0x1DF1E, 0x41, 0x42, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, + 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x1D157, 0x1D165, 0x1D158, 0x1D165, 0x1D158, 0x1D165, 0x1D16E, 0x1D158, + 0x1D165, 0x1D16F, 0x1D158, 0x1D165, 0x1D170, 0x1D158, 0x1D165, 0x1D171, 0x1D158, 0x1D165, + 0x1D172, 0x1D1B9, 0x1D165, 0x1D1BA, 0x1D165, 0x1D1B9, 0x1D165, 0x1D16E, 0x1D1BA, 0x1D165, + 0x1D16E, 0x1D1B9, 0x1D165, 0x1D16F, 0x1D1BA, 0x1D165, 0x1D16F, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, + 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, + 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7A, 0x41, 0x43, 0x44, 0x47, 0x4A, 0x4B, 0x4E, 0x4F, + 0x50, 0x51, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, + 0x61, 0x62, 0x63, 0x64, 0x66, 0x68, 0x69, 0x6A, 0x6B, 0x6C, + 0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x44, 0x45, 0x46, + 0x47, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x61, 0x62, 0x63, 0x64, + 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, + 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7A, 0x41, 0x42, 0x44, 0x45, 0x46, 0x47, 0x49, 0x4A, + 0x4B, 0x4C, 0x4D, 0x4F, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, + 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, + 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, + 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, + 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, + 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, + 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, + 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x131, + 0x237, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, + 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, + 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, + 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, + 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, + 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, + 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, + 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, + 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, + 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, + 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, + 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, + 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, + 0x39E, 0x39F, 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, + 0x3A8, 0x3A9, 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, + 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, + 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, + 0x3B8, 0x3BA, 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, 0x394, 0x395, + 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, + 0x3A0, 0x3A1, 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, + 0x2207, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, + 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, + 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, + 0x3C6, 0x3C1, 0x3C0, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, + 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, + 0x398, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x2207, 0x3B1, + 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, + 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, + 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x2202, 0x3B5, 0x3B8, 0x3BA, 0x3C6, 0x3C1, + 0x3C0, 0x3DC, 0x3DD, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x430, 0x431, 0x432, 0x433, 0x434, 0x435, 0x436, + 0x437, 0x438, 0x43A, 0x43B, 0x43C, 0x43E, 0x43F, 0x440, 0x441, 0x442, + 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x44B, 0x44D, 0x44E, 0xA689, + 0x4D9, 0x456, 0x458, 0x4E9, 0x4AF, 0x4CF, 0x430, 0x431, 0x432, 0x433, + 0x434, 0x435, 0x436, 0x437, 0x438, 0x43A, 0x43B, 0x43E, 0x43F, 0x441, + 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x44A, 0x44B, 0x491, 0x456, + 0x455, 0x45F, 0x4AB, 0xA651, 0x4B1, 0x627, 0x628, 0x62C, 0x62F, 0x648, + 0x632, 0x62D, 0x637, 0x64A, 0x643, 0x644, 0x645, 0x646, 0x633, 0x639, + 0x641, 0x635, 0x642, 0x631, 0x634, 0x62A, 0x62B, 0x62E, 0x630, 0x636, + 0x638, 0x63A, 0x66E, 0x6BA, 0x6A1, 0x66F, 0x628, 0x62C, 0x647, 0x62D, + 0x64A, 0x643, 0x644, 0x645, 0x646, 0x633, 0x639, 0x641, 0x635, 0x642, + 0x634, 0x62A, 0x62B, 0x62E, 0x636, 0x63A, 0x62C, 0x62D, 0x64A, 0x644, + 0x646, 0x633, 0x639, 0x635, 0x642, 0x634, 0x62E, 0x636, 0x63A, 0x6BA, + 0x66F, 0x628, 0x62C, 0x647, 0x62D, 0x637, 0x64A, 0x643, 0x645, 0x646, + 0x633, 0x639, 0x641, 0x635, 0x642, 0x634, 0x62A, 0x62B, 0x62E, 0x636, + 0x638, 0x63A, 0x66E, 0x6A1, 0x627, 0x628, 0x62C, 0x62F, 0x647, 0x648, + 0x632, 0x62D, 0x637, 0x64A, 0x644, 0x645, 0x646, 0x633, 0x639, 0x641, + 0x635, 0x642, 0x631, 0x634, 0x62A, 0x62B, 0x62E, 0x630, 0x636, 0x638, + 0x63A, 0x628, 0x62C, 0x62F, 0x648, 0x632, 0x62D, 0x637, 0x64A, 0x644, + 0x645, 0x646, 0x633, 0x639, 0x641, 0x635, 0x642, 0x631, 0x634, 0x62A, + 0x62B, 0x62E, 0x630, 0x636, 0x638, 0x63A, 0x30, 0x2E, 0x30, 0x2C, + 0x31, 0x2C, 0x32, 0x2C, 0x33, 0x2C, 0x34, 0x2C, 0x35, 0x2C, + 0x36, 0x2C, 0x37, 0x2C, 0x38, 0x2C, 0x39, 0x2C, 0x28, 0x41, + 0x29, 0x28, 0x42, 0x29, 0x28, 0x43, 0x29, 0x28, 0x44, 0x29, + 0x28, 0x45, 0x29, 0x28, 0x46, 0x29, 0x28, 0x47, 0x29, 0x28, + 0x48, 0x29, 0x28, 0x49, 0x29, 0x28, 0x4A, 0x29, 0x28, 0x4B, + 0x29, 0x28, 0x4C, 0x29, 0x28, 0x4D, 0x29, 0x28, 0x4E, 0x29, + 0x28, 0x4F, 0x29, 0x28, 0x50, 0x29, 0x28, 0x51, 0x29, 0x28, + 0x52, 0x29, 0x28, 0x53, 0x29, 0x28, 0x54, 0x29, 0x28, 0x55, + 0x29, 0x28, 0x56, 0x29, 0x28, 0x57, 0x29, 0x28, 0x58, 0x29, + 0x28, 0x59, 0x29, 0x28, 0x5A, 0x29, 0x3014, 0x53, 0x3015, 0x43, + 0x52, 0x43, 0x44, 0x57, 0x5A, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5A, 0x48, 0x56, 0x4D, 0x56, 0x53, 0x44, 0x53, 0x53, 0x50, + 0x50, 0x56, 0x57, 0x43, 0x4D, 0x43, 0x4D, 0x44, 0x4D, 0x52, + 0x44, 0x4A, 0x307B, 0x304B, 0x30B3, 0x30B3, 0x30B5, 0x624B, 0x5B57, 0x53CC, + 0x30C7, 0x4E8C, 0x591A, 0x89E3, 0x5929, 0x4EA4, 0x6620, 0x7121, 0x6599, 0x524D, + 0x5F8C, 0x518D, 0x65B0, 0x521D, 0x7D42, 0x751F, 0x8CA9, 0x58F0, 0x5439, 0x6F14, + 0x6295, 0x6355, 0x4E00, 0x4E09, 0x904A, 0x5DE6, 0x4E2D, 0x53F3, 0x6307, 0x8D70, + 0x6253, 0x7981, 0x7A7A, 0x5408, 0x6E80, 0x6709, 0x6708, 0x7533, 0x5272, 0x55B6, + 0x914D, 0x3014, 0x672C, 0x3015, 0x3014, 0x4E09, 0x3015, 0x3014, 0x4E8C, 0x3015, + 0x3014, 0x5B89, 0x3015, 0x3014, 0x70B9, 0x3015, 0x3014, 0x6253, 0x3015, 0x3014, + 0x76D7, 0x3015, 0x3014, 0x52DD, 0x3015, 0x3014, 0x6557, 0x3015, 0x5F97, 0x53EF, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, + 0x4E3D, 0x4E38, 0x4E41, 0x20122, 0x4F60, 0x4FAE, 0x4FBB, 0x5002, 0x507A, 0x5099, + 0x50E7, 0x50CF, 0x349E, 0x2063A, 0x514D, 0x5154, 0x5164, 0x5177, 0x2051C, 0x34B9, + 0x5167, 0x518D, 0x2054B, 0x5197, 0x51A4, 0x4ECC, 0x51AC, 0x51B5, 0x291DF, 0x51F5, + 0x5203, 0x34DF, 0x523B, 0x5246, 0x5272, 0x5277, 0x3515, 0x52C7, 0x52C9, 0x52E4, + 0x52FA, 0x5305, 0x5306, 0x5317, 0x5349, 0x5351, 0x535A, 0x5373, 0x537D, 0x537F, + 0x537F, 0x537F, 0x20A2C, 0x7070, 0x53CA, 0x53DF, 0x20B63, 0x53EB, 0x53F1, 0x5406, + 0x549E, 0x5438, 0x5448, 0x5468, 0x54A2, 0x54F6, 0x5510, 0x5553, 0x5563, 0x5584, + 0x5584, 0x5599, 0x55AB, 0x55B3, 0x55C2, 0x5716, 0x5606, 0x5717, 0x5651, 0x5674, + 0x5207, 0x58EE, 0x57CE, 0x57F4, 0x580D, 0x578B, 0x5832, 0x5831, 0x58AC, 0x214E4, + 0x58F2, 0x58F7, 0x5906, 0x591A, 0x5922, 0x5962, 0x216A8, 0x216EA, 0x59EC, 0x5A1B, + 0x5A27, 0x59D8, 0x5A66, 0x36EE, 0x36FC, 0x5B08, 0x5B3E, 0x5B3E, 0x219C8, 0x5BC3, + 0x5BD8, 0x5BE7, 0x5BF3, 0x21B18, 0x5BFF, 0x5C06, 0x5F53, 0x5C22, 0x3781, 0x5C60, + 0x5C6E, 0x5CC0, 0x5C8D, 0x21DE4, 0x5D43, 0x21DE6, 0x5D6E, 0x5D6B, 0x5D7C, 0x5DE1, + 0x5DE2, 0x382F, 0x5DFD, 0x5E28, 0x5E3D, 0x5E69, 0x3862, 0x22183, 0x387C, 0x5EB0, + 0x5EB3, 0x5EB6, 0x5ECA, 0x2A392, 0x5EFE, 0x22331, 0x22331, 0x8201, 0x5F22, 0x5F22, + 0x38C7, 0x232B8, 0x261DA, 0x5F62, 0x5F6B, 0x38E3, 0x5F9A, 0x5FCD, 0x5FD7, 0x5FF9, + 0x6081, 0x393A, 0x391C, 0x6094, 0x226D4, 0x60C7, 0x6148, 0x614C, 0x614E, 0x614C, + 0x617A, 0x618E, 0x61B2, 0x61A4, 0x61AF, 0x61DE, 0x61F2, 0x61F6, 0x6210, 0x621B, + 0x625D, 0x62B1, 0x62D4, 0x6350, 0x22B0C, 0x633D, 0x62FC, 0x6368, 0x6383, 0x63E4, + 0x22BF1, 0x6422, 0x63C5, 0x63A9, 0x3A2E, 0x6469, 0x647E, 0x649D, 0x6477, 0x3A6C, + 0x654F, 0x656C, 0x2300A, 0x65E3, 0x66F8, 0x6649, 0x3B19, 0x6691, 0x3B08, 0x3AE4, + 0x5192, 0x5195, 0x6700, 0x669C, 0x80AD, 0x43D9, 0x6717, 0x671B, 0x6721, 0x675E, + 0x6753, 0x233C3, 0x3B49, 0x67FA, 0x6785, 0x6852, 0x6885, 0x2346D, 0x688E, 0x681F, + 0x6914, 0x3B9D, 0x6942, 0x69A3, 0x69EA, 0x6AA8, 0x236A3, 0x6ADB, 0x3C18, 0x6B21, + 0x238A7, 0x6B54, 0x3C4E, 0x6B72, 0x6B9F, 0x6BBA, 0x6BBB, 0x23A8D, 0x21D0B, 0x23AFA, + 0x6C4E, 0x23CBC, 0x6CBF, 0x6CCD, 0x6C67, 0x6D16, 0x6D3E, 0x6D77, 0x6D41, 0x6D69, + 0x6D78, 0x6D85, 0x23D1E, 0x6D34, 0x6E2F, 0x6E6E, 0x3D33, 0x6ECB, 0x6EC7, 0x23ED1, + 0x6DF9, 0x6F6E, 0x23F5E, 0x23F8E, 0x6FC6, 0x7039, 0x701E, 0x701B, 0x3D96, 0x704A, + 0x707D, 0x7077, 0x70AD, 0x20525, 0x7145, 0x24263, 0x719C, 0x243AB, 0x7228, 0x7235, + 0x7250, 0x24608, 0x7280, 0x7295, 0x24735, 0x24814, 0x737A, 0x738B, 0x3EAC, 0x73A5, + 0x3EB8, 0x3EB8, 0x7447, 0x745C, 0x7471, 0x7485, 0x74CA, 0x3F1B, 0x7524, 0x24C36, + 0x753E, 0x24C92, 0x7570, 0x2219F, 0x7610, 0x24FA1, 0x24FB8, 0x25044, 0x3FFC, 0x4008, + 0x76F4, 0x250F3, 0x250F2, 0x25119, 0x25133, 0x771E, 0x771F, 0x771F, 0x774A, 0x4039, + 0x778B, 0x4046, 0x4096, 0x2541D, 0x784E, 0x788C, 0x78CC, 0x40E3, 0x25626, 0x7956, + 0x2569A, 0x256C5, 0x798F, 0x79EB, 0x412F, 0x7A40, 0x7A4A, 0x7A4F, 0x2597C, 0x25AA7, + 0x25AA7, 0x7AEE, 0x4202, 0x25BAB, 0x7BC6, 0x7BC9, 0x4227, 0x25C80, 0x7CD2, 0x42A0, + 0x7CE8, 0x7CE3, 0x7D00, 0x25F86, 0x7D63, 0x4301, 0x7DC7, 0x7E02, 0x7E45, 0x4334, + 0x26228, 0x26247, 0x4359, 0x262D9, 0x7F7A, 0x2633E, 0x7F95, 0x7FFA, 0x8005, 0x264DA, + 0x26523, 0x8060, 0x265A8, 0x8070, 0x2335F, 0x43D5, 0x80B2, 0x8103, 0x440B, 0x813E, + 0x5AB5, 0x267A7, 0x267B5, 0x23393, 0x2339C, 0x8201, 0x8204, 0x8F9E, 0x446B, 0x8291, + 0x828B, 0x829D, 0x52B3, 0x82B1, 0x82B3, 0x82BD, 0x82E6, 0x26B3C, 0x82E5, 0x831D, + 0x8363, 0x83AD, 0x8323, 0x83BD, 0x83E7, 0x8457, 0x8353, 0x83CA, 0x83CC, 0x83DC, + 0x26C36, 0x26D6B, 0x26CD5, 0x452B, 0x84F1, 0x84F3, 0x8516, 0x273CA, 0x8564, 0x26F2C, + 0x455D, 0x4561, 0x26FB1, 0x270D2, 0x456B, 0x8650, 0x865C, 0x8667, 0x8669, 0x86A9, + 0x8688, 0x870E, 0x86E2, 0x8779, 0x8728, 0x876B, 0x8786, 0x45D7, 0x87E1, 0x8801, + 0x45F9, 0x8860, 0x8863, 0x27667, 0x88D7, 0x88DE, 0x4635, 0x88FA, 0x34BB, 0x278AE, + 0x27966, 0x46BE, 0x46C7, 0x8AA0, 0x8AED, 0x8B8A, 0x8C55, 0x27CA8, 0x8CAB, 0x8CC1, + 0x8D1B, 0x8D77, 0x27F2F, 0x20804, 0x8DCB, 0x8DBC, 0x8DF0, 0x208DE, 0x8ED4, 0x8F38, + 0x285D2, 0x285ED, 0x9094, 0x90F1, 0x9111, 0x2872E, 0x911B, 0x9238, 0x92D7, 0x92D8, + 0x927C, 0x93F9, 0x9415, 0x28BFA, 0x958B, 0x4995, 0x95B7, 0x28D77, 0x49E6, 0x96C3, + 0x5DB2, 0x9723, 0x29145, 0x2921A, 0x4A6E, 0x4A76, 0x97E0, 0x2940A, 0x4AB2, 0x29496, + 0x980B, 0x980B, 0x9829, 0x295B6, 0x98E2, 0x4B33, 0x9929, 0x99A7, 0x99C2, 0x99FE, + 0x4BCE, 0x29B30, 0x9B12, 0x9C40, 0x9CFD, 0x4CCE, 0x4CED, 0x9D67, 0x2A0CE, 0x4CF8, + 0x2A105, 0x2A20E, 0x2A291, 0x9EBB, 0x4D56, 0x9EF9, 0x9EFE, 0x9F05, 0x9F0F, 0x9F16, + 0x9F3B, 0x2A600, +}; + +const pdf_ucomp_t PDF_UCOMP[] = { + {0x3C, 0x338, 0x226E}, {0x3D, 0x338, 0x2260}, {0x3E, 0x338, 0x226F}, + {0x41, 0x300, 0xC0}, {0x41, 0x301, 0xC1}, {0x41, 0x302, 0xC2}, + {0x41, 0x303, 0xC3}, {0x41, 0x304, 0x100}, {0x41, 0x306, 0x102}, + {0x41, 0x307, 0x226}, {0x41, 0x308, 0xC4}, {0x41, 0x309, 0x1EA2}, + {0x41, 0x30A, 0xC5}, {0x41, 0x30C, 0x1CD}, {0x41, 0x30F, 0x200}, + {0x41, 0x311, 0x202}, {0x41, 0x323, 0x1EA0}, {0x41, 0x325, 0x1E00}, + {0x41, 0x328, 0x104}, {0x42, 0x307, 0x1E02}, {0x42, 0x323, 0x1E04}, + {0x42, 0x331, 0x1E06}, {0x43, 0x301, 0x106}, {0x43, 0x302, 0x108}, + {0x43, 0x307, 0x10A}, {0x43, 0x30C, 0x10C}, {0x43, 0x327, 0xC7}, + {0x44, 0x307, 0x1E0A}, {0x44, 0x30C, 0x10E}, {0x44, 0x323, 0x1E0C}, + {0x44, 0x327, 0x1E10}, {0x44, 0x32D, 0x1E12}, {0x44, 0x331, 0x1E0E}, + {0x45, 0x300, 0xC8}, {0x45, 0x301, 0xC9}, {0x45, 0x302, 0xCA}, + {0x45, 0x303, 0x1EBC}, {0x45, 0x304, 0x112}, {0x45, 0x306, 0x114}, + {0x45, 0x307, 0x116}, {0x45, 0x308, 0xCB}, {0x45, 0x309, 0x1EBA}, + {0x45, 0x30C, 0x11A}, {0x45, 0x30F, 0x204}, {0x45, 0x311, 0x206}, + {0x45, 0x323, 0x1EB8}, {0x45, 0x327, 0x228}, {0x45, 0x328, 0x118}, + {0x45, 0x32D, 0x1E18}, {0x45, 0x330, 0x1E1A}, {0x46, 0x307, 0x1E1E}, + {0x47, 0x301, 0x1F4}, {0x47, 0x302, 0x11C}, {0x47, 0x304, 0x1E20}, + {0x47, 0x306, 0x11E}, {0x47, 0x307, 0x120}, {0x47, 0x30C, 0x1E6}, + {0x47, 0x327, 0x122}, {0x48, 0x302, 0x124}, {0x48, 0x307, 0x1E22}, + {0x48, 0x308, 0x1E26}, {0x48, 0x30C, 0x21E}, {0x48, 0x323, 0x1E24}, + {0x48, 0x327, 0x1E28}, {0x48, 0x32E, 0x1E2A}, {0x49, 0x300, 0xCC}, + {0x49, 0x301, 0xCD}, {0x49, 0x302, 0xCE}, {0x49, 0x303, 0x128}, + {0x49, 0x304, 0x12A}, {0x49, 0x306, 0x12C}, {0x49, 0x307, 0x130}, + {0x49, 0x308, 0xCF}, {0x49, 0x309, 0x1EC8}, {0x49, 0x30C, 0x1CF}, + {0x49, 0x30F, 0x208}, {0x49, 0x311, 0x20A}, {0x49, 0x323, 0x1ECA}, + {0x49, 0x328, 0x12E}, {0x49, 0x330, 0x1E2C}, {0x4A, 0x302, 0x134}, + {0x4B, 0x301, 0x1E30}, {0x4B, 0x30C, 0x1E8}, {0x4B, 0x323, 0x1E32}, + {0x4B, 0x327, 0x136}, {0x4B, 0x331, 0x1E34}, {0x4C, 0x301, 0x139}, + {0x4C, 0x30C, 0x13D}, {0x4C, 0x323, 0x1E36}, {0x4C, 0x327, 0x13B}, + {0x4C, 0x32D, 0x1E3C}, {0x4C, 0x331, 0x1E3A}, {0x4D, 0x301, 0x1E3E}, + {0x4D, 0x307, 0x1E40}, {0x4D, 0x323, 0x1E42}, {0x4E, 0x300, 0x1F8}, + {0x4E, 0x301, 0x143}, {0x4E, 0x303, 0xD1}, {0x4E, 0x307, 0x1E44}, + {0x4E, 0x30C, 0x147}, {0x4E, 0x323, 0x1E46}, {0x4E, 0x327, 0x145}, + {0x4E, 0x32D, 0x1E4A}, {0x4E, 0x331, 0x1E48}, {0x4F, 0x300, 0xD2}, + {0x4F, 0x301, 0xD3}, {0x4F, 0x302, 0xD4}, {0x4F, 0x303, 0xD5}, + {0x4F, 0x304, 0x14C}, {0x4F, 0x306, 0x14E}, {0x4F, 0x307, 0x22E}, + {0x4F, 0x308, 0xD6}, {0x4F, 0x309, 0x1ECE}, {0x4F, 0x30B, 0x150}, + {0x4F, 0x30C, 0x1D1}, {0x4F, 0x30F, 0x20C}, {0x4F, 0x311, 0x20E}, + {0x4F, 0x31B, 0x1A0}, {0x4F, 0x323, 0x1ECC}, {0x4F, 0x328, 0x1EA}, + {0x50, 0x301, 0x1E54}, {0x50, 0x307, 0x1E56}, {0x52, 0x301, 0x154}, + {0x52, 0x307, 0x1E58}, {0x52, 0x30C, 0x158}, {0x52, 0x30F, 0x210}, + {0x52, 0x311, 0x212}, {0x52, 0x323, 0x1E5A}, {0x52, 0x327, 0x156}, + {0x52, 0x331, 0x1E5E}, {0x53, 0x301, 0x15A}, {0x53, 0x302, 0x15C}, + {0x53, 0x307, 0x1E60}, {0x53, 0x30C, 0x160}, {0x53, 0x323, 0x1E62}, + {0x53, 0x326, 0x218}, {0x53, 0x327, 0x15E}, {0x54, 0x307, 0x1E6A}, + {0x54, 0x30C, 0x164}, {0x54, 0x323, 0x1E6C}, {0x54, 0x326, 0x21A}, + {0x54, 0x327, 0x162}, {0x54, 0x32D, 0x1E70}, {0x54, 0x331, 0x1E6E}, + {0x55, 0x300, 0xD9}, {0x55, 0x301, 0xDA}, {0x55, 0x302, 0xDB}, + {0x55, 0x303, 0x168}, {0x55, 0x304, 0x16A}, {0x55, 0x306, 0x16C}, + {0x55, 0x308, 0xDC}, {0x55, 0x309, 0x1EE6}, {0x55, 0x30A, 0x16E}, + {0x55, 0x30B, 0x170}, {0x55, 0x30C, 0x1D3}, {0x55, 0x30F, 0x214}, + {0x55, 0x311, 0x216}, {0x55, 0x31B, 0x1AF}, {0x55, 0x323, 0x1EE4}, + {0x55, 0x324, 0x1E72}, {0x55, 0x328, 0x172}, {0x55, 0x32D, 0x1E76}, + {0x55, 0x330, 0x1E74}, {0x56, 0x303, 0x1E7C}, {0x56, 0x323, 0x1E7E}, + {0x57, 0x300, 0x1E80}, {0x57, 0x301, 0x1E82}, {0x57, 0x302, 0x174}, + {0x57, 0x307, 0x1E86}, {0x57, 0x308, 0x1E84}, {0x57, 0x323, 0x1E88}, + {0x58, 0x307, 0x1E8A}, {0x58, 0x308, 0x1E8C}, {0x59, 0x300, 0x1EF2}, + {0x59, 0x301, 0xDD}, {0x59, 0x302, 0x176}, {0x59, 0x303, 0x1EF8}, + {0x59, 0x304, 0x232}, {0x59, 0x307, 0x1E8E}, {0x59, 0x308, 0x178}, + {0x59, 0x309, 0x1EF6}, {0x59, 0x323, 0x1EF4}, {0x5A, 0x301, 0x179}, + {0x5A, 0x302, 0x1E90}, {0x5A, 0x307, 0x17B}, {0x5A, 0x30C, 0x17D}, + {0x5A, 0x323, 0x1E92}, {0x5A, 0x331, 0x1E94}, {0x61, 0x300, 0xE0}, + {0x61, 0x301, 0xE1}, {0x61, 0x302, 0xE2}, {0x61, 0x303, 0xE3}, + {0x61, 0x304, 0x101}, {0x61, 0x306, 0x103}, {0x61, 0x307, 0x227}, + {0x61, 0x308, 0xE4}, {0x61, 0x309, 0x1EA3}, {0x61, 0x30A, 0xE5}, + {0x61, 0x30C, 0x1CE}, {0x61, 0x30F, 0x201}, {0x61, 0x311, 0x203}, + {0x61, 0x323, 0x1EA1}, {0x61, 0x325, 0x1E01}, {0x61, 0x328, 0x105}, + {0x62, 0x307, 0x1E03}, {0x62, 0x323, 0x1E05}, {0x62, 0x331, 0x1E07}, + {0x63, 0x301, 0x107}, {0x63, 0x302, 0x109}, {0x63, 0x307, 0x10B}, + {0x63, 0x30C, 0x10D}, {0x63, 0x327, 0xE7}, {0x64, 0x307, 0x1E0B}, + {0x64, 0x30C, 0x10F}, {0x64, 0x323, 0x1E0D}, {0x64, 0x327, 0x1E11}, + {0x64, 0x32D, 0x1E13}, {0x64, 0x331, 0x1E0F}, {0x65, 0x300, 0xE8}, + {0x65, 0x301, 0xE9}, {0x65, 0x302, 0xEA}, {0x65, 0x303, 0x1EBD}, + {0x65, 0x304, 0x113}, {0x65, 0x306, 0x115}, {0x65, 0x307, 0x117}, + {0x65, 0x308, 0xEB}, {0x65, 0x309, 0x1EBB}, {0x65, 0x30C, 0x11B}, + {0x65, 0x30F, 0x205}, {0x65, 0x311, 0x207}, {0x65, 0x323, 0x1EB9}, + {0x65, 0x327, 0x229}, {0x65, 0x328, 0x119}, {0x65, 0x32D, 0x1E19}, + {0x65, 0x330, 0x1E1B}, {0x66, 0x307, 0x1E1F}, {0x67, 0x301, 0x1F5}, + {0x67, 0x302, 0x11D}, {0x67, 0x304, 0x1E21}, {0x67, 0x306, 0x11F}, + {0x67, 0x307, 0x121}, {0x67, 0x30C, 0x1E7}, {0x67, 0x327, 0x123}, + {0x68, 0x302, 0x125}, {0x68, 0x307, 0x1E23}, {0x68, 0x308, 0x1E27}, + {0x68, 0x30C, 0x21F}, {0x68, 0x323, 0x1E25}, {0x68, 0x327, 0x1E29}, + {0x68, 0x32E, 0x1E2B}, {0x68, 0x331, 0x1E96}, {0x69, 0x300, 0xEC}, + {0x69, 0x301, 0xED}, {0x69, 0x302, 0xEE}, {0x69, 0x303, 0x129}, + {0x69, 0x304, 0x12B}, {0x69, 0x306, 0x12D}, {0x69, 0x308, 0xEF}, + {0x69, 0x309, 0x1EC9}, {0x69, 0x30C, 0x1D0}, {0x69, 0x30F, 0x209}, + {0x69, 0x311, 0x20B}, {0x69, 0x323, 0x1ECB}, {0x69, 0x328, 0x12F}, + {0x69, 0x330, 0x1E2D}, {0x6A, 0x302, 0x135}, {0x6A, 0x30C, 0x1F0}, + {0x6B, 0x301, 0x1E31}, {0x6B, 0x30C, 0x1E9}, {0x6B, 0x323, 0x1E33}, + {0x6B, 0x327, 0x137}, {0x6B, 0x331, 0x1E35}, {0x6C, 0x301, 0x13A}, + {0x6C, 0x30C, 0x13E}, {0x6C, 0x323, 0x1E37}, {0x6C, 0x327, 0x13C}, + {0x6C, 0x32D, 0x1E3D}, {0x6C, 0x331, 0x1E3B}, {0x6D, 0x301, 0x1E3F}, + {0x6D, 0x307, 0x1E41}, {0x6D, 0x323, 0x1E43}, {0x6E, 0x300, 0x1F9}, + {0x6E, 0x301, 0x144}, {0x6E, 0x303, 0xF1}, {0x6E, 0x307, 0x1E45}, + {0x6E, 0x30C, 0x148}, {0x6E, 0x323, 0x1E47}, {0x6E, 0x327, 0x146}, + {0x6E, 0x32D, 0x1E4B}, {0x6E, 0x331, 0x1E49}, {0x6F, 0x300, 0xF2}, + {0x6F, 0x301, 0xF3}, {0x6F, 0x302, 0xF4}, {0x6F, 0x303, 0xF5}, + {0x6F, 0x304, 0x14D}, {0x6F, 0x306, 0x14F}, {0x6F, 0x307, 0x22F}, + {0x6F, 0x308, 0xF6}, {0x6F, 0x309, 0x1ECF}, {0x6F, 0x30B, 0x151}, + {0x6F, 0x30C, 0x1D2}, {0x6F, 0x30F, 0x20D}, {0x6F, 0x311, 0x20F}, + {0x6F, 0x31B, 0x1A1}, {0x6F, 0x323, 0x1ECD}, {0x6F, 0x328, 0x1EB}, + {0x70, 0x301, 0x1E55}, {0x70, 0x307, 0x1E57}, {0x72, 0x301, 0x155}, + {0x72, 0x307, 0x1E59}, {0x72, 0x30C, 0x159}, {0x72, 0x30F, 0x211}, + {0x72, 0x311, 0x213}, {0x72, 0x323, 0x1E5B}, {0x72, 0x327, 0x157}, + {0x72, 0x331, 0x1E5F}, {0x73, 0x301, 0x15B}, {0x73, 0x302, 0x15D}, + {0x73, 0x307, 0x1E61}, {0x73, 0x30C, 0x161}, {0x73, 0x323, 0x1E63}, + {0x73, 0x326, 0x219}, {0x73, 0x327, 0x15F}, {0x74, 0x307, 0x1E6B}, + {0x74, 0x308, 0x1E97}, {0x74, 0x30C, 0x165}, {0x74, 0x323, 0x1E6D}, + {0x74, 0x326, 0x21B}, {0x74, 0x327, 0x163}, {0x74, 0x32D, 0x1E71}, + {0x74, 0x331, 0x1E6F}, {0x75, 0x300, 0xF9}, {0x75, 0x301, 0xFA}, + {0x75, 0x302, 0xFB}, {0x75, 0x303, 0x169}, {0x75, 0x304, 0x16B}, + {0x75, 0x306, 0x16D}, {0x75, 0x308, 0xFC}, {0x75, 0x309, 0x1EE7}, + {0x75, 0x30A, 0x16F}, {0x75, 0x30B, 0x171}, {0x75, 0x30C, 0x1D4}, + {0x75, 0x30F, 0x215}, {0x75, 0x311, 0x217}, {0x75, 0x31B, 0x1B0}, + {0x75, 0x323, 0x1EE5}, {0x75, 0x324, 0x1E73}, {0x75, 0x328, 0x173}, + {0x75, 0x32D, 0x1E77}, {0x75, 0x330, 0x1E75}, {0x76, 0x303, 0x1E7D}, + {0x76, 0x323, 0x1E7F}, {0x77, 0x300, 0x1E81}, {0x77, 0x301, 0x1E83}, + {0x77, 0x302, 0x175}, {0x77, 0x307, 0x1E87}, {0x77, 0x308, 0x1E85}, + {0x77, 0x30A, 0x1E98}, {0x77, 0x323, 0x1E89}, {0x78, 0x307, 0x1E8B}, + {0x78, 0x308, 0x1E8D}, {0x79, 0x300, 0x1EF3}, {0x79, 0x301, 0xFD}, + {0x79, 0x302, 0x177}, {0x79, 0x303, 0x1EF9}, {0x79, 0x304, 0x233}, + {0x79, 0x307, 0x1E8F}, {0x79, 0x308, 0xFF}, {0x79, 0x309, 0x1EF7}, + {0x79, 0x30A, 0x1E99}, {0x79, 0x323, 0x1EF5}, {0x7A, 0x301, 0x17A}, + {0x7A, 0x302, 0x1E91}, {0x7A, 0x307, 0x17C}, {0x7A, 0x30C, 0x17E}, + {0x7A, 0x323, 0x1E93}, {0x7A, 0x331, 0x1E95}, {0xA8, 0x300, 0x1FED}, + {0xA8, 0x301, 0x385}, {0xA8, 0x342, 0x1FC1}, {0xC2, 0x300, 0x1EA6}, + {0xC2, 0x301, 0x1EA4}, {0xC2, 0x303, 0x1EAA}, {0xC2, 0x309, 0x1EA8}, + {0xC4, 0x304, 0x1DE}, {0xC5, 0x301, 0x1FA}, {0xC6, 0x301, 0x1FC}, + {0xC6, 0x304, 0x1E2}, {0xC7, 0x301, 0x1E08}, {0xCA, 0x300, 0x1EC0}, + {0xCA, 0x301, 0x1EBE}, {0xCA, 0x303, 0x1EC4}, {0xCA, 0x309, 0x1EC2}, + {0xCF, 0x301, 0x1E2E}, {0xD4, 0x300, 0x1ED2}, {0xD4, 0x301, 0x1ED0}, + {0xD4, 0x303, 0x1ED6}, {0xD4, 0x309, 0x1ED4}, {0xD5, 0x301, 0x1E4C}, + {0xD5, 0x304, 0x22C}, {0xD5, 0x308, 0x1E4E}, {0xD6, 0x304, 0x22A}, + {0xD8, 0x301, 0x1FE}, {0xDC, 0x300, 0x1DB}, {0xDC, 0x301, 0x1D7}, + {0xDC, 0x304, 0x1D5}, {0xDC, 0x30C, 0x1D9}, {0xE2, 0x300, 0x1EA7}, + {0xE2, 0x301, 0x1EA5}, {0xE2, 0x303, 0x1EAB}, {0xE2, 0x309, 0x1EA9}, + {0xE4, 0x304, 0x1DF}, {0xE5, 0x301, 0x1FB}, {0xE6, 0x301, 0x1FD}, + {0xE6, 0x304, 0x1E3}, {0xE7, 0x301, 0x1E09}, {0xEA, 0x300, 0x1EC1}, + {0xEA, 0x301, 0x1EBF}, {0xEA, 0x303, 0x1EC5}, {0xEA, 0x309, 0x1EC3}, + {0xEF, 0x301, 0x1E2F}, {0xF4, 0x300, 0x1ED3}, {0xF4, 0x301, 0x1ED1}, + {0xF4, 0x303, 0x1ED7}, {0xF4, 0x309, 0x1ED5}, {0xF5, 0x301, 0x1E4D}, + {0xF5, 0x304, 0x22D}, {0xF5, 0x308, 0x1E4F}, {0xF6, 0x304, 0x22B}, + {0xF8, 0x301, 0x1FF}, {0xFC, 0x300, 0x1DC}, {0xFC, 0x301, 0x1D8}, + {0xFC, 0x304, 0x1D6}, {0xFC, 0x30C, 0x1DA}, {0x102, 0x300, 0x1EB0}, + {0x102, 0x301, 0x1EAE}, {0x102, 0x303, 0x1EB4}, {0x102, 0x309, 0x1EB2}, + {0x103, 0x300, 0x1EB1}, {0x103, 0x301, 0x1EAF}, {0x103, 0x303, 0x1EB5}, + {0x103, 0x309, 0x1EB3}, {0x112, 0x300, 0x1E14}, {0x112, 0x301, 0x1E16}, + {0x113, 0x300, 0x1E15}, {0x113, 0x301, 0x1E17}, {0x14C, 0x300, 0x1E50}, + {0x14C, 0x301, 0x1E52}, {0x14D, 0x300, 0x1E51}, {0x14D, 0x301, 0x1E53}, + {0x15A, 0x307, 0x1E64}, {0x15B, 0x307, 0x1E65}, {0x160, 0x307, 0x1E66}, + {0x161, 0x307, 0x1E67}, {0x168, 0x301, 0x1E78}, {0x169, 0x301, 0x1E79}, + {0x16A, 0x308, 0x1E7A}, {0x16B, 0x308, 0x1E7B}, {0x17F, 0x307, 0x1E9B}, + {0x1A0, 0x300, 0x1EDC}, {0x1A0, 0x301, 0x1EDA}, {0x1A0, 0x303, 0x1EE0}, + {0x1A0, 0x309, 0x1EDE}, {0x1A0, 0x323, 0x1EE2}, {0x1A1, 0x300, 0x1EDD}, + {0x1A1, 0x301, 0x1EDB}, {0x1A1, 0x303, 0x1EE1}, {0x1A1, 0x309, 0x1EDF}, + {0x1A1, 0x323, 0x1EE3}, {0x1AF, 0x300, 0x1EEA}, {0x1AF, 0x301, 0x1EE8}, + {0x1AF, 0x303, 0x1EEE}, {0x1AF, 0x309, 0x1EEC}, {0x1AF, 0x323, 0x1EF0}, + {0x1B0, 0x300, 0x1EEB}, {0x1B0, 0x301, 0x1EE9}, {0x1B0, 0x303, 0x1EEF}, + {0x1B0, 0x309, 0x1EED}, {0x1B0, 0x323, 0x1EF1}, {0x1B7, 0x30C, 0x1EE}, + {0x1EA, 0x304, 0x1EC}, {0x1EB, 0x304, 0x1ED}, {0x226, 0x304, 0x1E0}, + {0x227, 0x304, 0x1E1}, {0x228, 0x306, 0x1E1C}, {0x229, 0x306, 0x1E1D}, + {0x22E, 0x304, 0x230}, {0x22F, 0x304, 0x231}, {0x292, 0x30C, 0x1EF}, + {0x391, 0x300, 0x1FBA}, {0x391, 0x301, 0x386}, {0x391, 0x304, 0x1FB9}, + {0x391, 0x306, 0x1FB8}, {0x391, 0x313, 0x1F08}, {0x391, 0x314, 0x1F09}, + {0x391, 0x345, 0x1FBC}, {0x395, 0x300, 0x1FC8}, {0x395, 0x301, 0x388}, + {0x395, 0x313, 0x1F18}, {0x395, 0x314, 0x1F19}, {0x397, 0x300, 0x1FCA}, + {0x397, 0x301, 0x389}, {0x397, 0x313, 0x1F28}, {0x397, 0x314, 0x1F29}, + {0x397, 0x345, 0x1FCC}, {0x399, 0x300, 0x1FDA}, {0x399, 0x301, 0x38A}, + {0x399, 0x304, 0x1FD9}, {0x399, 0x306, 0x1FD8}, {0x399, 0x308, 0x3AA}, + {0x399, 0x313, 0x1F38}, {0x399, 0x314, 0x1F39}, {0x39F, 0x300, 0x1FF8}, + {0x39F, 0x301, 0x38C}, {0x39F, 0x313, 0x1F48}, {0x39F, 0x314, 0x1F49}, + {0x3A1, 0x314, 0x1FEC}, {0x3A5, 0x300, 0x1FEA}, {0x3A5, 0x301, 0x38E}, + {0x3A5, 0x304, 0x1FE9}, {0x3A5, 0x306, 0x1FE8}, {0x3A5, 0x308, 0x3AB}, + {0x3A5, 0x314, 0x1F59}, {0x3A9, 0x300, 0x1FFA}, {0x3A9, 0x301, 0x38F}, + {0x3A9, 0x313, 0x1F68}, {0x3A9, 0x314, 0x1F69}, {0x3A9, 0x345, 0x1FFC}, + {0x3AC, 0x345, 0x1FB4}, {0x3AE, 0x345, 0x1FC4}, {0x3B1, 0x300, 0x1F70}, + {0x3B1, 0x301, 0x3AC}, {0x3B1, 0x304, 0x1FB1}, {0x3B1, 0x306, 0x1FB0}, + {0x3B1, 0x313, 0x1F00}, {0x3B1, 0x314, 0x1F01}, {0x3B1, 0x342, 0x1FB6}, + {0x3B1, 0x345, 0x1FB3}, {0x3B5, 0x300, 0x1F72}, {0x3B5, 0x301, 0x3AD}, + {0x3B5, 0x313, 0x1F10}, {0x3B5, 0x314, 0x1F11}, {0x3B7, 0x300, 0x1F74}, + {0x3B7, 0x301, 0x3AE}, {0x3B7, 0x313, 0x1F20}, {0x3B7, 0x314, 0x1F21}, + {0x3B7, 0x342, 0x1FC6}, {0x3B7, 0x345, 0x1FC3}, {0x3B9, 0x300, 0x1F76}, + {0x3B9, 0x301, 0x3AF}, {0x3B9, 0x304, 0x1FD1}, {0x3B9, 0x306, 0x1FD0}, + {0x3B9, 0x308, 0x3CA}, {0x3B9, 0x313, 0x1F30}, {0x3B9, 0x314, 0x1F31}, + {0x3B9, 0x342, 0x1FD6}, {0x3BF, 0x300, 0x1F78}, {0x3BF, 0x301, 0x3CC}, + {0x3BF, 0x313, 0x1F40}, {0x3BF, 0x314, 0x1F41}, {0x3C1, 0x313, 0x1FE4}, + {0x3C1, 0x314, 0x1FE5}, {0x3C5, 0x300, 0x1F7A}, {0x3C5, 0x301, 0x3CD}, + {0x3C5, 0x304, 0x1FE1}, {0x3C5, 0x306, 0x1FE0}, {0x3C5, 0x308, 0x3CB}, + {0x3C5, 0x313, 0x1F50}, {0x3C5, 0x314, 0x1F51}, {0x3C5, 0x342, 0x1FE6}, + {0x3C9, 0x300, 0x1F7C}, {0x3C9, 0x301, 0x3CE}, {0x3C9, 0x313, 0x1F60}, + {0x3C9, 0x314, 0x1F61}, {0x3C9, 0x342, 0x1FF6}, {0x3C9, 0x345, 0x1FF3}, + {0x3CA, 0x300, 0x1FD2}, {0x3CA, 0x301, 0x390}, {0x3CA, 0x342, 0x1FD7}, + {0x3CB, 0x300, 0x1FE2}, {0x3CB, 0x301, 0x3B0}, {0x3CB, 0x342, 0x1FE7}, + {0x3CE, 0x345, 0x1FF4}, {0x3D2, 0x301, 0x3D3}, {0x3D2, 0x308, 0x3D4}, + {0x406, 0x308, 0x407}, {0x410, 0x306, 0x4D0}, {0x410, 0x308, 0x4D2}, + {0x413, 0x301, 0x403}, {0x415, 0x300, 0x400}, {0x415, 0x306, 0x4D6}, + {0x415, 0x308, 0x401}, {0x416, 0x306, 0x4C1}, {0x416, 0x308, 0x4DC}, + {0x417, 0x308, 0x4DE}, {0x418, 0x300, 0x40D}, {0x418, 0x304, 0x4E2}, + {0x418, 0x306, 0x419}, {0x418, 0x308, 0x4E4}, {0x41A, 0x301, 0x40C}, + {0x41E, 0x308, 0x4E6}, {0x423, 0x304, 0x4EE}, {0x423, 0x306, 0x40E}, + {0x423, 0x308, 0x4F0}, {0x423, 0x30B, 0x4F2}, {0x427, 0x308, 0x4F4}, + {0x42B, 0x308, 0x4F8}, {0x42D, 0x308, 0x4EC}, {0x430, 0x306, 0x4D1}, + {0x430, 0x308, 0x4D3}, {0x433, 0x301, 0x453}, {0x435, 0x300, 0x450}, + {0x435, 0x306, 0x4D7}, {0x435, 0x308, 0x451}, {0x436, 0x306, 0x4C2}, + {0x436, 0x308, 0x4DD}, {0x437, 0x308, 0x4DF}, {0x438, 0x300, 0x45D}, + {0x438, 0x304, 0x4E3}, {0x438, 0x306, 0x439}, {0x438, 0x308, 0x4E5}, + {0x43A, 0x301, 0x45C}, {0x43E, 0x308, 0x4E7}, {0x443, 0x304, 0x4EF}, + {0x443, 0x306, 0x45E}, {0x443, 0x308, 0x4F1}, {0x443, 0x30B, 0x4F3}, + {0x447, 0x308, 0x4F5}, {0x44B, 0x308, 0x4F9}, {0x44D, 0x308, 0x4ED}, + {0x456, 0x308, 0x457}, {0x474, 0x30F, 0x476}, {0x475, 0x30F, 0x477}, + {0x4D8, 0x308, 0x4DA}, {0x4D9, 0x308, 0x4DB}, {0x4E8, 0x308, 0x4EA}, + {0x4E9, 0x308, 0x4EB}, {0x627, 0x653, 0x622}, {0x627, 0x654, 0x623}, + {0x627, 0x655, 0x625}, {0x648, 0x654, 0x624}, {0x64A, 0x654, 0x626}, + {0x6C1, 0x654, 0x6C2}, {0x6D2, 0x654, 0x6D3}, {0x6D5, 0x654, 0x6C0}, + {0x928, 0x93C, 0x929}, {0x930, 0x93C, 0x931}, {0x933, 0x93C, 0x934}, + {0x9C7, 0x9BE, 0x9CB}, {0x9C7, 0x9D7, 0x9CC}, {0xB47, 0xB3E, 0xB4B}, + {0xB47, 0xB56, 0xB48}, {0xB47, 0xB57, 0xB4C}, {0xB92, 0xBD7, 0xB94}, + {0xBC6, 0xBBE, 0xBCA}, {0xBC6, 0xBD7, 0xBCC}, {0xBC7, 0xBBE, 0xBCB}, + {0xC46, 0xC56, 0xC48}, {0xCBF, 0xCD5, 0xCC0}, {0xCC6, 0xCC2, 0xCCA}, + {0xCC6, 0xCD5, 0xCC7}, {0xCC6, 0xCD6, 0xCC8}, {0xCCA, 0xCD5, 0xCCB}, + {0xD46, 0xD3E, 0xD4A}, {0xD46, 0xD57, 0xD4C}, {0xD47, 0xD3E, 0xD4B}, + {0xDD9, 0xDCA, 0xDDA}, {0xDD9, 0xDCF, 0xDDC}, {0xDD9, 0xDDF, 0xDDE}, + {0xDDC, 0xDCA, 0xDDD}, {0x1025, 0x102E, 0x1026}, {0x1100, 0x1161, 0xAC00}, + {0x1100, 0x1162, 0xAC1C}, {0x1100, 0x1163, 0xAC38}, {0x1100, 0x1164, 0xAC54}, + {0x1100, 0x1165, 0xAC70}, {0x1100, 0x1166, 0xAC8C}, {0x1100, 0x1167, 0xACA8}, + {0x1100, 0x1168, 0xACC4}, {0x1100, 0x1169, 0xACE0}, {0x1100, 0x116A, 0xACFC}, + {0x1100, 0x116B, 0xAD18}, {0x1100, 0x116C, 0xAD34}, {0x1100, 0x116D, 0xAD50}, + {0x1100, 0x116E, 0xAD6C}, {0x1100, 0x116F, 0xAD88}, {0x1100, 0x1170, 0xADA4}, + {0x1100, 0x1171, 0xADC0}, {0x1100, 0x1172, 0xADDC}, {0x1100, 0x1173, 0xADF8}, + {0x1100, 0x1174, 0xAE14}, {0x1100, 0x1175, 0xAE30}, {0x1101, 0x1161, 0xAE4C}, + {0x1101, 0x1162, 0xAE68}, {0x1101, 0x1163, 0xAE84}, {0x1101, 0x1164, 0xAEA0}, + {0x1101, 0x1165, 0xAEBC}, {0x1101, 0x1166, 0xAED8}, {0x1101, 0x1167, 0xAEF4}, + {0x1101, 0x1168, 0xAF10}, {0x1101, 0x1169, 0xAF2C}, {0x1101, 0x116A, 0xAF48}, + {0x1101, 0x116B, 0xAF64}, {0x1101, 0x116C, 0xAF80}, {0x1101, 0x116D, 0xAF9C}, + {0x1101, 0x116E, 0xAFB8}, {0x1101, 0x116F, 0xAFD4}, {0x1101, 0x1170, 0xAFF0}, + {0x1101, 0x1171, 0xB00C}, {0x1101, 0x1172, 0xB028}, {0x1101, 0x1173, 0xB044}, + {0x1101, 0x1174, 0xB060}, {0x1101, 0x1175, 0xB07C}, {0x1102, 0x1161, 0xB098}, + {0x1102, 0x1162, 0xB0B4}, {0x1102, 0x1163, 0xB0D0}, {0x1102, 0x1164, 0xB0EC}, + {0x1102, 0x1165, 0xB108}, {0x1102, 0x1166, 0xB124}, {0x1102, 0x1167, 0xB140}, + {0x1102, 0x1168, 0xB15C}, {0x1102, 0x1169, 0xB178}, {0x1102, 0x116A, 0xB194}, + {0x1102, 0x116B, 0xB1B0}, {0x1102, 0x116C, 0xB1CC}, {0x1102, 0x116D, 0xB1E8}, + {0x1102, 0x116E, 0xB204}, {0x1102, 0x116F, 0xB220}, {0x1102, 0x1170, 0xB23C}, + {0x1102, 0x1171, 0xB258}, {0x1102, 0x1172, 0xB274}, {0x1102, 0x1173, 0xB290}, + {0x1102, 0x1174, 0xB2AC}, {0x1102, 0x1175, 0xB2C8}, {0x1103, 0x1161, 0xB2E4}, + {0x1103, 0x1162, 0xB300}, {0x1103, 0x1163, 0xB31C}, {0x1103, 0x1164, 0xB338}, + {0x1103, 0x1165, 0xB354}, {0x1103, 0x1166, 0xB370}, {0x1103, 0x1167, 0xB38C}, + {0x1103, 0x1168, 0xB3A8}, {0x1103, 0x1169, 0xB3C4}, {0x1103, 0x116A, 0xB3E0}, + {0x1103, 0x116B, 0xB3FC}, {0x1103, 0x116C, 0xB418}, {0x1103, 0x116D, 0xB434}, + {0x1103, 0x116E, 0xB450}, {0x1103, 0x116F, 0xB46C}, {0x1103, 0x1170, 0xB488}, + {0x1103, 0x1171, 0xB4A4}, {0x1103, 0x1172, 0xB4C0}, {0x1103, 0x1173, 0xB4DC}, + {0x1103, 0x1174, 0xB4F8}, {0x1103, 0x1175, 0xB514}, {0x1104, 0x1161, 0xB530}, + {0x1104, 0x1162, 0xB54C}, {0x1104, 0x1163, 0xB568}, {0x1104, 0x1164, 0xB584}, + {0x1104, 0x1165, 0xB5A0}, {0x1104, 0x1166, 0xB5BC}, {0x1104, 0x1167, 0xB5D8}, + {0x1104, 0x1168, 0xB5F4}, {0x1104, 0x1169, 0xB610}, {0x1104, 0x116A, 0xB62C}, + {0x1104, 0x116B, 0xB648}, {0x1104, 0x116C, 0xB664}, {0x1104, 0x116D, 0xB680}, + {0x1104, 0x116E, 0xB69C}, {0x1104, 0x116F, 0xB6B8}, {0x1104, 0x1170, 0xB6D4}, + {0x1104, 0x1171, 0xB6F0}, {0x1104, 0x1172, 0xB70C}, {0x1104, 0x1173, 0xB728}, + {0x1104, 0x1174, 0xB744}, {0x1104, 0x1175, 0xB760}, {0x1105, 0x1161, 0xB77C}, + {0x1105, 0x1162, 0xB798}, {0x1105, 0x1163, 0xB7B4}, {0x1105, 0x1164, 0xB7D0}, + {0x1105, 0x1165, 0xB7EC}, {0x1105, 0x1166, 0xB808}, {0x1105, 0x1167, 0xB824}, + {0x1105, 0x1168, 0xB840}, {0x1105, 0x1169, 0xB85C}, {0x1105, 0x116A, 0xB878}, + {0x1105, 0x116B, 0xB894}, {0x1105, 0x116C, 0xB8B0}, {0x1105, 0x116D, 0xB8CC}, + {0x1105, 0x116E, 0xB8E8}, {0x1105, 0x116F, 0xB904}, {0x1105, 0x1170, 0xB920}, + {0x1105, 0x1171, 0xB93C}, {0x1105, 0x1172, 0xB958}, {0x1105, 0x1173, 0xB974}, + {0x1105, 0x1174, 0xB990}, {0x1105, 0x1175, 0xB9AC}, {0x1106, 0x1161, 0xB9C8}, + {0x1106, 0x1162, 0xB9E4}, {0x1106, 0x1163, 0xBA00}, {0x1106, 0x1164, 0xBA1C}, + {0x1106, 0x1165, 0xBA38}, {0x1106, 0x1166, 0xBA54}, {0x1106, 0x1167, 0xBA70}, + {0x1106, 0x1168, 0xBA8C}, {0x1106, 0x1169, 0xBAA8}, {0x1106, 0x116A, 0xBAC4}, + {0x1106, 0x116B, 0xBAE0}, {0x1106, 0x116C, 0xBAFC}, {0x1106, 0x116D, 0xBB18}, + {0x1106, 0x116E, 0xBB34}, {0x1106, 0x116F, 0xBB50}, {0x1106, 0x1170, 0xBB6C}, + {0x1106, 0x1171, 0xBB88}, {0x1106, 0x1172, 0xBBA4}, {0x1106, 0x1173, 0xBBC0}, + {0x1106, 0x1174, 0xBBDC}, {0x1106, 0x1175, 0xBBF8}, {0x1107, 0x1161, 0xBC14}, + {0x1107, 0x1162, 0xBC30}, {0x1107, 0x1163, 0xBC4C}, {0x1107, 0x1164, 0xBC68}, + {0x1107, 0x1165, 0xBC84}, {0x1107, 0x1166, 0xBCA0}, {0x1107, 0x1167, 0xBCBC}, + {0x1107, 0x1168, 0xBCD8}, {0x1107, 0x1169, 0xBCF4}, {0x1107, 0x116A, 0xBD10}, + {0x1107, 0x116B, 0xBD2C}, {0x1107, 0x116C, 0xBD48}, {0x1107, 0x116D, 0xBD64}, + {0x1107, 0x116E, 0xBD80}, {0x1107, 0x116F, 0xBD9C}, {0x1107, 0x1170, 0xBDB8}, + {0x1107, 0x1171, 0xBDD4}, {0x1107, 0x1172, 0xBDF0}, {0x1107, 0x1173, 0xBE0C}, + {0x1107, 0x1174, 0xBE28}, {0x1107, 0x1175, 0xBE44}, {0x1108, 0x1161, 0xBE60}, + {0x1108, 0x1162, 0xBE7C}, {0x1108, 0x1163, 0xBE98}, {0x1108, 0x1164, 0xBEB4}, + {0x1108, 0x1165, 0xBED0}, {0x1108, 0x1166, 0xBEEC}, {0x1108, 0x1167, 0xBF08}, + {0x1108, 0x1168, 0xBF24}, {0x1108, 0x1169, 0xBF40}, {0x1108, 0x116A, 0xBF5C}, + {0x1108, 0x116B, 0xBF78}, {0x1108, 0x116C, 0xBF94}, {0x1108, 0x116D, 0xBFB0}, + {0x1108, 0x116E, 0xBFCC}, {0x1108, 0x116F, 0xBFE8}, {0x1108, 0x1170, 0xC004}, + {0x1108, 0x1171, 0xC020}, {0x1108, 0x1172, 0xC03C}, {0x1108, 0x1173, 0xC058}, + {0x1108, 0x1174, 0xC074}, {0x1108, 0x1175, 0xC090}, {0x1109, 0x1161, 0xC0AC}, + {0x1109, 0x1162, 0xC0C8}, {0x1109, 0x1163, 0xC0E4}, {0x1109, 0x1164, 0xC100}, + {0x1109, 0x1165, 0xC11C}, {0x1109, 0x1166, 0xC138}, {0x1109, 0x1167, 0xC154}, + {0x1109, 0x1168, 0xC170}, {0x1109, 0x1169, 0xC18C}, {0x1109, 0x116A, 0xC1A8}, + {0x1109, 0x116B, 0xC1C4}, {0x1109, 0x116C, 0xC1E0}, {0x1109, 0x116D, 0xC1FC}, + {0x1109, 0x116E, 0xC218}, {0x1109, 0x116F, 0xC234}, {0x1109, 0x1170, 0xC250}, + {0x1109, 0x1171, 0xC26C}, {0x1109, 0x1172, 0xC288}, {0x1109, 0x1173, 0xC2A4}, + {0x1109, 0x1174, 0xC2C0}, {0x1109, 0x1175, 0xC2DC}, {0x110A, 0x1161, 0xC2F8}, + {0x110A, 0x1162, 0xC314}, {0x110A, 0x1163, 0xC330}, {0x110A, 0x1164, 0xC34C}, + {0x110A, 0x1165, 0xC368}, {0x110A, 0x1166, 0xC384}, {0x110A, 0x1167, 0xC3A0}, + {0x110A, 0x1168, 0xC3BC}, {0x110A, 0x1169, 0xC3D8}, {0x110A, 0x116A, 0xC3F4}, + {0x110A, 0x116B, 0xC410}, {0x110A, 0x116C, 0xC42C}, {0x110A, 0x116D, 0xC448}, + {0x110A, 0x116E, 0xC464}, {0x110A, 0x116F, 0xC480}, {0x110A, 0x1170, 0xC49C}, + {0x110A, 0x1171, 0xC4B8}, {0x110A, 0x1172, 0xC4D4}, {0x110A, 0x1173, 0xC4F0}, + {0x110A, 0x1174, 0xC50C}, {0x110A, 0x1175, 0xC528}, {0x110B, 0x1161, 0xC544}, + {0x110B, 0x1162, 0xC560}, {0x110B, 0x1163, 0xC57C}, {0x110B, 0x1164, 0xC598}, + {0x110B, 0x1165, 0xC5B4}, {0x110B, 0x1166, 0xC5D0}, {0x110B, 0x1167, 0xC5EC}, + {0x110B, 0x1168, 0xC608}, {0x110B, 0x1169, 0xC624}, {0x110B, 0x116A, 0xC640}, + {0x110B, 0x116B, 0xC65C}, {0x110B, 0x116C, 0xC678}, {0x110B, 0x116D, 0xC694}, + {0x110B, 0x116E, 0xC6B0}, {0x110B, 0x116F, 0xC6CC}, {0x110B, 0x1170, 0xC6E8}, + {0x110B, 0x1171, 0xC704}, {0x110B, 0x1172, 0xC720}, {0x110B, 0x1173, 0xC73C}, + {0x110B, 0x1174, 0xC758}, {0x110B, 0x1175, 0xC774}, {0x110C, 0x1161, 0xC790}, + {0x110C, 0x1162, 0xC7AC}, {0x110C, 0x1163, 0xC7C8}, {0x110C, 0x1164, 0xC7E4}, + {0x110C, 0x1165, 0xC800}, {0x110C, 0x1166, 0xC81C}, {0x110C, 0x1167, 0xC838}, + {0x110C, 0x1168, 0xC854}, {0x110C, 0x1169, 0xC870}, {0x110C, 0x116A, 0xC88C}, + {0x110C, 0x116B, 0xC8A8}, {0x110C, 0x116C, 0xC8C4}, {0x110C, 0x116D, 0xC8E0}, + {0x110C, 0x116E, 0xC8FC}, {0x110C, 0x116F, 0xC918}, {0x110C, 0x1170, 0xC934}, + {0x110C, 0x1171, 0xC950}, {0x110C, 0x1172, 0xC96C}, {0x110C, 0x1173, 0xC988}, + {0x110C, 0x1174, 0xC9A4}, {0x110C, 0x1175, 0xC9C0}, {0x110D, 0x1161, 0xC9DC}, + {0x110D, 0x1162, 0xC9F8}, {0x110D, 0x1163, 0xCA14}, {0x110D, 0x1164, 0xCA30}, + {0x110D, 0x1165, 0xCA4C}, {0x110D, 0x1166, 0xCA68}, {0x110D, 0x1167, 0xCA84}, + {0x110D, 0x1168, 0xCAA0}, {0x110D, 0x1169, 0xCABC}, {0x110D, 0x116A, 0xCAD8}, + {0x110D, 0x116B, 0xCAF4}, {0x110D, 0x116C, 0xCB10}, {0x110D, 0x116D, 0xCB2C}, + {0x110D, 0x116E, 0xCB48}, {0x110D, 0x116F, 0xCB64}, {0x110D, 0x1170, 0xCB80}, + {0x110D, 0x1171, 0xCB9C}, {0x110D, 0x1172, 0xCBB8}, {0x110D, 0x1173, 0xCBD4}, + {0x110D, 0x1174, 0xCBF0}, {0x110D, 0x1175, 0xCC0C}, {0x110E, 0x1161, 0xCC28}, + {0x110E, 0x1162, 0xCC44}, {0x110E, 0x1163, 0xCC60}, {0x110E, 0x1164, 0xCC7C}, + {0x110E, 0x1165, 0xCC98}, {0x110E, 0x1166, 0xCCB4}, {0x110E, 0x1167, 0xCCD0}, + {0x110E, 0x1168, 0xCCEC}, {0x110E, 0x1169, 0xCD08}, {0x110E, 0x116A, 0xCD24}, + {0x110E, 0x116B, 0xCD40}, {0x110E, 0x116C, 0xCD5C}, {0x110E, 0x116D, 0xCD78}, + {0x110E, 0x116E, 0xCD94}, {0x110E, 0x116F, 0xCDB0}, {0x110E, 0x1170, 0xCDCC}, + {0x110E, 0x1171, 0xCDE8}, {0x110E, 0x1172, 0xCE04}, {0x110E, 0x1173, 0xCE20}, + {0x110E, 0x1174, 0xCE3C}, {0x110E, 0x1175, 0xCE58}, {0x110F, 0x1161, 0xCE74}, + {0x110F, 0x1162, 0xCE90}, {0x110F, 0x1163, 0xCEAC}, {0x110F, 0x1164, 0xCEC8}, + {0x110F, 0x1165, 0xCEE4}, {0x110F, 0x1166, 0xCF00}, {0x110F, 0x1167, 0xCF1C}, + {0x110F, 0x1168, 0xCF38}, {0x110F, 0x1169, 0xCF54}, {0x110F, 0x116A, 0xCF70}, + {0x110F, 0x116B, 0xCF8C}, {0x110F, 0x116C, 0xCFA8}, {0x110F, 0x116D, 0xCFC4}, + {0x110F, 0x116E, 0xCFE0}, {0x110F, 0x116F, 0xCFFC}, {0x110F, 0x1170, 0xD018}, + {0x110F, 0x1171, 0xD034}, {0x110F, 0x1172, 0xD050}, {0x110F, 0x1173, 0xD06C}, + {0x110F, 0x1174, 0xD088}, {0x110F, 0x1175, 0xD0A4}, {0x1110, 0x1161, 0xD0C0}, + {0x1110, 0x1162, 0xD0DC}, {0x1110, 0x1163, 0xD0F8}, {0x1110, 0x1164, 0xD114}, + {0x1110, 0x1165, 0xD130}, {0x1110, 0x1166, 0xD14C}, {0x1110, 0x1167, 0xD168}, + {0x1110, 0x1168, 0xD184}, {0x1110, 0x1169, 0xD1A0}, {0x1110, 0x116A, 0xD1BC}, + {0x1110, 0x116B, 0xD1D8}, {0x1110, 0x116C, 0xD1F4}, {0x1110, 0x116D, 0xD210}, + {0x1110, 0x116E, 0xD22C}, {0x1110, 0x116F, 0xD248}, {0x1110, 0x1170, 0xD264}, + {0x1110, 0x1171, 0xD280}, {0x1110, 0x1172, 0xD29C}, {0x1110, 0x1173, 0xD2B8}, + {0x1110, 0x1174, 0xD2D4}, {0x1110, 0x1175, 0xD2F0}, {0x1111, 0x1161, 0xD30C}, + {0x1111, 0x1162, 0xD328}, {0x1111, 0x1163, 0xD344}, {0x1111, 0x1164, 0xD360}, + {0x1111, 0x1165, 0xD37C}, {0x1111, 0x1166, 0xD398}, {0x1111, 0x1167, 0xD3B4}, + {0x1111, 0x1168, 0xD3D0}, {0x1111, 0x1169, 0xD3EC}, {0x1111, 0x116A, 0xD408}, + {0x1111, 0x116B, 0xD424}, {0x1111, 0x116C, 0xD440}, {0x1111, 0x116D, 0xD45C}, + {0x1111, 0x116E, 0xD478}, {0x1111, 0x116F, 0xD494}, {0x1111, 0x1170, 0xD4B0}, + {0x1111, 0x1171, 0xD4CC}, {0x1111, 0x1172, 0xD4E8}, {0x1111, 0x1173, 0xD504}, + {0x1111, 0x1174, 0xD520}, {0x1111, 0x1175, 0xD53C}, {0x1112, 0x1161, 0xD558}, + {0x1112, 0x1162, 0xD574}, {0x1112, 0x1163, 0xD590}, {0x1112, 0x1164, 0xD5AC}, + {0x1112, 0x1165, 0xD5C8}, {0x1112, 0x1166, 0xD5E4}, {0x1112, 0x1167, 0xD600}, + {0x1112, 0x1168, 0xD61C}, {0x1112, 0x1169, 0xD638}, {0x1112, 0x116A, 0xD654}, + {0x1112, 0x116B, 0xD670}, {0x1112, 0x116C, 0xD68C}, {0x1112, 0x116D, 0xD6A8}, + {0x1112, 0x116E, 0xD6C4}, {0x1112, 0x116F, 0xD6E0}, {0x1112, 0x1170, 0xD6FC}, + {0x1112, 0x1171, 0xD718}, {0x1112, 0x1172, 0xD734}, {0x1112, 0x1173, 0xD750}, + {0x1112, 0x1174, 0xD76C}, {0x1112, 0x1175, 0xD788}, {0x1B05, 0x1B35, 0x1B06}, + {0x1B07, 0x1B35, 0x1B08}, {0x1B09, 0x1B35, 0x1B0A}, {0x1B0B, 0x1B35, 0x1B0C}, + {0x1B0D, 0x1B35, 0x1B0E}, {0x1B11, 0x1B35, 0x1B12}, {0x1B3A, 0x1B35, 0x1B3B}, + {0x1B3C, 0x1B35, 0x1B3D}, {0x1B3E, 0x1B35, 0x1B40}, {0x1B3F, 0x1B35, 0x1B41}, + {0x1B42, 0x1B35, 0x1B43}, {0x1E36, 0x304, 0x1E38}, {0x1E37, 0x304, 0x1E39}, + {0x1E5A, 0x304, 0x1E5C}, {0x1E5B, 0x304, 0x1E5D}, {0x1E62, 0x307, 0x1E68}, + {0x1E63, 0x307, 0x1E69}, {0x1EA0, 0x302, 0x1EAC}, {0x1EA0, 0x306, 0x1EB6}, + {0x1EA1, 0x302, 0x1EAD}, {0x1EA1, 0x306, 0x1EB7}, {0x1EB8, 0x302, 0x1EC6}, + {0x1EB9, 0x302, 0x1EC7}, {0x1ECC, 0x302, 0x1ED8}, {0x1ECD, 0x302, 0x1ED9}, + {0x1F00, 0x300, 0x1F02}, {0x1F00, 0x301, 0x1F04}, {0x1F00, 0x342, 0x1F06}, + {0x1F00, 0x345, 0x1F80}, {0x1F01, 0x300, 0x1F03}, {0x1F01, 0x301, 0x1F05}, + {0x1F01, 0x342, 0x1F07}, {0x1F01, 0x345, 0x1F81}, {0x1F02, 0x345, 0x1F82}, + {0x1F03, 0x345, 0x1F83}, {0x1F04, 0x345, 0x1F84}, {0x1F05, 0x345, 0x1F85}, + {0x1F06, 0x345, 0x1F86}, {0x1F07, 0x345, 0x1F87}, {0x1F08, 0x300, 0x1F0A}, + {0x1F08, 0x301, 0x1F0C}, {0x1F08, 0x342, 0x1F0E}, {0x1F08, 0x345, 0x1F88}, + {0x1F09, 0x300, 0x1F0B}, {0x1F09, 0x301, 0x1F0D}, {0x1F09, 0x342, 0x1F0F}, + {0x1F09, 0x345, 0x1F89}, {0x1F0A, 0x345, 0x1F8A}, {0x1F0B, 0x345, 0x1F8B}, + {0x1F0C, 0x345, 0x1F8C}, {0x1F0D, 0x345, 0x1F8D}, {0x1F0E, 0x345, 0x1F8E}, + {0x1F0F, 0x345, 0x1F8F}, {0x1F10, 0x300, 0x1F12}, {0x1F10, 0x301, 0x1F14}, + {0x1F11, 0x300, 0x1F13}, {0x1F11, 0x301, 0x1F15}, {0x1F18, 0x300, 0x1F1A}, + {0x1F18, 0x301, 0x1F1C}, {0x1F19, 0x300, 0x1F1B}, {0x1F19, 0x301, 0x1F1D}, + {0x1F20, 0x300, 0x1F22}, {0x1F20, 0x301, 0x1F24}, {0x1F20, 0x342, 0x1F26}, + {0x1F20, 0x345, 0x1F90}, {0x1F21, 0x300, 0x1F23}, {0x1F21, 0x301, 0x1F25}, + {0x1F21, 0x342, 0x1F27}, {0x1F21, 0x345, 0x1F91}, {0x1F22, 0x345, 0x1F92}, + {0x1F23, 0x345, 0x1F93}, {0x1F24, 0x345, 0x1F94}, {0x1F25, 0x345, 0x1F95}, + {0x1F26, 0x345, 0x1F96}, {0x1F27, 0x345, 0x1F97}, {0x1F28, 0x300, 0x1F2A}, + {0x1F28, 0x301, 0x1F2C}, {0x1F28, 0x342, 0x1F2E}, {0x1F28, 0x345, 0x1F98}, + {0x1F29, 0x300, 0x1F2B}, {0x1F29, 0x301, 0x1F2D}, {0x1F29, 0x342, 0x1F2F}, + {0x1F29, 0x345, 0x1F99}, {0x1F2A, 0x345, 0x1F9A}, {0x1F2B, 0x345, 0x1F9B}, + {0x1F2C, 0x345, 0x1F9C}, {0x1F2D, 0x345, 0x1F9D}, {0x1F2E, 0x345, 0x1F9E}, + {0x1F2F, 0x345, 0x1F9F}, {0x1F30, 0x300, 0x1F32}, {0x1F30, 0x301, 0x1F34}, + {0x1F30, 0x342, 0x1F36}, {0x1F31, 0x300, 0x1F33}, {0x1F31, 0x301, 0x1F35}, + {0x1F31, 0x342, 0x1F37}, {0x1F38, 0x300, 0x1F3A}, {0x1F38, 0x301, 0x1F3C}, + {0x1F38, 0x342, 0x1F3E}, {0x1F39, 0x300, 0x1F3B}, {0x1F39, 0x301, 0x1F3D}, + {0x1F39, 0x342, 0x1F3F}, {0x1F40, 0x300, 0x1F42}, {0x1F40, 0x301, 0x1F44}, + {0x1F41, 0x300, 0x1F43}, {0x1F41, 0x301, 0x1F45}, {0x1F48, 0x300, 0x1F4A}, + {0x1F48, 0x301, 0x1F4C}, {0x1F49, 0x300, 0x1F4B}, {0x1F49, 0x301, 0x1F4D}, + {0x1F50, 0x300, 0x1F52}, {0x1F50, 0x301, 0x1F54}, {0x1F50, 0x342, 0x1F56}, + {0x1F51, 0x300, 0x1F53}, {0x1F51, 0x301, 0x1F55}, {0x1F51, 0x342, 0x1F57}, + {0x1F59, 0x300, 0x1F5B}, {0x1F59, 0x301, 0x1F5D}, {0x1F59, 0x342, 0x1F5F}, + {0x1F60, 0x300, 0x1F62}, {0x1F60, 0x301, 0x1F64}, {0x1F60, 0x342, 0x1F66}, + {0x1F60, 0x345, 0x1FA0}, {0x1F61, 0x300, 0x1F63}, {0x1F61, 0x301, 0x1F65}, + {0x1F61, 0x342, 0x1F67}, {0x1F61, 0x345, 0x1FA1}, {0x1F62, 0x345, 0x1FA2}, + {0x1F63, 0x345, 0x1FA3}, {0x1F64, 0x345, 0x1FA4}, {0x1F65, 0x345, 0x1FA5}, + {0x1F66, 0x345, 0x1FA6}, {0x1F67, 0x345, 0x1FA7}, {0x1F68, 0x300, 0x1F6A}, + {0x1F68, 0x301, 0x1F6C}, {0x1F68, 0x342, 0x1F6E}, {0x1F68, 0x345, 0x1FA8}, + {0x1F69, 0x300, 0x1F6B}, {0x1F69, 0x301, 0x1F6D}, {0x1F69, 0x342, 0x1F6F}, + {0x1F69, 0x345, 0x1FA9}, {0x1F6A, 0x345, 0x1FAA}, {0x1F6B, 0x345, 0x1FAB}, + {0x1F6C, 0x345, 0x1FAC}, {0x1F6D, 0x345, 0x1FAD}, {0x1F6E, 0x345, 0x1FAE}, + {0x1F6F, 0x345, 0x1FAF}, {0x1F70, 0x345, 0x1FB2}, {0x1F74, 0x345, 0x1FC2}, + {0x1F7C, 0x345, 0x1FF2}, {0x1FB6, 0x345, 0x1FB7}, {0x1FBF, 0x300, 0x1FCD}, + {0x1FBF, 0x301, 0x1FCE}, {0x1FBF, 0x342, 0x1FCF}, {0x1FC6, 0x345, 0x1FC7}, + {0x1FF6, 0x345, 0x1FF7}, {0x1FFE, 0x300, 0x1FDD}, {0x1FFE, 0x301, 0x1FDE}, + {0x1FFE, 0x342, 0x1FDF}, {0x2190, 0x338, 0x219A}, {0x2192, 0x338, 0x219B}, + {0x2194, 0x338, 0x21AE}, {0x21D0, 0x338, 0x21CD}, {0x21D2, 0x338, 0x21CF}, + {0x21D4, 0x338, 0x21CE}, {0x2203, 0x338, 0x2204}, {0x2208, 0x338, 0x2209}, + {0x220B, 0x338, 0x220C}, {0x2223, 0x338, 0x2224}, {0x2225, 0x338, 0x2226}, + {0x223C, 0x338, 0x2241}, {0x2243, 0x338, 0x2244}, {0x2245, 0x338, 0x2247}, + {0x2248, 0x338, 0x2249}, {0x224D, 0x338, 0x226D}, {0x2261, 0x338, 0x2262}, + {0x2264, 0x338, 0x2270}, {0x2265, 0x338, 0x2271}, {0x2272, 0x338, 0x2274}, + {0x2273, 0x338, 0x2275}, {0x2276, 0x338, 0x2278}, {0x2277, 0x338, 0x2279}, + {0x227A, 0x338, 0x2280}, {0x227B, 0x338, 0x2281}, {0x227C, 0x338, 0x22E0}, + {0x227D, 0x338, 0x22E1}, {0x2282, 0x338, 0x2284}, {0x2283, 0x338, 0x2285}, + {0x2286, 0x338, 0x2288}, {0x2287, 0x338, 0x2289}, {0x2291, 0x338, 0x22E2}, + {0x2292, 0x338, 0x22E3}, {0x22A2, 0x338, 0x22AC}, {0x22A8, 0x338, 0x22AD}, + {0x22A9, 0x338, 0x22AE}, {0x22AB, 0x338, 0x22AF}, {0x22B2, 0x338, 0x22EA}, + {0x22B3, 0x338, 0x22EB}, {0x22B4, 0x338, 0x22EC}, {0x22B5, 0x338, 0x22ED}, + {0x3046, 0x3099, 0x3094}, {0x304B, 0x3099, 0x304C}, {0x304D, 0x3099, 0x304E}, + {0x304F, 0x3099, 0x3050}, {0x3051, 0x3099, 0x3052}, {0x3053, 0x3099, 0x3054}, + {0x3055, 0x3099, 0x3056}, {0x3057, 0x3099, 0x3058}, {0x3059, 0x3099, 0x305A}, + {0x305B, 0x3099, 0x305C}, {0x305D, 0x3099, 0x305E}, {0x305F, 0x3099, 0x3060}, + {0x3061, 0x3099, 0x3062}, {0x3064, 0x3099, 0x3065}, {0x3066, 0x3099, 0x3067}, + {0x3068, 0x3099, 0x3069}, {0x306F, 0x3099, 0x3070}, {0x306F, 0x309A, 0x3071}, + {0x3072, 0x3099, 0x3073}, {0x3072, 0x309A, 0x3074}, {0x3075, 0x3099, 0x3076}, + {0x3075, 0x309A, 0x3077}, {0x3078, 0x3099, 0x3079}, {0x3078, 0x309A, 0x307A}, + {0x307B, 0x3099, 0x307C}, {0x307B, 0x309A, 0x307D}, {0x309D, 0x3099, 0x309E}, + {0x30A6, 0x3099, 0x30F4}, {0x30AB, 0x3099, 0x30AC}, {0x30AD, 0x3099, 0x30AE}, + {0x30AF, 0x3099, 0x30B0}, {0x30B1, 0x3099, 0x30B2}, {0x30B3, 0x3099, 0x30B4}, + {0x30B5, 0x3099, 0x30B6}, {0x30B7, 0x3099, 0x30B8}, {0x30B9, 0x3099, 0x30BA}, + {0x30BB, 0x3099, 0x30BC}, {0x30BD, 0x3099, 0x30BE}, {0x30BF, 0x3099, 0x30C0}, + {0x30C1, 0x3099, 0x30C2}, {0x30C4, 0x3099, 0x30C5}, {0x30C6, 0x3099, 0x30C7}, + {0x30C8, 0x3099, 0x30C9}, {0x30CF, 0x3099, 0x30D0}, {0x30CF, 0x309A, 0x30D1}, + {0x30D2, 0x3099, 0x30D3}, {0x30D2, 0x309A, 0x30D4}, {0x30D5, 0x3099, 0x30D6}, + {0x30D5, 0x309A, 0x30D7}, {0x30D8, 0x3099, 0x30D9}, {0x30D8, 0x309A, 0x30DA}, + {0x30DB, 0x3099, 0x30DC}, {0x30DB, 0x309A, 0x30DD}, {0x30EF, 0x3099, 0x30F7}, + {0x30F0, 0x3099, 0x30F8}, {0x30F1, 0x3099, 0x30F9}, {0x30F2, 0x3099, 0x30FA}, + {0x30FD, 0x3099, 0x30FE}, {0x105D2, 0x307, 0x105C9}, {0x105DA, 0x307, 0x105E4}, + {0x11099, 0x110BA, 0x1109A}, {0x1109B, 0x110BA, 0x1109C}, {0x110A5, 0x110BA, 0x110AB}, + {0x11131, 0x11127, 0x1112E}, {0x11132, 0x11127, 0x1112F}, {0x11347, 0x1133E, 0x1134B}, + {0x11347, 0x11357, 0x1134C}, {0x11382, 0x113C9, 0x11383}, {0x11384, 0x113BB, 0x11385}, + {0x1138B, 0x113C2, 0x1138E}, {0x11390, 0x113C9, 0x11391}, {0x113C2, 0x113B8, 0x113C7}, + {0x113C2, 0x113C2, 0x113C5}, {0x113C2, 0x113C9, 0x113C8}, {0x114B9, 0x114B0, 0x114BC}, + {0x114B9, 0x114BA, 0x114BB}, {0x114B9, 0x114BD, 0x114BE}, {0x115B8, 0x115AF, 0x115BA}, + {0x115B9, 0x115AF, 0x115BB}, {0x11935, 0x11930, 0x11938}, {0x1611E, 0x1611E, 0x16121}, + {0x1611E, 0x1611F, 0x16123}, {0x1611E, 0x16120, 0x16125}, {0x1611E, 0x16129, 0x16122}, + {0x16121, 0x1611F, 0x16126}, {0x16121, 0x16120, 0x16128}, {0x16122, 0x1611F, 0x16127}, + {0x16129, 0x1611F, 0x16124}, {0x16D63, 0x16D67, 0x16D69}, {0x16D67, 0x16D67, 0x16D68}, + {0x16D69, 0x16D67, 0x16D6A}, +}; +const int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0])); + +const pdf_urange_t PDF_U_SPACE[] = { + {0x9, 0xD}, {0x1C, 0x20}, {0x85, 0x85}, {0xA0, 0xA0}, {0x1680, 0x1680}, + {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, +}; +const int PDF_U_SPACE_COUNT = (int)(sizeof(PDF_U_SPACE) / sizeof(PDF_U_SPACE[0])); + +const pdf_urange_t PDF_U_WORD[] = { + {0x30, 0x39}, {0x41, 0x5A}, {0x5F, 0x5F}, {0x61, 0x7A}, + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, + {0xBC, 0xBE}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, + {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, + {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, + {0x559, 0x559}, {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, + {0x620, 0x64A}, {0x660, 0x669}, {0x66E, 0x66F}, {0x671, 0x6D3}, + {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, {0x6FF, 0x6FF}, + {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, + {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, + {0x860, 0x86A}, {0x870, 0x887}, {0x889, 0x88E}, {0x8A0, 0x8C9}, + {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, + {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, + {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, + {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, {0x9FC, 0x9FC}, {0xA05, 0xA0A}, + {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, {0xA32, 0xA33}, + {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, + {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, + {0xABD, 0xABD}, {0xAD0, 0xAD0}, {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, + {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, + {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, + {0xB83, 0xB83}, {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, + {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, + {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xBE6, 0xBF2}, + {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, + {0xC66, 0xC6F}, {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, + {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, + {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, {0xCE6, 0xCEF}, + {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, + {0xD66, 0xD78}, {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, + {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, + {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, {0xE50, 0xE59}, + {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, + {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, + {0xF00, 0xF00}, {0xF20, 0xF33}, {0xF40, 0xF47}, {0xF49, 0xF6C}, + {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, {0x1050, 0x1055}, + {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, + {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, + {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, + {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, + {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, + {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, + {0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, + {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, + {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, + {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, + {0x1900, 0x191E}, {0x1946, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB}, + {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, + {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, + {0x1C00, 0x1C23}, {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C8A}, + {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, + {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, + {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, + {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, + {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, {0x2074, 0x2079}, + {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, + {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, + {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, {0x2150, 0x2189}, + {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, {0x2C00, 0x2CE4}, + {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, + {0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, + {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, + {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, {0x3021, 0x3029}, + {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, + {0x3192, 0x3195}, {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, + {0x3248, 0x324F}, {0x3251, 0x325F}, {0x3280, 0x3289}, {0x32B1, 0x32BF}, + {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, + {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CD}, {0xA7D0, 0xA7D1}, + {0xA7D3, 0xA7D3}, {0xA7D5, 0xA7DC}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, + {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA830, 0xA835}, {0xA840, 0xA873}, + {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, + {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, + {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, + {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, + {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, + {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, + {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, + {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, + {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, + {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, + {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, + {0x10107, 0x10133}, {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A}, + {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, + {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, + {0x1057C, 0x1058A}, {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, + {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, + {0x10858, 0x10876}, {0x10879, 0x1089E}, {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, + {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, + {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, + {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D23}, + {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, {0x10D6F, 0x10D85}, {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC4}, {0x10F00, 0x10F27}, + {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, + {0x11075, 0x11075}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, + {0x11103, 0x11126}, {0x11136, 0x1113F}, {0x11144, 0x11144}, {0x11147, 0x11147}, + {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, + {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, + {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x112F0, 0x112F9}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, + {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, + {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11380, 0x11389}, {0x1138B, 0x1138B}, + {0x1138E, 0x1138E}, {0x11390, 0x113B5}, {0x113B7, 0x113B7}, {0x113D1, 0x113D1}, + {0x113D3, 0x113D3}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x11450, 0x11459}, + {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, + {0x11644, 0x11644}, {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, + {0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A}, {0x11730, 0x1173B}, + {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, + {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, + {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, + {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, + {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11BF0, 0x11BF9}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, + {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, + {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, + {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, + {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, {0x12000, 0x12399}, {0x12400, 0x1246E}, + {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, {0x13441, 0x13446}, + {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, {0x16130, 0x16139}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A70, 0x16ABE}, + {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, + {0x16D40, 0x16D6C}, {0x16D70, 0x16D79}, {0x16E40, 0x16E96}, {0x16F00, 0x16F4A}, + {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18CFF, 0x18D08}, {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, + {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, + {0x1CCF0, 0x1CCF9}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, + {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, + {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, + {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, {0x1DF00, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, + {0x1E2F0, 0x1E2F9}, {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E5D0, 0x1E5ED}, + {0x1E5F0, 0x1E5FA}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, + {0x1E94B, 0x1E94B}, {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, + {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, + {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, + {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1F100, 0x1F10C}, {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, + {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, + {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, +}; +const int PDF_U_WORD_COUNT = (int)(sizeof(PDF_U_WORD) / sizeof(PDF_U_WORD[0])); + +const pdf_urange_t PDF_U_ALPHA[] = { + {0x41, 0x5A}, {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, + {0xBA, 0xBA}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, + {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, + {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, + {0x559, 0x559}, {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, + {0x620, 0x64A}, {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, + {0x6E5, 0x6E6}, {0x6EE, 0x6EF}, {0x6FA, 0x6FC}, {0x6FF, 0x6FF}, + {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7CA, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, + {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, + {0x860, 0x86A}, {0x870, 0x887}, {0x889, 0x88E}, {0x8A0, 0x8C9}, + {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, + {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, + {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9F0, 0x9F1}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, + {0xA2A, 0xA30}, {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, + {0xA59, 0xA5C}, {0xA5E, 0xA5E}, {0xA72, 0xA74}, {0xA85, 0xA8D}, + {0xA8F, 0xA91}, {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, + {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, {0xAE0, 0xAE1}, + {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, + {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB71, 0xB71}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, + {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, + {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, + {0xC12, 0xC28}, {0xC2A, 0xC39}, {0xC3D, 0xC3D}, {0xC58, 0xC5A}, + {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC80, 0xC80}, {0xC85, 0xC8C}, + {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, + {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, {0xCF1, 0xCF2}, + {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, + {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD5F, 0xD61}, {0xD7A, 0xD7F}, + {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, + {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, + {0x103F, 0x103F}, {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, + {0x1065, 0x1066}, {0x106E, 0x1070}, {0x1075, 0x1081}, {0x108E, 0x108E}, + {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, + {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, + {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, + {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, + {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, + {0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16F1, 0x16F8}, + {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, + {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, + {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1950, 0x196D}, {0x1970, 0x1974}, + {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, + {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4C}, {0x1B83, 0x1BA0}, + {0x1BAE, 0x1BAF}, {0x1BBA, 0x1BE5}, {0x1C00, 0x1C23}, {0x1C4D, 0x1C4F}, + {0x1C5A, 0x1C7D}, {0x1C80, 0x1C8A}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, + {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, + {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, + {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, + {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, {0x2102, 0x2102}, + {0x2107, 0x2107}, {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, + {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, + {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, + {0x2183, 0x2184}, {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, + {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, + {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, + {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3006}, + {0x3031, 0x3035}, {0x303B, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, + {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA61F}, {0xA62A, 0xA62B}, + {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6E5}, {0xA717, 0xA71F}, + {0xA722, 0xA788}, {0xA78B, 0xA7CD}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7DC}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, + {0xA80C, 0xA822}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA90A, 0xA925}, {0xA930, 0xA946}, + {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9CF}, {0xA9E0, 0xA9E4}, + {0xA9E6, 0xA9EF}, {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, + {0xAA44, 0xAA4B}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, + {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, + {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, + {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, + {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, + {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, + {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, + {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, + {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, + {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, {0x10300, 0x1031F}, {0x1032D, 0x10340}, {0x10342, 0x10349}, + {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, + {0x10400, 0x1049D}, {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, + {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, + {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0}, {0x107B2, 0x107BA}, + {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838}, + {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10860, 0x10876}, {0x10880, 0x1089E}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x10900, 0x10915}, {0x10920, 0x10939}, + {0x10980, 0x109B7}, {0x109BE, 0x109BF}, {0x10A00, 0x10A00}, {0x10A10, 0x10A13}, + {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A60, 0x10A7C}, {0x10A80, 0x10A9C}, + {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, + {0x10B60, 0x10B72}, {0x10B80, 0x10B91}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, + {0x10CC0, 0x10CF2}, {0x10D00, 0x10D23}, {0x10D4A, 0x10D65}, {0x10D6F, 0x10D85}, + {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC4}, {0x10F00, 0x10F1C}, + {0x10F27, 0x10F27}, {0x10F30, 0x10F45}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FC4}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11071, 0x11072}, {0x11075, 0x11075}, + {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x11103, 0x11126}, {0x11144, 0x11144}, + {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, + {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, + {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, + {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, + {0x1135D, 0x11361}, {0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, + {0x11390, 0x113B5}, {0x113B7, 0x113B7}, {0x113D1, 0x113D1}, {0x113D3, 0x113D3}, + {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, + {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, + {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, + {0x11700, 0x1171A}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118DF}, + {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, + {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, + {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, + {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, + {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11BC0, 0x11BE0}, {0x11C00, 0x11C08}, + {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, + {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65}, + {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, + {0x11F02, 0x11F02}, {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11FB0, 0x11FB0}, + {0x12000, 0x12399}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, + {0x13441, 0x13446}, {0x13460, 0x143FA}, {0x14400, 0x14646}, {0x16100, 0x1611D}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A70, 0x16ABE}, {0x16AD0, 0x16AED}, + {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, + {0x16D40, 0x16D6C}, {0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, + {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, {0x18CFF, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, + {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, + {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, + {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7CB}, {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, + {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, + {0x1E2C0, 0x1E2EB}, {0x1E4D0, 0x1E4EB}, {0x1E5D0, 0x1E5ED}, {0x1E5F0, 0x1E5F0}, + {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, + {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, + {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, + {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, + {0x31350, 0x323AF}, +}; +const int PDF_U_ALPHA_COUNT = (int)(sizeof(PDF_U_ALPHA) / sizeof(PDF_U_ALPHA[0])); + +const pdf_urange_t PDF_U_UPPER[] = { + {0x41, 0x5A}, {0xC0, 0xD6}, {0xD8, 0xDE}, {0x100, 0x100}, + {0x102, 0x102}, {0x104, 0x104}, {0x106, 0x106}, {0x108, 0x108}, + {0x10A, 0x10A}, {0x10C, 0x10C}, {0x10E, 0x10E}, {0x110, 0x110}, + {0x112, 0x112}, {0x114, 0x114}, {0x116, 0x116}, {0x118, 0x118}, + {0x11A, 0x11A}, {0x11C, 0x11C}, {0x11E, 0x11E}, {0x120, 0x120}, + {0x122, 0x122}, {0x124, 0x124}, {0x126, 0x126}, {0x128, 0x128}, + {0x12A, 0x12A}, {0x12C, 0x12C}, {0x12E, 0x12E}, {0x130, 0x130}, + {0x132, 0x132}, {0x134, 0x134}, {0x136, 0x136}, {0x139, 0x139}, + {0x13B, 0x13B}, {0x13D, 0x13D}, {0x13F, 0x13F}, {0x141, 0x141}, + {0x143, 0x143}, {0x145, 0x145}, {0x147, 0x147}, {0x14A, 0x14A}, + {0x14C, 0x14C}, {0x14E, 0x14E}, {0x150, 0x150}, {0x152, 0x152}, + {0x154, 0x154}, {0x156, 0x156}, {0x158, 0x158}, {0x15A, 0x15A}, + {0x15C, 0x15C}, {0x15E, 0x15E}, {0x160, 0x160}, {0x162, 0x162}, + {0x164, 0x164}, {0x166, 0x166}, {0x168, 0x168}, {0x16A, 0x16A}, + {0x16C, 0x16C}, {0x16E, 0x16E}, {0x170, 0x170}, {0x172, 0x172}, + {0x174, 0x174}, {0x176, 0x176}, {0x178, 0x179}, {0x17B, 0x17B}, + {0x17D, 0x17D}, {0x181, 0x182}, {0x184, 0x184}, {0x186, 0x187}, + {0x189, 0x18B}, {0x18E, 0x191}, {0x193, 0x194}, {0x196, 0x198}, + {0x19C, 0x19D}, {0x19F, 0x1A0}, {0x1A2, 0x1A2}, {0x1A4, 0x1A4}, + {0x1A6, 0x1A7}, {0x1A9, 0x1A9}, {0x1AC, 0x1AC}, {0x1AE, 0x1AF}, + {0x1B1, 0x1B3}, {0x1B5, 0x1B5}, {0x1B7, 0x1B8}, {0x1BC, 0x1BC}, + {0x1C4, 0x1C4}, {0x1C7, 0x1C7}, {0x1CA, 0x1CA}, {0x1CD, 0x1CD}, + {0x1CF, 0x1CF}, {0x1D1, 0x1D1}, {0x1D3, 0x1D3}, {0x1D5, 0x1D5}, + {0x1D7, 0x1D7}, {0x1D9, 0x1D9}, {0x1DB, 0x1DB}, {0x1DE, 0x1DE}, + {0x1E0, 0x1E0}, {0x1E2, 0x1E2}, {0x1E4, 0x1E4}, {0x1E6, 0x1E6}, + {0x1E8, 0x1E8}, {0x1EA, 0x1EA}, {0x1EC, 0x1EC}, {0x1EE, 0x1EE}, + {0x1F1, 0x1F1}, {0x1F4, 0x1F4}, {0x1F6, 0x1F8}, {0x1FA, 0x1FA}, + {0x1FC, 0x1FC}, {0x1FE, 0x1FE}, {0x200, 0x200}, {0x202, 0x202}, + {0x204, 0x204}, {0x206, 0x206}, {0x208, 0x208}, {0x20A, 0x20A}, + {0x20C, 0x20C}, {0x20E, 0x20E}, {0x210, 0x210}, {0x212, 0x212}, + {0x214, 0x214}, {0x216, 0x216}, {0x218, 0x218}, {0x21A, 0x21A}, + {0x21C, 0x21C}, {0x21E, 0x21E}, {0x220, 0x220}, {0x222, 0x222}, + {0x224, 0x224}, {0x226, 0x226}, {0x228, 0x228}, {0x22A, 0x22A}, + {0x22C, 0x22C}, {0x22E, 0x22E}, {0x230, 0x230}, {0x232, 0x232}, + {0x23A, 0x23B}, {0x23D, 0x23E}, {0x241, 0x241}, {0x243, 0x246}, + {0x248, 0x248}, {0x24A, 0x24A}, {0x24C, 0x24C}, {0x24E, 0x24E}, + {0x370, 0x370}, {0x372, 0x372}, {0x376, 0x376}, {0x37F, 0x37F}, + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x38F}, + {0x391, 0x3A1}, {0x3A3, 0x3AB}, {0x3CF, 0x3CF}, {0x3D2, 0x3D4}, + {0x3D8, 0x3D8}, {0x3DA, 0x3DA}, {0x3DC, 0x3DC}, {0x3DE, 0x3DE}, + {0x3E0, 0x3E0}, {0x3E2, 0x3E2}, {0x3E4, 0x3E4}, {0x3E6, 0x3E6}, + {0x3E8, 0x3E8}, {0x3EA, 0x3EA}, {0x3EC, 0x3EC}, {0x3EE, 0x3EE}, + {0x3F4, 0x3F4}, {0x3F7, 0x3F7}, {0x3F9, 0x3FA}, {0x3FD, 0x42F}, + {0x460, 0x460}, {0x462, 0x462}, {0x464, 0x464}, {0x466, 0x466}, + {0x468, 0x468}, {0x46A, 0x46A}, {0x46C, 0x46C}, {0x46E, 0x46E}, + {0x470, 0x470}, {0x472, 0x472}, {0x474, 0x474}, {0x476, 0x476}, + {0x478, 0x478}, {0x47A, 0x47A}, {0x47C, 0x47C}, {0x47E, 0x47E}, + {0x480, 0x480}, {0x48A, 0x48A}, {0x48C, 0x48C}, {0x48E, 0x48E}, + {0x490, 0x490}, {0x492, 0x492}, {0x494, 0x494}, {0x496, 0x496}, + {0x498, 0x498}, {0x49A, 0x49A}, {0x49C, 0x49C}, {0x49E, 0x49E}, + {0x4A0, 0x4A0}, {0x4A2, 0x4A2}, {0x4A4, 0x4A4}, {0x4A6, 0x4A6}, + {0x4A8, 0x4A8}, {0x4AA, 0x4AA}, {0x4AC, 0x4AC}, {0x4AE, 0x4AE}, + {0x4B0, 0x4B0}, {0x4B2, 0x4B2}, {0x4B4, 0x4B4}, {0x4B6, 0x4B6}, + {0x4B8, 0x4B8}, {0x4BA, 0x4BA}, {0x4BC, 0x4BC}, {0x4BE, 0x4BE}, + {0x4C0, 0x4C1}, {0x4C3, 0x4C3}, {0x4C5, 0x4C5}, {0x4C7, 0x4C7}, + {0x4C9, 0x4C9}, {0x4CB, 0x4CB}, {0x4CD, 0x4CD}, {0x4D0, 0x4D0}, + {0x4D2, 0x4D2}, {0x4D4, 0x4D4}, {0x4D6, 0x4D6}, {0x4D8, 0x4D8}, + {0x4DA, 0x4DA}, {0x4DC, 0x4DC}, {0x4DE, 0x4DE}, {0x4E0, 0x4E0}, + {0x4E2, 0x4E2}, {0x4E4, 0x4E4}, {0x4E6, 0x4E6}, {0x4E8, 0x4E8}, + {0x4EA, 0x4EA}, {0x4EC, 0x4EC}, {0x4EE, 0x4EE}, {0x4F0, 0x4F0}, + {0x4F2, 0x4F2}, {0x4F4, 0x4F4}, {0x4F6, 0x4F6}, {0x4F8, 0x4F8}, + {0x4FA, 0x4FA}, {0x4FC, 0x4FC}, {0x4FE, 0x4FE}, {0x500, 0x500}, + {0x502, 0x502}, {0x504, 0x504}, {0x506, 0x506}, {0x508, 0x508}, + {0x50A, 0x50A}, {0x50C, 0x50C}, {0x50E, 0x50E}, {0x510, 0x510}, + {0x512, 0x512}, {0x514, 0x514}, {0x516, 0x516}, {0x518, 0x518}, + {0x51A, 0x51A}, {0x51C, 0x51C}, {0x51E, 0x51E}, {0x520, 0x520}, + {0x522, 0x522}, {0x524, 0x524}, {0x526, 0x526}, {0x528, 0x528}, + {0x52A, 0x52A}, {0x52C, 0x52C}, {0x52E, 0x52E}, {0x531, 0x556}, + {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x13A0, 0x13F5}, + {0x1C89, 0x1C89}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1E00, 0x1E00}, + {0x1E02, 0x1E02}, {0x1E04, 0x1E04}, {0x1E06, 0x1E06}, {0x1E08, 0x1E08}, + {0x1E0A, 0x1E0A}, {0x1E0C, 0x1E0C}, {0x1E0E, 0x1E0E}, {0x1E10, 0x1E10}, + {0x1E12, 0x1E12}, {0x1E14, 0x1E14}, {0x1E16, 0x1E16}, {0x1E18, 0x1E18}, + {0x1E1A, 0x1E1A}, {0x1E1C, 0x1E1C}, {0x1E1E, 0x1E1E}, {0x1E20, 0x1E20}, + {0x1E22, 0x1E22}, {0x1E24, 0x1E24}, {0x1E26, 0x1E26}, {0x1E28, 0x1E28}, + {0x1E2A, 0x1E2A}, {0x1E2C, 0x1E2C}, {0x1E2E, 0x1E2E}, {0x1E30, 0x1E30}, + {0x1E32, 0x1E32}, {0x1E34, 0x1E34}, {0x1E36, 0x1E36}, {0x1E38, 0x1E38}, + {0x1E3A, 0x1E3A}, {0x1E3C, 0x1E3C}, {0x1E3E, 0x1E3E}, {0x1E40, 0x1E40}, + {0x1E42, 0x1E42}, {0x1E44, 0x1E44}, {0x1E46, 0x1E46}, {0x1E48, 0x1E48}, + {0x1E4A, 0x1E4A}, {0x1E4C, 0x1E4C}, {0x1E4E, 0x1E4E}, {0x1E50, 0x1E50}, + {0x1E52, 0x1E52}, {0x1E54, 0x1E54}, {0x1E56, 0x1E56}, {0x1E58, 0x1E58}, + {0x1E5A, 0x1E5A}, {0x1E5C, 0x1E5C}, {0x1E5E, 0x1E5E}, {0x1E60, 0x1E60}, + {0x1E62, 0x1E62}, {0x1E64, 0x1E64}, {0x1E66, 0x1E66}, {0x1E68, 0x1E68}, + {0x1E6A, 0x1E6A}, {0x1E6C, 0x1E6C}, {0x1E6E, 0x1E6E}, {0x1E70, 0x1E70}, + {0x1E72, 0x1E72}, {0x1E74, 0x1E74}, {0x1E76, 0x1E76}, {0x1E78, 0x1E78}, + {0x1E7A, 0x1E7A}, {0x1E7C, 0x1E7C}, {0x1E7E, 0x1E7E}, {0x1E80, 0x1E80}, + {0x1E82, 0x1E82}, {0x1E84, 0x1E84}, {0x1E86, 0x1E86}, {0x1E88, 0x1E88}, + {0x1E8A, 0x1E8A}, {0x1E8C, 0x1E8C}, {0x1E8E, 0x1E8E}, {0x1E90, 0x1E90}, + {0x1E92, 0x1E92}, {0x1E94, 0x1E94}, {0x1E9E, 0x1E9E}, {0x1EA0, 0x1EA0}, + {0x1EA2, 0x1EA2}, {0x1EA4, 0x1EA4}, {0x1EA6, 0x1EA6}, {0x1EA8, 0x1EA8}, + {0x1EAA, 0x1EAA}, {0x1EAC, 0x1EAC}, {0x1EAE, 0x1EAE}, {0x1EB0, 0x1EB0}, + {0x1EB2, 0x1EB2}, {0x1EB4, 0x1EB4}, {0x1EB6, 0x1EB6}, {0x1EB8, 0x1EB8}, + {0x1EBA, 0x1EBA}, {0x1EBC, 0x1EBC}, {0x1EBE, 0x1EBE}, {0x1EC0, 0x1EC0}, + {0x1EC2, 0x1EC2}, {0x1EC4, 0x1EC4}, {0x1EC6, 0x1EC6}, {0x1EC8, 0x1EC8}, + {0x1ECA, 0x1ECA}, {0x1ECC, 0x1ECC}, {0x1ECE, 0x1ECE}, {0x1ED0, 0x1ED0}, + {0x1ED2, 0x1ED2}, {0x1ED4, 0x1ED4}, {0x1ED6, 0x1ED6}, {0x1ED8, 0x1ED8}, + {0x1EDA, 0x1EDA}, {0x1EDC, 0x1EDC}, {0x1EDE, 0x1EDE}, {0x1EE0, 0x1EE0}, + {0x1EE2, 0x1EE2}, {0x1EE4, 0x1EE4}, {0x1EE6, 0x1EE6}, {0x1EE8, 0x1EE8}, + {0x1EEA, 0x1EEA}, {0x1EEC, 0x1EEC}, {0x1EEE, 0x1EEE}, {0x1EF0, 0x1EF0}, + {0x1EF2, 0x1EF2}, {0x1EF4, 0x1EF4}, {0x1EF6, 0x1EF6}, {0x1EF8, 0x1EF8}, + {0x1EFA, 0x1EFA}, {0x1EFC, 0x1EFC}, {0x1EFE, 0x1EFE}, {0x1F08, 0x1F0F}, + {0x1F18, 0x1F1D}, {0x1F28, 0x1F2F}, {0x1F38, 0x1F3F}, {0x1F48, 0x1F4D}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F5F}, + {0x1F68, 0x1F6F}, {0x1FB8, 0x1FBB}, {0x1FC8, 0x1FCB}, {0x1FD8, 0x1FDB}, + {0x1FE8, 0x1FEC}, {0x1FF8, 0x1FFB}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210B, 0x210D}, {0x2110, 0x2112}, {0x2115, 0x2115}, {0x2119, 0x211D}, + {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, + {0x2130, 0x2133}, {0x213E, 0x213F}, {0x2145, 0x2145}, {0x2160, 0x216F}, + {0x2183, 0x2183}, {0x24B6, 0x24CF}, {0x2C00, 0x2C2F}, {0x2C60, 0x2C60}, + {0x2C62, 0x2C64}, {0x2C67, 0x2C67}, {0x2C69, 0x2C69}, {0x2C6B, 0x2C6B}, + {0x2C6D, 0x2C70}, {0x2C72, 0x2C72}, {0x2C75, 0x2C75}, {0x2C7E, 0x2C80}, + {0x2C82, 0x2C82}, {0x2C84, 0x2C84}, {0x2C86, 0x2C86}, {0x2C88, 0x2C88}, + {0x2C8A, 0x2C8A}, {0x2C8C, 0x2C8C}, {0x2C8E, 0x2C8E}, {0x2C90, 0x2C90}, + {0x2C92, 0x2C92}, {0x2C94, 0x2C94}, {0x2C96, 0x2C96}, {0x2C98, 0x2C98}, + {0x2C9A, 0x2C9A}, {0x2C9C, 0x2C9C}, {0x2C9E, 0x2C9E}, {0x2CA0, 0x2CA0}, + {0x2CA2, 0x2CA2}, {0x2CA4, 0x2CA4}, {0x2CA6, 0x2CA6}, {0x2CA8, 0x2CA8}, + {0x2CAA, 0x2CAA}, {0x2CAC, 0x2CAC}, {0x2CAE, 0x2CAE}, {0x2CB0, 0x2CB0}, + {0x2CB2, 0x2CB2}, {0x2CB4, 0x2CB4}, {0x2CB6, 0x2CB6}, {0x2CB8, 0x2CB8}, + {0x2CBA, 0x2CBA}, {0x2CBC, 0x2CBC}, {0x2CBE, 0x2CBE}, {0x2CC0, 0x2CC0}, + {0x2CC2, 0x2CC2}, {0x2CC4, 0x2CC4}, {0x2CC6, 0x2CC6}, {0x2CC8, 0x2CC8}, + {0x2CCA, 0x2CCA}, {0x2CCC, 0x2CCC}, {0x2CCE, 0x2CCE}, {0x2CD0, 0x2CD0}, + {0x2CD2, 0x2CD2}, {0x2CD4, 0x2CD4}, {0x2CD6, 0x2CD6}, {0x2CD8, 0x2CD8}, + {0x2CDA, 0x2CDA}, {0x2CDC, 0x2CDC}, {0x2CDE, 0x2CDE}, {0x2CE0, 0x2CE0}, + {0x2CE2, 0x2CE2}, {0x2CEB, 0x2CEB}, {0x2CED, 0x2CED}, {0x2CF2, 0x2CF2}, + {0xA640, 0xA640}, {0xA642, 0xA642}, {0xA644, 0xA644}, {0xA646, 0xA646}, + {0xA648, 0xA648}, {0xA64A, 0xA64A}, {0xA64C, 0xA64C}, {0xA64E, 0xA64E}, + {0xA650, 0xA650}, {0xA652, 0xA652}, {0xA654, 0xA654}, {0xA656, 0xA656}, + {0xA658, 0xA658}, {0xA65A, 0xA65A}, {0xA65C, 0xA65C}, {0xA65E, 0xA65E}, + {0xA660, 0xA660}, {0xA662, 0xA662}, {0xA664, 0xA664}, {0xA666, 0xA666}, + {0xA668, 0xA668}, {0xA66A, 0xA66A}, {0xA66C, 0xA66C}, {0xA680, 0xA680}, + {0xA682, 0xA682}, {0xA684, 0xA684}, {0xA686, 0xA686}, {0xA688, 0xA688}, + {0xA68A, 0xA68A}, {0xA68C, 0xA68C}, {0xA68E, 0xA68E}, {0xA690, 0xA690}, + {0xA692, 0xA692}, {0xA694, 0xA694}, {0xA696, 0xA696}, {0xA698, 0xA698}, + {0xA69A, 0xA69A}, {0xA722, 0xA722}, {0xA724, 0xA724}, {0xA726, 0xA726}, + {0xA728, 0xA728}, {0xA72A, 0xA72A}, {0xA72C, 0xA72C}, {0xA72E, 0xA72E}, + {0xA732, 0xA732}, {0xA734, 0xA734}, {0xA736, 0xA736}, {0xA738, 0xA738}, + {0xA73A, 0xA73A}, {0xA73C, 0xA73C}, {0xA73E, 0xA73E}, {0xA740, 0xA740}, + {0xA742, 0xA742}, {0xA744, 0xA744}, {0xA746, 0xA746}, {0xA748, 0xA748}, + {0xA74A, 0xA74A}, {0xA74C, 0xA74C}, {0xA74E, 0xA74E}, {0xA750, 0xA750}, + {0xA752, 0xA752}, {0xA754, 0xA754}, {0xA756, 0xA756}, {0xA758, 0xA758}, + {0xA75A, 0xA75A}, {0xA75C, 0xA75C}, {0xA75E, 0xA75E}, {0xA760, 0xA760}, + {0xA762, 0xA762}, {0xA764, 0xA764}, {0xA766, 0xA766}, {0xA768, 0xA768}, + {0xA76A, 0xA76A}, {0xA76C, 0xA76C}, {0xA76E, 0xA76E}, {0xA779, 0xA779}, + {0xA77B, 0xA77B}, {0xA77D, 0xA77E}, {0xA780, 0xA780}, {0xA782, 0xA782}, + {0xA784, 0xA784}, {0xA786, 0xA786}, {0xA78B, 0xA78B}, {0xA78D, 0xA78D}, + {0xA790, 0xA790}, {0xA792, 0xA792}, {0xA796, 0xA796}, {0xA798, 0xA798}, + {0xA79A, 0xA79A}, {0xA79C, 0xA79C}, {0xA79E, 0xA79E}, {0xA7A0, 0xA7A0}, + {0xA7A2, 0xA7A2}, {0xA7A4, 0xA7A4}, {0xA7A6, 0xA7A6}, {0xA7A8, 0xA7A8}, + {0xA7AA, 0xA7AE}, {0xA7B0, 0xA7B4}, {0xA7B6, 0xA7B6}, {0xA7B8, 0xA7B8}, + {0xA7BA, 0xA7BA}, {0xA7BC, 0xA7BC}, {0xA7BE, 0xA7BE}, {0xA7C0, 0xA7C0}, + {0xA7C2, 0xA7C2}, {0xA7C4, 0xA7C7}, {0xA7C9, 0xA7C9}, {0xA7CB, 0xA7CC}, + {0xA7D0, 0xA7D0}, {0xA7D6, 0xA7D6}, {0xA7D8, 0xA7D8}, {0xA7DA, 0xA7DA}, + {0xA7DC, 0xA7DC}, {0xA7F5, 0xA7F5}, {0xFF21, 0xFF3A}, {0x10400, 0x10427}, + {0x104B0, 0x104D3}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, + {0x10594, 0x10595}, {0x10C80, 0x10CB2}, {0x10D50, 0x10D65}, {0x118A0, 0x118BF}, + {0x16E40, 0x16E5F}, {0x1D400, 0x1D419}, {0x1D434, 0x1D44D}, {0x1D468, 0x1D481}, + {0x1D49C, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B5}, {0x1D4D0, 0x1D4E9}, {0x1D504, 0x1D505}, + {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D538, 0x1D539}, + {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D56C, 0x1D585}, {0x1D5A0, 0x1D5B9}, {0x1D5D4, 0x1D5ED}, {0x1D608, 0x1D621}, + {0x1D63C, 0x1D655}, {0x1D670, 0x1D689}, {0x1D6A8, 0x1D6C0}, {0x1D6E2, 0x1D6FA}, + {0x1D71C, 0x1D734}, {0x1D756, 0x1D76E}, {0x1D790, 0x1D7A8}, {0x1D7CA, 0x1D7CA}, + {0x1E900, 0x1E921}, {0x1F130, 0x1F149}, {0x1F150, 0x1F169}, {0x1F170, 0x1F189}, +}; +const int PDF_U_UPPER_COUNT = (int)(sizeof(PDF_U_UPPER) / sizeof(PDF_U_UPPER[0])); + +const pdf_urange_t PDF_U_LOWER[] = { + {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, {0xBA, 0xBA}, + {0xDF, 0xF6}, {0xF8, 0xFF}, {0x101, 0x101}, {0x103, 0x103}, + {0x105, 0x105}, {0x107, 0x107}, {0x109, 0x109}, {0x10B, 0x10B}, + {0x10D, 0x10D}, {0x10F, 0x10F}, {0x111, 0x111}, {0x113, 0x113}, + {0x115, 0x115}, {0x117, 0x117}, {0x119, 0x119}, {0x11B, 0x11B}, + {0x11D, 0x11D}, {0x11F, 0x11F}, {0x121, 0x121}, {0x123, 0x123}, + {0x125, 0x125}, {0x127, 0x127}, {0x129, 0x129}, {0x12B, 0x12B}, + {0x12D, 0x12D}, {0x12F, 0x12F}, {0x131, 0x131}, {0x133, 0x133}, + {0x135, 0x135}, {0x137, 0x138}, {0x13A, 0x13A}, {0x13C, 0x13C}, + {0x13E, 0x13E}, {0x140, 0x140}, {0x142, 0x142}, {0x144, 0x144}, + {0x146, 0x146}, {0x148, 0x149}, {0x14B, 0x14B}, {0x14D, 0x14D}, + {0x14F, 0x14F}, {0x151, 0x151}, {0x153, 0x153}, {0x155, 0x155}, + {0x157, 0x157}, {0x159, 0x159}, {0x15B, 0x15B}, {0x15D, 0x15D}, + {0x15F, 0x15F}, {0x161, 0x161}, {0x163, 0x163}, {0x165, 0x165}, + {0x167, 0x167}, {0x169, 0x169}, {0x16B, 0x16B}, {0x16D, 0x16D}, + {0x16F, 0x16F}, {0x171, 0x171}, {0x173, 0x173}, {0x175, 0x175}, + {0x177, 0x177}, {0x17A, 0x17A}, {0x17C, 0x17C}, {0x17E, 0x180}, + {0x183, 0x183}, {0x185, 0x185}, {0x188, 0x188}, {0x18C, 0x18D}, + {0x192, 0x192}, {0x195, 0x195}, {0x199, 0x19B}, {0x19E, 0x19E}, + {0x1A1, 0x1A1}, {0x1A3, 0x1A3}, {0x1A5, 0x1A5}, {0x1A8, 0x1A8}, + {0x1AA, 0x1AB}, {0x1AD, 0x1AD}, {0x1B0, 0x1B0}, {0x1B4, 0x1B4}, + {0x1B6, 0x1B6}, {0x1B9, 0x1BA}, {0x1BD, 0x1BF}, {0x1C6, 0x1C6}, + {0x1C9, 0x1C9}, {0x1CC, 0x1CC}, {0x1CE, 0x1CE}, {0x1D0, 0x1D0}, + {0x1D2, 0x1D2}, {0x1D4, 0x1D4}, {0x1D6, 0x1D6}, {0x1D8, 0x1D8}, + {0x1DA, 0x1DA}, {0x1DC, 0x1DD}, {0x1DF, 0x1DF}, {0x1E1, 0x1E1}, + {0x1E3, 0x1E3}, {0x1E5, 0x1E5}, {0x1E7, 0x1E7}, {0x1E9, 0x1E9}, + {0x1EB, 0x1EB}, {0x1ED, 0x1ED}, {0x1EF, 0x1F0}, {0x1F3, 0x1F3}, + {0x1F5, 0x1F5}, {0x1F9, 0x1F9}, {0x1FB, 0x1FB}, {0x1FD, 0x1FD}, + {0x1FF, 0x1FF}, {0x201, 0x201}, {0x203, 0x203}, {0x205, 0x205}, + {0x207, 0x207}, {0x209, 0x209}, {0x20B, 0x20B}, {0x20D, 0x20D}, + {0x20F, 0x20F}, {0x211, 0x211}, {0x213, 0x213}, {0x215, 0x215}, + {0x217, 0x217}, {0x219, 0x219}, {0x21B, 0x21B}, {0x21D, 0x21D}, + {0x21F, 0x21F}, {0x221, 0x221}, {0x223, 0x223}, {0x225, 0x225}, + {0x227, 0x227}, {0x229, 0x229}, {0x22B, 0x22B}, {0x22D, 0x22D}, + {0x22F, 0x22F}, {0x231, 0x231}, {0x233, 0x239}, {0x23C, 0x23C}, + {0x23F, 0x240}, {0x242, 0x242}, {0x247, 0x247}, {0x249, 0x249}, + {0x24B, 0x24B}, {0x24D, 0x24D}, {0x24F, 0x293}, {0x295, 0x2B8}, + {0x2C0, 0x2C1}, {0x2E0, 0x2E4}, {0x345, 0x345}, {0x371, 0x371}, + {0x373, 0x373}, {0x377, 0x377}, {0x37A, 0x37D}, {0x390, 0x390}, + {0x3AC, 0x3CE}, {0x3D0, 0x3D1}, {0x3D5, 0x3D7}, {0x3D9, 0x3D9}, + {0x3DB, 0x3DB}, {0x3DD, 0x3DD}, {0x3DF, 0x3DF}, {0x3E1, 0x3E1}, + {0x3E3, 0x3E3}, {0x3E5, 0x3E5}, {0x3E7, 0x3E7}, {0x3E9, 0x3E9}, + {0x3EB, 0x3EB}, {0x3ED, 0x3ED}, {0x3EF, 0x3F3}, {0x3F5, 0x3F5}, + {0x3F8, 0x3F8}, {0x3FB, 0x3FC}, {0x430, 0x45F}, {0x461, 0x461}, + {0x463, 0x463}, {0x465, 0x465}, {0x467, 0x467}, {0x469, 0x469}, + {0x46B, 0x46B}, {0x46D, 0x46D}, {0x46F, 0x46F}, {0x471, 0x471}, + {0x473, 0x473}, {0x475, 0x475}, {0x477, 0x477}, {0x479, 0x479}, + {0x47B, 0x47B}, {0x47D, 0x47D}, {0x47F, 0x47F}, {0x481, 0x481}, + {0x48B, 0x48B}, {0x48D, 0x48D}, {0x48F, 0x48F}, {0x491, 0x491}, + {0x493, 0x493}, {0x495, 0x495}, {0x497, 0x497}, {0x499, 0x499}, + {0x49B, 0x49B}, {0x49D, 0x49D}, {0x49F, 0x49F}, {0x4A1, 0x4A1}, + {0x4A3, 0x4A3}, {0x4A5, 0x4A5}, {0x4A7, 0x4A7}, {0x4A9, 0x4A9}, + {0x4AB, 0x4AB}, {0x4AD, 0x4AD}, {0x4AF, 0x4AF}, {0x4B1, 0x4B1}, + {0x4B3, 0x4B3}, {0x4B5, 0x4B5}, {0x4B7, 0x4B7}, {0x4B9, 0x4B9}, + {0x4BB, 0x4BB}, {0x4BD, 0x4BD}, {0x4BF, 0x4BF}, {0x4C2, 0x4C2}, + {0x4C4, 0x4C4}, {0x4C6, 0x4C6}, {0x4C8, 0x4C8}, {0x4CA, 0x4CA}, + {0x4CC, 0x4CC}, {0x4CE, 0x4CF}, {0x4D1, 0x4D1}, {0x4D3, 0x4D3}, + {0x4D5, 0x4D5}, {0x4D7, 0x4D7}, {0x4D9, 0x4D9}, {0x4DB, 0x4DB}, + {0x4DD, 0x4DD}, {0x4DF, 0x4DF}, {0x4E1, 0x4E1}, {0x4E3, 0x4E3}, + {0x4E5, 0x4E5}, {0x4E7, 0x4E7}, {0x4E9, 0x4E9}, {0x4EB, 0x4EB}, + {0x4ED, 0x4ED}, {0x4EF, 0x4EF}, {0x4F1, 0x4F1}, {0x4F3, 0x4F3}, + {0x4F5, 0x4F5}, {0x4F7, 0x4F7}, {0x4F9, 0x4F9}, {0x4FB, 0x4FB}, + {0x4FD, 0x4FD}, {0x4FF, 0x4FF}, {0x501, 0x501}, {0x503, 0x503}, + {0x505, 0x505}, {0x507, 0x507}, {0x509, 0x509}, {0x50B, 0x50B}, + {0x50D, 0x50D}, {0x50F, 0x50F}, {0x511, 0x511}, {0x513, 0x513}, + {0x515, 0x515}, {0x517, 0x517}, {0x519, 0x519}, {0x51B, 0x51B}, + {0x51D, 0x51D}, {0x51F, 0x51F}, {0x521, 0x521}, {0x523, 0x523}, + {0x525, 0x525}, {0x527, 0x527}, {0x529, 0x529}, {0x52B, 0x52B}, + {0x52D, 0x52D}, {0x52F, 0x52F}, {0x560, 0x588}, {0x10D0, 0x10FA}, + {0x10FC, 0x10FF}, {0x13F8, 0x13FD}, {0x1C80, 0x1C88}, {0x1C8A, 0x1C8A}, + {0x1D00, 0x1DBF}, {0x1E01, 0x1E01}, {0x1E03, 0x1E03}, {0x1E05, 0x1E05}, + {0x1E07, 0x1E07}, {0x1E09, 0x1E09}, {0x1E0B, 0x1E0B}, {0x1E0D, 0x1E0D}, + {0x1E0F, 0x1E0F}, {0x1E11, 0x1E11}, {0x1E13, 0x1E13}, {0x1E15, 0x1E15}, + {0x1E17, 0x1E17}, {0x1E19, 0x1E19}, {0x1E1B, 0x1E1B}, {0x1E1D, 0x1E1D}, + {0x1E1F, 0x1E1F}, {0x1E21, 0x1E21}, {0x1E23, 0x1E23}, {0x1E25, 0x1E25}, + {0x1E27, 0x1E27}, {0x1E29, 0x1E29}, {0x1E2B, 0x1E2B}, {0x1E2D, 0x1E2D}, + {0x1E2F, 0x1E2F}, {0x1E31, 0x1E31}, {0x1E33, 0x1E33}, {0x1E35, 0x1E35}, + {0x1E37, 0x1E37}, {0x1E39, 0x1E39}, {0x1E3B, 0x1E3B}, {0x1E3D, 0x1E3D}, + {0x1E3F, 0x1E3F}, {0x1E41, 0x1E41}, {0x1E43, 0x1E43}, {0x1E45, 0x1E45}, + {0x1E47, 0x1E47}, {0x1E49, 0x1E49}, {0x1E4B, 0x1E4B}, {0x1E4D, 0x1E4D}, + {0x1E4F, 0x1E4F}, {0x1E51, 0x1E51}, {0x1E53, 0x1E53}, {0x1E55, 0x1E55}, + {0x1E57, 0x1E57}, {0x1E59, 0x1E59}, {0x1E5B, 0x1E5B}, {0x1E5D, 0x1E5D}, + {0x1E5F, 0x1E5F}, {0x1E61, 0x1E61}, {0x1E63, 0x1E63}, {0x1E65, 0x1E65}, + {0x1E67, 0x1E67}, {0x1E69, 0x1E69}, {0x1E6B, 0x1E6B}, {0x1E6D, 0x1E6D}, + {0x1E6F, 0x1E6F}, {0x1E71, 0x1E71}, {0x1E73, 0x1E73}, {0x1E75, 0x1E75}, + {0x1E77, 0x1E77}, {0x1E79, 0x1E79}, {0x1E7B, 0x1E7B}, {0x1E7D, 0x1E7D}, + {0x1E7F, 0x1E7F}, {0x1E81, 0x1E81}, {0x1E83, 0x1E83}, {0x1E85, 0x1E85}, + {0x1E87, 0x1E87}, {0x1E89, 0x1E89}, {0x1E8B, 0x1E8B}, {0x1E8D, 0x1E8D}, + {0x1E8F, 0x1E8F}, {0x1E91, 0x1E91}, {0x1E93, 0x1E93}, {0x1E95, 0x1E9D}, + {0x1E9F, 0x1E9F}, {0x1EA1, 0x1EA1}, {0x1EA3, 0x1EA3}, {0x1EA5, 0x1EA5}, + {0x1EA7, 0x1EA7}, {0x1EA9, 0x1EA9}, {0x1EAB, 0x1EAB}, {0x1EAD, 0x1EAD}, + {0x1EAF, 0x1EAF}, {0x1EB1, 0x1EB1}, {0x1EB3, 0x1EB3}, {0x1EB5, 0x1EB5}, + {0x1EB7, 0x1EB7}, {0x1EB9, 0x1EB9}, {0x1EBB, 0x1EBB}, {0x1EBD, 0x1EBD}, + {0x1EBF, 0x1EBF}, {0x1EC1, 0x1EC1}, {0x1EC3, 0x1EC3}, {0x1EC5, 0x1EC5}, + {0x1EC7, 0x1EC7}, {0x1EC9, 0x1EC9}, {0x1ECB, 0x1ECB}, {0x1ECD, 0x1ECD}, + {0x1ECF, 0x1ECF}, {0x1ED1, 0x1ED1}, {0x1ED3, 0x1ED3}, {0x1ED5, 0x1ED5}, + {0x1ED7, 0x1ED7}, {0x1ED9, 0x1ED9}, {0x1EDB, 0x1EDB}, {0x1EDD, 0x1EDD}, + {0x1EDF, 0x1EDF}, {0x1EE1, 0x1EE1}, {0x1EE3, 0x1EE3}, {0x1EE5, 0x1EE5}, + {0x1EE7, 0x1EE7}, {0x1EE9, 0x1EE9}, {0x1EEB, 0x1EEB}, {0x1EED, 0x1EED}, + {0x1EEF, 0x1EEF}, {0x1EF1, 0x1EF1}, {0x1EF3, 0x1EF3}, {0x1EF5, 0x1EF5}, + {0x1EF7, 0x1EF7}, {0x1EF9, 0x1EF9}, {0x1EFB, 0x1EFB}, {0x1EFD, 0x1EFD}, + {0x1EFF, 0x1F07}, {0x1F10, 0x1F15}, {0x1F20, 0x1F27}, {0x1F30, 0x1F37}, + {0x1F40, 0x1F45}, {0x1F50, 0x1F57}, {0x1F60, 0x1F67}, {0x1F70, 0x1F7D}, + {0x1F80, 0x1F87}, {0x1F90, 0x1F97}, {0x1FA0, 0x1FA7}, {0x1FB0, 0x1FB4}, + {0x1FB6, 0x1FB7}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FC7}, + {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FD7}, {0x1FE0, 0x1FE7}, {0x1FF2, 0x1FF4}, + {0x1FF6, 0x1FF7}, {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, + {0x210A, 0x210A}, {0x210E, 0x210F}, {0x2113, 0x2113}, {0x212F, 0x212F}, + {0x2134, 0x2134}, {0x2139, 0x2139}, {0x213C, 0x213D}, {0x2146, 0x2149}, + {0x214E, 0x214E}, {0x2170, 0x217F}, {0x2184, 0x2184}, {0x24D0, 0x24E9}, + {0x2C30, 0x2C5F}, {0x2C61, 0x2C61}, {0x2C65, 0x2C66}, {0x2C68, 0x2C68}, + {0x2C6A, 0x2C6A}, {0x2C6C, 0x2C6C}, {0x2C71, 0x2C71}, {0x2C73, 0x2C74}, + {0x2C76, 0x2C7D}, {0x2C81, 0x2C81}, {0x2C83, 0x2C83}, {0x2C85, 0x2C85}, + {0x2C87, 0x2C87}, {0x2C89, 0x2C89}, {0x2C8B, 0x2C8B}, {0x2C8D, 0x2C8D}, + {0x2C8F, 0x2C8F}, {0x2C91, 0x2C91}, {0x2C93, 0x2C93}, {0x2C95, 0x2C95}, + {0x2C97, 0x2C97}, {0x2C99, 0x2C99}, {0x2C9B, 0x2C9B}, {0x2C9D, 0x2C9D}, + {0x2C9F, 0x2C9F}, {0x2CA1, 0x2CA1}, {0x2CA3, 0x2CA3}, {0x2CA5, 0x2CA5}, + {0x2CA7, 0x2CA7}, {0x2CA9, 0x2CA9}, {0x2CAB, 0x2CAB}, {0x2CAD, 0x2CAD}, + {0x2CAF, 0x2CAF}, {0x2CB1, 0x2CB1}, {0x2CB3, 0x2CB3}, {0x2CB5, 0x2CB5}, + {0x2CB7, 0x2CB7}, {0x2CB9, 0x2CB9}, {0x2CBB, 0x2CBB}, {0x2CBD, 0x2CBD}, + {0x2CBF, 0x2CBF}, {0x2CC1, 0x2CC1}, {0x2CC3, 0x2CC3}, {0x2CC5, 0x2CC5}, + {0x2CC7, 0x2CC7}, {0x2CC9, 0x2CC9}, {0x2CCB, 0x2CCB}, {0x2CCD, 0x2CCD}, + {0x2CCF, 0x2CCF}, {0x2CD1, 0x2CD1}, {0x2CD3, 0x2CD3}, {0x2CD5, 0x2CD5}, + {0x2CD7, 0x2CD7}, {0x2CD9, 0x2CD9}, {0x2CDB, 0x2CDB}, {0x2CDD, 0x2CDD}, + {0x2CDF, 0x2CDF}, {0x2CE1, 0x2CE1}, {0x2CE3, 0x2CE4}, {0x2CEC, 0x2CEC}, + {0x2CEE, 0x2CEE}, {0x2CF3, 0x2CF3}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0xA641, 0xA641}, {0xA643, 0xA643}, {0xA645, 0xA645}, + {0xA647, 0xA647}, {0xA649, 0xA649}, {0xA64B, 0xA64B}, {0xA64D, 0xA64D}, + {0xA64F, 0xA64F}, {0xA651, 0xA651}, {0xA653, 0xA653}, {0xA655, 0xA655}, + {0xA657, 0xA657}, {0xA659, 0xA659}, {0xA65B, 0xA65B}, {0xA65D, 0xA65D}, + {0xA65F, 0xA65F}, {0xA661, 0xA661}, {0xA663, 0xA663}, {0xA665, 0xA665}, + {0xA667, 0xA667}, {0xA669, 0xA669}, {0xA66B, 0xA66B}, {0xA66D, 0xA66D}, + {0xA681, 0xA681}, {0xA683, 0xA683}, {0xA685, 0xA685}, {0xA687, 0xA687}, + {0xA689, 0xA689}, {0xA68B, 0xA68B}, {0xA68D, 0xA68D}, {0xA68F, 0xA68F}, + {0xA691, 0xA691}, {0xA693, 0xA693}, {0xA695, 0xA695}, {0xA697, 0xA697}, + {0xA699, 0xA699}, {0xA69B, 0xA69D}, {0xA723, 0xA723}, {0xA725, 0xA725}, + {0xA727, 0xA727}, {0xA729, 0xA729}, {0xA72B, 0xA72B}, {0xA72D, 0xA72D}, + {0xA72F, 0xA731}, {0xA733, 0xA733}, {0xA735, 0xA735}, {0xA737, 0xA737}, + {0xA739, 0xA739}, {0xA73B, 0xA73B}, {0xA73D, 0xA73D}, {0xA73F, 0xA73F}, + {0xA741, 0xA741}, {0xA743, 0xA743}, {0xA745, 0xA745}, {0xA747, 0xA747}, + {0xA749, 0xA749}, {0xA74B, 0xA74B}, {0xA74D, 0xA74D}, {0xA74F, 0xA74F}, + {0xA751, 0xA751}, {0xA753, 0xA753}, {0xA755, 0xA755}, {0xA757, 0xA757}, + {0xA759, 0xA759}, {0xA75B, 0xA75B}, {0xA75D, 0xA75D}, {0xA75F, 0xA75F}, + {0xA761, 0xA761}, {0xA763, 0xA763}, {0xA765, 0xA765}, {0xA767, 0xA767}, + {0xA769, 0xA769}, {0xA76B, 0xA76B}, {0xA76D, 0xA76D}, {0xA76F, 0xA778}, + {0xA77A, 0xA77A}, {0xA77C, 0xA77C}, {0xA77F, 0xA77F}, {0xA781, 0xA781}, + {0xA783, 0xA783}, {0xA785, 0xA785}, {0xA787, 0xA787}, {0xA78C, 0xA78C}, + {0xA78E, 0xA78E}, {0xA791, 0xA791}, {0xA793, 0xA795}, {0xA797, 0xA797}, + {0xA799, 0xA799}, {0xA79B, 0xA79B}, {0xA79D, 0xA79D}, {0xA79F, 0xA79F}, + {0xA7A1, 0xA7A1}, {0xA7A3, 0xA7A3}, {0xA7A5, 0xA7A5}, {0xA7A7, 0xA7A7}, + {0xA7A9, 0xA7A9}, {0xA7AF, 0xA7AF}, {0xA7B5, 0xA7B5}, {0xA7B7, 0xA7B7}, + {0xA7B9, 0xA7B9}, {0xA7BB, 0xA7BB}, {0xA7BD, 0xA7BD}, {0xA7BF, 0xA7BF}, + {0xA7C1, 0xA7C1}, {0xA7C3, 0xA7C3}, {0xA7C8, 0xA7C8}, {0xA7CA, 0xA7CA}, + {0xA7CD, 0xA7CD}, {0xA7D1, 0xA7D1}, {0xA7D3, 0xA7D3}, {0xA7D5, 0xA7D5}, + {0xA7D7, 0xA7D7}, {0xA7D9, 0xA7D9}, {0xA7DB, 0xA7DB}, {0xA7F2, 0xA7F4}, + {0xA7F6, 0xA7F6}, {0xA7F8, 0xA7FA}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, + {0xAB70, 0xABBF}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFF41, 0xFF5A}, + {0x10428, 0x1044F}, {0x104D8, 0x104FB}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, + {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x10780, 0x10780}, {0x10783, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10CC0, 0x10CF2}, {0x10D70, 0x10D85}, + {0x118C0, 0x118DF}, {0x16E60, 0x16E7F}, {0x1D41A, 0x1D433}, {0x1D44E, 0x1D454}, + {0x1D456, 0x1D467}, {0x1D482, 0x1D49B}, {0x1D4B6, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D4CF}, {0x1D4EA, 0x1D503}, {0x1D51E, 0x1D537}, + {0x1D552, 0x1D56B}, {0x1D586, 0x1D59F}, {0x1D5BA, 0x1D5D3}, {0x1D5EE, 0x1D607}, + {0x1D622, 0x1D63B}, {0x1D656, 0x1D66F}, {0x1D68A, 0x1D6A5}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6E1}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D71B}, {0x1D736, 0x1D74E}, + {0x1D750, 0x1D755}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D78F}, {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7C9}, {0x1D7CB, 0x1D7CB}, {0x1DF00, 0x1DF09}, {0x1DF0B, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E922, 0x1E943}, +}; +const int PDF_U_LOWER_COUNT = (int)(sizeof(PDF_U_LOWER) / sizeof(PDF_U_LOWER[0])); + +const pdf_urange_t PDF_U_TITLE[] = { + {0x1C5, 0x1C5}, {0x1C8, 0x1C8}, {0x1CB, 0x1CB}, {0x1F2, 0x1F2}, {0x1F88, 0x1F8F}, + {0x1F98, 0x1F9F}, {0x1FA8, 0x1FAF}, {0x1FBC, 0x1FBC}, {0x1FCC, 0x1FCC}, {0x1FFC, 0x1FFC}, +}; +const int PDF_U_TITLE_COUNT = (int)(sizeof(PDF_U_TITLE) / sizeof(PDF_U_TITLE[0])); diff --git a/internal/cbm/pdf/pdf_unicode.h b/internal/cbm/pdf/pdf_unicode.h new file mode 100644 index 0000000000..12194b9c55 --- /dev/null +++ b/internal/cbm/pdf/pdf_unicode.h @@ -0,0 +1,103 @@ +/* + * pdf_unicode.h — the Unicode facts the PDF mention scanner needs, exactly + * as CPython answers them (pdf_unicode.c is generated from CPython's + * unicodedata; the field-tested scanner ran on CPython's str and re). + */ +#ifndef CBM_PDF_UNICODE_H +#define CBM_PDF_UNICODE_H + +#include +#include + +typedef struct { + uint32_t cp; + uint16_t off; /* into PDF_NFKC_SEQ */ + uint8_t len; +} pdf_nfkc_t; + +typedef struct { + uint32_t first; + uint32_t second; + uint32_t composite; +} pdf_ucomp_t; + +typedef struct { + uint32_t lo; + uint32_t hi; +} pdf_urange_t; + +extern const pdf_nfkc_t PDF_NFKC[]; /* sorted by cp: NFKC of one code point where it differs */ +extern const int PDF_NFKC_COUNT; +extern const uint32_t PDF_NFKC_SEQ[]; +extern const pdf_ucomp_t PDF_UCOMP[]; /* sorted by (first, second): canonical compositions */ +extern const int PDF_UCOMP_COUNT; +extern const pdf_urange_t PDF_U_SPACE[]; /* str.isspace */ +extern const int PDF_U_SPACE_COUNT; +extern const pdf_urange_t PDF_U_WORD[]; /* re's \w: str.isalnum or '_' */ +extern const int PDF_U_WORD_COUNT; +extern const pdf_urange_t PDF_U_ALPHA[]; /* str.isalpha */ +extern const int PDF_U_ALPHA_COUNT; +extern const pdf_urange_t PDF_U_UPPER[]; /* one-character str.isupper */ +extern const int PDF_U_UPPER_COUNT; +extern const pdf_urange_t PDF_U_LOWER[]; /* one-character str.islower */ +extern const int PDF_U_LOWER_COUNT; +extern const pdf_urange_t PDF_U_TITLE[]; /* titlecase letters (neither upper nor lower) */ +extern const int PDF_U_TITLE_COUNT; + +static inline bool pdf_u_in(const pdf_urange_t *r, int n, uint32_t cp) { + int lo = 0; + int hi = n - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + if (r[mid].hi < cp) { + lo = mid + 1; + } else if (r[mid].lo > cp) { + hi = mid - 1; + } else { + return true; + } + } + return false; +} + +static inline bool pdf_u_space(uint32_t c) { + if (c < 0x80) { + return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F); + } + return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, c); +} + +static inline bool pdf_u_word(uint32_t c) { + if (c < 0x80) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_'; + } + return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, c); +} + +static inline bool pdf_u_alpha(uint32_t c) { + if (c < 0x80) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + return pdf_u_in(PDF_U_ALPHA, PDF_U_ALPHA_COUNT, c); +} + +static inline bool pdf_u_upper(uint32_t c) { + if (c < 0x80) { + return c >= 'A' && c <= 'Z'; + } + return pdf_u_in(PDF_U_UPPER, PDF_U_UPPER_COUNT, c); +} + +static inline bool pdf_u_lower(uint32_t c) { + if (c < 0x80) { + return c >= 'a' && c <= 'z'; + } + return pdf_u_in(PDF_U_LOWER, PDF_U_LOWER_COUNT, c); +} + +static inline bool pdf_u_title(uint32_t c) { + return c >= 0x80 && pdf_u_in(PDF_U_TITLE, PDF_U_TITLE_COUNT, c); +} + +#endif /* CBM_PDF_UNICODE_H */ diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 9691b6c961..438a5d5354 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -276,6 +276,7 @@ static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) { cr_str(c, &d->http_client); cr_str(c, &d->http_base_url); cr_str(c, &d->variants); + cr_list(c, &d->extra_props); } static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { diff --git a/scripts/ci/generate-sbom.py b/scripts/ci/generate-sbom.py index 31f333c147..ff2e97efd9 100644 --- a/scripts/ci/generate-sbom.py +++ b/scripts/ci/generate-sbom.py @@ -49,6 +49,9 @@ {"SPDXID": "SPDXRef-Package-simplecpp", "name": "simplecpp", "versionInfo": "1.x", "licenseDeclared": "0BSD", "downloadLocation": "https://github.com/danmar/simplecpp", "filesAnalyzed": False}, {"SPDXID": "SPDXRef-Package-verstable", "name": "verstable", "versionInfo": "2.2.1", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/JacksonAllan/Verstable", "filesAnalyzed": False}, {"SPDXID": "SPDXRef-Package-wyhash", "name": "wyhash", "versionInfo": "final-4.3", "licenseDeclared": "Unlicense", "downloadLocation": "https://github.com/wangyi-fudan/wyhash", "filesAnalyzed": False}, + {"SPDXID": "SPDXRef-Package-adobe-glyph-list", "name": "adobe-glyph-list", "versionInfo": "2.0", "licenseDeclared": "BSD-3-Clause", "downloadLocation": "https://github.com/adobe-type-tools/agl-aglfn", "filesAnalyzed": False, "comment": "Glyph-name subset in internal/cbm/pdf/pdf_tables.c (PDF text layer); see THIRD_PARTY.md"}, + {"SPDXID": "SPDXRef-Package-adobe-core14-metrics", "name": "adobe-core14-afm-metrics", "versionInfo": "1997", "licenseDeclared": "NOASSERTION", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "comment": "Printable-ASCII advance widths of Helvetica and Times-Roman in internal/cbm/pdf/pdf_tables.c; (c) 1985-1997 Adobe Systems Incorporated; the AFM files are not distributed; see THIRD_PARTY.md"}, + {"SPDXID": "SPDXRef-Package-unicode-ucd", "name": "unicode-character-database", "versionInfo": "16.0.0", "licenseDeclared": "Unicode-3.0", "downloadLocation": "https://www.unicode.org/Public/16.0.0/ucd/", "filesAnalyzed": False, "comment": "NFKC, compositions and character classes in internal/cbm/pdf/pdf_unicode.c, generated by scripts/gen-pdf-unicode.py; see THIRD_PARTY.md"}, {"SPDXID": "SPDXRef-Package-nomic-embed-code", "name": "nomic-embed-code-token-embeddings", "versionInfo": "1.0", "licenseDeclared": "Apache-2.0", "downloadLocation": "https://huggingface.co/nomic-ai/nomic-embed-code", "filesAnalyzed": False, "comment": "Derived int8 token embeddings; see vendored/nomic/NOTICE"}, {"SPDXID": "SPDXRef-Package-tree-sitter-grammars", "name": "tree-sitter-grammars-aggregate", "versionInfo": f"{n_grammars}-grammars", "licenseDeclared": "MIT", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "comment": f"Aggregate of {n_grammars} vendored tree-sitter grammars, compiled statically. Predominantly MIT; non-MIT families present include CC0-1.0 (clojure, fennel), Apache-2.0 (elixir, erlang, gleam, hcl, ini, jinja2, just, pkl, sway, wit), ISC (pine, templ), Unlicense (fish); first-party grammars (c) DeusData, MIT. This summary is non-exhaustive; the authoritative complete per-grammar license list is internal/cbm/vendored/grammars/MANIFEST.md, and full license texts ship in THIRD_PARTY_NOTICES.md inside each archive."}, ], diff --git a/scripts/gen-pdf-unicode.py b/scripts/gen-pdf-unicode.py new file mode 100644 index 0000000000..d31417e2c9 --- /dev/null +++ b/scripts/gen-pdf-unicode.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Generate internal/cbm/pdf/pdf_unicode.c: the Unicode facts the PDF mention +scanner needs, exactly as CPython sees them (the field test's scanner ran on +CPython's str and re). + +usage: scripts/gen-pdf-unicode.py internal/cbm/pdf/pdf_unicode.c (then clang-format -i it) +The output depends on the CPython version (its Unicode data): the committed +tables were made with CPython 3.14.6 (Unicode 16.0.0). +""" +import sys +import unicodedata + +MAXCP = 0x110000 + + +def is_surrogate(c): + return 0xD800 <= c <= 0xDFFF + + +def ranges(pred): + out = [] + start = None + for c in range(MAXCP): + ok = not is_surrogate(c) and pred(chr(c)) + if ok and start is None: + start = c + elif not ok and start is not None: + out.append((start, c - 1)) + start = None + if start is not None: + out.append((start, MAXCP - 1)) + return out + + +def emit_ranges(f, name, rs): + f.write("const pdf_urange_t %s[] = {\n" % name) + for i in range(0, len(rs), 4): + f.write(" " + " ".join("{0x%X, 0x%X}," % r for r in rs[i:i + 4]) + "\n") + f.write("};\nconst int %s_COUNT = (int)(sizeof(%s) / sizeof(%s[0]));\n\n" % (name, name, name)) + + +def main(): + out = sys.argv[1] + # per-character NFKC + nfkc = [] + seq = [] + for c in range(MAXCP): + if is_surrogate(c): + continue + ch = chr(c) + m = unicodedata.normalize("NFKC", ch) + if m != ch: + nfkc.append((c, len(seq), len(m))) + seq.extend(ord(x) for x in m) + # canonical composition pairs: a + b -> c when NFC composes them + comp = [] + for c in range(MAXCP): + if is_surrogate(c): + continue + d = unicodedata.decomposition(chr(c)) + if not d or d.startswith("<"): + continue + parts = [int(x, 16) for x in d.split()] + if len(parts) != 2: + continue + a, b = parts + if unicodedata.normalize("NFC", chr(a) + chr(b)) == chr(c): + comp.append((a, b, c)) + comp.sort() + with open(out, "w", encoding="utf-8") as f: + f.write("/*\n * pdf_unicode.c -- Unicode %s facts for the PDF mention scanner (generated).\n" + " *\n * The field test's scanner ran on CPython's str and re; these tables are what\n" + " * CPython %s answers: NFKC of each code point, the canonical compositions,\n" + " * str.isspace, the re module's \\w (str.isalnum or '_'), str.isalpha, and the\n" + " * one-character str.isupper / str.islower / titlecase. Generated by\n" + " * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode\n" + " * Character Database, (c) Unicode, Inc., Unicode License v3\n" + " * (THIRD_PARTY.md).\n */\n" + % (unicodedata.unidata_version, sys.version.split()[0])) + f.write("#include \"pdf_unicode.h\"\n\n") + f.write("const pdf_nfkc_t PDF_NFKC[] = {\n") + for i in range(0, len(nfkc), 4): + f.write(" " + " ".join("{0x%X, %d, %d}," % e for e in nfkc[i:i + 4]) + "\n") + f.write("};\nconst int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0]));\n\n") + f.write("const uint32_t PDF_NFKC_SEQ[] = {\n") + for i in range(0, len(seq), 10): + f.write(" " + " ".join("0x%X," % x for x in seq[i:i + 10]) + "\n") + f.write("};\n\n") + f.write("const pdf_ucomp_t PDF_UCOMP[] = {\n") + for i in range(0, len(comp), 3): + f.write(" " + " ".join("{0x%X, 0x%X, 0x%X}," % e for e in comp[i:i + 3]) + "\n") + f.write("};\nconst int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0]));\n\n") + emit_ranges(f, "PDF_U_SPACE", ranges(lambda c: c.isspace())) + emit_ranges(f, "PDF_U_WORD", ranges(lambda c: c.isalnum() or c == "_")) + emit_ranges(f, "PDF_U_ALPHA", ranges(lambda c: c.isalpha())) + emit_ranges(f, "PDF_U_UPPER", ranges(lambda c: c.isupper())) + emit_ranges(f, "PDF_U_LOWER", ranges(lambda c: c.islower())) + emit_ranges(f, "PDF_U_TITLE", ranges(lambda c: c.istitle() and not c.isupper())) + print("nfkc", len(nfkc), "seq", len(seq), "comp", len(comp)) + + +if __name__ == "__main__": + main() diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index a44ce0d2b8..5bd6dbd401 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -79,7 +79,7 @@ src/pipeline/pass_usages.c 4 src/pipeline/path_alias.c 42 src/pipeline/pipeline.c 134 src/pipeline/pipeline_delta.c 19 -src/pipeline/pipeline_incremental.c 144 +src/pipeline/pipeline_incremental.c 143 src/pipeline/registry.c 17 src/pipeline/worker_pool.c 2 src/store/store.c 358 diff --git a/src/discover/discover.c b/src/discover/discover.c index fa8e4bf2b5..c8fcb850a8 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -15,6 +15,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/mem_core.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -950,13 +951,159 @@ static int safe_stat(const char *abs_path, struct stat *st, bool *is_symlink) { #endif } +/* ── Sphinx documentation sets ──────────────────────────────────── + * A directory with a conf.py is a Sphinx documentation set, and its + * `source_suffix` says which file suffixes are reStructuredText (Django's + * docs are `.txt`). A file of such a suffix in the set's subtree, of no + * language by its name, is reST; a nested conf.py starts its own set. conf.py + * is read as text (the first SPHINX_CONF_READ bytes) and never run. */ + +enum { + SPHINX_SUFFIXES = 4, /* extra reST suffixes one set declares */ + SPHINX_SUFFIX_LEN = 16, /* ".txt", ".rest": a longer one is no suffix */ + SPHINX_CONF_READ = 65536, +}; + +typedef struct { + char ext[SPHINX_SUFFIXES][SPHINX_SUFFIX_LEN]; + int count; +} sphinx_suffixes_t; + +/* The quoted string at s[i] ('...' or "..."): its span; false otherwise. */ +static bool sphinx_quoted(const char *s, size_t n, size_t i, size_t *a, size_t *b) { + if (i >= n || (s[i] != '\'' && s[i] != '"')) { + return false; + } + const char *close = memchr(s + i + SKIP_ONE, s[i], n - i - SKIP_ONE); + if (!close) { + return false; + } + *a = i + SKIP_ONE; + *b = (size_t)(close - s); + return true; +} + +static void sphinx_add(sphinx_suffixes_t *out, const char *s, size_t n) { + if (n < PAIR_LEN || n >= SPHINX_SUFFIX_LEN || s[0] != '.' || out->count >= SPHINX_SUFFIXES || + (n == strlen(".rst") && memcmp(s, ".rst", n) == 0)) { + return; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!isalnum((unsigned char)s[i]) && s[i] != '_' && s[i] != '-') { + return; + } + } + memcpy(out->ext[out->count], s, n); + out->ext[out->count][n] = '\0'; + out->count++; +} + +/* `source_suffix = '.txt'`, `= ['.rst', '.txt']` or + * `= {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}` (a dict + * entry counts only when it maps to restructuredtext). */ +static void sphinx_parse(const char *s, size_t n, sphinx_suffixes_t *out) { + static const char key[] = "source_suffix"; + size_t kl = strlen(key); + for (size_t line = 0; line < n;) { + const char *nl = memchr(s + line, '\n', n - line); + size_t end = nl ? (size_t)(nl - s) : n; + size_t i = line + kl; + if (end >= i && memcmp(s + line, key, kl) == 0) { + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + if (i < n && s[i] == '=') { + i++; + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + char open = i < n ? s[i] : '\0'; + char close = '\0'; + if (open == '[' || open == '(' || open == '{') { + close = open == '[' ? ']' : (open == '(' ? ')' : '}'); + } + /* the value: up to its closing bracket, or this line */ + size_t stop = end; + if (close) { + const char *c = memchr(s + i, close, n - i); + stop = c ? (size_t)(c - s) : end; + } + size_t k = i; + while (k < stop) { + size_t a; + size_t b; + if (!sphinx_quoted(s, stop, k, &a, &b)) { + k++; + continue; + } + k = b + SKIP_ONE; + if (open != '{') { + sphinx_add(out, s + a, b - a); + continue; + } + while (k < stop && (s[k] == ' ' || s[k] == ':')) { + k++; + } + size_t va; + size_t vb; + if (sphinx_quoted(s, stop, k, &va, &vb)) { + if (vb - va == strlen("restructuredtext") && + memcmp(s + va, "restructuredtext", vb - va) == 0) { + sphinx_add(out, s + a, b - a); + } + k = vb + SKIP_ONE; + } + } + return; /* the first assignment */ + } + } + line = end + SKIP_ONE; + } +} + +/* The documentation set this directory starts, if it has a conf.py. */ +static bool sphinx_load(const char *dir, sphinx_suffixes_t *out) { + char conf[CBM_SZ_4K]; + if (snprintf(conf, sizeof(conf), "%s/conf.py", dir) >= (int)sizeof(conf)) { + return false; + } + struct stat st; + if (wide_stat(conf, &st) != 0 || !S_ISREG(st.st_mode)) { + return false; + } + FILE *f = cbm_fopen(conf, "rb"); + if (!f) { + return false; + } + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, SPHINX_CONF_READ); + size_t n = buf ? fread(buf, SKIP_ONE, SPHINX_CONF_READ, f) : 0; + (void)fclose(f); + memset(out, 0, sizeof(*out)); + if (buf) { + sphinx_parse(buf, n, out); + } + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return true; +} + +static bool sphinx_is_rst(const sphinx_suffixes_t *set, const char *name) { + size_t nl = strlen(name); + for (int i = 0; set && i < set->count; i++) { + size_t el = strlen(set->ext[i]); + if (nl > el && strcmp(name + nl - el, set->ext[i]) == 0) { + return true; + } + } + return false; +} + /* Process a single regular file entry during directory walk. */ static void walk_dir_process_file(const char *abs_path, const char *rel_path, const char *name, const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, const struct stat *st, - file_list_t *out) { + const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx, + const struct stat *st, file_list_t *out) { const char *skip_reason = file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); if (skip_reason) { @@ -968,6 +1115,9 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co return; } CBMLanguage lang = detect_file_language(name, abs_path); + if (lang == CBM_LANG_COUNT && sphinx_is_rst(sphinx, name)) { + lang = CBM_LANG_RST; /* its documentation set's source_suffix */ + } if (lang == CBM_LANG_COUNT) { return; } @@ -978,6 +1128,7 @@ typedef struct { char dir[CBM_SZ_4K]; char prefix[CBM_SZ_4K]; const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */ + sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */ } walk_frame_t; /* Initial capacity only — the stack grows on demand. A single directory can * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855 @@ -1035,6 +1186,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char return; } slot->ignore_chain = parent->ignore_chain; + slot->sphinx = parent->sphinx; ws->top++; } @@ -1083,7 +1235,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, &st, out); + cbmignore, &frame->sphinx, &st, out); } } @@ -1177,6 +1329,11 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis frame.ignore_chain = link; } + sphinx_suffixes_t set; + if (sphinx_load(frame.dir, &set)) { + frame.sphinx = set; /* a (nested) documentation set starts here */ + } + cbm_dir_t *d = cbm_opendir(frame.dir); if (!d) { if (out->count_only) { diff --git a/src/discover/language.c b/src/discover/language.c index dcc41dafd0..f0bad96e0f 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -613,6 +613,10 @@ static const ext_entry_t EXT_TABLE[] = { {".clsp", CBM_LANG_CHIALISP}, {".clib", CBM_LANG_CHIALISP}, {".clinc", CBM_LANG_CHIALISP}, + {".pdf", CBM_LANG_PDF}, + /* AsciiDoc; `.asc` is left out: it is as often an ASCII-armored signature */ + {".adoc", CBM_LANG_ASCIIDOC}, + {".asciidoc", CBM_LANG_ASCIIDOC}, /* Slang */ {".slang", CBM_LANG_SLANG}, @@ -824,6 +828,8 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_NIM] = "Nim", [CBM_LANG_SCHEME] = "Scheme", [CBM_LANG_CHIALISP] = "Chialisp", + [CBM_LANG_PDF] = "PDF", + [CBM_LANG_ASCIIDOC] = "AsciiDoc", [CBM_LANG_FENNEL] = "Fennel", [CBM_LANG_FISH] = "Fish", [CBM_LANG_AWK] = "AWK", diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index 8ba63d5bd9..4c8f8f99c5 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -143,6 +143,8 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = { {"mojo", CBM_LANG_MOJO}, {"plsql", CBM_LANG_PLSQL}, {"chialisp", CBM_LANG_CHIALISP}, + {"pdf", CBM_LANG_PDF}, + {"asciidoc", CBM_LANG_ASCIIDOC}, {"lean", CBM_LANG_LEAN}, {"form", CBM_LANG_FORM}, {"magma", CBM_LANG_MAGMA}, diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c799282f1e..43acff7c3c 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -12477,6 +12477,36 @@ static char *resolve_path_owner(const char *path, char *derived) { return owner; } +/* A PDF is read as its text layer at index time (doc_pdf.c); its bytes are no + * source text. Its page Sections carry the page's text as their docstring. */ +static bool snippet_is_pdf(const char *file_path) { + size_t n = file_path ? strlen(file_path) : 0; + return n >= 4 && file_path[n - 4] == '.' && (file_path[n - 3] | 0x20) == 'p' && + (file_path[n - 2] | 0x20) == 'd' && (file_path[n - 1] | 0x20) == 'f'; +} + +static char *snippet_pdf_text(const cbm_node_t *node) { + if (!node->properties_json) { + return NULL; + } + yyjson_doc *pd = yyjson_read(node->properties_json, strlen(node->properties_json), 0); + if (!pd) { + return NULL; + } + yyjson_val *ds = yyjson_obj_get(yyjson_doc_get_root(pd), "docstring"); + const char *text = yyjson_is_str(ds) ? yyjson_get_str(ds) : NULL; + char *out = NULL; + if (text && text[0]) { + /* the same builder read_file_lines returns its text from */ + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_sb_append_n(&sb, text, strlen(text)); + out = cbm_sb_finish(&sb); + } + yyjson_doc_free(pd); + return out; +} + static char *resolve_snippet_source(const char *root_path, const char *file_path, int start, int end, char **out_abs_path) { *out_abs_path = NULL; @@ -12827,8 +12857,14 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, snippet_clipped = true; } char *abs_path = NULL; - char *source = - outline ? NULL : resolve_snippet_source(root_path, node->file_path, start, end, &abs_path); + char *source = NULL; + if (outline) { + source = NULL; + } else if (snippet_is_pdf(node->file_path)) { + source = snippet_pdf_text(node); /* never the file's bytes */ + } else { + source = resolve_snippet_source(root_path, node->file_path, start, end, &abs_path); + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root_obj = yyjson_mut_obj(doc); diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 8ac85fcb57..c5023658d0 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -45,7 +45,8 @@ const char *cbm_doclink_reason_name(int reason) { /* One pointer per language with a resolver: the one line a language leg adds * to this file. */ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { - &cbm_doclink_cs_resolver, + &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, + &cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; @@ -89,6 +90,10 @@ static const cbm_doclink_resolver_t *resolver_of_scope(const char *scope) { /* ── Incremental scope rules ─────────────────────────────────────── */ +bool cbm_doclinks_binds_lines(CBMLanguage lang) { + return lang == CBM_LANG_MARKDOWN || lang == CBM_LANG_RST || lang == CBM_LANG_ASCIIDOC; +} + bool cbm_doclinks_is_scope_input(const char *rel_path) { for (int i = 0; rel_path && i < DOCLINK_RESOLVER_COUNT; i++) { if (DOCLINK_RESOLVERS[i]->scope_input && DOCLINK_RESOLVERS[i]->scope_input(rel_path)) { @@ -328,6 +333,9 @@ typedef struct { int64_t src; int64_t tgt; uint32_t line; + uint32_t target_first; /* the target lines the reference names; 0: none */ + uint32_t target_last; + const char *via; uint16_t syntax; bool exact; } doclink_mention_t; @@ -335,6 +343,10 @@ typedef struct { static int mention_cmp(const void *a, const void *b) { const doclink_mention_t *x = (const doclink_mention_t *)a; const doclink_mention_t *y = (const doclink_mention_t *)b; + int edge = strcmp(cbm_doclink_syntax_edge(x->syntax), cbm_doclink_syntax_edge(y->syntax)); + if (edge != 0) { + return edge; + } if (x->src != y->src) { return x->src < y->src ? -1 : 1; } @@ -368,7 +380,8 @@ void cbm_doclinks_test_fail_edge_insert_after(int nth) { #endif /* Insert one MENTIONS edge; 0 when it could not be stored. */ -static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *props) { +static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *type, + const char *props) { #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS int n = atomic_load(&doclinks_test_edge_fail_after); while (n > 0) { @@ -380,10 +393,10 @@ static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, co } } #endif - return cbm_gbuf_insert_edge(gb, src, tgt, "MENTIONS", props); + return cbm_gbuf_insert_edge(gb, src, tgt, type, props); } -/* Emit one MENTIONS edge per (source, target): first line, its syntax, the +/* Emit one edge per (edge type, source, target): first line, its syntax, the * mention count, and tier exact when any mention bound exactly. A failed * insert fails the layer: the edge is not there. */ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t *m, int n, @@ -393,17 +406,24 @@ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t while (i < n) { int j = i; bool exact = false; - while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt) { + const char *type = cbm_doclink_syntax_edge(m[i].syntax); + while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt && + strcmp(cbm_doclink_syntax_edge(m[j].syntax), type) == 0) { exact = exact || m[j].exact; j++; } char props[CBM_SZ_256]; + char lines[CBM_SZ_64] = ""; + if (m[i].target_first > 0) { + snprintf(lines, sizeof(lines), ",\"target_lines\":[%u,%u]", m[i].target_first, + m[i].target_last); + } snprintf(props, sizeof(props), - "{\"via\":\"doc_comment\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u," - "\"count\":%d}", - cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", m[i].line, - j - i); - if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, props) == 0) { + "{\"via\":\"%s\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u," + "\"count\":%d%s}", + m[i].via, cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", + m[i].line, j - i, lines); + if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, type, props) == 0) { mark_failed(dl, rel, "alloc"); /* an edge that is not there is not counted */ } else { atomic_fetch_add_explicit(&dl->edges, 1, memory_order_relaxed); @@ -446,7 +466,12 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe * file-level doc resolves */ const cbm_gbuf_node_t *file_node = NULL; bool file_node_looked_up = false; - void *state = (R && index && R->file_begin) ? R->file_begin(index, file_idx) : NULL; + void *state = NULL; + if (R && index && R->file_prepare) { + state = R->file_prepare(index, file_idx, result->doc_links.items, n, graph); + } else if (R && index && R->file_begin) { + state = R->file_begin(index, file_idx); + } for (int i = 0; i < n; i++) { const CBMDocLink *link = &result->doc_links.items[i]; cbm_doclink_outcome_t out = {.kind = CBM_DOCLINK_UNRESOLVED, @@ -488,6 +513,9 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe mentions[nm++] = (doclink_mention_t){.src = src->id, .tgt = out.target->id, .line = link->line, + .target_first = out.target_first, + .target_last = out.target_last, + .via = R->via ? R->via : "doc_comment", .syntax = link->syntax, .exact = out.exact}; continue; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index 20f9fbfaa5..77a5dfb67b 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -67,6 +67,10 @@ typedef struct { const cbm_gbuf_node_t *target; bool exact; /* tier: "exact" (qualified/doc ID/alias) vs "unique" (scope lookup) */ int reason; + /* Lines of the target that the reference names (a document's `file#L3-L9`): + * the edge's "target_lines". 0 and 0: the reference names no lines. */ + uint32_t target_first; + uint32_t target_last; } cbm_doclink_outcome_t; /* A file's stored doc-link scope (incremental runs: files not re-extracted). */ @@ -194,6 +198,12 @@ typedef struct { * of the file, released after its last. A NULL state (no hook, or memory * ran out) changes the cost of resolving, never an outcome. */ void *(*file_begin)(const void *index, int run_file); + /* Optional, instead of file_begin: the file's state built from ALL its + * references before any is resolved (a PDF's cross-line joins decide + * which line fragments stand). `links` stays valid until file_end, and + * resolve's `link` points into it. */ + void *(*file_prepare)(const void *index, int run_file, const CBMDocLink *links, int n, + const cbm_gbuf_t *graph); void (*file_end)(void *state); /* Resolve one reference of run file `run_file` (its own scope is in the * index). Thread-safe: the index is read-only after build; `state` is the @@ -222,9 +232,40 @@ typedef struct { * the run. */ int (*scope_accepted)(const char *scope); char *(*rejected_scope)(const char *scope); + /* The `via` of its MENTIONS edges: what kind of text the reference is + * written in. NULL: "doc_comment". */ + const char *via; } cbm_doclink_resolver_t; extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver; +/* Markdown documents (doc_links_md.c): via "markdown". */ +extern const cbm_doclink_resolver_t cbm_doclink_md_resolver; +/* PDF documents (doc_links_pdf.c): via "pdf". */ +extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver; +/* reStructuredText documents and the Python scope they read (doc_links_rst.c): + * via "rst". */ +extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver; +/* AsciiDoc documents and the Antora YAML scope they read (doc_links_adoc.c): + * via "asciidoc". */ +extern const cbm_doclink_resolver_t cbm_doclink_adoc_resolver; + +/* Over a Markdown resolver index (cbm_doclink_md_resolver.build), for the + * other document resolvers: the innermost code definition of `path` holding + * lines [first, last] (or the one named `member`), NULL for none; the Folder + * node at `path`; whether `path` is test code (a test directory or file). */ +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member); +const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path); +/* The definition a region [first, last] of `path` (its text in `text`) binds, + * by the field test's rule (H8 bind_range): edge blank and comment lines + * trimmed; the innermost enclosing definition, unless it is a type (or there + * is none) and the definitions inside cover every other line of the region + * (an annotation right above one counts) -- then that one definition (the + * type for several). NULL: the file. */ +const cbm_gbuf_node_t *cbm_doclink_md_region(const void *md_index, const char *path, + const char *text, size_t text_len, uint32_t first, + uint32_t last); +bool cbm_doclink_md_test_path(const char *path); #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS /* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver @@ -248,6 +289,13 @@ bool cbm_doclink_cs_test_scope_parses(const char *scope); void cbm_doclink_cs_test_unit_memo(bool on); #endif +/* True for a document language whose references may bind the LINES of a file + * (Markdown `file#L3-L9`, reST :lines:, AsciiDoc tags and lines=): a body + * edit of that file keeps its + * names but can move what those lines hold, so an incremental run + * re-resolves such documents with the edited file. */ +bool cbm_doclinks_binds_lines(CBMLanguage lang); + /* True when `rel_path` is a scope input of some language (scope_input). */ bool cbm_doclinks_is_scope_input(const char *rel_path); diff --git a/src/pipeline/doc_links_adoc.c b/src/pipeline/doc_links_adoc.c new file mode 100644 index 0000000000..863462e41e --- /dev/null +++ b/src/pipeline/doc_links_adoc.c @@ -0,0 +1,919 @@ +/* + * doc_links_adoc.c — AsciiDoc references -> MENTIONS edges, via "asciidoc" + * (the resolving half of internal/cbm/doc_adoc.c), with the Antora rules the + * field test's harvester measured (H8: includes 97.6-100 % resolved, ~100 % + * precise; javadoc attributes 320/320). + * + * Antora: a component is the directory of an antora.yml; its pages live under + * /modules//pages. Its attributes and collector scans, and + * a playbook's attributes, come from the YAML scope blobs (doc_adoc.c), so an + * incremental run resolves as a full one. + * + * include::target[...]: references the page left are expanded from the + * component's attributes, then the playbook's. A resource ID + * ([[version@]component:]module:]family$path) names + * /modules///, or a file a collector scan + * contributes there; any other target is relative to the including file. Its + * `lines=` bind the innermost definition holding them; its `tag=` / `tags=` + * regions -- the included file's own `tag::x[]` .. `end::x[]` comments, read + * from that indexed file -- likewise; a tag the file does not have binds the + * file (the document stays a dependent of that file, so it re-resolves when + * the file changes, on an incremental run as on a full one). + * + * {attribute} references whose value is an Antora javadoc link + * ({javadoc-root}///.html[#member]) bind the + * Java or Kotlin type (or its member) that the repository's main sources + * declare under that package path. + * + * Monospace spans: the Markdown resolver's code-span rules (doc_links_md.c). + */ +#include "pipeline/doc_links.h" + +#include "discover/discover.h" /* cbm_language_for_filename */ +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/compat_fs.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + RA_PATH_CAP = 1024, + RA_FIELDS = 6, + RA_EXPAND_PASSES = 10, /* nested attribute references (H8's depth) */ + RA_TAG_FILE_MAX = 8 * 1024 * 1024, /* an included file read for its tags */ + RA_TAGS = 16, /* tags of one include */ + RA_TAG_LEN = 128, /* a tag name */ + RA_INIT = 8, +}; + +typedef struct { + const char *key; + const char *value; +} ra_attr_t; + +typedef struct { + const char *dir; + const char *into; +} ra_scan_t; + +typedef struct { + const char *root; /* the antora.yml's directory ("" for the repository root) */ + const char *name; + ra_attr_t *attrs; + int nattrs; + ra_scan_t *scans; + int nscans; +} ra_comp_t; + +typedef struct { + const char *project; + const char *repo; /* the repository on disk: included files are read for their tags */ + void *md; + const char **run_paths; + int run_count; + CBMArena arena; + ra_comp_t *comps; + int ncomps; + ra_attr_t *pattrs; /* playbook attributes, the first definition of a key wins */ + int npattrs; +} ra_index_t; + +static void ra_destroy(void *index); + +static const char *ra_base(const char *path) { + const char *s = strrchr(path, '/'); + return s ? s + SKIP_ONE : path; +} + +static bool ra_ends(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n >= k && strcmp(s + n - k, suffix) == 0; +} + +static bool ra_grow(CBMArena *a, void **items, int count, int *cap, size_t size) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : RA_INIT; + void *grown = cbm_arena_alloc(a, (size_t)ncap * size); + if (!grown) { + return false; + } + if (count > 0) { + memcpy(grown, *items, (size_t)count * size); + } + *items = grown; + *cap = ncap; + return true; +} + +/* One YAML scope blob: a component (C) or a playbook (P). */ +static bool ra_read_scope(ra_index_t *x, const char *path, const char *scope, int *comp_cap, + int *pattr_cap) { + char *copy = cbm_arena_strdup(&x->arena, scope); + if (!copy) { + return false; + } + char *save = NULL; + (void)strtok_r(copy, "\n", &save); /* the tag line */ + char *line = strtok_r(NULL, "\n", &save); + if (!line || (strcmp(line, "C") != 0 && strcmp(line, "P") != 0)) { + return true; + } + bool component = line[0] == 'C'; + ra_comp_t comp = {0}; + int acap = 0; + int scap = 0; + if (component) { + const char *b = ra_base(path); + comp.root = + cbm_arena_strndup(&x->arena, path, b > path ? (size_t)(b - path - SKIP_ONE) : 0); + comp.name = ""; + if (!comp.root) { + return false; + } + } + for (line = strtok_r(NULL, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { + char *tab = strchr(line, '\t'); + if (!tab) { + continue; + } + *tab = '\0'; + char *v = tab + SKIP_ONE; + if (strcmp(line, "N") == 0 && component) { + comp.name = v; + } else if (strcmp(line, "A") == 0) { + char *t2 = strchr(v, '\t'); + if (!t2) { + continue; + } + *t2 = '\0'; + ra_attr_t at = {v, t2 + SKIP_ONE}; + if (component) { + if (!ra_grow(&x->arena, (void **)&comp.attrs, comp.nattrs, &acap, sizeof(at))) { + return false; + } + comp.attrs[comp.nattrs++] = at; + } else { + bool seen = false; + for (int i = 0; i < x->npattrs; i++) { + seen = seen || strcmp(x->pattrs[i].key, at.key) == 0; + } + if (!seen) { + if (!ra_grow(&x->arena, (void **)&x->pattrs, x->npattrs, pattr_cap, + sizeof(at))) { + return false; + } + x->pattrs[x->npattrs++] = at; + } + } + } else if (strcmp(line, "D") == 0 && component) { + if (!ra_grow(&x->arena, (void **)&comp.scans, comp.nscans, &scap, sizeof(ra_scan_t))) { + return false; + } + comp.scans[comp.nscans++] = (ra_scan_t){v, ""}; + } else if (strcmp(line, "I") == 0 && component && comp.nscans > 0) { + comp.scans[comp.nscans - 1].into = v; + } + } + if (component) { + if (!ra_grow(&x->arena, (void **)&x->comps, x->ncomps, comp_cap, sizeof(comp))) { + return false; + } + x->comps[x->ncomps++] = comp; + } + return true; +} + +static void *ra_build(const cbm_doclink_build_in_t *in) { + ra_index_t *x = (ra_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->repo = in->ctx ? in->ctx->repo_path : NULL; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->md = cbm_doclink_md_resolver.build(in); + if ((in->run_file_count > 0 && !x->run_paths) || !x->md || !x->project) { + ra_destroy(x); + return NULL; + } + int comp_cap = 0; + int pattr_cap = 0; + for (int i = 0; i < in->file_count; i++) { + const cbm_doclink_file_t *f = &in->files[i]; + if (f->run_file >= 0 && f->run_file < x->run_count) { + x->run_paths[f->run_file] = f->rel_path; /* valid until destroy */ + } + if (f->scope && + strncmp(f->scope, CBM_DOCLINK_ADOC_SCOPE_TAG "\n", + strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n")) == 0 && + !ra_read_scope(x, f->rel_path, f->scope, &comp_cap, &pattr_cap)) { + ra_destroy(x); + return NULL; + } + } + return x; +} + +static void ra_destroy(void *index) { + ra_index_t *x = (ra_index_t *)index; + if (!x) { + return; + } + if (x->md) { + cbm_doclink_md_resolver.destroy(x->md); + } + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* H8 component_of: the deepest component whose modules/ hold the document. */ +static const ra_comp_t *ra_component_of(const ra_index_t *x, const char *doc) { + const ra_comp_t *best = NULL; + for (int i = 0; i < x->ncomps; i++) { + const ra_comp_t *c = &x->comps[i]; + char pref[RA_PATH_CAP]; + snprintf(pref, sizeof(pref), "%s%smodules/", c->root, c->root[0] ? "/" : ""); + if (strncmp(doc, pref, strlen(pref)) == 0 && + (!best || strlen(c->root) > strlen(best->root))) { + best = c; + } + } + return best; +} + +static const char *ra_attr(const ra_index_t *x, const ra_comp_t *c, const char *key, size_t n) { + for (int i = 0; c && i < c->nattrs; i++) { + if (strlen(c->attrs[i].key) == n && memcmp(c->attrs[i].key, key, n) == 0) { + return c->attrs[i].value; + } + } + for (int i = 0; i < x->npattrs; i++) { + if (strlen(x->pattrs[i].key) == n && memcmp(x->pattrs[i].key, key, n) == 0) { + return x->pattrs[i].value; + } + } + return NULL; +} + +static bool ra_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The text with the component's and the playbook's attribute references + * substituted, pass after pass. False when one stays undefined or it does + * not fit. */ +static bool ra_expand(const ra_index_t *x, const ra_comp_t *c, const char *s, char *out, + size_t cap) { + char cur[RA_PATH_CAP]; + char nxt[RA_PATH_CAP]; + if (snprintf(cur, sizeof(cur), "%s", s) >= (int)sizeof(cur)) { + return false; + } + bool undefined = false; + for (int pass = 0; pass < RA_EXPAND_PASSES; pass++) { + size_t w = 0; + bool changed = false; + undefined = false; + for (size_t i = 0; cur[i]; i++) { + const char *v = NULL; + size_t e = i + SKIP_ONE; + if (cur[i] == '{' && (i == 0 || cur[i - SKIP_ONE] != '\\') && ra_word(cur[e])) { + while (cur[e] && (ra_word(cur[e]) || cur[e] == '-')) { + e++; + } + if (cur[e] == '}') { + v = ra_attr(x, c, cur + i + SKIP_ONE, e - i - SKIP_ONE); + undefined = undefined || !v; + } + } + if (v) { + size_t vl = strlen(v); + if (w + vl >= sizeof(nxt)) { + return false; + } + memcpy(nxt + w, v, vl); + w += vl; + i = e; + changed = true; + continue; + } + if (w + SKIP_ONE >= sizeof(nxt)) { + return false; + } + nxt[w++] = cur[i]; + } + nxt[w] = '\0'; + memcpy(cur, nxt, w + SKIP_ONE); + if (!changed) { + break; + } + } + if (undefined || snprintf(out, cap, "%s", cur) >= (int)cap) { + return false; + } + return true; +} + +/* os.path.normpath(base/p); false when it leaves the repository. */ +static bool ra_norm(const char *base, const char *p, char *out, size_t cap) { + char tmp[RA_PATH_CAP * PAIR_LEN]; + snprintf(tmp, sizeof(tmp), "%s%s%s", base, base[0] && p[0] ? "/" : "", p); + size_t w = 0; + out[0] = '\0'; + for (char *s = tmp; *s;) { + char *e = strchr(s, '/'); + size_t l = e ? (size_t)(e - s) : strlen(s); + if (l == PAIR_LEN && s[0] == '.' && s[1] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + if (w > 0) { + w--; + } + out[w] = '\0'; + } else if (l > 0 && !(l == SKIP_ONE && s[0] == '.')) { + if (w + l + PAIR_LEN > cap) { + return false; + } + if (w > 0) { + out[w++] = '/'; + } + memcpy(out + w, s, l); + w += l; + out[w] = '\0'; + } + if (!e) { + break; + } + s = e + SKIP_ONE; + } + return true; +} + +static void ra_unresolved(cbm_doclink_outcome_t *out, int reason) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = reason; +} + +static void ra_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *n, uint32_t a, uint32_t b) { + out->kind = CBM_DOCLINK_EDGE; + out->target = n; + out->exact = true; + out->target_first = a; + out->target_last = b; +} + +/* A path of a type the indexer does not read (a service file, an image): no + * reference to code, decided by its name as the Markdown resolver does -- never + * by the disk, which an incremental run does not watch for such files. */ +static bool ra_unindexable(const char *path) { + return cbm_language_for_filename(ra_base(path)) == CBM_LANG_COUNT; +} + +/* A collector scan of build output (`./build/...`): what it contributes is + * generated, not a file of the repository (H8). */ +static bool ra_build_dir(const char *dir) { + while (dir[0] == '.' && dir[1] == '/') { + dir += PAIR_LEN; + } + return strcmp(dir, "build") == 0 || strncmp(dir, "build/", strlen("build/")) == 0 || + strstr(dir, "/build/") != NULL; +} + +static const char *const RA_FAMILIES[][2] = {{"example", "examples"}, + {"partial", "partials"}, + {"page", "pages"}, + {"image", "images"}, + {"attachment", "attachments"}}; + +/* H8 resolve_target: an Antora resource ID or a path relative to the including + * file. Returns the repository path in `out`, or false with `out->kind` set. */ +static bool ra_target(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_comp_t *comp, const char *target, char *path, size_t cap, + cbm_doclink_outcome_t *out) { + out->kind = CBM_DOCLINK_LOCAL; + if (strstr(target, "://")) { + return false; /* a URL include */ + } + const char *dollar = strchr(target, '$'); + if (dollar && comp) { + char pre[RA_PATH_CAP]; + snprintf(pre, sizeof(pre), "%.*s", (int)(dollar - target), target); + const char *at = strrchr(pre, '@'); + char *segs = at ? pre + (at - pre) + SKIP_ONE : pre; + /* family is the last segment; module and component before it */ + char *fam = strrchr(segs, ':'); + const char *module = NULL; + if (fam) { + *fam = '\0'; + fam++; + char *comp_sep = strchr(segs, ':'); + if (comp_sep) { + *comp_sep = '\0'; + if (strcmp(segs, comp->name) != 0) { + return false; /* another component's resource */ + } + module = comp_sep + SKIP_ONE; + } else { + module = segs; + } + } else { + fam = segs; + } + char doc_module[RA_PATH_CAP] = "ROOT"; + if (!module) { + /* the including page's own module: modules//... */ + const char *rel = doc + strlen(comp->root) + (comp->root[0] ? SKIP_ONE : 0); + if (strncmp(rel, "modules/", strlen("modules/")) == 0) { + const char *m = rel + strlen("modules/"); + const char *slash = strchr(m, '/'); + if (slash) { + snprintf(doc_module, sizeof(doc_module), "%.*s", (int)(slash - m), m); + } + } + module = doc_module; + } + const char *famdir = NULL; + for (size_t i = 0; i < sizeof(RA_FAMILIES) / sizeof(RA_FAMILIES[0]); i++) { + if (strcmp(fam, RA_FAMILIES[i][0]) == 0) { + famdir = RA_FAMILIES[i][1]; + } + } + if (!famdir) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return false; + } + char virt[RA_PATH_CAP]; + snprintf(virt, sizeof(virt), "modules/%s/%s/%s", module, famdir, dollar + SKIP_ONE); + if (ra_norm(comp->root, virt, path, cap) && + cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + /* a file a collector scan contributes into the component */ + for (int i = 0; i < comp->nscans; i++) { + const char *into = comp->scans[i].into; + while (into[0] == '.' && into[1] == '/') { + into += PAIR_LEN; + } + size_t il = strlen(into); + while (il > 0 && into[il - SKIP_ONE] == '/') { + il--; + } + if (il == 0 || strncmp(virt, into, il) != 0 || virt[il] != '/') { + continue; + } + char sub[RA_PATH_CAP]; + snprintf(sub, sizeof(sub), "%s/%s", comp->scans[i].dir, virt + il + SKIP_ONE); + if (ra_norm(comp->root, sub, path, cap) && + cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + if (ra_build_dir(comp->scans[i].dir)) { + return false; /* build output (H8 generated): no file of the repository */ + } + } + if (ra_unindexable(virt)) { + return false; + } + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return false; + } + char dir[RA_PATH_CAP]; + const char *b = ra_base(doc); + snprintf(dir, sizeof(dir), "%.*s", b > doc ? (int)(b - doc - SKIP_ONE) : 0, doc); + if (!ra_norm(dir, target, path, cap)) { + return false; /* above the repository root */ + } + if (cbm_pipeline_file_node(graph, x->project, path)) { + return true; + } + char parent[RA_PATH_CAP]; + const char *pb = ra_base(path); + snprintf(parent, sizeof(parent), "%.*s", pb > path ? (int)(pb - path - SKIP_ONE) : 0, path); + if (!ra_unindexable(path) && parent[0] && cbm_doclink_md_folder(x->md, parent)) { + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } + return false; +} + +/* Asciidoctor's TagDirectiveRx \b(?:tag|(e)nd)::(\S+?)\[\](?=$|[ \r]) over the + * file: the 1-based lines from the first region's first content line to the + * last region's last one (directive lines excluded). False when a tag has no + * region. */ +static bool ra_tag_span(const char *src, size_t n, const char *const *tags, int ntags, + uint32_t *first, uint32_t *last) { + *first = 0; + *last = 0; + for (int t = 0; t < ntags; t++) { + size_t tl = strlen(tags[t]); + uint32_t line = 1; + uint32_t start = 0; + bool found = false; + size_t pos = 0; + while (pos < n) { + const char *nl = memchr(src + pos, '\n', n - pos); + size_t end = nl ? (size_t)(nl - src) : n; + for (size_t i = pos; i + PAIR_LEN < end; i++) { + bool is_end = false; + size_t k; + if (i + strlen("tag::") <= end && memcmp(src + i, "tag::", strlen("tag::")) == 0) { + k = i + strlen("tag::"); + } else if (i + strlen("end::") <= end && + memcmp(src + i, "end::", strlen("end::")) == 0) { + k = i + strlen("end::"); + is_end = true; + } else { + continue; + } + if (i > pos && ra_word(src[i - SKIP_ONE])) { + continue; /* \b */ + } + if (k + tl + PAIR_LEN > end || memcmp(src + k, tags[t], tl) != 0 || + src[k + tl] != '[' || src[k + tl + SKIP_ONE] != ']') { + continue; + } + size_t after = k + tl + PAIR_LEN; + if (after != end && src[after] != ' ' && src[after] != '\r') { + continue; + } + if (!is_end && start == 0) { + start = line + SKIP_ONE; + } else if (is_end && start != 0) { + if (start <= line - SKIP_ONE) { + *first = (*first == 0 || start < *first) ? start : *first; + *last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last; + found = true; + } + start = 0; + } + break; + } + line++; + pos = end + SKIP_ONE; + } + if (start != 0 && start <= line - SKIP_ONE) { + *first = (*first == 0 || start < *first) ? start : *first; + *last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last; + found = true; + } + if (!found) { + return false; + } + } + return *first > 0; +} + +/* The included file's text, read from the indexed file on disk (at most + * RA_TAG_FILE_MAX bytes): its tag regions and the lines of a region. NULL + * when it cannot be read; the caller frees it (cbm_free, OTHER). */ +static char *ra_read(const ra_index_t *x, const char *path, size_t *n) { + *n = 0; + if (!x->repo) { + return NULL; + } + char abs[RA_PATH_CAP * PAIR_LEN]; + snprintf(abs, sizeof(abs), "%s/%s", x->repo, path); + FILE *f = cbm_fopen(abs, "rb"); + if (!f) { + return NULL; + } + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, RA_TAG_FILE_MAX); + *n = buf ? fread(buf, SKIP_ONE, RA_TAG_FILE_MAX, f) : 0; + (void)fclose(f); + return buf; +} + +/* `lines=a..b;c;d..-1`: the span of all ranges (-1: the file's end). */ +static bool ra_lines(const char *v, uint32_t *first, uint32_t *last) { + *first = 0; + *last = 0; + const char *p = v; + while (*p) { + while (*p == ';' || *p == ',' || *p == ' ' || *p == '"') { + p++; + } + if (!*p) { + break; + } + if (*p < '0' || *p > '9') { + return false; + } + uint32_t a = (uint32_t)strtoul(p, (char **)&p, 10); + uint32_t b = a; + if (p[0] == '.' && p[1] == '.') { + p += PAIR_LEN; + if (p[0] == '-' && p[1] == '1') { + b = UINT32_MAX; + p += PAIR_LEN; + } else if (*p >= '0' && *p <= '9') { + b = (uint32_t)strtoul(p, (char **)&p, 10); + } else { + return false; + } + } + if (a == 0 || b < a) { + return false; + } + *first = (*first == 0 || a < *first) ? a : *first; + *last = b > *last ? b : *last; + } + return *first > 0; +} + +typedef struct { + char buf[RA_PATH_CAP * PAIR_LEN]; + char *f[RA_FIELDS]; + int n; +} ra_rec_t; + +static bool ra_record(const char *raw, ra_rec_t *r) { + if (snprintf(r->buf, sizeof(r->buf), "%s", raw) >= (int)sizeof(r->buf)) { + return false; + } + r->n = 0; + char *p = r->buf; + while (r->n < RA_FIELDS) { + r->f[r->n++] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + p = tab + SKIP_ONE; + } + for (int i = r->n; i < RA_FIELDS; i++) { + r->f[i] = ""; + } + return true; +} + +static void ra_include(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_rec_t *r, cbm_doclink_outcome_t *out) { + const ra_comp_t *comp = ra_component_of(x, doc); + char target[RA_PATH_CAP]; + if (strchr(r->f[1], '{')) { + if (!comp || !ra_expand(x, comp, r->f[1], target, sizeof(target))) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); /* an undefined attribute */ + return; + } + } else { + snprintf(target, sizeof(target), "%s", r->f[1]); + } + char path[RA_PATH_CAP]; + if (!ra_target(x, graph, doc, comp, target, path, sizeof(path), out)) { + return; + } + const cbm_gbuf_node_t *file = cbm_pipeline_file_node(graph, x->project, path); + if (!file) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + uint32_t first = 0; + uint32_t last = 0; + const char *tagv = r->f[2]; + if (!tagv[0] && r->f[3][0] && !ra_lines(r->f[3], &first, &last)) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + size_t tn = 0; + char *text = (tagv[0] || first > 0) ? ra_read(x, path, &tn) : NULL; + if (tagv[0]) { + /* H8: negated and wildcard tags leave the positive ones; none left + * includes the whole file */ + char tags[RA_TAGS][RA_TAG_LEN]; + const char *tagp[RA_TAGS]; + int nt = 0; + for (const char *p = tagv; *p && nt < RA_TAGS;) { + size_t l = strcspn(p, ";,"); + if (l > 0 && l < RA_TAG_LEN && p[0] != '!' && !memchr(p, '*', l)) { + snprintf(tags[nt], RA_TAG_LEN, "%.*s", (int)l, p); + tagp[nt] = tags[nt]; + nt++; + } + p += l + (p[l] ? SKIP_ONE : 0); + } + if (nt == 0 || !text || !ra_tag_span(text, tn, tagp, nt, &first, &last)) { + first = 0; /* wildcards only, or a tag the file lacks: the file */ + last = 0; + } + } + if (first > 0) { + const cbm_gbuf_node_t *seg = text + ? cbm_doclink_md_region(x->md, path, text, tn, first, last) + : cbm_doclink_md_segment(x->md, path, first, last, NULL); + cbm_free(CBM_MEM_CLASS_OTHER, text); + ra_edge(out, seg ? seg : file, first, last == UINT32_MAX ? 0 : last); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, text); + ra_edge(out, file, 0, 0); +} + +static bool ra_class_label(const char *label) { + static const char *const labels[] = {"Class", "Interface", "Enum", "Type", "Trait", NULL}; + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +static bool ra_member_label(const char *label) { + static const char *const labels[] = {"Method", "Field", "Variable", "Function", NULL}; + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* H8 javadoc: xref:attachment$api///.html[#member]. */ +static void ra_javadoc(const ra_index_t *x, const cbm_gbuf_t *graph, const char *val, + cbm_doclink_outcome_t *out) { + static const char key[] = "xref:attachment$api/"; + const char *p = strstr(val, key); + if (!p) { + return; + } + p += strlen(key); + const char *slash = strchr(p, '/'); /* after the JPMS module */ + if (!slash) { + return; + } + const char *path = slash + SKIP_ONE; + const char *html = strstr(path, ".html"); + if (!html || memchr(path, '[', (size_t)(html - path))) { + return; + } + char rel[RA_PATH_CAP]; + snprintf(rel, sizeof(rel), "%.*s", (int)(html - path), path); + const char *tname = ra_base(rel); + if (strcmp(tname, "package-summary") == 0 || strcmp(tname, "module-summary") == 0 || + strcmp(rel, "index") == 0) { + return; /* a package or module page: no code node */ + } + char anchor[RA_PATH_CAP] = ""; + if (html[strlen(".html")] == '#') { + const char *a = html + strlen(".html") + SKIP_ONE; + size_t al = strcspn(a, "[(-"); + snprintf(anchor, sizeof(anchor), "%.*s", (int)al, a); + } + char pkgdir[RA_PATH_CAP]; + snprintf(pkgdir, sizeof(pkgdir), "%.*s", (int)(tname - rel), rel); /* "org/x/" */ + char outer[RA_PATH_CAP]; + snprintf(outer, sizeof(outer), "%.*s", (int)strcspn(tname, "."), tname); + const char *want = strrchr(tname, '.') ? strrchr(tname, '.') + SKIP_ONE : tname; + /* the source files declaring /.java|.kt, main sources first */ + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, outer, &nodes, &count) != 0) { + return; + } + const char *files[2] = {NULL, NULL}; + int nfiles = 0; + for (int pass = 0; pass < PAIR_LEN && nfiles == 0; pass++) { + bool main_only = pass == 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *fp = n->file_path; + if (!ra_class_label(n->label) || !fp) { + continue; + } + char tail[RA_PATH_CAP * PAIR_LEN]; + bool java = ra_ends(fp, ".java"); + snprintf(tail, sizeof(tail), "%s%s%s", pkgdir, outer, java ? ".java" : ".kt"); + size_t fl = strlen(fp); + size_t tl = strlen(tail); + if (!(java || ra_ends(fp, ".kt")) || fl < tl || strcmp(fp + fl - tl, tail) != 0 || + (fl > tl && fp[fl - tl - SKIP_ONE] != '/') || + (main_only && !strstr(fp, "/src/main/"))) { + continue; + } + bool seen = false; + for (int k = 0; k < nfiles; k++) { + seen = seen || strcmp(files[k], fp) == 0; + } + if (!seen && nfiles < PAIR_LEN) { + files[nfiles++] = fp; + } else if (!seen) { + nfiles++; + } + } + } + if (nfiles == 0) { + ra_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + if (nfiles > SKIP_ONE) { + ra_unresolved(out, CBM_DOCLINK_REASON_AMBIGUOUS); + return; + } + /* the type in that file (the first by line for a repeated name) */ + const cbm_gbuf_node_t *type = NULL; + if (cbm_gbuf_find_by_name(graph, want, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (ra_class_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 && + (!type || n->start_line < type->start_line || + (n->start_line == type->start_line && n->id < type->id))) { + type = n; + } + } + } + if (!type) { + ra_unresolved(out, CBM_DOCLINK_REASON_GRAPH_GAP); + return; + } + const cbm_gbuf_node_t *member = NULL; + int members = 0; + if (anchor[0] && cbm_gbuf_find_by_name(graph, anchor, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (ra_member_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 && + n->start_line >= type->start_line && n->start_line <= type->end_line) { + member = n; + members++; + } + } + } + /* an overloaded member: the type that declares it */ + ra_edge(out, members == SKIP_ONE ? member : type, 0, 0); +} + +static void ra_attribute(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const ra_rec_t *r, cbm_doclink_outcome_t *out) { + const ra_comp_t *comp = ra_component_of(x, doc); + if (!comp) { + return; /* outside an Antora component: no API attributes (H8) */ + } + const char *raw = r->f[2][0] ? r->f[2] : ra_attr(x, comp, r->f[1], strlen(r->f[1])); + if (!raw || !strstr(raw, "{javadoc-root}")) { + return; + } + char val[RA_PATH_CAP]; + if (!ra_expand(x, comp, r->f[2][0] ? r->f[3] : raw, val, sizeof(val))) { + return; + } + ra_javadoc(x, graph, val, out); +} + +static void ra_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const ra_index_t *x = (const ra_index_t *)index; + out->kind = CBM_DOCLINK_LOCAL; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (link->syntax == CBM_DOCLINK_ADOC_CODE_PATH || link->syntax == CBM_DOCLINK_ADOC_CODE_NAME) { + CBMDocLink md = *link; + md.syntax = link->syntax == CBM_DOCLINK_ADOC_CODE_PATH ? CBM_DOCLINK_MD_CODE_PATH + : CBM_DOCLINK_MD_CODE_NAME; + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, &md, graph, out); + return; + } + ra_rec_t *r = (ra_rec_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*r)); + if (!r || !ra_record(link->raw, r)) { + ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + cbm_free(CBM_MEM_CLASS_OTHER, r); + return; + } + if (link->syntax == CBM_DOCLINK_ADOC_INCLUDE) { + ra_include(x, graph, doc, r, out); + } else if (link->syntax == CBM_DOCLINK_ADOC_ATTRIBUTE) { + ra_attribute(x, graph, doc, r, out); + } + cbm_free(CBM_MEM_CLASS_OTHER, r); +} + +const cbm_doclink_resolver_t cbm_doclink_adoc_resolver = { + .langs = {CBM_LANG_ASCIIDOC, CBM_LANG_YAML}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG, + .build = ra_build, + .destroy = ra_destroy, + .resolve = ra_resolve, + .via = "asciidoc", +}; diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c new file mode 100644 index 0000000000..4eaee1ce86 --- /dev/null +++ b/src/pipeline/doc_links_md.c @@ -0,0 +1,1596 @@ +/* + * doc_links_md.c — Markdown references -> MENTIONS edges: the resolving half. + * + * A Markdown reference (doclink_md.c) names a repository PATH, so it is looked + * up in the graph as written: the File node, or the Folder node, at that path, + * read relative to the document's own directory and -- for a path with a + * directory part -- relative to the repository root as well (documents are + * written both ways). A file name alone is read relative to the document + * only: a file of that name elsewhere in the repository is not what the + * document says (the field tests' rule for bare file names). + * + * Into the file: a line range picks the innermost code definition that holds + * the WHOLE range; a member (`path::Name`, `path::Owner::name`) picks the one + * definition of that name in the file. Otherwise the edge goes to the file, + * and the range stays on it ("target_lines"). Every edge is exact: the + * document wrote the path. + * + * Unresolved: + * missing the path is not in the graph, but its directory is: the + * document names something that is not there (doc rot) + * ambiguous a file name alone that is not next to the document but exists + * elsewhere in the repository: the document does not say which + * Not a reference to this repository at all (LOCAL: no edge, no row): a link + * back into the same document; a path that is neither indexed nor of a type + * the indexer reads (an image, an archive, an extensionless page); a path + * whose directory is not in the repository either (a file a tutorial asks + * the reader to create, a system path, a path above the root); and, in a + * code span, a rooted single segment without a file extension, which is a + * URL route (`/docs`, `/health`), not the directory of that name. + */ +#include "pipeline/doc_links.h" + +#include "discover/discover.h" /* cbm_language_for_filename */ +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + MDR_PATH_CAP = 1024, /* the longest repository path looked up */ + MDR_TRIES = 2, /* relative to the document, relative to the root */ + MDR_BASENAME_INIT = 1024, /* first capacity of the file-name table */ + MDR_HEX_BASE = 16, + MDR_PCT_LEN = 3, /* `%XX` */ + MDR_ADR_DIGITS = 5, /* an ADR number has at most this many digits */ + MDR_PRIO_FIELD = 3, /* H8's label priorities: types 0, Type 1, callables 2 */ + MDR_PRIO_OTHER = 4, +}; + +/* Labels whose nodes are code a range or a member can name. */ +/* Labels whose nodes are code a range, a member or a name can bind. Every one + * of them is proxied on the incremental route (pipeline_delta.c preseed), so + * a re-resolved document binds what a full build binds; Macro nodes are not + * proxied, and are no target here. */ +static const char *const MDR_CODE_LABELS[] = { + "Function", "Method", "Class", "Struct", "Interface", "Trait", "Enum", "Type", + "Variable", "Constant", "Field", "Table", "View", "Trigger", NULL}; + +static unsigned char mdr_norm_c(unsigned char c); +static bool mdr_in(const char *s, size_t n, const char *const *list); +static void mdr_destroy(void *index); + +/* A module or package a qualified name can name: a code file without its + * extension (`pkg/routing.py` is `pkg.routing`), a package directory through + * its index file (`pkg/__init__.py`, `mod.rs`, `index.ts`, `lib.rs`), or a + * directory. */ +typedef struct mdr_pkg { + const char *chain; /* the pieces before its name, '.'-joined */ + const cbm_gbuf_node_t *node; + const char *file; /* its code file (NULL: a directory) */ + struct mdr_pkg *next; /* the next one of the same name */ +} mdr_pkg_t; + +/* A code definition and its file: the by-file list ranges and members are + * read from. */ +typedef struct { + const char *file; + const cbm_gbuf_node_t *node; +} mdr_def_t; + +typedef struct { + const char *project; + size_t project_len; + const char **run_paths; /* run_file -> rel_path (NULL: not a Markdown file of this run) */ + int run_count; + /* base name -> number of File nodes with it (a file name alone that is + * not next to the document: ambiguous when the name exists anywhere) */ + CBMHashTable *basenames; + CBMHashTable *pkgs; /* name -> mdr_pkg_t list */ + CBMHashTable *top; /* normalized top-level directory and code-file names */ + CBMArena arena; /* the strings and records of pkgs and top */ + mdr_def_t *defs; /* every code definition, sorted by (file, start line, id) */ + int ndefs; + CBMHashTable *adr_by_file; /* file path -> its ADR node (doc_adr.c) */ + CBMHashTable *folders; /* directory path -> its Folder node */ +} mdr_index_t; + +static const char *mdr_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +static bool mdr_code_file(const char *path) { + CBMLanguage lang = cbm_language_for_filename(mdr_base(path)); + return lang != CBM_LANG_COUNT && lang != CBM_LANG_MARKDOWN && lang != CBM_LANG_RST && + lang != CBM_LANG_ASCIIDOC && lang != CBM_LANG_PDF && lang != CBM_LANG_HTML && + lang != CBM_LANG_CSS && lang != CBM_LANG_SCSS && !cbm_has_config_extension(path); +} + +/* The stems of a package directory's index file. */ +static const char *const MDR_INDEX_STEMS[] = {"__init__", "mod", "index", "lib", NULL}; + +/* Lower-cased copy, '-' read as '_'. */ +static char *mdr_norm_dup(CBMArena *a, const char *s, size_t n) { + char *out = cbm_arena_strndup(a, s, n); + for (size_t i = 0; out && i < n; i++) { + out[i] = (char)mdr_norm_c((unsigned char)out[i]); + } + return out; +} + +/* Add the module `key` (a path without extension, '/'-separated, whose last + * piece may hold dots) unless one is there. False when memory ran out. */ +static bool mdr_add_pkg(mdr_index_t *x, CBMHashTable *seen, const char *key, size_t klen, + const cbm_gbuf_node_t *node, const char *file) { + char *seen_key = cbm_arena_strndup(&x->arena, key, klen); /* borrowed by `seen` */ + char *k = seen_key ? cbm_arena_strndup(&x->arena, key, klen) : NULL; + if (!k) { + return false; + } + if (cbm_ht_get(seen, seen_key)) { + return true; /* the first file of a key wins (files come sorted by path) */ + } + cbm_ht_set(seen, seen_key, (void *)node); + /* pieces: '/'- and '.'-separated; the name is the last one */ + for (char *c = k; *c; c++) { + if (*c == '/') { + *c = '.'; + } + } + char *last = strrchr(k, '.'); + const char *name = last ? last + SKIP_ONE : k; + if (!name[0]) { + return true; + } + mdr_pkg_t *p = (mdr_pkg_t *)cbm_arena_alloc(&x->arena, sizeof(*p)); + if (!p) { + return false; + } + if (last) { + *last = '\0'; + } + p->chain = last ? k : ""; + p->node = node; + p->file = file; + p->next = (mdr_pkg_t *)cbm_ht_get(x->pkgs, name); + cbm_ht_set(x->pkgs, name, p); + return true; +} + +static bool mdr_add_top(mdr_index_t *x, const char *s, size_t n) { + char *k = mdr_norm_dup(&x->arena, s, n); + if (!k) { + return false; + } + cbm_ht_set(x->top, k, (void *)k); + return true; +} + +static int mdr_def_cmp(const void *a, const void *b) { + const mdr_def_t *x = (const mdr_def_t *)a; + const mdr_def_t *y = (const mdr_def_t *)b; + int c = strcmp(x->file, y->file); + if (c != 0) { + return c; + } + if (x->node->start_line != y->node->start_line) { + return x->node->start_line < y->node->start_line ? -1 : 1; + } + return (x->node->id > y->node->id) - (x->node->id < y->node->id); +} + +/* Every code definition of the graph, sorted by file. False when memory ran + * out. */ +static bool mdr_build_defs(mdr_index_t *x, const cbm_gbuf_t *graph) { + /* the ADR records first: a repository of documents alone has records but + * no code definitions, and its supersedes links still need them */ + const cbm_gbuf_node_t **adrs = NULL; + int nadr = 0; + if (cbm_gbuf_find_by_label(graph, "ADR", &adrs, &nadr) == 0) { + for (int i = 0; i < nadr; i++) { + if (adrs[i]->file_path && adrs[i]->file_path[0]) { + cbm_ht_set(x->adr_by_file, adrs[i]->file_path, (void *)adrs[i]); + } + } + } + int total = 0; + for (size_t l = 0; MDR_CODE_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, MDR_CODE_LABELS[l], &nodes, &count) == 0) { + total += count; + } + } + if (total == 0) { + return true; + } + x->defs = (mdr_def_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)total * sizeof(*x->defs)); + if (!x->defs) { + return false; + } + for (size_t l = 0; MDR_CODE_LABELS[l]; l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, MDR_CODE_LABELS[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count && x->ndefs < total; i++) { + if (nodes[i]->file_path && nodes[i]->file_path[0]) { + x->defs[x->ndefs++] = (mdr_def_t){.file = nodes[i]->file_path, .node = nodes[i]}; + } + } + } + qsort(x->defs, (size_t)x->ndefs, sizeof(*x->defs), mdr_def_cmp); + return true; +} + +/* The module index over the graph's code files and directories. */ +static bool mdr_build_pkgs(mdr_index_t *x, const cbm_gbuf_t *graph) { + CBMHashTable *seen = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + if (!seen) { + return false; + } + bool ok = true; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, "File", &nodes, &count) == 0) { + for (int i = 0; ok && i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !fp[0] || !mdr_code_file(fp)) { + continue; + } + const char *base = mdr_base(fp); + const char *dot = strrchr(base, '.'); + size_t stem_len = dot && dot != base ? (size_t)(dot - base) : strlen(base); + size_t dir_len = base > fp ? (size_t)(base - fp - SKIP_ONE) : 0; + ok = mdr_add_pkg(x, seen, fp, (size_t)(base - fp) + stem_len, nodes[i], fp); + if (ok && dir_len > 0 && mdr_in(base, stem_len, MDR_INDEX_STEMS)) { + ok = mdr_add_pkg(x, seen, fp, dir_len, nodes[i], fp); + } + if (ok && dir_len == 0) { + ok = mdr_add_top(x, base, stem_len); + } + } + } + if (ok && cbm_gbuf_find_by_label(graph, "Folder", &nodes, &count) == 0) { + for (int i = 0; ok && i < count; i++) { + const char *dp = nodes[i]->file_path; + if (!dp || !dp[0]) { + continue; + } + ok = mdr_add_pkg(x, seen, dp, strlen(dp), nodes[i], NULL); + if (ok && !strchr(dp, '/')) { + ok = mdr_add_top(x, dp, strlen(dp)); + } + } + } + cbm_ht_free(seen); + return ok; +} + +static void *mdr_build(const cbm_doclink_build_in_t *in) { + mdr_index_t *x = (mdr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->project_len = x->project ? strlen(x->project) : 0; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->basenames = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->pkgs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->top = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->adr_by_file = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + x->folders = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, MDR_BASENAME_INIT); + if ((in->run_file_count > 0 && !x->run_paths) || !x->basenames || !x->pkgs || !x->top || + !x->adr_by_file || !x->folders || !x->project || !mdr_build_pkgs(x, in->graph) || + !mdr_build_defs(x, in->graph)) { + mdr_destroy(x); + return NULL; + } + for (int i = 0; i < in->file_count; i++) { + int rf = in->files[i].run_file; + if (rf >= 0 && rf < x->run_count) { + x->run_paths[rf] = in->files[i].rel_path; /* valid until destroy */ + } + } + const cbm_gbuf_node_t **folders = NULL; + int nfolders = 0; + if (cbm_gbuf_find_by_label(in->graph, "Folder", &folders, &nfolders) == 0) { + for (int i = 0; i < nfolders; i++) { + if (folders[i]->file_path && folders[i]->file_path[0]) { + /* the graph outlives the index: its strings may be keys */ + cbm_ht_set(x->folders, folders[i]->file_path, (void *)folders[i]); + } + } + } + const cbm_gbuf_node_t **files = NULL; + int nfiles = 0; + if (cbm_gbuf_find_by_label(in->graph, "File", &files, &nfiles) == 0) { + for (int i = 0; i < nfiles; i++) { + if (!files[i]->file_path || !files[i]->file_path[0]) { + continue; + } + const char *base = mdr_base(files[i]->file_path); /* the graph outlives the index */ + uintptr_t n = (uintptr_t)cbm_ht_get(x->basenames, base); + cbm_ht_set(x->basenames, base, (void *)(n + SKIP_ONE)); + } + } + return x; +} + +static void mdr_destroy(void *index) { + mdr_index_t *x = (mdr_index_t *)index; + if (!x) { + return; + } + cbm_ht_free(x->basenames); + cbm_ht_free(x->pkgs); + cbm_ht_free(x->top); + cbm_ht_free(x->adr_by_file); + cbm_ht_free(x->folders); + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->defs); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* ── Paths ───────────────────────────────────────────────────────── */ + +static int mdr_hex(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + CBM_DECIMAL_BASE; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + CBM_DECIMAL_BASE; + } + return CBM_NOT_FOUND; +} + +/* `base` + "/" + `p`, normalized: "." dropped, ".." pops a segment. False + * when the path climbs above the root or does not fit. */ +static bool mdr_join(const char *base, const char *p, char *out, size_t cap) { + size_t w = 0; + const char *parts[PAIR_LEN] = {base, p}; + for (int k = 0; k < PAIR_LEN; k++) { + const char *s = parts[k]; + while (s && *s) { + const char *e = strchr(s, '/'); + size_t n = e ? (size_t)(e - s) : strlen(s); + if (n == 0 || (n == SKIP_ONE && s[0] == '.')) { + /* empty or "." */ + } else if (n == PAIR_LEN && s[0] == '.' && s[SKIP_ONE] == '.') { + if (w == 0) { + return false; + } + while (w > 0 && out[w - SKIP_ONE] != '/') { + w--; + } + if (w > 0) { + w--; /* the slash before the popped segment */ + } + } else { + if (w + n + PAIR_LEN > cap) { + return false; + } + if (w > 0) { + out[w++] = '/'; + } + memcpy(out + w, s, n); + w += n; + } + s = e ? e + SKIP_ONE : NULL; + } + } + out[w] = '\0'; + return true; +} + +static const cbm_gbuf_node_t *mdr_folder(const mdr_index_t *x, const cbm_gbuf_t *graph, + const char *path) { + (void)graph; + return path[0] ? (const cbm_gbuf_node_t *)cbm_ht_get(x->folders, path) : NULL; +} + +static bool mdr_code_label(const char *label) { + for (size_t i = 0; label && MDR_CODE_LABELS[i]; i++) { + if (strcmp(label, MDR_CODE_LABELS[i]) == 0) { + return true; + } + } + return false; +} + +/* The member text's last segment (after "::" or "."), and the one before it. */ +static void mdr_member_names(const char *member, const char **name, size_t *name_len, + const char **owner, size_t *owner_len) { + const char *segs[PAIR_LEN] = {NULL, NULL}; + size_t lens[PAIR_LEN] = {0, 0}; + const char *s = member; + while (*s) { + size_t n = strcspn(s, ":."); + if (n > 0) { + segs[0] = segs[SKIP_ONE]; + lens[0] = lens[SKIP_ONE]; + segs[SKIP_ONE] = s; + lens[SKIP_ONE] = n; + } + s += n; + while (*s == ':' || *s == '.') { + s++; + } + } + *name = segs[SKIP_ONE]; + *name_len = lens[SKIP_ONE]; + *owner = segs[0]; + *owner_len = lens[0]; +} + +/* The QN segment before the node's own name ends with `owner`. */ +static bool mdr_owned_by(const cbm_gbuf_node_t *n, const char *owner, size_t owner_len) { + const char *qn = n->qualified_name; + size_t ql = qn ? strlen(qn) : 0; + size_t nl = n->name ? strlen(n->name) : 0; + if (!owner || ql < nl + owner_len + PAIR_LEN) { + return false; + } + size_t end = ql - nl - SKIP_ONE; /* the '.' before the name */ + if (qn[end] != '.') { + return false; + } + size_t start = end - owner_len; + return memcmp(qn + start, owner, owner_len) == 0 && (start == 0 || qn[start - SKIP_ONE] == '.'); +} + +/* The code definitions of `path`: [*lo, *hi) of the index's sorted list. */ +static void mdr_file_defs(const mdr_index_t *x, const char *path, int *lo, int *hi) { + int a = 0; + int b = x->ndefs; + while (a < b) { + int mid = a + ((b - a) / PAIR_LEN); + if (strcmp(x->defs[mid].file, path) < 0) { + a = mid + SKIP_ONE; + } else { + b = mid; + } + } + int e = a; + while (e < x->ndefs && strcmp(x->defs[e].file, path) == 0) { + e++; + } + *lo = a; + *hi = e; +} + +/* The code definition of file `path` that the range or the member names, or + * NULL. Read from the index's definitions by file, never from DEFINES edges: + * an incremental run's graph holds the edges of the files it re-extracts + * only, while every node is there. */ +static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path, uint32_t first, + uint32_t last, const char *member) { + int lo = 0; + int hi = 0; + mdr_file_defs(x, path, &lo, &hi); + const cbm_gbuf_node_t *best = NULL; + if (first > 0) { + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (n->start_line <= 0 || (uint32_t)n->start_line > first || + (uint32_t)n->end_line < last) { + continue; + } + int span = n->end_line - n->start_line; + int best_span = best ? best->end_line - best->start_line : 0; + if (!best || span < best_span || + (span == best_span && (n->start_line > best->start_line || + (n->start_line == best->start_line && n->id < best->id)))) { + best = n; + } + } + if (best) { + return best; + } + } + if (!member) { + return NULL; + } + const char *name; + const char *owner; + size_t name_len; + size_t owner_len; + mdr_member_names(member, &name, &name_len, &owner, &owner_len); + if (!name) { + return NULL; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *owned = NULL; + int owned_hits = 0; + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (!n->name || strlen(n->name) != name_len || memcmp(n->name, name, name_len) != 0) { + continue; + } + hit = n; + hits++; + if (owner && mdr_owned_by(n, owner, owner_len)) { + owned = n; + owned_hits++; + } + } + if (owned_hits == SKIP_ONE) { + return owned; + } + return owned_hits == 0 && hits == SKIP_ONE ? hit : NULL; +} + +/* Is `path` of a file type the indexer reads? */ +static bool mdr_indexable(const char *path) { + return cbm_language_for_filename(mdr_base(path)) != CBM_LANG_COUNT; +} + +/* ── Qualified names ─────────────────────────────────────────────── */ + +/* Path pieces a written qualifier may leave out (`pkg.Foo` for + * `pkg/src/Foo`, `pkg.mod.Foo` for `pkg/mod.rs`'s Foo). */ +static const char *const MDR_TRANSPARENT[] = { + "src", "lib", "mod", "__init__", "index", "main", "java", "kotlin", "scala", "groovy", NULL}; + +/* Directory names and file names of test code. */ +static const char *const MDR_TEST_DIRS[] = {"test", + "tests", + "testing", + "__tests__", + "spec", + "specs", + "testdata", + "test_data", + "fixtures", + "e2e", + "integration_tests", + "testutil", + "testutils", + "benchmark", + "benchmarks", + "bench", + "mocks", + "mock", + "fakes", + "fake", + "testlib", + NULL}; + +static unsigned char mdr_norm_c(unsigned char c) { + if (c >= 'A' && c <= 'Z') { + return (unsigned char)(c - 'A' + 'a'); + } + return c == '-' ? '_' : c; +} + +/* Equal after lower-casing and reading '-' as '_'. */ +static bool mdr_norm_eq(const char *a, size_t al, const char *b, size_t bl) { + if (al != bl) { + return false; + } + for (size_t i = 0; i < al; i++) { + if (mdr_norm_c((unsigned char)a[i]) != mdr_norm_c((unsigned char)b[i])) { + return false; + } + } + return true; +} + +/* s[0..n) ends with `suffix`. */ +static bool mdr_ends(const char *s, size_t n, const char *suffix) { + size_t k = strlen(suffix); + return n >= k && memcmp(s + n - k, suffix, k) == 0; +} + +static bool mdr_in(const char *s, size_t n, const char *const *list) { + for (size_t i = 0; list[i]; i++) { + if (mdr_norm_eq(s, n, list[i], strlen(list[i]))) { + return true; + } + } + return false; +} + +/* A test file or a file in a test directory. */ +static bool mdr_test_path(const char *path) { + const char *s = path; + for (const char *slash = strchr(s, '/'); slash; slash = strchr(s, '/')) { + if (mdr_in(s, (size_t)(slash - s), MDR_TEST_DIRS)) { + return true; + } + s = slash + SKIP_ONE; + } + const char *base = s; + if (strncmp(base, "test_", strlen("test_")) == 0 || strcmp(base, "conftest.py") == 0) { + return true; + } + const char *dot = strrchr(base, '.'); + if (!dot) { + return false; + } + size_t stem = (size_t)(dot - base); + const char *ext = dot + SKIP_ONE; + bool lower_ext = ext[0] != '\0'; + for (const char *e = ext; *e; e++) { + lower_ext = lower_ext && *e >= 'a' && *e <= 'z'; + } + static const char *const jvm_like[] = {"java", "kt", "cs", "scala", "groovy", "php", NULL}; + return (lower_ext && (mdr_ends(base, stem, "_test") || mdr_ends(base, stem, ".test") || + mdr_ends(base, stem, ".spec"))) || + (strcmp(ext, "rb") == 0 && mdr_ends(base, stem, "_spec")) || + (mdr_in(ext, strlen(ext), jvm_like) && + (mdr_ends(base, stem, "Test") || mdr_ends(base, stem, "Tests"))); +} + +typedef struct { + const char *path; /* as written (no fragment, decoded) */ + uint32_t first; + uint32_t last; + const char *member; + bool name_only; /* a file name alone: the document's directory only */ + bool dir; /* written as a directory */ + bool link; /* a link: relative to the document by definition */ +} mdr_ref_t; + +static void mdr_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *target, uint32_t first, + uint32_t last) { + out->kind = CBM_DOCLINK_EDGE; + out->target = target; + out->exact = true; + out->target_first = first; + out->target_last = last; +} + +enum { MDR_MAX_PIECES = 80 }; /* a code span is at most 150 bytes */ + +typedef struct { + const char *s[MDR_MAX_PIECES]; + size_t n[MDR_MAX_PIECES]; + int count; +} mdr_pieces_t; + +/* The '.'-separated pieces of s. False when there are too many. */ +static bool mdr_split(const char *s, mdr_pieces_t *p) { + p->count = 0; + for (;;) { + if (p->count == MDR_MAX_PIECES) { + return false; + } + size_t n = strcspn(s, "."); + p->s[p->count] = s; + p->n[p->count] = n; + p->count++; + if (!s[n]) { + return true; + } + s += n + SKIP_ONE; + } +} + +/* Is the written qualifier q[0..nq) a piece-aligned suffix of `chain` (a + * node's '.'-joined qualifier pieces), when chain pieces in MDR_TRANSPARENT + * may be skipped? `exact`: every piece as written; otherwise lower-cased and + * '-' read as '_' (`rootMulti.Store` names package rootmulti), except that + * with `types_exact` a capitalized piece (a type) must be written as is + * (`debug.log` is no member of class Debug). `left`: where the chain left of + * the match ends. */ +static bool mdr_chain_match(const char *chain, size_t clen, const mdr_pieces_t *q, int nq, + bool exact, bool types_exact, size_t *left) { + size_t ce = clen; + bool more = clen > 0; + int j = nq - SKIP_ONE; + while (j >= 0) { + if (!more) { + return false; + } + size_t cs = ce; + while (cs > 0 && chain[cs - SKIP_ONE] != '.') { + cs--; + } + const char *piece = chain + cs; + size_t pl = ce - cs; + bool as_written = pl == q->n[j] && memcmp(piece, q->s[j], pl) == 0; + bool same = exact ? as_written : mdr_norm_eq(piece, pl, q->s[j], q->n[j]); + if (same && !as_written && types_exact && piece[0] >= 'A' && piece[0] <= 'Z') { + same = false; /* a type is named in its own case */ + } + if (same) { + j--; + } else if (!mdr_in(piece, pl, MDR_TRANSPARENT)) { + return false; + } + more = cs > 0; + ce = more ? cs - SKIP_ONE : 0; + } + if (left) { + *left = more ? ce : 0; + } + return true; +} + +/* Reverse-domain roots of JVM packages: a name written from one of them is + * absolute (`com.google.protobuf.Message`), never the tail of a relocated + * copy (`org.apache.hadoop.hbase.shaded.com.google.protobuf.Message`). */ +static const char *const MDR_JVM_ROOTS[] = {"com", "org", "net", "io", "edu", + "gov", "java", "javax", "jakarta", "android", + "androidx", "kotlin", "kotlinx", NULL}; + +/* After a match that left chain[0, left): the written name starts the chain + * or follows a source-root piece (`src/main/java`), so it is the package as + * declared, not a relocated copy's tail. */ +static bool mdr_rooted(const char *chain, size_t left) { + if (left == 0) { + return true; + } + size_t s = left; + while (s > 0 && chain[s - SKIP_ONE] != '.') { + s--; + } + return mdr_in(chain + s, left - s, MDR_TRANSPARENT); +} + +static bool mdr_ext_in(const char *path, const char *const *exts) { + const char *dot = strrchr(path, '.'); + return dot && !strchr(dot, '/') && mdr_in(dot + SKIP_ONE, strlen(dot + SKIP_ONE), exts); +} + +/* Languages whose types are written in their own case (`Debug`, never `debug`). */ +static bool mdr_cased_types_file(const char *path) { + static const char *const exts[] = {"java", "kt", "kts", "scala", "groovy", "cs", NULL}; + return mdr_ext_in(path, exts); +} + +static bool mdr_jvm_file(const char *path) { + static const char *const exts[] = {"java", "kt", "kts", "scala", "groovy", NULL}; + return mdr_ext_in(path, exts); +} + +static bool mdr_c_family_file(const char *path) { + static const char *const exts[] = {"c", "h", "cc", "cpp", "cxx", "hpp", + "hh", "hxx", "ipp", "inl", "tpp", NULL}; + return mdr_ext_in(path, exts); +} + +/* Is `dir` a JVM package folder: is the first code definition at or below it + * in a Java/Kotlin/Scala/Groovy file? A dotted name in documentation that + * matches such a folder is far more often a configuration key, an OSGi or JMX + * id or a Maven coordinate than the package (`snapshot.mode` of + * io/debezium/snapshot/mode), so a code name never binds one; a Python + * package is what its dotted name says and keeps binding. */ +static bool mdr_jvm_folder(const mdr_index_t *x, const char *dir) { + char prefix[MDR_PATH_CAP]; + int n = snprintf(prefix, sizeof(prefix), "%s/", dir); + if (n < 0 || (size_t)n >= sizeof(prefix)) { + return false; + } + int a = 0; + int b = x->ndefs; + while (a < b) { + int mid = a + ((b - a) / PAIR_LEN); + if (strcmp(x->defs[mid].file, prefix) < 0) { + a = mid + SKIP_ONE; + } else { + b = mid; + } + } + return a < x->ndefs && strncmp(x->defs[a].file, prefix, (size_t)n) == 0 && + mdr_jvm_file(x->defs[a].file); +} + +/* A repository can carry several copies of one project (snapshots side by + * side, a vendored copy). A definition of another copy never answers the + * document: below the directory where the document's path and the file's part, + * the document's own branch holds the same rest of the path, at least two + * directories deep, as a file with definitions. A shared file name alone + * (`__init__.py` of two packages) is no copy, and different modules of one + * build hold different paths there, so their links stay. */ +static bool mdr_other_copy(const mdr_index_t *x, const char *doc, const char *file) { + enum { MIN_SHARED_SLASHES = 3 }; /* "/dir/dir/file" */ + size_t common = 0; + for (size_t i = 0; doc[i] && doc[i] == file[i]; i++) { + if (doc[i] == '/') { + common = i + SKIP_ONE; + } + } + const char *dslash = strchr(doc + common, '/'); + const char *fslash = strchr(file + common, '/'); + if (!dslash || !fslash) { + return false; + } + int slashes = 0; + for (const char *c = fslash; *c; c++) { + slashes += *c == '/'; + } + if (slashes < MIN_SHARED_SLASHES) { + return false; + } + char twin[MDR_PATH_CAP]; + int n = snprintf(twin, sizeof(twin), "%.*s%s", (int)(dslash - doc), doc, fslash); + if (n < 0 || (size_t)n >= sizeof(twin)) { + return false; + } + int lo = 0; + int hi = 0; + mdr_file_defs(x, twin, &lo, &hi); + return hi > lo; +} + +/* The node's qualifier chain (its QN without the project and its own name); + * false when the QN does not have that shape. */ +static bool mdr_node_chain(const mdr_index_t *x, const cbm_gbuf_node_t *n, const char **chain, + size_t *clen) { + const char *qn = n->qualified_name; + size_t name_len = n->name ? strlen(n->name) : 0; + if (!qn || name_len == 0 || strncmp(qn, x->project, x->project_len) != 0 || + qn[x->project_len] != '.') { + return false; + } + const char *rest = qn + x->project_len + SKIP_ONE; + size_t rl = strlen(rest); + if (rl < name_len || strcmp(rest + rl - name_len, n->name) != 0) { + return false; + } + if (rl == name_len) { + *chain = rest; + *clen = 0; + return true; + } + if (rest[rl - name_len - SKIP_ONE] != '.') { + return false; + } + *chain = rest; + *clen = rl - name_len - SKIP_ONE; + return true; +} + +static bool mdr_python_file(const char *path) { + return mdr_ends(path, strlen(path), ".py") || mdr_ends(path, strlen(path), ".pyi"); +} + +static bool mdr_classlike(const char *label) { + static const char *const labels[] = {"Class", "Struct", "Interface", "Trait", + "Enum", "Type", NULL}; + for (size_t i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* Does the repository itself define the written root (`fastapi` of + * `fastapi.routing.X`): a top-level directory or code file, or a type? */ +static bool mdr_root_known(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *root, + size_t len) { + char key[MDR_PATH_CAP]; + if (len >= sizeof(key)) { + return false; + } + for (size_t i = 0; i < len; i++) { + key[i] = (char)mdr_norm_c((unsigned char)root[i]); + } + key[len] = '\0'; + if (cbm_ht_get(x->top, key)) { + return true; + } + memcpy(key, root, len); + key[len] = '\0'; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, key, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + if (mdr_classlike(nodes[i]->label) && nodes[i]->file_path && + mdr_code_file(nodes[i]->file_path)) { + return true; + } + } + } + return false; +} + +/* The one non-member definition named `name` in the module file `file`. */ +static const cbm_gbuf_node_t *mdr_module_item(const mdr_index_t *x, const char *file, + const char *name) { + int lo = 0; + int hi = 0; + mdr_file_defs(x, file, &lo, &hi); + const cbm_gbuf_node_t *hit = NULL; + for (int i = lo; i < hi; i++) { + const cbm_gbuf_node_t *n = x->defs[i].node; + if (!n->name || strcmp(n->name, name) != 0 || strcmp(n->label, "Method") == 0 || + strcmp(n->label, "Field") == 0) { + continue; + } + if (hit) { + return NULL; + } + hit = n; + } + return hit; +} + +/* Candidates of one class: the first one, and how many (counting stops at + * two: one is an edge, two are ambiguous). */ +typedef struct { + const cbm_gbuf_node_t *node; + const mdr_pkg_t *pkg; + int hits; +} mdr_tally_t; + +static void mdr_tally(mdr_tally_t *t, const cbm_gbuf_node_t *node, const mdr_pkg_t *pkg) { + if (t->hits == 0) { + t->node = node; + t->pkg = pkg; + } + if (t->hits < PAIR_LEN) { + t->hits++; + } +} + +/* A qualified name written in a code span: EXACT only. The one code + * definition named like its last piece whose qualifier chain ends with the + * written qualifier, or the one module or package so named. A match with + * every piece as written outranks one that needs case folding, so + * `fastapi.middleware` names the package fastapi/middleware, not the method + * FastAPI.middleware. Order: definitions as written, modules as written, + * definitions folded, modules folded; the first class with a candidate + * decides. */ +static void mdr_resolve_name(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const CBMDocLinkMdPath *p, cbm_doclink_outcome_t *out) { + mdr_pieces_t q; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + if (!mdr_split(p->path, &q) || q.count < PAIR_LEN) { + return; + } + int nq = q.count - SKIP_ONE; /* the qualifier pieces */ + char name[MDR_PATH_CAP]; + size_t name_len = q.n[nq]; + if (name_len >= sizeof(name)) { + return; + } + memcpy(name, q.s[nq], name_len); + name[name_len] = '\0'; + bool doc_test = mdr_test_path(doc); + /* `std::hash` is the standard library's, never a C/C++ definition of this + * repository (a specialization of it, a header named like it) */ + bool std_root = q.n[0] == strlen("std") && memcmp(q.s[0], "std", q.n[0]) == 0; + bool jvm_root = mdr_in(q.s[0], q.n[0], MDR_JVM_ROOTS); + enum { DEF_EXACT, MOD_EXACT, DEF_FOLDED, MOD_FOLDED, TALLIES }; + mdr_tally_t tally[TALLIES] = {{0}}; + int test_hits = 0; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + count = 0; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *chain = NULL; + size_t clen = 0; + if (!mdr_code_label(n->label) || !n->file_path || !mdr_code_file(n->file_path) || + !mdr_node_chain(x, n, &chain, &clen)) { + continue; + } + if (p->colon && !mdr_python_file(n->file_path)) { + continue; /* `module:attr` is Python's form */ + } + if (strcmp(n->label, "Field") == 0) { + /* a field only through its type, written as the type is named: + * never through an instance (`self.x`, `app.state`) */ + const char *owner = chain + clen; + while (owner > chain && owner[-SKIP_ONE] != '.') { + owner--; + } + size_t ol = (size_t)(chain + clen - owner); + if (p->instance || ol != q.n[nq - SKIP_ONE] || + memcmp(owner, q.s[nq - SKIP_ONE], ol) != 0) { + continue; + } + } + size_t left = 0; + bool exact = mdr_chain_match(chain, clen, &q, nq, true, false, &left); + bool folded = !exact && mdr_chain_match(chain, clen, &q, nq, false, false, &left); + if (!exact && !folded) { + continue; + } + /* The name-scope rules only ever take a link away: a refused candidate + * still counts (two candidates stay ambiguous) but is never linked. */ + bool refused = (std_root && mdr_c_family_file(n->file_path)) || + (folded && mdr_cased_types_file(n->file_path) && + !mdr_chain_match(chain, clen, &q, nq, false, true, NULL)) || + (jvm_root && mdr_jvm_file(n->file_path) && !mdr_rooted(chain, left)) || + mdr_other_copy(x, doc, n->file_path); + if (!doc_test && mdr_test_path(n->file_path)) { + test_hits++; + continue; + } + mdr_tally(&tally[exact ? DEF_EXACT : DEF_FOLDED], refused ? NULL : n, NULL); + } + /* modules and packages so named */ + for (const mdr_pkg_t *m = (const mdr_pkg_t *)cbm_ht_get(x->pkgs, name); m; m = m->next) { + size_t cl = strlen(m->chain); + const char *mfile = m->file ? m->file : m->node->file_path; + size_t left = 0; + bool exact = mdr_chain_match(m->chain, cl, &q, nq, true, false, &left); + if (!exact && !mdr_chain_match(m->chain, cl, &q, nq, false, false, &left)) { + continue; + } + bool refused = + mfile && ((std_root && mdr_c_family_file(mfile)) || + (jvm_root && mdr_jvm_file(mfile) && !mdr_rooted(m->chain, left)) || + mdr_other_copy(x, doc, mfile) || + (!m->file && m->node->label && strcmp(m->node->label, "Folder") == 0 && + mdr_jvm_folder(x, mfile))); + if (!doc_test && mdr_test_path(m->file ? m->file : m->node->file_path)) { + test_hits++; + continue; + } + if (refused) { + mdr_tally(&tally[exact ? MOD_EXACT : MOD_FOLDED], NULL, NULL); + } else { + mdr_tally(&tally[exact ? MOD_EXACT : MOD_FOLDED], m->node, m); + } + } + for (int k = 0; k < TALLIES; k++) { + if (tally[k].hits == 0) { + continue; + } + if (tally[k].hits > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + if (!tally[k].node) { + break; /* its one candidate is refused: no link, and no later class decides */ + } + const mdr_pkg_t *mod = tally[k].pkg; + const cbm_gbuf_node_t *item = mod && mod->file ? mdr_module_item(x, mod->file, name) : NULL; + mdr_edge(out, item ? item : tally[k].node, 0, 0); + return; + } + if (test_hits > 0) { + out->reason = CBM_DOCLINK_REASON_TEST_ONLY; + return; + } + if (mdr_root_known(x, graph, q.s[0], q.n[0])) { + out->reason = CBM_DOCLINK_REASON_MISSING; /* the repository's name, but nothing so named */ + return; + } + out->kind = CBM_DOCLINK_LOCAL; /* code of another project (a library, the standard one) */ +} + +static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const mdr_ref_t *r, cbm_doclink_outcome_t *out) { + const char *p = r->path; + bool rooted = p[0] == '/'; + bool relative = strncmp(p, "./", PAIR_LEN) == 0 || strncmp(p, "../", PAIR_LEN + 1) == 0; + char doc_dir[MDR_PATH_CAP]; + const char *slash = strrchr(doc, '/'); + size_t dl = slash ? (size_t)(slash - doc) : 0; + if (dl >= sizeof(doc_dir)) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + memcpy(doc_dir, doc, dl); + doc_dir[dl] = '\0'; + char tries[MDR_TRIES][MDR_PATH_CAP]; + int nt = 0; + int attempted = 0; + if (!rooted) { + attempted++; + nt += mdr_join(doc_dir, p, tries[nt], MDR_PATH_CAP); + } + if (!relative && !r->name_only) { + attempted++; + const char *q = p; + while (*q == '/') { + q++; + } + if (mdr_join("", q, tries[nt], MDR_PATH_CAP) && (nt == 0 || strcmp(tries[0], tries[nt]))) { + nt++; + } + } + for (int i = 0; i < nt; i++) { + const char *t = tries[i]; + if (!t[0]) { + continue; /* the repository root itself */ + } + const cbm_gbuf_node_t *file = cbm_pipeline_file_node(graph, x->project, t); + if (file) { + if (strcmp(t, doc) == 0) { + out->kind = CBM_DOCLINK_LOCAL; /* a link into the same document */ + return; + } + const cbm_gbuf_node_t *seg = (r->first > 0 || r->member) + ? mdr_segment(x, t, r->first, r->last, r->member) + : NULL; + mdr_edge(out, seg ? seg : file, r->first, r->last); + return; + } + const cbm_gbuf_node_t *folder = mdr_folder(x, graph, t); + if (folder) { + mdr_edge(out, folder, 0, 0); + return; + } + } + out->kind = CBM_DOCLINK_UNRESOLVED; + if ((attempted > 0 && nt == 0) || (!r->dir && !mdr_indexable(p))) { + /* above the repository root; a file or a page the indexer does not read */ + out->kind = CBM_DOCLINK_LOCAL; + return; + } + if (r->name_only && !r->link && cbm_ht_get(x->basenames, mdr_base(p))) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; + return; + } + /* doc rot when the reference points into the repository: its directory is + * there (a bare name in a link: the document's own directory) */ + for (int i = 0; i < nt; i++) { + char parent[MDR_PATH_CAP]; + const char *ps = strrchr(tries[i], '/'); + size_t pl = ps ? (size_t)(ps - tries[i]) : 0; + memcpy(parent, tries[i], pl); + parent[pl] = '\0'; + if (pl > 0 && mdr_folder(x, graph, parent) && (strchr(p, '/') || r->link)) { + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + if (pl == 0 && r->link && !rooted) { + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + } + out->kind = CBM_DOCLINK_LOCAL; /* not in this repository, nor its directory */ +} + +/* A link destination: `<...>` dropped, %XX decoded, `#L` fragment read, query + * dropped. False when nothing of a path is left (an anchor or a query). */ +static bool mdr_link_path(const char *raw, char *buf, size_t cap, mdr_ref_t *r) { + size_t n = strlen(raw); + while (n > 0 && (raw[0] == '<' || raw[0] == '>')) { + raw++; + n--; + } + while (n > 0 && (raw[n - SKIP_ONE] == '<' || raw[n - SKIP_ONE] == '>')) { + n--; + } + size_t w = 0; + for (size_t i = 0; i < n; i++) { + if (w + SKIP_ONE >= cap) { + return false; + } + int h1 = i + PAIR_LEN < n && raw[i] == '%' ? mdr_hex(raw[i + SKIP_ONE]) : CBM_NOT_FOUND; + int h2 = h1 >= 0 ? mdr_hex(raw[i + PAIR_LEN]) : CBM_NOT_FOUND; + if (h1 >= 0 && h2 >= 0 && (h1 * MDR_HEX_BASE) + h2 != 0) { + buf[w++] = (char)((h1 * MDR_HEX_BASE) + h2); + i += MDR_PCT_LEN - SKIP_ONE; + } else { + buf[w++] = raw[i]; + } + } + buf[w] = '\0'; + char *hash = strchr(buf, '#'); + if (hash) { + *hash = '\0'; + (void)cbm_doclink_md_line_fragment(hash + SKIP_ONE, &r->first, &r->last); + } + char *query = strchr(buf, '?'); + if (query) { + *query = '\0'; + } + r->path = buf; + r->dir = buf[0] && buf[strlen(buf) - SKIP_ONE] == '/'; + r->name_only = !strchr(buf, '/'); + return buf[0] != '\0'; +} + +/* `ADR-12`, `ADR 012`, `adr_7` (the whole text): the canonical node name. */ +static bool mdr_adr_id(const char *raw, char *name, size_t cap) { + size_t n = strlen(raw); + if (n < PAIR_LEN * PAIR_LEN || mdr_norm_c((unsigned char)raw[0]) != 'a' || + mdr_norm_c((unsigned char)raw[SKIP_ONE]) != 'd' || + mdr_norm_c((unsigned char)raw[PAIR_LEN]) != 'r' || + (raw[PAIR_LEN + SKIP_ONE] != '-' && raw[PAIR_LEN + SKIP_ONE] != '_' && + raw[PAIR_LEN + SKIP_ONE] != ' ')) { + return false; + } + unsigned v = 0; + size_t digits = 0; + for (size_t i = PAIR_LEN * PAIR_LEN; i < n; i++) { + if (raw[i] < '0' || raw[i] > '9' || digits >= MDR_ADR_DIGITS) { + return false; + } + v = (v * CBM_DECIMAL_BASE) + (unsigned)(raw[i] - '0'); + digits++; + } + return digits > 0 && snprintf(name, cap, "ADR-%u", v) > 0; +} + +/* An ADR's own "supersedes X": X by its id, or the file a link names, to its + * ADR node. */ +static void mdr_resolve_adr(const mdr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const char *raw, cbm_doclink_outcome_t *out) { + char name[MDR_PATH_CAP]; + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_MISSING; + if (mdr_adr_id(raw, name, sizeof(name))) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + if (nodes[i]->label && strcmp(nodes[i]->label, "ADR") == 0) { + hit = nodes[i]; + hits++; + } + } + } + if (hits == SKIP_ONE) { + mdr_edge(out, hit, 0, 0); + } else if (hits > SKIP_ONE) { + out->reason = CBM_DOCLINK_REASON_AMBIGUOUS; /* two ADR logs number alike */ + } + return; + } + char buf[MDR_PATH_CAP]; + mdr_ref_t r = {.link = true}; + if (!mdr_link_path(raw, buf, sizeof(buf), &r)) { + return; + } + mdr_resolve_path(x, graph, doc, &r, out); + if (out->kind != CBM_DOCLINK_EDGE) { + return; /* missing, ambiguous, or no file of this repository */ + } + const cbm_gbuf_node_t *adr = + out->target->file_path + ? (const cbm_gbuf_node_t *)cbm_ht_get(x->adr_by_file, out->target->file_path) + : NULL; + if (!adr) { + out->kind = CBM_DOCLINK_UNRESOLVED; /* the file is there, but it is no record */ + out->reason = CBM_DOCLINK_REASON_MISSING; + return; + } + mdr_edge(out, adr, 0, 0); +} + +static void mdr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const mdr_index_t *x = (const mdr_index_t *)index; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + char buf[MDR_PATH_CAP]; + mdr_ref_t r = {0}; + switch (link->syntax) { + case CBM_DOCLINK_MD_SUPERSEDES: + case CBM_DOCLINK_MD_SUPERSEDES_PROSE: + mdr_resolve_adr(x, graph, doc, link->raw, out); + return; + case CBM_DOCLINK_MD_LINK: + r.link = true; + if (!mdr_link_path(link->raw, buf, sizeof(buf), &r)) { + out->kind = CBM_DOCLINK_LOCAL; /* an anchor, a query: nothing of a path */ + return; + } + break; + case CBM_DOCLINK_MD_PATH: { + size_t n = strlen(link->raw); + if (n >= sizeof(buf)) { + break; /* longer than any path looked up: unparseable */ + } + memcpy(buf, link->raw, n + SKIP_ONE); + r.path = buf; + break; + } + case CBM_DOCLINK_MD_CODE_NAME: { + CBMDocLinkMdPath p; + if (cbm_doclink_md_classify_span(link->raw, strlen(link->raw), buf, sizeof(buf), &p) && + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + mdr_resolve_name(x, graph, doc, &p, out); + return; + } + break; + } + case CBM_DOCLINK_MD_BARE_PATH: + case CBM_DOCLINK_MD_CODE_PATH: { + CBMDocLinkMdPath p; + if (!cbm_doclink_md_classify_span(link->raw, strlen(link->raw), buf, sizeof(buf), &p) || + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + break; + } + r.path = p.path; + r.member = p.member; + r.first = p.first_line; + r.last = p.last_line; + r.name_only = p.shape == CBM_DOCLINK_MD_FILENAME; + r.dir = p.shape == CBM_DOCLINK_MD_DIR; + if (r.dir && r.path[0] == '/' && !strchr(r.path + SKIP_ONE, '/')) { + /* `/docs`, `/health`: one rooted segment is a URL route, not the + * directory of that name (`/x/bank` stays a path) */ + out->kind = CBM_DOCLINK_LOCAL; + return; + } + break; + } + default: + break; + } + if (!r.path || !r.path[0]) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = CBM_DOCLINK_REASON_UNPARSEABLE; + return; + } + mdr_resolve_path(x, graph, doc, &r, out); +} + +/* ── Shared with the other document resolvers (doc_links.h) ──────── */ + +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member) { + return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; +} + +/* ── A region of a file's text (the field test's H8 bind_range) ─────── */ + +/* A blank or comment-only line (H8 _skippable). */ +static bool mdr_skippable(const char *s, size_t n) { + size_t i = 0; + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r')) { + i++; + } + if (i == n) { + return true; + } + const char *t = s + i; + size_t r = n - i; + if ((r >= PAIR_LEN && t[0] == '/' && (t[1] == '/' || t[1] == '*')) || t[0] == '*' || + t[0] == ';' || t[0] == '%' || (r >= PAIR_LEN && t[0] == '\'' && t[1] == ' ') || + (r >= PAIR_LEN && t[0] == '-' && t[1] == '-' && r > PAIR_LEN && + (t[PAIR_LEN] == ' ' || t[PAIR_LEN] == '\t')) || + (r >= strlen("\n" /* 22 */ + "app/html.py in HTML, `cfg.Name` through an instance.\n"; /* 23 */ + +static void md_write_fixture(const char *tmp) { + th_write_file(TH_PATH(tmp, "docs/guide.md"), MD_GUIDE); + th_write_file(TH_PATH(tmp, "app/__init__.py"), ""); + th_write_file(TH_PATH(tmp, "app/routing.py"), "class APIRouter:\n" /* 1 */ + " prefix = ''\n" /* 2 */ + "\n" /* 3 */ + " def add(self, p):\n" /* 4 */ + " return p\n" /* 5 */ + "\n" /* 6 */ + "\n" /* 7 */ + "def helper():\n" /* 8 */ + " return 1\n"); /* 9 */ + th_write_file(TH_PATH(tmp, "app/applications.py"), "class App:\n" + " def middleware(self):\n" + " return None\n"); + th_write_file(TH_PATH(tmp, "app/middleware/__init__.py"), "VALUE = 1\n"); + th_write_file(TH_PATH(tmp, "tests/test_routing.py"), "def only_in_tests():\n" + " return 0\n"); + th_write_file(TH_PATH(tmp, "lib/main.py"), "def run():\n" + " return None\n"); + th_write_file(TH_PATH(tmp, "pkg/config.go"), "package pkg\n" + "\n" + "type Config struct {\n" + "\tName string\n" + "}\n" + "\n" + "func Util() {}\n"); +} + +/* ── extraction ──────────────────────────────────────────────────── */ + +static const CBMDocLink *md_token(const CBMFileResult *r, const char *raw) { + return dm_find_token(r, raw); +} + +static bool md_ends_with(const char *s, const char *suffix) { + size_t a = s ? strlen(s) : 0; + size_t b = strlen(suffix); + return a >= b && strcmp(s + a - b, suffix) == 0; +} + +TEST(doc_links_md_extract_tokens) { + CBMFileResult *r = dm_extract(MD_GUIDE, CBM_LANG_MARKDOWN, "docs/guide.md"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->doc_links.failed); + /* before the first heading: the file is the source */ + const CBMDocLink *t = md_token(r, "pkg/config.go"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 1); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH); + ASSERT_TRUE((t->flags & CBM_DOCLINK_FLAG_FILE) != 0); + /* an inline link and a link reference definition, under "Guide" */ + t = md_token(r, "../pkg/config.go"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 5); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK); + ASSERT_TRUE(md_ends_with(t->source_qn, "Guide")); + ASSERT_EQ(t->def_line, 3); + ASSERT_EQ(t->flags & CBM_DOCLINK_FLAG_FILE, 0); + t = md_token(r, "../app/routing.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 7); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK); + /* paths under "Paths" */ + t = md_token(r, "app/routing.py#L4-L5"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 11); + ASSERT_TRUE(md_ends_with(t->source_qn, "Paths")); + ASSERT_NOT_NULL(md_token(r, "app/routing.py:1-9")); + t = md_token(r, "app/applications.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_PATH); + t = md_token(r, "app/middleware/"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH); + /* a token (the resolver calls it a route); one directory name alone is a + * bare path: in a held-out audit `doc/` was another project's as often as not */ + t = md_token(r, "/docs"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH); + /* an image, a URL: no tokens */ + ASSERT_NULL(md_token(r, "img/a.png")); + ASSERT_NULL(md_token(r, "https://example.org/a/b.py")); + ASSERT_NULL(md_token(r, "a/b.py")); + /* qualified names under "Names" */ + t = md_token(r, "app.routing.APIRouter.add"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_NAME); + ASSERT_EQ(t->line, 16); + ASSERT_TRUE(md_ends_with(t->source_qn, "Names")); + ASSERT_NOT_NULL(md_token(r, "pkg.Config.Name")); + ASSERT_NULL(md_token(r, "self.Name")); /* one identifier after `self.`: a bare name */ + ASSERT_NOT_NULL(md_token(r, "os.path.join")); + /* a file name alone is its own family: in the held-out audit every wrong + * code path was one (`config.yaml`: the reader's own file) */ + t = md_token(r, "main.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH); + /* fenced code and HTML comments are not scanned; HTML is */ + ASSERT_NULL(md_token(r, "app/fenced.py")); + ASSERT_NULL(md_token(r, "app/commented.py")); + t = md_token(r, "app/html.py"); + ASSERT_NOT_NULL(t); + ASSERT_EQ(t->line, 23); + cbm_free_result(r); + PASS(); +} + +/* Two headings with one title share one Section node, which owns only one of + * them: their references belong to the file, never to the other heading. */ +TEST(doc_links_md_repeated_headings) { + const char *src = "# Changes\n" /* 1 */ + "\n" /* 2 */ + "## Fixes\n" /* 3 */ + "\n" /* 4 */ + "See `app/one.py`.\n" /* 5 */ + "\n" /* 6 */ + "## Fixes\n" /* 7 */ + "\n" /* 8 */ + "See `app/two.py`.\n" /* 9 */ + "\n" /* 10 */ + "## Unique\n" /* 11 */ + "\n" /* 12 */ + "See `app/three.py`.\n"; /* 13 */ + CBMFileResult *r = dm_extract(src, CBM_LANG_MARKDOWN, "CHANGES.md"); + ASSERT_NOT_NULL(r); + const CBMDocLink *one = md_token(r, "app/one.py"); + const CBMDocLink *two = md_token(r, "app/two.py"); + const CBMDocLink *three = md_token(r, "app/three.py"); + ASSERT_NOT_NULL(one); + ASSERT_NOT_NULL(two); + ASSERT_NOT_NULL(three); + ASSERT_TRUE((one->flags & CBM_DOCLINK_FLAG_FILE) != 0); + ASSERT_TRUE((two->flags & CBM_DOCLINK_FLAG_FILE) != 0); + ASSERT_EQ(three->flags & CBM_DOCLINK_FLAG_FILE, 0); + ASSERT_TRUE(md_ends_with(three->source_qn, "Unique")); + cbm_free_result(r); + PASS(); +} + +typedef struct { + const char *text; + CBMDocLinkMdShape shape; + const char *path; + const char *member; + uint32_t first; + uint32_t last; + bool instance; + bool colon; +} md_case_t; + +TEST(doc_links_md_classify_span) { + static const md_case_t cases[] = { + {"src/app/main.go", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 0, 0, false, false}, + {"src/app/", CBM_DOCLINK_MD_DIR, "src/app/", NULL, 0, 0, false, false}, + {"src/app/main.go#L3-L9", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 3, 9, false, false}, + {"src/app/main.go:12", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 12, 12, false, false}, + {"src/lib.rs::Type::run", CBM_DOCLINK_MD_FILE, "src/lib.rs", "Type::run", 0, 0, false, + false}, + {"src\\win\\file.cs", CBM_DOCLINK_MD_FILE, "src/win/file.cs", NULL, 0, 0, false, false}, + {"main.go:7-8", CBM_DOCLINK_MD_FILENAME, "main.go", NULL, 7, 8, false, false}, + {"README.md", CBM_DOCLINK_MD_FILENAME, "README.md", NULL, 0, 0, false, false}, + {"Makefile", CBM_DOCLINK_MD_FILENAME, "Makefile", NULL, 0, 0, false, false}, + /* a built or shipped file is a file name, never a member (`OliveTin.exe` + * took the module stem of OliveTin.exe.manifest in a held-out audit) */ + {"OliveTin.exe", CBM_DOCLINK_MD_FILENAME, "OliveTin.exe", NULL, 0, 0, false, false}, + {"app.manifest", CBM_DOCLINK_MD_FILENAME, "app.manifest", NULL, 0, 0, false, false}, + {"main.rs::run", CBM_DOCLINK_MD_FILENAME, "main.rs", "run", 0, 0, false, false}, + {"fastapi.routing.APIRouter", CBM_DOCLINK_MD_QUALIFIED, "fastapi.routing.APIRouter", NULL, + 0, 0, false, false}, + {"crate::extract::Json", CBM_DOCLINK_MD_QUALIFIED, "extract.Json", NULL, 0, 0, false, + false}, + {"Vec::new", CBM_DOCLINK_MD_QUALIFIED, "Vec.new", NULL, 0, 0, false, false}, + {"self.app.state", CBM_DOCLINK_MD_QUALIFIED, "app.state", NULL, 0, 0, true, false}, + {"pkg.module:attr", CBM_DOCLINK_MD_QUALIFIED, "pkg.module.attr", NULL, 0, 0, false, true}, + {"Foo#bar", CBM_DOCLINK_MD_QUALIFIED, "Foo.bar", NULL, 0, 0, false, false}, + {"App\\Http\\Kernel", CBM_DOCLINK_MD_QUALIFIED, "App.Http.Kernel", NULL, 0, 0, false, + false}, + {"server.New()", CBM_DOCLINK_MD_QUALIFIED, "server.New", NULL, 0, 0, false, false}, + /* no path and no qualified name */ + {"foo", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"self.x", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"Node.js", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"github.com/x/y", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"https://x.org/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"$HOME/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"PATH=/usr/bin", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"--verbose", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"go run main.go", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"\"a/b.py\"", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"pkg/Type.Member", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"@scope/pkg", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + /* a traversal or a root names no directory (a held-out link: `../` in a + * warning about attacker-controlled paths) */ + {"../", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"./", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + {"../..", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + const md_case_t *c = &cases[i]; + char buf[256]; + CBMDocLinkMdPath p; + bool ok = cbm_doclink_md_classify_span(c->text, strlen(c->text), buf, sizeof(buf), &p); + if (ok != (c->shape != CBM_DOCLINK_MD_NONE) || p.shape != c->shape) { + printf(" case %s: shape %d, expected %d\n", c->text, (int)p.shape, (int)c->shape); + FAIL("classification differs"); + } + if (!ok) { + continue; + } + if (strcmp(p.path, c->path) != 0 || + (c->member ? !p.member || strcmp(p.member, c->member) != 0 : p.member != NULL) || + p.first_line != c->first || p.last_line != c->last || p.instance != c->instance || + p.colon != c->colon) { + printf(" case %s: path %s member %s lines %u-%u instance %d colon %d\n", c->text, + p.path, p.member ? p.member : "(none)", p.first_line, p.last_line, p.instance, + p.colon); + FAIL("classification differs"); + } + } + /* a name never fits a buffer by being cut */ + char tiny[8]; + CBMDocLinkMdPath p; + ASSERT_FALSE(cbm_doclink_md_classify_span("src/app/main.go", 15, tiny, sizeof(tiny), &p)); + PASS(); +} + +/* ── resolution ──────────────────────────────────────────────────── */ + +/* Properties of the MENTIONS edge from the section or file whose qualified + * name ends with `src` to the node whose qualified name ends with `tgt`. */ +static int md_edge(const char *db, const char *src, const char *tgt, char *props, size_t cap) { + int n = 0; + dm_edge(db, src, tgt, props, cap, &n); + return n; +} + +TEST(doc_links_md_resolve) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + md_write_fixture(tmp); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + char reason[64]; + char syntax[64]; + + /* a link and a link reference definition: their files */ + ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"via\":\"markdown\"")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"link\"")); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":5")); + ASSERT_EQ(md_edge(db, "Guide", "routing.py.__file__", props, sizeof(props)), 1); + /* the reference before the first heading: from the File node */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.label = 'File' AND s.file_path = 'docs/guide.md' " + "AND t.file_path = 'pkg/config.go' AND t.label = 'File'"), + 1); + /* a range held by one definition: that definition, the range on the edge */ + ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"target_lines\":[4,5]")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_path\"")); + /* a range across definitions: the file, with the range */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Paths' AND t.label = 'File' " + "AND t.file_path = 'app/routing.py' " + "AND e.properties LIKE '%\"target_lines\":[1,9]%'"), + 1); + /* a bare path in prose; a directory */ + ASSERT_EQ(md_edge(db, "Paths", "applications.py.__file__", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"path\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Paths' AND t.label = 'Folder' " + "AND t.file_path = 'app/middleware'"), + 1); + /* a route, an image, a URL: neither edges nor rows */ + dm_row(db, "docs/guide.md", "/docs", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'MENTIONS' AND t.label = 'Folder' " + "AND t.file_path = 'docs'"), + 0); + dm_row(db, "docs/guide.md", "img/a.png", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + + /* qualified names: a method, a field through its type */ + ASSERT_EQ(md_edge(db, "Names", "routing.APIRouter.add", props, sizeof(props)), 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_name\"")); + ASSERT_EQ(md_edge(db, "Names", "Config.Name", props, sizeof(props)), 1); + /* `app.middleware`: the package as written, not the method App.middleware */ + ASSERT_EQ(md_edge(db, "Names", "App.middleware", props, sizeof(props)), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Names' " + "AND t.file_path = 'app/middleware/__init__.py'"), + 1); + /* a field never through an instance */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'MENTIONS' AND t.label = 'Field' " + "AND e.properties LIKE '%\"line\":23%'"), + 0); + + /* unresolved, by reason */ + dm_row(db, "docs/guide.md", "app.Nope", reason, sizeof(reason), syntax, sizeof(syntax)); + ASSERT_STR_EQ(reason, "missing"); + ASSERT_STR_EQ(syntax, "code_name"); + dm_row(db, "docs/guide.md", "test_routing.only_in_tests", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "test_only_target"); + dm_row(db, "docs/guide.md", "main.py", reason, sizeof(reason), syntax, sizeof(syntax)); + ASSERT_STR_EQ(reason, "ambiguous"); /* lib/main.py, but not next to the document */ + ASSERT_STR_EQ(syntax, "bare_path"); + dm_row(db, "docs/guide.md", "../app/gone.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + dm_row(db, "docs/guide.md", "app/html.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + /* another project's code is no reference to this repository */ + dm_row(db, "docs/guide.md", "os.path.join", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, ""); + + /* back navigation: the sections that mention a method, one hop away */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND t.qualified_name LIKE '%.APIRouter.add' AND s.label = 'Section'"), + 2); + + th_cleanup(tmp); + PASS(); +} + +/* ── incremental == full ─────────────────────────────────────────── */ + +TEST(doc_links_md_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + md_write_fixture(repo); + char db[512]; + char full_db[512]; + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + + /* a body edit of a linked code file: its linking document re-resolves */ + th_write_file(TH_PATH(repo, "app/routing.py"), "class APIRouter:\n" + " prefix = ''\n" + "\n" + " def add(self, p):\n" + " return p + p\n" + "\n" + "\n" + "def helper():\n" + " return 2\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "code body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + /* the document itself changes: a new link, a reference gone */ + char guide[sizeof(MD_GUIDE) + 64]; + snprintf(guide, sizeof(guide), "%s\nSee also [helpers](../app/routing.py#L8-L9).\n", MD_GUIDE); + th_write_file(TH_PATH(repo, "docs/guide.md"), guide); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* a body edit that moves lines: the document bound by lines (#L8-L9, the + * helper until now) re-resolves, though no name changed */ + th_write_file(TH_PATH(repo, "app/routing.py"), "# moved\n" + "# down\n" + "class APIRouter:\n" + " prefix = ''\n" + "\n" + " def add(self, p):\n" + " return p + p\n" + "\n" + "\n" + "def helper():\n" + " return 2\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* the name a row calls missing is added: the run goes FULL and the row + * becomes an edge, exactly as a full index has it */ + th_write_file(TH_PATH(repo, "app/nope.py"), "def Nope():\n" + " return None\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "missing name added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + th_cleanup(tmp); + PASS(); +} + +/* ── ADRs ────────────────────────────────────────────────────────── */ + +static void md_write_adrs(const char *tmp, const char *first_status) { + char first[1024]; + snprintf(first, sizeof(first), + "---\n" + "status: %s\n" + "date: 2024-03-05\n" + "deciders: Ana, Bo\n" + "---\n" + "# Use X for storage\n" + "\n" + "## Context and Problem Statement\n" + "\n" + "We need storage.\n" + "\n" + "## Decision Outcome\n" + "\n" + "We use X, see `app/routing.py`.\n" + "\n" + "Superseded by [ADR-0002](0002-use-y.md).\n", + first_status); + th_write_file(TH_PATH(tmp, "docs/adr/0001-use-x.md"), first); + th_write_file(TH_PATH(tmp, "docs/adr/0002-use-y.md"), "# 2. Use Y instead\n" + "\n" + "Date: 12 May 2024\n" + "\n" + "## Status\n" + "\n" + "Accepted\n" + "\n" + "Supersedes [ADR-0001](0001-use-x.md).\n" + "\n" + "## Context\n" + "\n" + "X was slow.\n" + "\n" + "## Decision\n" + "\n" + "We use Y.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0003-cache.md"), + "# Cache reads\n" + "\n" + "## Status\n" + "\n" + "Proposed\n" + "\n" + "- Replaces ADR-2\n" + "\n" + "## Decision\n" + "\n" + "The cache replaces the reader described in [ADR-1](0001-use-x.md).\n"); + th_write_file(TH_PATH(tmp, "docs/adr/README.md"), "# Decisions\n\n## Status\n\nAccepted\n"); + th_write_file(TH_PATH(tmp, "docs/adr/template.md"), + "# Title\n\n## Status\n\n{proposed | accepted}\n\n## Decision\n\nTBD\n"); +} + +/* Properties of the ADR node of `file`; "" when there is none. */ +static void md_adr_props(const char *db, const char *file, char *out, size_t cap) { + out[0] = '\0'; + sqlite3 *h = NULL; + if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(h, + "SELECT name || ' ' || properties FROM nodes WHERE label = 'ADR' " + "AND file_path = ?1", + -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, file, -1, SQLITE_TRANSIENT); + if (sqlite3_step(st) == SQLITE_ROW) { + snprintf(out, cap, "%s", (const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + sqlite3_close(h); +} + +TEST(doc_links_md_adr) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_adr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + md_write_fixture(tmp); + md_write_adrs(tmp, "superseded"); + char db[512]; + snprintf(db, sizeof(db), "%s/adr.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[2048]; + + /* three records; README and template are none */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'ADR'"), 3); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'DEFINES' AND t.label = 'ADR'"), + 3); + /* facts: front matter */ + md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props)); + ASSERT_TRUE(strncmp(props, "ADR-1 ", strlen("ADR-1 ")) == 0); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"superseded\"")); + ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-03-05\"")); + ASSERT_NOT_NULL(strstr(props, "\"deciders\":\"Ana, Bo\"")); + ASSERT_NOT_NULL(strstr(props, "\"title\":\"Use X for storage\"")); + ASSERT_NOT_NULL(strstr(props, "\"superseded_by\":\"0002-use-y.md\"")); + ASSERT_NOT_NULL(strstr(props, "We use X")); /* the decision, searchable */ + /* facts: header field date, Status section */ + md_adr_props(db, "docs/adr/0002-use-y.md", props, sizeof(props)); + ASSERT_TRUE(strncmp(props, "ADR-2 ", strlen("ADR-2 ")) == 0); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"accepted\"")); + ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-05-12\"")); + md_adr_props(db, "docs/adr/README.md", props, sizeof(props)); + ASSERT_STR_EQ(props, ""); + + /* SUPERSEDES: by link, and by id at a line head; never from prose that + * uses "replaces" as a verb, never from the superseded record */ + const char *sup = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' " + "AND s.file_path = '%s' AND t.file_path = '%s'"; + char q[512]; + snprintf(q, sizeof(q), sup, "docs/adr/0002-use-y.md", "docs/adr/0001-use-x.md"); + ASSERT_EQ(dm_count(db, q), 1); + snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0002-use-y.md"); + ASSERT_EQ(dm_count(db, q), 1); + snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0001-use-x.md"); + ASSERT_EQ(dm_count(db, q), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'SUPERSEDES'"), 2); + /* the record's sections still link to code */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.file_path = 'docs/adr/0001-use-x.md' " + "AND t.file_path = 'app/routing.py'"), + 1); + + /* incremental == full: a status edit, then a new record that a statement + * names by id */ + char full_db[512]; + snprintf(full_db, sizeof(full_db), "%s/adr-full.db", tmp); + md_write_adrs(tmp, "deprecated"); + ASSERT_EQ(dm_step(tmp, db, full_db, "ADR status edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props)); + ASSERT_NOT_NULL(strstr(props, "\"status\":\"deprecated\"")); + th_write_file(TH_PATH(tmp, "docs/adr/0004-later.md"), "# Later\n\n## Status\n\nAccepted\n\n" + "Supersedes ADR-3.\n\n## Decision\n\n" + "Later.\n"); + ASSERT_EQ(dm_step(tmp, db, full_db, "ADR added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + snprintf(q, sizeof(q), sup, "docs/adr/0004-later.md", "docs/adr/0003-cache.md"); + ASSERT_EQ(dm_count(db, q), 1); + th_cleanup(tmp); + PASS(); +} + +/* The production gate (doclink.h): families held back by the held-out audit + * write rows, not edges; the families that passed still write edges. */ +static int md_supersedes_from(const char *db, const char *file, const char *target); + +TEST(doc_links_md_ship_gate) { + for (size_t i = 0; i < sizeof(DM_HELD_FAMILIES) / sizeof(DM_HELD_FAMILIES[0]); i++) { + ASSERT_FALSE(cbm_doclink_syntax_ships(DM_HELD_FAMILIES[i])); + } + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_LINK)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_ROLE)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_ADOC_INCLUDE)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_LITERALINCLUDE)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_CODE_PATH)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_CODE_PATH)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_SUPERSEDES)); + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_OBJECT)); + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdgate_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + md_write_fixture(tmp); + /* a file name alone, next to its file: it resolves, and its tier holds it */ + th_write_file(TH_PATH(tmp, "pkg/README.md"), "# Pkg\n\nSee `config.go`.\n"); + /* an ADR's statement ships as SUPERSEDES; the same words in prose are held */ + th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"), + "# Base\n\n## Status\n\nSuperseded\n\n## Decision\n\nWe use A.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"), + "# Next\n\n## Status\n\nAccepted\n\nSupersedes [ADR-0001](0001-base.md)\n\n" + "## Decision\n\nWe use B.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"), + "# Last\n\n## Status\n\nAccepted\n\n## Decision\n\n" + "This record also supersedes ADR-0001 for the cache.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "reason = 'below_bar_tier' AND syntax = 'code_path'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "reason = 'below_bar_tier' AND syntax = 'bare_path' " + "AND rel_path = 'pkg/README.md'"), + 1); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND " + "properties LIKE '%\"syntax\":\"bare_path\"%'"), + 0); + ASSERT_EQ(md_supersedes_from(db, "0002-next.md", "0001-base.md"), 1); + ASSERT_EQ(md_supersedes_from(db, "0003-last.md", "%"), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "reason = 'below_bar_tier' AND syntax = 'supersedes_prose' " + "AND rel_path = 'docs/adr/0003-last.md'"), + 1); + th_cleanup(tmp); + PASS(); +} + +/* A held-out finding: a spec template's combined field `Supersedes / Depends + * on: [Spec 2](...)` states no supersession (the record depended on the one it + * named); the plain statement next to it still does. */ +TEST(doc_links_md_adr_label_choice) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrl_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"), + "# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"), + "# Next\n\n## Status\n\nAccepted\n\n" + "Supersedes / Depends on: [ADR-0001](0001-base.md)\n\n" + "## Decision\n\nWe add B.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"), "# Last\n\n## Status\n\nAccepted\n\n" + "Supersedes [ADR-0001](0001-base.md).\n\n" + "## Decision\n\nWe use C.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/adr.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0002-next.md'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0003-last.md'"), + 1); + th_cleanup(tmp); + PASS(); +} + +static int md_supersedes_from(const char *db, const char *file, const char *target) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' " + "AND s.file_path = 'docs/adr/%s' AND t.file_path LIKE 'docs/adr/%s'", + file, target); + return dm_count(db, sql); +} + +static int md_supersedes_syntax(const char *db, const char *file, const char *syntax) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/%s' " + "AND e.properties LIKE '%%\"syntax\":\"%s\"%%'", + file, syntax); + return dm_count(db, sql); +} + +/* Held-out findings: a field whose value is "none" goes on to name records it + * extends; a record reports another record's supersession (`ADR 0002 + * supersedes ...`, a table cell `0002 (supersedes ...)`); the record after a + * statement's full stop is no target. The record's own number as the subject + * is its own statement. */ +TEST(doc_links_md_adr_reported) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"), + "# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"), + "# Next\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use B.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0003-none.md"), + "# None\n\n- Status: Accepted\n" + "- Supersedes: none (it extends [ADR 0001](0001-base.md))\n\n" + "## Decision\n\nWe add C.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0004-na.md"), + "# NA\n\n- Status: Accepted\n- Supersedes: **N/A** -- refines ADR-0002\n\n" + "## Decision\n\nWe add D.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0005-report.md"), + "# Report\n\n## Status\n\nAccepted\n\n## Context\n\n" + "ADR 0002 supersedes ADR 0001's first decision.\n\n" + "| Row | Owner |\n|---|---|\n| retries | 0002 (supersedes ADR 0001's rule) |\n\n" + "[ADR 0002](0002-next.md) supersedes ADR 0001 too.\n\n" + "## Decision\n\nWe audit.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0006-own.md"), + "# Own\n\n## Status\n\nAccepted\n\n## Decision\n\n" + "ADR 0006 supersedes ADR 0001.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0007-stop.md"), + "# Stop\n\n## Status\n\nAccepted\n\n## Decision\n\n" + "> This record supersedes ADR-0001. ADR 0002 is an orthogonal decision.\n"); + /* adr-tools' own form: the link text holds the record's number and title */ + th_write_file(TH_PATH(tmp, "docs/adr/0008-tools.md"), + "# 8. Tools\n\n## Status\n\nAccepted\n\n" + "Supersedes [2. Next](0002-next.md)\n\n## Decision\n\nWe use E.\n"); + /* statements in a field table and a bold field */ + th_write_file(TH_PATH(tmp, "docs/adr/0009-table.md"), + "# Table\n\n| Field | Value |\n|---|---|\n| Status | Accepted |\n" + "| Supersedes | [ADR 0001](0001-base.md) |\n\n## Decision\n\nWe use F.\n"); + th_write_file(TH_PATH(tmp, "docs/adr/0010-field.md"), + "# Field\n\n- Status: Accepted\n- **Supersedes:** ADR-0002\n\n" + "## Decision\n\nWe use G.\n"); + /* a paragraph wrapped before the phrase: its subject is the line before */ + th_write_file( + TH_PATH(tmp, "docs/adr/0011-wrap.md"), + "# Wrap\n\n## Status\n\nAccepted\n\n## Context\n\nThe sealed store of\n" + "[ADR 0002](0002-next.md)\nsupersedes ADR-0001 and leaves this record standing.\n\n" + "## Decision\n\nWe use I.\n"); + /* field lines one under the other: each its own statement */ + th_write_file(TH_PATH(tmp, "docs/adr/0012-fields.md"), + "# Fields\n\nStatus: Accepted\nSupersedes [ADR-0001](0001-base.md)\n\n" + "## Decision\n\nWe use J.\n"); + /* a `Supersedes:` field right under running text is still a field */ + th_write_file(TH_PATH(tmp, "docs/adr/0013-colon.md"), + "# Colon\n\n## Status\n\nAccepted after a long review of the store\n" + "Supersedes: [ADR 0002](0002-next.md)\n\n## Decision\n\nWe use K.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/adr.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + ASSERT_EQ(md_supersedes_from(db, "0003-none.md", "%"), 0); + ASSERT_EQ(md_supersedes_from(db, "0004-na.md", "%"), 0); + ASSERT_EQ(md_supersedes_from(db, "0005-report.md", "%"), 0); + ASSERT_EQ(md_supersedes_from(db, "0006-own.md", "0001-base.md"), 1); + ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0001-base.md"), 1); + ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0002-next.md"), 0); + ASSERT_EQ(md_supersedes_from(db, "0008-tools.md", "0002-next.md"), 1); + ASSERT_EQ(md_supersedes_from(db, "0009-table.md", "0001-base.md"), 1); + ASSERT_EQ(md_supersedes_from(db, "0010-field.md", "0002-next.md"), 1); + /* a statement opens its line or field and names a record directly; the + * same words in running prose are the held family supersedes_prose */ + ASSERT_EQ(md_supersedes_syntax(db, "0006-own.md", "supersedes_prose"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0007-stop.md", "supersedes_prose"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0008-tools.md", "supersedes"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0009-table.md", "supersedes"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0010-field.md", "supersedes"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes"), 0); + ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes_prose"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0012-fields.md", "supersedes"), 1); + ASSERT_EQ(md_supersedes_syntax(db, "0013-colon.md", "supersedes"), 1); + th_cleanup(tmp); + PASS(); +} + +/* Held-out findings (AsciiDoc code names, which resolve like Markdown's): a + * `std::` name is the standard library's, not a specialization or header of + * this repository; a Java name written from its root package is absolute, not + * a relocated copy's tail; a type is named in its own case (`debug.log` is a + * log file, not Debug.log); a definition of another copy of the document's + * project (snapshots side by side) never answers the document. Each with a + * control that keeps its link. */ +TEST(doc_links_md_name_scope_rules) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdscope_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + /* a compatibility header named like the standard one (Boost.Typeof's) */ + th_write_file(TH_PATH(tmp, "lib/include/mylib/std/complex.hpp"), + "#include \nnamespace mylib { using cplx = int; }\n"); + th_write_file(TH_PATH(tmp, "lib/include/mylib/widget.hpp"), + "namespace mylib {\nclass Widget {\n public:\n int size() const;\n};\n}\n"); + th_write_file( + TH_PATH(tmp, "shaded/src/main/java/org/acme/shaded/com/google/protobuf/Message.java"), + "package org.acme.shaded.com.google.protobuf;\npublic class Message {}\n"); + th_write_file(TH_PATH(tmp, "core/src/main/java/com/acme/Widget.java"), + "package com.acme;\npublic class Widget {}\n"); + th_write_file(TH_PATH(tmp, "core/src/main/java/com/acme/Ref.java"), + "package com.acme;\npublic class Ref {\n public static class Debug {\n" + " public static void log(String s) {}\n }\n}\n"); + th_write_file( + TH_PATH(tmp, "v1/src/main/java/org/acme/Store.java"), + "package org.acme;\npublic class Store {\n public static class Reader {}\n}\n"); + th_write_file(TH_PATH(tmp, "v2/src/main/java/org/acme/Store.java"), + "package org.acme;\npublic interface Store {}\n"); + th_write_file( + TH_PATH(tmp, "docs/guide.md"), + "# Keys\n\nKeys use `std::complex`, and `mylib::Widget` holds them.\n\n" + "# Wire\n\nProtobuf's `com.google.protobuf.Message`; ours is `com.acme.Widget`.\n\n" + "# Logs\n\nRead `debug.log`.\n\n" + "# Debug\n\nCall `Debug.log`.\n"); + th_write_file(TH_PATH(tmp, "v2/docs/design.md"), "# Design\n\nThe `Store.Reader` reads.\n"); + th_write_file(TH_PATH(tmp, "v1/docs/notes.md"), "# Notes\n\nThe `Store.Reader` reads.\n"); + /* a configuration key that spells a Java package's tail; a Python package */ + th_write_file(TH_PATH(tmp, "core/src/main/java/io/acme/snapshot/mode/SnapshotMode.java"), + "package io.acme.snapshot.mode;\npublic enum SnapshotMode { INITIAL }\n"); + th_write_file(TH_PATH(tmp, "pyapp/__init__.py"), ""); + th_write_file(TH_PATH(tmp, "pyapp/sub/__init__.py"), "from .core import run\n"); + th_write_file(TH_PATH(tmp, "pyapp/sub/core.py"), "def run():\n return 1\n"); + th_write_file(TH_PATH(tmp, "docs/config.md"), + "# Config\n\nSet `snapshot.mode` to `initial`.\n\n" + "# Py\n\nThe `pyapp.sub` package runs it.\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + ASSERT_EQ(md_edge(db, "guide.Keys", "complex.hpp.__file__", props, sizeof(props)), 0); + ASSERT_EQ(md_edge(db, "guide.Keys", "mylib.Widget", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "guide.Wire", "protobuf.Message", props, sizeof(props)), 0); + ASSERT_EQ(md_edge(db, "guide.Wire", "com.acme.Widget", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "guide.Logs", "Debug.log", props, sizeof(props)), 0); + ASSERT_EQ(md_edge(db, "guide.Debug", "Debug.log", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "design.Design", "Store.Reader", props, sizeof(props)), 0); + ASSERT_EQ(md_edge(db, "notes.Notes", "Store.Reader", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "config.Config", "snapshot.mode", props, sizeof(props)), 0); + ASSERT_EQ(md_edge(db, "config.Py", "sub.__init__.py.__file__", props, sizeof(props)), 1); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_md) { + dm_ship_held_families(); + RUN_TEST(doc_links_md_extract_tokens); + RUN_TEST(doc_links_md_repeated_headings); + RUN_TEST(doc_links_md_classify_span); + RUN_TEST(doc_links_md_resolve); + RUN_TEST(doc_links_md_incremental); + RUN_TEST(doc_links_md_adr); + RUN_TEST(doc_links_md_adr_label_choice); + RUN_TEST(doc_links_md_adr_reported); + RUN_TEST(doc_links_md_name_scope_rules); + cbm_doclink_test_reset_ships(); + RUN_TEST(doc_links_md_ship_gate); +} diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c new file mode 100644 index 0000000000..1a78b72547 --- /dev/null +++ b/tests/test_doc_links_pdf.c @@ -0,0 +1,1317 @@ +/* + * test_doc_links_pdf.c — PDF documents -> Sections and MENTIONS edges. + * + * The text-layer extractor (internal/cbm/pdf: fonts, encodings, CMaps, + * cross-reference forms, filters, layout, hostile input), the mention scanner + * (doc_pdf.c, through its test seam: expected tokens are the field-test + * scanner's own output for the same text), the resolver (doc_links_pdf.c: + * tiers, owner-qualified Go methods, hygiene, unresolved reasons), the + * snippet of a page, and incremental == full. PDFs are built here, byte by + * byte, with their cross-reference offsets computed. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "foundation/compat_thread.h" +#include "mcp/mcp.h" +#include "pdf/pdf.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include +#include + +/* ── building PDFs ───────────────────────────────────────────────── */ + +typedef struct { + unsigned char *p; + size_t n; + size_t cap; +} tp_buf_t; + +static void tp_put(tp_buf_t *b, const void *s, size_t n) { + if (b->n + n + 1 > b->cap) { + size_t cap = b->cap ? b->cap * 2 : 4096; + while (cap < b->n + n + 1) { + cap *= 2; + } + b->p = (unsigned char *)realloc(b->p, cap); + b->cap = cap; + } + memcpy(b->p + b->n, s, n); + b->n += n; + b->p[b->n] = 0; +} + +static void tp_puts(tp_buf_t *b, const char *s) { + tp_put(b, s, strlen(s)); +} + +/* One object's body: a value, or a dict plus stream data. */ +typedef struct { + const char *dict; /* the value, or the stream's dict without /Length */ + const void *data; /* stream data, or NULL */ + size_t len; +} tp_obj_t; + +/* A PDF of objects 1..n (objs[i] is object i+1) with a classic xref table; + * `trailer_extra` is added to the trailer dict. Free with free(). */ +static unsigned char *tp_pdf(const tp_obj_t *objs, int n, const char *trailer_extra, + size_t *out_len) { + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n%\xe2\xe3\xcf\xd3\n"); + size_t *off = (size_t *)calloc((size_t)n + 1, sizeof(size_t)); + char line[256]; + for (int i = 0; i < n; i++) { + off[i + 1] = b.n; + snprintf(line, sizeof(line), "%d 0 obj\n", i + 1); + tp_puts(&b, line); + if (objs[i].data) { + snprintf(line, sizeof(line), "<< /Length %zu ", objs[i].len); + tp_puts(&b, line); + tp_puts(&b, objs[i].dict); + tp_puts(&b, " >>\nstream\n"); + tp_put(&b, objs[i].data, objs[i].len); + tp_puts(&b, "\nendstream"); + } else { + tp_puts(&b, objs[i].dict); + } + tp_puts(&b, "\nendobj\n"); + } + size_t xref = b.n; + snprintf(line, sizeof(line), "xref\n0 %d\n0000000000 65535 f \n", n + 1); + tp_puts(&b, line); + for (int i = 1; i <= n; i++) { + snprintf(line, sizeof(line), "%010zu 00000 n \n", off[i]); + tp_puts(&b, line); + } + snprintf(line, sizeof(line), + "trailer\n<< /Size %d /Root 1 0 R %s >>\nstartxref\n%zu\n%%%%EOF\n", n + 1, + trailer_extra ? trailer_extra : "", xref); + tp_puts(&b, line); + free(off); + *out_len = b.n; + return b.p; +} + +static unsigned char *tp_flate(const void *src, size_t n, size_t *out_len) { + uLongf cap = compressBound((uLong)n); + unsigned char *out = (unsigned char *)malloc(cap); + compress(out, &cap, (const Bytef *)src, (uLong)n); + *out_len = cap; + return out; +} + +#define TP_CATALOG "<< /Type /Catalog /Pages 2 0 R >>" +#define TP_HELV "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" + +/* A one-page PDF whose content is `content` with font F1 = Helvetica. */ +static unsigned char *tp_simple(const char *content, size_t *len) { + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 " + "0 R >> >> /Contents 5 0 R >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", content, strlen(content)}, + }; + return tp_pdf(o, 5, NULL, len); +} + +typedef struct { + CBMArena arena; + cbm_pdf_result_t r; +} tp_text_t; + +static bool tp_extract(const unsigned char *pdf, size_t len, tp_text_t *t) { + cbm_arena_init(&t->arena); + return cbm_pdf_extract(pdf, len, &t->arena, &t->r); +} + +/* The smallest stack a thread of this program runs on (the daemon's). */ +#define TP_SMALL_STACK (256 * 1024) + +typedef struct { + const unsigned char *pdf; + size_t len; + tp_text_t *t; + bool ok; +} tp_job_t; + +static void *tp_extract_job(void *arg) { + tp_job_t *j = (tp_job_t *)arg; + j->ok = tp_extract(j->pdf, j->len, j->t); + return NULL; +} + +/* tp_extract on a thread with the smallest stack: an input that nests calls + * per object crashes here long before it would on a worker's 8 MiB. */ +static bool tp_extract_small_stack(const unsigned char *pdf, size_t len, tp_text_t *t) { + tp_job_t j = {pdf, len, t, false}; + cbm_thread_t th; + if (cbm_thread_create(&th, TP_SMALL_STACK, tp_extract_job, &j) != 0) { + return false; + } + cbm_thread_join(&th); + return j.ok; +} + +/* "N 0 obj\n" for object n at the end of b; records its offset. */ +static void tp_obj_head(tp_buf_t *b, size_t *off, int n) { + char line[64]; + off[n] = b->n; + snprintf(line, sizeof(line), "%d 0 obj\n", n); + tp_puts(b, line); +} + +/* A classic xref table for objects 1..n at off[] and the trailer. */ +static void tp_classic_tail(tp_buf_t *b, const size_t *off, int n) { + char line[128]; + size_t xref = b->n; + snprintf(line, sizeof(line), "xref\n0 %d\n0000000000 65535 f \n", n + 1); + tp_puts(b, line); + for (int i = 1; i <= n; i++) { + snprintf(line, sizeof(line), "%010zu 00000 n \n", off[i]); + tp_puts(b, line); + } + snprintf(line, sizeof(line), "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%zu\n%%%%EOF\n", + n + 1, xref); + tp_puts(b, line); +} + +#define TP_PAGE_F1_5 \ + "<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + +/* A page whose content stream's /Length names a stream whose /Length names + * the next one, `depth` streams deep (the last one's /Length is a number). */ +static unsigned char *tp_length_chain(int depth, size_t *out_len) { + int n = 4 + depth; + size_t *off = (size_t *)calloc((size_t)n + 1, sizeof(size_t)); + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + const char *head[4] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5, + TP_HELV}; + for (int i = 1; i <= 4; i++) { + tp_obj_head(&b, off, i); + tp_puts(&b, head[i - 1]); + tp_puts(&b, "\nendobj\n"); + } + char line[96]; + for (int i = 5; i <= n; i++) { + tp_obj_head(&b, off, i); + if (i < n) { + snprintf(line, sizeof(line), "<< /Length %d 0 R >>\nstream\n", i + 1); + } else { + snprintf(line, sizeof(line), "<< /Length 1 >>\nstream\n"); + } + tp_puts(&b, line); + tp_puts(&b, i == 5 ? "BT /F1 12 Tf 72 700 Td (chain) Tj ET" : "x"); + tp_puts(&b, "\nendstream\nendobj\n"); + } + tp_classic_tail(&b, off, n); + free(off); + *out_len = b.n; + return b.p; +} + +/* The page object lies in object stream C1, whose /Length lies in object + * stream C2, whose /Length lies in C3, ... `depth` streams deep; a + * cross-reference stream places them. Objects: 1-5 as tp_simple (3 in C1), + * C_i = 5 + i, L_i (C_i's length, in C_{i+1}) = 5 + depth + i, the xref stream + * last. */ +static unsigned char *tp_objstm_chain(int depth, size_t *out_len) { + int x = 5 + 2 * depth; /* the xref stream */ + size_t *off = (size_t *)calloc((size_t)x + 1, sizeof(size_t)); + size_t *data_len = (size_t *)calloc((size_t)depth + 1, sizeof(size_t)); + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + const char *plain[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, TP_HELV, + NULL}; + char line[160]; + for (int i = 1; i <= 5; i++) { + if (i == 3) { + continue; + } + tp_obj_head(&b, off, i); + if (i == 5) { + const char *content = "BT /F1 12 Tf 72 700 Td (objstm chain) Tj ET"; + snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content)); + tp_puts(&b, line); + tp_puts(&b, content); + tp_puts(&b, "\nendstream"); + } else { + tp_puts(&b, plain[i - 1]); + } + tp_puts(&b, "\nendobj\n"); + } + for (int i = 1; i <= depth; i++) { + char hdr[48]; + char body[160]; + if (i == 1) { + snprintf(hdr, sizeof(hdr), "3 0 "); + snprintf(body, sizeof(body), "%s", TP_PAGE_F1_5); + } else { + snprintf(hdr, sizeof(hdr), "%d 0 ", 5 + depth + i - 1); + snprintf(body, sizeof(body), "%zu", data_len[i - 1]); + } + data_len[i] = strlen(hdr) + strlen(body); + tp_obj_head(&b, off, 5 + i); + if (i < depth) { + snprintf(line, sizeof(line), + "<< /Type /ObjStm /N 1 /First %zu /Length %d 0 R >>\nstream\n", strlen(hdr), + 5 + depth + i); + } else { + snprintf(line, sizeof(line), + "<< /Type /ObjStm /N 1 /First %zu /Length %zu >>\nstream\n", strlen(hdr), + data_len[i]); + } + tp_puts(&b, line); + tp_puts(&b, hdr); + tp_puts(&b, body); + tp_puts(&b, "\nendstream\nendobj\n"); + } + off[x] = b.n; + /* rows W [1 4 2]: type, offset or containing stream, generation or index */ + size_t nrows = (size_t)x + 1; + unsigned char *rows = (unsigned char *)calloc(nrows, 7); + for (int i = 1; i <= x; i++) { + unsigned char *r = rows + (size_t)i * 7; + bool in_stm = i == 3 || (i > 5 + depth && i < x); + uint32_t f2 = in_stm ? (uint32_t)(i == 3 ? 6 : i - depth + 1) : (uint32_t)off[i]; + r[0] = in_stm ? 2 : 1; + r[1] = (unsigned char)(f2 >> 24); + r[2] = (unsigned char)(f2 >> 16); + r[3] = (unsigned char)(f2 >> 8); + r[4] = (unsigned char)f2; + } + snprintf(line, sizeof(line), + "%d 0 obj\n<< /Type /XRef /Size %d /W [1 4 2] /Root 1 0 R /Length %zu >>\nstream\n", x, + x + 1, nrows * 7); + tp_puts(&b, line); + tp_put(&b, rows, nrows * 7); + tp_puts(&b, "\nendstream\nendobj\n"); + snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", off[x]); + tp_puts(&b, line); + free(rows); + free(data_len); + free(off); + *out_len = b.n; + return b.p; +} + +/* ── extractor ───────────────────────────────────────────────────── */ + +TEST(pdf_extract_layout) { + /* inherited resources, two pages, a line break by Td, a word break by TJ + * kerning, a new text object further down */ + const char *c1 = "BT /F1 12 Tf 72 700 Td (Hello src/auth/handler.go) Tj 0 -14 Td (second line) " + "Tj ET\nBT /F1 12 Tf 72 600 Td [(word)-1000(gap)] TJ ET"; + const char *c2 = "BT /F1 12 Tf 72 700 Td (page two) Tj ET"; + tp_obj_t o[7] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 /Resources << /Font << /F1 4 0 R >> >> " + "/MediaBox [0 0 612 792] >>", + NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Contents 5 0 R >>", NULL, 0}, + {TP_HELV, NULL, 0}, + {"", c1, strlen(c1)}, + {"<< /Type /Page /Parent 2 0 R /Contents 7 0 R >>", NULL, 0}, + {"", c2, strlen(c2)}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 7, NULL, &len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 2); + ASSERT_STR_EQ(t.r.pages[0].text, "Hello src/auth/handler.go\nsecond line\nword gap"); + ASSERT_STR_EQ(t.r.pages[1].text, "page two"); + ASSERT_EQ(t.r.pages[0].unmapped, 0); + ASSERT_TRUE(t.r.pages[0].glyphs > 30); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +TEST(pdf_extract_fonts) { + /* Type0 Identity-H through a ToUnicode CMap (bfchar, a ligature, a + * bfrange, an unmapped code), and a simple font with /Differences */ + static const char cmap[] = "/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + "1 begincodespacerange <0000> endcodespacerange\n" + "2 beginbfchar <0001> <0041> <0002> <00660069> endbfchar\n" + "1 beginbfrange <0003> <0005> <0061> endbfrange\n" + "endcmap CMapName currentdict /CMap defineresource pop end end\n"; + const char *c = "BT /F1 10 Tf 10 700 Td <000100020003000400050006> Tj ET\n" + "BT /F2 10 Tf 10 600 Td (AB) Tj ET"; + tp_obj_t o[8] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 " + "R /F2 8 0 R >> >> /Contents 5 0 R >>", + NULL, 0}, + {"<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEE+Foo /Encoding /Identity-H " + "/DescendantFonts [6 0 R] /ToUnicode 7 0 R >>", + NULL, 0}, + {"", c, strlen(c)}, + {"<< /Type /Font /Subtype /CIDFontType2 /BaseFont /Foo /DW 500 /W [1 [600 600] 3 5 " + "700] >>", + NULL, 0}, + {"", cmap, sizeof(cmap) - 1}, + {"<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding << /BaseEncoding " + "/WinAnsiEncoding /Differences [65 /B /A] >> >>", + NULL, 0}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 8, NULL, &len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "Afiabc\xEF\xBF\xBD\nBA"); + ASSERT_EQ(t.r.pages[0].unmapped, 1); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* A PDF whose page and font live in a Flate object stream, indexed by a + * PNG-predicted xref stream, with no classic table at all. */ +static unsigned char *tp_compressed(size_t *out_len) { + const char *content = "BT /F1 12 Tf 72 700 Td (from an object stream) Tj ET"; + size_t clen; + unsigned char *cz = tp_flate(content, strlen(content), &clen); + /* object stream: objects 3 (page) and 4 (font) */ + const char *o3 = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << " + "/F1 4 0 R >> >> /Contents 5 0 R >>"; + const char *o4 = TP_HELV; + char hdr[64]; + snprintf(hdr, sizeof(hdr), "3 0 4 %zu ", strlen(o3) + 1); + tp_buf_t stm = {0}; + tp_puts(&stm, hdr); + size_t first = stm.n; + tp_puts(&stm, o3); + tp_puts(&stm, " "); + tp_puts(&stm, o4); + size_t sz; + unsigned char *stmz = tp_flate(stm.p, stm.n, &sz); + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + size_t off[8] = {0}; + char line[256]; + off[1] = b.n; + tp_puts(&b, "1 0 obj\n" TP_CATALOG "\nendobj\n"); + off[2] = b.n; + tp_puts(&b, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"); + off[5] = b.n; + snprintf(line, sizeof(line), "5 0 obj\n<< /Length %zu /Filter /FlateDecode >>\nstream\n", clen); + tp_puts(&b, line); + tp_put(&b, cz, clen); + tp_puts(&b, "\nendstream\nendobj\n"); + off[6] = b.n; + snprintf(line, sizeof(line), + "6 0 obj\n<< /Type /ObjStm /N 2 /First %zu /Length %zu /Filter /FlateDecode " + ">>\nstream\n", + first, sz); + tp_puts(&b, line); + tp_put(&b, stmz, sz); + tp_puts(&b, "\nendstream\nendobj\n"); + off[7] = b.n; + /* xref stream rows: W [1 2 1], PNG Up/None rows of 4 bytes + filter byte */ + unsigned char rows[8][5]; + for (int i = 0; i < 8; i++) { + unsigned type = (i == 0) ? 0 : (i == 3 || i == 4) ? 2 : 1; + unsigned f2 = (type == 1) ? (unsigned)off[i] : (type == 2) ? 6U : 0U; + unsigned f3 = (i == 4) ? 1U : 0U; + rows[i][0] = 0; /* PNG filter: None */ + rows[i][1] = (unsigned char)type; + rows[i][2] = (unsigned char)(f2 >> 8); + rows[i][3] = (unsigned char)f2; + rows[i][4] = (unsigned char)f3; + } + size_t xz; + unsigned char *xrz = tp_flate(rows, sizeof(rows), &xz); + snprintf(line, sizeof(line), + "7 0 obj\n<< /Type /XRef /Size 8 /W [1 2 1] /Root 1 0 R /Length %zu /Filter " + "/FlateDecode /DecodeParms << /Predictor 12 /Columns 4 >> >>\nstream\n", + xz); + tp_puts(&b, line); + tp_put(&b, xrz, xz); + tp_puts(&b, "\nendstream\nendobj\n"); + snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", off[7]); + tp_puts(&b, line); + free(cz); + free(stm.p); + free(stmz); + free(xrz); + *out_len = b.n; + return b.p; +} + +TEST(pdf_extract_structure) { + size_t len; + unsigned char *pdf = tp_compressed(&len); + tp_text_t t; + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "from an object stream"); + ASSERT_EQ(t.r.counts.xref_reconstructed, 0); + cbm_arena_destroy(&t.arena); + free(pdf); + + /* a wrong startxref: the xref section is found near it */ + unsigned char *ok = tp_simple("BT /F1 12 Tf 72 700 Td (rebuilt) Tj ET", &len); + char *sx = strstr((char *)ok, "startxref\n"); + ASSERT_NOT_NULL(sx); + sx[strlen("startxref\n")] = '9'; + ASSERT_TRUE(tp_extract(ok, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "rebuilt"); + ASSERT_EQ(t.r.counts.xref_offset_fixed, 1); + ASSERT_EQ(t.r.counts.xref_reconstructed, 0); + cbm_arena_destroy(&t.arena); + /* no xref section at all: rebuilt from the object headers, same text */ + char *xr = strstr((char *)ok, "xref\n0 "); + ASSERT_NOT_NULL(xr); + memcpy(xr, "XXXX", 4); + ASSERT_TRUE(tp_extract(ok, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "rebuilt"); + ASSERT_EQ(t.r.counts.xref_reconstructed, 1); + cbm_arena_destroy(&t.arena); + free(ok); + + /* a Form XObject with its own resources, a self-referencing form, an + * inline image with binary data, content split over two streams */ + const char *page_a = "q 1 0 0 1 100 500 cm /Fm1 Do Q /Fm2 Do\n"; + const char *page_b = "BI /W 2 /H 2 /BPC 8 /CS /G ID \x01\xff(junk)\xfe EI\n" + "BT /F1 12 Tf 72 400 Td (after image) Tj ET"; + const char *form = "BT /F1 12 Tf 0 0 Td (in form) Tj ET"; + const char *self = "/Fm2 Do BT /F1 12 Tf 0 300 Td (self) Tj ET"; + tp_obj_t o[9] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /XObject << /Fm1 6 " + "0 R /Fm2 9 0 R >> /Font << /F1 4 0 R >> >> /Contents [5 0 R 7 0 R] >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", page_a, strlen(page_a)}, + {"/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Resources << /Font << /F1 8 0 R " + ">> >>", + form, strlen(form)}, + {"", page_b, strlen(page_b)}, + {TP_HELV, NULL, 0}, + {"/Type /XObject /Subtype /Form /BBox [0 0 200 200]", self, strlen(self)}, + }; + unsigned char *fp = tp_pdf(o, 9, NULL, &len); + ASSERT_TRUE(tp_extract(fp, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "in form")); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "self")); + ASSERT_NOT_NULL(strstr(t.r.pages[0].text, "after image")); + ASSERT_NULL(strstr(t.r.pages[0].text, "junk")); + cbm_arena_destroy(&t.arena); + free(fp); + PASS(); +} + +static void tp_ahx(tp_buf_t *b, const char *s) { + static const char hex[] = "0123456789abcdef"; + for (; *s; s++) { + char h[3] = {hex[(unsigned char)*s >> 4], hex[(unsigned char)*s & 15], ' '}; + tp_put(b, h, 3); + } + tp_puts(b, ">"); +} + +static void tp_a85(tp_buf_t *b, const unsigned char *s, size_t n) { + tp_puts(b, "<~"); + for (size_t i = 0; i < n; i += 4) { + unsigned char g[4] = {0, 0, 0, 0}; + size_t k = n - i < 4 ? n - i : 4; + memcpy(g, s + i, k); + uint32_t v = ((uint32_t)g[0] << 24) | ((uint32_t)g[1] << 16) | ((uint32_t)g[2] << 8) | g[3]; + char out[5]; + for (int j = 4; j >= 0; j--) { + out[j] = (char)('!' + v % 85); + v /= 85; + } + tp_put(b, out, k + 1); + } + tp_puts(b, "~>"); +} + +TEST(pdf_extract_filters_and_hostile) { + size_t len; + tp_text_t t; + const char *content = "BT /F1 12 Tf 72 700 Td (filtered text) Tj ET"; + /* ASCIIHex, then ASCII85 over Flate */ + tp_buf_t h = {0}; + tp_ahx(&h, content); + size_t zl; + unsigned char *z = tp_flate(content, strlen(content), &zl); + tp_buf_t a = {0}; + tp_a85(&a, z, zl); + const char *dicts[2] = {"/Filter /ASCIIHexDecode", "/Filter [/ASCII85Decode /FlateDecode]"}; + const tp_buf_t *datas[2] = {&h, &a}; + for (int i = 0; i < 2; i++) { + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 " + "R >>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {dicts[i], datas[i]->p, datas[i]->n}, + }; + unsigned char *pdf = tp_pdf(o, 5, NULL, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_STR_EQ(t.r.pages[0].text, "filtered text"); + cbm_arena_destroy(&t.arena); + free(pdf); + } + free(h.p); + free(a.p); + free(z); + + /* not a PDF; encrypted; truncated anywhere; nested a hundred thousand deep */ + ASSERT_TRUE(tp_extract((const unsigned char *)"hello", 5, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_NOT_PDF); + cbm_arena_destroy(&t.arena); + tp_obj_t enc[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Contents 5 0 R >>", NULL, 0}, + {"<< /Filter /Standard /V 1 /R 2 /O <00> /U <00> /P -4 >>", NULL, 0}, + {"", "BT ET", 5}, + }; + unsigned char *pdf = tp_pdf(enc, 5, "/Encrypt 4 0 R", &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_ENCRYPTED); + ASSERT_EQ(t.r.npages, 0); + cbm_arena_destroy(&t.arena); + free(pdf); + pdf = tp_simple("BT /F1 12 Tf 72 700 Td (cut short) Tj ET", &len); + for (size_t cut = 0; cut < len; cut += 7) { + ASSERT_TRUE(tp_extract(pdf, cut, &t)); /* any status; never a crash */ + cbm_arena_destroy(&t.arena); + } + free(pdf); + enum { DEEP = 100000 }; + char *deep = (char *)malloc(DEEP * 2 + 64); + memset(deep, '[', DEEP); + strcpy(deep + DEEP, " BT /F1 12 Tf (x) Tj ET"); + pdf = tp_simple(deep, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.counts.page_errors, 1); /* the content stops at the nesting limit */ + cbm_arena_destroy(&t.arena); + free(pdf); + free(deep); + /* a page tree that contains itself, a /Prev loop */ + tp_obj_t loop[5] = { + {TP_CATALOG, NULL, 0}, + {"<< /Type /Pages /Kids [2 0 R 3 0 R] /Count 1 >>", NULL, 0}, + {"<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R " + ">>", + NULL, 0}, + {TP_HELV, NULL, 0}, + {"", "BT /F1 12 Tf (loops) Tj ET", 26}, + }; + pdf = tp_pdf(loop, 5, "/Prev 9", &len); + char *sx = strstr((char *)pdf, "startxref\n"); + char prev[32]; + snprintf(prev, sizeof(prev), "/Prev %ld", strtol(sx + strlen("startxref\n"), NULL, 10)); + char *pv = strstr((char *)pdf, "/Prev 9"); + ASSERT_NOT_NULL(pv); + /* the same width: point /Prev at this very section (a loop) */ + if (strlen(prev) <= strlen("/Prev 9") + 6) { + tp_buf_t nb = {0}; + tp_put(&nb, pdf, (size_t)(pv - (char *)pdf)); + tp_puts(&nb, prev); + tp_puts(&nb, pv + strlen("/Prev 9")); + free(pdf); + pdf = nb.p; + len = nb.n; + } + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "loops"); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* Objects 1-5 as tp_simple placed by one cross-reference stream (object 6), + * then `sections` classic sections, each naming that stream as its /XRefStm, + * chained by /Prev. */ +static unsigned char *tp_xrefstm_sections(int sections, size_t *out_len) { + size_t off[7] = {0}; + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + const char *content = "BT /F1 12 Tf 72 700 Td (xrefstm) Tj ET"; + const char *body[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5, + TP_HELV, NULL}; + char line[160]; + for (int i = 1; i <= 5; i++) { + tp_obj_head(&b, off, i); + if (i == 5) { + snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content)); + tp_puts(&b, line); + tp_puts(&b, content); + tp_puts(&b, "\nendstream"); + } else { + tp_puts(&b, body[i - 1]); + } + tp_puts(&b, "\nendobj\n"); + } + off[6] = b.n; + unsigned char rows[7][7] = {{0}}; + for (int i = 1; i <= 6; i++) { + uint32_t f2 = (uint32_t)off[i]; + rows[i][0] = 1; + rows[i][1] = (unsigned char)(f2 >> 24); + rows[i][2] = (unsigned char)(f2 >> 16); + rows[i][3] = (unsigned char)(f2 >> 8); + rows[i][4] = (unsigned char)f2; + } + snprintf(line, sizeof(line), + "6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Length %zu >>\nstream\n", sizeof(rows)); + tp_puts(&b, line); + tp_put(&b, rows, sizeof(rows)); + tp_puts(&b, "\nendstream\nendobj\n"); + size_t prev = 0; + for (int s = 0; s < sections; s++) { + size_t at = b.n; + tp_puts(&b, "xref\n0 1\n0000000000 65535 f \ntrailer\n"); + if (s == 0) { + snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu >>\n", off[6]); + } else { + snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu /Prev %zu >>\n", + off[6], prev); + } + tp_puts(&b, line); + prev = at; + } + snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", prev); + tp_puts(&b, line); + *out_len = b.n; + return b.p; +} + +/* A page drawing form 1, form k drawing form k+1 `fan` times, `depth` forms, + * each padded with a 64 KiB comment; the last one shows "x". */ +static unsigned char *tp_form_ladder(int depth, int fan, size_t *out_len) { + int n = 5 + depth; + tp_obj_t *o = (tp_obj_t *)calloc((size_t)n, sizeof(tp_obj_t)); + char **dicts = (char **)calloc((size_t)n, sizeof(char *)); + tp_buf_t *bodies = (tp_buf_t *)calloc((size_t)n, sizeof(tp_buf_t)); + o[0] = (tp_obj_t){TP_CATALOG, NULL, 0}; + o[1] = (tp_obj_t){"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}; + o[2] = (tp_obj_t){"<< /Type /Page /Parent 2 0 R /Resources << /XObject << /F 6 0 R >> >> " + "/Contents 5 0 R >>", + NULL, 0}; + o[3] = (tp_obj_t){TP_HELV, NULL, 0}; + o[4] = (tp_obj_t){"", "q /F Do Q", 9}; + char *pad = (char *)malloc(64 * 1024 + 3); + pad[0] = '%'; + memset(pad + 1, 'x', 64 * 1024); + pad[64 * 1024 + 1] = '\n'; + pad[64 * 1024 + 2] = 0; + for (int k = 0; k < depth; k++) { + int i = 5 + k; /* object i + 1 */ + dicts[i] = (char *)malloc(256); + snprintf(dicts[i], 256, + "/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /XObject << /F " + "%d 0 R >> /Font << /F1 4 0 R >> >>", + i + 2); + tp_puts(&bodies[i], pad); + if (k + 1 < depth) { + for (int f = 0; f < fan; f++) { + tp_puts(&bodies[i], "q /F Do Q\n"); + } + } else { + tp_puts(&bodies[i], "BT /F1 12 Tf 72 700 Td (x) Tj ET"); + } + o[i] = (tp_obj_t){dicts[i], bodies[i].p, bodies[i].n}; + } + unsigned char *pdf = tp_pdf(o, n, NULL, out_len); + for (int i = 0; i < n; i++) { + free(dicts[i]); + free(bodies[i].p); + } + free(pad); + free(bodies); + free(dicts); + free(o); + return pdf; +} + +/* Forms drawing the next one four times, seven levels: the first reading of + * each form is whole, readings again stop at PDF_MAX_FORM_REREAD (about 1.3 + * GB of form content read again before, and draws ^ depth in general). */ +TEST(pdf_extract_form_ladder_bounded) { + size_t len; + tp_text_t t; + unsigned char *pdf = tp_form_ladder(7, 4, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_TRUE(t.r.counts.form_rereads_cut > 0); + ASSERT_TRUE(t.r.pages[0].glyphs >= 1); + ASSERT_TRUE((size_t)t.r.pages[0].glyphs <= (256U << 20) / (64U * 1024U) + 1); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* tp_simple's page, then `n` objects that each open a string and never close + * it, and no cross-reference: the reader rebuilds it from the object headers. */ +static unsigned char *tp_open_strings(int n, size_t *out_len) { + tp_buf_t b = {0}; + tp_puts(&b, "%PDF-1.7\n"); + const char *content = "BT /F1 12 Tf 72 700 Td (open strings) Tj ET"; + const char *body[4] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5, + TP_HELV}; + char line[128]; + for (int i = 1; i <= 4; i++) { + snprintf(line, sizeof(line), "%d 0 obj\n", i); + tp_puts(&b, line); + tp_puts(&b, body[i - 1]); + tp_puts(&b, "\nendobj\n"); + } + snprintf(line, sizeof(line), "5 0 obj\n<< /Length %zu >>\nstream\n", strlen(content)); + tp_puts(&b, line); + tp_puts(&b, content); + tp_puts(&b, "\nendstream\nendobj\n"); + for (int i = 0; i < n; i++) { + snprintf(line, sizeof(line), "%d 0 obj\n(", 6 + i); + tp_puts(&b, line); + for (int k = 0; k < 100; k++) { + tp_puts(&b, "a"); + } + tp_puts(&b, "\n"); + } + *out_len = b.n; + return b.p; +} + +/* Objects whose strings never end cost what the file costs: each object's + * value is read up to the next object, not to the end of the file again (a + * thousand objects asked for about 60 MB). */ +TEST(pdf_extract_open_strings_linear) { + size_t len; + tp_text_t t; + unsigned char *pdf = tp_open_strings(1000, &len); + cbm_mem_class_reset_peaks(); + size_t arena0 = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_ARENA); + size_t extract0 = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + size_t grew = (cbm_mem_class_peak_bytes(CBM_MEM_CLASS_ARENA) - arena0) + + (cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT) - extract0); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.counts.xref_reconstructed, 1); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "open strings"); + if (grew > 16 * len + (1U << 20)) { + fprintf(stderr, "open strings: %zu bytes for a %zu-byte file\n", grew, len); + } + ASSERT_TRUE(grew <= 16 * len + (1U << 20)); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* A thousand sections naming one /XRefStm decode it once: a later (older) + * section adds nothing, and every copy used to stay until the document closed. */ +TEST(pdf_extract_xrefstm_once) { + size_t len; + tp_text_t t; + unsigned char *pdf = tp_xrefstm_sections(1000, &len); + ASSERT_TRUE(tp_extract(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "xrefstm"); + ASSERT_EQ(t.r.counts.xref_streams, 1); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* A chain of objects, each needed to load the one before, costs no stack per + * object: a stream's /Length is read as a value (a stream there is no length), + * and the objects that open an object stream are no objects of one. Run on the + * smallest stack in the program. */ +TEST(pdf_extract_object_chains) { + size_t len; + tp_text_t t; + unsigned char *pdf = tp_length_chain(20000, &len); + ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "chain"); + ASSERT_TRUE(t.r.counts.stream_length_recovered >= 1); + cbm_arena_destroy(&t.arena); + free(pdf); + pdf = tp_objstm_chain(3000, &len); + ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t)); + ASSERT_EQ(t.r.status, CBM_PDF_OK); + ASSERT_EQ(t.r.npages, 1); + ASSERT_STR_EQ(t.r.pages[0].text, "objstm chain"); + ASSERT_TRUE(t.r.counts.objstm_nested >= 1); + cbm_arena_destroy(&t.arena); + free(pdf); + PASS(); +} + +/* ── scanner (expected: the field-test scanner on the same text) ─── */ + +TEST(pdf_scan_mentions) { + const char *page = "See src/auth/handler.go and ./pkg/x.py, http://example.com/a/b.py and " + "and/or 2024/01/02.\n" + "Call pkg.Config.Name(), Handler::run and obj->field, e.g. x.y and " + "example.com.\n" + "The file config.py and setup.cfg; f(a, opts.debug=True).\n" + "long/path/to/mod-\n" + "ule.py and Handler::\n" + "serve here\n" + "\xEF\xAC\x81le_utils.py ligature\n"; + static const struct { + uint32_t line; + int syntax; + uint16_t flags; + const char *raw; + } want[] = { + {1, CBM_DOCLINK_PDF_PATH, 0, "src/auth/handler.go"}, + {1, CBM_DOCLINK_PDF_PATH, 0, "pkg/x.py"}, + {2, CBM_DOCLINK_PDF_QN, 0, "pkg.Config.Name"}, + {2, CBM_DOCLINK_PDF_QN, 0, "Handler.run"}, + {2, CBM_DOCLINK_PDF_QN, 0, "obj.field"}, + {3, CBM_DOCLINK_PDF_FILE, 0, "config.py"}, + {3, CBM_DOCLINK_PDF_FILE, 0, "setup.cfg"}, + {4, CBM_DOCLINK_PDF_PATH, 0, "long/path/to/mod"}, + {5, CBM_DOCLINK_PDF_FILE, 0, "ule.py"}, + {7, CBM_DOCLINK_PDF_FILE, 0, "file_utils.py"}, + {4, CBM_DOCLINK_PDF_PATH, CBM_DOCLINK_FLAG_JOIN, + "long/path/to/module.py\x1flong/path/to/mod-ule.py\x1e" + "7,8"}, + {5, CBM_DOCLINK_PDF_QN, CBM_DOCLINK_FLAG_JOIN, "Handler.serve\x1e"}, + }; + CBMFileResult *r = (CBMFileResult *)calloc(1, sizeof(*r)); + cbm_arena_init(&r->arena); + CBMExtractCtx ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.arena = &r->arena; + ctx.result = r; + cbm_pdf_test_scan_page(&ctx, page, strlen(page), 1, "p.doc.page_1"); + ASSERT_FALSE(r->doc_links.failed); + ASSERT_EQ(r->doc_links.count, (int)(sizeof(want) / sizeof(want[0]))); + for (int i = 0; i < r->doc_links.count; i++) { + const CBMDocLink *l = &r->doc_links.items[i]; + ASSERT_EQ(l->line, want[i].line); + ASSERT_EQ(l->syntax, want[i].syntax); + ASSERT_EQ(l->flags, want[i].flags); + ASSERT_STR_EQ(l->raw, want[i].raw); + ASSERT_EQ(l->def_line, 1); + ASSERT_STR_EQ(l->source_qn, "p.doc.page_1"); + } + cbm_free_result(r); + PASS(); +} + +/* Join work on a page of `n` lines that all join, eight names a line. */ +static size_t tp_join_work(int n) { + tp_buf_t b = {0}; + for (int i = 0; i < n; i++) { + tp_puts(&b, "pkg.mod.Alpha pkg.mod.Beta dir/sub/f1.py dir/sub/f2.py pkg.mod.Gamma " + "pkg.mod.Delta dir/sub/f3.py dir/sub/f4.py end_\n"); + } + CBMFileResult *r = (CBMFileResult *)calloc(1, sizeof(*r)); + cbm_arena_init(&r->arena); + CBMExtractCtx ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.arena = &r->arena; + ctx.result = r; + size_t before = cbm_pdf_test_join_steps(); + cbm_pdf_test_scan_page(&ctx, (const char *)b.p, b.n, 1, "p.doc.page_1"); + size_t work = cbm_pdf_test_join_steps() - before; + bool ok = !r->doc_links.failed && r->doc_links.count > n * 8; + cbm_free_result(r); + free(b.p); + return ok ? work : 0; +} + +/* A join compares only the candidates of its two lines: twice the page, about + * twice the work (every candidate of the page per join was four times). */ +TEST(pdf_scan_join_linear) { + size_t small = tp_join_work(300); + size_t large = tp_join_work(600); + ASSERT_TRUE(small > 0); + if (large > small * 5 / 2) { + fprintf(stderr, "join work: %zu for 300 lines, %zu for 600\n", small, large); + } + ASSERT_TRUE(large <= small * 5 / 2); + PASS(); +} + +/* ── the pipeline ────────────────────────────────────────────────── */ + +/* A PDF whose pages each show the given lines (Helvetica, one Tj a line). */ +static void tp_write_pdf(const char *path, const char *const *pages, int npages) { + int n = 3 + 2 * npages; + tp_obj_t *o = (tp_obj_t *)calloc((size_t)n, sizeof(*o)); + char **bufs = (char **)calloc((size_t)npages * 2 + 1, sizeof(char *)); + o[0].dict = TP_CATALOG; + tp_buf_t kids = {0}; + tp_puts(&kids, "<< /Type /Pages /Count 1 /Kids ["); + for (int i = 0; i < npages; i++) { + char k[32]; + snprintf(k, sizeof(k), "%d 0 R ", 4 + 2 * i); + tp_puts(&kids, k); + } + tp_puts(&kids, "] >>"); + o[1].dict = (const char *)kids.p; + o[2].dict = TP_HELV; + for (int i = 0; i < npages; i++) { + char *pg = (char *)malloc(256); + snprintf(pg, 256, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 " + "3 0 R >> >> /Contents %d 0 R >>", + 5 + 2 * i); + bufs[2 * i] = pg; + o[3 + 2 * i].dict = pg; + tp_buf_t c = {0}; + tp_puts(&c, "BT /F1 10 Tf 72 720 Td 12 TL\n"); + for (const char *s = pages[i]; *s;) { + const char *e = strchr(s, '\n'); + size_t k = e ? (size_t)(e - s) : strlen(s); + tp_puts(&c, "("); + tp_put(&c, s, k); + tp_puts(&c, ") Tj T*\n"); + s += k + (e ? 1 : 0); + } + tp_puts(&c, "ET"); + bufs[2 * i + 1] = (char *)c.p; + o[4 + 2 * i].dict = ""; + o[4 + 2 * i].data = c.p; + o[4 + 2 * i].len = c.n; + } + size_t len; + unsigned char *pdf = tp_pdf(o, n, NULL, &len); + char dir[512]; + snprintf(dir, sizeof(dir), "%s", path); + char *slash = strrchr(dir, '/'); + if (slash) { + *slash = '\0'; + cbm_mkdir_p(dir, 0755); + } + FILE *f = fopen(path, "wb"); + fwrite(pdf, 1, len, f); + fclose(f); + free(pdf); + free(kids.p); + for (int i = 0; i < npages * 2; i++) { + free(bufs[i]); + } + free(bufs); + free(o); +} + +static const char *const DESIGN_P1 = + "Design: pkg/server/handler.go serves via server.Handler.Serve\n" + "and app.models.User.save stores users; see\n" + "tests/test_api.py and config/settings.json.\n" + "Gone: pkg/server/gone.go and other/thing.go."; +static const char *const DESIGN_P2 = + "Factory server.NewHandler and models.py build it; app.models holds it."; + +static void tp_write_repo(const char *repo, const char *p2) { + th_write_file(TH_PATH(repo, "pkg/server/handler.go"), + "package server\n" + "\n" + "type Handler struct{}\n" + "\n" + "func (h *Handler) Serve() {}\n" + "\n" + "func NewHandler() *Handler { return nil }\n"); + th_write_file(TH_PATH(repo, "app/__init__.py"), ""); + th_write_file(TH_PATH(repo, "app/models.py"), "class User:\n" + " def save(self):\n" + " return None\n"); + th_write_file(TH_PATH(repo, "tests/test_api.py"), "def test_x():\n" + " return None\n"); + th_write_file(TH_PATH(repo, "config/settings.json"), "{\"a\": 1}\n"); + char path[1024]; + snprintf(path, sizeof(path), "%s/docs/design.pdf", repo); + const char *pages[2] = {DESIGN_P1, p2}; + tp_write_pdf(path, pages, 2); + snprintf(path, sizeof(path), "%s/tests/fixtures/sample.pdf", repo); + const char *fx[1] = {"Fixture names pkg/server/handler.go too."}; + tp_write_pdf(path, fx, 1); +} + +TEST(pdf_links_pipeline) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdf_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + tp_write_repo(repo, DESIGN_P2); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* one Section per page, the page's text as its docstring */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND " + "file_path='docs/design.pdf'"), + 2); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND name='page 1' " + "AND file_path='docs/design.pdf' AND properties LIKE " + "'%pkg/server/handler.go serves%'"), + 1); + char props[512]; + int n; + /* path-exact */ + dm_edge(db, "docs.design.page_1", "pkg.server.handler.go.__file__", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"via\":\"pdf\"")); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_path\"")); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":1")); + /* qualified names (pdf_qn) passed a second held-out audit (97 of 99 + * correct, Wilson 95 % low 0.929) once a member whose QN leaves its owner + * out binds only through the owner. A Go method through its receiver: */ + dm_edge(db, "docs.design.page_1", "pkg.server.Serve", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\"")); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"unique\"")); + /* qn-exact */ + dm_edge(db, "docs.design.page_1", "app.models.User.save", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\"")); + ASSERT_NOT_NULL(strstr(props, "\"line\":2")); + /* page 2: a Go function, a file name */ + dm_edge(db, "docs.design.page_2", "pkg.server.NewHandler", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + dm_edge(db, "docs.design.page_2", "app.models.py.__file__", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_file\"")); + /* a module by its qualified name (a Module node is named by its path) */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.label='Module' AND t.file_path='app/models.py' AND s.name='page 2' " + "AND e.properties LIKE '%\"tier\":\"exact\"%'"), + 1); + /* back navigation: the method knows the page that names it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND t.name='Serve' " + "AND s.name='page 1'"), + 1); + /* hygiene: a test target is a row, a data file nothing; a fixture PDF links nothing */ + char reason[64]; + dm_row(db, "docs/design.pdf", "tests/test_api.py", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "test_only_target"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path='config/settings.json'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "raw='config/settings.json'"), + 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id WHERE " + "e.type='MENTIONS' AND s.file_path='tests/fixtures/sample.pdf'"), + 0); + /* unresolved: a missing file under a known directory; another repo's path is no row */ + dm_row(db, "docs/design.pdf", "pkg/server/gone.go", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw='other/thing.go'"), + 0); + /* back navigation: the file knows the page that names it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.label='File' AND t.file_path='pkg/server/handler.go' AND " + "s.name='page 1'"), + 1); + th_cleanup(tmp); + PASS(); +} + +/* A held-out finding of the PDF audit: a Go method's QN has no receiver, so + * `time.Now` (the standard library's function, in a spec) matched the method + * Now of a type that the repository's own package time declares. A method is + * named through its owner: `time.Provider.Now` still binds it. */ +TEST(pdf_qn_method_through_owner) { + cbm_doclink_test_set_ships(CBM_DOCLINK_PDF_QN, true); + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdfq_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "pkg/time/provider.go"), + "package time\n" + "\n" + "type Provider struct{}\n" + "\n" + "func (p *Provider) Now() int { return 0 }\n"); + char path[1024]; + snprintf(path, sizeof(path), "%s/docs/spec.pdf", repo); + const char *pages[2] = {"Go's clock: time.Now is the standard library's.", + "Our clock: time.Provider.Now wraps it."}; + tp_write_pdf(path, pages, 2); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + char props[512]; + int n; + dm_edge(db, "docs.spec.page_1", "pkg.time.Now", props, sizeof(props), &n); + ASSERT_EQ(n, 0); + dm_edge(db, "docs.spec.page_2", "pkg.time.Now", props, sizeof(props), &n); + ASSERT_EQ(n, 1); + ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\"")); + th_cleanup(tmp); + cbm_doclink_test_reset_ships(); + PASS(); +} + +/* A page's docstring keeps 256 KiB of its text and says where it was cut; a + * page's mentions still come from the whole text. */ +TEST(pdf_page_doc_bounded) { + enum { WORDS = 45000 }; /* 315 KB of text */ + tp_buf_t c = {0}; + tp_puts(&c, "BT /F1 12 Tf 72 700 Td ("); + for (int i = 0; i < WORDS; i++) { + tp_puts(&c, "filler "); + } + tp_puts(&c, "src/tail/last.py) Tj ET"); + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {TP_PAGE_F1_5, NULL, 0}, {TP_HELV, NULL, 0}, + {"", c.p, c.n}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 5, NULL, &len); + CBMFileResult *r = + cbm_extract_file((const char *)pdf, (int)len, CBM_LANG_PDF, "p", "doc.pdf", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + const CBMDefinition *page = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Section") == 0) { + page = &r->defs.items[i]; + } + } + ASSERT_NOT_NULL(page); + ASSERT_NOT_NULL(page->docstring); + size_t dl = strlen(page->docstring); + ASSERT_TRUE(dl > 256 * 1024 && dl < 256 * 1024 + 128); + ASSERT_NOT_NULL(strstr(page->docstring, "[page text cut at 262144 of ")); + bool tail = false; + for (int i = 0; i < r->doc_links.count; i++) { + tail = tail || strcmp(r->doc_links.items[i].raw, "src/tail/last.py") == 0; + } + ASSERT_TRUE(tail); + cbm_free_result(r); + free(pdf); + free(c.p); + PASS(); +} + +TEST(pdf_snippet_is_page_text) { + char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[400]; + char cache[400]; + char db[1024]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(cache, sizeof(cache), "%s/cache", tmp); + tp_write_repo(repo, DESIGN_P2); + cbm_mkdir_p(cache, 0700); + char *project = cbm_project_name_from_path(repo); + ASSERT_NOT_NULL(project); + const char *saved = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved ? strdup(saved) : NULL; + snprintf(db, sizeof(db), "%s/%s.db", cache, project); + cbm_setenv("CBM_CACHE_DIR", cache, 1); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + char args[1024]; + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"qualified_name\":\"%s.docs.design.page_2\"}", project, project); + char *resp = cbm_mcp_handle_tool(srv, "get_code_snippet", args); + char *text = dm_tool_text(resp); + ASSERT_NOT_NULL(text); + ASSERT_NOT_NULL( + strstr(text, "Factory server.NewHandler and models.py build it; app.models holds it.")); + ASSERT_NULL(strstr(text, "%PDF")); + free(text); + free(resp); + cbm_mcp_server_free(srv); + if (saved_copy) { + cbm_setenv("CBM_CACHE_DIR", saved_copy, 1); + } else { + cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_copy); + free(project); + th_cleanup(tmp); + PASS(); +} + +TEST(pdf_links_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdf_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + tp_write_repo(repo, DESIGN_P2); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a body edit of a linked code file: the document re-resolves */ + th_write_file(TH_PATH(repo, "app/models.py"), "class User:\n" + " def save(self):\n" + " return 1\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "code body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + /* the document changes while the Go file does not: its method binds + * through the owner a proxy carries */ + char path[1024]; + snprintf(path, sizeof(path), "%s/docs/design.pdf", repo); + const char *pages[2] = {DESIGN_P1, "Now only server.Handler.Serve and pkg/server/new.go."}; + tp_write_pdf(path, pages, 2); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + /* the missing file appears: the row becomes an edge, as in a full index */ + th_write_file(TH_PATH(repo, "pkg/server/gone.go"), "package server\n" + "\n" + "func Gone() {}\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "missing file added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), + 0); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_pdf) { + RUN_TEST(pdf_extract_layout); + RUN_TEST(pdf_extract_fonts); + RUN_TEST(pdf_extract_structure); + RUN_TEST(pdf_extract_filters_and_hostile); + RUN_TEST(pdf_extract_object_chains); + RUN_TEST(pdf_extract_xrefstm_once); + RUN_TEST(pdf_extract_open_strings_linear); + RUN_TEST(pdf_extract_form_ladder_bounded); + RUN_TEST(pdf_scan_mentions); + RUN_TEST(pdf_scan_join_linear); + RUN_TEST(pdf_links_pipeline); + RUN_TEST(pdf_qn_method_through_owner); + RUN_TEST(pdf_page_doc_bounded); + RUN_TEST(pdf_snippet_is_page_text); + RUN_TEST(pdf_links_incremental); +} diff --git a/tests/test_doc_links_rst.c b/tests/test_doc_links_rst.c new file mode 100644 index 0000000000..714f9a5dcd --- /dev/null +++ b/tests/test_doc_links_rst.c @@ -0,0 +1,533 @@ +/* + * test_doc_links_rst.c — reStructuredText documents -> Sections and MENTIONS + * edges (doclink_rst.c, doclink_py.c, doc_links_rst.c). + * + * The line model (titles, literal blocks, directives, comments), the tokens + * and their resolution contexts, the Python scope blob (package re-exports, + * Sphinx conf.py), the resolver's rules as the field test measured them + * (forms and re-exports, a module never answers a member's role, bare members + * are never searched, a name written in full binds even test code while a + * search leaves it out, intersphinx and other projects are no rows), the C + * domain, include / literalinclude / kernel-doc, extlinks, inline literals, + * Sphinx's source_suffix in discovery, reST ADRs, incremental == full. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include + +/* ── the scanner ─────────────────────────────────────────────────── */ + +static const char *const RST_DOC = + "Intro names :class:`pkg.Top` before any title.\n" /* 1 */ + "\n" /* 2 */ + "=========\n" /* 3 */ + " The API\n" /* 4 */ + "=========\n" /* 5 */ + "\n" /* 6 */ + ".. module:: pkg\n" /* 7 */ + "\n" /* 8 */ + "Use :class:`Model` and :meth:`~pkg.Model.save`::\n" /* 9 */ + "\n" /* 10 */ + " :class:`NotARole` in a literal block\n" /* 11 */ + " Fake Title\n" /* 12 */ + " ==========\n" /* 13 */ + "\n" /* 14 */ + ".. code-block:: python\n" /* 15 */ + "\n" /* 16 */ + " x = :func:`nope`\n" /* 17 */ + "\n" /* 18 */ + ".. class:: Model(name)\n" /* 19 */ + "\n" /* 20 */ + " .. method:: save(force=False)\n" /* 21 */ + "\n" /* 22 */ + " Saves; see :attr:`name`.\n" /* 23 */ + "\n" /* 24 */ + "Usage\n" /* 25 */ + "-----\n" /* 26 */ + "\n" /* 27 */ + ".. autofunction:: pkg.base.top\n" /* 28 */ + "\n" /* 29 */ + ".. literalinclude:: ../pkg/base.py\n" /* 30 */ + " :lines: 2-3\n" /* 31 */ + "\n" /* 32 */ + ".. include:: ../pkg/snippet.py\n" /* 33 */ + "\n" /* 34 */ + ".. include:: other.rst\n" /* 35 */ + "\n" /* 36 */ + ".. kernel-doc:: drivers/x.c\n" /* 37 */ + " :identifiers: foo bar\n" /* 38 */ + "\n" /* 39 */ + ".. a comment with :class:`Hidden`\n" /* 40 */ + "\n" /* 41 */ + "Text ``see :class:`Hidden2` there`` and ``pkg/base.py``,\n" /* 42 */ + "``pkg.base.top`` and :source:`a file `.\n"; /* 43 */ + +static int rst_count(const CBMFileResult *r, int syntax) { + int n = 0; + for (int i = 0; i < r->doc_links.count; i++) { + n += r->doc_links.items[i].syntax == syntax; + } + return n; +} + +static bool rst_raw_has(const CBMFileResult *r, const char *needle) { + for (int i = 0; i < r->doc_links.count; i++) { + if (strstr(r->doc_links.items[i].raw, needle)) { + return true; + } + } + return false; +} + +static const CBMDefinition *rst_def(const CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && strcmp(d->label, label) == 0 && d->name && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +TEST(rst_scan_structure) { + CBMFileResult *r = dm_extract(RST_DOC, CBM_LANG_RST, "docs/api.rst"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->doc_links.failed); + /* titles: over+under and under only; a title shape in a literal block is none */ + const CBMDefinition *api = rst_def(r, "Section", "The API"); + const CBMDefinition *usage = rst_def(r, "Section", "Usage"); + ASSERT_NOT_NULL(api); + ASSERT_NOT_NULL(usage); + ASSERT_NULL(rst_def(r, "Section", "Fake Title")); + ASSERT_EQ(api->start_line, 4); + ASSERT_EQ(api->end_line, 23); + ASSERT_EQ(usage->start_line, 25); + ASSERT_STR_EQ(api->qualified_name, "p.docs.api.The-API"); + ASSERT_NOT_NULL(api->docstring); + ASSERT_NOT_NULL(strstr(api->docstring, "Use :class:`Model`")); + /* nothing from a literal block, a code directive, a comment or an inline literal */ + ASSERT_FALSE(rst_raw_has(r, "NotARole")); + ASSERT_FALSE(rst_raw_has(r, "nope")); + ASSERT_FALSE(rst_raw_has(r, "Hidden")); + /* text before the first title belongs to the file */ + const CBMDocLink *top = dm_find_token(r, ":class:`pkg.Top`\tclass\tpkg.Top\t\t\t"); + ASSERT_NOT_NULL(top); + ASSERT_EQ(top->flags, CBM_DOCLINK_FLAG_FILE); + ASSERT_EQ(top->line, 1); + /* roles carry the module and class in force */ + const CBMDocLink *model = dm_find_token(r, ":class:`Model`\tclass\tModel\tpkg\t\t"); + ASSERT_NOT_NULL(model); + ASSERT_STR_EQ(model->source_qn, "p.docs.api.The-API"); + ASSERT_EQ(model->line, 9); + ASSERT_NOT_NULL(dm_find_token(r, ":attr:`name`\tattr\tname\tpkg\tModel\t")); + /* object directives: the signature and Python's full name in its class */ + ASSERT_NOT_NULL(dm_find_token(r, ".. class:: Model(name)\tclass\tModel(name)\tpkg\t\t\tModel")); + ASSERT_NOT_NULL(dm_find_token( + r, ".. method:: save(force=False)\tmethod\tsave(force=False)\tpkg\tModel\t\tModel.save")); + ASSERT_NOT_NULL(dm_find_token(r, ".. module:: pkg\tmodule\tpkg\tpkg\t\t\t")); + /* autodoc, literalinclude, an include of code (not of a document), kernel-doc names */ + ASSERT_NOT_NULL( + dm_find_token(r, ".. autofunction:: pkg.base.top\tautofunction\tpkg.base.top\tpkg\t")); + ASSERT_NOT_NULL( + dm_find_token(r, ".. literalinclude:: ../pkg/base.py\t../pkg/base.py\t2-3\t\t")); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_INCLUDE), 1); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_KERNEL_DOC), 3); + ASSERT_NOT_NULL(dm_find_token(r, ".. kernel-doc:: drivers/x.c\tdrivers/x.c\tbar")); + /* inline literals naming a path or a qualified name; a titled extlink */ + ASSERT_NOT_NULL(dm_find_token(r, "pkg/base.py")); + ASSERT_EQ(dm_find_token(r, "pkg/base.py")->syntax, CBM_DOCLINK_RST_CODE_PATH); + ASSERT_EQ(dm_find_token(r, "pkg.base.top")->syntax, CBM_DOCLINK_RST_CODE_NAME); + ASSERT_NOT_NULL( + dm_find_token(r, ":source:`a file `\tsource\ta file \tpkg\t\t")); + cbm_free_result(r); + PASS(); +} + +/* ── Python's scope ──────────────────────────────────────────────── */ + +TEST(rst_python_scope_blob) { + CBMFileResult *r = dm_extract("from .base import Model\n" + "from .sub.thing import helper as aid, other\n" + "from ..up import x\n" + "from os import *\n" + "from django.db import models\n" + "try:\n" + " from .opt import Nested\n" + "except ImportError:\n" + " pass\n", + CBM_LANG_PYTHON, "pkg/__init__.py"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "I\t1\tbase\tModel\tModel\n" + "I\t1\tsub.thing\thelper\taid\n" + "I\t1\tsub.thing\tother\tother\n" + "I\t2\tup\tx\tx\n" + "I\t0\tdjango.db\tmodels\tmodels\n"); + ASSERT_EQ(r->doc_links.count, 0); + cbm_free_result(r); + r = dm_extract("project = 'x'\n" + "primary_domain = 'c'\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + " \"sphinx\": (\"https://www.sphinx-doc.org/en/master/\", None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + " 'issue': ('https://github.com/o/r/issues/%s', '#%s'),\n" + "}\n" + "extlinks['tree'] = ('https://github.com/o/r/tree/main/%s', '%s')\n", + CBM_LANG_PYTHON, "docs/conf.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "C\n" + "D\tc\n" + "S\tpython\n" + "S\tsphinx\n" + "X\tsource\n" + "X\ttree\n"); + cbm_free_result(r); + r = dm_extract("from .x import y\n", CBM_LANG_PYTHON, "pkg/mod.py"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(r->doc_scope); + cbm_free_result(r); + PASS(); +} + +/* ── the pipeline ────────────────────────────────────────────────── */ + +static const char *const API_RST = + "Models\n" /* 1 */ + "======\n" /* 2 */ + "\n" /* 3 */ + ".. module:: pkg\n" /* 4 */ + "\n" /* 5 */ + "Use :class:`Model`, :func:`pkg.aid` and :meth:`save`.\n" /* 6 */ + "Not :class:`python:dict`, :class:`otherlib.Thing`; gone\n" /* 7 */ + ":class:`pkg.Missing`.\n" /* 8 */ + "\n" /* 9 */ + ".. currentmodule:: pkg.views\n" /* 10 */ + "\n" /* 11 */ + ".. class:: View\n" /* 12 */ + "\n" /* 13 */ + " Calls :meth:`dispatch`.\n" /* 14 */ + "\n" /* 15 */ + "Testing\n" /* 16 */ + "=======\n" /* 17 */ + "\n" /* 18 */ + "Run :class:`pkg.test.runner.Runner`, not :class:`Runner`.\n" /* 19 */ + "\n" /* 20 */ + "Code\n" /* 21 */ + "====\n" /* 22 */ + "\n" /* 23 */ + "See :source:`pkg/base.py`, ``pkg.base.top`` and :c:func:`c_func`.\n" /* 24 */ + "\n" /* 25 */ + ".. autofunction:: pkg.base.top\n" /* 26 */ + "\n" /* 27 */ + ".. literalinclude:: ../pkg/base.py\n" /* 28 */ + " :lines: 2-3\n" /* 29 */ + "\n" /* 30 */ + ".. include:: /../pkg/sub/thing.py\n" /* 31 */ + "\n" /* 32 */ + ".. kernel-doc:: csrc/lib.c\n" /* 33 */ + " :identifiers: c_func\n"; /* 34 */ + +static void rst_write_repo(const char *repo, const char *init) { + th_write_file(TH_PATH(repo, "pkg/__init__.py"), init); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " def save(self):\n" + " return None\n" + "\n" + "\n" + "def top():\n" + " return 1\n"); + th_write_file(TH_PATH(repo, "pkg/other.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "pkg/sub/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/sub/thing.py"), "def helper():\n" + " return 2\n"); + th_write_file(TH_PATH(repo, "pkg/views.py"), "class View:\n" + " def dispatch(self):\n" + " return None\n"); + th_write_file(TH_PATH(repo, "pkg/test/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/test/runner.py"), "class Runner:\n" + " pass\n"); + /* tests/ is no package: tests/dispatch's import name is `dispatch` */ + th_write_file(TH_PATH(repo, "tests/dispatch/__init__.py"), ""); + th_write_file(TH_PATH(repo, "tests/dispatch/test_x.py"), "def test_x():\n" + " return None\n"); + th_write_file(TH_PATH(repo, "csrc/lib.c"), "int c_func(void) {\n" + " return 0;\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/conf.py"), + "project = 'x'\n" + "source_suffix = {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/api.rst"), API_RST); + th_write_file(TH_PATH(repo, "docs/topics/guide.txt"), "Guide\n" + "=====\n" + "\n" + "The :class:`pkg.views.View`.\n"); + th_write_file(TH_PATH(repo, "notes/readme.txt"), "Title\n" + "=====\n" + "\n" + "Plain :class:`pkg.views.View`.\n"); +} + +static const char *const INIT_PY = "from .base import Model\n" + "from .sub.thing import helper as aid\n"; + +static bool rst_edge(const char *db, const char *src, const char *tgt, const char *needle) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + return n == 1 && (!needle || strstr(props, needle)); +} + +static int rst_rows(const char *db, const char *raw_prefix, const char *reason) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE '%s%%' AND reason='%s'", + raw_prefix, reason); + return dm_count(db, sql); +} + +TEST(rst_links_pipeline) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a re-export through the module context: pkg.Model is pkg/base.py's */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", + "{\"via\":\"rst\",\"syntax\":\"role\",\"tier\":\"exact\",\"line\":6")); + /* an aliased re-export: pkg.aid is helper */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.sub.thing.helper", "\"tier\":\"exact\"")); + /* a bare member is never searched; intersphinx and another project: no rows */ + ASSERT_EQ(dm_count(db, + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE ':meth:`save`%' " + "OR raw LIKE ':class:`python:dict`%' OR raw LIKE " + "':class:`otherlib.Thing`%'"), + 0); + ASSERT_EQ(rst_rows(db, ":class:`pkg.Missing`", "missing"), 1); + /* a module never answers :meth: -- the class context does */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View.dispatch", "\"tier\":\"exact\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path LIKE 'tests/%'"), + 0); + /* the object directive declares View in its module */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\"")); + /* a name written in full binds test code; a search for it does not */ + ASSERT_TRUE(rst_edge(db, "docs.api.Testing", "pkg.test.runner.Runner", "\"tier\":\"exact\"")); + ASSERT_EQ(rst_rows(db, ":class:`Runner`", "test_only_target"), 1); + /* extlink, inline literal, C domain, autodoc */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.py.__file__", "\"syntax\":\"role\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.top", "\"syntax\":\"code_name\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c_func", NULL)); + /* literalinclude :lines: the innermost definition, the lines on the edge */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* include from the documentation set's directory; kernel-doc file */ + ASSERT_TRUE( + rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", "\"syntax\":\"include\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c.__file__", "\"syntax\":\"kernel_doc\"")); + /* conf.py's source_suffix: .txt is reST in its documentation set only */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND " + "file_path='docs/topics/guide.txt' AND name='Guide'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.topics.guide.Guide", "pkg.views.View", "\"via\":\"rst\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE file_path='notes/readme.txt'"), 0); + /* back navigation: the class knows the sections that name it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.name='View' AND t.label='Class' AND s.label='Section'"), + 2); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_adr_record) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_adr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "docs/decisions/0002-use-models.rst"), "2. Use models\n" + "=============\n" + "\n" + "Status\n" + "------\n" + "\n" + "Accepted\n" + "\n" + "Context\n" + "-------\n" + "\n" + "We store records.\n" + "\n" + "Decision\n" + "--------\n" + "\n" + "Use ``pkg.base.Model``.\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='ADR' AND name='ADR-2' AND " + "json_extract(properties,'$.status')='accepted'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.decisions.0002-use-models.Decision", "pkg.base.Model", + "\"syntax\":\"code_name\"")); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_links_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* a body edit moves the lines a literalinclude names: the document + * re-resolves with the code file (its names did not change) */ + th_write_file(TH_PATH(repo, "pkg/base.py"), "# moved\n" + "# down\n" + "class Model:\n" + " def save(self):\n" + " return None\n" + "\n" + "\n" + "def top():\n" + " return 1\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_FALSE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* the document changes alone: it re-resolves through the stored re-exports */ + th_write_file(TH_PATH(repo, "docs/api.rst"), "Models\n" + "======\n" + "\n" + ".. module:: pkg\n" + "\n" + "Only :class:`Model` and :func:`aid`.\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", NULL)); + /* the package re-exports another Model: the unchanged document follows */ + th_write_file(TH_PATH(repo, "pkg/__init__.py"), "from .other import Model\n" + "from .sub.thing import helper as aid\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "re-export change", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.other.Model", NULL)); + th_cleanup(tmp); + PASS(); +} + +/* Two held-out findings (supplementary audit): `.. py:function:: autofit` on a + * page without a module bound the example script module examples/autofit.py + * (an object directive declares a definition, never a module), and + * `:lines: 5-` (line 5 to the file's end) was read as line 5 alone. */ +TEST(rst_object_and_open_lines) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrsto_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "examples/autofit.py"), "print('autofit example')\n"); + th_write_file(TH_PATH(repo, "examples/tail.py"), "import os\n" /* 1 */ + "\n" /* 2 */ + "\n" /* 3 */ + "class Tail:\n" /* 4 */ + " def a(self):\n" /* 5 */ + " return 1\n" /* 6 */ + "\n" /* 7 */ + " def b(self):\n" /* 8 */ + " return 2\n"); /* 9 */ + th_write_file(TH_PATH(repo, "docs/conf.py"), "project = 'x'\n"); + th_write_file(TH_PATH(repo, "docs/worksheet.rst"), "Worksheet\n" + "=========\n" + "\n" + ".. py:function:: autofit()\n" + "\n" + " Fit the columns.\n" + "\n" + ".. literalinclude:: ../examples/tail.py\n" + " :lines: 5-\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path='examples/autofit.py'"), + 0); + ASSERT_EQ(rst_rows(db, ".. py:function:: autofit", "missing"), 1); + /* lines 5 to 9 lie in class Tail; line 5 alone lies in method a */ + ASSERT_TRUE( + rst_edge(db, "docs.worksheet.Worksheet", "examples.tail.Tail", "\"target_lines\":[5,9]")); + th_cleanup(tmp); + PASS(); +} + +/* The production gate (doclink.h): object directives ship since their + * held-out audit; include and kernel-doc are still held, so in the same + * documents they write rows, not edges. Runs after the suite's forced + * families are reset. */ +TEST(rst_ship_gate) { + ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_OBJECT)); + ASSERT_FALSE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_INCLUDE)); + ASSERT_FALSE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_KERNEL_DOC)); + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_gate_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* `.. class:: View` in pkg.views: an edge */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\"")); + /* the include resolves, and its family holds it as a row */ + ASSERT_FALSE(rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", NULL)); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE " + "reason = 'below_bar_tier' AND syntax = 'include'"), + 1); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND " + "properties LIKE '%\"syntax\":\"kernel_doc\"%'"), + 0); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_rst) { + dm_ship_held_families(); + RUN_TEST(rst_scan_structure); + RUN_TEST(rst_python_scope_blob); + RUN_TEST(rst_links_pipeline); + RUN_TEST(rst_object_and_open_lines); + RUN_TEST(rst_adr_record); + RUN_TEST(rst_links_incremental); + cbm_doclink_test_reset_ships(); + RUN_TEST(rst_ship_gate); +} diff --git a/tests/test_doc_mentions_helpers.h b/tests/test_doc_mentions_helpers.h index 66fbcd52ce..9c2eca104a 100644 --- a/tests/test_doc_mentions_helpers.h +++ b/tests/test_doc_mentions_helpers.h @@ -411,4 +411,20 @@ static inline int dm_reason_lines(const char *db, const char *block) { return n; } +/* The document families held back from shipping (doclink.h: below the held-out + * audit's bar, or not yet audited). Their resolution is tested with them on, + * so it is the resolution an audit would ship; doc_links_md_ship_gate tests + * the production gate itself. */ +static const int DM_HELD_FAMILIES[] = { + CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_DOCLINK_RST_INCLUDE, + CBM_DOCLINK_RST_KERNEL_DOC, CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, + CBM_DOCLINK_ADOC_CODE_NAME, +}; + +static inline void dm_ship_held_families(void) { + for (size_t i = 0; i < sizeof(DM_HELD_FAMILIES) / sizeof(DM_HELD_FAMILIES[0]); i++) { + cbm_doclink_test_set_ships(DM_HELD_FAMILIES[i], true); + } +} + #endif /* TEST_DOC_MENTIONS_HELPERS_H */ diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 6f99acd066..634a5ec0df 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -586,6 +586,48 @@ TEST(java_class) { PASS(); } +/* A class field is one Field. Java, C# and Solidity list the field + * declaration as a variable kind too, and the class-body variable pass minted + * a second node: a Variable with the module's qualified name (Query.filters + * became `demo.query.filters`, the name of a package a documentation code + * span then bound). Languages whose class variables are no Field keep them. */ +TEST(class_field_is_one_field_not_also_a_variable) { + CBMFileResult *r = extract("package demo.query;\npublic class Query {\n" + " private Object filters;\n" + " private static final String PROP = \"v\";\n}\n", + CBM_LANG_JAVA, "t", "Query.java"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Field", "filters"), 1); + ASSERT_EQ(count_defs_named(r, "Variable", "filters"), 0); + /* an initialized field is named by its declarator's name, not its text + * (`PROP = "v"` was the Field's name, so only the duplicate was findable) */ + ASSERT_EQ(count_defs_named(r, "Field", "PROP"), 1); + ASSERT_EQ(count_defs_named(r, "Variable", "PROP"), 0); + cbm_free_result(r); + r = extract("namespace Demo {\n public class Q {\n private int count;\n }\n}\n", + CBM_LANG_CSHARP, "t", "Q.cs"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Field", "count"), 1); + ASSERT_EQ(count_defs_named(r, "Variable", "count"), 0); + cbm_free_result(r); + r = extract("pragma solidity ^0.8.0;\ncontract Q {\n uint256 public total;\n}\n", + CBM_LANG_SOLIDITY, "t", "Q.sol"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Field", "total"), 1); + ASSERT_EQ(count_defs_named(r, "Variable", "total"), 0); + cbm_free_result(r); + /* no Field for these: their class variables stay */ + r = extract("class Q {\n var count: Int = 0\n}\n", CBM_LANG_SWIFT, "t", "Q.swift"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Variable", "count"), 1); + cbm_free_result(r); + r = extract("extends Node\nclass Inner:\n var x = 1\n", CBM_LANG_GDSCRIPT, "t", "q.gd"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Variable", "x"), 1); + cbm_free_result(r); + PASS(); +} + TEST(java_method) { CBMFileResult *r = extract( "public class Svc { public void doWork() {} public int compute(int x) { return x; } }", @@ -5011,10 +5053,11 @@ TEST(swift_non_url_constructor_untouched_issue1892) { * the per-file constant map and resolved at the call site, for both return * statements and arrow expression bodies. */ TEST(extract_ts_await_generic_call_issue2210) { - CBMFileResult *r = extract("function parseJsonBody() { return {} as T; }\n" - "async function plain() { return await parseJsonBody(); }\n" - "async function generic() { return await parseJsonBody(); }\n", - CBM_LANG_TYPESCRIPT, "t", "await.ts"); + CBMFileResult *r = + extract("function parseJsonBody() { return {} as T; }\n" + "async function plain() { return await parseJsonBody(); }\n" + "async function generic() { return await parseJsonBody(); }\n", + CBM_LANG_TYPESCRIPT, "t", "await.ts"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT_EQ(count_calls_named(r, "parseJsonBody"), 2); @@ -7071,6 +7114,37 @@ static const CBMDefinition *c1_def_qn(CBMFileResult *r, const char *qn) { return NULL; } +static bool qn_ends_with(const CBMDefinition *d, const char *tail) { + size_t n = d && d->qualified_name ? strlen(d->qualified_name) : 0; + size_t t = strlen(tail); + return n >= t && strcmp(d->qualified_name + n - t, tail) == 0; +} + +/* A field and a same-named method of one class body would share one qualified + * name (one node per QN): the field is fenced `.#field` and the + * method keeps the plain name. A field without a namesake keeps its name, and + * so does a nested class's field named like a method of the outer class. */ +TEST(field_named_like_method_is_fenced) { + CBMFileResult *r = extract("package demo;\npublic class Starter {\n" + " private String executable;\n" + " private String dir;\n" + " public Starter executable(String p) { return this; }\n" + " public void size() {}\n" + " static class Inner { int size; }\n}\n", + CBM_LANG_JAVA, "t", "Starter.java"); + ASSERT_NOT_NULL(r); + const CBMDefinition *field = c1_def(r, "Field", "executable"); + const CBMDefinition *method = c1_def(r, "Method", "executable"); + ASSERT_NOT_NULL(field); + ASSERT_NOT_NULL(method); + ASSERT_TRUE(qn_ends_with(field, "Starter.executable#field")); + ASSERT_TRUE(qn_ends_with(method, "Starter.executable")); + ASSERT_TRUE(qn_ends_with(c1_def(r, "Field", "dir"), "Starter.dir")); + ASSERT_TRUE(qn_ends_with(c1_def(r, "Field", "size"), "Inner.size")); + cbm_free_result(r); + PASS(); +} + /* The def with this label and name that starts on `line`. */ static const CBMDefinition *c1_def_at(CBMFileResult *r, const char *label, const char *name, int line) { @@ -10036,6 +10110,8 @@ SUITE(extraction) { /* OOP */ RUN_TEST(java_class); RUN_TEST(java_method); + RUN_TEST(class_field_is_one_field_not_also_a_variable); + RUN_TEST(field_named_like_method_is_fenced); RUN_TEST(java_interface); RUN_TEST(java_interface_no_duplicate_function_issue1234); RUN_TEST(java_enum_dedup_preserves_calls_issue1234); diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 7818c4c445..7246e5b958 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -195,7 +195,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"css", "Module:1"}, {"scss", "Module:1,Variable:1"}, {"markdown", "Module:1,Section:1"}, - {"rst", "Module:1"}, + {"rst", "Module:1,Section:1"}, {"dockerfile", "Module:1"}, {"makefile", "Function:1,Module:1"}, {"cmake", "Function:1,Module:1"}, diff --git a/tests/test_grammar_probe_g.c b/tests/test_grammar_probe_g.c index d0f99d4e0e..215b4acee0 100644 --- a/tests/test_grammar_probe_g.c +++ b/tests/test_grammar_probe_g.c @@ -399,12 +399,13 @@ TEST(probe_markdown_no_class_variable) { /* ══════════════════════════════════════════════════════════════════ * GROUP 6 — RST (.rst) * - * RST golden histogram: Module:1 (pure-data). - * reStructuredText headings are NOT extracted as Section nodes (unlike Markdown). + * RST golden histogram: Module:1,Section:1. + * reStructuredText titles are Section nodes, like Markdown headings + * (doclink_rst.c). * ══════════════════════════════════════════════════════════════════ */ -/* RST: document with sections → Module:1, no Section nodes. */ -TEST(probe_rst_module_only) { +/* RST: a document with two titles → Module:1 and a Section per title. */ +TEST(probe_rst_sections) { GpgMetrics m = gpg_metrics("docs.rst", "Project Title\n" "=============\n" "\n" @@ -415,9 +416,9 @@ TEST(probe_rst_module_only) { "\n" "Run ``pip install mypackage``.\n"); ASSERT_TRUE(m.ok); - /* GREEN: RST produces only a Module node (no Section nodes extracted). */ + /* GREEN: a Module node and the two titles' Section nodes. */ ASSERT_TRUE(m.modules == 1); - ASSERT_TRUE(m.sections == 0); + ASSERT_TRUE(m.sections == 2); PASS(); } @@ -1063,7 +1064,7 @@ SUITE(grammar_probe_g) { RUN_TEST(probe_markdown_no_class_variable); /* RST */ - RUN_TEST(probe_rst_module_only); + RUN_TEST(probe_rst_sections); RUN_TEST(probe_rst_no_crash); /* JSON */ diff --git a/tests/test_main.c b/tests/test_main.c index 8bd1a81d5a..bce1fd57c0 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -972,6 +972,10 @@ extern void suite_doclinks(void); extern void suite_doc_mentions(void); extern void suite_doc_mentions_cost(void); extern void suite_doc_mentions_msbuild(void); +extern void suite_doc_links_md(void); +extern void suite_doc_links_pdf(void); +extern void suite_doc_links_rst(void); +extern void suite_doc_links_adoc(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -1343,6 +1347,10 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions); RUN_SELECTED_SUITE(doc_mentions_cost); RUN_SELECTED_SUITE(doc_mentions_msbuild); + RUN_SELECTED_SUITE(doc_links_md); + RUN_SELECTED_SUITE(doc_links_pdf); + RUN_SELECTED_SUITE(doc_links_rst); + RUN_SELECTED_SUITE(doc_links_adoc); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0bbf1ba03a..05b2e9e4ad 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1690,6 +1690,74 @@ static int fixture_node_count(cbm_store_t *store, const char *project, const cha return found; } +/* Edges of `edge_type` from the node named `source_name` to the node named + * `target_name` that carries `target_label`. */ +static int labeled_edge_count(cbm_store_t *s, const char *project, const char *edge_type, + const char *source_name, const char *target_name, + const char *target_label) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + if (cbm_store_find_edges_by_type(s, project, edge_type, &edges, &edge_count) != CBM_STORE_OK) { + return -1; + } + int matches = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t source = {0}; + cbm_node_t target = {0}; + int source_ok = cbm_store_find_node_by_id(s, edges[i].source_id, &source) == CBM_STORE_OK; + int target_ok = cbm_store_find_node_by_id(s, edges[i].target_id, &target) == CBM_STORE_OK; + if (source_ok && target_ok && source.name && target.name && target.label && + strcmp(source.name, source_name) == 0 && strcmp(target.name, target_name) == 0 && + strcmp(target.label, target_label) == 0) { + matches++; + } + cbm_node_free_fields(&source); + cbm_node_free_fields(&target); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); + } + return matches; +} + +/* A Java field and a same-named method of one class keep their own nodes (the + * field is fenced `.#field`), and a write or a read of the field + * reaches the field, never the method; the method keeps its calls. The setter + * writing its own namesake was a self-edge before, so it was dropped. */ +TEST(pipeline_field_named_like_method_keeps_its_reads_and_writes) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_fieldfence_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + th_write_file(TH_PATH(tmp, "src/demo/Starter.java"), + "package demo;\n\npublic class Starter {\n" + " private String executable;\n\n" + " public Starter executable(String path) {\n" + " this.executable = path;\n" + " return this;\n }\n\n" + " public String show() {\n return executable;\n }\n\n" + " public static void run() {\n" + " new Starter().executable(\"x\").show();\n }\n}\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/fence.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + ASSERT_EQ(named_node_count(store, project, "executable"), 2); + ASSERT_EQ(labeled_edge_count(store, project, "WRITES", "executable", "executable", "Field"), 1); + ASSERT_EQ(labeled_edge_count(store, project, "WRITES", "executable", "executable", "Method"), + 0); + ASSERT_EQ(labeled_edge_count(store, project, "USAGE", "show", "executable", "Field"), 1); + ASSERT_EQ(labeled_edge_count(store, project, "USAGE", "show", "executable", "Method"), 0); + ASSERT_EQ(labeled_edge_count(store, project, "CALLS", "run", "executable", "Method"), 1); + cbm_store_close(store); + cbm_pipeline_free(p); + th_cleanup(tmp); + PASS(); +} + typedef struct { int run_rc; bool store_opened; @@ -18112,6 +18180,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_nix_scoped_binding_calls_resolve); RUN_TEST(pipeline_hcl_block_reference_resolves_to_its_block); RUN_TEST(pipeline_incremental_preserves_cross_file_calls); + RUN_TEST(pipeline_field_named_like_method_keeps_its_reads_and_writes); RUN_TEST(pipeline_objectscript_export_preserves_calls_sequential_parallel); RUN_TEST(pipeline_objectscript_export_incremental_matches_full_relationships); RUN_TEST(pipeline_objectscript_export_aggregate_exceeds_arena_block_table); diff --git a/tests/test_registry.c b/tests/test_registry.c index 278e97d9b0..b87600b815 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -213,6 +213,27 @@ TEST(registry_label_of) { PASS(); } +/* A read, write or value reference that resolved to a Method goes to the + * method's fenced `#field` twin when the class has one; nothing else moves. */ +TEST(registry_value_target_field_twin) { + cbm_registry_t *r = cbm_registry_new(); + cbm_registry_add(r, "count", "p.C.count", "Method"); + cbm_registry_add(r, "count", "p.C.count#field", "Field"); + cbm_registry_add(r, "size", "p.C.size", "Method"); + cbm_registry_add(r, "name", "p.C.name", "Field"); + ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.count"), "p.C.count#field"); + ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.size"), "p.C.size"); + ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.name"), "p.C.name"); + ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.none"), "p.C.none"); + /* the fenced field is still found by its plain name */ + const char **qns = NULL; + int n = 0; + ASSERT_EQ(cbm_registry_find_by_name(r, "count", &qns, &n), 0); + ASSERT_EQ(n, 2); + cbm_registry_free(r); + PASS(); +} + TEST(registry_find_by_name) { cbm_registry_t *r = cbm_registry_new(); cbm_registry_add(r, "main", "proj.cmd.main", "Function"); @@ -1312,6 +1333,7 @@ SUITE(registry) { RUN_TEST(registry_free_null); RUN_TEST(registry_add_and_exists); RUN_TEST(registry_label_of); + RUN_TEST(registry_value_target_field_twin); RUN_TEST(registry_find_by_name); RUN_TEST(registry_no_duplicates); /* Resolution */