Repository navigation
Expand file tree
/
Copy pathfallback.js
More file actions
94 lines (83 loc) · 3.4 KB
/
Copy pathfallback.js
File metadata and controls
94 lines (83 loc) · 3.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
function unescapeHtml(str) {
if (!str) return "";
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/ /g, " ").replace(/&#(\d+);/g, (match, dec) => String.fromCharCode(dec));
}
export const SHELL_CONTENT_FLOOR_BYTES = 280;
const ARTICLE_TYPES = new Set(["Article", "NewsArticle", "BlogPosting", "Report", "WebPage"]);
const encoder = new TextEncoder();
function plainText(value) {
if (typeof value !== "string") return "";
return unescapeHtml(value).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
}
function usable(source, paragraphs) {
const markdown = paragraphs.filter(Boolean).join("\n\n");
return encoder.encode(markdown).length >= SHELL_CONTENT_FLOOR_BYTES
? { source, markdown } : null;
}
function scripts(rawHtml) {
return rawHtml.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi);
}
function attribute(attributes, name) {
return attributes.match(new RegExp(`(?:^|\\s)${name}\\s*=\\s*(["'])(.*?)\\1`, "i"))?.[2] ?? "";
}
function articleParagraphs(value, output) {
if (Array.isArray(value)) {
for (const item of value) articleParagraphs(item, output);
return;
}
if (!value || typeof value !== "object") return;
const types = Array.isArray(value["@type"]) ? value["@type"] : [value["@type"]];
if (types.some((type) => ARTICLE_TYPES.has(type))) {
const heading = plainText(value.headline);
const body = plainText(value.articleBody) || plainText(value.text) || plainText(value.description);
if (heading) output.push(`# ${heading}`);
if (body) output.push(body);
}
if (value["@graph"]) articleParagraphs(value["@graph"], output);
}
function longStrings(value, output, seen) {
if (typeof value === "string") {
const paragraph = plainText(value);
if (paragraph.length >= 80 && !/^https?:\/\//i.test(paragraph) && !seen.has(paragraph)) {
seen.add(paragraph);
output.push(paragraph);
}
return;
}
if (Array.isArray(value)) {
for (const item of value) longStrings(item, output, seen);
} else if (value && typeof value === "object") {
for (const item of Object.values(value)) longStrings(item, output, seen);
}
}
export function structuredText(rawHtml) {
if (typeof rawHtml !== "string" || !rawHtml) return null;
const blocks = [...scripts(rawHtml)];
const articles = [];
for (const [, attributes, body] of blocks) {
if (attribute(attributes, "type").toLowerCase() !== "application/ld+json") continue;
try { articleParagraphs(JSON.parse(body), articles); } catch { /* Skip malformed data. */ }
}
const jsonLd = usable("json_ld", articles);
if (jsonLd) return jsonLd;
for (const [, attributes, body] of blocks) {
if (attribute(attributes, "id") !== "__NEXT_DATA__") continue;
try {
const paragraphs = [];
longStrings(JSON.parse(body)?.props?.pageProps, paragraphs, new Set());
const next = usable("next_data", paragraphs);
if (next) return next;
} catch { /* Skip malformed data. */ }
}
for (const [, , body] of blocks) {
const assignment = /^\s*window\.__NUXT__\s*=\s*([\s\S]*?)\s*;?\s*$/.exec(body);
if (!assignment) continue;
try {
const paragraphs = [];
longStrings(JSON.parse(assignment[1]), paragraphs, new Set());
const nuxt = usable("nuxt_data", paragraphs);
if (nuxt) return nuxt;
} catch { /* A JavaScript object literal is not JSON. */ }
}
return null;
}