| 1 | // Run: tsx src/__tests__/markdown-pipeline.test.tsx |
| 2 | // |
| 3 | // Parse-parity goldens for the isomorphic markdown pipeline (Phase E): the |
| 4 | // worker/fallback pipeline must render byte-identical static markup to the |
| 5 | // production react-markdown path for the same document, both unsliced and |
| 6 | // sliced into blocks (footnotes/reference definitions resolve across blocks |
| 7 | // because parsing is whole-document). |
| 8 | |
| 9 | import { createElement, Fragment, type ReactNode } from "react"; |
| 10 | import { renderToStaticMarkup } from "react-dom/server"; |
| 11 | import ReactMarkdown, { defaultUrlTransform } from "react-markdown"; |
| 12 | import { JSDOM } from "jsdom"; |
| 13 | import { normalizeMath } from "../components/mathNormalize"; |
| 14 | import { createComponents } from "../components/markdownComponents"; |
| 15 | import { LocaleProvider } from "../lib/i18n"; |
| 16 | import { reasonixRehypePlugins, reasonixRemarkPlugins } from "../components/markdownRemarkPlugins"; |
| 17 | import { hastBlockToJsx } from "../lib/hastJsx"; |
| 18 | import { visibleMarkdownBlockCount } from "../lib/markdownDomBudget"; |
| 19 | import { |
| 20 | defaultMarkdownUrlTransform, |
| 21 | estimateHastBytes, |
| 22 | markdownContentRevision, |
| 23 | markdownUrlTransform, |
| 24 | parseMarkdown, |
| 25 | parseMarkdownToBlocks, |
| 26 | parseMarkdownToHast, |
| 27 | sliceHastBlocks, |
| 28 | type MarkdownBlock, |
| 29 | } from "../lib/markdownPipeline"; |
| 30 | |
| 31 | let passed = 0; |
| 32 | let failed = 0; |
| 33 | |
| 34 | function ok(value: unknown, label: string) { |
| 35 | if (value) { |
| 36 | process.stdout.write(` PASS ${label}\n`); |
| 37 | passed += 1; |
| 38 | } else { |
| 39 | process.stdout.write(` FAIL ${label}\n`); |
| 40 | failed += 1; |
| 41 | } |
| 42 | } |
| 43 | |
| 44 | function eq(actual: unknown, expected: unknown, label: string) { |
| 45 | if (actual === expected) ok(true, label); |
| 46 | else ok(false, `${label}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); |
| 47 | } |
| 48 | |
| 49 | function renderCurrent(text: string): string { |
| 50 | return renderToStaticMarkup( |
| 51 | createElement(LocaleProvider, null, createElement(ReactMarkdown, { |
| 52 | remarkPlugins: reasonixRemarkPlugins, |
| 53 | rehypePlugins: reasonixRehypePlugins, |
| 54 | components: createComponents(false), |
| 55 | urlTransform: markdownUrlTransform, |
| 56 | children: normalizeMath(text), |
| 57 | })), |
| 58 | ); |
| 59 | } |
| 60 | |
| 61 | function renderBlocks(blocks: MarkdownBlock[]): string { |
| 62 | const components = createComponents(false); |
| 63 | return renderToStaticMarkup( |
| 64 | createElement(LocaleProvider, null, createElement(Fragment, { |
| 65 | children: blocks.map((block) => |
| 66 | createElement(Fragment, { key: block.key, children: hastBlockToJsx(block, components) as ReactNode })), |
| 67 | })), |
| 68 | ); |
| 69 | } |
| 70 | |
| 71 | function projectRenderedBlocks(blocks: MarkdownBlock[]): string { |
| 72 | const dom = new JSDOM(`<!doctype html><body><div id="root" data-transcript-selectable>${renderBlocks(blocks)}</div></body>`); |
| 73 | globalThis.Node = dom.window.Node; |
| 74 | globalThis.Element = dom.window.Element; |
| 75 | globalThis.HTMLElement = dom.window.HTMLElement; |
| 76 | const root = dom.window.document.getElementById("root") as HTMLElement; |
| 77 | const selection = dom.window.getSelection()!; |
| 78 | selection.selectAllChildren(root); |
| 79 | const projected = selection.toString(); |
| 80 | dom.window.close(); |
| 81 | return projected; |
| 82 | } |
| 83 | |
| 84 | console.log("\nmarkdown pipeline parity"); |
| 85 | |
| 86 | const bigCode = "```js\n" + "const value = compute(index); // keep this line long enough\n".repeat(4000) + "```"; |
| 87 | ok(bigCode.length > 100_000, "code-fence fixture exceeds 100KB"); |
| 88 | |
| 89 | const fixtures: Record<string, string> = { |
| 90 | headingsAndProse: "# Title\n\nHello **world**, this is [a link](https://example.com).\n\nSecond paragraph with `code`.", |
| 91 | gfmTable: "| name | value |\n| --- | ---: |\n| a | 1 |\n| b | 2 |", |
| 92 | taskList: "- [x] done\n- [ ] todo\n- [ ] item with **bold** and `code`", |
| 93 | strikethroughAutolink: "~~gone~~ and https://example.com/auto plus www.example.com", |
| 94 | mathInline: "Price is $5 and math $x^2$ works. Also \\(y=1\\) and $E=mc^2$.", |
| 95 | mathBlockRepair: "Before\n\n\\[\n\\int_0^1 x\\,dx\n\\]\n\nAfter\n\n$$\n\\frac{a}{b}\n$$", |
| 96 | mathInlinePipe: "| formula | note |\n| --- | --- |\n| $|x|$ cell | pipe inside math |", |
| 97 | footnotes: "Text with a note[^1] and another[^long].\n\n[^1]: first note\n\n[^long]: second note with [ref link][r]\n\n[r]: https://example.com/ref", |
| 98 | crossBlockRefs: "Use [shared] here.\n\n## Section\n\nMore text.\n\n## Later\n\nAgain [shared] and [other].\n\n[shared]: https://example.com/shared\n[other]: https://example.com/other", |
| 99 | mermaidFence: "```mermaid\ngraph TD\nA-->B\n```", |
| 100 | bigCodeFence: bigCode, |
| 101 | cjk: "中文段落，包含「引号」和路径 D:\\work\\项目\\文件.md。\n\n- 列表项一\n- 列表项二", |
| 102 | rawHtml: "Before <div class=\"x\">raw</div> after\n\n<script>alert(1)</script>", |
| 103 | unsafeAndFileLinks: "[bad](javascript:alert(1)) and [file](file:///tmp/a%20b.txt) and D:\\src\\app.ts", |
| 104 | manyBlocks: Array.from({ length: 40 }, (_, i) => `## Part ${i}\n\nParagraph ${i} with *emphasis*.\n`).join("\n"), |
| 105 | }; |
| 106 | |
| 107 | for (const [name, text] of Object.entries(fixtures)) { |
| 108 | const expected = renderCurrent(text); |
| 109 | const root = parseMarkdownToHast(text); |
| 110 | const whole = renderBlocks([{ key: "whole", children: root.children }]); |
| 111 | eq(whole, expected, `${name}: pipeline render matches react-markdown`); |
| 112 | const blocks = sliceHastBlocks(root); |
| 113 | const sliced = renderBlocks(blocks); |
| 114 | eq(sliced, expected, `${name}: sliced blocks render identically (${blocks.length} blocks)`); |
| 115 | } |
| 116 | |
| 117 | // Empty fenced blocks are formatting placeholders. They must not create a |
| 118 | // bordered CodeViewer, while comment-only and whitespace-adjacent blocks stay |
| 119 | // visible as real code content. |
| 120 | { |
| 121 | const emptyFences = "Before\n\n```\n\n```\n\nAfter\n\n```ts\n \n```"; |
| 122 | const html = renderCurrent(emptyFences); |
| 123 | ok(!html.includes("code-block"), "empty fenced blocks do not render phantom code cards"); |
| 124 | ok(html.includes("Before") && html.includes("After"), "text around empty fenced blocks remains visible"); |
| 125 | |
| 126 | const comments = "```ts\n// keep this comment\nconst stable = true;\n```"; |
| 127 | const commentHtml = renderCurrent(comments); |
| 128 | ok(commentHtml.includes("keep this comment"), "non-empty comment code blocks remain visible"); |
| 129 | ok(commentHtml.includes("code-block"), "non-empty comment code keeps its code-block surface"); |
| 130 | } |
| 131 | |
| 132 | // A lone tilde writes a range; two ranges on one line must not strike out the |
| 133 | // text between them. Only ~~ strikes, on the live and the worker path alike. |
| 134 | { |
| 135 | const text = "转速 500~1000 或 2000~3000 转，~~旧值~~"; |
| 136 | const live = renderCurrent(text); |
| 137 | const worker = renderBlocks([{ key: "whole", children: parseMarkdownToHast(text).children }]); |
| 138 | for (const [path, html] of [["live", live], ["worker", worker]] as const) { |
| 139 | eq((html.match(/<del>/g) ?? []).length, 1, `${path}: only the doubled tilde strikes`); |
| 140 | ok(html.includes("500~1000 或 2000~3000"), `${path}: single-tilde ranges stay literal`); |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // Block keys are stable top-level indexes. |
| 145 | { |
| 146 | const blocks = parseMarkdownToBlocks("one\n\ntwo\n\nthree"); |
| 147 | eq(blocks.map((b) => b.key).join(","), "b0,b1,b2", "block keys are stable indexes"); |
| 148 | ok(blocks.every((block) => (block.elementCount ?? 0) > 0), "parse stamps DOM element counts with block fingerprints"); |
| 149 | } |
| 150 | |
| 151 | // Progressive DOM publication keeps semantic blocks whole and advances by an |
| 152 | // explicit element budget instead of mounting an unbounded parsed document. |
| 153 | { |
| 154 | const blocks = parseMarkdownToBlocks(Array.from({ length: 12 }, (_, index) => |
| 155 | `## Part ${index}\n\nParagraph with **bold** and [link](https://example.com/${index}).`).join("\n\n")); |
| 156 | const first = visibleMarkdownBlockCount(blocks, 8); |
| 157 | ok(first > 0 && first < blocks.length, "small DOM budget publishes a strict leading block page"); |
| 158 | ok(visibleMarkdownBlockCount(blocks, 10_000) === blocks.length, "larger DOM budget makes every block reachable"); |
| 159 | } |
| 160 | |
| 161 | // Footnote definitions survive slicing as a trailing block with working refs. |
| 162 | { |
| 163 | const text = "first[^a]\n\n## middle\n\nsecond[^b]\n\n[^a]: note a\n[^b]: note b"; |
| 164 | const blocks = parseMarkdownToBlocks(text); |
| 165 | const last = blocks[blocks.length - 1]; |
| 166 | const lastHtml = renderBlocks([last]); |
| 167 | ok(lastHtml.includes("data-footnotes"), "footnote section is the trailing block"); |
| 168 | const sliced = renderBlocks(blocks); |
| 169 | ok(sliced.includes('href="#user-content-fn-a"'), "footnote reference links survive slicing"); |
| 170 | ok(sliced.includes('id="user-content-fn-a"'), "footnote definition anchors survive slicing"); |
| 171 | } |
| 172 | |
| 173 | // The copied defaultUrlTransform must match react-markdown's across protocols. |
| 174 | { |
| 175 | const corpus = [ |
| 176 | "https://example.com/a?b=c#d", |
| 177 | "http://example.com", |
| 178 | "mailto:user@example.com", |
| 179 | "javascript:alert(1)", |
| 180 | "vbscript:x", |
| 181 | "data:text/html,boom", |
| 182 | "./relative/path", |
| 183 | "../up", |
| 184 | "/absolute", |
| 185 | "#fragment", |
| 186 | "query?only", |
| 187 | "ftp://example.com/file", |
| 188 | "file:///etc/passwd", |
| 189 | "HTTPS://EXAMPLE.COM/upper", |
| 190 | "ircs://irc.example.com/chan", |
| 191 | "xmpp:user@example.com", |
| 192 | "a:b:c", |
| 193 | "", |
| 194 | "C:\\src\\app.ts", |
| 195 | ]; |
| 196 | for (const url of corpus) { |
| 197 | eq(defaultMarkdownUrlTransform(url), defaultUrlTransform(url), `urlTransform parity for ${JSON.stringify(url)}`); |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // Authority-form UNC links use the same strict local-file allowlist in the |
| 202 | // worker pipeline as canonical file:/// links do. |
| 203 | { |
| 204 | const unc = "file://nas/share/report.md"; |
| 205 | eq(markdownUrlTransform(unc), unc, "authority-form UNC survives pipeline URL sanitization"); |
| 206 | const root = parseMarkdownToHast(`[report](${unc})`); |
| 207 | const html = renderBlocks([{ key: "unc", children: root.children }]); |
| 208 | ok(html.includes(`href="${unc}"`), "authority-form UNC href survives HAST rendering"); |
| 209 | } |
| 210 | |
| 211 | // Windows device namespaces and alternate data streams must not be restored |
| 212 | // after react-markdown's default URL sanitizer rejects their file: scheme. |
| 213 | for (const unsafe of [ |
| 214 | "file://./PhysicalDrive0", |
| 215 | "file:////?/C:/Windows", |
| 216 | "file:///C:/safe.txt:payload", |
| 217 | "file:///tmp/report.md?download=1", |
| 218 | "file:///tmp/report.md#section", |
| 219 | ]) { |
| 220 | eq(markdownUrlTransform(unsafe), "", `unsafe local URL is blanked: ${unsafe}`); |
| 221 | } |
| 222 | |
| 223 | // Content revision + byte weight. |
| 224 | { |
| 225 | eq(markdownContentRevision("alpha") === markdownContentRevision("alpha"), true, "content revision is deterministic"); |
| 226 | eq(markdownContentRevision("alpha") !== markdownContentRevision("beta"), true, "content revision distinguishes texts"); |
| 227 | const blocks = parseMarkdownToBlocks("hello **world**"); |
| 228 | const bytes = estimateHastBytes(blocks); |
| 229 | ok(bytes > 0 && bytes < 100_000, "hast byte estimate is positive and bounded"); |
| 230 | const bigBlocks = parseMarkdownToBlocks(bigCode); |
| 231 | ok(estimateHastBytes(bigBlocks) > bytes, "hast byte estimate grows with content"); |
| 232 | } |
| 233 | |
| 234 | console.log("\nmarkdown selection projection"); |
| 235 | |
| 236 | { |
| 237 | const result = parseMarkdown([ |
| 238 | "# 标题 😀", |
| 239 | "", |
| 240 | "段落 [链接文字](https://example.com) 与 $x^2$。", |
| 241 | "", |
| 242 | "内联 **粗体** *斜体*。", |
| 243 | "", |
| 244 | "- 第一项", |
| 245 | "- 第二项", |
| 246 | "", |
| 247 | "```ts", |
| 248 | "const value = 1;", |
| 249 | "```", |
| 250 | "", |
| 251 | "| 名称 | 值 |", |
| 252 | "| --- | --- |", |
| 253 | "| 一 | 1 |", |
| 254 | ].join("\n")); |
| 255 | eq( |
| 256 | result.selectionText, |
| 257 | "标题 😀\n段落 链接文字 与 $x^2$。\n内联 粗体 斜体。\n第一项\n第二项\nconst value = 1;\n名称\t值\n一\t1", |
| 258 | "selection projection preserves readable structure, code, tables, CJK, emoji and LaTeX", |
| 259 | ); |
| 260 | const selected = projectRenderedBlocks(result.blocks); |
| 261 | ok(selected.includes("标题 😀") && selected.includes("链接文字") && selected.includes("const value = 1;"), |
| 262 | "native DOM selection includes Unicode, links and complete code text"); |
| 263 | eq(result.selectionRevision, markdownContentRevision(result.selectionText), "selection revision fingerprints projected UTF-16 text"); |
| 264 | } |
| 265 | |
| 266 | { |
| 267 | const rows = Array.from({ length: 52 }, (_, index) => `| row-${index} | ${index} |`).join("\n"); |
| 268 | const result = parseMarkdown(`| name | value |\n| --- | --- |\n${rows}`); |
| 269 | ok(result.blocks.some((block) => block.virtualTable), "large plain table uses the virtual table representation"); |
| 270 | ok(result.selectionText.includes("row-51\t51"), "virtual table projection includes rows that never mount in the DOM"); |
| 271 | } |
| 272 | |
| 273 | // ── block fingerprints ────────────────────────────────────────────────────── |
| 274 | // The render path keeps a previous AST object when key and fingerprint match, |
| 275 | // so the fingerprint must be equal exactly when the block's rendered content is |
| 276 | // equal. A false "unchanged" leaves a stale block on screen. |
| 277 | { |
| 278 | const same = parseMarkdown("paragraph one\n\n- a\n- b"); |
| 279 | const again = parseMarkdown("paragraph one\n\n- a\n- b"); |
| 280 | eq( |
| 281 | again.blocks.map(block => block.fingerprint).join(","), |
| 282 | same.blocks.map(block => block.fingerprint).join(","), |
| 283 | "identical sources fingerprint identically", |
| 284 | ); |
| 285 | ok(same.blocks.every(block => Number.isInteger(block.fingerprint)), "every block is stamped with a fingerprint"); |
| 286 | |
| 287 | // A block that changed must not be mistaken for its previous self. |
| 288 | const appended = parseMarkdown("paragraph one\n\n- a\n- b\n- c"); |
| 289 | ok(appended.blocks[0].fingerprint === same.blocks[0].fingerprint, "an unchanged leading block keeps its fingerprint"); |
| 290 | ok(appended.blocks[1].fingerprint !== same.blocks[1].fingerprint, "a grown list block changes its fingerprint"); |
| 291 | |
| 292 | // Appending a reference definition rewrites an EARLIER paragraph's link. A |
| 293 | // positional-only identity would keep the stale block here. |
| 294 | const unresolved = parseMarkdown("see [docs][ref]\n\nand more"); |
| 295 | const resolved = parseMarkdown("see [docs][ref]\n\nand more\n\n[ref]: https://example.com/doc"); |
| 296 | ok(unresolved.blocks[0].fingerprint !== resolved.blocks[0].fingerprint, |
| 297 | "a reference definition that resolves an earlier link changes that block's fingerprint"); |
| 298 | |
| 299 | // Structure the markdown source does not spell out still separates blocks. |
| 300 | const inline = parseMarkdown("a **bold** word"); |
| 301 | const plain = parseMarkdown("a bold word"); |
| 302 | ok(inline.blocks[0].fingerprint !== plain.blocks[0].fingerprint, "inline emphasis changes the fingerprint"); |
| 303 | |
| 304 | // Distinct content must not collide, including across block boundaries. |
| 305 | const boundary = parseMarkdown("ab\n\nc"); |
| 306 | const shifted = parseMarkdown("a\n\nbc"); |
| 307 | ok(boundary.blocks[0].fingerprint !== shifted.blocks[0].fingerprint, "a shorter first block fingerprints differently"); |
| 308 | } |
| 309 | |
| 310 | console.log(`\n${passed} passed, ${failed} failed, ${passed + failed} total`); |
| 311 | if (failed > 0) process.exit(1); |
| 312 |