| 1 | // Deterministic pre-pass that repairs LLM-typical math syntax before |
| 2 | // remark-math parses Markdown. Semantic inline classification lives in |
| 3 | // remarkMathPolicy, where code boundaries and surrounding AST context are |
| 4 | // already known. |
| 5 | // |
| 6 | // 1. Protect Markdown code spans/fences from all math rewrites. |
| 7 | // 2. Protect LaTeX line-break spacing (\\[...]) from the LLM-delimiter rewrite. |
| 8 | // 3. \(...)/\[...] → $/$$. |
| 9 | // 4. Expand \yng/\young to KaTeX-compatible \boxed{array} forms. |
| 10 | // Stateful: tracks `$…$` so bare macros in prose get wrapped in |
| 11 | // `$…$` and macros already inside math just substitute. |
| 12 | // 5. Inline `$$` glued to prose gets a blank line inserted before it |
| 13 | // (CommonMark requires that block math be paragraph-separated). |
| 14 | // 6. Protect pipes inside inline math before GFM table tokenisation. |
| 15 | // 7. Restore placeholders for remark-math. KaTeX-specific normalisation is |
| 16 | // handled by the AST policy after parsing. |
| 17 | |
| 18 | import { expandYoungDiagrams } from "./youngDiagrams"; |
| 19 | |
| 20 | // Matches $\cmd{...}...$ where the body may contain $ and one level of nested |
| 21 | // braces. Group 1 captures the full \cmd{...} including the outer }. After |
| 22 | // the closing }, [^$]*? consumes any trailing content (e.g. " + x^2") up to |
| 23 | // the closing $, so patterns like $\text{a} + x^2$ are handled as a whole |
| 24 | // rather than split at stray $ signs inside \text{}. |
| 25 | const TEXT_MODE_PAIR = /\$\s*(\\[A-Za-z]+\{(?:[^{}]|\{[^{}]*\})*\}[^$]*?)\s*\$/g; |
| 26 | |
| 27 | const DM = "__REASONIX_MATH_DISPLAY__"; |
| 28 | const IM = "__REASONIX_MATH_INLINE__"; |
| 29 | const LB = "__REASONIX_LATEX_LINEBREAK__"; |
| 30 | const ED_BASE = "REASONIXESCAPEDDOLLAR"; |
| 31 | const INLINE_SOURCE_PREFIX = "\\reasonixInternalSourceV1{"; |
| 32 | const INLINE_RENDER_SOURCE_PREFIX = "\\reasonixInternalRenderV1"; |
| 33 | const INLINE_RENDER_SOURCE_RE = /\\reasonixInternalRenderV1\{([^}]*)\}\{([^}]*)\}/g; |
| 34 | |
| 35 | export function normalizeMath(s: string): string { |
| 36 | const protectedCode = protectMarkdownCode(s); |
| 37 | let r = normalizeMathText(protectedCode.text); |
| 38 | for (let i = 0; i < protectedCode.segments.length; i += 1) { |
| 39 | r = r.split(`${protectedCode.prefix}${i}__`).join(protectedCode.segments[i]); |
| 40 | } |
| 41 | return r; |
| 42 | } |
| 43 | |
| 44 | function normalizeMathText(s: string): string { |
| 45 | // Step 1: protect LaTeX line-break spacing (\\[4pt], \\[2ex], ...) so the |
| 46 | // \[ → $$ rewrite below doesn't swallow it. |
| 47 | let r = s.replace(/\\\\\[/g, LB); |
| 48 | |
| 49 | // Step 2: convert LLM-native delimiters to standard $/$$ syntax. Arrow |
| 50 | // functions are required because "$$" in a JS replace string means a |
| 51 | // single literal $. |
| 52 | r = r |
| 53 | .replace(/\\\[/g, () => "$$") |
| 54 | .replace(/\\\]/g, () => "$$") |
| 55 | .replace(/\\\(/g, () => "$") |
| 56 | .replace(/\\\)/g, () => "$"); |
| 57 | r = r.replace(new RegExp(LB, "g"), "\\\\["); |
| 58 | |
| 59 | // Step 2.5: expand \yng/\young macros to KaTeX-compatible \boxed{array} |
| 60 | // forms after LLM-native delimiters have been converted, so macros inside |
| 61 | // \(...\) or \[...\] are correctly recognised as already being in math. |
| 62 | r = expandYoungDiagrams(r, ({ source, rendered }) => { |
| 63 | return protectInlineMathRenderSource(source, rendered); |
| 64 | }); |
| 65 | |
| 66 | // Escaped dollars are literal prose dollars, not math delimiters. Hide them |
| 67 | // while the structural dollar scans below run, then restore them before |
| 68 | // remark-math parses the result. |
| 69 | const escapedDollarToken = unusedEscapedDollarToken(r); |
| 70 | r = r.split("\\$").join(escapedDollarToken); |
| 71 | |
| 72 | // Step 3+4: normalise display $$ block structure. remark-math requires |
| 73 | // opening and closing $$ to sit on their own lines; LLMs often emit |
| 74 | // single-line displays, opening fences glued to prose, and adjacent display |
| 75 | // blocks separated by prose. A line parser avoids the old cross-block regex |
| 76 | // capture that swallowed prose between two display blocks. |
| 77 | r = normaliseDisplayBlocks(r); |
| 78 | |
| 79 | // Step 5: preserve the complete source of $\cmd{...}$ pairs containing a |
| 80 | // stray inner $ (e.g. $\text{price is $5}$) in a parser-safe marker. The |
| 81 | // AST policy restores and normalises it after remark-math has established |
| 82 | // the real formula boundary. |
| 83 | r = r.replace(TEXT_MODE_PAIR, (match, math: string) => { |
| 84 | return math.includes("$") ? `${IM}${protectInlineMathSource(math)}${IM}` : match; |
| 85 | }); |
| 86 | |
| 87 | // Step 6: GFM identifies table cells before remark plugins can transform the |
| 88 | // AST. Normalise only inline spans containing pipes at this syntax boundary |
| 89 | // so formulas such as `$|x|$` cannot be split into separate cells. A pipe is |
| 90 | // already an explicit math signal in the semantic classifier; all other |
| 91 | // inline decisions remain deferred to remarkMathPolicy. |
| 92 | r = protectInlineMathPipesForGfm(r); |
| 93 | |
| 94 | // Step 7: restore standard $/$$ delimiters for remark-math to parse. |
| 95 | return r |
| 96 | .replace(new RegExp(DM, "g"), () => "$$") |
| 97 | .replace(new RegExp(IM, "g"), "$") |
| 98 | .split(escapedDollarToken).join("\\$"); |
| 99 | } |
| 100 | |
| 101 | function protectInlineMathPipesForGfm(s: string): string { |
| 102 | return s.replace(/\$([^$\n]+)\$/g, (match, math: string) => { |
| 103 | let hasUnescapedPipe = false; |
| 104 | let backslashes = 0; |
| 105 | for (const char of math) { |
| 106 | if (char === "|" && backslashes % 2 === 0) { |
| 107 | hasUnescapedPipe = true; |
| 108 | break; |
| 109 | } |
| 110 | backslashes = char === "\\" ? backslashes + 1 : 0; |
| 111 | } |
| 112 | return hasUnescapedPipe ? `$${protectInlineMathSource(math)}$` : match; |
| 113 | }); |
| 114 | } |
| 115 | |
| 116 | function protectInlineMathSource(source: string): string { |
| 117 | return `${INLINE_SOURCE_PREFIX}${encodeURIComponent(source)}}`; |
| 118 | } |
| 119 | |
| 120 | function protectInlineMathRenderSource(source: string, rendered: string): string { |
| 121 | return `${INLINE_RENDER_SOURCE_PREFIX}{${encodeURIComponent(source)}}{${encodeURIComponent(rendered)}}`; |
| 122 | } |
| 123 | |
| 124 | export interface ResolvedInlineMathSource { |
| 125 | source: string; |
| 126 | rendered: string; |
| 127 | } |
| 128 | |
| 129 | export function resolveProtectedInlineMathSource(value: string): ResolvedInlineMathSource { |
| 130 | let payload = value; |
| 131 | if (payload.startsWith(INLINE_SOURCE_PREFIX) && payload.endsWith("}")) { |
| 132 | const encoded = payload.slice(INLINE_SOURCE_PREFIX.length, -1); |
| 133 | try { |
| 134 | payload = decodeURIComponent(encoded); |
| 135 | } catch { |
| 136 | payload = value; |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | const replaceRenderSource = (part: "source" | "rendered"): string => { |
| 141 | return payload.replace( |
| 142 | INLINE_RENDER_SOURCE_RE, |
| 143 | (match, encodedSource: string, encodedRendered: string) => { |
| 144 | try { |
| 145 | return decodeURIComponent(part === "source" ? encodedSource : encodedRendered); |
| 146 | } catch { |
| 147 | return match; |
| 148 | } |
| 149 | }, |
| 150 | ); |
| 151 | }; |
| 152 | |
| 153 | return { |
| 154 | source: replaceRenderSource("source"), |
| 155 | rendered: replaceRenderSource("rendered"), |
| 156 | }; |
| 157 | } |
| 158 | |
| 159 | export function restoreProtectedInlineMathSource(source: string): string { |
| 160 | return resolveProtectedInlineMathSource(source).source; |
| 161 | } |
| 162 | |
| 163 | function unusedEscapedDollarToken(s: string): string { |
| 164 | let token = ED_BASE; |
| 165 | let n = 0; |
| 166 | while (s.includes(token)) { |
| 167 | n += 1; |
| 168 | token = `${ED_BASE}${n}`; |
| 169 | } |
| 170 | return token; |
| 171 | } |
| 172 | |
| 173 | function protectMarkdownCode(s: string): { text: string; prefix: string; segments: string[] } { |
| 174 | const prefix = unusedPlaceholderPrefix(s); |
| 175 | const segments: string[] = []; |
| 176 | let out = ""; |
| 177 | let i = 0; |
| 178 | |
| 179 | const pushSegment = (segment: string) => { |
| 180 | const token = `${prefix}${segments.length}__`; |
| 181 | segments.push(segment); |
| 182 | out += token; |
| 183 | }; |
| 184 | |
| 185 | while (i < s.length) { |
| 186 | const fenceEnd = fencedCodeEnd(s, i); |
| 187 | if (fenceEnd > i) { |
| 188 | pushSegment(s.slice(i, fenceEnd)); |
| 189 | i = fenceEnd; |
| 190 | continue; |
| 191 | } |
| 192 | |
| 193 | if (s[i] === "`") { |
| 194 | const tickEnd = inlineCodeEnd(s, i); |
| 195 | if (tickEnd > i) { |
| 196 | pushSegment(s.slice(i, tickEnd)); |
| 197 | i = tickEnd; |
| 198 | continue; |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | out += s[i]; |
| 203 | i += 1; |
| 204 | } |
| 205 | |
| 206 | return { text: out, prefix, segments }; |
| 207 | } |
| 208 | |
| 209 | function unusedPlaceholderPrefix(s: string): string { |
| 210 | let prefix = "__REASONIX_PROTECTED_CODE__"; |
| 211 | let n = 0; |
| 212 | while (s.includes(prefix)) { |
| 213 | n += 1; |
| 214 | prefix = `__REASONIX_PROTECTED_CODE_${n}__`; |
| 215 | } |
| 216 | return prefix; |
| 217 | } |
| 218 | |
| 219 | function fencedCodeEnd(s: string, start: number): number { |
| 220 | // Fence must be at the start of a line (or the document) — CommonMark |
| 221 | // requirement. Allowing mid-line fences would swallow prose like |
| 222 | // "wrap code in ```blocks``` here" into the code region. |
| 223 | if (start !== 0 && s[start - 1] !== "\n") return -1; |
| 224 | |
| 225 | let markerStart = start; |
| 226 | let spaces = 0; |
| 227 | while (spaces < 4 && s[markerStart] === " ") { |
| 228 | markerStart += 1; |
| 229 | spaces += 1; |
| 230 | } |
| 231 | |
| 232 | const marker = s[markerStart]; |
| 233 | if (marker !== "`" && marker !== "~") return -1; |
| 234 | |
| 235 | let fenceLen = 0; |
| 236 | while (s[markerStart + fenceLen] === marker) fenceLen += 1; |
| 237 | if (fenceLen < 3) return -1; |
| 238 | |
| 239 | const openingLineEnd = lineEnd(s, markerStart + fenceLen); |
| 240 | |
| 241 | // Single-line doc: treat the next matching fence as the closing fence. |
| 242 | if (openingLineEnd >= s.length) { |
| 243 | const fencePattern = marker.repeat(fenceLen); |
| 244 | const nextFence = s.indexOf(fencePattern, markerStart + fenceLen); |
| 245 | if (nextFence === -1) return s.length; |
| 246 | return nextFence + fenceLen; |
| 247 | } |
| 248 | |
| 249 | let lineStart = openingLineEnd + 1; |
| 250 | while (lineStart < s.length) { |
| 251 | const currentLineEnd = lineEnd(s, lineStart); |
| 252 | if (isClosingFenceLine(s, lineStart, currentLineEnd, marker, fenceLen)) { |
| 253 | return currentLineEnd < s.length ? currentLineEnd + 1 : currentLineEnd; |
| 254 | } |
| 255 | lineStart = currentLineEnd < s.length ? currentLineEnd + 1 : currentLineEnd; |
| 256 | } |
| 257 | |
| 258 | return s.length; |
| 259 | } |
| 260 | |
| 261 | function isClosingFenceLine(s: string, start: number, end: number, marker: string, minLen: number): boolean { |
| 262 | let i = start; |
| 263 | let spaces = 0; |
| 264 | while (spaces < 4 && s[i] === " ") { |
| 265 | i += 1; |
| 266 | spaces += 1; |
| 267 | } |
| 268 | |
| 269 | let count = 0; |
| 270 | while (s[i + count] === marker) count += 1; |
| 271 | if (count < minLen) return false; |
| 272 | |
| 273 | for (let j = i + count; j < end; j += 1) { |
| 274 | if (s[j] !== " " && s[j] !== "\t") return false; |
| 275 | } |
| 276 | return true; |
| 277 | } |
| 278 | |
| 279 | function inlineCodeEnd(s: string, start: number): number { |
| 280 | let tickLen = 0; |
| 281 | while (s[start + tickLen] === "`") tickLen += 1; |
| 282 | |
| 283 | const ticks = "`".repeat(tickLen); |
| 284 | const end = s.indexOf(ticks, start + tickLen); |
| 285 | return end < 0 ? -1 : end + tickLen; |
| 286 | } |
| 287 | |
| 288 | function lineEnd(s: string, start: number): number { |
| 289 | const end = s.indexOf("\n", start); |
| 290 | return end < 0 ? s.length : end; |
| 291 | } |
| 292 | |
| 293 | function normaliseDisplayBlocks(s: string): string { |
| 294 | const lines = s.split("\n"); |
| 295 | const out: string[] = []; |
| 296 | let i = 0; |
| 297 | |
| 298 | while (i < lines.length) { |
| 299 | const line = lines[i]; |
| 300 | const $$idx = line.indexOf("$$"); |
| 301 | |
| 302 | if ($$idx >= 0 && line.indexOf("$$", $$idx + 2) >= 0 |
| 303 | && !($$idx > 0 && /\d/.test(line[$$idx - 1]))) { |
| 304 | const m = line.match(/^(.*?)\$\$([^\n]*?)\$\$(.*)$/); |
| 305 | if (m) { |
| 306 | const quote = blockquotePrefix(m[1]); |
| 307 | pushDisplayBefore(out, m[1]); |
| 308 | out.push(DM); |
| 309 | out.push(m[2]); |
| 310 | out.push(DM); |
| 311 | if (m[3]) out.push(normaliseDisplayBlocks(quote ? quote + m[3].trimStart() : m[3])); |
| 312 | i += 1; |
| 313 | continue; |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | if ($$idx >= 0 && line.indexOf("$$", $$idx + 2) < 0 |
| 318 | && !($$idx > 0 && /\d/.test(line[$$idx - 1]))) { |
| 319 | const before = line.slice(0, $$idx); |
| 320 | const afterOpen = line.slice($$idx + 2); |
| 321 | const quote = blockquotePrefix(before); |
| 322 | |
| 323 | const formulaLines: string[] = []; |
| 324 | if (afterOpen) formulaLines.push(afterOpen); |
| 325 | |
| 326 | let j = i + 1; |
| 327 | let found = false; |
| 328 | while (j < lines.length) { |
| 329 | const rawLine = lines[j]; |
| 330 | const fLine = quote ? stripBlockquotePrefix(rawLine, quote) : rawLine; |
| 331 | const closeIdx = fLine.indexOf("$$"); |
| 332 | if (closeIdx >= 0 && fLine.indexOf("$$", closeIdx + 2) < 0) { |
| 333 | const formulaPart = fLine.slice(0, closeIdx); |
| 334 | const afterClose = fLine.slice(closeIdx + 2); |
| 335 | pushDisplayBefore(out, before); |
| 336 | if (formulaPart) formulaLines.push(formulaPart); |
| 337 | const formula = formulaLines.join("\n"); |
| 338 | out.push(DM); |
| 339 | out.push(formula); |
| 340 | out.push(DM); |
| 341 | if (afterClose) out.push(quote ? quote + afterClose.trimStart() : afterClose); |
| 342 | i = j + 1; |
| 343 | found = true; |
| 344 | break; |
| 345 | } |
| 346 | formulaLines.push(fLine); |
| 347 | j += 1; |
| 348 | } |
| 349 | |
| 350 | if (found) continue; |
| 351 | pushDisplayBefore(out, before); |
| 352 | out.push("$$"); |
| 353 | if (afterOpen) out.push(afterOpen); |
| 354 | i += 1; |
| 355 | continue; |
| 356 | } |
| 357 | |
| 358 | out.push(line); |
| 359 | i += 1; |
| 360 | } |
| 361 | |
| 362 | return out.join("\n"); |
| 363 | } |
| 364 | |
| 365 | function pushDisplayBefore(out: string[], before: string): void { |
| 366 | if (!before.trim()) return; |
| 367 | out.push(before); |
| 368 | } |
| 369 | |
| 370 | function blockquotePrefix(before: string): string | null { |
| 371 | const m = before.match(/^(\s*>\s*)/); |
| 372 | return m ? m[1] : null; |
| 373 | } |
| 374 | |
| 375 | function stripBlockquotePrefix(line: string, prefix: string): string { |
| 376 | if (line.startsWith(prefix)) return line.slice(prefix.length); |
| 377 | const marker = prefix.trimEnd(); |
| 378 | if (marker && line.startsWith(marker)) { |
| 379 | const rest = line.slice(marker.length); |
| 380 | return rest.startsWith(" ") ? rest.slice(1) : rest; |
| 381 | } |
| 382 | return line; |
| 383 | } |
| 384 |