| 1 | /** |
| 2 | * Fetch external content sources (web articles + GitHub repos) server-side and |
| 3 | * turn them into Markdown the agent can read. |
| 4 | * |
| 5 | * Why server-side: the studio's agents (claude --print / cursor-agent / codex / |
| 6 | * the Messages API) have no network access and only consume a plain-text |
| 7 | * prompt. So when a user pastes a link, the server fetches + flattens it here, |
| 8 | * stores it as a text asset, and lets the existing attachment→prompt pipeline |
| 9 | * feed it to the agent. |
| 10 | * |
| 11 | * Zero runtime deps (matches the CLI package's minimalism): native fetch + |
| 12 | * a lean regex HTML→Markdown pass, GitHub's public REST API for repos. |
| 13 | */ |
| 14 | |
| 15 | const ARTICLE_MAX = 8_000; // chars of markdown kept from an article |
| 16 | const README_MAX = 10_000; // chars of README kept from a repo |
| 17 | const FETCH_TIMEOUT_MS = 12_000; |
| 18 | const UA = |
| 19 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36'; |
| 20 | |
| 21 | export interface FetchedSource { |
| 22 | url: string; |
| 23 | title: string; |
| 24 | markdown: string; |
| 25 | kind: 'article' | 'repo'; |
| 26 | truncated: boolean; |
| 27 | } |
| 28 | |
| 29 | /** Extract up to `max` distinct http(s) URLs from free text (in order). */ |
| 30 | export function extractUrls(text: string, max = 3): string[] { |
| 31 | if (!text) return []; |
| 32 | const re = /https?:\/\/[^\s<>"'`)\]}]+/gi; |
| 33 | const seen = new Set<string>(); |
| 34 | const out: string[] = []; |
| 35 | for (const m of text.matchAll(re)) { |
| 36 | // Trim common trailing punctuation that isn't part of the URL. |
| 37 | const u = m[0].replace(/[.,;:!?]+$/, ''); |
| 38 | if (!seen.has(u)) { |
| 39 | seen.add(u); |
| 40 | out.push(u); |
| 41 | if (out.length >= max) break; |
| 42 | } |
| 43 | } |
| 44 | return out; |
| 45 | } |
| 46 | |
| 47 | /** |
| 48 | * Reject URLs that point at localhost / link-local / private network ranges |
| 49 | * (SSRF guard). Only plain http(s) public hosts are allowed. |
| 50 | */ |
| 51 | export function assertPublicHttpUrl(raw: string): URL { |
| 52 | let u: URL; |
| 53 | try { |
| 54 | u = new URL(raw); |
| 55 | } catch { |
| 56 | throw new Error(`invalid URL: ${raw}`); |
| 57 | } |
| 58 | if (u.protocol !== 'http:' && u.protocol !== 'https:') { |
| 59 | throw new Error(`only http(s) URLs are allowed (got ${u.protocol})`); |
| 60 | } |
| 61 | const host = u.hostname.toLowerCase(); |
| 62 | if ( |
| 63 | host === 'localhost' || |
| 64 | host === '0.0.0.0' || |
| 65 | host === '::1' || |
| 66 | host.endsWith('.localhost') || |
| 67 | host.endsWith('.internal') || |
| 68 | host.endsWith('.local') |
| 69 | ) { |
| 70 | throw new Error(`refusing to fetch local host: ${host}`); |
| 71 | } |
| 72 | // IPv4 private / loopback / link-local ranges. |
| 73 | const m = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); |
| 74 | if (m) { |
| 75 | const [a, b] = [Number(m[1]), Number(m[2])]; |
| 76 | if ( |
| 77 | a === 127 || // loopback |
| 78 | a === 10 || // private |
| 79 | (a === 172 && b >= 16 && b <= 31) || // private |
| 80 | (a === 192 && b === 168) || // private |
| 81 | (a === 169 && b === 254) || // link-local (cloud metadata) |
| 82 | a === 0 |
| 83 | ) { |
| 84 | throw new Error(`refusing to fetch private IP: ${host}`); |
| 85 | } |
| 86 | } |
| 87 | return u; |
| 88 | } |
| 89 | |
| 90 | /** Dispatch: GitHub repo URL → repo summary, anything else → article. */ |
| 91 | export async function fetchSource(rawUrl: string, signal?: AbortSignal): Promise<FetchedSource> { |
| 92 | const u = assertPublicHttpUrl(rawUrl); |
| 93 | const repo = parseGithubRepo(u); |
| 94 | if (repo) return fetchRepo(repo.owner, repo.repo, rawUrl, signal); |
| 95 | return fetchArticle(rawUrl, signal); |
| 96 | } |
| 97 | |
| 98 | // -------------------------------------------------------------------------- |
| 99 | // Article |
| 100 | // -------------------------------------------------------------------------- |
| 101 | |
| 102 | async function fetchArticle(url: string, signal?: AbortSignal): Promise<FetchedSource> { |
| 103 | const html = await fetchText(url, { accept: 'text/html,application/xhtml+xml' }, signal); |
| 104 | const title = extractTitle(html); |
| 105 | let body = htmlToMarkdown(extractMainHtml(html)); |
| 106 | const truncated = body.length > ARTICLE_MAX; |
| 107 | if (truncated) body = body.slice(0, ARTICLE_MAX); |
| 108 | const markdown = `# ${title || url}\n\nSource: ${url}\n\n${body}`.trim(); |
| 109 | return { url, title, markdown, kind: 'article', truncated }; |
| 110 | } |
| 111 | |
| 112 | /** Extract the inner HTML of the first element matching `openTagRe`, scanning |
| 113 | * forward and balancing nested `<tag>`/`</tag>` so we capture the WHOLE |
| 114 | * container — not just up to the first inner close tag. A naive |
| 115 | * `(.*?)</tag>` regex collapses on deeply-nested markup (e.g. WeChat's |
| 116 | * #js_content wraps hundreds of nested <div>/<section>), which is why the |
| 117 | * old single-regex approach returned an almost-empty body. */ |
| 118 | function extractBalanced(html: string, tag: string, openTagRe: RegExp): string | null { |
| 119 | const m = openTagRe.exec(html); |
| 120 | if (!m) return null; |
| 121 | const start = m.index + m[0].length; |
| 122 | const tagRe = new RegExp(`<(/)?${tag}\\b[^>]*>`, 'gi'); |
| 123 | tagRe.lastIndex = start; |
| 124 | let depth = 1; |
| 125 | let t: RegExpExecArray | null; |
| 126 | while ((t = tagRe.exec(html))) { |
| 127 | if (t[1]) { |
| 128 | depth--; |
| 129 | if (depth === 0) return html.slice(start, t.index); |
| 130 | } else if (!/\/>$/.test(t[0])) { |
| 131 | depth++; |
| 132 | } |
| 133 | } |
| 134 | return html.slice(start); // unbalanced — take the rest |
| 135 | } |
| 136 | |
| 137 | /** Prefer the article's main content container when we can spot one |
| 138 | * (WeChat's #js_content, <article>, <main>), else the whole document. */ |
| 139 | function extractMainHtml(html: string): string { |
| 140 | // WeChat official-account articles are server-rendered into #js_content |
| 141 | // (class attribute precedes id, and the open tag spans newlines). |
| 142 | const wx = extractBalanced(html, 'div', /<div[^>]*\bid=["']js_content["'][^>]*>/i); |
| 143 | if (wx && wx.length > 200) return wx; |
| 144 | const article = extractBalanced(html, 'article', /<article[^>]*>/i); |
| 145 | if (article && article.length > 200) return article; |
| 146 | const main = extractBalanced(html, 'main', /<main[^>]*>/i); |
| 147 | if (main && main.length > 200) return main; |
| 148 | // Fall back to <body> so we don't carry <head> noise. |
| 149 | const bodyM = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i); |
| 150 | return bodyM && bodyM[1] ? bodyM[1] : html; |
| 151 | } |
| 152 | |
| 153 | function extractTitle(html: string): string { |
| 154 | const og = html.match(/<meta[^>]+property=["']og:title["'][^>]*content=["']([^"']+)["']/i); |
| 155 | if (og && og[1]) return decodeEntities(og[1]).trim(); |
| 156 | const t = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i); |
| 157 | return t && t[1] ? decodeEntities(t[1]).replace(/\s+/g, ' ').trim() : ''; |
| 158 | } |
| 159 | |
| 160 | /** Lean, dependency-free HTML→Markdown. Not a full converter — just enough to |
| 161 | * give the agent readable prose with headings, lists, links kept. */ |
| 162 | export function htmlToMarkdown(html: string): string { |
| 163 | let s = html; |
| 164 | // Drop non-content elements entirely. |
| 165 | s = s.replace(/<(script|style|noscript|svg|head|nav|footer|form|iframe)[^>]*>[\s\S]*?<\/\1>/gi, ''); |
| 166 | s = s.replace(/<!--[\s\S]*?-->/g, ''); |
| 167 | // Block-ish → newlines / markers. |
| 168 | s = s.replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_m, lvl, inner) => `\n\n${'#'.repeat(Number(lvl))} ${strip(inner)}\n`); |
| 169 | s = s.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, inner) => `\n- ${strip(inner)}`); |
| 170 | s = s.replace(/<(p|div|section|article|tr|h[1-6]|ul|ol|blockquote)[^>]*>/gi, '\n'); |
| 171 | s = s.replace(/<\/(p|div|section|article|tr|li|ul|ol|blockquote)>/gi, '\n'); |
| 172 | s = s.replace(/<br\s*\/?>/gi, '\n'); |
| 173 | // Inline: links + images keep their target. |
| 174 | s = s.replace(/<a[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_m, href, inner) => { |
| 175 | const text = strip(inner); |
| 176 | return text ? `[${text}](${href})` : ''; |
| 177 | }); |
| 178 | s = s.replace(/<img[^>]*alt=["']([^"']*)["'][^>]*src=["']([^"']+)["'][^>]*>/gi, (_m, alt, src) => ``); |
| 179 | s = s.replace(/<img[^>]*src=["']([^"']+)["'][^>]*>/gi, (_m, src) => ``); |
| 180 | // Strip every remaining tag. |
| 181 | s = s.replace(/<[^>]+>/g, ''); |
| 182 | s = decodeEntities(s); |
| 183 | // Collapse whitespace: trim each line, drop 3+ blank lines. |
| 184 | s = s |
| 185 | .split('\n') |
| 186 | .map((l) => l.replace(/[ \t ]+/g, ' ').trimEnd()) |
| 187 | .join('\n') |
| 188 | .replace(/\n{3,}/g, '\n\n') |
| 189 | .trim(); |
| 190 | return s; |
| 191 | } |
| 192 | |
| 193 | /** Strip tags + decode entities + collapse spaces (for inline fragments). */ |
| 194 | function strip(html: string): string { |
| 195 | return decodeEntities(html.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim(); |
| 196 | } |
| 197 | |
| 198 | function decodeEntities(s: string): string { |
| 199 | return s |
| 200 | .replace(/ /g, ' ') |
| 201 | .replace(/&/g, '&') |
| 202 | .replace(/</g, '<') |
| 203 | .replace(/>/g, '>') |
| 204 | .replace(/"/g, '"') |
| 205 | .replace(/�?39;/g, "'") |
| 206 | .replace(/'/g, "'") |
| 207 | .replace(/&#x([0-9a-f]+);/gi, (_m, h) => safeCodePoint(parseInt(h, 16))) |
| 208 | .replace(/&#(\d+);/g, (_m, d) => safeCodePoint(parseInt(d, 10))); |
| 209 | } |
| 210 | |
| 211 | function safeCodePoint(n: number): string { |
| 212 | try { |
| 213 | return Number.isFinite(n) && n > 0 && n <= 0x10ffff ? String.fromCodePoint(n) : ''; |
| 214 | } catch { |
| 215 | return ''; |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // -------------------------------------------------------------------------- |
| 220 | // GitHub repo |
| 221 | // -------------------------------------------------------------------------- |
| 222 | |
| 223 | function parseGithubRepo(u: URL): { owner: string; repo: string } | null { |
| 224 | if (u.hostname.toLowerCase() !== 'github.com') return null; |
| 225 | const parts = u.pathname.split('/').filter(Boolean); |
| 226 | if (parts.length < 2) return null; |
| 227 | // Skip non-repo paths (search, marketplace, etc. have reserved first segments, |
| 228 | // but a 2-segment owner/repo is the common case; reject known non-repo roots). |
| 229 | const reserved = new Set(['search', 'marketplace', 'topics', 'collections', 'sponsors', 'about', 'features']); |
| 230 | if (reserved.has(parts[0]!.toLowerCase())) return null; |
| 231 | return { owner: parts[0]!, repo: parts[1]!.replace(/\.git$/, '') }; |
| 232 | } |
| 233 | |
| 234 | async function fetchRepo(owner: string, repo: string, url: string, signal?: AbortSignal): Promise<FetchedSource> { |
| 235 | const api = `https://api.github.com/repos/${owner}/${repo}`; |
| 236 | const ghHeaders = { accept: 'application/vnd.github+json', 'x-github-api-version': '2022-11-28' }; |
| 237 | |
| 238 | // Repo metadata (required — fails loudly if repo is private/missing). |
| 239 | const metaRaw = await fetchText(api, ghHeaders, signal); |
| 240 | const meta = JSON.parse(metaRaw) as { |
| 241 | full_name?: string; |
| 242 | description?: string; |
| 243 | language?: string; |
| 244 | stargazers_count?: number; |
| 245 | topics?: string[]; |
| 246 | license?: { spdx_id?: string }; |
| 247 | homepage?: string; |
| 248 | }; |
| 249 | |
| 250 | // README (raw) + top-level tree — best-effort, don't fail the whole thing. |
| 251 | const readme = await fetchText(`${api}/readme`, { ...ghHeaders, accept: 'application/vnd.github.raw' }, signal).catch( |
| 252 | () => '', |
| 253 | ); |
| 254 | const tree = await fetchTopLevelTree(api, ghHeaders, signal).catch(() => [] as string[]); |
| 255 | |
| 256 | const title = meta.full_name || `${owner}/${repo}`; |
| 257 | const lines: string[] = [`# ${title}`, '', `Source: ${url}`, '']; |
| 258 | if (meta.description) lines.push(`> ${meta.description}`, ''); |
| 259 | const facts: string[] = []; |
| 260 | if (meta.language) facts.push(`Language: ${meta.language}`); |
| 261 | if (typeof meta.stargazers_count === 'number') facts.push(`Stars: ${meta.stargazers_count.toLocaleString('en-US')}`); |
| 262 | if (meta.license?.spdx_id && meta.license.spdx_id !== 'NOASSERTION') facts.push(`License: ${meta.license.spdx_id}`); |
| 263 | if (meta.homepage) facts.push(`Homepage: ${meta.homepage}`); |
| 264 | if (meta.topics?.length) facts.push(`Topics: ${meta.topics.join(', ')}`); |
| 265 | if (facts.length) lines.push(...facts.map((f) => `- ${f}`), ''); |
| 266 | if (tree.length) { |
| 267 | lines.push('## Top-level structure', '', ...tree.map((t) => `- ${t}`), ''); |
| 268 | } |
| 269 | |
| 270 | let readmeMd = readme.trim(); |
| 271 | const truncated = readmeMd.length > README_MAX; |
| 272 | if (truncated) readmeMd = readmeMd.slice(0, README_MAX); |
| 273 | if (readmeMd) lines.push('## README', '', readmeMd); |
| 274 | |
| 275 | return { url, title, markdown: lines.join('\n').trim(), kind: 'repo', truncated }; |
| 276 | } |
| 277 | |
| 278 | async function fetchTopLevelTree( |
| 279 | api: string, |
| 280 | headers: Record<string, string>, |
| 281 | signal?: AbortSignal, |
| 282 | ): Promise<string[]> { |
| 283 | const raw = await fetchText(`${api}/contents`, headers, signal); |
| 284 | const items = JSON.parse(raw) as { name?: string; type?: string }[]; |
| 285 | return items |
| 286 | .filter((i) => i.name) |
| 287 | .slice(0, 40) |
| 288 | .map((i) => (i.type === 'dir' ? `${i.name}/` : i.name!)); |
| 289 | } |
| 290 | |
| 291 | // -------------------------------------------------------------------------- |
| 292 | // shared fetch |
| 293 | // -------------------------------------------------------------------------- |
| 294 | |
| 295 | async function fetchText(url: string, extraHeaders: Record<string, string>, signal?: AbortSignal): Promise<string> { |
| 296 | assertPublicHttpUrl(url); |
| 297 | const res = await fetch(url, { |
| 298 | headers: { 'user-agent': UA, ...extraHeaders }, |
| 299 | redirect: 'follow', |
| 300 | signal: signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS), |
| 301 | }); |
| 302 | if (!res.ok) { |
| 303 | throw new Error(`fetch ${url} → HTTP ${res.status}`); |
| 304 | } |
| 305 | return res.text(); |
| 306 | } |
| 307 |