| 1 | /** |
| 2 | * content-watch.ts — two daily watchers that catch site drift the mechanical |
| 3 | * facts pipeline misses: |
| 4 | * |
| 5 | * runLinkCheck — pings every external URL referenced in the site copy, |
| 6 | * writes a draft per broken link (4xx/5xx). Stores a |
| 7 | * `linkcheck:last` summary so /admin can show last status. |
| 8 | * |
| 9 | * runSemanticDrift — reads recent CHANGELOG / commits, asks deepseek-flash |
| 10 | * whether any specific claims on the site look out of |
| 11 | * date, writes review-required drafts. |
| 12 | * |
| 13 | * Both surface as drafts in CURATED_KV under `draft:linkcheck:<...>` and |
| 14 | * `draft:semantic-drift:<...>`, picked up by the existing /admin listing. |
| 15 | */ |
| 16 | import { fetchBoundedText } from "./bounded-body"; |
| 17 | import { agentChat, draftStorageKey, getDraft, saveDraft, type AgentDraft, type DeepSeekEnv, VOICE_CONSTRAINTS } from "./community-agent"; |
| 18 | |
| 19 | interface KVNamespace { |
| 20 | get(k: string): Promise<string | null>; |
| 21 | put(k: string, v: string, o?: { expirationTtl?: number }): Promise<void>; |
| 22 | list(o?: { prefix?: string; limit?: number }): Promise<{ keys: { name: string }[] }>; |
| 23 | delete(k: string): Promise<void>; |
| 24 | } |
| 25 | |
| 26 | // --- Canonical draft identities --- |
| 27 | // |
| 28 | // Watcher draft IDs are deterministic: a readable slug plus a ~64-bit |
| 29 | // SHA-256 suffix over the finding's full identity, capped at 80 characters. |
| 30 | // Re-running a watcher over an unchanged finding reproduces the same key (so |
| 31 | // the open-draft dedup check skips it); any change to the finding's identity |
| 32 | // produces a new key (so changed findings are drafted again instead of being |
| 33 | // silently swallowed by a truncated-prefix collision). |
| 34 | const WATCH_DRAFT_ID_MAX = 80; |
| 35 | const WATCH_DRAFT_HASH_HEX = 16; // 64 bits of SHA-256 |
| 36 | |
| 37 | async function sha256Hex(input: string): Promise<string> { |
| 38 | const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input)); |
| 39 | return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| 40 | } |
| 41 | |
| 42 | function slugify(input: string): string { |
| 43 | return input.replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").toLowerCase(); |
| 44 | } |
| 45 | |
| 46 | export async function watchDraftId(slugSource: string, identity: string): Promise<string> { |
| 47 | const hash = (await sha256Hex(identity)).slice(0, WATCH_DRAFT_HASH_HEX); |
| 48 | const slug = slugify(slugSource).slice(0, WATCH_DRAFT_ID_MAX - WATCH_DRAFT_HASH_HEX - 1); |
| 49 | return `${slug}-${hash}`; |
| 50 | } |
| 51 | |
| 52 | interface WatchEnv { |
| 53 | CURATED_KV?: KVNamespace; |
| 54 | DEEPSEEK_API_KEY?: string; |
| 55 | DEEPSEEK_BASE_URL?: string; |
| 56 | DEEPSEEK_MODEL?: string; |
| 57 | GITHUB_TOKEN?: string; |
| 58 | } |
| 59 | |
| 60 | function dsEnv(env: WatchEnv): DeepSeekEnv { |
| 61 | return { |
| 62 | baseUrl: env.DEEPSEEK_BASE_URL ?? process.env.DEEPSEEK_BASE_URL, |
| 63 | model: env.DEEPSEEK_MODEL ?? process.env.DEEPSEEK_MODEL, |
| 64 | }; |
| 65 | } |
| 66 | |
| 67 | // --- Link checker --- |
| 68 | |
| 69 | // Targets to probe daily. For registries that block bot HEAD/GET (npm, crates.io) |
| 70 | // we hit the public JSON API instead — same upstream, doesn't 403. |
| 71 | const LINK_TARGETS: { url: string; label: string }[] = [ |
| 72 | { url: "https://github.com/codewhale-hq/CodeWhale", label: "Main repo" }, |
| 73 | { url: "https://github.com/codewhale-hq/CodeWhale/issues", label: "Issues" }, |
| 74 | { url: "https://github.com/codewhale-hq/CodeWhale/pulls", label: "Pull Requests" }, |
| 75 | { url: "https://github.com/codewhale-hq/CodeWhale/discussions", label: "Discussions" }, |
| 76 | { url: "https://github.com/codewhale-hq/CodeWhale/releases", label: "Releases" }, |
| 77 | { url: "https://github.com/codewhale-hq/CodeWhale/blob/main/LICENSE", label: "License file" }, |
| 78 | { url: "https://github.com/codewhale-hq/CodeWhale/blob/main/.github/CODE_OF_CONDUCT.md", label: "Code of Conduct" }, |
| 79 | { url: "https://github.com/codewhale-hq/CodeWhale/blob/main/.github/SECURITY.md", label: "Security policy" }, |
| 80 | { url: "https://github.com/codewhale-hq/CodeWhale/blob/main/CONTRIBUTING.md", label: "Contributing guide" }, |
| 81 | { url: "https://github.com/codewhale-hq/CodeWhale/blob/main/.github/PULL_REQUEST_TEMPLATE.md", label: "PR template" }, |
| 82 | { url: "https://github.com/Hmbown/homebrew-deepseek-tui", label: "Homebrew tap" }, |
| 83 | { url: "https://github.com/sponsors/Hmbown", label: "Support link (GitHub Sponsors)" }, |
| 84 | { url: "https://buymeacoffee.com/hmbown", label: "Support link (BMC)" }, |
| 85 | { url: "https://registry.npmjs.org/codewhale", label: "npm package (registry API)" }, |
| 86 | // crates.io intentionally not in this list — both their HTML and JSON API return 403 to |
| 87 | // Cloudflare Workers, so the check produces false positives. The crate links on the site |
| 88 | // still work for human users. |
| 89 | ]; |
| 90 | |
| 91 | export interface LinkCheckResult { |
| 92 | url: string; |
| 93 | label: string; |
| 94 | status: number | "error"; |
| 95 | ok: boolean; |
| 96 | ms: number; |
| 97 | } |
| 98 | |
| 99 | const PROBE_TIMEOUT_MS = 10_000; |
| 100 | |
| 101 | async function probe(target: { url: string; label: string }): Promise<LinkCheckResult> { |
| 102 | const start = Date.now(); |
| 103 | try { |
| 104 | // Use HEAD where possible; fall back to GET on 405/403 since some hosts |
| 105 | // (e.g. Cloudflare-protected) reject HEAD. |
| 106 | // Each request has its own deadline; only the status matters, so a GET |
| 107 | // body is discarded unread. |
| 108 | let r = await fetch(target.url, { method: "HEAD", redirect: "follow", signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) }); |
| 109 | if (r.status === 405 || r.status === 403 || r.status === 404) { |
| 110 | // Some sites return 404 to HEAD but 200 to GET (e.g. NPM) |
| 111 | r = await fetch(target.url, { method: "GET", redirect: "follow", signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) }); |
| 112 | await r.body?.cancel().catch(() => undefined); |
| 113 | } |
| 114 | return { url: target.url, label: target.label, status: r.status, ok: r.ok, ms: Date.now() - start }; |
| 115 | } catch { |
| 116 | return { url: target.url, label: target.label, status: "error", ok: false, ms: Date.now() - start }; |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | export async function runLinkCheck(env: WatchEnv): Promise<{ ok: boolean; checked: number; broken: number; results?: LinkCheckResult[] }> { |
| 121 | if (!env.CURATED_KV) return { ok: false, checked: 0, broken: 0 }; |
| 122 | |
| 123 | const results = await Promise.all(LINK_TARGETS.map(probe)); |
| 124 | const broken = results.filter((r) => !r.ok); |
| 125 | |
| 126 | await env.CURATED_KV.put("linkcheck:last", JSON.stringify({ |
| 127 | at: new Date().toISOString(), |
| 128 | checked: results.length, |
| 129 | broken: broken.length, |
| 130 | results, |
| 131 | }), { expirationTtl: 60 * 60 * 24 * 14 }); |
| 132 | |
| 133 | // Write drafts ONLY for new breakages — dedup on the canonical draft key, |
| 134 | // derived from the full URL identity (the hash suffix keeps long URLs with |
| 135 | // identical slug prefixes distinct). |
| 136 | for (const b of broken) { |
| 137 | const id = await watchDraftId(b.url, `linkcheck\n${b.url}`); |
| 138 | const draft: AgentDraft = { |
| 139 | id, |
| 140 | type: "linkcheck", |
| 141 | targetUrl: b.url, |
| 142 | bodyEn: `**Broken link** (auto-detected by daily watch cron)\n\n- Label: **${b.label}**\n- URL: ${b.url}\n- HTTP status: ${b.status}\n- Latency: ${b.ms}ms\n\nThis URL is referenced in codewhale.net copy. Update the source page or fix the destination.\n\n— drafted by community assistant, pending maintainer review`, |
| 143 | bodyZh: `**链接失效**（每日巡检自动发现）\n\n- 名称：**${b.label}**\n- 地址：${b.url}\n- HTTP 状态：${b.status}\n- 延迟：${b.ms}ms\n\n该地址被 codewhale.net 文案引用，请更新源页面或修复目标。\n\n— 由社区助理草拟，待维护者审阅`, |
| 144 | generatedAt: new Date().toISOString(), |
| 145 | posted: false, |
| 146 | }; |
| 147 | const existing = await getDraft(env.CURATED_KV, draftStorageKey(draft)); |
| 148 | if (existing) continue; // already flagged; don't churn |
| 149 | |
| 150 | await saveDraft(env.CURATED_KV, draft); |
| 151 | } |
| 152 | |
| 153 | return { ok: true, checked: results.length, broken: broken.length, results: broken }; |
| 154 | } |
| 155 | |
| 156 | // --- Semantic drift --- |
| 157 | |
| 158 | const SEMANTIC_DRIFT_PROMPT = `You are reviewing copy on a community website (codewhale.net) for the open-source Codewhale project. |
| 159 | |
| 160 | Given: |
| 161 | 1. The CHANGELOG entries below (most recent first) |
| 162 | 2. The current homepage and docs page text below |
| 163 | 3. Recent commit messages |
| 164 | |
| 165 | Identify any factual claims on the site that are CONTRADICTED by recent changes. Be conservative — only flag claims you can directly tie to a CHANGELOG line or commit. Don't speculate. |
| 166 | |
| 167 | Return ONLY this JSON shape (no prose, no markdown fences): |
| 168 | { |
| 169 | "drifts": [ |
| 170 | { |
| 171 | "page": "homepage" | "docs" | "install" | "contribute" | "roadmap", |
| 172 | "claim": "exact text on the site that is now inaccurate", |
| 173 | "evidence": "the CHANGELOG line or commit hash that contradicts it", |
| 174 | "suggested_replacement": "what the site should say instead" |
| 175 | } |
| 176 | ] |
| 177 | } |
| 178 | |
| 179 | If nothing is drifted, return { "drifts": [] }. |
| 180 | |
| 181 | ${VOICE_CONSTRAINTS}`; |
| 182 | |
| 183 | function startsWithAsciiCI(input: string, index: number, needle: string): boolean { |
| 184 | if (index + needle.length > input.length) return false; |
| 185 | return input.slice(index, index + needle.length).toLowerCase() === needle; |
| 186 | } |
| 187 | |
| 188 | function isWhitespace(c: string | undefined): boolean { |
| 189 | return c === " " || c === "\n" || c === "\r" || c === "\t" || c === "\f"; |
| 190 | } |
| 191 | |
| 192 | function tagNameBoundary(input: string, index: number): boolean { |
| 193 | const c = input[index]; |
| 194 | return c === undefined || c === ">" || c === "/" || isWhitespace(c); |
| 195 | } |
| 196 | |
| 197 | function findClosingRawTextTag(input: string, from: number, tagName: "script" | "style"): number { |
| 198 | const closePrefix = `</${tagName}`; |
| 199 | for (let i = from; i < input.length; i += 1) { |
| 200 | if (startsWithAsciiCI(input, i, closePrefix) && tagNameBoundary(input, i + closePrefix.length)) { |
| 201 | const close = input.indexOf(">", i + closePrefix.length); |
| 202 | return close === -1 ? input.length : close + 1; |
| 203 | } |
| 204 | } |
| 205 | return input.length; |
| 206 | } |
| 207 | |
| 208 | function collapseWhitespace(input: string): string { |
| 209 | let out = ""; |
| 210 | let pendingSpace = false; |
| 211 | for (const c of input) { |
| 212 | if (isWhitespace(c)) { |
| 213 | pendingSpace = out.length > 0; |
| 214 | continue; |
| 215 | } |
| 216 | if (pendingSpace) out += " "; |
| 217 | out += c; |
| 218 | pendingSpace = false; |
| 219 | } |
| 220 | return out.trim(); |
| 221 | } |
| 222 | |
| 223 | function stripHtmlForPrompt(input: string): string { |
| 224 | let out = ""; |
| 225 | for (let i = 0; i < input.length;) { |
| 226 | if (input[i] !== "<") { |
| 227 | out += input[i]; |
| 228 | i += 1; |
| 229 | continue; |
| 230 | } |
| 231 | |
| 232 | if (startsWithAsciiCI(input, i, "<script") && tagNameBoundary(input, i + "<script".length)) { |
| 233 | out += " "; |
| 234 | const openEnd = input.indexOf(">", i + 1); |
| 235 | i = openEnd === -1 ? input.length : findClosingRawTextTag(input, openEnd + 1, "script"); |
| 236 | continue; |
| 237 | } |
| 238 | if (startsWithAsciiCI(input, i, "<style") && tagNameBoundary(input, i + "<style".length)) { |
| 239 | out += " "; |
| 240 | const openEnd = input.indexOf(">", i + 1); |
| 241 | i = openEnd === -1 ? input.length : findClosingRawTextTag(input, openEnd + 1, "style"); |
| 242 | continue; |
| 243 | } |
| 244 | |
| 245 | out += " "; |
| 246 | const tagEnd = input.indexOf(">", i + 1); |
| 247 | i = tagEnd === -1 ? input.length : tagEnd + 1; |
| 248 | } |
| 249 | return collapseWhitespace(out).slice(0, 8000); |
| 250 | } |
| 251 | |
| 252 | // The model's drift list is untrusted input: bound it before any KV fanout so |
| 253 | // a runaway or hostile response cannot flood the draft queue, and drop entries |
| 254 | // whose shape does not match the prompt's contract. |
| 255 | const MAX_DRIFT_DRAFTS_PER_RUN = 10; |
| 256 | const DRIFT_FIELD_MAX = 2_000; |
| 257 | const DRIFT_PAGES = new Set(["homepage", "docs", "install", "contribute", "roadmap"]); |
| 258 | |
| 259 | interface DriftFinding { |
| 260 | page: string; |
| 261 | claim: string; |
| 262 | evidence: string; |
| 263 | suggested_replacement: string; |
| 264 | } |
| 265 | |
| 266 | function validateDriftFindings(raw: unknown): DriftFinding[] { |
| 267 | if (!Array.isArray(raw)) return []; |
| 268 | const findings: DriftFinding[] = []; |
| 269 | for (const entry of raw) { |
| 270 | if (findings.length >= MAX_DRIFT_DRAFTS_PER_RUN) break; |
| 271 | if (typeof entry !== "object" || entry === null) continue; |
| 272 | const { page, claim, evidence, suggested_replacement } = entry as Record<string, unknown>; |
| 273 | if (typeof page !== "string" || !DRIFT_PAGES.has(page)) continue; |
| 274 | if (typeof claim !== "string" || !claim.trim()) continue; |
| 275 | if (typeof evidence !== "string" || !evidence.trim()) continue; |
| 276 | if (typeof suggested_replacement !== "string" || !suggested_replacement.trim()) continue; |
| 277 | findings.push({ |
| 278 | page, |
| 279 | claim: claim.slice(0, DRIFT_FIELD_MAX), |
| 280 | evidence: evidence.slice(0, DRIFT_FIELD_MAX), |
| 281 | suggested_replacement: suggested_replacement.slice(0, DRIFT_FIELD_MAX), |
| 282 | }); |
| 283 | } |
| 284 | return findings; |
| 285 | } |
| 286 | |
| 287 | export async function runSemanticDrift(env: WatchEnv): Promise<{ ok: boolean; drafted: number; reason?: string }> { |
| 288 | if (!env.CURATED_KV || !env.DEEPSEEK_API_KEY) { |
| 289 | return { ok: false, drafted: 0, reason: "missing CURATED_KV or DEEPSEEK_API_KEY" }; |
| 290 | } |
| 291 | |
| 292 | const ghHeaders: Record<string, string> = { |
| 293 | Accept: "application/vnd.github+json", |
| 294 | "User-Agent": "codewhale-web-semantic-drift", |
| 295 | }; |
| 296 | if (env.GITHUB_TOKEN) ghHeaders["Authorization"] = `Bearer ${env.GITHUB_TOKEN}`; |
| 297 | |
| 298 | // Fetch CHANGELOG (truncated), recent commits, and live homepage HTML. |
| 299 | const [changelog, commits, homepageHtml, docsHtml] = await Promise.all([ |
| 300 | fetchBoundedText("https://raw.githubusercontent.com/codewhale-hq/CodeWhale/main/CHANGELOG.md", { headers: ghHeaders }).then((r) => r.text).catch(() => ""), |
| 301 | fetchBoundedText("https://api.github.com/repos/codewhale-hq/CodeWhale/commits?per_page=30", { headers: ghHeaders }).then((r) => r.ok ? JSON.parse(r.text) as { commit: { message: string }; sha: string }[] : []).catch(() => []), |
| 302 | fetchBoundedText("https://codewhale.net/en", { headers: { "User-Agent": "codewhale-watch" } }).then((r) => r.text).catch(() => ""), |
| 303 | fetchBoundedText("https://codewhale.net/en/docs", { headers: { "User-Agent": "codewhale-watch" } }).then((r) => r.text).catch(() => ""), |
| 304 | ]); |
| 305 | |
| 306 | if (!changelog && (!commits || commits.length === 0)) { |
| 307 | return { ok: false, drafted: 0, reason: "no changelog or commits available" }; |
| 308 | } |
| 309 | |
| 310 | const homepageText = stripHtmlForPrompt(homepageHtml); |
| 311 | const docsText = stripHtmlForPrompt(docsHtml); |
| 312 | const changelogHead = changelog.slice(0, 4000); |
| 313 | const commitMsgs = commits.slice(0, 30).map((c) => `- ${c.sha.slice(0, 7)}: ${c.commit.message.split("\n")[0]}`).join("\n"); |
| 314 | |
| 315 | const userMessage = `## Recent CHANGELOG entries |
| 316 | ${changelogHead || "(no CHANGELOG.md fetched)"} |
| 317 | |
| 318 | ## Last 30 commits |
| 319 | ${commitMsgs || "(no commits fetched)"} |
| 320 | |
| 321 | ## Homepage text (HTML stripped) |
| 322 | ${homepageText} |
| 323 | |
| 324 | ## Docs page text (HTML stripped) |
| 325 | ${docsText}`; |
| 326 | |
| 327 | let response: { content: string; usage: { input: number; output: number } }; |
| 328 | try { |
| 329 | response = await agentChat( |
| 330 | [ |
| 331 | { role: "system", content: SEMANTIC_DRIFT_PROMPT }, |
| 332 | { role: "user", content: userMessage }, |
| 333 | ], |
| 334 | env.DEEPSEEK_API_KEY, |
| 335 | true, |
| 336 | dsEnv(env), |
| 337 | ); |
| 338 | } catch (e) { |
| 339 | return { ok: false, drafted: 0, reason: `LLM call failed: ${e}` }; |
| 340 | } |
| 341 | |
| 342 | // Extract JSON (jsonMode usually returns clean JSON, but defend against fences) |
| 343 | let parsed: { drifts?: unknown }; |
| 344 | try { |
| 345 | const trimmed = response.content.replace(/^```(?:json)?\s*/i, "").replace(/\s*```\s*$/i, "").trim(); |
| 346 | parsed = JSON.parse(trimmed); |
| 347 | } catch { |
| 348 | return { ok: false, drafted: 0, reason: "LLM returned non-JSON" }; |
| 349 | } |
| 350 | |
| 351 | const drifts = validateDriftFindings(parsed.drifts); |
| 352 | let drafted = 0; |
| 353 | for (const d of drifts) { |
| 354 | // Identity covers the full finding — page, claim, evidence, and |
| 355 | // replacement — so a re-run over an unchanged finding dedups, while a |
| 356 | // changed finding lands under a new key as a fresh draft. |
| 357 | const identity = [d.page, d.claim, d.evidence, d.suggested_replacement].join("\n"); |
| 358 | const id = await watchDraftId(`${d.page}-${d.claim.slice(0, 40)}`, `semantic-drift\n${identity}`); |
| 359 | const body = `Page: **${d.page}**\n\nClaim that may be drifted:\n> ${d.claim}\n\nEvidence:\n> ${d.evidence}\n\nSuggested replacement:\n> ${d.suggested_replacement}\n\n— drafted by community assistant, pending maintainer review`; |
| 360 | const draft: AgentDraft = { |
| 361 | id, |
| 362 | type: "semantic-drift", |
| 363 | targetUrl: `https://codewhale.net/en/${d.page === "homepage" ? "" : d.page}`, |
| 364 | bodyEn: body, |
| 365 | bodyZh: body, |
| 366 | generatedAt: new Date().toISOString(), |
| 367 | posted: false, |
| 368 | }; |
| 369 | const existing = await getDraft(env.CURATED_KV, draftStorageKey(draft)); |
| 370 | if (existing) continue; |
| 371 | |
| 372 | // saveDraft refuses findings the maintainer already discarded. |
| 373 | if (await saveDraft(env.CURATED_KV, draft)) drafted++; |
| 374 | } |
| 375 | |
| 376 | return { ok: true, drafted }; |
| 377 | } |
| 378 |