| 1 | import { afterEach, describe, expect, it, vi } from "vitest"; |
| 2 | import { readFileSync } from "node:fs"; |
| 3 | import { isRepoFacts } from "./facts"; |
| 4 | import { deriveFactsFromRemote, runFactsDrift } from "./facts-drift"; |
| 5 | |
| 6 | const REVISION = "b".repeat(40); |
| 7 | |
| 8 | function response(body: string, status = 200): Response { |
| 9 | return new Response(body, { status }); |
| 10 | } |
| 11 | |
| 12 | const VALID_GENERATED_FACTS = |
| 13 | 'export const FACTS: RepoFacts = {"toolCount":73,"models":[]};'; |
| 14 | |
| 15 | function installGitHubFixture( |
| 16 | toolCountSource: string | null, |
| 17 | releaseHtmlUrl = "https://github.com/codewhale-hq/CodeWhale/releases/tag/v0.9.0", |
| 18 | sourceOverrides: Record<string, string> = {}, |
| 19 | ): void { |
| 20 | vi.stubGlobal( |
| 21 | "fetch", |
| 22 | vi.fn(async (input: string | URL | Request) => { |
| 23 | const url = String(input); |
| 24 | if (url.endsWith("/commits/main")) { |
| 25 | return response( |
| 26 | JSON.stringify({ |
| 27 | sha: REVISION, |
| 28 | commit: { committer: { date: "2026-07-21T23:00:00Z" } }, |
| 29 | }), |
| 30 | ); |
| 31 | } |
| 32 | if (url.endsWith("/releases/latest")) { |
| 33 | return response( |
| 34 | JSON.stringify({ |
| 35 | tag_name: "v0.9.0", |
| 36 | published_at: "2026-07-16T20:05:39Z", |
| 37 | html_url: releaseHtmlUrl, |
| 38 | }), |
| 39 | ); |
| 40 | } |
| 41 | const rawPath = url.split(`/${REVISION}/`)[1]; |
| 42 | const sources: Record<string, string> = { |
| 43 | "Cargo.toml": 'version = "0.9.2"\nmembers = ["crates/tui"]', |
| 44 | "crates/config/assets/provider_descriptors.json": JSON.stringify({ |
| 45 | schema_version: 3, |
| 46 | providers: [{ kind: "Deepseek", id: "deepseek", label: "Remote DeepSeek", default_model: "remote-model", env_vars: ["REMOTE_KEY"], retired: false, selectable: true, web: { order: 0 }, tui_wire_tag: "deepseek", catalog_id: "deepseek", catalog_source_id: "deepseek" }], |
| 47 | constant_refs: { DEFAULT_TEXT_MODEL: { provider: "deepseek", field: "default_model" } }, |
| 48 | }), |
| 49 | "crates/config/assets/models_dev.bundled.json": JSON.stringify({ _reviewed: { |
| 50 | revision: "remote-reviewed", intrinsic: { "deepseek-v4-pro": { source: "exact remote fixture", context_window: 1000000, max_output: 128000, reasoning: true } }, |
| 51 | public_models: [{ id: "deepseek-v4-pro", label: "DeepSeek", added_at: "2026-07-01", aliases: [] }], |
| 52 | } }), |
| 53 | "crates/tui/src/sandbox/mod.rs": ` |
| 54 | pub const PUBLIC_SANDBOX_BACKENDS: &[&str] = &[ |
| 55 | "seatbelt (macOS, when available)", |
| 56 | "bubblewrap (Linux, opt-in when installed)", |
| 57 | ]; |
| 58 | `, |
| 59 | "npm/codewhale/package.json": JSON.stringify({ engines: { node: ">=18" } }), |
| 60 | LICENSE: "MIT License\n", |
| 61 | ...sourceOverrides, |
| 62 | }; |
| 63 | if (rawPath === "web/lib/facts.generated.ts") { |
| 64 | return toolCountSource === null ? response("not found", 404) : response(toolCountSource); |
| 65 | } |
| 66 | return rawPath && rawPath in sources |
| 67 | ? response(sources[rawPath]) |
| 68 | : response("not found", 404); |
| 69 | }), |
| 70 | ); |
| 71 | } |
| 72 | |
| 73 | afterEach(() => { |
| 74 | vi.unstubAllGlobals(); |
| 75 | }); |
| 76 | |
| 77 | describe("deriveFactsFromRemote", () => { |
| 78 | it("derives tool count and model rows from the same exact remote revision", async () => { |
| 79 | installGitHubFixture( |
| 80 | 'export const FACTS: RepoFacts = {"toolCount":73,"models":[' + |
| 81 | '{"id":"deepseek-v4-pro","provider":"DeepSeek","contextWindow":1000000,' + |
| 82 | '"maxOutput":128000,"reasoning":true,"addedAt":"2026-07-01"}' + |
| 83 | "]};", |
| 84 | ); |
| 85 | |
| 86 | const facts = await deriveFactsFromRemote(); |
| 87 | |
| 88 | expect(facts?.sourceRevision).toBe(REVISION); |
| 89 | expect(facts?.version).toBe("0.9.2"); |
| 90 | expect(facts?.toolCount).toBe(73); |
| 91 | expect(facts?.defaultModel).toBe("remote-model"); |
| 92 | expect(facts?.providers).toEqual([{ id: "deepseek", label: "Remote DeepSeek", env: "REMOTE_KEY" }]); |
| 93 | expect(facts?.models).toEqual([ |
| 94 | { |
| 95 | id: "deepseek-v4-pro", |
| 96 | provider: "DeepSeek", |
| 97 | contextWindow: 1000000, |
| 98 | maxOutput: 128000, |
| 99 | reasoning: true, |
| 100 | addedAt: "2026-07-01", |
| 101 | }, |
| 102 | ]); |
| 103 | expect(facts?.sandboxBackends).toEqual([ |
| 104 | "seatbelt (macOS, when available)", |
| 105 | "bubblewrap (Linux, opt-in when installed)", |
| 106 | ]); |
| 107 | const fetchMock = vi.mocked(fetch); |
| 108 | expect( |
| 109 | fetchMock.mock.calls.some(([input]) => String(input).includes("/contents/")), |
| 110 | ).toBe(false); |
| 111 | }); |
| 112 | |
| 113 | it("refuses missing, malformed or incompatible exact-revision provider metadata", async () => { |
| 114 | for (const metadata of ["not json", "{}", '{"schema_version":1,"providers":[]}']) { |
| 115 | installGitHubFixture(VALID_GENERATED_FACTS, undefined, { |
| 116 | "crates/config/assets/provider_descriptors.json": metadata, |
| 117 | }); |
| 118 | await expect(deriveFactsFromRemote()).resolves.toBeNull(); |
| 119 | } |
| 120 | }); |
| 121 | |
| 122 | it("fails derivation when the exact revision has no valid tool count", async () => { |
| 123 | installGitHubFixture(null); |
| 124 | |
| 125 | await expect(deriveFactsFromRemote()).resolves.toBeNull(); |
| 126 | }); |
| 127 | |
| 128 | it("refuses missing or malformed model metadata from the exact source revision", async () => { |
| 129 | for (const catalog of ["not json", "{}", '{"_reviewed":{"revision":"x","intrinsic":{},"public_models":[{"id":42}]}}']) { |
| 130 | installGitHubFixture(VALID_GENERATED_FACTS, undefined, { "crates/config/assets/models_dev.bundled.json": catalog }); |
| 131 | await expect(deriveFactsFromRemote()).resolves.toBeNull(); |
| 132 | } |
| 133 | }); |
| 134 | |
| 135 | it("stores a canonical release URL when GitHub answers with the repo's other casing", async () => { |
| 136 | installGitHubFixture( |
| 137 | VALID_GENERATED_FACTS, |
| 138 | "https://github.com/codewhale-hq/Codewhale/releases/tag/v0.9.0", |
| 139 | ); |
| 140 | |
| 141 | const facts = await deriveFactsFromRemote(); |
| 142 | |
| 143 | expect(facts?.latestPublishedRelease?.url).toBe( |
| 144 | "https://github.com/codewhale-hq/CodeWhale/releases/tag/v0.9.0", |
| 145 | ); |
| 146 | expect(isRepoFacts(facts)).toBe(true); |
| 147 | }); |
| 148 | }); |
| 149 | |
| 150 | describe("runFactsDrift", () => { |
| 151 | it("writes a KV snapshot that getFacts() accepts", async () => { |
| 152 | installGitHubFixture( |
| 153 | VALID_GENERATED_FACTS, |
| 154 | "https://github.com/codewhale-hq/Codewhale/releases/tag/v0.9.0", |
| 155 | ); |
| 156 | const store = new Map<string, string>(); |
| 157 | const kv = { |
| 158 | get: async (key: string) => store.get(key) ?? null, |
| 159 | put: async (key: string, value: string) => { |
| 160 | store.set(key, value); |
| 161 | }, |
| 162 | }; |
| 163 | |
| 164 | const result = await runFactsDrift({ CURATED_KV: kv }); |
| 165 | |
| 166 | expect(result.ok).toBe(true); |
| 167 | expect(isRepoFacts(JSON.parse(store.get("facts:current") ?? "null"))).toBe(true); |
| 168 | }); |
| 169 | |
| 170 | it("never replaces a snapshot from a newer source commit with an older one", async () => { |
| 171 | installGitHubFixture(VALID_GENERATED_FACTS); |
| 172 | const newer = JSON.stringify({ sourceCommittedAt: "2026-07-22T00:00:00Z", version: "9.9.9" }); |
| 173 | const store = new Map<string, string>([["facts:current", newer]]); |
| 174 | const kv = { |
| 175 | get: async (key: string) => store.get(key) ?? null, |
| 176 | put: async (key: string, value: string) => { store.set(key, value); }, |
| 177 | }; |
| 178 | // The fixture's source commit is 2026-07-21T23:00:00Z: an overlapping, |
| 179 | // slower run finishing after a newer one. |
| 180 | expect(await runFactsDrift({ CURATED_KV: kv })).toEqual({ ok: true, changed: false }); |
| 181 | expect(store.get("facts:current")).toBe(newer); |
| 182 | }); |
| 183 | |
| 184 | // The scheduled handler discards the result, so the log line is the only |
| 185 | // signal that the cron stopped refreshing KV. |
| 186 | it("warns and writes nothing when the derived facts fail validation", async () => { |
| 187 | installGitHubFixture(VALID_GENERATED_FACTS, undefined, { |
| 188 | // A non-string engines.node survives derivation but fails isRepoFacts. |
| 189 | "npm/codewhale/package.json": JSON.stringify({ engines: { node: 18 } }), |
| 190 | }); |
| 191 | const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); |
| 192 | const store = new Map<string, string>(); |
| 193 | const kv = { |
| 194 | get: async (key: string) => store.get(key) ?? null, |
| 195 | put: async (key: string, value: string) => { |
| 196 | store.set(key, value); |
| 197 | }, |
| 198 | }; |
| 199 | |
| 200 | const result = await runFactsDrift({ CURATED_KV: kv }); |
| 201 | |
| 202 | expect(result).toEqual({ ok: false, reason: "remote facts failed validation" }); |
| 203 | expect(store.size).toBe(0); |
| 204 | expect(warn).toHaveBeenCalledWith( |
| 205 | expect.stringContaining("[facts-drift] remote facts failed validation"), |
| 206 | ); |
| 207 | warn.mockRestore(); |
| 208 | }); |
| 209 | }); |
| 210 | |
| 211 | describe("complete remote provider roster", () => { |
| 212 | it("keeps all public providers with the labels and credentials from that exact revision", async () => { |
| 213 | const metadata = JSON.parse(readFileSync(new URL("../../crates/config/assets/provider_descriptors.json", import.meta.url), "utf8")); |
| 214 | for (const row of metadata.providers) { |
| 215 | if (row.web && !row.retired) { |
| 216 | row.web.label = `Pinned ${row.id}`; |
| 217 | row.web.env = `PINNED_${row.id}`; |
| 218 | } |
| 219 | } |
| 220 | installGitHubFixture(VALID_GENERATED_FACTS, undefined, { |
| 221 | "crates/tui/src/config.rs": readFileSync(new URL("../../crates/tui/src/config.rs", import.meta.url), "utf8"), |
| 222 | "crates/config/assets/provider_descriptors.json": JSON.stringify(metadata), |
| 223 | }); |
| 224 | const facts = await deriveFactsFromRemote(); |
| 225 | expect(facts?.sourceRevision).toBe(REVISION); |
| 226 | expect(facts?.providers).toHaveLength(50); |
| 227 | expect(new Set(facts?.providers.map(row => row.id)).size).toBe(50); |
| 228 | expect(facts?.providers.every(row => row.label === `Pinned ${row.id}` && row.env === `PINNED_${row.id}`)).toBe(true); |
| 229 | }); |
| 230 | }); |
| 231 |