| 1 | // Run: tsx src/__tests__/provider-model-refresh.test.ts |
| 2 | |
| 3 | import { |
| 4 | apiKeyEnvFromProviderName, |
| 5 | inferredVisionModels, |
| 6 | isLikelyChatModel, |
| 7 | isLikelyVisionModel, |
| 8 | mergedFetchedProviderModels, |
| 9 | mergeProviderModelContextWindows, |
| 10 | providerApiKeyEnvForSave, |
| 11 | providerDefaultModel, |
| 12 | providerIsConfigured, |
| 13 | providerModelCandidates, |
| 14 | providerModelContextWindowDrafts, |
| 15 | providerModelContextWindowIsSmall, |
| 16 | providerRequiresKey, |
| 17 | } from "../lib/providerModels"; |
| 18 | |
| 19 | let passed = 0; |
| 20 | let failed = 0; |
| 21 | |
| 22 | function eq(a: unknown, b: unknown, label: string) { |
| 23 | if (JSON.stringify(a) === JSON.stringify(b)) { |
| 24 | process.stdout.write(` PASS ${label}\n`); |
| 25 | passed += 1; |
| 26 | } else { |
| 27 | process.stdout.write(` FAIL ${label}: expected ${JSON.stringify(b)}, got ${JSON.stringify(a)}\n`); |
| 28 | failed += 1; |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | console.log("\nprovider model refresh"); |
| 33 | |
| 34 | eq( |
| 35 | mergedFetchedProviderModels(["coding-pro"], ["coding-pro", "chat", "vision"]), |
| 36 | ["coding-pro", "chat", "vision"], |
| 37 | "appends discovered models without removing curated ones", |
| 38 | ); |
| 39 | |
| 40 | eq( |
| 41 | mergedFetchedProviderModels(["coding-pro"], ["coding-pro", "chat", "vision"], { preserveCurated: true }), |
| 42 | ["coding-pro"], |
| 43 | "background refresh preserves manually curated model list", |
| 44 | ); |
| 45 | |
| 46 | eq( |
| 47 | mergedFetchedProviderModels(["coding-pro"], ["chat", "vision"], { preserveCurated: true }), |
| 48 | ["coding-pro"], |
| 49 | "background refresh does not re-add deleted models", |
| 50 | ); |
| 51 | |
| 52 | eq( |
| 53 | mergedFetchedProviderModels(["mimo-v2.5-pro"], ["mimo-v2.5", "mimo-v2.5-pro"], { preserveCurated: true }), |
| 54 | ["mimo-v2.5-pro"], |
| 55 | "manual access refresh preserves selected MiMo model instead of importing provider catalog", |
| 56 | ); |
| 57 | |
| 58 | eq( |
| 59 | providerModelCandidates(["mimo-v2.5-pro"], ["mimo-v2.5", "mimo-v2.5-pro"]), |
| 60 | ["mimo-v2.5-pro", "mimo-v2.5"], |
| 61 | "manual access refresh can show provider catalog as unsaved candidates", |
| 62 | ); |
| 63 | |
| 64 | eq( |
| 65 | providerModelCandidates(["mimo-v2.5-pro"], ["mimo-v2.5-asr", "mimo-v2.5-tts", "mimo-v2.5", "mimo-v2.5-pro"]), |
| 66 | ["mimo-v2.5-pro", "mimo-v2.5"], |
| 67 | "manual access refresh filters non-chat candidates before saving", |
| 68 | ); |
| 69 | |
| 70 | eq( |
| 71 | [ |
| 72 | isLikelyChatModel("mimo-v2.5-pro"), |
| 73 | isLikelyChatModel("mimo-v2.5-asr"), |
| 74 | isLikelyChatModel("mimo-v2.5-tts"), |
| 75 | isLikelyChatModel("text-embedding-3-small"), |
| 76 | ], |
| 77 | [true, false, false, false], |
| 78 | "matches backend non-chat model heuristic", |
| 79 | ); |
| 80 | |
| 81 | eq( |
| 82 | [ |
| 83 | isLikelyVisionModel("gpt-4o"), |
| 84 | isLikelyVisionModel("gpt-4o-audio-preview"), |
| 85 | isLikelyVisionModel("gpt-4o-mini-audio-preview"), |
| 86 | isLikelyVisionModel("mimo-v2.5"), |
| 87 | isLikelyVisionModel("mimo-v2-omni"), |
| 88 | isLikelyVisionModel("qwen2.5-vl-72b-instruct"), |
| 89 | isLikelyVisionModel("mimo-v2.5-asr"), |
| 90 | ], |
| 91 | [true, false, false, true, true, true, false], |
| 92 | "detects likely image-capable chat model IDs", |
| 93 | ); |
| 94 | |
| 95 | eq( |
| 96 | inferredVisionModels([ |
| 97 | "mimo-v2.5-pro", |
| 98 | "mimo-v2.5", |
| 99 | "mimo-v2-omni", |
| 100 | "qwen-vl-plus", |
| 101 | "mimo-v2.5-asr", |
| 102 | "audio-omni-tts", |
| 103 | "gpt-4o-audio-preview", |
| 104 | "gpt-4o-mini-audio-preview", |
| 105 | ]), |
| 106 | ["mimo-v2.5", "mimo-v2-omni", "qwen-vl-plus"], |
| 107 | "infers image-capable models without importing audio models", |
| 108 | ); |
| 109 | |
| 110 | eq( |
| 111 | mergedFetchedProviderModels([], ["coding-pro", "chat"], { preserveCurated: true }), |
| 112 | ["coding-pro", "chat"], |
| 113 | "background refresh can populate an empty model list", |
| 114 | ); |
| 115 | |
| 116 | eq( |
| 117 | providerDefaultModel("coding-pro", ["coding-pro", "chat"]), |
| 118 | "coding-pro", |
| 119 | "preserves current default when it remains available", |
| 120 | ); |
| 121 | |
| 122 | eq( |
| 123 | providerDefaultModel("deleted", ["coding-pro", "chat"]), |
| 124 | "coding-pro", |
| 125 | "falls back to first saved model when default is unavailable", |
| 126 | ); |
| 127 | |
| 128 | const contextOverrides = [ |
| 129 | { |
| 130 | model: "short-model", |
| 131 | reasoningProtocol: "none", |
| 132 | supportedEfforts: [], |
| 133 | defaultEffort: "", |
| 134 | vision: null, |
| 135 | contextWindow: 32768, |
| 136 | }, |
| 137 | { |
| 138 | model: "long-model", |
| 139 | reasoningProtocol: "", |
| 140 | supportedEfforts: [], |
| 141 | defaultEffort: "", |
| 142 | vision: null, |
| 143 | contextWindow: 1000000, |
| 144 | }, |
| 145 | ]; |
| 146 | |
| 147 | eq( |
| 148 | providerModelContextWindowDrafts(contextOverrides), |
| 149 | { "short-model": "32768", "long-model": "1000000" }, |
| 150 | "loads positive per-model context windows into editable drafts", |
| 151 | ); |
| 152 | |
| 153 | eq( |
| 154 | mergeProviderModelContextWindows(contextOverrides, ["short-model", "new-model"], { |
| 155 | "short-model": "65536", |
| 156 | "new-model": "131072", |
| 157 | }), |
| 158 | [ |
| 159 | { ...contextOverrides[0], contextWindow: 65536 }, |
| 160 | { |
| 161 | model: "new-model", |
| 162 | reasoningProtocol: "", |
| 163 | supportedEfforts: [], |
| 164 | defaultEffort: "", |
| 165 | vision: null, |
| 166 | contextWindow: 131072, |
| 167 | }, |
| 168 | ], |
| 169 | "merges model context edits while preserving other overrides and dropping removed models", |
| 170 | ); |
| 171 | |
| 172 | eq( |
| 173 | mergeProviderModelContextWindows([], ["invalid-model"], { "invalid-model": "Infinity" }), |
| 174 | [], |
| 175 | "drops non-finite model context values before crossing the desktop bridge", |
| 176 | ); |
| 177 | |
| 178 | eq( |
| 179 | [ |
| 180 | providerModelContextWindowIsSmall("8192"), |
| 181 | providerModelContextWindowIsSmall("16384"), |
| 182 | providerModelContextWindowIsSmall(""), |
| 183 | providerModelContextWindowIsSmall("Infinity"), |
| 184 | ], |
| 185 | [true, false, false, false], |
| 186 | "warns only for positive context windows below 16384 tokens", |
| 187 | ); |
| 188 | |
| 189 | eq( |
| 190 | providerApiKeyEnvForSave("Local Gateway", "", ""), |
| 191 | "", |
| 192 | "keeps custom provider keyless when no key env or key value is supplied", |
| 193 | ); |
| 194 | |
| 195 | eq( |
| 196 | providerApiKeyEnvForSave("Local Gateway", "", "sk-test"), |
| 197 | "LOCAL_GATEWAY_API_KEY", |
| 198 | "creates a key env when saving an inline key for a new custom provider", |
| 199 | ); |
| 200 | |
| 201 | eq( |
| 202 | providerApiKeyEnvForSave("9router", "", "sk-test"), |
| 203 | "CUSTOM_9ROUTER_API_KEY", |
| 204 | "creates a valid key env for a digit-leading custom provider", |
| 205 | ); |
| 206 | |
| 207 | eq( |
| 208 | providerApiKeyEnvForSave("Local Gateway", "GATEWAY_KEY", ""), |
| 209 | "GATEWAY_KEY", |
| 210 | "preserves an explicitly configured key env", |
| 211 | ); |
| 212 | |
| 213 | eq( |
| 214 | [ |
| 215 | apiKeyEnvFromProviderName("9router"), |
| 216 | apiKeyEnvFromProviderName("商汤"), |
| 217 | apiKeyEnvFromProviderName("通义千问"), |
| 218 | ], |
| 219 | ["CUSTOM_9ROUTER_API_KEY", "CUSTOM_d39b9067_API_KEY", "CUSTOM_e995c4c9_API_KEY"], |
| 220 | "generates valid stable key envs for digit-leading and non-ASCII provider names", |
| 221 | ); |
| 222 | |
| 223 | eq( |
| 224 | providerApiKeyEnvForSave("商汤", "CUSTOM_API_KEY", "sk-test"), |
| 225 | "CUSTOM_API_KEY", |
| 226 | "preserves an explicitly configured legacy custom key env", |
| 227 | ); |
| 228 | |
| 229 | eq( |
| 230 | [ |
| 231 | providerRequiresKey({ apiKeyEnv: "" }), |
| 232 | providerIsConfigured({ apiKeyEnv: "", keySet: false }), |
| 233 | providerIsConfigured({ apiKeyEnv: "LOCAL_API_KEY", keySet: false, requiresKey: false }), |
| 234 | providerIsConfigured({ apiKeyEnv: "REMOTE_API_KEY", keySet: false, requiresKey: true }), |
| 235 | providerIsConfigured({ apiKeyEnv: "REMOTE_API_KEY", keySet: true, requiresKey: true }), |
| 236 | ], |
| 237 | [false, true, true, false, true], |
| 238 | "separates provider selectability from key presence for no-auth providers", |
| 239 | ); |
| 240 | |
| 241 | console.log(`\n${passed} passed, ${failed} failed, ${passed + failed} total`); |
| 242 | if (failed > 0) process.exit(1); |
| 243 |