| 1 | //! `model resolve` must report the route the runtime would actually take. |
| 2 | //! |
| 3 | //! Regression coverage for #4832, where a Z.ai config reported |
| 4 | //! `provider: deepseek` because the subcommand read only the CLI flags and |
| 5 | //! never consulted the resolved runtime. A diagnostic that confidently |
| 6 | //! reports the wrong provider is worse than one that reports nothing, so |
| 7 | //! every provider is asserted here rather than DeepSeek alone. |
| 8 | |
| 9 | use std::collections::BTreeMap; |
| 10 | use std::fs; |
| 11 | use std::path::PathBuf; |
| 12 | use std::process::Command; |
| 13 | |
| 14 | use tempfile::TempDir; |
| 15 | |
| 16 | /// Run `model resolve` against a sealed HOME containing `config`. |
| 17 | /// |
| 18 | /// `env_clear` plus a temporary HOME keeps this off the real |
| 19 | /// `~/.codewhale/config.toml`; the suite has written to real user state before |
| 20 | /// (#4831) and this test must never be the one that does it again. |
| 21 | fn resolve_with_config(config: &str, args: &[&str]) -> BTreeMap<String, String> { |
| 22 | let fixture = TempDir::new().expect("fixture root"); |
| 23 | let home = fixture.path().join("sealed-home"); |
| 24 | fs::create_dir_all(home.join(".codewhale")).expect("sealed config dir"); |
| 25 | fs::write(home.join(".codewhale").join("config.toml"), config).expect("seed config"); |
| 26 | |
| 27 | let mut command = Command::new(codewhale_binary()); |
| 28 | command.arg("model").arg("resolve").args(args); |
| 29 | let output = command |
| 30 | .env_clear() |
| 31 | .env("HOME", &home) |
| 32 | .env("USERPROFILE", &home) |
| 33 | .env("CODEWHALE_HOME", home.join(".codewhale")) |
| 34 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 35 | .output() |
| 36 | .expect("run model resolve"); |
| 37 | |
| 38 | assert!( |
| 39 | output.status.success(), |
| 40 | "model resolve {args:?} failed\nstdout:\n{}\nstderr:\n{}", |
| 41 | String::from_utf8_lossy(&output.stdout), |
| 42 | String::from_utf8_lossy(&output.stderr) |
| 43 | ); |
| 44 | |
| 45 | String::from_utf8_lossy(&output.stdout) |
| 46 | .lines() |
| 47 | .filter_map(|line| line.split_once(": ")) |
| 48 | .map(|(key, value)| (key.trim().to_string(), value.trim().to_string())) |
| 49 | .collect() |
| 50 | } |
| 51 | |
| 52 | #[test] |
| 53 | fn resolve_reports_the_configured_provider_not_a_deepseek_fallback() { |
| 54 | let report = resolve_with_config( |
| 55 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 56 | &[], |
| 57 | ); |
| 58 | |
| 59 | assert_eq!( |
| 60 | report.get("provider").map(String::as_str), |
| 61 | Some("zai"), |
| 62 | "configured provider must survive to the diagnostic: {report:?}" |
| 63 | ); |
| 64 | assert_eq!( |
| 65 | report.get("provider_source").map(String::as_str), |
| 66 | Some("config"), |
| 67 | "provenance must name the config file: {report:?}" |
| 68 | ); |
| 69 | } |
| 70 | |
| 71 | #[test] |
| 72 | fn resolve_reports_a_provider_scoped_model_as_explicitly_configured() { |
| 73 | let report = resolve_with_config( |
| 74 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\nmodel = \"kimi-k3-turbo\"\n", |
| 75 | &[], |
| 76 | ); |
| 77 | |
| 78 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 79 | assert_eq!( |
| 80 | report.get("requested").map(String::as_str), |
| 81 | Some("kimi-k3-turbo"), |
| 82 | "a configured model is a request, not a fallback: {report:?}" |
| 83 | ); |
| 84 | assert_eq!( |
| 85 | report.get("used_fallback").map(String::as_str), |
| 86 | Some("false"), |
| 87 | "{report:?}" |
| 88 | ); |
| 89 | assert_eq!( |
| 90 | report.get("model_source").map(String::as_str), |
| 91 | Some("config [providers.*].model"), |
| 92 | "{report:?}" |
| 93 | ); |
| 94 | } |
| 95 | |
| 96 | #[test] |
| 97 | fn resolve_admits_when_nothing_was_configured() { |
| 98 | // The honest answer to "what did the user ask for" is "nothing". The |
| 99 | // built-in default may still be shown, but it must be labelled as ours. |
| 100 | let report = resolve_with_config("", &[]); |
| 101 | |
| 102 | assert_eq!( |
| 103 | report.get("requested").map(String::as_str), |
| 104 | Some(""), |
| 105 | "an unconfigured model must not be presented as a request: {report:?}" |
| 106 | ); |
| 107 | assert_eq!( |
| 108 | report.get("used_fallback").map(String::as_str), |
| 109 | Some("true"), |
| 110 | "{report:?}" |
| 111 | ); |
| 112 | assert_eq!( |
| 113 | report.get("model_source").map(String::as_str), |
| 114 | Some("provider default"), |
| 115 | "{report:?}" |
| 116 | ); |
| 117 | } |
| 118 | |
| 119 | #[test] |
| 120 | fn an_explicit_model_argument_still_answers_the_hypothetical() { |
| 121 | // Naming a model asks "what would this resolve to", which must keep |
| 122 | // working even when the configured provider is something else. |
| 123 | let report = resolve_with_config( |
| 124 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 125 | &["deepseek-v4-flash"], |
| 126 | ); |
| 127 | |
| 128 | assert_eq!( |
| 129 | report.get("requested").map(String::as_str), |
| 130 | Some("deepseek-v4-flash"), |
| 131 | "{report:?}" |
| 132 | ); |
| 133 | assert_eq!( |
| 134 | report.get("model_source").map(String::as_str), |
| 135 | Some("argument"), |
| 136 | "{report:?}" |
| 137 | ); |
| 138 | assert_eq!( |
| 139 | report.get("used_fallback").map(String::as_str), |
| 140 | Some("false"), |
| 141 | "{report:?}" |
| 142 | ); |
| 143 | } |
| 144 | |
| 145 | #[test] |
| 146 | fn an_explicit_provider_flag_is_reported_as_the_source() { |
| 147 | let report = resolve_with_config( |
| 148 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 149 | &["--provider", "moonshot"], |
| 150 | ); |
| 151 | |
| 152 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 153 | assert_eq!( |
| 154 | report.get("provider_source").map(String::as_str), |
| 155 | Some("--provider"), |
| 156 | "{report:?}" |
| 157 | ); |
| 158 | } |
| 159 | |
| 160 | /// Run `model resolve` with global flags placed before the subcommand, which |
| 161 | /// is where `--provider` / `--model` actually go. |
| 162 | fn resolve_with_global_flags( |
| 163 | config: &str, |
| 164 | global: &[&str], |
| 165 | args: &[&str], |
| 166 | ) -> BTreeMap<String, String> { |
| 167 | let fixture = TempDir::new().expect("fixture root"); |
| 168 | let home = fixture.path().join("sealed-home"); |
| 169 | fs::create_dir_all(home.join(".codewhale")).expect("sealed config dir"); |
| 170 | fs::write(home.join(".codewhale").join("config.toml"), config).expect("seed config"); |
| 171 | |
| 172 | let mut command = Command::new(codewhale_binary()); |
| 173 | command.args(global).arg("model").arg("resolve").args(args); |
| 174 | let output = command |
| 175 | .env_clear() |
| 176 | .env("HOME", &home) |
| 177 | .env("USERPROFILE", &home) |
| 178 | .env("CODEWHALE_HOME", home.join(".codewhale")) |
| 179 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 180 | .output() |
| 181 | .expect("run model resolve"); |
| 182 | |
| 183 | assert!( |
| 184 | output.status.success(), |
| 185 | "model resolve {global:?} {args:?} failed\nstdout:\n{}\nstderr:\n{}", |
| 186 | String::from_utf8_lossy(&output.stdout), |
| 187 | String::from_utf8_lossy(&output.stderr) |
| 188 | ); |
| 189 | |
| 190 | String::from_utf8_lossy(&output.stdout) |
| 191 | .lines() |
| 192 | .filter_map(|line| line.split_once(": ")) |
| 193 | .map(|(key, value)| (key.trim().to_string(), value.trim().to_string())) |
| 194 | .collect() |
| 195 | } |
| 196 | |
| 197 | /// v0.9.1 kimi-k3 dogfood report: `codewhale --provider moonshot --model kimi-k3 model resolve` |
| 198 | /// reported `kimi-k2.7-code`. The top-level flags are the route this process |
| 199 | /// is on, not a hypothetical, so the diagnostic has to answer with the runtime |
| 200 | /// resolution instead of re-deriving a registry default and ignoring `--model`. |
| 201 | #[test] |
| 202 | fn top_level_provider_and_model_flags_report_the_runtime_route() { |
| 203 | let report = resolve_with_global_flags( |
| 204 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 205 | &["--provider", "moonshot", "--model", "kimi-k3"], |
| 206 | &[], |
| 207 | ); |
| 208 | |
| 209 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 210 | assert_eq!( |
| 211 | report.get("resolved").map(String::as_str), |
| 212 | Some("kimi-k3"), |
| 213 | "the diagnostic must not contradict the model the run will use: {report:?}" |
| 214 | ); |
| 215 | assert_eq!( |
| 216 | report.get("requested").map(String::as_str), |
| 217 | Some("kimi-k3"), |
| 218 | "{report:?}" |
| 219 | ); |
| 220 | assert_eq!( |
| 221 | report.get("used_fallback").map(String::as_str), |
| 222 | Some("false"), |
| 223 | "{report:?}" |
| 224 | ); |
| 225 | assert_eq!( |
| 226 | report.get("model_source").map(String::as_str), |
| 227 | Some("--model"), |
| 228 | "{report:?}" |
| 229 | ); |
| 230 | } |
| 231 | |
| 232 | /// Moonshot ships `kimi-k3` on the direct platform API and `k3` on the Kimi |
| 233 | /// Code coding-plan API. Both must resolve, and neither may be answered by |
| 234 | /// another provider's identically named model (OpenCode Go also serves a |
| 235 | /// `kimi-k3`). |
| 236 | #[test] |
| 237 | fn moonshot_k3_products_resolve_without_crossing_providers() { |
| 238 | for model in ["kimi-k3", "k3"] { |
| 239 | let report = resolve_with_global_flags( |
| 240 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 241 | &[], |
| 242 | &[model, "--provider", "moonshot"], |
| 243 | ); |
| 244 | |
| 245 | assert_eq!( |
| 246 | report.get("provider").map(String::as_str), |
| 247 | Some("moonshot"), |
| 248 | "a Moonshot question must not be answered by another provider: {report:?}" |
| 249 | ); |
| 250 | assert_eq!( |
| 251 | report.get("resolved").map(String::as_str), |
| 252 | Some(model), |
| 253 | "{report:?}" |
| 254 | ); |
| 255 | assert_eq!( |
| 256 | report.get("used_fallback").map(String::as_str), |
| 257 | Some("false"), |
| 258 | "{report:?}" |
| 259 | ); |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | /// An id the selected provider cannot serve must be reported as a fallback, |
| 264 | /// never as if the request had been honoured. |
| 265 | #[test] |
| 266 | fn an_unservable_model_on_the_selected_provider_is_reported_as_a_fallback() { |
| 267 | let report = resolve_with_global_flags( |
| 268 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 269 | &[], |
| 270 | &["glm-5.2", "--provider", "moonshot"], |
| 271 | ); |
| 272 | |
| 273 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 274 | assert_eq!( |
| 275 | report.get("used_fallback").map(String::as_str), |
| 276 | Some("true"), |
| 277 | "an unservable id must not be presented as an honoured request: {report:?}" |
| 278 | ); |
| 279 | } |
| 280 | |
| 281 | /// Adding a model to the catalog must make it servable on the provider that |
| 282 | /// carries it and nowhere else. `glm-5.3` was added as a peer of `glm-5.2`, so |
| 283 | /// it has to answer on Z.ai without a fallback while a Moonshot-scoped question |
| 284 | /// still refuses it — the same cross-provider boundary the `glm-5.2` case above |
| 285 | /// pins, asserted on the newest sibling so the boundary cannot rot as the |
| 286 | /// family grows. |
| 287 | #[test] |
| 288 | fn a_new_glm_sibling_is_servable_on_zai_but_not_on_moonshot() { |
| 289 | let served = resolve_with_global_flags( |
| 290 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 291 | &[], |
| 292 | &["glm-5.3", "--provider", "zai"], |
| 293 | ); |
| 294 | |
| 295 | assert_eq!(served.get("provider").map(String::as_str), Some("zai")); |
| 296 | assert_eq!( |
| 297 | served.get("resolved").map(String::as_str), |
| 298 | Some("GLM-5.3"), |
| 299 | "a catalogued model must resolve to itself, not to the provider default: {served:?}" |
| 300 | ); |
| 301 | assert_eq!( |
| 302 | served.get("used_fallback").map(String::as_str), |
| 303 | Some("false"), |
| 304 | "a model the provider serves must not be reported as a fallback: {served:?}" |
| 305 | ); |
| 306 | |
| 307 | let refused = resolve_with_global_flags( |
| 308 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 309 | &[], |
| 310 | &["glm-5.3", "--provider", "moonshot"], |
| 311 | ); |
| 312 | |
| 313 | assert_eq!( |
| 314 | refused.get("provider").map(String::as_str), |
| 315 | Some("moonshot") |
| 316 | ); |
| 317 | assert_eq!( |
| 318 | refused.get("used_fallback").map(String::as_str), |
| 319 | Some("true"), |
| 320 | "a Z.ai id must not be presented as honoured by Moonshot: {refused:?}" |
| 321 | ); |
| 322 | let resolved = refused |
| 323 | .get("resolved") |
| 324 | .map(String::as_str) |
| 325 | .unwrap_or_default(); |
| 326 | assert!( |
| 327 | !resolved.to_ascii_lowercase().contains("glm"), |
| 328 | "a provider that cannot serve GLM must not be handed a fabricated GLM id: {refused:?}" |
| 329 | ); |
| 330 | } |
| 331 | |
| 332 | /// The OpenRouter sibling carries a different wire id (`z-ai/glm-5.3`) than the |
| 333 | /// direct Z.ai row (`GLM-5.3`), so the bare family alias has to be rewritten |
| 334 | /// per provider rather than passed through. This pins the OpenRouter half of |
| 335 | /// that rewrite, which the Z.ai case above cannot observe, and pins that adding |
| 336 | /// the sibling left the OpenRouter default alone. |
| 337 | #[test] |
| 338 | fn the_openrouter_glm_sibling_resolves_to_its_own_gateway_wire_id() { |
| 339 | let served = resolve_with_global_flags( |
| 340 | "provider = \"openrouter\"\n\n[providers.openrouter]\napi_key = \"k\"\n", |
| 341 | &[], |
| 342 | &["glm-5.3", "--provider", "openrouter"], |
| 343 | ); |
| 344 | |
| 345 | assert_eq!( |
| 346 | served.get("provider").map(String::as_str), |
| 347 | Some("openrouter") |
| 348 | ); |
| 349 | assert_eq!( |
| 350 | served.get("resolved").map(String::as_str), |
| 351 | Some("z-ai/glm-5.3"), |
| 352 | "the bare alias must be rewritten to the OpenRouter wire id, not passed through: {served:?}" |
| 353 | ); |
| 354 | assert_eq!( |
| 355 | served.get("used_fallback").map(String::as_str), |
| 356 | Some("false"), |
| 357 | "a gateway row the provider serves must not be reported as a fallback: {served:?}" |
| 358 | ); |
| 359 | |
| 360 | let default_route = resolve_with_config( |
| 361 | "provider = \"openrouter\"\n\n[providers.openrouter]\napi_key = \"k\"\n", |
| 362 | &[], |
| 363 | ); |
| 364 | let resolved = default_route |
| 365 | .get("resolved") |
| 366 | .map(String::as_str) |
| 367 | .unwrap_or_default(); |
| 368 | assert!( |
| 369 | !resolved.to_ascii_lowercase().contains("glm"), |
| 370 | "adding a GLM sibling must not make GLM the OpenRouter default: {default_route:?}" |
| 371 | ); |
| 372 | } |
| 373 | |
| 374 | /// Adding a sibling must not move anyone's route. A Z.ai config that names no |
| 375 | /// model still has to land on `GLM-5.2`: the newer `glm-5.3` is catalogued but |
| 376 | /// deliberately not the default, and this is the surface where that would |
| 377 | /// silently change under a user. |
| 378 | #[test] |
| 379 | fn adding_a_glm_sibling_leaves_the_zai_default_route_untouched() { |
| 380 | let report = resolve_with_config( |
| 381 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 382 | &[], |
| 383 | ); |
| 384 | |
| 385 | assert_eq!( |
| 386 | report.get("resolved").map(String::as_str), |
| 387 | Some("GLM-5.2"), |
| 388 | "the Z.ai default must stay GLM-5.2 after a newer sibling is added: {report:?}" |
| 389 | ); |
| 390 | assert_eq!( |
| 391 | report.get("model_source").map(String::as_str), |
| 392 | Some("provider default"), |
| 393 | "{report:?}" |
| 394 | ); |
| 395 | } |
| 396 | |
| 397 | fn codewhale_binary() -> PathBuf { |
| 398 | if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale") { |
| 399 | return PathBuf::from(path); |
| 400 | } |
| 401 | if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale") { |
| 402 | return PathBuf::from(path); |
| 403 | } |
| 404 | |
| 405 | let mut path = std::env::current_exe().expect("current test executable path"); |
| 406 | path.pop(); |
| 407 | if path.ends_with("deps") { |
| 408 | path.pop(); |
| 409 | } |
| 410 | path.push(format!("codewhale{}", std::env::consts::EXE_SUFFIX)); |
| 411 | path |
| 412 | } |
| 413 |