| 1 | //! Operator-approved route replacement at the first-request seam. |
| 2 | //! |
| 3 | //! The observed failure: a saved reviewer pin answered its first request with |
| 4 | //! `Authorization failed: You have run out of credits or need a Grok |
| 5 | //! subscription.` and no review was produced. |
| 6 | use super::*; |
| 7 | |
| 8 | const REFUSAL: &str = "You have run out of credits or need a Grok subscription."; |
| 9 | |
| 10 | /// A provider fixture that refuses every request with HTTP 403. |
| 11 | async fn refusing_chat_server() -> (String, Arc<AtomicUsize>) { |
| 12 | let calls = Arc::new(AtomicUsize::new(0)); |
| 13 | let app = Router::new().route( |
| 14 | "/{*path}", |
| 15 | post({ |
| 16 | let calls = Arc::clone(&calls); |
| 17 | move |Json(_body): Json<Value>| { |
| 18 | let calls = Arc::clone(&calls); |
| 19 | async move { |
| 20 | calls.fetch_add(1, Ordering::SeqCst); |
| 21 | ( |
| 22 | StatusCode::FORBIDDEN, |
| 23 | Json(json!({"error": {"message": REFUSAL}})), |
| 24 | ) |
| 25 | .into_response() |
| 26 | } |
| 27 | } |
| 28 | }), |
| 29 | ); |
| 30 | let listener = tokio::net::TcpListener::bind("127.0.0.1:0") |
| 31 | .await |
| 32 | .expect("bind refusing server"); |
| 33 | let addr = listener.local_addr().expect("refusing server addr"); |
| 34 | tokio::spawn(async move { |
| 35 | let _ = axum::serve(listener, app).await; |
| 36 | }); |
| 37 | (format!("http://{addr}/v1"), calls) |
| 38 | } |
| 39 | |
| 40 | async fn write_replacement_config( |
| 41 | path: &std::path::Path, |
| 42 | pin_url: &str, |
| 43 | backup_url: &str, |
| 44 | pin: &str, |
| 45 | ) { |
| 46 | tokio::fs::write( |
| 47 | path, |
| 48 | format!( |
| 49 | r#" |
| 50 | provider = "deepseek" |
| 51 | model = "deepseek-v4-flash" |
| 52 | api_key = "fixture-key" |
| 53 | base_url = "{backup_url}" |
| 54 | |
| 55 | [retry] |
| 56 | enabled = false |
| 57 | max_retries = 0 |
| 58 | |
| 59 | [providers.PinRoute] |
| 60 | kind = "openai-compatible" |
| 61 | api_key = "fixture-pin-key" |
| 62 | base_url = "{pin_url}" |
| 63 | model = "fixture-pin-model" |
| 64 | |
| 65 | [providers.BackupRoute] |
| 66 | kind = "openai-compatible" |
| 67 | api_key = "fixture-backup-key" |
| 68 | base_url = "{backup_url}" |
| 69 | model = "fixture-backup-model" |
| 70 | |
| 71 | {pin} |
| 72 | "# |
| 73 | ), |
| 74 | ) |
| 75 | .await |
| 76 | .unwrap(); |
| 77 | } |
| 78 | |
| 79 | async fn reviewer_tool( |
| 80 | root: &std::path::Path, |
| 81 | pin: &str, |
| 82 | ) -> ( |
| 83 | AgentTool, |
| 84 | ToolContext, |
| 85 | SharedSubAgentManager, |
| 86 | Arc<AtomicUsize>, |
| 87 | Arc<AtomicUsize>, |
| 88 | ) { |
| 89 | reviewer_tool_with(root, pin, false).await |
| 90 | } |
| 91 | |
| 92 | /// [`reviewer_tool`], optionally with the parent route refusing as well. |
| 93 | async fn reviewer_tool_with( |
| 94 | root: &std::path::Path, |
| 95 | pin: &str, |
| 96 | parent_refuses: bool, |
| 97 | ) -> ( |
| 98 | AgentTool, |
| 99 | ToolContext, |
| 100 | SharedSubAgentManager, |
| 101 | Arc<AtomicUsize>, |
| 102 | Arc<AtomicUsize>, |
| 103 | ) { |
| 104 | let (backup, backup_calls, _, _) = delayed_chat_client(Duration::ZERO, "review done").await; |
| 105 | let (pin_url, pin_calls) = refusing_chat_server().await; |
| 106 | let config_path = root.join("config.toml"); |
| 107 | let parent_url = if parent_refuses { |
| 108 | pin_url.as_str() |
| 109 | } else { |
| 110 | backup.base_url() |
| 111 | }; |
| 112 | write_replacement_config(&config_path, &pin_url, parent_url, pin).await; |
| 113 | let config = crate::config::Config::load(Some(config_path), None).unwrap(); |
| 114 | let client = CodewhaleClient::new(&config).unwrap(); |
| 115 | let manager = new_shared_subagent_manager(root.to_path_buf(), 2); |
| 116 | let context = ToolContext::new(root).with_state_namespace("route-replacement"); |
| 117 | let runtime = SubAgentRuntime::new( |
| 118 | client, |
| 119 | "deepseek-v4-flash".into(), |
| 120 | context.clone(), |
| 121 | false, |
| 122 | None, |
| 123 | manager.clone(), |
| 124 | ) |
| 125 | .with_api_config(config); |
| 126 | ( |
| 127 | AgentTool::new(manager.clone(), runtime), |
| 128 | context, |
| 129 | manager, |
| 130 | pin_calls, |
| 131 | backup_calls, |
| 132 | ) |
| 133 | } |
| 134 | |
| 135 | async fn run_reviewer(pin: &str) -> (Value, SubAgentResult, usize, usize) { |
| 136 | let root = tempdir().unwrap(); |
| 137 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 138 | let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek"); |
| 139 | let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash"); |
| 140 | let (tool, context, manager, pin_calls, backup_calls) = reviewer_tool(root.path(), pin).await; |
| 141 | let started = tool |
| 142 | .execute( |
| 143 | json!({"type": "reviewer", "prompt": "Review the change."}), |
| 144 | &context, |
| 145 | ) |
| 146 | .await |
| 147 | .unwrap(); |
| 148 | let meta = started.metadata.clone().unwrap(); |
| 149 | let id = meta["agent_id"].as_str().unwrap().to_string(); |
| 150 | let result = tokio::time::timeout(Duration::from_secs(10), async { |
| 151 | loop { |
| 152 | let result = manager.read().await.get_result(&id).expect("registered"); |
| 153 | if result.status != SubAgentStatus::Running { |
| 154 | return result; |
| 155 | } |
| 156 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 157 | } |
| 158 | }) |
| 159 | .await |
| 160 | .expect("child settles"); |
| 161 | ( |
| 162 | meta, |
| 163 | result, |
| 164 | pin_calls.load(Ordering::SeqCst), |
| 165 | backup_calls.load(Ordering::SeqCst), |
| 166 | ) |
| 167 | } |
| 168 | |
| 169 | #[tokio::test] |
| 170 | async fn approved_replacement_takes_a_refused_first_request_and_is_receipted() { |
| 171 | let _env = crate::test_support::lock_test_env(); |
| 172 | let (meta, result, pin_calls, backup_calls) = run_reviewer( |
| 173 | r#" |
| 174 | [subagents.roles.reviewer] |
| 175 | model = "PinRoute/fixture-pin-model" |
| 176 | replacements = ["BackupRoute/fixture-backup-model"] |
| 177 | "#, |
| 178 | ) |
| 179 | .await; |
| 180 | assert_eq!(meta["child_route"]["provider_id"], "PinRoute"); |
| 181 | assert_eq!(pin_calls, 1, "the refused route is asked exactly once"); |
| 182 | assert!(backup_calls >= 1, "the approved route took the request"); |
| 183 | assert_eq!( |
| 184 | result.status, |
| 185 | SubAgentStatus::Completed, |
| 186 | "{:?}", |
| 187 | result.status |
| 188 | ); |
| 189 | assert_eq!(result.result.as_deref(), Some("review done")); |
| 190 | let route = result.child_route.expect("route receipt"); |
| 191 | assert_eq!(route.provider_id, "BackupRoute"); |
| 192 | assert_eq!(route.model_id, "fixture-backup-model"); |
| 193 | assert_eq!(route.route_source, "role.replacement"); |
| 194 | let note = route.fallback_note.expect("replacement note"); |
| 195 | for fact in [ |
| 196 | "fixture-pin-model", |
| 197 | "provider refused authorization", |
| 198 | "run out of credits", |
| 199 | "BackupRoute/fixture-backup-model", |
| 200 | "attempt 1 of 1", |
| 201 | ] { |
| 202 | assert!(note.contains(fact), "{fact} missing from {note}"); |
| 203 | } |
| 204 | assert!(!note.contains("fixture-backup-key") && !note.contains("fixture-pin-key")); |
| 205 | } |
| 206 | |
| 207 | #[tokio::test] |
| 208 | async fn a_pin_without_approved_replacements_stays_exact() { |
| 209 | let _env = crate::test_support::lock_test_env(); |
| 210 | let (_meta, result, pin_calls, backup_calls) = run_reviewer( |
| 211 | r#" |
| 212 | [subagents.roles.reviewer] |
| 213 | model = "PinRoute/fixture-pin-model" |
| 214 | "#, |
| 215 | ) |
| 216 | .await; |
| 217 | assert_eq!(pin_calls, 1); |
| 218 | assert_eq!(backup_calls, 0, "no provider was asked without approval"); |
| 219 | let SubAgentStatus::Failed(error) = &result.status else { |
| 220 | panic!("an exact refused pin fails: {:?}", result.status); |
| 221 | }; |
| 222 | for fact in [ |
| 223 | "Authorization failed", |
| 224 | "[redacted]", |
| 225 | "requested model `fixture-pin-model`", |
| 226 | "config.toml role pin for \"reviewer\" routes PinRoute/fixture-pin-model", |
| 227 | ] { |
| 228 | assert!(error.contains(fact), "{fact} missing from {error}"); |
| 229 | } |
| 230 | assert!(!error.contains("fixture-pin-key") && !error.contains("fixture-backup-key")); |
| 231 | assert_eq!(result.child_route.unwrap().provider_id, "PinRoute"); |
| 232 | } |
| 233 | |
| 234 | #[test] |
| 235 | fn replacement_reasons_are_typed_never_message_matched() { |
| 236 | let refusal = |status| anyhow::Error::new(LlmError::from_http_response(status, REFUSAL)); |
| 237 | assert_eq!( |
| 238 | route_replacement_reason(&refusal(403)), |
| 239 | Some("provider refused authorization") |
| 240 | ); |
| 241 | assert_eq!( |
| 242 | route_replacement_reason(&refusal(401)), |
| 243 | Some("credentials rejected") |
| 244 | ); |
| 245 | // A credits-themed message without typed evidence is not a route refusal. |
| 246 | assert_eq!(route_replacement_reason(&anyhow!(REFUSAL)), None); |
| 247 | assert_eq!( |
| 248 | route_replacement_reason(&anyhow::Error::new(LlmError::ContentPolicyError( |
| 249 | "blocked".into() |
| 250 | ))), |
| 251 | None, |
| 252 | "content refusals are never shopped to another provider" |
| 253 | ); |
| 254 | assert_eq!( |
| 255 | route_replacement_reason(&anyhow::Error::new(LlmError::ContextLengthError( |
| 256 | "too long".into() |
| 257 | ))), |
| 258 | None |
| 259 | ); |
| 260 | } |
| 261 | |
| 262 | #[test] |
| 263 | fn replacements_must_name_their_provider() { |
| 264 | let config: crate::config::Config = toml::from_str( |
| 265 | r#" |
| 266 | [subagents.roles.reviewer] |
| 267 | model = "deepseek-v4-pro" |
| 268 | replacements = ["deepseek/deepseek-v4-flash", "bare-model"] |
| 269 | "#, |
| 270 | ) |
| 271 | .unwrap(); |
| 272 | let routes = config.subagent_route_replacements("reviewer"); |
| 273 | assert_eq!(routes.len(), 2); |
| 274 | assert_eq!(routes[0].provider.as_deref(), Some("deepseek")); |
| 275 | assert_eq!(routes[1].provider, None); |
| 276 | assert!(config.subagent_route_replacements("builder").is_empty()); |
| 277 | } |
| 278 | |
| 279 | #[tokio::test] |
| 280 | async fn a_replacement_without_an_explicit_provider_fails_at_spawn() { |
| 281 | let _env = crate::test_support::lock_test_env(); |
| 282 | let root = tempdir().unwrap(); |
| 283 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 284 | let (tool, context, manager, pin_calls, backup_calls) = reviewer_tool( |
| 285 | root.path(), |
| 286 | r#" |
| 287 | [subagents.roles.reviewer] |
| 288 | model = "PinRoute/fixture-pin-model" |
| 289 | replacements = ["fixture-backup-model"] |
| 290 | "#, |
| 291 | ) |
| 292 | .await; |
| 293 | let error = tool |
| 294 | .execute(json!({"type": "reviewer", "prompt": "Review."}), &context) |
| 295 | .await |
| 296 | .unwrap_err(); |
| 297 | assert!(error.to_string().contains("provider/model"), "{error}"); |
| 298 | assert!(manager.read().await.agents.is_empty()); |
| 299 | assert_eq!( |
| 300 | pin_calls.load(Ordering::SeqCst) + backup_calls.load(Ordering::SeqCst), |
| 301 | 0 |
| 302 | ); |
| 303 | } |
| 304 | |
| 305 | /// The founder's shape: a saved agent profile (not a config pin) routes the |
| 306 | /// reviewer role to a provider whose account refuses authorization, while |
| 307 | /// the parent route works. |
| 308 | async fn write_refusing_reviewer_profile(root: &std::path::Path) { |
| 309 | let dir = root.join(".codewhale/agents"); |
| 310 | tokio::fs::create_dir_all(&dir).await.unwrap(); |
| 311 | tokio::fs::write( |
| 312 | dir.join("reviewer.toml"), |
| 313 | "id = \"reviewer\"\ndisplay_name = \"reviewer\"\nprovider = \"PinRoute\"\nmodel = \"fixture-pin-model\"\nrole_hint = \"reviewer\"\n", |
| 314 | ) |
| 315 | .await |
| 316 | .unwrap(); |
| 317 | } |
| 318 | |
| 319 | struct ProjectProfiles(bool); |
| 320 | impl ProjectProfiles { |
| 321 | fn enabled() -> Self { |
| 322 | let previous = crate::fleet::roster::project_agent_profiles_enabled(); |
| 323 | crate::fleet::roster::set_project_agent_profiles_enabled(true); |
| 324 | Self(previous) |
| 325 | } |
| 326 | } |
| 327 | impl Drop for ProjectProfiles { |
| 328 | fn drop(&mut self) { |
| 329 | crate::fleet::roster::set_project_agent_profiles_enabled(self.0); |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | async fn settle(manager: &SharedSubAgentManager, id: &str) -> SubAgentResult { |
| 334 | tokio::time::timeout(Duration::from_secs(10), async { |
| 335 | loop { |
| 336 | let result = manager.read().await.get_result(id).expect("registered"); |
| 337 | if result.status != SubAgentStatus::Running { |
| 338 | return result; |
| 339 | } |
| 340 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 341 | } |
| 342 | }) |
| 343 | .await |
| 344 | .expect("child settles") |
| 345 | } |
| 346 | |
| 347 | #[tokio::test] |
| 348 | async fn refused_saved_profile_pin_runs_on_the_parent_route_visibly_and_once() { |
| 349 | let _env = crate::test_support::lock_test_env(); |
| 350 | let _profiles = ProjectProfiles::enabled(); |
| 351 | let root = tempdir().unwrap(); |
| 352 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 353 | let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek"); |
| 354 | let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash"); |
| 355 | write_refusing_reviewer_profile(root.path()).await; |
| 356 | let (tool, context, manager, pin_calls, backup_calls) = reviewer_tool(root.path(), "").await; |
| 357 | |
| 358 | // First worker: the pin is asked once, refuses, and the same request runs |
| 359 | // on the parent route with the reason on the receipt. |
| 360 | let started = tool |
| 361 | .execute( |
| 362 | json!({"type": "reviewer", "prompt": "Review the change."}), |
| 363 | &context, |
| 364 | ) |
| 365 | .await |
| 366 | .unwrap(); |
| 367 | let meta = started.metadata.clone().unwrap(); |
| 368 | assert_eq!(meta["child_route"]["route_source"], "agent_profile.model"); |
| 369 | assert_eq!(meta["child_route"]["provider_id"], "PinRoute"); |
| 370 | let first = settle(&manager, meta["agent_id"].as_str().unwrap()).await; |
| 371 | assert_eq!( |
| 372 | first.status, |
| 373 | SubAgentStatus::Completed, |
| 374 | "{:?}", |
| 375 | first.status |
| 376 | ); |
| 377 | assert_eq!(first.result.as_deref(), Some("review done")); |
| 378 | assert_eq!(pin_calls.load(Ordering::SeqCst), 1, "the pin is asked once"); |
| 379 | assert!( |
| 380 | backup_calls.load(Ordering::SeqCst) >= 1, |
| 381 | "the parent route ran it" |
| 382 | ); |
| 383 | let route = first.child_route.expect("route receipt"); |
| 384 | assert_eq!(route.route_source, "session.fallback"); |
| 385 | assert_eq!(route.provider_id, "deepseek"); |
| 386 | assert_eq!(route.model_id, "deepseek-v4-flash"); |
| 387 | let note = route.fallback_note.expect("fallback note"); |
| 388 | for fact in [ |
| 389 | "saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model", |
| 390 | "failed authorization", |
| 391 | "run out of credits", |
| 392 | "ran on deepseek/deepseek-v4-flash instead", |
| 393 | ] { |
| 394 | assert!(note.contains(fact), "{fact} missing from {note}"); |
| 395 | } |
| 396 | assert!(!note.contains("fixture-pin-key") && !note.contains("fixture-backup-key")); |
| 397 | |
| 398 | // Second worker in the same session goes straight to the parent route: |
| 399 | // the known-bad pin is not asked again. |
| 400 | let started = tool |
| 401 | .execute( |
| 402 | json!({"type": "reviewer", "prompt": "Review the other change."}), |
| 403 | &context, |
| 404 | ) |
| 405 | .await |
| 406 | .unwrap(); |
| 407 | let meta = started.metadata.clone().unwrap(); |
| 408 | assert_eq!(meta["child_route"]["route_source"], "session.fallback"); |
| 409 | assert_eq!(meta["child_route"]["provider_id"], "deepseek"); |
| 410 | let note = meta["child_route"]["fallback_note"].as_str().unwrap(); |
| 411 | assert!(note.contains("earlier this session"), "{note}"); |
| 412 | assert!(note.contains("PinRoute/fixture-pin-model"), "{note}"); |
| 413 | let second = settle(&manager, meta["agent_id"].as_str().unwrap()).await; |
| 414 | assert_eq!( |
| 415 | second.status, |
| 416 | SubAgentStatus::Completed, |
| 417 | "{:?}", |
| 418 | second.status |
| 419 | ); |
| 420 | assert_eq!(pin_calls.load(Ordering::SeqCst), 1, "no second refusal"); |
| 421 | } |
| 422 | |
| 423 | #[tokio::test] |
| 424 | async fn strict_saved_profile_pin_stays_exact_and_names_its_source() { |
| 425 | let _env = crate::test_support::lock_test_env(); |
| 426 | let _profiles = ProjectProfiles::enabled(); |
| 427 | let root = tempdir().unwrap(); |
| 428 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 429 | write_refusing_reviewer_profile(root.path()).await; |
| 430 | let (tool, _context, _manager, _pin, _backup) = reviewer_tool(root.path(), "").await; |
| 431 | let runtime = tool.runtime.clone(); |
| 432 | let bind = |allow_fallback: bool| { |
| 433 | let runtime = runtime.clone(); |
| 434 | async move { |
| 435 | let roster = spawn_roster(&runtime); |
| 436 | let mut request = |
| 437 | parse_spawn_request(&json!({"type": "reviewer", "prompt": "Review."})).unwrap(); |
| 438 | let member = resolve_spawn_route_profile(&runtime, &mut request, &roster).unwrap(); |
| 439 | let mut child = runtime.child_runtime(); |
| 440 | bind_spawn_model_route(&mut child, &request, member.as_ref(), true, allow_fallback) |
| 441 | .await |
| 442 | .unwrap(); |
| 443 | child |
| 444 | } |
| 445 | }; |
| 446 | |
| 447 | let fallback = bind(true).await; |
| 448 | let origin = fallback.route_origin.as_deref().expect("origin"); |
| 449 | assert_eq!( |
| 450 | origin.parent.as_ref().map(|parent| parent.label.as_str()), |
| 451 | Some("deepseek/deepseek-v4-flash") |
| 452 | ); |
| 453 | |
| 454 | // An exact Fleet binding forbids the substitution: no parent route is |
| 455 | // armed, and the failure names the saved profile and how to change it. |
| 456 | let strict = bind(false).await; |
| 457 | let origin = strict.route_origin.as_deref().expect("origin"); |
| 458 | assert!(origin.parent.is_none()); |
| 459 | let refusal = anyhow::Error::new(LlmError::from_http_response(403, REFUSAL)); |
| 460 | let message = annotate_child_model_error_with_origin( |
| 461 | &subagent_failure_message(&refusal), |
| 462 | &strict.model, |
| 463 | strict.client.api_provider(), |
| 464 | &ModelRoute::Fixed(strict.model.clone()), |
| 465 | Some(origin), |
| 466 | ); |
| 467 | for fact in [ |
| 468 | "run out of credits", |
| 469 | "saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model", |
| 470 | "/fleet members", |
| 471 | "reviewer.toml", |
| 472 | "forbids falling back to the parent route", |
| 473 | ] { |
| 474 | assert!(message.contains(fact), "{fact} missing from {message}"); |
| 475 | } |
| 476 | assert!( |
| 477 | !message.contains("explicit child model override"), |
| 478 | "no override was given: {message}" |
| 479 | ); |
| 480 | } |
| 481 | |
| 482 | #[test] |
| 483 | fn only_typed_auth_and_credit_refusals_move_a_saved_pin() { |
| 484 | let refusal = |status| anyhow::Error::new(LlmError::from_http_response(status, REFUSAL)); |
| 485 | assert!(pin_refusal_reason(&refusal(403)).is_some()); |
| 486 | assert!(pin_refusal_reason(&refusal(401)).is_some()); |
| 487 | // Transient failures keep the ordinary retry path; no route change. |
| 488 | for transient in [ |
| 489 | LlmError::RateLimited { |
| 490 | message: "slow down".into(), |
| 491 | retry_after: None, |
| 492 | }, |
| 493 | LlmError::NetworkError("connection reset".into()), |
| 494 | LlmError::ServerError { |
| 495 | status: 503, |
| 496 | message: "unavailable".into(), |
| 497 | }, |
| 498 | LlmError::ModelError("no such model".into()), |
| 499 | ] { |
| 500 | assert_eq!(pin_refusal_reason(&anyhow::Error::new(transient)), None); |
| 501 | } |
| 502 | assert_eq!(pin_refusal_reason(&anyhow!(REFUSAL)), None); |
| 503 | } |
| 504 | |
| 505 | /// Finding on #6717: after the fallback, a failure on the parent route was |
| 506 | /// still reported against the refused pin ("change it in /fleet members"). |
| 507 | #[tokio::test] |
| 508 | async fn a_failure_after_the_fallback_names_the_route_that_failed() { |
| 509 | let _env = crate::test_support::lock_test_env(); |
| 510 | let _profiles = ProjectProfiles::enabled(); |
| 511 | let root = tempdir().unwrap(); |
| 512 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 513 | let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek"); |
| 514 | let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash"); |
| 515 | write_refusing_reviewer_profile(root.path()).await; |
| 516 | let (tool, context, manager, calls, _backup) = |
| 517 | reviewer_tool_with(root.path(), "", /* parent_refuses */ true).await; |
| 518 | let started = tool |
| 519 | .execute( |
| 520 | json!({"type": "reviewer", "prompt": "Review the change."}), |
| 521 | &context, |
| 522 | ) |
| 523 | .await |
| 524 | .unwrap(); |
| 525 | let meta = started.metadata.clone().unwrap(); |
| 526 | let result = settle(&manager, meta["agent_id"].as_str().unwrap()).await; |
| 527 | assert_eq!(calls.load(Ordering::SeqCst), 2, "pin once, parent once"); |
| 528 | let SubAgentStatus::Failed(error) = &result.status else { |
| 529 | panic!("both routes refused: {:?}", result.status); |
| 530 | }; |
| 531 | for fact in [ |
| 532 | "requested model `deepseek-v4-flash`", |
| 533 | "deepseek/deepseek-v4-flash failed too", |
| 534 | "saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model", |
| 535 | ] { |
| 536 | assert!(error.contains(fact), "{fact} missing from {error}"); |
| 537 | } |
| 538 | assert!( |
| 539 | !error.contains("requested model `fixture-pin-model`"), |
| 540 | "the refused pin did not make the failing request: {error}" |
| 541 | ); |
| 542 | assert!( |
| 543 | manager.read().await.rerouted_requests.is_empty(), |
| 544 | "taken when the task settles" |
| 545 | ); |
| 546 | } |
| 547 | |
| 548 | /// Finding on #6717: a resumed child skips the spawn's route binding, so its |
| 549 | /// origin comes from the saved receipt instead of a guess from the model. |
| 550 | #[test] |
| 551 | fn a_resumed_child_names_its_saved_route_source() { |
| 552 | let receipt: ChildRouteReceipt = serde_json::from_value(json!({ |
| 553 | "requested_type": "reviewer", |
| 554 | "requested_profile": null, |
| 555 | "resolved_profile_id": "reviewer", |
| 556 | "profile_origin": null, |
| 557 | "canonical_role": "reviewer", |
| 558 | "provider_id": "PinRoute", |
| 559 | "model_id": "fixture-pin-model", |
| 560 | "route_source": "agent_profile.model", |
| 561 | "fallback_note": null, |
| 562 | "requested_reasoning": "auto", |
| 563 | "effective_reasoning": null, |
| 564 | "runtime_version": "0", |
| 565 | "runtime_build_sha": "0" |
| 566 | })) |
| 567 | .expect("receipt"); |
| 568 | let origin = route_origin_from_receipt(&receipt).expect("known source"); |
| 569 | let refusal = anyhow::Error::new(LlmError::from_http_response(403, REFUSAL)); |
| 570 | let message = annotate_child_model_error_with_origin( |
| 571 | &subagent_failure_message(&refusal), |
| 572 | "fixture-pin-model", |
| 573 | crate::config::ProviderKind::Deepseek, |
| 574 | &ModelRoute::Fixed("fixture-pin-model".into()), |
| 575 | Some(&origin), |
| 576 | ); |
| 577 | assert!( |
| 578 | message.contains("saved agent profile \"reviewer\" pins PinRoute/fixture-pin-model"), |
| 579 | "{message}" |
| 580 | ); |
| 581 | assert!(message.contains("/fleet members"), "{message}"); |
| 582 | assert!( |
| 583 | !message.contains("explicit child model override"), |
| 584 | "no override was given: {message}" |
| 585 | ); |
| 586 | // Without any origin, a fixed route no longer claims an override either. |
| 587 | let message = annotate_child_model_error( |
| 588 | &subagent_failure_message(&refusal), |
| 589 | "fixture-pin-model", |
| 590 | crate::config::ProviderKind::Deepseek, |
| 591 | &ModelRoute::Fixed("fixture-pin-model".into()), |
| 592 | ); |
| 593 | assert!( |
| 594 | !message.contains("explicit child model override"), |
| 595 | "{message}" |
| 596 | ); |
| 597 | } |
| 598 | |
| 599 | /// The approved replacement owns the response grammar. Its distinct Responses |
| 600 | /// item IDs do not make a repeated call_id safe to dispatch under Chat rules. |
| 601 | #[tokio::test] |
| 602 | async fn responses_replacement_rejects_pairing_collisions_using_its_actual_route() { |
| 603 | let _env = crate::test_support::lock_test_env(); |
| 604 | let root = tempdir().unwrap(); |
| 605 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path().join("state")); |
| 606 | let _provider = crate::test_support::EnvVarGuard::set("CODEWHALE_PROVIDER", "deepseek"); |
| 607 | let _model = crate::test_support::EnvVarGuard::set("CODEWHALE_MODEL", "deepseek-v4-flash"); |
| 608 | let (pin_url, pin_calls) = refusing_chat_server().await; |
| 609 | let backup = wiremock::MockServer::start().await; |
| 610 | let mut frames = vec![ |
| 611 | json!({"type":"response.output_item.added", "item":{"type":"message", "id":"message"}}), |
| 612 | json!({"type":"response.output_text.delta", "delta":"Retained replacement response text."}), |
| 613 | json!({"type":"response.output_item.done"}), |
| 614 | ]; |
| 615 | for item in ["item_first", "item_second"] { |
| 616 | frames.extend([ |
| 617 | json!({"type":"response.output_item.added", "item":{ |
| 618 | "type":"function_call", "id":item, "call_id":"same_call", "name":"bash" |
| 619 | }}), |
| 620 | json!({"type":"response.function_call_arguments.delta", "delta":r#"{"command":"pwd"}"#}), |
| 621 | json!({"type":"response.output_item.done"}), |
| 622 | ]); |
| 623 | } |
| 624 | frames.push(json!({"type":"response.completed", "response":{ |
| 625 | "status":"completed", "usage":{"input_tokens":10, "output_tokens":5} |
| 626 | }})); |
| 627 | let body = frames |
| 628 | .into_iter() |
| 629 | .map(|frame| format!("data: {frame}\n\n")) |
| 630 | .collect::<String>(); |
| 631 | wiremock::Mock::given(wiremock::matchers::method("POST")) |
| 632 | .and(wiremock::matchers::path("/v1/responses")) |
| 633 | .respond_with( |
| 634 | wiremock::ResponseTemplate::new(200) |
| 635 | .insert_header("Content-Type", "text/event-stream") |
| 636 | .set_body_string(body), |
| 637 | ) |
| 638 | .expect(1) |
| 639 | .mount(&backup) |
| 640 | .await; |
| 641 | let config_path = root.path().join("config.toml"); |
| 642 | write_replacement_config( |
| 643 | &config_path, |
| 644 | &pin_url, |
| 645 | &format!("{}/v1", backup.uri()), |
| 646 | r#" |
| 647 | wire = "responses" |
| 648 | |
| 649 | [subagents.roles.reviewer] |
| 650 | model = "PinRoute/fixture-pin-model" |
| 651 | replacements = ["BackupRoute/fixture-backup-model"] |
| 652 | "#, |
| 653 | ) |
| 654 | .await; |
| 655 | let config = crate::config::Config::load(Some(config_path), None).unwrap(); |
| 656 | let client = CodewhaleClient::new(&config).unwrap(); |
| 657 | assert_eq!( |
| 658 | client.wire_format(), |
| 659 | codewhale_config::provider::WireFormat::ChatCompletions |
| 660 | ); |
| 661 | let manager = new_shared_subagent_manager(root.path().to_path_buf(), 2); |
| 662 | let context = ToolContext::new(root.path()).with_state_namespace("route-identity"); |
| 663 | let runtime = SubAgentRuntime::new( |
| 664 | client, |
| 665 | "deepseek-v4-flash".into(), |
| 666 | context.clone(), |
| 667 | false, |
| 668 | None, |
| 669 | manager.clone(), |
| 670 | ) |
| 671 | .with_api_config(config); |
| 672 | let tool = AgentTool::new(manager.clone(), runtime); |
| 673 | let started = tool |
| 674 | .execute( |
| 675 | json!({"type":"reviewer", "prompt":"Inspect the workspace."}), |
| 676 | &context, |
| 677 | ) |
| 678 | .await |
| 679 | .unwrap(); |
| 680 | let metadata = started.metadata.as_ref().unwrap(); |
| 681 | let id = metadata["agent_id"].as_str().unwrap(); |
| 682 | assert_eq!(metadata["child_route"]["provider_id"], "PinRoute"); |
| 683 | let result = tokio::time::timeout(Duration::from_secs(10), async { |
| 684 | loop { |
| 685 | let result = manager.read().await.get_result(id).unwrap(); |
| 686 | if result.status != SubAgentStatus::Running { |
| 687 | return result; |
| 688 | } |
| 689 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 690 | } |
| 691 | }) |
| 692 | .await |
| 693 | .expect("replacement pairing refusal settles"); |
| 694 | let SubAgentStatus::Failed(error) = &result.status else { |
| 695 | panic!("ambiguous replacement must fail: {:?}", result.status); |
| 696 | }; |
| 697 | assert!(error.contains("pairing id"), "{error}"); |
| 698 | assert_eq!(pin_calls.load(Ordering::SeqCst), 1); |
| 699 | let requests = backup.received_requests().await.unwrap(); |
| 700 | assert_eq!( |
| 701 | requests.len(), |
| 702 | 1, |
| 703 | "no tool result round follows refused admission" |
| 704 | ); |
| 705 | let request: Value = serde_json::from_slice(&requests[0].body).unwrap(); |
| 706 | assert!(request.get("input").is_some()); |
| 707 | assert!(request.get("messages").is_none()); |
| 708 | assert_eq!(request["model"], "fixture-backup-model"); |
| 709 | assert!( |
| 710 | result.usage.as_ref().and_then(|usage| usage.total_tokens) == Some(15), |
| 711 | "the refused response remains billed" |
| 712 | ); |
| 713 | let checkpoint = result |
| 714 | .checkpoint |
| 715 | .as_ref() |
| 716 | .expect("durable refused response"); |
| 717 | assert!(checkpoint.messages.iter().flat_map(|m| &m.content).any( |
| 718 | |block| matches!(block, ContentBlock::Text { text, .. } if text == "Retained replacement response text.") |
| 719 | )); |
| 720 | assert!( |
| 721 | checkpoint |
| 722 | .messages |
| 723 | .iter() |
| 724 | .flat_map(|m| &m.content) |
| 725 | .all(|block| !matches!( |
| 726 | block, |
| 727 | ContentBlock::ToolUse { .. } | ContentBlock::ToolResult { .. } |
| 728 | )) |
| 729 | ); |
| 730 | let route = result.child_route.unwrap(); |
| 731 | assert_eq!(route.provider_id, "BackupRoute"); |
| 732 | assert_eq!(route.route_source, "role.replacement"); |
| 733 | } |
| 734 | |
| 735 | #[path = "persona_receipt.rs"] |
| 736 | mod persona_receipt; |
| 737 |