| 1 | //! Local-only release runtime QA through real pseudo-terminals. |
| 2 | //! |
| 3 | //! These scenarios cover the live TUI checks that unit tests cannot prove: |
| 4 | //! six-worker fanout liveness/cancellation, multi-terminal route isolation, |
| 5 | //! and the explicit Enter-queue / Ctrl+Enter-steer contract. Every provider is a loopback wiremock |
| 6 | //! server and every process receives a sealed HOME. |
| 7 | |
| 8 | #![cfg(unix)] |
| 9 | |
| 10 | #[path = "support/qa_harness/mod.rs"] |
| 11 | mod qa_harness; |
| 12 | |
| 13 | use std::sync::Arc; |
| 14 | use std::sync::atomic::{AtomicUsize, Ordering}; |
| 15 | use std::time::{Duration, Instant}; |
| 16 | |
| 17 | use anyhow::{Result, anyhow}; |
| 18 | use qa_harness::harness::{Harness, SealedWorkspace, make_sealed_workspace}; |
| 19 | use qa_harness::keys; |
| 20 | use serde_json::{Value, json}; |
| 21 | use wiremock::matchers::{method, path}; |
| 22 | use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; |
| 23 | |
| 24 | const BOOT_TIMEOUT: Duration = Duration::from_secs(20); |
| 25 | const INTERACTION_TIMEOUT: Duration = Duration::from_secs(15); |
| 26 | const PASTE_GUARD_SETTLE: Duration = Duration::from_millis(180); |
| 27 | const COMPOSER_READY_TEXT: &str = "Write a task"; |
| 28 | const MUSE_MODEL: &str = "muse-spark-1.1"; |
| 29 | const GPT_MODEL: &str = "gpt-5.6-terra"; |
| 30 | const DEEPSEEK_TEST_MODEL: &str = "deepseek-v4-pro"; |
| 31 | static RELEASE_RUNTIME_QA_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); |
| 32 | |
| 33 | fn sse_chunk(value: Value) -> String { |
| 34 | format!( |
| 35 | "data: {}\n\n", |
| 36 | serde_json::to_string(&value).expect("SSE JSON") |
| 37 | ) |
| 38 | } |
| 39 | |
| 40 | fn text_sse(model: &str, text: &str) -> String { |
| 41 | [ |
| 42 | sse_chunk(json!({ |
| 43 | "id": "chatcmpl-local-qa", |
| 44 | "object": "chat.completion.chunk", |
| 45 | "model": model, |
| 46 | "choices": [{ |
| 47 | "index": 0, |
| 48 | "delta": { "content": text }, |
| 49 | "finish_reason": null |
| 50 | }] |
| 51 | })), |
| 52 | sse_chunk(json!({ |
| 53 | "id": "chatcmpl-local-qa", |
| 54 | "object": "chat.completion.chunk", |
| 55 | "model": model, |
| 56 | "choices": [{ |
| 57 | "index": 0, |
| 58 | "delta": {}, |
| 59 | "finish_reason": "stop" |
| 60 | }], |
| 61 | "usage": { |
| 62 | "prompt_tokens": 12, |
| 63 | "completion_tokens": 4, |
| 64 | "total_tokens": 16 |
| 65 | } |
| 66 | })), |
| 67 | "data: [DONE]\n\n".to_string(), |
| 68 | ] |
| 69 | .join("") |
| 70 | } |
| 71 | |
| 72 | fn fanout_tool_call_sse() -> String { |
| 73 | fanout_tool_call_sse_n(6) |
| 74 | } |
| 75 | |
| 76 | fn fanout_tool_call_sse_n(count: usize) -> String { |
| 77 | let tool_calls = (1..=count) |
| 78 | .map(|worker| { |
| 79 | json!({ |
| 80 | "index": worker - 1, |
| 81 | "id": format!("call_agent_{worker}"), |
| 82 | "type": "function", |
| 83 | "function": { |
| 84 | "name": "agent", |
| 85 | "arguments": serde_json::to_string(&json!({ |
| 86 | "message": format!("stay busy worker {worker} until the parent QA turn is cancelled"), |
| 87 | "agent_type": "explorer", |
| 88 | // Explicit fresh context: this harness dispatches mock |
| 89 | // responses on request content, and an auto-forked |
| 90 | // child would carry the parent conversation (including |
| 91 | // the parent prompt) in its requests. Explicit false |
| 92 | // always wins over the auto-fork policy. |
| 93 | "fork_context": false, |
| 94 | "session_name": format!("qa-worker-{worker}") |
| 95 | })) |
| 96 | .expect("agent arguments") |
| 97 | } |
| 98 | }) |
| 99 | }) |
| 100 | .collect::<Vec<_>>(); |
| 101 | |
| 102 | [ |
| 103 | sse_chunk(json!({ |
| 104 | "id": "chatcmpl-fanout", |
| 105 | "object": "chat.completion.chunk", |
| 106 | "model": DEEPSEEK_TEST_MODEL, |
| 107 | "choices": [{ |
| 108 | "index": 0, |
| 109 | "delta": { "tool_calls": tool_calls }, |
| 110 | "finish_reason": null |
| 111 | }] |
| 112 | })), |
| 113 | sse_chunk(json!({ |
| 114 | "id": "chatcmpl-fanout", |
| 115 | "object": "chat.completion.chunk", |
| 116 | "model": DEEPSEEK_TEST_MODEL, |
| 117 | "choices": [{ |
| 118 | "index": 0, |
| 119 | "delta": {}, |
| 120 | "finish_reason": "tool_calls" |
| 121 | }], |
| 122 | "usage": { |
| 123 | "prompt_tokens": 20, |
| 124 | "completion_tokens": 12, |
| 125 | "total_tokens": 32 |
| 126 | } |
| 127 | })), |
| 128 | "data: [DONE]\n\n".to_string(), |
| 129 | ] |
| 130 | .join("") |
| 131 | } |
| 132 | |
| 133 | fn fleet_role_tool_call_sse() -> String { |
| 134 | let roles = ["worker", "scout", "reviewer", "verifier"]; |
| 135 | let tool_calls = roles |
| 136 | .iter() |
| 137 | .enumerate() |
| 138 | .map(|(index, role)| { |
| 139 | json!({ |
| 140 | "index": index, |
| 141 | "id": format!("call_role_{role}"), |
| 142 | "type": "function", |
| 143 | "function": { |
| 144 | "name": "agent", |
| 145 | "arguments": serde_json::to_string(&json!({ |
| 146 | "action": "start", |
| 147 | "prompt": format!("role-probe-{role}"), |
| 148 | "type": role, |
| 149 | "fork_context": false, |
| 150 | "session_name": format!("qa-{role}"), |
| 151 | "workspace_policy": "shared", |
| 152 | "write_authority": "read_only", |
| 153 | "expected_artifact": "one role launch receipt", |
| 154 | "deliberate": true |
| 155 | })) |
| 156 | .expect("Fleet role arguments") |
| 157 | } |
| 158 | }) |
| 159 | }) |
| 160 | .collect::<Vec<_>>(); |
| 161 | |
| 162 | [ |
| 163 | sse_chunk(json!({ |
| 164 | "id": "chatcmpl-fleet-roles", |
| 165 | "object": "chat.completion.chunk", |
| 166 | "model": DEEPSEEK_TEST_MODEL, |
| 167 | "choices": [{ |
| 168 | "index": 0, |
| 169 | "delta": { "tool_calls": tool_calls }, |
| 170 | "finish_reason": null |
| 171 | }] |
| 172 | })), |
| 173 | sse_chunk(json!({ |
| 174 | "id": "chatcmpl-fleet-roles", |
| 175 | "object": "chat.completion.chunk", |
| 176 | "model": DEEPSEEK_TEST_MODEL, |
| 177 | "choices": [{ |
| 178 | "index": 0, |
| 179 | "delta": {}, |
| 180 | "finish_reason": "tool_calls" |
| 181 | }], |
| 182 | "usage": { |
| 183 | "prompt_tokens": 20, |
| 184 | "completion_tokens": 12, |
| 185 | "total_tokens": 32 |
| 186 | } |
| 187 | })), |
| 188 | "data: [DONE]\n\n".to_string(), |
| 189 | ] |
| 190 | .join("") |
| 191 | } |
| 192 | |
| 193 | fn sse_response(body: String) -> ResponseTemplate { |
| 194 | ResponseTemplate::new(200) |
| 195 | .insert_header("content-type", "text/event-stream") |
| 196 | .insert_header("cache-control", "no-cache") |
| 197 | .set_body_string(body) |
| 198 | } |
| 199 | |
| 200 | fn json_response(value: Value) -> ResponseTemplate { |
| 201 | ResponseTemplate::new(200) |
| 202 | .insert_header("content-type", "application/json") |
| 203 | .set_body_json(value) |
| 204 | } |
| 205 | |
| 206 | async fn mount_models(server: &MockServer, models: &[&str]) { |
| 207 | Mock::given(method("GET")) |
| 208 | .and(path("/v1/models")) |
| 209 | .respond_with(json_response(json!({ |
| 210 | "object": "list", |
| 211 | "data": models |
| 212 | .iter() |
| 213 | .map(|model| json!({ "id": model, "object": "model" })) |
| 214 | .collect::<Vec<_>>() |
| 215 | }))) |
| 216 | .mount(server) |
| 217 | .await; |
| 218 | } |
| 219 | |
| 220 | async fn mount_text_model(server: &MockServer, model: &str, answer: &str) { |
| 221 | mount_models(server, &[model]).await; |
| 222 | Mock::given(method("POST")) |
| 223 | .and(path("/v1/chat/completions")) |
| 224 | .respond_with(sse_response(text_sse(model, answer))) |
| 225 | .mount(server) |
| 226 | .await; |
| 227 | } |
| 228 | |
| 229 | fn common_tui_builder(ws: &SealedWorkspace) -> qa_harness::harness::HarnessBuilder { |
| 230 | Harness::builder(Harness::cargo_bin("codewhale-tui")) |
| 231 | .cwd(ws.workspace()) |
| 232 | .clear_env() |
| 233 | .seal_home(ws.home()) |
| 234 | .env("RUST_LOG", "warn") |
| 235 | .args([ |
| 236 | "--workspace", |
| 237 | ws.workspace().to_str().expect("utf-8 workspace path"), |
| 238 | "--no-project-config", |
| 239 | "--skip-onboarding", |
| 240 | ]) |
| 241 | .size(42, 150) |
| 242 | } |
| 243 | |
| 244 | /// Release scenarios exercise the direct-session runtime. The optional launch |
| 245 | /// screen is not enabled in these sealed homes. |
| 246 | fn enter_launch_session(harness: &mut Harness) -> Result<()> { |
| 247 | harness.wait_for_text(COMPOSER_READY_TEXT, BOOT_TIMEOUT)?; |
| 248 | Ok(()) |
| 249 | } |
| 250 | |
| 251 | fn wait_for_counter( |
| 252 | harness: &mut Harness, |
| 253 | counter: &AtomicUsize, |
| 254 | expected: usize, |
| 255 | timeout: Duration, |
| 256 | ) -> Result<()> { |
| 257 | let deadline = Instant::now() + timeout; |
| 258 | loop { |
| 259 | harness.pump(); |
| 260 | if counter.load(Ordering::SeqCst) >= expected { |
| 261 | return Ok(()); |
| 262 | } |
| 263 | // A dead child renders as a hang: `pump()` only feeds *new* bytes into a |
| 264 | // retained frame, so `debug_dump()` keeps painting the last frame the TUI |
| 265 | // drew before it died. Without this probe a SIGABRT (stack overflow aborts |
| 266 | // as 134) burns the whole timeout and then reports a live-looking screen. |
| 267 | if let Some(code) = harness.wait_for_exit(Duration::from_millis(0)) { |
| 268 | return Err(anyhow!( |
| 269 | "TUI exited with {code} before the counter reached {expected}; observed {}\n{}", |
| 270 | counter.load(Ordering::SeqCst), |
| 271 | harness.debug_dump() |
| 272 | )); |
| 273 | } |
| 274 | if Instant::now() >= deadline { |
| 275 | return Err(anyhow!( |
| 276 | "counter did not reach {expected} within {timeout:?}; observed {}\n{}", |
| 277 | counter.load(Ordering::SeqCst), |
| 278 | harness.debug_dump() |
| 279 | )); |
| 280 | } |
| 281 | std::thread::sleep(Duration::from_millis(40)); |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | fn type_and_submit(harness: &mut Harness, text: &str) -> Result<()> { |
| 286 | harness.send(keys::key::text(text))?; |
| 287 | // Rapid PTY writes intentionally exercise paste-burst detection. Wait |
| 288 | // beyond its 120 ms trailing-Enter suppression window before submitting. |
| 289 | // Ambient ocean life keeps repainting even when the runtime is idle, so |
| 290 | // visual frame stability is not a valid readiness signal. |
| 291 | harness.wait_for_text(text, Duration::from_secs(3))?; |
| 292 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 293 | harness.pump(); |
| 294 | harness.send(keys::key::enter())?; |
| 295 | Ok(()) |
| 296 | } |
| 297 | |
| 298 | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] |
| 299 | async fn underwater_footer_moves_from_working_through_one_shot_completion() -> Result<()> { |
| 300 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 301 | let server = MockServer::start().await; |
| 302 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 303 | Mock::given(method("POST")) |
| 304 | .and(path("/v1/chat/completions")) |
| 305 | .respond_with( |
| 306 | sse_response(text_sse(DEEPSEEK_TEST_MODEL, "local phase proof")) |
| 307 | .set_delay(Duration::from_millis(850)), |
| 308 | ) |
| 309 | .mount(&server) |
| 310 | .await; |
| 311 | |
| 312 | let ws = make_sealed_workspace()?; |
| 313 | let mut tui = common_tui_builder(&ws) |
| 314 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 315 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 316 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 317 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 318 | .spawn()?; |
| 319 | match tui.wait_for_text(COMPOSER_READY_TEXT, BOOT_TIMEOUT) { |
| 320 | Ok(()) => {} |
| 321 | Err(e) => { |
| 322 | eprintln!("[dogfood] launch wait error debug: {e:?}"); |
| 323 | return Err(anyhow::anyhow!("launch wait failed: {e:#}")); |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | type_and_submit(&mut tui, "show the underwater phase transition")?; |
| 328 | // TUI-DOG-008: live phases (working/finishing/done) render on the phase |
| 329 | // strip ABOVE the composer, so the bottom row is no longer the phase |
| 330 | // owner. Assert the phase words anywhere in the frame — the mock reply |
| 331 | // ("local phase proof") and the prompt contain none of them. |
| 332 | tui.wait_for(|frame| frame.contains("working"), INTERACTION_TIMEOUT)?; |
| 333 | tui.wait_for( |
| 334 | |frame| frame.contains("finishing") || frame.contains("✓ done"), |
| 335 | INTERACTION_TIMEOUT, |
| 336 | )?; |
| 337 | tui.wait_for(|frame| frame.contains("✓ done"), INTERACTION_TIMEOUT)?; |
| 338 | |
| 339 | let _ = tui.shutdown(); |
| 340 | Ok(()) |
| 341 | } |
| 342 | |
| 343 | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] |
| 344 | async fn underwater_theme_picker_emits_each_live_palette_to_the_terminal() -> Result<()> { |
| 345 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 346 | let ws = make_sealed_workspace()?; |
| 347 | let mut tui = common_tui_builder(&ws) |
| 348 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 349 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 350 | .env("DEEPSEEK_BASE_URL", "http://127.0.0.1:1") |
| 351 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 352 | .env("COLORTERM", "truecolor") |
| 353 | .env("RUST_BACKTRACE", "1") |
| 354 | .spawn()?; |
| 355 | enter_launch_session(&mut tui)?; |
| 356 | // A bracketed paste plus trailing space makes this an explicit command |
| 357 | // invocation, outside both autocomplete and unbracketed burst handling. |
| 358 | tui.paste("/theme ")?; |
| 359 | tui.wait_for_text("/theme", Duration::from_secs(3))?; |
| 360 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 361 | tui.pump(); |
| 362 | tui.send(keys::key::enter())?; |
| 363 | std::thread::sleep(Duration::from_millis(300)); |
| 364 | tui.pump(); |
| 365 | if let Some(status) = tui.wait_for_exit(Duration::from_millis(1)) { |
| 366 | let logs = std::fs::read_dir(ws.home().join(".codewhale/logs")) |
| 367 | .ok() |
| 368 | .into_iter() |
| 369 | .flatten() |
| 370 | .filter_map(Result::ok) |
| 371 | .filter_map(|entry| std::fs::read_to_string(entry.path()).ok()) |
| 372 | .collect::<Vec<_>>() |
| 373 | .join("\n"); |
| 374 | return Err(anyhow!( |
| 375 | "theme picker process exited with {status}:\n{}\nlogs:\n{logs}", |
| 376 | tui.debug_dump(), |
| 377 | )); |
| 378 | } |
| 379 | if tui |
| 380 | .wait_for_text("live preview", Duration::from_secs(1)) |
| 381 | .is_err() |
| 382 | { |
| 383 | // A PTY can deliver the first Enter inside the paste guard's trailing |
| 384 | // suppression window. Once that window expires, the next deliberate |
| 385 | // Enter must execute the retained draft. |
| 386 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 387 | tui.pump(); |
| 388 | tui.send(keys::key::enter())?; |
| 389 | tui.wait_for_text("live preview", INTERACTION_TIMEOUT)?; |
| 390 | } |
| 391 | |
| 392 | let labels = [ |
| 393 | "System", |
| 394 | "Terminal", |
| 395 | "Blue Stage", |
| 396 | "Blue Stage Light", |
| 397 | "Grayscale", |
| 398 | "Catppuccin Mocha", |
| 399 | "Tokyo Night", |
| 400 | "Dracula", |
| 401 | "Gruvbox Dark", |
| 402 | "Claude", |
| 403 | "Matrix", |
| 404 | "Solarized Light", |
| 405 | ]; |
| 406 | let mut previous_signature = None; |
| 407 | for (index, label) in labels.iter().enumerate() { |
| 408 | let selected = format!("▸ {}.", index + 1); |
| 409 | tui.wait_for( |
| 410 | |frame| frame.text().contains(&selected), |
| 411 | INTERACTION_TIMEOUT, |
| 412 | )?; |
| 413 | let frame = tui.frame(); |
| 414 | let signature = ( |
| 415 | frame.colors_at(0, 0).expect("theme surface cell"), |
| 416 | frame |
| 417 | .first_symbol_colors("▸") |
| 418 | .expect("selected theme pointer cell"), |
| 419 | ); |
| 420 | assert!( |
| 421 | frame.text().contains(label), |
| 422 | "missing theme row {label}:\n{}", |
| 423 | frame.debug_dump() |
| 424 | ); |
| 425 | if let Some(previous) = previous_signature { |
| 426 | assert_ne!( |
| 427 | signature, |
| 428 | previous, |
| 429 | "live ANSI palette did not change from {} to {label}", |
| 430 | labels[index - 1] |
| 431 | ); |
| 432 | } |
| 433 | previous_signature = Some(signature); |
| 434 | if index + 1 < labels.len() { |
| 435 | tui.send(b"\x1b[B")?; |
| 436 | std::thread::sleep(Duration::from_millis(250)); |
| 437 | tui.pump(); |
| 438 | if let Some(status) = tui.wait_for_exit(Duration::from_millis(1)) { |
| 439 | let logs = std::fs::read_dir(ws.home().join(".codewhale/logs")) |
| 440 | .ok() |
| 441 | .into_iter() |
| 442 | .flatten() |
| 443 | .filter_map(Result::ok) |
| 444 | .filter_map(|entry| std::fs::read_to_string(entry.path()).ok()) |
| 445 | .collect::<Vec<_>>() |
| 446 | .join("\n"); |
| 447 | return Err(anyhow!( |
| 448 | "theme preview exited with {status}:\n{}\nlogs:\n{logs}", |
| 449 | tui.debug_dump() |
| 450 | )); |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | tui.send(b"\x1b")?; |
| 456 | let _ = tui.shutdown(); |
| 457 | Ok(()) |
| 458 | } |
| 459 | |
| 460 | fn chat_requests(requests: &[Request]) -> Vec<Value> { |
| 461 | requests |
| 462 | .iter() |
| 463 | .filter(|request| request.url.path().ends_with("/chat/completions")) |
| 464 | .map(|request| request.body_json().expect("chat body JSON")) |
| 465 | .collect() |
| 466 | } |
| 467 | |
| 468 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 469 | async fn release_multi_terminal_muse_and_gpt_routes_stay_isolated() -> Result<()> { |
| 470 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 471 | let meta_server = MockServer::start().await; |
| 472 | let openai_server = MockServer::start().await; |
| 473 | mount_text_model(&meta_server, MUSE_MODEL, "meta-route-ok").await; |
| 474 | mount_models(&openai_server, &["gpt-5.6-luna", GPT_MODEL]).await; |
| 475 | Mock::given(method("POST")) |
| 476 | .and(path("/v1/chat/completions")) |
| 477 | .respond_with(sse_response(text_sse(GPT_MODEL, "openai-route-ok"))) |
| 478 | .mount(&openai_server) |
| 479 | .await; |
| 480 | |
| 481 | let ws = make_sealed_workspace()?; |
| 482 | let openai_base_url = openai_server.uri(); |
| 483 | let meta_base_url = meta_server.uri(); |
| 484 | let shared_openai_env = [ |
| 485 | ("OPENAI_API_KEY", "openai-local-test-key"), |
| 486 | ("OPENAI_BASE_URL", openai_base_url.as_str()), |
| 487 | ("OPENAI_MODEL", "gpt-5.6-luna"), |
| 488 | ]; |
| 489 | let shared_meta_env = [ |
| 490 | ("META_MODEL_API_KEY", "meta-local-test-key"), |
| 491 | ("MODEL_API_KEY", "meta-local-test-key"), |
| 492 | ("META_MODEL_API_BASE_URL", meta_base_url.as_str()), |
| 493 | ("META_MODEL_API_MODEL", MUSE_MODEL), |
| 494 | ]; |
| 495 | |
| 496 | let mut meta_builder = common_tui_builder(&ws).env("CODEWHALE_PROVIDER", "meta"); |
| 497 | let mut openai_builder = common_tui_builder(&ws).env("CODEWHALE_PROVIDER", "openai"); |
| 498 | for (key, value) in shared_openai_env.into_iter().chain(shared_meta_env) { |
| 499 | meta_builder = meta_builder.env(key, value); |
| 500 | openai_builder = openai_builder.env(key, value); |
| 501 | } |
| 502 | |
| 503 | let mut meta_tui = meta_builder.spawn()?; |
| 504 | let mut openai_tui = openai_builder.spawn()?; |
| 505 | enter_launch_session(&mut meta_tui)?; |
| 506 | enter_launch_session(&mut openai_tui)?; |
| 507 | |
| 508 | // Change terminal B's model through the live command path while terminal A |
| 509 | // remains open on Meta. Both processes share one sealed settings file. |
| 510 | type_and_submit(&mut openai_tui, "/model gpt-5.6-terra")?; |
| 511 | openai_tui.wait_for( |
| 512 | |frame| frame.row(0).contains(GPT_MODEL), |
| 513 | INTERACTION_TIMEOUT, |
| 514 | )?; |
| 515 | assert!( |
| 516 | meta_tui.frame().contains(MUSE_MODEL), |
| 517 | "terminal A route changed when terminal B selected a model:\n{}", |
| 518 | meta_tui.debug_dump() |
| 519 | ); |
| 520 | |
| 521 | type_and_submit(&mut meta_tui, "route probe from meta terminal")?; |
| 522 | type_and_submit(&mut openai_tui, "route probe from openai terminal")?; |
| 523 | meta_tui.wait_for_text("meta-route-ok", INTERACTION_TIMEOUT)?; |
| 524 | openai_tui.wait_for_text("openai-route-ok", INTERACTION_TIMEOUT)?; |
| 525 | |
| 526 | let meta_requests = meta_server.received_requests().await.unwrap_or_default(); |
| 527 | let openai_requests = openai_server.received_requests().await.unwrap_or_default(); |
| 528 | let meta_chat = chat_requests(&meta_requests); |
| 529 | let openai_chat = chat_requests(&openai_requests); |
| 530 | assert_eq!( |
| 531 | meta_chat.len(), |
| 532 | 1, |
| 533 | "unexpected Meta chat requests: {meta_chat:#?}" |
| 534 | ); |
| 535 | assert_eq!( |
| 536 | openai_chat.len(), |
| 537 | 1, |
| 538 | "unexpected OpenAI chat requests: {openai_chat:#?}" |
| 539 | ); |
| 540 | assert_eq!(meta_chat[0]["model"], MUSE_MODEL); |
| 541 | assert_eq!(openai_chat[0]["model"], GPT_MODEL); |
| 542 | assert!( |
| 543 | meta_chat[0] |
| 544 | .to_string() |
| 545 | .contains("route probe from meta terminal") |
| 546 | ); |
| 547 | assert!(!meta_chat[0].to_string().contains("openai terminal")); |
| 548 | assert!( |
| 549 | openai_chat[0] |
| 550 | .to_string() |
| 551 | .contains("route probe from openai terminal") |
| 552 | ); |
| 553 | assert!(!openai_chat[0].to_string().contains("meta terminal")); |
| 554 | |
| 555 | let _ = meta_tui.shutdown(); |
| 556 | let _ = openai_tui.shutdown(); |
| 557 | Ok(()) |
| 558 | } |
| 559 | |
| 560 | #[derive(Clone)] |
| 561 | struct FanoutResponder { |
| 562 | child_requests: Arc<AtomicUsize>, |
| 563 | } |
| 564 | |
| 565 | #[derive(Clone)] |
| 566 | struct FleetRoleResponder { |
| 567 | launched: Arc<AtomicUsize>, |
| 568 | canonical_prompts: Arc<AtomicUsize>, |
| 569 | worker: Arc<AtomicUsize>, |
| 570 | scout: Arc<AtomicUsize>, |
| 571 | reviewer: Arc<AtomicUsize>, |
| 572 | verifier: Arc<AtomicUsize>, |
| 573 | } |
| 574 | |
| 575 | impl Respond for FleetRoleResponder { |
| 576 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 577 | let body = request.body_json::<Value>().unwrap_or(Value::Null); |
| 578 | let raw = body.to_string(); |
| 579 | let role_markers = [ |
| 580 | ("role-probe-worker", "Fleet worker", &self.worker), |
| 581 | ("role-probe-scout", "Fleet scout", &self.scout), |
| 582 | ("role-probe-reviewer", "Fleet reviewer", &self.reviewer), |
| 583 | ("role-probe-verifier", "Fleet verifier", &self.verifier), |
| 584 | ]; |
| 585 | let matched = role_markers |
| 586 | .iter() |
| 587 | .filter(|(marker, _, _)| raw.contains(marker)) |
| 588 | .collect::<Vec<_>>(); |
| 589 | if matched.len() == 1 { |
| 590 | let (_, expected_prompt, counter) = matched[0]; |
| 591 | self.launched.fetch_add(1, Ordering::SeqCst); |
| 592 | counter.fetch_add(1, Ordering::SeqCst); |
| 593 | if raw.contains(expected_prompt) { |
| 594 | self.canonical_prompts.fetch_add(1, Ordering::SeqCst); |
| 595 | } |
| 596 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "role-launch-complete")); |
| 597 | } |
| 598 | |
| 599 | if raw.contains("launch four canonical read-only Fleet roles") { |
| 600 | return sse_response(fleet_role_tool_call_sse()); |
| 601 | } |
| 602 | |
| 603 | sse_response(text_sse( |
| 604 | DEEPSEEK_TEST_MODEL, |
| 605 | "fleet-role-receipts-complete", |
| 606 | )) |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | impl Respond for FanoutResponder { |
| 611 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 612 | let body = request.body_json::<Value>().unwrap_or(Value::Null); |
| 613 | let raw = body.to_string(); |
| 614 | |
| 615 | if raw.contains("stay busy worker") && !raw.contains("launch six QA workers") { |
| 616 | self.child_requests.fetch_add(1, Ordering::SeqCst); |
| 617 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "child-finished-too-soon")) |
| 618 | .set_delay(Duration::from_secs(20)); |
| 619 | } |
| 620 | |
| 621 | if raw.contains("launch six QA workers") { |
| 622 | return sse_response(fanout_tool_call_sse()); |
| 623 | } |
| 624 | |
| 625 | sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request")) |
| 626 | } |
| 627 | } |
| 628 | |
| 629 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 630 | async fn release_six_worker_fanout_keeps_typing_render_and_esc_cancel_live() -> Result<()> { |
| 631 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 632 | let server = MockServer::start().await; |
| 633 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 634 | let child_requests = Arc::new(AtomicUsize::new(0)); |
| 635 | Mock::given(method("POST")) |
| 636 | .and(path("/v1/chat/completions")) |
| 637 | .respond_with(FanoutResponder { |
| 638 | child_requests: Arc::clone(&child_requests), |
| 639 | }) |
| 640 | .mount(&server) |
| 641 | .await; |
| 642 | |
| 643 | let ws = make_sealed_workspace()?; |
| 644 | std::fs::write( |
| 645 | ws.home().join(".codewhale").join("config.toml"), |
| 646 | "[subagents]\nmax_concurrent = 6\nlaunch_concurrency = 6\nmax_admitted = 6\n", |
| 647 | )?; |
| 648 | let mut tui = common_tui_builder(&ws) |
| 649 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 650 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 651 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 652 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 653 | .args(["--yolo", "--max-subagents", "6"]) |
| 654 | .spawn()?; |
| 655 | enter_launch_session(&mut tui)?; |
| 656 | |
| 657 | type_and_submit( |
| 658 | &mut tui, |
| 659 | "launch six QA workers and keep the parent turn open", |
| 660 | )?; |
| 661 | wait_for_counter(&mut tui, &child_requests, 6, INTERACTION_TIMEOUT)?; |
| 662 | tui.wait_for( |
| 663 | |frame| { |
| 664 | let text = frame.text(); |
| 665 | text.matches("Agent ").count() >= 6 |
| 666 | || text.matches("delegate scout [running]").count() >= 6 |
| 667 | }, |
| 668 | Duration::from_secs(5), |
| 669 | )?; |
| 670 | |
| 671 | let fanout_frame = tui.debug_dump(); |
| 672 | assert!( |
| 673 | fanout_frame.matches("Agent ").count() >= 6 |
| 674 | || fanout_frame.matches("delegate scout [running]").count() >= 6, |
| 675 | "all six workers were not visible in the live runtime projection:\n{fanout_frame}" |
| 676 | ); |
| 677 | |
| 678 | // The provider is deliberately holding every child open. Prove keyboard |
| 679 | // input and rendering remain live during the storm, then interrupt the |
| 680 | // still-live orchestration turn directly with Esc. |
| 681 | tui.send(keys::key::text("fanout-live-marker"))?; |
| 682 | tui.wait_for_text("fanout-live-marker", Duration::from_secs(3))?; |
| 683 | let before_cancel = tui.debug_dump(); |
| 684 | assert!( |
| 685 | before_cancel.contains("Agent") || before_cancel.contains("agent"), |
| 686 | "fanout UI did not expose agent activity:\n{before_cancel}" |
| 687 | ); |
| 688 | |
| 689 | let cancel_started = Instant::now(); |
| 690 | tui.send(b"\x1b")?; |
| 691 | tui.wait_for( |
| 692 | |frame| { |
| 693 | let text = frame.text().to_ascii_lowercase(); |
| 694 | text.contains("cancelled") || text.contains("interrupted") |
| 695 | }, |
| 696 | Duration::from_secs(5), |
| 697 | )?; |
| 698 | assert!( |
| 699 | cancel_started.elapsed() < Duration::from_secs(5), |
| 700 | "Esc cancellation exceeded the five-second liveness budget" |
| 701 | ); |
| 702 | |
| 703 | // Let the raw-key paste-burst window from the pre-cancel marker expire. |
| 704 | // Without this guard, the first character of the next marker can remain |
| 705 | // retained while cancellation repaints, making this a paste-heuristic |
| 706 | // race instead of the intended post-cancel composer-liveness assertion. |
| 707 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 708 | tui.pump(); |
| 709 | tui.send(keys::key::text("post-cancel-live"))?; |
| 710 | tui.wait_for_text("post-cancel-live", Duration::from_secs(3))?; |
| 711 | assert_eq!(child_requests.load(Ordering::SeqCst), 6); |
| 712 | |
| 713 | let _ = tui.shutdown(); |
| 714 | Ok(()) |
| 715 | } |
| 716 | |
| 717 | struct SingleDispatchResponder { |
| 718 | child_requests: Arc<AtomicUsize>, |
| 719 | } |
| 720 | |
| 721 | impl Respond for SingleDispatchResponder { |
| 722 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 723 | let body = request.body_json::<Value>().unwrap_or(Value::Null); |
| 724 | let raw = body.to_string(); |
| 725 | |
| 726 | if raw.contains("stay busy worker") && !raw.contains("dispatch one QA worker") { |
| 727 | self.child_requests.fetch_add(1, Ordering::SeqCst); |
| 728 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "child-acknowledged")); |
| 729 | } |
| 730 | |
| 731 | if raw.contains("dispatch one QA worker") { |
| 732 | return sse_response(fanout_tool_call_sse_n(1)); |
| 733 | } |
| 734 | |
| 735 | sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request")) |
| 736 | } |
| 737 | } |
| 738 | |
| 739 | /// Dispatching `agent` must not kill the process. |
| 740 | /// |
| 741 | /// Regression guard for the 0.9.4 release blocker: the Tokio runtime was built |
| 742 | /// by `#[tokio::main]`, so every worker thread carried tokio's 2 MiB default |
| 743 | /// while only the `codewhale-main` owner thread got `CODEWHALE_MAIN_STACK_BYTES`. |
| 744 | /// The engine runs on a worker, and a debug-build `agent` dispatch measured a |
| 745 | /// stack high-water mark between 2.25 and 2.5 MiB — it overflowed the guard page |
| 746 | /// and aborted the process with 134, mid-dispatch, before any child request was |
| 747 | /// ever issued. |
| 748 | /// |
| 749 | /// This asserts the invariant that the default violated (the process survives an |
| 750 | /// `agent` dispatch) rather than re-asserting a child counter, which a dead |
| 751 | /// process also fails — but fails slowly and for the wrong stated reason. |
| 752 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 753 | async fn release_agent_dispatch_never_aborts_the_runtime() -> Result<()> { |
| 754 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 755 | let server = MockServer::start().await; |
| 756 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 757 | let child_requests = Arc::new(AtomicUsize::new(0)); |
| 758 | Mock::given(method("POST")) |
| 759 | .and(path("/v1/chat/completions")) |
| 760 | .respond_with(SingleDispatchResponder { |
| 761 | child_requests: Arc::clone(&child_requests), |
| 762 | }) |
| 763 | .mount(&server) |
| 764 | .await; |
| 765 | |
| 766 | let ws = make_sealed_workspace()?; |
| 767 | std::fs::write( |
| 768 | ws.home().join(".codewhale").join("config.toml"), |
| 769 | "[subagents]\nmax_concurrent = 1\nlaunch_concurrency = 1\nmax_admitted = 1\n", |
| 770 | )?; |
| 771 | let mut tui = common_tui_builder(&ws) |
| 772 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 773 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 774 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 775 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 776 | .args(["--yolo", "--max-subagents", "1"]) |
| 777 | .spawn()?; |
| 778 | enter_launch_session(&mut tui)?; |
| 779 | |
| 780 | type_and_submit(&mut tui, "dispatch one QA worker for the stack guard check")?; |
| 781 | |
| 782 | // The abort lands inside the first `agent` dispatch, so the process is |
| 783 | // already reaped by the time the child request would have been issued. |
| 784 | // Probe liveness first: it names the mechanism in the failure text instead |
| 785 | // of leaving a 15s timeout over a retained frame of a dead TUI. |
| 786 | assert!( |
| 787 | tui.wait_for_exit(Duration::from_millis(250)).is_none(), |
| 788 | "codewhale-tui exited during `agent` dispatch (a stack overflow aborts as 134); \ |
| 789 | the Tokio runtime must carry CODEWHALE_MAIN_STACK_BYTES — see main.rs build_runtime()" |
| 790 | ); |
| 791 | |
| 792 | wait_for_counter(&mut tui, &child_requests, 1, INTERACTION_TIMEOUT)?; |
| 793 | |
| 794 | let _ = tui.shutdown(); |
| 795 | Ok(()) |
| 796 | } |
| 797 | |
| 798 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 799 | async fn release_four_read_only_fleet_roles_launch_with_canonical_prompts() -> Result<()> { |
| 800 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 801 | let server = MockServer::start().await; |
| 802 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 803 | let launched = Arc::new(AtomicUsize::new(0)); |
| 804 | let canonical_prompts = Arc::new(AtomicUsize::new(0)); |
| 805 | let worker = Arc::new(AtomicUsize::new(0)); |
| 806 | let scout = Arc::new(AtomicUsize::new(0)); |
| 807 | let reviewer = Arc::new(AtomicUsize::new(0)); |
| 808 | let verifier = Arc::new(AtomicUsize::new(0)); |
| 809 | Mock::given(method("POST")) |
| 810 | .and(path("/v1/chat/completions")) |
| 811 | .respond_with(FleetRoleResponder { |
| 812 | launched: Arc::clone(&launched), |
| 813 | canonical_prompts: Arc::clone(&canonical_prompts), |
| 814 | worker: Arc::clone(&worker), |
| 815 | scout: Arc::clone(&scout), |
| 816 | reviewer: Arc::clone(&reviewer), |
| 817 | verifier: Arc::clone(&verifier), |
| 818 | }) |
| 819 | .mount(&server) |
| 820 | .await; |
| 821 | |
| 822 | let ws = make_sealed_workspace()?; |
| 823 | std::fs::write( |
| 824 | ws.home().join(".codewhale").join("config.toml"), |
| 825 | "[subagents]\nmax_concurrent = 4\nlaunch_concurrency = 4\nmax_admitted = 4\n", |
| 826 | )?; |
| 827 | let mut tui = common_tui_builder(&ws) |
| 828 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 829 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 830 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 831 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 832 | .args(["--yolo", "--max-subagents", "4"]) |
| 833 | .spawn()?; |
| 834 | enter_launch_session(&mut tui)?; |
| 835 | |
| 836 | type_and_submit(&mut tui, "launch four canonical read-only Fleet roles")?; |
| 837 | wait_for_counter(&mut tui, &launched, 4, INTERACTION_TIMEOUT)?; |
| 838 | |
| 839 | assert_eq!( |
| 840 | worker.load(Ordering::SeqCst), |
| 841 | 1, |
| 842 | "worker did not launch once" |
| 843 | ); |
| 844 | assert_eq!(scout.load(Ordering::SeqCst), 1, "scout did not launch once"); |
| 845 | assert_eq!( |
| 846 | reviewer.load(Ordering::SeqCst), |
| 847 | 1, |
| 848 | "reviewer did not launch once" |
| 849 | ); |
| 850 | assert_eq!( |
| 851 | verifier.load(Ordering::SeqCst), |
| 852 | 1, |
| 853 | "verifier did not launch once" |
| 854 | ); |
| 855 | assert_eq!( |
| 856 | canonical_prompts.load(Ordering::SeqCst), |
| 857 | 4, |
| 858 | "each live child request must contain its canonical Fleet role prompt" |
| 859 | ); |
| 860 | |
| 861 | let _ = tui.shutdown(); |
| 862 | Ok(()) |
| 863 | } |
| 864 | |
| 865 | #[derive(Clone)] |
| 866 | struct SteeringResponder { |
| 867 | initial_requests: Arc<AtomicUsize>, |
| 868 | steer_requests: Arc<AtomicUsize>, |
| 869 | } |
| 870 | |
| 871 | impl Respond for SteeringResponder { |
| 872 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 873 | let body = request.body_json::<Value>().unwrap_or(Value::Null); |
| 874 | let raw = body.to_string(); |
| 875 | if raw.contains("queued steering from enter") { |
| 876 | self.steer_requests.fetch_add(1, Ordering::SeqCst); |
| 877 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "steering-applied")); |
| 878 | } |
| 879 | if raw.contains("portable steering from enter") { |
| 880 | self.steer_requests.fetch_add(1, Ordering::SeqCst); |
| 881 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "portable-steering-applied")); |
| 882 | } |
| 883 | if raw.contains("initial slow turn") { |
| 884 | self.initial_requests.fetch_add(1, Ordering::SeqCst); |
| 885 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "initial-turn-output")) |
| 886 | // Leave enough room for the real launch transition plus the |
| 887 | // queued-preview assertion on slower release-gate machines. |
| 888 | .set_delay(Duration::from_secs(8)); |
| 889 | } |
| 890 | sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request")) |
| 891 | } |
| 892 | } |
| 893 | |
| 894 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 895 | async fn release_empty_enter_promotes_queued_follow_up() -> Result<()> { |
| 896 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 897 | let server = MockServer::start().await; |
| 898 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 899 | let initial_requests = Arc::new(AtomicUsize::new(0)); |
| 900 | let steer_requests = Arc::new(AtomicUsize::new(0)); |
| 901 | Mock::given(method("POST")) |
| 902 | .and(path("/v1/chat/completions")) |
| 903 | .respond_with(SteeringResponder { |
| 904 | initial_requests: Arc::clone(&initial_requests), |
| 905 | steer_requests: Arc::clone(&steer_requests), |
| 906 | }) |
| 907 | .mount(&server) |
| 908 | .await; |
| 909 | |
| 910 | let ws = make_sealed_workspace()?; |
| 911 | let mut tui = common_tui_builder(&ws) |
| 912 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 913 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 914 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 915 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 916 | .spawn()?; |
| 917 | enter_launch_session(&mut tui)?; |
| 918 | |
| 919 | type_and_submit(&mut tui, "initial slow turn")?; |
| 920 | // Use the same bounded interaction budget as the rest of this PTY gate. |
| 921 | // Cold debug binaries can take more than three seconds to reach the |
| 922 | // loopback server while release builds and workspace tests run in parallel. |
| 923 | // A dead engine still fails closed because the counter never advances. |
| 924 | wait_for_counter(&mut tui, &initial_requests, 1, INTERACTION_TIMEOUT)?; |
| 925 | |
| 926 | tui.send(keys::key::text("queued steering from enter"))?; |
| 927 | tui.wait_for_text("queued steering from enter", Duration::from_secs(3))?; |
| 928 | tui.send(b"\t")?; |
| 929 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 930 | tui.pump(); |
| 931 | assert!( |
| 932 | tui.frame().contains("queued steering from enter"), |
| 933 | "Tab must leave a busy-turn draft in the composer:\n{}", |
| 934 | tui.debug_dump() |
| 935 | ); |
| 936 | tui.send(keys::key::enter())?; |
| 937 | tui.wait_for_text("Enter send now", Duration::from_secs(5))?; |
| 938 | assert!( |
| 939 | tui.frame().contains("queued steering from enter"), |
| 940 | "queued steering preview was not readable:\n{}", |
| 941 | tui.debug_dump() |
| 942 | ); |
| 943 | |
| 944 | tui.send(keys::key::text("stash this draft, do not steer"))?; |
| 945 | tui.wait_for_text("stash this draft, do not steer", Duration::from_secs(3))?; |
| 946 | tui.send(keys::key::ctrl_g())?; |
| 947 | tui.wait_for_text("Draft stashed", Duration::from_secs(3))?; |
| 948 | assert_eq!( |
| 949 | steer_requests.load(Ordering::SeqCst), |
| 950 | 0, |
| 951 | "Ctrl+G must not send a queued follow-up" |
| 952 | ); |
| 953 | tui.wait_for_text("Enter send now", Duration::from_secs(3))?; |
| 954 | |
| 955 | let steer_started = Instant::now(); |
| 956 | tui.send(keys::key::enter())?; |
| 957 | wait_for_counter(&mut tui, &steer_requests, 1, INTERACTION_TIMEOUT)?; |
| 958 | tui.wait_for_text("steering-applied", INTERACTION_TIMEOUT)?; |
| 959 | assert!( |
| 960 | steer_started.elapsed() < Duration::from_secs(10), |
| 961 | "empty Enter queue promotion was not incorporated promptly" |
| 962 | ); |
| 963 | |
| 964 | let _ = tui.shutdown(); |
| 965 | Ok(()) |
| 966 | } |
| 967 | |
| 968 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 969 | async fn release_enter_queue_then_enter_steers_running_turn() -> Result<()> { |
| 970 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 971 | let server = MockServer::start().await; |
| 972 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 973 | let initial_requests = Arc::new(AtomicUsize::new(0)); |
| 974 | let steer_requests = Arc::new(AtomicUsize::new(0)); |
| 975 | Mock::given(method("POST")) |
| 976 | .and(path("/v1/chat/completions")) |
| 977 | .respond_with(SteeringResponder { |
| 978 | initial_requests: Arc::clone(&initial_requests), |
| 979 | steer_requests: Arc::clone(&steer_requests), |
| 980 | }) |
| 981 | .mount(&server) |
| 982 | .await; |
| 983 | |
| 984 | let ws = make_sealed_workspace()?; |
| 985 | let mut tui = common_tui_builder(&ws) |
| 986 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 987 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 988 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 989 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 990 | .spawn()?; |
| 991 | enter_launch_session(&mut tui)?; |
| 992 | |
| 993 | type_and_submit(&mut tui, "initial slow turn")?; |
| 994 | wait_for_counter(&mut tui, &initial_requests, 1, INTERACTION_TIMEOUT)?; |
| 995 | |
| 996 | tui.send(keys::key::text("busy-shift-line"))?; |
| 997 | tui.send(keys::key::shift_enter())?; |
| 998 | tui.send(keys::key::text("busy-alt-line"))?; |
| 999 | tui.send(keys::key::alt_enter())?; |
| 1000 | tui.send(keys::key::text("busy-ctrl-j-line"))?; |
| 1001 | tui.send(keys::key::ctrl_j())?; |
| 1002 | tui.send(keys::key::text("portable steering from enter"))?; |
| 1003 | tui.wait_for_text("portable steering from enter", Duration::from_secs(3))?; |
| 1004 | let frame = tui.frame(); |
| 1005 | let rows = [ |
| 1006 | "busy-shift-line", |
| 1007 | "busy-alt-line", |
| 1008 | "busy-ctrl-j-line", |
| 1009 | "portable steering from enter", |
| 1010 | ] |
| 1011 | .map(|line| { |
| 1012 | frame |
| 1013 | .find_text(line) |
| 1014 | .expect("busy multiline draft stays visible") |
| 1015 | .0 |
| 1016 | }); |
| 1017 | assert!( |
| 1018 | rows.windows(2).all(|pair| pair[0] < pair[1]), |
| 1019 | "newline chords must stay newlines during a running turn:\n{}", |
| 1020 | frame.debug_dump() |
| 1021 | ); |
| 1022 | tui.wait_for_text("then ↵ steer", Duration::from_secs(3))?; |
| 1023 | let steer_started = Instant::now(); |
| 1024 | // The first portable Enter queues the completed draft. The queued preview |
| 1025 | // uses the already-visible "then Enter" contract; the second Enter |
| 1026 | // promotes it into the active turn even when a multiline preview consumes |
| 1027 | // the compact control row. |
| 1028 | tui.send(keys::key::enter())?; |
| 1029 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 1030 | tui.pump(); |
| 1031 | tui.send(keys::key::enter())?; |
| 1032 | wait_for_counter(&mut tui, &steer_requests, 1, INTERACTION_TIMEOUT)?; |
| 1033 | tui.wait_for_text("portable-steering-applied", INTERACTION_TIMEOUT)?; |
| 1034 | assert!( |
| 1035 | steer_started.elapsed() < Duration::from_secs(10), |
| 1036 | "two-Enter steering was not incorporated promptly" |
| 1037 | ); |
| 1038 | |
| 1039 | let _ = tui.shutdown(); |
| 1040 | Ok(()) |
| 1041 | } |
| 1042 | |
| 1043 | /// Records, for every chat request, the highest-numbered follow-up marker |
| 1044 | /// present in the serialized body. Because history accumulates, request `k` |
| 1045 | /// contains markers `1..=k`, so the sequence of maxima is an exact record of |
| 1046 | /// which follow-up each request carried — which makes a dropped message and a |
| 1047 | /// double-sent message both visible, and distinguishable from each other. |
| 1048 | #[derive(Clone)] |
| 1049 | struct QueueOrderResponder { |
| 1050 | markers: Vec<String>, |
| 1051 | observed: Arc<std::sync::Mutex<Vec<usize>>>, |
| 1052 | initial_delay: Duration, |
| 1053 | } |
| 1054 | |
| 1055 | impl QueueOrderResponder { |
| 1056 | fn observed(&self) -> Vec<usize> { |
| 1057 | self.observed |
| 1058 | .lock() |
| 1059 | .unwrap_or_else(|poison| poison.into_inner()) |
| 1060 | .clone() |
| 1061 | } |
| 1062 | } |
| 1063 | |
| 1064 | impl Respond for QueueOrderResponder { |
| 1065 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 1066 | let raw = request |
| 1067 | .body_json::<Value>() |
| 1068 | .unwrap_or(Value::Null) |
| 1069 | .to_string(); |
| 1070 | let highest = self |
| 1071 | .markers |
| 1072 | .iter() |
| 1073 | .enumerate() |
| 1074 | .filter(|(_, marker)| raw.contains(marker.as_str())) |
| 1075 | .map(|(index, _)| index + 1) |
| 1076 | .max() |
| 1077 | .unwrap_or(0); |
| 1078 | self.observed |
| 1079 | .lock() |
| 1080 | .unwrap_or_else(|poison| poison.into_inner()) |
| 1081 | .push(highest); |
| 1082 | if highest == 0 { |
| 1083 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "initial-turn-output")) |
| 1084 | .set_delay(self.initial_delay); |
| 1085 | } |
| 1086 | sse_response(text_sse( |
| 1087 | DEEPSEEK_TEST_MODEL, |
| 1088 | &format!("follow-up-{highest}-done"), |
| 1089 | )) |
| 1090 | } |
| 1091 | } |
| 1092 | |
| 1093 | /// The running-turn contract, end to end: while a turn is in flight, bare |
| 1094 | /// Enter queues rather than steering, the composer says so, and every queued |
| 1095 | /// follow-up dispatches exactly once, in order, after the turn completes. |
| 1096 | /// |
| 1097 | /// This is the mailbox-backpressure row of #3758. Queueing six follow-ups |
| 1098 | /// against a busy engine puts several ops in flight behind the |
| 1099 | /// `dispatch_in_flight` guard (#4605); the failure modes it rules out are a |
| 1100 | /// silently dropped follow-up and a follow-up sent twice, which look identical |
| 1101 | /// on the transcript but are opposite bugs. |
| 1102 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 1103 | async fn release_queued_follow_ups_dispatch_exactly_once_and_in_order() -> Result<()> { |
| 1104 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 1105 | let server = MockServer::start().await; |
| 1106 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 1107 | |
| 1108 | // Six markers, none a prefix of another, so "contains" cannot confuse |
| 1109 | // marker 1 with marker 10. |
| 1110 | let markers: Vec<String> = (1..=6).map(|n| format!("queue-marker-{n}-end")).collect(); |
| 1111 | let responder = QueueOrderResponder { |
| 1112 | markers: markers.clone(), |
| 1113 | observed: Arc::new(std::sync::Mutex::new(Vec::new())), |
| 1114 | initial_delay: Duration::from_secs(14), |
| 1115 | }; |
| 1116 | Mock::given(method("POST")) |
| 1117 | .and(path("/v1/chat/completions")) |
| 1118 | .respond_with(responder.clone()) |
| 1119 | .mount(&server) |
| 1120 | .await; |
| 1121 | |
| 1122 | let ws = make_sealed_workspace()?; |
| 1123 | let mut tui = common_tui_builder(&ws) |
| 1124 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 1125 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 1126 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 1127 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 1128 | .spawn()?; |
| 1129 | enter_launch_session(&mut tui)?; |
| 1130 | |
| 1131 | type_and_submit(&mut tui, "queue backpressure initial turn")?; |
| 1132 | // Start the busy-state clock when the loopback server has actually |
| 1133 | // received the request. Cold debug launches may spend most of the generic |
| 1134 | // interaction budget before the request reaches wiremock; the responder's |
| 1135 | // 14-second delay begins only after this signal. |
| 1136 | let request_deadline = Instant::now() + INTERACTION_TIMEOUT; |
| 1137 | while responder.observed().is_empty() { |
| 1138 | tui.pump(); |
| 1139 | if Instant::now() >= request_deadline { |
| 1140 | return Err(anyhow!( |
| 1141 | "initial queue-order request never reached the mock server\n{}", |
| 1142 | tui.debug_dump() |
| 1143 | )); |
| 1144 | } |
| 1145 | std::thread::sleep(Duration::from_millis(40)); |
| 1146 | } |
| 1147 | let first_marker = markers.first().expect("queue matrix has a marker"); |
| 1148 | tui.send(keys::key::text(first_marker))?; |
| 1149 | tui.wait_for_text(first_marker, Duration::from_secs(3))?; |
| 1150 | tui.wait_for_text("then ↵ steer", Duration::from_secs(3))?; |
| 1151 | |
| 1152 | // While the turn is running the composer must advertise queueing, and it |
| 1153 | // must not advertise the stash chords as a way to send (#440 / #3758). |
| 1154 | let busy_frame = tui.frame(); |
| 1155 | let busy_dump = busy_frame.debug_dump(); |
| 1156 | assert!( |
| 1157 | busy_frame.contains("↵ queue"), |
| 1158 | "busy composer must say Enter queues:\n{busy_dump}" |
| 1159 | ); |
| 1160 | for line in busy_frame.text().lines() { |
| 1161 | if !(line.contains("Ctrl+G") || line.contains("Ctrl+S")) { |
| 1162 | continue; |
| 1163 | } |
| 1164 | let lowered = line.to_ascii_lowercase(); |
| 1165 | for forbidden in ["send", "queue", "steer", "submit"] { |
| 1166 | assert!( |
| 1167 | !lowered.contains(forbidden), |
| 1168 | "stash chords must not be advertised as a send/queue/steer path: {line:?}" |
| 1169 | ); |
| 1170 | } |
| 1171 | } |
| 1172 | |
| 1173 | std::thread::sleep(PASTE_GUARD_SETTLE); |
| 1174 | tui.pump(); |
| 1175 | tui.send(keys::key::enter())?; |
| 1176 | for marker in markers.iter().skip(1) { |
| 1177 | type_and_submit(&mut tui, marker)?; |
| 1178 | } |
| 1179 | |
| 1180 | // One request for the initial turn plus one per follow-up. |
| 1181 | let expected_requests = markers.len() + 1; |
| 1182 | let deadline = Instant::now() + Duration::from_secs(90); |
| 1183 | loop { |
| 1184 | tui.pump(); |
| 1185 | if responder.observed().len() >= expected_requests { |
| 1186 | break; |
| 1187 | } |
| 1188 | if Instant::now() >= deadline { |
| 1189 | return Err(anyhow!( |
| 1190 | "only {:?} of {expected_requests} requests arrived\n{}", |
| 1191 | responder.observed(), |
| 1192 | tui.debug_dump() |
| 1193 | )); |
| 1194 | } |
| 1195 | std::thread::sleep(Duration::from_millis(80)); |
| 1196 | } |
| 1197 | |
| 1198 | let observed = responder.observed(); |
| 1199 | let expected: Vec<usize> = (0..=markers.len()).collect(); |
| 1200 | assert_eq!( |
| 1201 | observed, |
| 1202 | expected, |
| 1203 | "queued follow-ups must dispatch exactly once each, in order; \ |
| 1204 | a missing index is a dropped message and a repeated one is a double send\n{}", |
| 1205 | tui.debug_dump() |
| 1206 | ); |
| 1207 | |
| 1208 | let _ = tui.shutdown(); |
| 1209 | Ok(()) |
| 1210 | } |
| 1211 | |
| 1212 | #[derive(Clone)] |
| 1213 | struct BenchFanoutResponder { |
| 1214 | child_requests: Arc<AtomicUsize>, |
| 1215 | workers: usize, |
| 1216 | } |
| 1217 | |
| 1218 | impl Respond for BenchFanoutResponder { |
| 1219 | fn respond(&self, request: &Request) -> ResponseTemplate { |
| 1220 | let body = request.body_json::<Value>().unwrap_or(Value::Null); |
| 1221 | let raw = body.to_string(); |
| 1222 | |
| 1223 | if raw.contains("stay busy worker") && !raw.contains("launch benchmark QA workers") { |
| 1224 | self.child_requests.fetch_add(1, Ordering::SeqCst); |
| 1225 | return sse_response(text_sse(DEEPSEEK_TEST_MODEL, "child-finished-too-soon")) |
| 1226 | .set_delay(Duration::from_secs(60)); |
| 1227 | } |
| 1228 | |
| 1229 | if raw.contains("launch benchmark QA workers") { |
| 1230 | return sse_response(fanout_tool_call_sse_n(self.workers)); |
| 1231 | } |
| 1232 | |
| 1233 | sse_response(text_sse(DEEPSEEK_TEST_MODEL, "unexpected-request")) |
| 1234 | } |
| 1235 | } |
| 1236 | |
| 1237 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 1238 | enum RssSample { |
| 1239 | Kib(u64), |
| 1240 | Unavailable(&'static str), |
| 1241 | } |
| 1242 | |
| 1243 | impl RssSample { |
| 1244 | fn required_kib(self, phase: &str) -> Result<u64> { |
| 1245 | match self { |
| 1246 | Self::Kib(value) => Ok(value), |
| 1247 | Self::Unavailable(reason) => Err(anyhow!( |
| 1248 | "RSS UNAVAILABLE during {phase}: {reason}; this Unix benchmark requires every sample" |
| 1249 | )), |
| 1250 | } |
| 1251 | } |
| 1252 | } |
| 1253 | |
| 1254 | fn rss_kib(pid: Option<u32>) -> RssSample { |
| 1255 | let Some(pid) = pid else { |
| 1256 | return RssSample::Unavailable("process_id_unavailable"); |
| 1257 | }; |
| 1258 | let out = match std::process::Command::new("ps") |
| 1259 | .args(["-o", "rss=", "-p", &pid.to_string()]) |
| 1260 | .output() |
| 1261 | { |
| 1262 | Ok(output) => output, |
| 1263 | Err(_) => return RssSample::Unavailable("ps_command_unavailable"), |
| 1264 | }; |
| 1265 | if !out.status.success() { |
| 1266 | return RssSample::Unavailable("ps_nonzero_or_process_exited"); |
| 1267 | } |
| 1268 | match std::str::from_utf8(&out.stdout) |
| 1269 | .ok() |
| 1270 | .map(str::trim) |
| 1271 | .filter(|value| !value.is_empty()) |
| 1272 | .and_then(|value| value.parse().ok()) |
| 1273 | { |
| 1274 | Some(value) => RssSample::Kib(value), |
| 1275 | None => RssSample::Unavailable("ps_output_invalid"), |
| 1276 | } |
| 1277 | } |
| 1278 | |
| 1279 | fn engine_turn_receipts(ws: &SealedWorkspace, pid: u32) -> Result<String> { |
| 1280 | let log_dir = ws.home().join(".codewhale").join("logs"); |
| 1281 | if !log_dir.is_dir() { |
| 1282 | return Ok(String::new()); |
| 1283 | } |
| 1284 | |
| 1285 | let pid_suffix = format!("-{pid}.log"); |
| 1286 | let mut receipts = String::new(); |
| 1287 | for entry in std::fs::read_dir(&log_dir)? { |
| 1288 | let entry = entry?; |
| 1289 | if entry.file_name().to_string_lossy().ends_with(&pid_suffix) { |
| 1290 | receipts.push_str(&std::fs::read_to_string(entry.path())?); |
| 1291 | } |
| 1292 | } |
| 1293 | Ok(receipts) |
| 1294 | } |
| 1295 | |
| 1296 | fn wait_for_interrupted_engine_turn_receipt( |
| 1297 | tui: &mut Harness, |
| 1298 | ws: &SealedWorkspace, |
| 1299 | timeout: Duration, |
| 1300 | ) -> Result<()> { |
| 1301 | let pid = tui |
| 1302 | .pid() |
| 1303 | .ok_or_else(|| anyhow!("engine completion receipt unavailable: process id missing"))?; |
| 1304 | let deadline = Instant::now() + timeout; |
| 1305 | loop { |
| 1306 | tui.pump(); |
| 1307 | let receipts = engine_turn_receipts(ws, pid)?; |
| 1308 | if receipts.lines().any(|line| { |
| 1309 | line.contains("engine turn completion settled") |
| 1310 | && line.contains("status=Interrupted") |
| 1311 | && line.contains("delivered=true") |
| 1312 | }) { |
| 1313 | return Ok(()); |
| 1314 | } |
| 1315 | if Instant::now() >= deadline { |
| 1316 | return Err(anyhow!( |
| 1317 | "typed engine TurnComplete(Interrupted) receipt did not arrive within {timeout:?}; \ |
| 1318 | rendered cancellation text is not a settlement receipt\nengine log:\n{receipts}\n{}", |
| 1319 | tui.debug_dump() |
| 1320 | )); |
| 1321 | } |
| 1322 | std::thread::sleep(Duration::from_millis(40)); |
| 1323 | } |
| 1324 | } |
| 1325 | |
| 1326 | /// #4014 acceptance benchmark: 32 concurrent loopback workers must keep the |
| 1327 | /// TUI live. Ignored by default (heavy storm); run explicitly with |
| 1328 | /// `cargo test -p codewhale-tui --test release_runtime_qa --locked -- \ |
| 1329 | /// --ignored bench_thirty_two --nocapture --test-threads=1`. |
| 1330 | #[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 1331 | #[ignore = "heavy 32-worker storm; run explicitly for #4014 evidence"] |
| 1332 | async fn release_bench_thirty_two_worker_fanout_stays_live() -> Result<()> { |
| 1333 | const WORKERS: usize = 32; |
| 1334 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 1335 | let server = MockServer::start().await; |
| 1336 | mount_models(&server, &[DEEPSEEK_TEST_MODEL]).await; |
| 1337 | let child_requests = Arc::new(AtomicUsize::new(0)); |
| 1338 | Mock::given(method("POST")) |
| 1339 | .and(path("/v1/chat/completions")) |
| 1340 | .respond_with(BenchFanoutResponder { |
| 1341 | child_requests: Arc::clone(&child_requests), |
| 1342 | workers: WORKERS, |
| 1343 | }) |
| 1344 | .mount(&server) |
| 1345 | .await; |
| 1346 | |
| 1347 | let ws = make_sealed_workspace()?; |
| 1348 | std::fs::write( |
| 1349 | ws.home().join(".codewhale").join("config.toml"), |
| 1350 | format!( |
| 1351 | "[subagents]\nmax_concurrent = {WORKERS}\nlaunch_concurrency = {WORKERS}\nmax_admitted = {WORKERS}\n" |
| 1352 | ), |
| 1353 | )?; |
| 1354 | let mut tui = common_tui_builder(&ws) |
| 1355 | .env("RUST_LOG", "warn,engine.turn=info") |
| 1356 | .env("CODEWHALE_PROVIDER", "deepseek") |
| 1357 | .env("DEEPSEEK_API_KEY", "deepseek-local-test-key") |
| 1358 | .env("DEEPSEEK_BASE_URL", server.uri()) |
| 1359 | .env("DEEPSEEK_MODEL", DEEPSEEK_TEST_MODEL) |
| 1360 | .args(["--yolo", "--max-subagents", &WORKERS.to_string()]) |
| 1361 | .spawn()?; |
| 1362 | enter_launch_session(&mut tui)?; |
| 1363 | let pid = tui.pid(); |
| 1364 | let rss_idle = rss_kib(pid); |
| 1365 | |
| 1366 | let spawn_started = Instant::now(); |
| 1367 | type_and_submit( |
| 1368 | &mut tui, |
| 1369 | "launch benchmark QA workers and keep the parent turn open", |
| 1370 | )?; |
| 1371 | wait_for_counter(&mut tui, &child_requests, WORKERS, Duration::from_secs(60))?; |
| 1372 | let all_children_live = spawn_started.elapsed(); |
| 1373 | // The Ocean work surface owns the exact copy around the aggregate count. |
| 1374 | // Keep this runtime benchmark coupled only to the typed count glyph in the |
| 1375 | // regular/wide phase strip; labels and available actions legitimately |
| 1376 | // change with layout, and compact layouts intentionally omit the count. |
| 1377 | tui.wait_for( |
| 1378 | |frame| { |
| 1379 | let text = frame.text(); |
| 1380 | text.contains(&format!("×{WORKERS}")) |
| 1381 | }, |
| 1382 | Duration::from_secs(10), |
| 1383 | )?; |
| 1384 | let aggregate_visible = spawn_started.elapsed(); |
| 1385 | let rss_storm = rss_kib(pid); |
| 1386 | |
| 1387 | // Echo latency under storm: three samples. |
| 1388 | let mut echo_samples = Vec::new(); |
| 1389 | for i in 0..3 { |
| 1390 | let marker = format!("bench-live-marker-{i}"); |
| 1391 | let t = Instant::now(); |
| 1392 | tui.send(keys::key::text(&marker))?; |
| 1393 | tui.wait_for_text(&marker, Duration::from_secs(5))?; |
| 1394 | echo_samples.push(t.elapsed()); |
| 1395 | // Clear the composer for the next sample. |
| 1396 | for _ in 0..marker.len() { |
| 1397 | tui.send(b"\x7f")?; |
| 1398 | } |
| 1399 | } |
| 1400 | |
| 1401 | let cancel_started = Instant::now(); |
| 1402 | tui.send(b"\x1b")?; |
| 1403 | wait_for_interrupted_engine_turn_receipt(&mut tui, &ws, Duration::from_secs(10))?; |
| 1404 | let cancel_latency = cancel_started.elapsed(); |
| 1405 | |
| 1406 | // Anchor retention evidence to the typed engine cancellation settlement, |
| 1407 | // then schedule absolute 1/3/5-second samples on another runtime worker so |
| 1408 | // the independent post-cancel input proof cannot shift their epoch. |
| 1409 | let post_cancel_observation_started = Instant::now(); |
| 1410 | let mut rss_retention_samples = Vec::with_capacity(4); |
| 1411 | rss_retention_samples.push(( |
| 1412 | Duration::ZERO, |
| 1413 | post_cancel_observation_started.elapsed(), |
| 1414 | rss_kib(pid), |
| 1415 | )); |
| 1416 | let rss_sampler = tokio::spawn(async move { |
| 1417 | let mut samples = Vec::with_capacity(3); |
| 1418 | for target in [ |
| 1419 | Duration::from_secs(1), |
| 1420 | Duration::from_secs(3), |
| 1421 | Duration::from_secs(5), |
| 1422 | ] { |
| 1423 | tokio::time::sleep_until(tokio::time::Instant::from_std( |
| 1424 | post_cancel_observation_started + target, |
| 1425 | )) |
| 1426 | .await; |
| 1427 | samples.push(( |
| 1428 | target, |
| 1429 | post_cancel_observation_started.elapsed(), |
| 1430 | rss_kib(pid), |
| 1431 | )); |
| 1432 | } |
| 1433 | samples |
| 1434 | }); |
| 1435 | |
| 1436 | tui.send(keys::key::text("post-cancel-live"))?; |
| 1437 | tui.wait_for_text("post-cancel-live", Duration::from_secs(5))?; |
| 1438 | let delayed_samples = tokio::time::timeout(Duration::from_secs(8), rss_sampler) |
| 1439 | .await |
| 1440 | .map_err(|_| anyhow!("RSS sampler exceeded its bounded post-cancel retention window"))? |
| 1441 | .map_err(|error| anyhow!("RSS sampler task failed: {error}"))?; |
| 1442 | rss_retention_samples.extend(delayed_samples); |
| 1443 | |
| 1444 | tui.send(keys::key::text("post-cancel-5s-live"))?; |
| 1445 | tui.wait_for_text("post-cancel-5s-live", Duration::from_secs(5))?; |
| 1446 | |
| 1447 | println!( |
| 1448 | "BENCH32: children_live={all_children_live:?} aggregate={aggregate_visible:?} \ |
| 1449 | echo={echo_samples:?} cancel={cancel_latency:?} \ |
| 1450 | rss_idle_kib={rss_idle:?} rss_storm_kib={rss_storm:?} \ |
| 1451 | rss_retention_samples={rss_retention_samples:?}" |
| 1452 | ); |
| 1453 | |
| 1454 | let worst_echo = echo_samples.iter().max().copied().unwrap_or_default(); |
| 1455 | assert!( |
| 1456 | worst_echo < Duration::from_secs(2), |
| 1457 | "typing echo exceeded 2s under a {WORKERS}-worker storm: {echo_samples:?}" |
| 1458 | ); |
| 1459 | assert!( |
| 1460 | cancel_latency < Duration::from_secs(5), |
| 1461 | "Esc cancellation exceeded 5s under a {WORKERS}-worker storm: {cancel_latency:?}" |
| 1462 | ); |
| 1463 | let idle = rss_idle.required_kib("idle baseline")?; |
| 1464 | let storm = rss_storm.required_kib("live worker storm")?; |
| 1465 | let rss_ceiling = idle.saturating_mul(6).max(idle + 1_500_000); |
| 1466 | assert!( |
| 1467 | storm < rss_ceiling, |
| 1468 | "RSS exploded under storm: idle={idle} KiB storm={storm} KiB" |
| 1469 | ); |
| 1470 | for (target, observed, sample) in &rss_retention_samples { |
| 1471 | assert!( |
| 1472 | *observed >= *target, |
| 1473 | "RSS sample preceded its target: target={target:?} observed={observed:?}" |
| 1474 | ); |
| 1475 | assert!( |
| 1476 | *observed <= *target + Duration::from_secs(2), |
| 1477 | "RSS sample missed its bounded target window: target={target:?} observed={observed:?}" |
| 1478 | ); |
| 1479 | let sample = sample.required_kib(&format!("post-cancel target {target:?}"))?; |
| 1480 | assert!( |
| 1481 | sample < rss_ceiling, |
| 1482 | "RSS exceeded the bounded storm ceiling at target={target:?} \ |
| 1483 | observed={observed:?}: idle={idle} KiB storm={storm} KiB sample={sample} KiB" |
| 1484 | ); |
| 1485 | } |
| 1486 | |
| 1487 | let _ = tui.shutdown(); |
| 1488 | Ok(()) |
| 1489 | } |
| 1490 | |
| 1491 | /// Dogfood of the named-Fleet journey against the release binary: |
| 1492 | /// migration -> session-only route change -> explicit /fleet save -> restart |
| 1493 | /// (selected Fleet operator applied) -> save-as. Every receipt is asserted |
| 1494 | /// on-screen and every claim is checked against the on-disk files. |
| 1495 | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] |
| 1496 | async fn release_fleet_route_save_journey() -> Result<()> { |
| 1497 | let _guard = RELEASE_RUNTIME_QA_LOCK.lock().await; |
| 1498 | let mock = MockServer::start().await; |
| 1499 | mount_text_model(&mock, "deepseek-v4-pro", "ok").await; |
| 1500 | let base_url = mock.uri(); |
| 1501 | |
| 1502 | let ws = make_sealed_workspace()?; |
| 1503 | let agents_dir = ws.workspace().join(".codewhale").join("agents"); |
| 1504 | std::fs::create_dir_all(&agents_dir) |
| 1505 | .map_err(|e| anyhow::anyhow!("agents dir {}: {e}", agents_dir.display()))?; |
| 1506 | std::fs::write( |
| 1507 | agents_dir.join("scout.toml"), |
| 1508 | r#"id = "scout" |
| 1509 | role_hint = "scout" |
| 1510 | model = "deepseek-v4-flash" |
| 1511 | provider = "deepseek" |
| 1512 | "#, |
| 1513 | ) |
| 1514 | .map_err(|e| anyhow::anyhow!("scout profile: {e}"))?; |
| 1515 | let home_dir = ws.home().join(".codewhale"); |
| 1516 | std::fs::create_dir_all(&home_dir) |
| 1517 | .map_err(|e| anyhow::anyhow!("home dir {}: {e}", home_dir.display()))?; |
| 1518 | std::fs::write( |
| 1519 | home_dir.join("config.toml"), |
| 1520 | format!( |
| 1521 | r#"provider = "deepseek" |
| 1522 | [providers.deepseek] |
| 1523 | api_key = "sk-release-qa" |
| 1524 | base_url = "{base_url}" |
| 1525 | "# |
| 1526 | ), |
| 1527 | ) |
| 1528 | .map_err(|e| anyhow::anyhow!("home config: {e}"))?; |
| 1529 | |
| 1530 | let mut tui = common_tui_builder(&ws).spawn()?; |
| 1531 | enter_launch_session(&mut tui)?; |
| 1532 | |
| 1533 | // 1. /fleet fleets (secondary named-Fleet picker) shows the migration |
| 1534 | // banner and no selection. Bare /fleet stays on the roster face. |
| 1535 | type_and_submit(&mut tui, "/fleet fleets")?; |
| 1536 | tui.wait_for_text("legacy role profile", INTERACTION_TIMEOUT)?; |
| 1537 | tui.wait_for_text("No Fleet selected", INTERACTION_TIMEOUT)?; |
| 1538 | // 2. m migrates with a receipt; Esc closes the pager, Esc closes the list. |
| 1539 | tui.send(keys::key::text("m"))?; |
| 1540 | tui.wait_for_text("Migrated", INTERACTION_TIMEOUT)?; |
| 1541 | tui.send(keys::key::esc())?; |
| 1542 | std::thread::sleep(Duration::from_millis(300)); |
| 1543 | tui.send(keys::key::esc())?; |
| 1544 | std::thread::sleep(Duration::from_millis(300)); |
| 1545 | tui.pump(); |
| 1546 | |
| 1547 | // 3. A route change is session-only and names the explicit commands. |
| 1548 | type_and_submit(&mut tui, "/model deepseek-v4-pro")?; |
| 1549 | tui.wait_for_text("session only", INTERACTION_TIMEOUT)?; |
| 1550 | tui.wait_for_text("/fleet save", INTERACTION_TIMEOUT)?; |
| 1551 | |
| 1552 | // 4. /fleet save writes the operator; the receipt names the file. |
| 1553 | type_and_submit(&mut tui, "/fleet save")?; |
| 1554 | tui.wait_for_text("now runs on deepseek/deepseek-v4-pro", INTERACTION_TIMEOUT)?; |
| 1555 | |
| 1556 | // 5. The updated Fleet file is the migrated Default (v2 schema with the |
| 1557 | // operator route pinned to the session model). |
| 1558 | let fleet_file = ws |
| 1559 | .home() |
| 1560 | .join(".codewhale") |
| 1561 | .join("fleets") |
| 1562 | .join("default.toml"); |
| 1563 | let fleet_text = std::fs::read_to_string(&fleet_file)?; |
| 1564 | assert!( |
| 1565 | fleet_text.contains("schema = \"fleet\"") && fleet_text.contains("deepseek-v4-pro"), |
| 1566 | "saved fleet must be v2 with the operator: {fleet_text}" |
| 1567 | ); |
| 1568 | |
| 1569 | // 6. Restart: the selected Fleet's operator is the session route. |
| 1570 | tui.shutdown() |
| 1571 | .ok_or_else(|| anyhow::anyhow!("graceful shutdown failed"))?; |
| 1572 | let mut tui = common_tui_builder(&ws).spawn()?; |
| 1573 | enter_launch_session(&mut tui)?; |
| 1574 | tui.wait_for_text("deepseek-v4-pro", INTERACTION_TIMEOUT)?; |
| 1575 | |
| 1576 | // 7. The named-Fleet picker shows the saved Fleet with its user scope |
| 1577 | // and selection (operator summary, not a filesystem path). |
| 1578 | type_and_submit(&mut tui, "/fleet fleets")?; |
| 1579 | tui.wait_for_text("DeepSeek", INTERACTION_TIMEOUT)?; |
| 1580 | tui.wait_for_text("[user]", INTERACTION_TIMEOUT)?; |
| 1581 | tui.wait_for_text("Selected", INTERACTION_TIMEOUT)?; |
| 1582 | tui.send(keys::key::esc())?; |
| 1583 | std::thread::sleep(Duration::from_millis(300)); |
| 1584 | tui.send(keys::key::esc())?; |
| 1585 | std::thread::sleep(Duration::from_millis(300)); |
| 1586 | tui.pump(); |
| 1587 | |
| 1588 | // 8. save-as creates and selects a second user-global Fleet. |
| 1589 | type_and_submit(&mut tui, "/model deepseek-v4-flash")?; |
| 1590 | tui.wait_for_text("session only", INTERACTION_TIMEOUT)?; |
| 1591 | type_and_submit(&mut tui, "/fleet save-as")?; |
| 1592 | // The receipt wraps across lines at this width; assert on fragments that |
| 1593 | // land on a single row. |
| 1594 | tui.wait_for_text("as new Fleet", INTERACTION_TIMEOUT)?; |
| 1595 | tui.wait_for_text("user-global default", INTERACTION_TIMEOUT)?; |
| 1596 | // The new Fleet is named after the route: `DeepSeek deepseek-v4-flash`. |
| 1597 | let second_file = ws |
| 1598 | .home() |
| 1599 | .join(".codewhale") |
| 1600 | .join("fleets") |
| 1601 | .join("deepseek-deepseek-v4-flash.toml"); |
| 1602 | assert!( |
| 1603 | second_file.is_file(), |
| 1604 | "save-as must create the second fleet file" |
| 1605 | ); |
| 1606 | // The legacy profile file was left untouched. |
| 1607 | assert!( |
| 1608 | std::fs::read_to_string( |
| 1609 | ws.workspace() |
| 1610 | .join(".codewhale") |
| 1611 | .join("agents") |
| 1612 | .join("scout.toml") |
| 1613 | )? |
| 1614 | .contains("deepseek-v4-flash") |
| 1615 | ); |
| 1616 | |
| 1617 | let _ = tui.shutdown(); |
| 1618 | Ok(()) |
| 1619 | } |
| 1620 |