| 1 | //! Provider-free acceptance lock for the public headless launch contract that |
| 2 | //! makes CodeWhale embeddable as a future Verifiers v1 harness (#4641). |
| 3 | //! |
| 4 | //! Everything here is loopback and sealed: a `wiremock` OpenAI-compatible |
| 5 | //! fixture stands in for the interception endpoint, `CODEWHALE_HOME` is a fresh |
| 6 | //! per-run directory, the credential is delivered only through the route's |
| 7 | //! `api_key_env`, and a sentinel secret must never escape the child process. |
| 8 | //! No provider credential, network egress, or installed CodeWhale is required. |
| 9 | |
| 10 | #![cfg(unix)] |
| 11 | |
| 12 | use std::io::Read; |
| 13 | use std::path::{Path, PathBuf}; |
| 14 | use std::process::{Command, Stdio}; |
| 15 | use std::time::Duration; |
| 16 | |
| 17 | use serde_json::{Value, json}; |
| 18 | use tempfile::TempDir; |
| 19 | use wait_timeout::ChildExt; |
| 20 | use wiremock::matchers::{method, path}; |
| 21 | use wiremock::{Mock, MockServer, ResponseTemplate}; |
| 22 | |
| 23 | const TEST_MODEL: &str = "verifiers-contract-model"; |
| 24 | const API_KEY_ENV: &str = "VF_CODEWHALE_API_KEY"; |
| 25 | /// A value that appears nowhere except the child environment; any sighting in |
| 26 | /// argv, stdout, stderr, the stream-json stream, or a written file is a leak. |
| 27 | const SENTINEL_SECRET: &str = "vf-sentinel-do-not-leak-8f3a2b1c9d7e"; |
| 28 | const APPEND_MARKER: &str = "VF-APPENDED-SYSTEM-PROMPT-MARKER"; |
| 29 | const RUN_TIMEOUT: Duration = Duration::from_secs(60); |
| 30 | |
| 31 | fn sse_chunk(value: Value) -> String { |
| 32 | format!( |
| 33 | "data: {}\n\n", |
| 34 | serde_json::to_string(&value).expect("SSE JSON") |
| 35 | ) |
| 36 | } |
| 37 | |
| 38 | fn text_sse(model: &str, text: &str) -> String { |
| 39 | [ |
| 40 | sse_chunk(json!({ |
| 41 | "id": "chatcmpl-vf", |
| 42 | "object": "chat.completion.chunk", |
| 43 | "model": model, |
| 44 | "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": null}] |
| 45 | })), |
| 46 | sse_chunk(json!({ |
| 47 | "id": "chatcmpl-vf", |
| 48 | "object": "chat.completion.chunk", |
| 49 | "model": model, |
| 50 | "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], |
| 51 | "usage": {"prompt_tokens": 11, "completion_tokens": 3, "total_tokens": 14} |
| 52 | })), |
| 53 | "data: [DONE]\n\n".to_string(), |
| 54 | ] |
| 55 | .join("") |
| 56 | } |
| 57 | |
| 58 | fn sse_response(body: String) -> ResponseTemplate { |
| 59 | ResponseTemplate::new(200) |
| 60 | .insert_header("content-type", "text/event-stream") |
| 61 | .insert_header("cache-control", "no-cache") |
| 62 | .set_body_string(body) |
| 63 | } |
| 64 | |
| 65 | fn json_response(value: Value) -> ResponseTemplate { |
| 66 | ResponseTemplate::new(200) |
| 67 | .insert_header("content-type", "application/json") |
| 68 | .set_body_json(value) |
| 69 | } |
| 70 | |
| 71 | async fn mount_models(server: &MockServer) { |
| 72 | Mock::given(method("GET")) |
| 73 | .and(path("/v1/models")) |
| 74 | .respond_with(json_response(json!({ |
| 75 | "object": "list", |
| 76 | "data": [{"id": TEST_MODEL, "object": "model"}] |
| 77 | }))) |
| 78 | .mount(server) |
| 79 | .await; |
| 80 | } |
| 81 | |
| 82 | struct Fixture { |
| 83 | _root: TempDir, |
| 84 | home: PathBuf, |
| 85 | codewhale_home: PathBuf, |
| 86 | workspace: PathBuf, |
| 87 | config_path: PathBuf, |
| 88 | mcp_config_path: PathBuf, |
| 89 | } |
| 90 | |
| 91 | impl Fixture { |
| 92 | /// Build a sealed launch environment whose only route lives in an explicit |
| 93 | /// `--config` file that names the credential env var but never the secret. |
| 94 | fn new(base_url: &str) -> Self { |
| 95 | let root = TempDir::new().expect("fixture root"); |
| 96 | let home = root.path().join("home"); |
| 97 | let codewhale_home = root.path().join("codewhale-home"); |
| 98 | let workspace = root.path().join("workspace"); |
| 99 | for dir in [&home, &codewhale_home, &workspace] { |
| 100 | std::fs::create_dir_all(dir).expect("create fixture dir"); |
| 101 | } |
| 102 | |
| 103 | // Route config: only the env-var NAME is stored, never the secret. |
| 104 | let config_path = root.path().join("config.toml"); |
| 105 | std::fs::write( |
| 106 | &config_path, |
| 107 | format!( |
| 108 | "provider = \"openai\"\n\n[providers.openai]\nbase_url = \"{base_url}/v1\"\nmodel = \"{TEST_MODEL}\"\napi_key_env = \"{API_KEY_ENV}\"\n" |
| 109 | ), |
| 110 | ) |
| 111 | .expect("write route config"); |
| 112 | |
| 113 | // Generated MCP file with no task servers: proves the URL-based MCP |
| 114 | // config surface loads cleanly from a fresh, generated file. |
| 115 | let mcp_config_path = root.path().join("vf-mcp.json"); |
| 116 | std::fs::write(&mcp_config_path, json!({"mcpServers": {}}).to_string()) |
| 117 | .expect("write mcp config"); |
| 118 | |
| 119 | Fixture { |
| 120 | _root: root, |
| 121 | home, |
| 122 | codewhale_home, |
| 123 | workspace, |
| 124 | config_path, |
| 125 | mcp_config_path, |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | /// The exact `#4641` launch shape, minus the dispatcher hop (this drives the |
| 130 | /// `codewhale-tui` runtime directly). `--no-project-config` precedes the |
| 131 | /// subcommand; the prompt follows `--`. |
| 132 | fn exec_argv(&self, prompt: &str) -> Vec<String> { |
| 133 | vec![ |
| 134 | "--config".into(), |
| 135 | self.config_path.to_string_lossy().into_owned(), |
| 136 | "--workspace".into(), |
| 137 | self.workspace.to_string_lossy().into_owned(), |
| 138 | "--no-project-config".into(), |
| 139 | "--skip-onboarding".into(), |
| 140 | "exec".into(), |
| 141 | "--auto".into(), |
| 142 | "--sandbox".into(), |
| 143 | "danger-full-access".into(), |
| 144 | "--append-system-prompt".into(), |
| 145 | APPEND_MARKER.into(), |
| 146 | "--disallowed-tools".into(), |
| 147 | "web_search".into(), |
| 148 | "--output-format".into(), |
| 149 | "stream-json".into(), |
| 150 | "--".into(), |
| 151 | prompt.into(), |
| 152 | ] |
| 153 | } |
| 154 | |
| 155 | fn run(&self, prompt: &str) -> std::process::Output { |
| 156 | let argv = self.exec_argv(prompt); |
| 157 | // The secret must never be an argument; only its env-var name is. |
| 158 | assert!( |
| 159 | !argv.iter().any(|arg| arg.contains(SENTINEL_SECRET)), |
| 160 | "sentinel secret must never appear in argv" |
| 161 | ); |
| 162 | |
| 163 | let mut command = Command::new(codewhale_tui_binary()); |
| 164 | preserve_host_env(&mut command); |
| 165 | command |
| 166 | .current_dir(&self.workspace) |
| 167 | .args(&argv) |
| 168 | .env("HOME", &self.home) |
| 169 | .env("USERPROFILE", &self.home) |
| 170 | .env("XDG_CONFIG_HOME", self.home.join(".config")) |
| 171 | .env("XDG_DATA_HOME", self.home.join(".local").join("share")) |
| 172 | .env("XDG_CACHE_HOME", self.home.join(".cache")) |
| 173 | .env("CODEWHALE_HOME", &self.codewhale_home) |
| 174 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 175 | .env("CODEWHALE_MEMORY", "false") |
| 176 | .env("CODEWHALE_TELEMETRY", "false") |
| 177 | .env("CODEWHALE_MCP_CONFIG", &self.mcp_config_path) |
| 178 | // The interception secret lives ONLY in the child environment, |
| 179 | // reached through the route's api_key_env. |
| 180 | .env(API_KEY_ENV, SENTINEL_SECRET) |
| 181 | .env("RUST_LOG", "warn") |
| 182 | .stdin(Stdio::null()) |
| 183 | .stdout(Stdio::piped()) |
| 184 | .stderr(Stdio::piped()); |
| 185 | |
| 186 | run_with_timeout(command, RUN_TIMEOUT) |
| 187 | } |
| 188 | |
| 189 | /// Every regular file under the sealed roots, for secret-leak scanning. |
| 190 | fn written_files(&self) -> Vec<PathBuf> { |
| 191 | let mut out = Vec::new(); |
| 192 | for base in [&self.home, &self.codewhale_home, &self.workspace] { |
| 193 | collect_files(base, &mut out); |
| 194 | } |
| 195 | out.push(self.config_path.clone()); |
| 196 | out.push(self.mcp_config_path.clone()); |
| 197 | out |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) { |
| 202 | let Ok(entries) = std::fs::read_dir(dir) else { |
| 203 | return; |
| 204 | }; |
| 205 | for entry in entries.flatten() { |
| 206 | let path = entry.path(); |
| 207 | match entry.file_type() { |
| 208 | Ok(ft) if ft.is_dir() => collect_files(&path, out), |
| 209 | Ok(ft) if ft.is_file() => out.push(path), |
| 210 | _ => {} |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | fn run_with_timeout(mut command: Command, timeout: Duration) -> std::process::Output { |
| 216 | let mut child = command.spawn().expect("spawn codewhale-tui exec"); |
| 217 | let stdout_reader = read_pipe_in_background(child.stdout.take().expect("stdout pipe")); |
| 218 | let stderr_reader = read_pipe_in_background(child.stderr.take().expect("stderr pipe")); |
| 219 | |
| 220 | let status = match child.wait_timeout(timeout).expect("wait for codewhale-tui") { |
| 221 | Some(status) => status, |
| 222 | None => { |
| 223 | let _ = child.kill(); |
| 224 | let _ = child.wait(); |
| 225 | let stdout = join_pipe_reader(stdout_reader, "stdout"); |
| 226 | let stderr = join_pipe_reader(stderr_reader, "stderr"); |
| 227 | panic!( |
| 228 | "codewhale-tui exec timed out after {timeout:?}\nstdout:\n{}\nstderr:\n{}", |
| 229 | String::from_utf8_lossy(&stdout), |
| 230 | String::from_utf8_lossy(&stderr) |
| 231 | ); |
| 232 | } |
| 233 | }; |
| 234 | |
| 235 | let stdout = join_pipe_reader(stdout_reader, "stdout"); |
| 236 | let stderr = join_pipe_reader(stderr_reader, "stderr"); |
| 237 | std::process::Output { |
| 238 | status, |
| 239 | stdout, |
| 240 | stderr, |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | fn read_pipe_in_background<R>(mut reader: R) -> std::thread::JoinHandle<std::io::Result<Vec<u8>>> |
| 245 | where |
| 246 | R: Read + Send + 'static, |
| 247 | { |
| 248 | std::thread::spawn(move || { |
| 249 | let mut output = Vec::new(); |
| 250 | reader.read_to_end(&mut output).map(|_| output) |
| 251 | }) |
| 252 | } |
| 253 | |
| 254 | fn join_pipe_reader( |
| 255 | handle: std::thread::JoinHandle<std::io::Result<Vec<u8>>>, |
| 256 | stream_name: &str, |
| 257 | ) -> Vec<u8> { |
| 258 | handle |
| 259 | .join() |
| 260 | .unwrap_or_else(|_| panic!("{stream_name} reader thread panicked")) |
| 261 | .unwrap_or_else(|err| panic!("read {stream_name}: {err}")) |
| 262 | } |
| 263 | |
| 264 | fn preserve_host_env(command: &mut Command) { |
| 265 | command.env_clear(); |
| 266 | for key in [ |
| 267 | "PATH", |
| 268 | "PATHEXT", |
| 269 | "SystemRoot", |
| 270 | "SystemDrive", |
| 271 | "WINDIR", |
| 272 | "COMSPEC", |
| 273 | "TEMP", |
| 274 | "TMP", |
| 275 | "TERM", |
| 276 | "COLORTERM", |
| 277 | "LANG", |
| 278 | "LC_ALL", |
| 279 | ] { |
| 280 | if let Some(value) = std::env::var_os(key) { |
| 281 | command.env(key, value); |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | fn codewhale_tui_binary() -> PathBuf { |
| 287 | if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale-tui") { |
| 288 | return PathBuf::from(path); |
| 289 | } |
| 290 | if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale-tui") { |
| 291 | return PathBuf::from(path); |
| 292 | } |
| 293 | let mut path = std::env::current_exe().expect("current test executable path"); |
| 294 | path.pop(); |
| 295 | if path.ends_with("deps") { |
| 296 | path.pop(); |
| 297 | } |
| 298 | path.push(format!("codewhale-tui{}", std::env::consts::EXE_SUFFIX)); |
| 299 | path |
| 300 | } |
| 301 | |
| 302 | /// The public headless launch reaches exactly the configured route/model, the |
| 303 | /// interception secret stays confined to the child environment, the appended |
| 304 | /// system prompt is delivered, the generated MCP config loads, and the run |
| 305 | /// exits cleanly — all provider-free. |
| 306 | #[tokio::test(flavor = "current_thread")] |
| 307 | async fn headless_launch_confines_secret_and_reaches_configured_route() { |
| 308 | let server = MockServer::start().await; |
| 309 | mount_models(&server).await; |
| 310 | Mock::given(method("POST")) |
| 311 | .and(path("/v1/chat/completions")) |
| 312 | .respond_with(sse_response(text_sse(TEST_MODEL, "contract acknowledged"))) |
| 313 | .mount(&server) |
| 314 | .await; |
| 315 | |
| 316 | let fixture = Fixture::new(&server.uri()); |
| 317 | let output = fixture.run("Reply with a short acknowledgement."); |
| 318 | |
| 319 | let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); |
| 320 | let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); |
| 321 | assert!( |
| 322 | output.status.success(), |
| 323 | "headless exec should exit 0\nstdout:\n{stdout}\nstderr:\n{stderr}" |
| 324 | ); |
| 325 | |
| 326 | let requests = server |
| 327 | .received_requests() |
| 328 | .await |
| 329 | .expect("recorded fixture requests"); |
| 330 | let chat: Vec<&wiremock::Request> = requests |
| 331 | .iter() |
| 332 | .filter(|req| req.url.path() == "/v1/chat/completions") |
| 333 | .collect(); |
| 334 | assert!( |
| 335 | !chat.is_empty(), |
| 336 | "expected at least one chat/completions request to the configured endpoint" |
| 337 | ); |
| 338 | |
| 339 | // Reaches the exact configured model, carrying the secret only as the |
| 340 | // Authorization bearer resolved from api_key_env. |
| 341 | let first = &chat[0]; |
| 342 | let body: Value = serde_json::from_slice(&first.body).expect("chat request body JSON"); |
| 343 | assert_eq!( |
| 344 | body["model"], TEST_MODEL, |
| 345 | "request must target the configured model" |
| 346 | ); |
| 347 | let auth = first |
| 348 | .headers |
| 349 | .get("authorization") |
| 350 | .map(|value| value.to_str().unwrap_or_default().to_string()) |
| 351 | .unwrap_or_default(); |
| 352 | assert_eq!( |
| 353 | auth, |
| 354 | format!("Bearer {SENTINEL_SECRET}"), |
| 355 | "the route must present the api_key_env secret to the model endpoint" |
| 356 | ); |
| 357 | |
| 358 | // The appended system prompt is delivered to the model. |
| 359 | let body_text = String::from_utf8_lossy(&first.body); |
| 360 | assert!( |
| 361 | body_text.contains(APPEND_MARKER), |
| 362 | "appended system prompt must reach the model request" |
| 363 | ); |
| 364 | |
| 365 | // Secret hygiene: the sentinel must not appear in argv, stdout (the |
| 366 | // stream-json stream), stderr, or any written file. |
| 367 | assert!( |
| 368 | !stdout.contains(SENTINEL_SECRET), |
| 369 | "secret leaked into stdout/stream-json" |
| 370 | ); |
| 371 | assert!( |
| 372 | !stderr.contains(SENTINEL_SECRET), |
| 373 | "secret leaked into stderr" |
| 374 | ); |
| 375 | for file in fixture.written_files() { |
| 376 | let Ok(bytes) = std::fs::read(&file) else { |
| 377 | continue; |
| 378 | }; |
| 379 | assert!( |
| 380 | !String::from_utf8_lossy(&bytes).contains(SENTINEL_SECRET), |
| 381 | "secret leaked into written file: {}", |
| 382 | file.display() |
| 383 | ); |
| 384 | } |
| 385 | |
| 386 | // Stdout is a valid stream-json stream (every non-empty line parses). |
| 387 | let mut saw_line = false; |
| 388 | for line in stdout.lines().filter(|line| !line.trim().is_empty()) { |
| 389 | saw_line = true; |
| 390 | serde_json::from_str::<Value>(line) |
| 391 | .unwrap_or_else(|err| panic!("stream-json line must parse: {err}\nline: {line}")); |
| 392 | } |
| 393 | assert!(saw_line, "expected a stream-json event stream on stdout"); |
| 394 | } |
| 395 | |
| 396 | /// A fixture model failure surfaces as a nonzero exit, so a harness can treat |
| 397 | /// the launch as failed rather than silently succeeding. |
| 398 | #[tokio::test(flavor = "current_thread")] |
| 399 | async fn fixture_model_failure_exits_nonzero() { |
| 400 | let server = MockServer::start().await; |
| 401 | mount_models(&server).await; |
| 402 | Mock::given(method("POST")) |
| 403 | .and(path("/v1/chat/completions")) |
| 404 | .respond_with(ResponseTemplate::new(500).set_body_string("upstream model failure")) |
| 405 | .mount(&server) |
| 406 | .await; |
| 407 | |
| 408 | let fixture = Fixture::new(&server.uri()); |
| 409 | let output = fixture.run("This request should fail at the model."); |
| 410 | assert!( |
| 411 | !output.status.success(), |
| 412 | "a fixture model failure must exit nonzero\nstdout:\n{}\nstderr:\n{}", |
| 413 | String::from_utf8_lossy(&output.stdout), |
| 414 | String::from_utf8_lossy(&output.stderr) |
| 415 | ); |
| 416 | } |
| 417 |