| 1 | //! Negative controls for the `mcp` family: each one changes a case (or the |
| 2 | //! dispatch) the way a regression would and shows the suite fails, for the |
| 3 | //! reason the case exists. A golden that no mutation can move pins nothing. |
| 4 | //! |
| 5 | //! Every control runs through [`run_case`] with `compare_only`, so a control |
| 6 | //! can never rewrite a golden, even under `CODEWHALE_CONFORMANCE_UPDATE=1`. |
| 7 | //! The existing stalled-server control |
| 8 | //! (`harness_timeout_rejects_a_real_unanswered_mcp_call`) lives with the |
| 9 | //! runner; these add the rest. |
| 10 | //! |
| 11 | //! Two controls swap the *dispatch* instead of the case: a dispatch that |
| 12 | //! replays a failed call, and one that refreshes its catalog behind the |
| 13 | //! model's back. They stand in for a replacement implementation getting |
| 14 | //! those behaviours wrong, and prove the `DISPATCHES` seam would catch it. |
| 15 | |
| 16 | use serde_json::{Value, json}; |
| 17 | use tokio_util::sync::CancellationToken; |
| 18 | |
| 19 | use super::{ |
| 20 | BEARER_ENV, CALL_DEADLINE, DispatchFactory, FAMILY, McpDispatchUnderTest, McpPoolDispatch, |
| 21 | assert_no_secret, run_case, |
| 22 | }; |
| 23 | use crate::conformance::golden; |
| 24 | use crate::tools::spec::{RichToolResult, ToolError}; |
| 25 | |
| 26 | /// Run `name` with `mutate` against both actual production backends. |
| 27 | fn mutated(name: &str, mutate: impl FnOnce(&mut Value)) -> Result<(), String> { |
| 28 | let mut case = golden::read_case(FAMILY, name); |
| 29 | mutate(&mut case); |
| 30 | let mut errors = Vec::new(); |
| 31 | for (dispatch, factory) in super::DISPATCHES { |
| 32 | match run_case(name, &case, *factory, CALL_DEADLINE, true) { |
| 33 | Ok(()) => return Ok(()), // one implementation missed the mutation |
| 34 | Err(error) => errors.push(format!("{dispatch}: {error}")), |
| 35 | } |
| 36 | } |
| 37 | Err(errors.join("\n")) |
| 38 | } |
| 39 | |
| 40 | fn mutated_via( |
| 41 | name: &str, |
| 42 | factory: DispatchFactory, |
| 43 | mutate: impl FnOnce(&mut Value), |
| 44 | ) -> Result<(), String> { |
| 45 | let mut case = golden::read_case(FAMILY, name); |
| 46 | mutate(&mut case); |
| 47 | run_case(name, &case, factory, CALL_DEADLINE, true) |
| 48 | } |
| 49 | |
| 50 | fn set(case: &mut Value, pointer: &str, value: Value) { |
| 51 | *case |
| 52 | .pointer_mut(pointer) |
| 53 | .unwrap_or_else(|| panic!("case has no `{pointer}`")) = value; |
| 54 | } |
| 55 | |
| 56 | #[track_caller] |
| 57 | fn drifts(result: Result<(), String>) -> String { |
| 58 | let error = result.expect_err("the changed case must not match its golden"); |
| 59 | eprintln!("control drift: {error}"); |
| 60 | assert!( |
| 61 | error.contains("golden drift"), |
| 62 | "expected a golden drift, got: {error}" |
| 63 | ); |
| 64 | error |
| 65 | } |
| 66 | |
| 67 | #[track_caller] |
| 68 | fn harness_rejects(result: Result<(), String>, reason: &str) { |
| 69 | let error = result.expect_err("the changed case must fail the harness"); |
| 70 | assert!(error.contains(reason), "expected `{reason}`, got: {error}"); |
| 71 | } |
| 72 | |
| 73 | // --- stdio --- |
| 74 | |
| 75 | #[cfg(unix)] |
| 76 | #[test] |
| 77 | fn control_stdio_server_that_survives_its_call_is_noticed() { |
| 78 | drifts(mutated("stdio_tools_and_exit", |case| { |
| 79 | set( |
| 80 | case, |
| 81 | "/server/tools~1call/3", |
| 82 | json!({"match": {"name": "dies"}, "result": {"content": [{"type": "text", "text": "survived"}]}}), |
| 83 | ); |
| 84 | })); |
| 85 | } |
| 86 | |
| 87 | #[cfg(unix)] |
| 88 | #[test] |
| 89 | fn control_stdio_changed_reply_bytes_are_noticed() { |
| 90 | drifts(mutated("stdio_tools_and_exit", |case| { |
| 91 | set( |
| 92 | case, |
| 93 | "/server/tools~1call/0/result/content/0/text", |
| 94 | json!("echo from stdio, changed"), |
| 95 | ); |
| 96 | })); |
| 97 | } |
| 98 | |
| 99 | // --- sessions --- |
| 100 | |
| 101 | #[test] |
| 102 | fn control_a_stale_session_that_never_happens_is_noticed() { |
| 103 | drifts(mutated("http_session_lifecycle", |case| { |
| 104 | case["server"]["tools/call"] |
| 105 | .as_array_mut() |
| 106 | .expect("candidates") |
| 107 | .remove(1); |
| 108 | })); |
| 109 | } |
| 110 | |
| 111 | #[test] |
| 112 | fn control_the_negotiated_protocol_revision_is_pinned() { |
| 113 | drifts(mutated("http_session_lifecycle", |case| { |
| 114 | set( |
| 115 | case, |
| 116 | "/server/initialize/result/protocolVersion", |
| 117 | json!("2024-11-05"), |
| 118 | ); |
| 119 | })); |
| 120 | } |
| 121 | |
| 122 | #[test] |
| 123 | fn control_a_dispatch_that_replays_a_failed_call_is_caught() { |
| 124 | // `session_error` answers a JSON-RPC error that mentions the session; the |
| 125 | // pool does not replay it (the server may have acted). A dispatch that |
| 126 | // calls again after any error shows up as a second `session_error` in |
| 127 | // `server_received`, and as a second `tools/call` request. |
| 128 | drifts(mutated_via( |
| 129 | "http_session_lifecycle", |
| 130 | |setup| { |
| 131 | Box::new(Deviant::new( |
| 132 | McpPoolDispatch::boxed(setup), |
| 133 | Deviation::ReplayOnError, |
| 134 | )) |
| 135 | }, |
| 136 | |_| {}, |
| 137 | )); |
| 138 | } |
| 139 | |
| 140 | #[test] |
| 141 | fn control_a_dispatch_that_refreshes_on_list_changed_is_caught() { |
| 142 | // The Rust pool ignores `notifications/tools/list_changed`; a replacement |
| 143 | // that re-lists on its own (here: on every catalog read) must not match. |
| 144 | drifts(mutated_via( |
| 145 | "http_session_lifecycle", |
| 146 | |setup| { |
| 147 | Box::new(Deviant::new( |
| 148 | McpPoolDispatch::boxed(setup), |
| 149 | Deviation::RefreshCatalog, |
| 150 | )) |
| 151 | }, |
| 152 | |_| {}, |
| 153 | )); |
| 154 | } |
| 155 | |
| 156 | #[test] |
| 157 | fn control_a_newer_accepted_protocol_revision_does_not_end_the_handshake() { |
| 158 | // The boundary of `MCP_CLIENT_ACCEPTED_PROTOCOL_VERSIONS`: the same case |
| 159 | // with a revision the client does implement connects, so the case fails |
| 160 | // the harness (it expects boot to fail) instead of drifting. |
| 161 | harness_rejects( |
| 162 | mutated("http_protocol_version_unsupported", |case| { |
| 163 | set( |
| 164 | case, |
| 165 | "/server/initialize/result/protocolVersion", |
| 166 | json!("2025-11-25"), |
| 167 | ); |
| 168 | case["server"]["tools/list"] = json!({"result": {"tools": []}}); |
| 169 | }), |
| 170 | "expected boot to fail", |
| 171 | ); |
| 172 | } |
| 173 | |
| 174 | // --- catalog caps --- |
| 175 | |
| 176 | #[test] |
| 177 | fn control_a_catalog_exactly_on_the_item_cap_still_connects() { |
| 178 | harness_rejects( |
| 179 | mutated("catalog_cap_items", |case| { |
| 180 | set( |
| 181 | case, |
| 182 | "/server/tools~1list/generate_tools/count", |
| 183 | json!(4096), |
| 184 | ); |
| 185 | }), |
| 186 | "expected boot to fail", |
| 187 | ); |
| 188 | } |
| 189 | |
| 190 | #[test] |
| 191 | fn control_a_catalog_exactly_on_the_page_cap_still_connects() { |
| 192 | harness_rejects( |
| 193 | mutated("catalog_cap_pages", |case| { |
| 194 | set(case, "/server/tools~1list/generate_pages/count", json!(64)); |
| 195 | }), |
| 196 | "expected boot to fail", |
| 197 | ); |
| 198 | } |
| 199 | |
| 200 | #[test] |
| 201 | fn control_a_cursor_chain_that_ends_is_not_a_repeat() { |
| 202 | harness_rejects( |
| 203 | mutated("catalog_cap_cursor_repeat", |case| { |
| 204 | case["server"]["tools/list"][0]["result"] |
| 205 | .as_object_mut() |
| 206 | .expect("loop page") |
| 207 | .remove("nextCursor"); |
| 208 | }), |
| 209 | "expected boot to fail", |
| 210 | ); |
| 211 | } |
| 212 | |
| 213 | #[test] |
| 214 | fn control_a_dropped_page_is_noticed() { |
| 215 | drifts(mutated("catalog_pagination", |case| { |
| 216 | case["server"]["tools/list"][2]["result"] |
| 217 | .as_object_mut() |
| 218 | .expect("first page") |
| 219 | .remove("nextCursor"); |
| 220 | })); |
| 221 | } |
| 222 | |
| 223 | #[test] |
| 224 | fn control_a_changed_approval_annotation_is_noticed() { |
| 225 | drifts(mutated("approval_hints", |case| { |
| 226 | set( |
| 227 | case, |
| 228 | "/server/tools~1list/result/tools/2/annotations/destructiveHint", |
| 229 | json!(false), |
| 230 | ); |
| 231 | })); |
| 232 | } |
| 233 | |
| 234 | // --- auth --- |
| 235 | |
| 236 | /// A server that accepts the handshake: what the 401 and redirect cases must |
| 237 | /// not turn into. |
| 238 | fn open_server() -> Value { |
| 239 | json!({ |
| 240 | "initialize": {"result": { |
| 241 | "protocolVersion": "2025-06-18", |
| 242 | "serverInfo": {"name": "open", "version": "1"}, |
| 243 | "capabilities": {"tools": {}}, |
| 244 | }}, |
| 245 | "tools/list": {"result": {"tools": []}}, |
| 246 | }) |
| 247 | } |
| 248 | |
| 249 | #[test] |
| 250 | fn control_a_server_that_accepts_the_login_is_not_needs_auth() { |
| 251 | harness_rejects( |
| 252 | mutated("auth_needs_login", |case| { |
| 253 | set(case, "/server", open_server()) |
| 254 | }), |
| 255 | "expected boot to fail", |
| 256 | ); |
| 257 | } |
| 258 | |
| 259 | #[test] |
| 260 | fn control_a_client_that_stops_sending_its_bearer_is_noticed() { |
| 261 | // Without the secret the harness exports no token and the server stops |
| 262 | // requiring one; what moves is the recorded `authorization` shape. |
| 263 | drifts(mutated("auth_bearer_never_recorded", |case| { |
| 264 | case.as_object_mut().expect("case").remove("bearer_secret"); |
| 265 | })); |
| 266 | } |
| 267 | |
| 268 | #[test] |
| 269 | fn control_a_followed_redirect_is_noticed() { |
| 270 | harness_rejects( |
| 271 | mutated("auth_redirect_cross_origin", |case| { |
| 272 | set(case, "/server", open_server()) |
| 273 | }), |
| 274 | "expected boot to fail", |
| 275 | ); |
| 276 | } |
| 277 | |
| 278 | #[test] |
| 279 | fn control_a_dispatch_that_leaks_the_credential_is_caught() { |
| 280 | let error = mutated_via( |
| 281 | "auth_bearer_never_recorded", |
| 282 | |setup| { |
| 283 | Box::new(Deviant::new( |
| 284 | McpPoolDispatch::boxed(setup), |
| 285 | Deviation::LeakCredential, |
| 286 | )) |
| 287 | }, |
| 288 | |_| {}, |
| 289 | ) |
| 290 | .expect_err("a leaked credential must fail the case"); |
| 291 | assert!(error.contains("secret leak"), "{error}"); |
| 292 | } |
| 293 | |
| 294 | #[test] |
| 295 | fn the_secret_scan_trips_on_every_spelling_of_the_token() { |
| 296 | let secret = "conformance-secret-token-0123456789"; |
| 297 | assert_no_secret( |
| 298 | secret, |
| 299 | "{\"requests\": [{\"authorization\": \"<bearer>\"}]}", |
| 300 | ) |
| 301 | .expect("a shape is not a secret"); |
| 302 | // Name the spelling that was missed, never print it: the failure message |
| 303 | // must not carry the token (or a string built from it) into a log. |
| 304 | for (spelling, leaked) in [ |
| 305 | ( |
| 306 | "an Authorization header", |
| 307 | format!("Authorization: Bearer {secret}"), |
| 308 | ), |
| 309 | ( |
| 310 | "a JSON error detail", |
| 311 | format!("{{\"detail\": \"rejected {secret}\"}}"), |
| 312 | ), |
| 313 | ("a URL query", format!("http://host/mcp?token={secret}")), |
| 314 | ] { |
| 315 | assert!( |
| 316 | assert_no_secret(secret, &leaked).is_err(), |
| 317 | "the scan missed the token in {spelling}" |
| 318 | ); |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | #[test] |
| 323 | fn no_committed_golden_carries_a_case_secret() { |
| 324 | let mut scanned = 0; |
| 325 | for name in golden::case_names(FAMILY) { |
| 326 | let case = golden::read_case(FAMILY, &name); |
| 327 | let Some(secret) = case["bearer_secret"].as_str() else { |
| 328 | continue; |
| 329 | }; |
| 330 | let path = golden::family_dir(FAMILY).join(format!("{name}.golden.json")); |
| 331 | let text = std::fs::read_to_string(&path) |
| 332 | .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); |
| 333 | assert_no_secret(secret, &text).unwrap_or_else(|error| panic!("{name}: {error}")); |
| 334 | scanned += 1; |
| 335 | } |
| 336 | assert!(scanned >= 2, "expected the auth cases to declare secrets"); |
| 337 | } |
| 338 | |
| 339 | // --- deny rules --- |
| 340 | |
| 341 | #[test] |
| 342 | fn control_dropping_the_deny_rules_is_noticed() { |
| 343 | drifts(mutated("deny_tool_rules", |case| { |
| 344 | set(case, "/disallowed_tools", json!([])); |
| 345 | })); |
| 346 | drifts(mutated("deny_server_wildcard", |case| { |
| 347 | set(case, "/disallowed_tools", json!([])); |
| 348 | })); |
| 349 | } |
| 350 | |
| 351 | // --- deviant dispatches --- |
| 352 | |
| 353 | #[derive(Clone, Copy)] |
| 354 | enum Deviation { |
| 355 | /// Call a failed tool a second time. |
| 356 | ReplayOnError, |
| 357 | /// Reconnect before every catalog read. |
| 358 | RefreshCatalog, |
| 359 | /// Put the bearer token into an error detail. |
| 360 | LeakCredential, |
| 361 | } |
| 362 | |
| 363 | struct Deviant { |
| 364 | inner: Box<dyn McpDispatchUnderTest>, |
| 365 | deviation: Deviation, |
| 366 | } |
| 367 | |
| 368 | impl Deviant { |
| 369 | fn new(inner: Box<dyn McpDispatchUnderTest>, deviation: Deviation) -> Self { |
| 370 | Self { inner, deviation } |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | #[async_trait::async_trait] |
| 375 | impl McpDispatchUnderTest for Deviant { |
| 376 | async fn boot(&self) -> Result<(), String> { |
| 377 | self.inner.boot().await |
| 378 | } |
| 379 | |
| 380 | async fn catalog(&self) -> Vec<codewhale_models::Tool> { |
| 381 | if matches!(self.deviation, Deviation::RefreshCatalog) { |
| 382 | self.inner.shutdown().await; |
| 383 | let _ = self.inner.boot().await; |
| 384 | } |
| 385 | self.inner.catalog().await |
| 386 | } |
| 387 | |
| 388 | async fn call( |
| 389 | &self, |
| 390 | model_name: &str, |
| 391 | input: Value, |
| 392 | cancel: CancellationToken, |
| 393 | ) -> Result<RichToolResult, ToolError> { |
| 394 | let first = self |
| 395 | .inner |
| 396 | .call(model_name, input.clone(), cancel.clone()) |
| 397 | .await; |
| 398 | match (self.deviation, first) { |
| 399 | (Deviation::ReplayOnError, Err(error)) |
| 400 | if !matches!(error, ToolError::Cancelled { .. }) => |
| 401 | { |
| 402 | self.inner.call(model_name, input, cancel).await |
| 403 | } |
| 404 | (Deviation::LeakCredential, Err(error)) => { |
| 405 | let token = std::env::var(BEARER_ENV).unwrap_or_default(); |
| 406 | Err(ToolError::execution_failed(format!( |
| 407 | "{error} (sent Authorization: Bearer {token})" |
| 408 | ))) |
| 409 | } |
| 410 | (_, result) => result, |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | async fn approval_hint(&self, model_name: &str) -> Option<&'static str> { |
| 415 | self.inner.approval_hint(model_name).await |
| 416 | } |
| 417 | |
| 418 | async fn shutdown(&self) { |
| 419 | self.inner.shutdown().await; |
| 420 | } |
| 421 | } |
| 422 |