| 1 | //! Fixture discovery, golden comparison, the update mode, and the masking |
| 2 | //! rules every family shares. Masking is deliberately small and named: a mask |
| 3 | //! hides a fact that genuinely varies between hosts or runs (temp paths, |
| 4 | //! UUIDs, clocks), never a fact the migration could change by accident. |
| 5 | |
| 6 | use std::collections::{BTreeMap, HashMap}; |
| 7 | use std::path::{Path, PathBuf}; |
| 8 | |
| 9 | use serde_json::Value; |
| 10 | |
| 11 | use crate::test_support::{EnvVarGuard, TestEnvLock, lock_test_env}; |
| 12 | use crate::tools::spec::ToolError; |
| 13 | |
| 14 | /// `CODEWHALE_CONFORMANCE_UPDATE=1` rewrites goldens from the current source. |
| 15 | pub(super) const UPDATE_ENV: &str = "CODEWHALE_CONFORMANCE_UPDATE"; |
| 16 | |
| 17 | pub(super) fn fixture_root() -> PathBuf { |
| 18 | Path::new(env!("CARGO_MANIFEST_DIR")) |
| 19 | .join("tests") |
| 20 | .join("fixtures") |
| 21 | .join("conformance") |
| 22 | } |
| 23 | |
| 24 | pub(super) fn family_dir(family: &str) -> PathBuf { |
| 25 | fixture_root().join(family) |
| 26 | } |
| 27 | |
| 28 | /// Case names (`<name>.case.json`) in one family, sorted. Panics on an empty |
| 29 | /// family: a filter that silently runs zero cases is not a pass. |
| 30 | pub(super) fn case_names(family: &str) -> Vec<String> { |
| 31 | let dir = family_dir(family); |
| 32 | let mut names: Vec<String> = std::fs::read_dir(&dir) |
| 33 | .unwrap_or_else(|error| panic!("read fixture dir {}: {error}", dir.display())) |
| 34 | .filter_map(|entry| { |
| 35 | let name = entry.ok()?.file_name().into_string().ok()?; |
| 36 | name.strip_suffix(".case.json").map(str::to_string) |
| 37 | }) |
| 38 | .collect(); |
| 39 | names.sort(); |
| 40 | assert!( |
| 41 | !names.is_empty(), |
| 42 | "conformance family `{family}` has no *.case.json fixtures in {}", |
| 43 | dir.display() |
| 44 | ); |
| 45 | names |
| 46 | } |
| 47 | |
| 48 | pub(super) fn read_case(family: &str, name: &str) -> Value { |
| 49 | let path = family_dir(family).join(format!("{name}.case.json")); |
| 50 | let text = std::fs::read_to_string(&path) |
| 51 | .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); |
| 52 | serde_json::from_str(&text).unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) |
| 53 | } |
| 54 | |
| 55 | pub(super) fn update_mode() -> bool { |
| 56 | let requested = std::env::var(UPDATE_ENV).is_ok_and(|value| value == "1"); |
| 57 | if requested && std::env::var_os("CI").is_some_and(|value| !value.is_empty()) { |
| 58 | panic!("{UPDATE_ENV}=1 is refused under CI: goldens are reviewed source, not build output"); |
| 59 | } |
| 60 | requested |
| 61 | } |
| 62 | |
| 63 | /// Compare `actual` with the golden at `path`; in update mode write it |
| 64 | /// instead. Returns a failure description rather than panicking so a family |
| 65 | /// can report every drifted case in one run. |
| 66 | pub(crate) fn check_golden(path: &Path, actual: &str) -> Result<(), String> { |
| 67 | let update = update_mode(); |
| 68 | let expected = std::fs::read_to_string(path).ok(); |
| 69 | if expected.as_deref() == Some(actual) { |
| 70 | return Ok(()); |
| 71 | } |
| 72 | if update { |
| 73 | if let Some(parent) = path.parent() { |
| 74 | std::fs::create_dir_all(parent).expect("create golden dir"); |
| 75 | } |
| 76 | std::fs::write(path, actual).expect("write golden"); |
| 77 | eprintln!("conformance: rewrote {}", path.display()); |
| 78 | return Ok(()); |
| 79 | } |
| 80 | let Some(expected) = expected else { |
| 81 | return Err(format!( |
| 82 | "missing golden {}; review the output and record it with {UPDATE_ENV}=1", |
| 83 | path.display() |
| 84 | )); |
| 85 | }; |
| 86 | Err(format!( |
| 87 | "golden drift at {}\n{}\nIf this change is intended, re-record with {UPDATE_ENV}=1 and review the diff.", |
| 88 | path.display(), |
| 89 | first_difference(&expected, actual) |
| 90 | )) |
| 91 | } |
| 92 | |
| 93 | /// A harness deadline is missing evidence, never a provider outcome or a |
| 94 | /// recordable golden. Keep this boundary shared by every asynchronous family. |
| 95 | pub(super) async fn complete_within<T>( |
| 96 | operation: &str, |
| 97 | deadline: std::time::Duration, |
| 98 | future: impl std::future::Future<Output = T>, |
| 99 | ) -> Result<T, String> { |
| 100 | tokio::time::timeout(deadline, future) |
| 101 | .await |
| 102 | .map_err(|_| format!("harness timeout: {operation} did not complete within {deadline:?}")) |
| 103 | } |
| 104 | |
| 105 | /// Line-oriented first difference with a little context — enough to see what |
| 106 | /// moved without dumping a whole transcript into the failure. |
| 107 | fn first_difference(expected: &str, actual: &str) -> String { |
| 108 | let expected_lines: Vec<&str> = expected.lines().collect(); |
| 109 | let actual_lines: Vec<&str> = actual.lines().collect(); |
| 110 | let index = expected_lines |
| 111 | .iter() |
| 112 | .zip(&actual_lines) |
| 113 | .position(|(left, right)| left != right) |
| 114 | .unwrap_or_else(|| expected_lines.len().min(actual_lines.len())); |
| 115 | let character = match (expected_lines.get(index), actual_lines.get(index)) { |
| 116 | (Some(left), Some(right)) => left |
| 117 | .chars() |
| 118 | .zip(right.chars()) |
| 119 | .take_while(|(left, right)| left == right) |
| 120 | .count(), |
| 121 | _ => 0, |
| 122 | }; |
| 123 | let start = character.saturating_sub(40); |
| 124 | let show = |lines: &[&str]| { |
| 125 | lines.get(index).map_or_else( |
| 126 | || "<end of file>".to_string(), |
| 127 | |line| { |
| 128 | let window: String = line.chars().skip(start).take(601).collect(); |
| 129 | format!( |
| 130 | "{}{}", |
| 131 | if start == 0 { "" } else { "…" }, |
| 132 | truncate(&window, 600) |
| 133 | ) |
| 134 | }, |
| 135 | ) |
| 136 | }; |
| 137 | format!( |
| 138 | "first difference at line {}, character {} (expected {} lines, got {}):\n expected: {}\n actual: {}", |
| 139 | index + 1, |
| 140 | character + 1, |
| 141 | expected_lines.len(), |
| 142 | actual_lines.len(), |
| 143 | show(&expected_lines), |
| 144 | show(&actual_lines) |
| 145 | ) |
| 146 | } |
| 147 | |
| 148 | fn truncate(line: &str, max: usize) -> String { |
| 149 | if line.chars().count() <= max { |
| 150 | return line.to_string(); |
| 151 | } |
| 152 | let head: String = line.chars().take(max).collect(); |
| 153 | format!("{head}…") |
| 154 | } |
| 155 | |
| 156 | /// Rebuild every object with keys in sorted order. Used where map order is an |
| 157 | /// implementation accident (HashMap-backed metadata); the prompt family does |
| 158 | /// not use it, because there key order is part of the cached prefix bytes. |
| 159 | pub(super) fn canonical(value: &Value) -> Value { |
| 160 | match value { |
| 161 | Value::Object(map) => { |
| 162 | let sorted: BTreeMap<&String, &Value> = map.iter().collect(); |
| 163 | Value::Object( |
| 164 | sorted |
| 165 | .into_iter() |
| 166 | .map(|(key, value)| (key.clone(), canonical(value))) |
| 167 | .collect(), |
| 168 | ) |
| 169 | } |
| 170 | Value::Array(items) => Value::Array(items.iter().map(canonical).collect()), |
| 171 | other => other.clone(), |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | /// One compact JSON document per line, trailing newline. |
| 176 | pub(super) fn jsonl(lines: &[Value]) -> String { |
| 177 | let mut out = String::new(); |
| 178 | for line in lines { |
| 179 | out.push_str(&serde_json::to_string(line).expect("serialize golden line")); |
| 180 | out.push('\n'); |
| 181 | } |
| 182 | out |
| 183 | } |
| 184 | |
| 185 | pub(super) fn pretty(value: &Value) -> String { |
| 186 | let mut out = serde_json::to_string_pretty(value).expect("serialize golden"); |
| 187 | out.push('\n'); |
| 188 | out |
| 189 | } |
| 190 | |
| 191 | /// Replaces host- and run-specific substrings in every string of a JSON tree. |
| 192 | /// |
| 193 | /// - literal path prefixes (workspace, home) → `<WORKSPACE>` / `<HOME>`, |
| 194 | /// including their canonicalized spellings (`/private/var` on macOS); |
| 195 | /// - UUIDs → `<uuid:N>`, numbered by first appearance so that two events |
| 196 | /// naming the same id still visibly agree; |
| 197 | /// - RFC 3339 timestamps → `<timestamp>` (applied first); |
| 198 | /// - values of the named volatile keys (durations, clocks) → `"<masked>"`. |
| 199 | pub(super) struct Masker { |
| 200 | literals: Vec<(String, String)>, |
| 201 | volatile_keys: &'static [&'static str], |
| 202 | uuids: HashMap<String, String>, |
| 203 | uuid_re: regex::Regex, |
| 204 | timestamp_re: regex::Regex, |
| 205 | } |
| 206 | |
| 207 | impl Masker { |
| 208 | pub(super) fn new(volatile_keys: &'static [&'static str]) -> Self { |
| 209 | Self { |
| 210 | literals: Vec::new(), |
| 211 | volatile_keys, |
| 212 | uuids: HashMap::new(), |
| 213 | uuid_re: regex::Regex::new( |
| 214 | r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", |
| 215 | ) |
| 216 | .expect("uuid regex"), |
| 217 | timestamp_re: regex::Regex::new( |
| 218 | r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})", |
| 219 | ) |
| 220 | .expect("timestamp regex"), |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | /// Mask `path` (and its canonical spelling) as `label`. Longer literals |
| 225 | /// are applied first so a home nested in a workspace cannot half-match. |
| 226 | pub(super) fn path(mut self, path: &Path, label: &str) -> Self { |
| 227 | let mut spellings = vec![path.to_string_lossy().into_owned()]; |
| 228 | if let Ok(canonical) = path.canonicalize() { |
| 229 | spellings.push(canonical.to_string_lossy().into_owned()); |
| 230 | } |
| 231 | for spelling in spellings { |
| 232 | if !spelling.is_empty() && !self.literals.iter().any(|(known, _)| *known == spelling) { |
| 233 | self.literals.push((spelling, label.to_string())); |
| 234 | } |
| 235 | } |
| 236 | self.literals |
| 237 | .sort_by(|(left, _), (right, _)| right.len().cmp(&left.len()).then(left.cmp(right))); |
| 238 | self |
| 239 | } |
| 240 | |
| 241 | /// Mask an arbitrary literal (a random session id, say). |
| 242 | pub(super) fn literal(mut self, literal: &str, label: &str) -> Self { |
| 243 | if !literal.is_empty() { |
| 244 | self.literals.push((literal.to_string(), label.to_string())); |
| 245 | self.literals.sort_by(|(left, _), (right, _)| { |
| 246 | right.len().cmp(&left.len()).then(left.cmp(right)) |
| 247 | }); |
| 248 | } |
| 249 | self |
| 250 | } |
| 251 | |
| 252 | pub(super) fn text(&mut self, text: &str) -> String { |
| 253 | // Timestamps first: a literal (today's date, say) must not split one. |
| 254 | let mut out = self |
| 255 | .timestamp_re |
| 256 | .replace_all(text, "<timestamp>") |
| 257 | .into_owned(); |
| 258 | for (literal, label) in &self.literals { |
| 259 | if out.contains(literal.as_str()) { |
| 260 | out = out.replace(literal.as_str(), label); |
| 261 | } |
| 262 | } |
| 263 | let uuids = &mut self.uuids; |
| 264 | self.uuid_re |
| 265 | .replace_all(&out, |captures: ®ex::Captures<'_>| { |
| 266 | let raw = captures[0].to_ascii_lowercase(); |
| 267 | let next = uuids.len() + 1; |
| 268 | uuids |
| 269 | .entry(raw) |
| 270 | .or_insert_with(|| format!("<uuid:{next}>")) |
| 271 | .clone() |
| 272 | }) |
| 273 | .into_owned() |
| 274 | } |
| 275 | |
| 276 | pub(super) fn value(&mut self, value: &mut Value) { |
| 277 | match value { |
| 278 | Value::String(text) => *text = self.text(text), |
| 279 | Value::Array(items) => { |
| 280 | for item in items { |
| 281 | self.value(item); |
| 282 | } |
| 283 | } |
| 284 | Value::Object(map) => { |
| 285 | for (key, item) in map.iter_mut() { |
| 286 | if self.volatile_keys.contains(&key.as_str()) && !item.is_null() { |
| 287 | *item = Value::String("<masked>".to_string()); |
| 288 | } else { |
| 289 | self.value(item); |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | _ => {} |
| 294 | } |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | /// Stable snake_case name of a `ToolError` variant. The golden also pins its |
| 299 | /// full detail bytes; migration does not weaken either part of the contract. |
| 300 | pub(super) fn tool_error_kind(error: &ToolError) -> &'static str { |
| 301 | match error { |
| 302 | ToolError::InvalidInput { .. } => "invalid_input", |
| 303 | ToolError::MissingField { .. } => "missing_field", |
| 304 | ToolError::PathEscape { .. } => "path_escape", |
| 305 | ToolError::ExecutionFailed { .. } => "execution_failed", |
| 306 | ToolError::Timeout { .. } => "timeout", |
| 307 | ToolError::Cancelled { .. } => "cancelled", |
| 308 | ToolError::NotAvailable { .. } => "not_available", |
| 309 | ToolError::PermissionDenied { .. } => "permission_denied", |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | /// A hermetic home + workspace for one case. Holds the process test |
| 314 | /// env lock for its lifetime; field order is drop order (guards restore the |
| 315 | /// environment before the lock is released). |
| 316 | pub(super) struct Sandbox { |
| 317 | _guards: Vec<EnvVarGuard>, |
| 318 | _lock: TestEnvLock, |
| 319 | pub(super) home: std::path::PathBuf, |
| 320 | pub(super) workspace: std::path::PathBuf, |
| 321 | root: tempfile::TempDir, |
| 322 | } |
| 323 | |
| 324 | impl Sandbox { |
| 325 | pub(super) fn new(case: &Value) -> Self { |
| 326 | let lock = lock_test_env(); |
| 327 | let root = tempfile::tempdir().expect("tempdir"); |
| 328 | let home = root.path().join("home"); |
| 329 | let workspace = root.path().join("workspace"); |
| 330 | std::fs::create_dir_all(&home).expect("home"); |
| 331 | std::fs::create_dir_all(&workspace).expect("workspace"); |
| 332 | let guards = vec![ |
| 333 | EnvVarGuard::set("HOME", &home), |
| 334 | EnvVarGuard::set("USERPROFILE", &home), |
| 335 | EnvVarGuard::set("CODEWHALE_HOME", home.join(".codewhale")), |
| 336 | // Model-visible host facts that would otherwise follow the |
| 337 | // developer's shell and locale. |
| 338 | EnvVarGuard::set("SHELL", "/bin/bash"), |
| 339 | EnvVarGuard::set("LC_ALL", "en_US.UTF-8"), |
| 340 | EnvVarGuard::set("LANG", "en_US.UTF-8"), |
| 341 | EnvVarGuard::remove("LC_MESSAGES"), |
| 342 | ]; |
| 343 | write_workspace(&workspace, case); |
| 344 | if case["trusted_workspace"].as_bool() == Some(true) { |
| 345 | // Repository instructions, commands and skills load only here. |
| 346 | crate::test_support::trust_workspace(&workspace); |
| 347 | } |
| 348 | Self { |
| 349 | _guards: guards, |
| 350 | _lock: lock, |
| 351 | home, |
| 352 | workspace, |
| 353 | root, |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | pub(super) fn masker(&self, volatile_keys: &'static [&'static str]) -> Masker { |
| 358 | // `<turn_meta>` states the local date; it is a clock, not a contract. |
| 359 | let today = chrono::Local::now().format("%Y-%m-%d").to_string(); |
| 360 | Masker::new(volatile_keys) |
| 361 | .path(&self.workspace, "<WORKSPACE>") |
| 362 | .path(&self.home, "<HOME>") |
| 363 | .path(self.root.path(), "<TMP>") |
| 364 | .literal(&today, "<today>") |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | pub(super) fn write_workspace(workspace: &Path, case: &Value) { |
| 369 | if let Some(files) = case.get("workspace_files").and_then(Value::as_object) { |
| 370 | for (relative, content) in files { |
| 371 | let path = workspace.join(relative); |
| 372 | if let Some(parent) = path.parent() { |
| 373 | std::fs::create_dir_all(parent).expect("create fixture dir"); |
| 374 | } |
| 375 | std::fs::write( |
| 376 | &path, |
| 377 | content.as_str().expect("workspace file content is text"), |
| 378 | ) |
| 379 | .expect("write fixture file"); |
| 380 | } |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | /// Collects per-case failures so one run reports every drifted case. |
| 385 | #[derive(Default)] |
| 386 | pub(super) struct Failures(Vec<String>); |
| 387 | |
| 388 | impl Failures { |
| 389 | pub(super) fn contains(&self, message: &str) -> bool { |
| 390 | self.0.iter().any(|failure| failure.contains(message)) |
| 391 | } |
| 392 | pub(super) fn record(&mut self, case: &str, result: Result<(), String>) { |
| 393 | if let Err(message) = result { |
| 394 | self.0.push(format!("[{case}] {message}")); |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | pub(super) fn push(&mut self, case: &str, message: impl Into<String>) { |
| 399 | self.0.push(format!("[{case}] {}", message.into())); |
| 400 | } |
| 401 | |
| 402 | pub(super) fn finish(self, family: &str, cases: usize) { |
| 403 | assert!(cases > 0, "conformance family `{family}` ran no cases"); |
| 404 | assert!( |
| 405 | self.0.is_empty(), |
| 406 | "conformance family `{family}`: {} failure(s) across {cases} case(s)\n\n{}", |
| 407 | self.0.len(), |
| 408 | self.0.join("\n\n") |
| 409 | ); |
| 410 | eprintln!("conformance family `{family}`: {cases} case(s) matched"); |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | #[test] |
| 415 | fn golden_comparison_rejects_changed_bytes_and_missing_output() { |
| 416 | let _lock = lock_test_env(); |
| 417 | let _update = EnvVarGuard::set(UPDATE_ENV, "0"); |
| 418 | let dir = tempfile::tempdir().expect("temporary golden"); |
| 419 | let path = dir.path().join("case.golden.txt"); |
| 420 | std::fs::write(&path, "expected\n").expect("write golden"); |
| 421 | assert!(check_golden(&path, "expected\n").is_ok()); |
| 422 | assert!( |
| 423 | check_golden(&path, "different\n") |
| 424 | .unwrap_err() |
| 425 | .contains("golden drift") |
| 426 | ); |
| 427 | assert!(check_golden(&path, "expected\r\n").is_err()); |
| 428 | assert!(check_golden(&dir.path().join("missing"), "").is_err()); |
| 429 | // A CRLF golden must not be silently normalized either. |
| 430 | std::fs::write(&path, "expected\r\n").expect("write CRLF golden"); |
| 431 | assert!(check_golden(&path, "expected\n").is_err()); |
| 432 | |
| 433 | // A large snapshot's common prefix must not hide its actual differing |
| 434 | // field; the comparison still rejects the entire changed document. |
| 435 | let prefix = "x".repeat(700); |
| 436 | let expected = jsonl(&[serde_json::json!({"prefix": prefix, "tail": "old"})]); |
| 437 | let actual = jsonl(&[serde_json::json!({"prefix": prefix, "tail": "new"})]); |
| 438 | std::fs::write(&path, expected).expect("write long JSON golden"); |
| 439 | let diagnostic = check_golden(&path, &actual).unwrap_err(); |
| 440 | assert!(diagnostic.contains("\"tail\":\"old\"")); |
| 441 | assert!(diagnostic.contains("\"tail\":\"new\"")); |
| 442 | } |
| 443 | |
| 444 | #[test] |
| 445 | #[should_panic(expected = "ran no cases")] |
| 446 | fn empty_family_cannot_pass() { |
| 447 | Failures::default().finish("empty_control", 0); |
| 448 | } |
| 449 | |
| 450 | #[test] |
| 451 | #[should_panic(expected = "is refused under CI")] |
| 452 | fn update_mode_is_refused_under_ci_even_for_matching_bytes() { |
| 453 | let _lock = lock_test_env(); |
| 454 | let _update = EnvVarGuard::set(UPDATE_ENV, "1"); |
| 455 | let _ci = EnvVarGuard::set("CI", "1"); |
| 456 | let dir = tempfile::tempdir().expect("temporary golden"); |
| 457 | let path = dir.path().join("matching.golden.txt"); |
| 458 | std::fs::write(&path, "matching\n").expect("write golden"); |
| 459 | let _ = check_golden(&path, "matching\n"); |
| 460 | } |
| 461 |