| 1 | //! Shared, host-bound test support for the FEAT-029 `debug::diagnostics` slice. |
| 2 | //! |
| 3 | //! This module lives at the `commands` root — outside `groups/debug`, which |
| 4 | //! FEAT-045 later moves into `codewhale-commands` — so the public-surface and |
| 5 | //! host-regression suites share one harness and one normalisation contract |
| 6 | //! instead of drifting copies. |
| 7 | //! |
| 8 | //! The frozen baseline fixtures under `fixtures/diagnostics/` were captured |
| 9 | //! from the untouched implementation at `origin/main` |
| 10 | //! `922679d6c0afe4556f3e5bc59073eb8e4cf93e07`. They are hand-reviewed source, |
| 11 | //! never regenerated from the migrated implementation. Only clock fields and |
| 12 | //! owned temporary paths are normalised. Context expectations account for the |
| 13 | //! host's platform/shell in the frozen environment block and its token subtotal. |
| 14 | |
| 15 | use std::path::{Path, PathBuf}; |
| 16 | use std::sync::OnceLock; |
| 17 | |
| 18 | use regex::Regex; |
| 19 | use tempfile::TempDir; |
| 20 | |
| 21 | use crate::config::ProviderKind; |
| 22 | use crate::tui::app::{App, TuiOptions}; |
| 23 | |
| 24 | /// The shared home seal, re-exported so the diagnostics suites keep one import |
| 25 | /// path. Take it first in any test whose output depends on the user's state |
| 26 | /// (global instructions, installed skills, persisted settings), before building |
| 27 | /// a [`DiagnosticsHarness`]: it holds the process-wide environment lock, which |
| 28 | /// the harness's settings loader re-enters. |
| 29 | pub(crate) use crate::test_support::SealedHome; |
| 30 | |
| 31 | /// Host-isolated fixture for the diagnostics command surface. |
| 32 | /// |
| 33 | /// The hermetic test-home wrapper isolates persisted settings. The owned |
| 34 | /// workspace and skills directories keep `/context` away from project data. |
| 35 | /// Do not hold either test-state mutex while constructing `App`: its settings |
| 36 | /// loader takes that mutex internally and the lock is not reentrant. |
| 37 | pub(crate) struct DiagnosticsHarness { |
| 38 | pub(crate) app: App, |
| 39 | pub(crate) workspace: PathBuf, |
| 40 | pub(crate) skills: PathBuf, |
| 41 | _temp: TempDir, |
| 42 | } |
| 43 | |
| 44 | impl DiagnosticsHarness { |
| 45 | pub(crate) fn new() -> Self { |
| 46 | let temp = TempDir::new().expect("diagnostics tempdir"); |
| 47 | |
| 48 | let workspace = temp.path().join("workspace"); |
| 49 | std::fs::create_dir_all(&workspace).expect("workspace dir"); |
| 50 | let skills = temp.path().join("skills"); |
| 51 | std::fs::create_dir_all(&skills).expect("skills dir"); |
| 52 | |
| 53 | let options = TuiOptions { |
| 54 | model: "deepseek-v4-pro".to_string(), |
| 55 | workspace: workspace.clone(), |
| 56 | skills_dir: skills.clone(), |
| 57 | memory_path: temp.path().join("memory.md"), |
| 58 | notes_path: temp.path().join("notes.txt"), |
| 59 | mcp_config_path: temp.path().join("mcp.json"), |
| 60 | use_memory: false, |
| 61 | ..crate::test_support::test_tui_options(temp.path()) |
| 62 | }; |
| 63 | let mut app = crate::test_support::test_app_with_options(options); |
| 64 | app.api_provider = ProviderKind::Deepseek; |
| 65 | |
| 66 | Self { |
| 67 | app, |
| 68 | workspace, |
| 69 | skills, |
| 70 | _temp: temp, |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | /// Apply the documented volatile-field normalisation for this harness. |
| 75 | pub(crate) fn normalize(&self, raw: &str) -> String { |
| 76 | normalize_generated_at(&normalize_harness_paths( |
| 77 | raw, |
| 78 | &self.workspace, |
| 79 | &self.skills, |
| 80 | &self._temp.path().join("memory.md"), |
| 81 | )) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | /// Read a frozen baseline fixture shipped with the slice. |
| 86 | pub(crate) fn fixture(name: &str) -> String { |
| 87 | let path = Path::new(env!("CARGO_MANIFEST_DIR")) |
| 88 | .join("src/commands/fixtures/diagnostics") |
| 89 | .join(name); |
| 90 | std::fs::read_to_string(&path).unwrap_or_else(|error| { |
| 91 | panic!( |
| 92 | "baseline fixture {} must be present and readable: {error}", |
| 93 | path.display() |
| 94 | ) |
| 95 | }) |
| 96 | } |
| 97 | |
| 98 | /// Assert the captured command output is byte-identical to its frozen baseline. |
| 99 | /// |
| 100 | /// Fixture files retain a stable `command: /...` provenance header captured by |
| 101 | /// the temporary baseline tool. The test identity and fixture name select that |
| 102 | /// command; the actual runtime value starts with `is_error`, so compare the |
| 103 | /// entire captured `CommandResult` body without manufacturing a header at run |
| 104 | /// time. |
| 105 | #[track_caller] |
| 106 | pub(crate) fn assert_fixture(name: &str, actual: &str) { |
| 107 | let expected = fixture(name); |
| 108 | let expected = if name.starts_with("context_") { |
| 109 | context_fixture_for_host( |
| 110 | &expected, |
| 111 | std::env::consts::OS, |
| 112 | crate::shell_dispatcher::global_dispatcher().kind().binary(), |
| 113 | ) |
| 114 | } else { |
| 115 | expected |
| 116 | }; |
| 117 | let (_, expected) = expected |
| 118 | .split_once('\n') |
| 119 | .unwrap_or_else(|| panic!("baseline fixture {name} is missing its provenance header")); |
| 120 | assert_eq!( |
| 121 | expected, actual, |
| 122 | "baseline parity drift in {name}; captured output changed" |
| 123 | ); |
| 124 | } |
| 125 | |
| 126 | /// The captured Linux environment contributes 21 tokens to the 2367 total. |
| 127 | /// Keep its text frozen independently of the production prompt renderer, but |
| 128 | /// substitute real host facts: Windows/pwsh.exe contributes 22, for example. |
| 129 | /// Never derive expectations from the report being tested or erase its counts. |
| 130 | fn context_fixture_for_host(expected: &str, platform: &str, shell: &str) -> String { |
| 131 | let environment = |
| 132 | format!("## Environment\n\n- lang: en\n- platform: {platform}\n- shell: {shell}"); |
| 133 | let tokens = environment.chars().count().div_ceil(3); |
| 134 | let total = 2367 - 21 + tokens; |
| 135 | expected |
| 136 | .replace( |
| 137 | "EnvironmentBlock: Runtime environment [<workspace>] - 21 tokens", |
| 138 | &format!("EnvironmentBlock: Runtime environment [<workspace>] - {tokens} tokens"), |
| 139 | ) |
| 140 | .replace( |
| 141 | "Runtime environment (21)", |
| 142 | &format!("Runtime environment ({tokens})"), |
| 143 | ) |
| 144 | .replace( |
| 145 | "\"estimated_tokens\": 21,", |
| 146 | &format!("\"estimated_tokens\": {tokens},"), |
| 147 | ) |
| 148 | .replace( |
| 149 | "Source-entry total: 2367 tokens", |
| 150 | &format!("Source-entry total: {total} tokens"), |
| 151 | ) |
| 152 | .replace( |
| 153 | "\"total_estimated_tokens\": 2367,", |
| 154 | &format!("\"total_estimated_tokens\": {total},"), |
| 155 | ) |
| 156 | } |
| 157 | |
| 158 | /// Replace the RFC-3339 `generated_at` report stamp so two captures of the same |
| 159 | /// source state compare byte-for-byte. |
| 160 | /// |
| 161 | /// `PromptSourceMap` / `PromptContext` stamp `Utc::now()` at construction; it is |
| 162 | /// the only wall-clock field on the diagnostics command surface. Production |
| 163 | /// timestamp semantics stay untouched — tests normalise the comparison, not the |
| 164 | /// adapter. |
| 165 | pub(crate) fn normalize_generated_at(raw: &str) -> String { |
| 166 | fn regex() -> &'static Regex { |
| 167 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 168 | RE.get_or_init(|| Regex::new(r#""generated_at":\s*"[^"]*""#).expect("generated_at regex")) |
| 169 | } |
| 170 | regex() |
| 171 | .replace_all(raw, "\"generated_at\": \"<timestamp>\"") |
| 172 | .into_owned() |
| 173 | } |
| 174 | |
| 175 | /// Replace the host-owned workspace/skills directory paths with stable |
| 176 | /// placeholders so the fixture does not embed a per-run temporary path. |
| 177 | pub(crate) fn normalize_harness_paths( |
| 178 | raw: &str, |
| 179 | workspace: &Path, |
| 180 | skills: &Path, |
| 181 | memory: &Path, |
| 182 | ) -> String { |
| 183 | fn replace_path(raw: &str, path: &str, placeholder: &str) -> String { |
| 184 | let json = serde_json::to_string(path).expect("serialize fixture path"); |
| 185 | raw.replace(&json[1..json.len() - 1], placeholder) |
| 186 | .replace(path, placeholder) |
| 187 | } |
| 188 | let workspace = workspace.display().to_string(); |
| 189 | let mut normalized = raw.to_string(); |
| 190 | // Path::join adds a native separator before this slash-containing suffix. |
| 191 | // Replace the whole owned path first, including JSON-escaped Windows paths. |
| 192 | for separator in ['/', '\\'] { |
| 193 | normalized = replace_path( |
| 194 | &normalized, |
| 195 | &format!("{workspace}{separator}.codewhale/handoff.md"), |
| 196 | "<workspace>/.codewhale/handoff.md", |
| 197 | ); |
| 198 | } |
| 199 | for (path, placeholder) in [ |
| 200 | (workspace, "<workspace>"), |
| 201 | (skills.display().to_string(), "<skills>"), |
| 202 | (memory.display().to_string(), "<memory>"), |
| 203 | ] { |
| 204 | normalized = replace_path(&normalized, &path, placeholder); |
| 205 | } |
| 206 | normalized |
| 207 | } |
| 208 | |
| 209 | /// Replace the trailing per-turn age cell in `/cache` history rows so a slow |
| 210 | /// runner cannot turn `0s` into `1s` and fail an otherwise identical capture. |
| 211 | /// |
| 212 | /// `humanize_age` renders `{secs}s` or `{mins}m {secs:02}s`; only that final |
| 213 | /// token on a telemetry row is volatile. |
| 214 | pub(crate) fn normalize_cache_ages(raw: &str) -> String { |
| 215 | fn regex() -> &'static Regex { |
| 216 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 217 | RE.get_or_init(|| Regex::new(r"(?m)(\d+m \d{2}s|\d+s)$").expect("cache-age regex")) |
| 218 | } |
| 219 | regex().replace_all(raw, "<age>").into_owned() |
| 220 | } |
| 221 | |
| 222 | /// The normalisers must replace *only* the documented volatile fields. A |
| 223 | /// normaliser that over-matched would turn the golden comparison into a |
| 224 | /// tautology, so pin its exact behaviour and prove an unrelated byte change |
| 225 | /// still fails equality. |
| 226 | #[test] |
| 227 | fn normalisers_replace_only_documented_volatile_fields() { |
| 228 | let raw = concat!( |
| 229 | "{\n", |
| 230 | " \"budget_used_percent\": 0.0048,\n", |
| 231 | " \"generated_at\": \"2026-09-23T19:02:15Z\",\n", |
| 232 | " \"note\": \"stable\"\n", |
| 233 | "}\n", |
| 234 | ); |
| 235 | let normalized = normalize_generated_at(raw); |
| 236 | assert!( |
| 237 | normalized.contains("\"generated_at\": \"<timestamp>\""), |
| 238 | "{normalized}" |
| 239 | ); |
| 240 | assert!(!normalized.contains("2026-09-23T19:02:15Z"), "{normalized}"); |
| 241 | // Every other byte survives exactly. |
| 242 | assert!( |
| 243 | normalized.contains("\"budget_used_percent\": 0.0048"), |
| 244 | "{normalized}" |
| 245 | ); |
| 246 | assert!(normalized.contains("\"note\": \"stable\""), "{normalized}"); |
| 247 | |
| 248 | // An unrelated byte change is not normalised away: it must still differ. |
| 249 | let mutated = raw.replace("\"stable\"", "\"drifted\""); |
| 250 | assert_ne!( |
| 251 | normalize_generated_at(raw), |
| 252 | normalize_generated_at(&mutated), |
| 253 | "a non-clock change must still fail the golden comparison" |
| 254 | ); |
| 255 | |
| 256 | // Path normalisation is exact-substring and cannot swallow unrelated text. |
| 257 | let paths = normalize_harness_paths( |
| 258 | "pack [/tmp/abc/workspace] skills [/tmp/abc/skills] memory [/tmp/abc/memory.md]", |
| 259 | Path::new("/tmp/abc/workspace"), |
| 260 | Path::new("/tmp/abc/skills"), |
| 261 | Path::new("/tmp/abc/memory.md"), |
| 262 | ); |
| 263 | assert_eq!( |
| 264 | paths, |
| 265 | "pack [<workspace>] skills [<skills>] memory [<memory>]" |
| 266 | ); |
| 267 | |
| 268 | // The age normaliser touches only the trailing age cell. |
| 269 | let rows = " 1 deepseek/x 4000 200 3000 1000 — — 75.0% — 0s\n\ |
| 270 | footer: sum_write: 0\n"; |
| 271 | let aged = normalize_cache_ages(rows); |
| 272 | assert!(aged.contains("75.0% — <age>\n"), "{aged}"); |
| 273 | assert!(aged.contains("footer: sum_write: 0"), "{aged}"); |
| 274 | } |
| 275 | |
| 276 | #[test] |
| 277 | fn context_expectations_preserve_counts_for_each_host() { |
| 278 | for name in [ |
| 279 | "context_report.txt", |
| 280 | "context_json.txt", |
| 281 | "context_summary.txt", |
| 282 | "context_prompt_json.txt", |
| 283 | ] { |
| 284 | let baseline = fixture(name); |
| 285 | assert_eq!( |
| 286 | context_fixture_for_host(&baseline, "linux", "/bin/bash"), |
| 287 | baseline |
| 288 | ); |
| 289 | // These literals reproduce the two changes seen in the Windows CI log. |
| 290 | let windows = baseline |
| 291 | .replace("- 21 tokens", "- 22 tokens") |
| 292 | .replace("environment (21)", "environment (22)") |
| 293 | .replace("\"estimated_tokens\": 21,", "\"estimated_tokens\": 22,") |
| 294 | .replace("2367", "2368"); |
| 295 | let expected = context_fixture_for_host(&baseline, "windows", "pwsh.exe"); |
| 296 | assert_eq!(expected, windows, "{name}"); |
| 297 | assert_ne!(expected, baseline, "{name} must retain the host difference"); |
| 298 | // A regression in another source or the environment count still fails. |
| 299 | assert_ne!(expected, windows.replace("1803", "1804")); |
| 300 | assert_ne!(expected, windows.replace("22", "23")); |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | #[test] |
| 305 | fn windows_paths_normalize_in_text_and_json_without_changing_other_data() { |
| 306 | let workspace = Path::new(r"C:\Users\runner\Temp\fixture\workspace"); |
| 307 | let skills = Path::new(r"C:\Users\runner\Temp\fixture\skills"); |
| 308 | let memory = Path::new(r"C:\Users\runner\Temp\fixture\memory.md"); |
| 309 | let handoff = format!("{}\\.codewhale/handoff.md", workspace.display()); |
| 310 | let raw = format!( |
| 311 | "handoff [{handoff}]\n{{\"handoff\":{},\"skills\":{},\"memory\":{},\"tokens\":22,\"other\":\"C:\\\\unrelated\"}}", |
| 312 | serde_json::to_string(&handoff).unwrap(), |
| 313 | serde_json::to_string(&skills.display().to_string()).unwrap(), |
| 314 | serde_json::to_string(&memory.display().to_string()).unwrap(), |
| 315 | ); |
| 316 | assert_eq!( |
| 317 | normalize_harness_paths(&raw, workspace, skills, memory), |
| 318 | "handoff [<workspace>/.codewhale/handoff.md]\n{\"handoff\":\"<workspace>/.codewhale/handoff.md\",\"skills\":\"<skills>\",\"memory\":\"<memory>\",\"tokens\":22,\"other\":\"C:\\\\unrelated\"}" |
| 319 | ); |
| 320 | } |
| 321 |