| 1 | //! Gherkin binary health and eval harness smoke test for command extraction. |
| 2 | //! |
| 3 | //! This runs the binary through `codewhale-tui eval` and verifies that the |
| 4 | //! executable still loads and reports a successful JSON evaluation after the |
| 5 | //! core/session command modules are extracted. |
| 6 | |
| 7 | use std::path::PathBuf; |
| 8 | use std::process::Command; |
| 9 | |
| 10 | use cucumber::{World as _, given, then, when, writer::Stats as _}; |
| 11 | use serde_json::Value; |
| 12 | use tempfile::TempDir; |
| 13 | |
| 14 | const FEATURE_NAME: &str = "Core and session command extraction"; |
| 15 | const FEATURE_PATH: &str = concat!( |
| 16 | env!("CARGO_MANIFEST_DIR"), |
| 17 | "/tests/features/core_session_command_extraction.feature" |
| 18 | ); |
| 19 | const CORE_SCENARIO: &str = "The binary loads and runs the evaluation harness after extraction"; |
| 20 | |
| 21 | #[derive(Debug, Default, cucumber::World)] |
| 22 | struct CoreSessionExtractionWorld { |
| 23 | record_dir: Option<TempDir>, |
| 24 | report: Option<Value>, |
| 25 | } |
| 26 | |
| 27 | #[given("a clean CodeWhale evaluation workspace")] |
| 28 | fn clean_codewhale_evaluation_workspace(world: &mut CoreSessionExtractionWorld) { |
| 29 | world.record_dir = Some(TempDir::new().expect("evaluation TempDir")); |
| 30 | } |
| 31 | |
| 32 | #[when("the evaluation harness runs a shell command")] |
| 33 | fn eval_harness_runs_shell_command(world: &mut CoreSessionExtractionWorld) { |
| 34 | let record_dir = world |
| 35 | .record_dir |
| 36 | .as_ref() |
| 37 | .expect("evaluation workspace should exist"); |
| 38 | |
| 39 | let output = Command::new(codewhale_tui_binary()) |
| 40 | .args([ |
| 41 | "eval", |
| 42 | "--json", |
| 43 | "--shell-command", |
| 44 | "echo eval-harness", |
| 45 | "--record", |
| 46 | ]) |
| 47 | .arg(record_dir.path()) |
| 48 | .output() |
| 49 | .expect("codewhale-tui eval should start"); |
| 50 | |
| 51 | assert!( |
| 52 | output.status.success(), |
| 53 | "codewhale-tui eval failed\nstderr:\n{}", |
| 54 | String::from_utf8_lossy(&output.stderr) |
| 55 | ); |
| 56 | |
| 57 | let report: Value = serde_json::from_slice(&output.stdout).unwrap_or_else(|err| { |
| 58 | panic!( |
| 59 | "eval --json should emit valid JSON: {err}\nstdout:\n{}", |
| 60 | String::from_utf8_lossy(&output.stdout) |
| 61 | ) |
| 62 | }); |
| 63 | |
| 64 | world.report = Some(report); |
| 65 | } |
| 66 | |
| 67 | #[then("the harness completes successfully")] |
| 68 | fn harness_completes_successfully(world: &mut CoreSessionExtractionWorld) { |
| 69 | let report = world.report.as_ref().expect("eval report should exist"); |
| 70 | |
| 71 | let success = report |
| 72 | .get("metrics") |
| 73 | .and_then(|metrics| metrics.get("success")) |
| 74 | .and_then(|value| value.as_bool()) |
| 75 | .unwrap_or(false); |
| 76 | assert!( |
| 77 | success, |
| 78 | "eval report 'metrics.success' should be true, got: {report:?}" |
| 79 | ); |
| 80 | } |
| 81 | |
| 82 | #[then("the JSON report contains a step with the expected kind")] |
| 83 | fn json_report_contains_step_with_expected_kind(world: &mut CoreSessionExtractionWorld) { |
| 84 | let report = world.report.as_ref().expect("eval report should exist"); |
| 85 | |
| 86 | let steps = report |
| 87 | .get("steps") |
| 88 | .and_then(|value| value.as_array()) |
| 89 | .expect("eval report should have a 'steps' array"); |
| 90 | |
| 91 | assert!( |
| 92 | !steps.is_empty(), |
| 93 | "eval report should have at least one step" |
| 94 | ); |
| 95 | |
| 96 | let first_step = &steps[0]; |
| 97 | let kind = first_step |
| 98 | .get("kind") |
| 99 | .and_then(|value| value.as_str()) |
| 100 | .expect("step should have a 'kind' field"); |
| 101 | |
| 102 | assert_eq!( |
| 103 | kind, "List", |
| 104 | "first step kind should be 'List', got: {kind}" |
| 105 | ); |
| 106 | |
| 107 | let step_success = first_step |
| 108 | .get("success") |
| 109 | .and_then(|value| value.as_bool()) |
| 110 | .unwrap_or(false); |
| 111 | assert!( |
| 112 | step_success, |
| 113 | "first step 'success' should be true, got: {first_step:?}" |
| 114 | ); |
| 115 | |
| 116 | let output = first_step |
| 117 | .get("output") |
| 118 | .and_then(|value| value.as_str()) |
| 119 | .unwrap_or(""); |
| 120 | assert!( |
| 121 | !output.is_empty(), |
| 122 | "step output should not be empty: {first_step:?}" |
| 123 | ); |
| 124 | } |
| 125 | |
| 126 | #[tokio::test(flavor = "current_thread")] |
| 127 | async fn codewhale_eval_runs_after_extraction() { |
| 128 | let writer = CoreSessionExtractionWorld::cucumber() |
| 129 | .fail_on_skipped() |
| 130 | .with_default_cli() |
| 131 | .filter_run(FEATURE_PATH, move |feature, _, scenario| { |
| 132 | feature.name == FEATURE_NAME && scenario.name == CORE_SCENARIO |
| 133 | }) |
| 134 | .await; |
| 135 | assert_eq!(writer.failed_steps(), 0, "scenario failed: {CORE_SCENARIO}"); |
| 136 | assert_eq!( |
| 137 | writer.skipped_steps(), |
| 138 | 0, |
| 139 | "scenario skipped steps: {CORE_SCENARIO}" |
| 140 | ); |
| 141 | assert_eq!( |
| 142 | writer.passed_steps(), |
| 143 | 4, |
| 144 | "scenario did not run: {CORE_SCENARIO}" |
| 145 | ); |
| 146 | } |
| 147 | |
| 148 | fn codewhale_tui_binary() -> PathBuf { |
| 149 | if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale-tui") { |
| 150 | return PathBuf::from(path); |
| 151 | } |
| 152 | if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale-tui") { |
| 153 | return PathBuf::from(path); |
| 154 | } |
| 155 | |
| 156 | let mut path = std::env::current_exe().expect("current test executable path"); |
| 157 | path.pop(); |
| 158 | if path.ends_with("deps") { |
| 159 | path.pop(); |
| 160 | } |
| 161 | path.push(format!("codewhale-tui{}", std::env::consts::EXE_SUFFIX)); |
| 162 | path |
| 163 | } |
| 164 |