| 1 | //! Integration tests for the offline evaluation harness. |
| 2 | |
| 3 | use std::fs; |
| 4 | |
| 5 | use tempfile::tempdir; |
| 6 | |
| 7 | #[path = "../src/eval.rs"] |
| 8 | mod eval; |
| 9 | #[path = "../src/shell_dispatcher.rs"] |
| 10 | mod shell_dispatcher; |
| 11 | |
| 12 | use eval::{EvalHarness, EvalHarnessConfig, FixtureRecord, ScenarioStepKind}; |
| 13 | |
| 14 | const HAPPY_PATH_TOOL_LOOP: [ScenarioStepKind; 6] = [ |
| 15 | ScenarioStepKind::List, |
| 16 | ScenarioStepKind::Read, |
| 17 | ScenarioStepKind::Search, |
| 18 | ScenarioStepKind::Edit, |
| 19 | ScenarioStepKind::ApplyPatch, |
| 20 | ScenarioStepKind::Bash, |
| 21 | ]; |
| 22 | |
| 23 | #[test] |
| 24 | fn runs_offline_tool_loop_successfully() { |
| 25 | let harness = EvalHarness::default(); |
| 26 | let run = harness.run().expect("eval harness run should succeed"); |
| 27 | assert_eq!( |
| 28 | ScenarioStepKind::parse("patch"), |
| 29 | Some(ScenarioStepKind::ApplyPatch) |
| 30 | ); |
| 31 | |
| 32 | assert!(run.metrics.success, "expected success metrics: {run:#?}"); |
| 33 | assert_eq!(run.metrics.tool_errors, 0); |
| 34 | assert_eq!(run.metrics.steps, 6); |
| 35 | assert!(run.metrics.duration.as_millis() > 0); |
| 36 | assert!(!run.scenario_name.is_empty()); |
| 37 | assert!(run.workspace_summary.file_count >= 3); |
| 38 | |
| 39 | for kind in HAPPY_PATH_TOOL_LOOP { |
| 40 | let stats = run |
| 41 | .metrics |
| 42 | .per_tool |
| 43 | .get(&kind) |
| 44 | .expect("missing per-tool stats"); |
| 45 | assert_eq!(stats.invocations, 1, "unexpected invocations for {kind:?}"); |
| 46 | assert_eq!(stats.errors, 0, "unexpected errors for {kind:?}"); |
| 47 | assert!(stats.total_duration.as_nanos() > 0); |
| 48 | } |
| 49 | |
| 50 | let notes_path = run.workspace_root().join("notes.txt"); |
| 51 | let notes = fs::read_to_string(¬es_path).expect("notes.txt should exist"); |
| 52 | assert!(notes.contains("edited = true")); |
| 53 | assert!(notes.contains("todo: offline metrics (patched)")); |
| 54 | |
| 55 | let report = run.to_report(); |
| 56 | assert_eq!(report.metrics.success, run.metrics.success); |
| 57 | } |
| 58 | |
| 59 | #[test] |
| 60 | fn acceptance_happy_path_records_simulated_llm_tool_plan() { |
| 61 | let record_dir = tempdir().expect("tempdir"); |
| 62 | let scenario_name = "issue-2791-happy-path-tool-loop"; |
| 63 | let config = EvalHarnessConfig { |
| 64 | scenario_name: scenario_name.to_string(), |
| 65 | record_dir: Some(record_dir.path().to_path_buf()), |
| 66 | ..EvalHarnessConfig::default() |
| 67 | }; |
| 68 | let harness = EvalHarness::new(config); |
| 69 | |
| 70 | let run = harness.run().expect("happy-path acceptance run"); |
| 71 | |
| 72 | assert!(run.metrics.success, "expected success metrics: {run:#?}"); |
| 73 | assert_eq!(run.metrics.tool_errors, 0); |
| 74 | assert_eq!(run.metrics.steps, HAPPY_PATH_TOOL_LOOP.len()); |
| 75 | |
| 76 | let actual_tool_names: Vec<&str> = run.steps.iter().map(|step| step.tool_name).collect(); |
| 77 | let expected_tool_names: Vec<&str> = HAPPY_PATH_TOOL_LOOP |
| 78 | .iter() |
| 79 | .map(|kind| kind.tool_name()) |
| 80 | .collect(); |
| 81 | assert_eq!(actual_tool_names, expected_tool_names); |
| 82 | |
| 83 | let scenario_file = record_dir.path().join(format!("{scenario_name}.jsonl")); |
| 84 | let records = read_fixture_records(&scenario_file); |
| 85 | assert_eq!(records.len(), HAPPY_PATH_TOOL_LOOP.len()); |
| 86 | |
| 87 | for (record, kind) in records.iter().zip(HAPPY_PATH_TOOL_LOOP) { |
| 88 | assert_eq!( |
| 89 | record.request.get("tool").and_then(|value| value.as_str()), |
| 90 | Some(kind.tool_name()) |
| 91 | ); |
| 92 | assert_eq!( |
| 93 | record |
| 94 | .request |
| 95 | .get("action") |
| 96 | .and_then(|value| value.as_str()), |
| 97 | kind.action() |
| 98 | ); |
| 99 | |
| 100 | let expected_kind = format!("{kind:?}"); |
| 101 | assert_eq!( |
| 102 | record.request.get("kind").and_then(|value| value.as_str()), |
| 103 | Some(expected_kind.as_str()) |
| 104 | ); |
| 105 | |
| 106 | let event = record |
| 107 | .response_events |
| 108 | .first() |
| 109 | .expect("simulated LLM fixture should include a response event"); |
| 110 | assert_eq!( |
| 111 | event.get("type").and_then(|value| value.as_str()), |
| 112 | Some("ok") |
| 113 | ); |
| 114 | assert!( |
| 115 | event |
| 116 | .get("output") |
| 117 | .and_then(|value| value.as_str()) |
| 118 | .is_some_and(|output| !output.is_empty()), |
| 119 | "fixture event should include non-empty tool output" |
| 120 | ); |
| 121 | } |
| 122 | |
| 123 | let notes_path = run.workspace_root().join("notes.txt"); |
| 124 | let notes = fs::read_to_string(¬es_path).expect("notes.txt should exist"); |
| 125 | assert!(notes.contains("edited = true")); |
| 126 | assert!(notes.contains("todo: offline metrics (patched)")); |
| 127 | } |
| 128 | |
| 129 | fn read_fixture_records(path: &std::path::Path) -> Vec<FixtureRecord> { |
| 130 | fs::read_to_string(path) |
| 131 | .expect("read fixture records") |
| 132 | .lines() |
| 133 | .filter(|line| !line.trim().is_empty()) |
| 134 | .map(|line| serde_json::from_str(line).expect("fixture line should parse")) |
| 135 | .collect() |
| 136 | } |
| 137 | |
| 138 | #[test] |
| 139 | fn records_tool_errors_when_step_fails() { |
| 140 | let config = EvalHarnessConfig { |
| 141 | fail_step: Some(ScenarioStepKind::ApplyPatch), |
| 142 | ..EvalHarnessConfig::default() |
| 143 | }; |
| 144 | let harness = EvalHarness::new(config); |
| 145 | |
| 146 | let run = harness |
| 147 | .run() |
| 148 | .expect("eval harness should return metrics even when a step fails"); |
| 149 | |
| 150 | assert!(!run.metrics.success); |
| 151 | assert!(run.metrics.tool_errors >= 1); |
| 152 | |
| 153 | let patch_stats = run |
| 154 | .metrics |
| 155 | .per_tool |
| 156 | .get(&ScenarioStepKind::ApplyPatch) |
| 157 | .expect("missing apply_patch stats"); |
| 158 | assert_eq!(patch_stats.invocations, 1); |
| 159 | assert_eq!(patch_stats.errors, 1); |
| 160 | |
| 161 | let patch_step = run |
| 162 | .steps |
| 163 | .iter() |
| 164 | .find(|step| step.kind == ScenarioStepKind::ApplyPatch) |
| 165 | .expect("missing apply_patch step"); |
| 166 | assert!(!patch_step.success); |
| 167 | assert!(patch_step.error.as_deref().is_some_and(|e| !e.is_empty())); |
| 168 | } |
| 169 | |
| 170 | #[test] |
| 171 | fn validation_can_fail_without_tool_errors() { |
| 172 | let config = EvalHarnessConfig { |
| 173 | shell_expect_token: "definitely-not-in-output".to_string(), |
| 174 | ..EvalHarnessConfig::default() |
| 175 | }; |
| 176 | let harness = EvalHarness::new(config); |
| 177 | |
| 178 | let run = harness.run().expect("eval harness run should complete"); |
| 179 | |
| 180 | assert_eq!(run.metrics.tool_errors, 0); |
| 181 | assert!( |
| 182 | !run.metrics.success, |
| 183 | "validation should fail due to shell token" |
| 184 | ); |
| 185 | } |
| 186 | |
| 187 | #[test] |
| 188 | fn record_flag_writes_one_jsonl_line_per_step() { |
| 189 | let dir = tempdir().expect("tempdir"); |
| 190 | let config = EvalHarnessConfig { |
| 191 | record_dir: Some(dir.path().to_path_buf()), |
| 192 | ..EvalHarnessConfig::default() |
| 193 | }; |
| 194 | let harness = EvalHarness::new(config); |
| 195 | let run = harness.run().expect("eval harness run should succeed"); |
| 196 | |
| 197 | let scenario_file = dir.path().join("offline-tool-loop.jsonl"); |
| 198 | assert!( |
| 199 | scenario_file.exists(), |
| 200 | "record_dir should contain {}", |
| 201 | scenario_file |
| 202 | .file_name() |
| 203 | .map(|n| n.to_string_lossy().into_owned()) |
| 204 | .unwrap_or_default(), |
| 205 | ); |
| 206 | |
| 207 | let contents = fs::read_to_string(&scenario_file).expect("read jsonl"); |
| 208 | let lines: Vec<&str> = contents.lines().filter(|l| !l.trim().is_empty()).collect(); |
| 209 | assert_eq!( |
| 210 | lines.len(), |
| 211 | run.metrics.steps, |
| 212 | "one JSONL line per step expected" |
| 213 | ); |
| 214 | |
| 215 | // Each line is a self-contained JSON object with the documented schema. |
| 216 | for line in lines { |
| 217 | let parsed: serde_json::Value = |
| 218 | serde_json::from_str(line).expect("each fixture line is valid JSON"); |
| 219 | assert!(parsed.get("request").is_some(), "missing request"); |
| 220 | let events = parsed |
| 221 | .get("response_events") |
| 222 | .and_then(|v| v.as_array()) |
| 223 | .expect("response_events must be an array"); |
| 224 | assert!(!events.is_empty(), "every fixture must have ≥1 event"); |
| 225 | } |
| 226 | } |
| 227 |