| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "testing" |
| 8 | |
| 9 | "reasonix/internal/event" |
| 10 | "reasonix/internal/evidence" |
| 11 | "reasonix/internal/provider" |
| 12 | ) |
| 13 | |
| 14 | func TestMetricsSinkUsesProviderCacheWriteCost(t *testing.T) { |
| 15 | s := &metricsSink{inner: event.Discard} |
| 16 | s.Emit(event.Event{ |
| 17 | Kind: event.Usage, |
| 18 | Usage: &provider.Usage{ |
| 19 | CacheMissTokens: 500_000, |
| 20 | CacheWriteTokens: 100_000, |
| 21 | CacheWriteBilledTokens: 200_000, |
| 22 | }, |
| 23 | Pricing: &provider.Pricing{Input: 2}, |
| 24 | }) |
| 25 | if got := s.Snapshot().Cost; got != 1.2 { |
| 26 | t.Fatalf("metrics cost = %f, want 1.2", got) |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | func TestMetricsSinkAccumulatesReadinessAudit(t *testing.T) { |
| 31 | s := &metricsSink{inner: event.Discard} |
| 32 | |
| 33 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 34 | Result: evidence.ReadinessBlocked, |
| 35 | MissingProjectChecks: 2, |
| 36 | IncompleteTodos: 3, |
| 37 | CommandMismatchMissing: 2, |
| 38 | MissingAcceptanceCriteria: 1, |
| 39 | MissingVerification: 1, |
| 40 | MissingReview: 1, |
| 41 | MissingSignoff: 1, |
| 42 | MissingActionEvidence: 1, |
| 43 | MissingMutation: 1, |
| 44 | }) |
| 45 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 46 | Result: evidence.ReadinessAllowed, |
| 47 | Recovered: true, |
| 48 | }) |
| 49 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 50 | Result: evidence.ReadinessErrored, |
| 51 | }) |
| 52 | |
| 53 | if s.m.ReadinessChecks != 3 { |
| 54 | t.Fatalf("readiness checks = %d, want 3", s.m.ReadinessChecks) |
| 55 | } |
| 56 | if s.m.ReadinessAllowed != 1 { |
| 57 | t.Fatalf("readiness allowed = %d, want 1", s.m.ReadinessAllowed) |
| 58 | } |
| 59 | if s.m.ReadinessBlocks != 1 { |
| 60 | t.Fatalf("readiness blocks = %d, want 1", s.m.ReadinessBlocks) |
| 61 | } |
| 62 | if s.m.ReadinessRecoveries != 1 { |
| 63 | t.Fatalf("readiness recoveries = %d, want 1", s.m.ReadinessRecoveries) |
| 64 | } |
| 65 | if s.m.ReadinessErrors != 1 { |
| 66 | t.Fatalf("readiness errors = %d, want 1", s.m.ReadinessErrors) |
| 67 | } |
| 68 | if s.m.ReadinessMissingProjectChecks != 2 { |
| 69 | t.Fatalf("missing project checks = %d, want 2", s.m.ReadinessMissingProjectChecks) |
| 70 | } |
| 71 | if s.m.ReadinessIncompleteTodos != 3 { |
| 72 | t.Fatalf("incomplete todos = %d, want 3", s.m.ReadinessIncompleteTodos) |
| 73 | } |
| 74 | if s.m.ReadinessCommandMismatches != 2 { |
| 75 | t.Fatalf("command mismatches = %d, want 2", s.m.ReadinessCommandMismatches) |
| 76 | } |
| 77 | if s.m.ReadinessMissingAcceptance != 1 || s.m.ReadinessMissingVerification != 1 || s.m.ReadinessMissingReview != 1 || s.m.ReadinessMissingSignoff != 1 { |
| 78 | t.Fatalf("delivery readiness misses = acceptance %d verification %d review %d signoff %d, want 1/1/1/1", |
| 79 | s.m.ReadinessMissingAcceptance, s.m.ReadinessMissingVerification, s.m.ReadinessMissingReview, s.m.ReadinessMissingSignoff) |
| 80 | } |
| 81 | if s.m.ReadinessMissingActionEvidence != 1 || s.m.ReadinessMissingMutation != 1 { |
| 82 | t.Fatalf("delivery work misses = action evidence %d mutation %d, want 1/1", s.m.ReadinessMissingActionEvidence, s.m.ReadinessMissingMutation) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func TestMetricsSinkAccumulatesSilentReasoningRecovery(t *testing.T) { |
| 87 | s := &metricsSink{inner: event.Discard} |
| 88 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningDetected}) |
| 89 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryAttempted}) |
| 90 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryRecovered}) |
| 91 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryReplaced}) |
| 92 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetrySuppressed}) |
| 93 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningFallback}) |
| 94 | |
| 95 | if s.m.MissingReasoningDetected != 1 || s.m.MissingReasoningRetries != 1 || s.m.MissingReasoningRecovered != 1 || s.m.MissingReasoningReplaced != 1 || s.m.MissingReasoningSuppressed != 1 || s.m.MissingReasoningFallbacks != 1 { |
| 96 | t.Fatalf("reasoning recovery metrics = %+v", s.m) |
| 97 | } |
| 98 | if s.m.Steps != 1 { |
| 99 | t.Fatalf("retry model-call step = %d, want 1", s.m.Steps) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | func TestMetricsSinkAccountsToolCallsAndRetries(t *testing.T) { |
| 104 | s := &metricsSink{inner: event.Discard} |
| 105 | |
| 106 | s.Emit(event.Event{Kind: event.ToolDispatch, Tool: event.Tool{Name: "read_file"}}) |
| 107 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "read_file", DurationMs: 12}}) |
| 108 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "read_file", DurationMs: 8}}) |
| 109 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "bash", Err: "exit status 1", DurationMs: 30}}) |
| 110 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "grep", ParentID: "task-1", DurationMs: 5}}) |
| 111 | s.Emit(event.Event{Kind: event.Retrying, RetryAttempt: 1, RetryMax: 3}) |
| 112 | |
| 113 | if s.m.ToolCalls != 4 { |
| 114 | t.Fatalf("tool calls = %d, want 4 (dispatch must not count)", s.m.ToolCalls) |
| 115 | } |
| 116 | if s.m.ToolFailures != 1 { |
| 117 | t.Fatalf("tool failures = %d, want 1", s.m.ToolFailures) |
| 118 | } |
| 119 | if s.m.ToolDurationMs != 55 { |
| 120 | t.Fatalf("tool duration = %dms, want 55", s.m.ToolDurationMs) |
| 121 | } |
| 122 | if s.m.SubagentToolCalls != 1 { |
| 123 | t.Fatalf("subagent tool calls = %d, want 1", s.m.SubagentToolCalls) |
| 124 | } |
| 125 | if s.m.Retries != 1 { |
| 126 | t.Fatalf("retries = %d, want 1", s.m.Retries) |
| 127 | } |
| 128 | if s.m.ToolCallsByName["read_file"] != 2 || s.m.ToolCallsByName["bash"] != 1 { |
| 129 | t.Fatalf("calls by name = %v", s.m.ToolCallsByName) |
| 130 | } |
| 131 | if s.m.ToolFailuresByName["bash"] != 1 { |
| 132 | t.Fatalf("failures by name = %v, want bash 1", s.m.ToolFailuresByName) |
| 133 | } |
| 134 | if _, ok := s.m.ToolFailuresByName["read_file"]; ok { |
| 135 | t.Fatalf("a successful tool must not appear in failures: %v", s.m.ToolFailuresByName) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | func TestWriteMetricsIncludesReadinessFields(t *testing.T) { |
| 140 | path := filepath.Join(t.TempDir(), "metrics.json") |
| 141 | if err := writeMetrics(path, RunMetrics{ |
| 142 | PromptTokens: 10, |
| 143 | CompletionTokens: 3, |
| 144 | CacheHitTokens: 7, |
| 145 | CacheMissTokens: 3, |
| 146 | Steps: 2, |
| 147 | ReadinessChecks: 1, |
| 148 | ReadinessAllowed: 1, |
| 149 | ReadinessBlocks: 0, |
| 150 | ReadinessRecoveries: 1, |
| 151 | ReadinessErrors: 0, |
| 152 | ReadinessMissingProjectChecks: 0, |
| 153 | ReadinessIncompleteTodos: 0, |
| 154 | ReadinessCommandMismatches: 0, |
| 155 | ReadinessMissingAcceptance: 0, |
| 156 | ReadinessMissingVerification: 0, |
| 157 | ReadinessMissingReview: 0, |
| 158 | ReadinessMissingSignoff: 0, |
| 159 | ReadinessMissingActionEvidence: 0, |
| 160 | ReadinessMissingMutation: 0, |
| 161 | }); err != nil { |
| 162 | t.Fatalf("writeMetrics: %v", err) |
| 163 | } |
| 164 | |
| 165 | b, err := os.ReadFile(path) |
| 166 | if err != nil { |
| 167 | t.Fatalf("ReadFile: %v", err) |
| 168 | } |
| 169 | var got map[string]any |
| 170 | if err := json.Unmarshal(b, &got); err != nil { |
| 171 | t.Fatalf("Unmarshal: %v", err) |
| 172 | } |
| 173 | for _, key := range []string{ |
| 174 | "readiness_checks", |
| 175 | "readiness_allowed", |
| 176 | "readiness_blocks", |
| 177 | "readiness_recoveries", |
| 178 | "readiness_errors", |
| 179 | "readiness_missing_project_checks", |
| 180 | "readiness_incomplete_todos", |
| 181 | "readiness_command_mismatches", |
| 182 | "readiness_missing_acceptance_criteria", |
| 183 | "readiness_missing_verification", |
| 184 | "readiness_missing_review", |
| 185 | "readiness_missing_signoff", |
| 186 | "readiness_missing_action_evidence", |
| 187 | "readiness_missing_mutation", |
| 188 | } { |
| 189 | if _, ok := got[key]; !ok { |
| 190 | t.Fatalf("metrics JSON missing %q: %s", key, string(b)) |
| 191 | } |
| 192 | } |
| 193 | } |
| 194 |