| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "reasonix/internal/agent" |
| 13 | "reasonix/internal/event" |
| 14 | "reasonix/internal/provider" |
| 15 | ) |
| 16 | |
| 17 | func TestRunOutputTextPrintsOnlyFinalMessage(t *testing.T) { |
| 18 | var out bytes.Buffer |
| 19 | sink := newRunOutputSink(&out, runOutputText) |
| 20 | sink.Emit(event.Event{Kind: event.Text, Text: "streamed "}) |
| 21 | sink.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "bash", Output: "noise"}}) |
| 22 | sink.Emit(event.Event{Kind: event.Message, Text: "final answer"}) |
| 23 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 24 | t.Fatal(err) |
| 25 | } |
| 26 | if got := out.String(); got != "final answer\n" { |
| 27 | t.Fatalf("text output = %q", got) |
| 28 | } |
| 29 | } |
| 30 | |
| 31 | func TestRunOutputJSONResult(t *testing.T) { |
| 32 | var out bytes.Buffer |
| 33 | sink := newRunOutputSink(&out, runOutputJSON) |
| 34 | sink.Emit(event.Event{Kind: event.Message, Text: "done"}) |
| 35 | sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{ |
| 36 | PromptTokens: 12, CompletionTokens: 3, CacheHitTokens: 8, CacheMissTokens: 4, Estimated: true, |
| 37 | }}) |
| 38 | sink.Emit(event.Event{Kind: event.TurnDone}) |
| 39 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | var result runResult |
| 43 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 44 | t.Fatalf("decode result: %v\n%s", err, out.String()) |
| 45 | } |
| 46 | if result.Type != "result" || result.Subtype != "success" || result.IsError || result.Result != "done" || result.SessionID != "abc" { |
| 47 | t.Fatalf("result = %+v", result) |
| 48 | } |
| 49 | if result.Usage.InputTokens != 12 || result.Usage.OutputTokens != 3 || result.Usage.CacheReadInputTokens != 8 || result.Usage.CacheCreationInputTokens != 4 { |
| 50 | t.Fatalf("usage = %+v", result.Usage) |
| 51 | } |
| 52 | if !result.Usage.Estimated { |
| 53 | t.Fatalf("usage lost estimated marker: %+v", result.Usage) |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | func TestRunOutputJSONIncludesCurrencyAwareCostFields(t *testing.T) { |
| 58 | for _, tt := range []struct { |
| 59 | name string |
| 60 | currency string |
| 61 | wantCode string |
| 62 | }{ |
| 63 | {name: "USD", currency: "$", wantCode: "USD"}, |
| 64 | {name: "CNY", currency: "¥", wantCode: "CNY"}, |
| 65 | } { |
| 66 | t.Run(tt.name, func(t *testing.T) { |
| 67 | var out bytes.Buffer |
| 68 | sink := newRunOutputSink(&out, runOutputJSON) |
| 69 | sink.Emit(event.Event{ |
| 70 | Kind: event.Usage, |
| 71 | Usage: &provider.Usage{PromptTokens: 1_000_000, CompletionTokens: 500_000}, |
| 72 | Pricing: &provider.Pricing{Input: 1, Output: 2, Currency: tt.currency}, |
| 73 | }) |
| 74 | if err := sink.Finalize("abc", time.Now(), nil); err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | var result runResult |
| 78 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | if result.TotalCost != 2 || result.TotalCostUSD != result.TotalCost || result.Currency != tt.wantCode { |
| 82 | t.Fatalf("currency-aware result = %+v", result) |
| 83 | } |
| 84 | }) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func TestRunOutputJSONRejectsMixedPricingCurrencies(t *testing.T) { |
| 89 | var out bytes.Buffer |
| 90 | sink := newRunOutputSink(&out, runOutputJSON) |
| 91 | for _, currency := range []string{"$", "¥"} { |
| 92 | sink.Emit(event.Event{ |
| 93 | Kind: event.Usage, |
| 94 | Usage: &provider.Usage{PromptTokens: 1_000_000}, |
| 95 | Pricing: &provider.Pricing{Input: 1, Currency: currency}, |
| 96 | }) |
| 97 | } |
| 98 | err := sink.Finalize("abc", time.Now(), nil) |
| 99 | if err == nil || !strings.Contains(err.Error(), "mixed pricing currencies") { |
| 100 | t.Fatalf("Finalize mixed currencies error = %v", err) |
| 101 | } |
| 102 | if out.Len() != 0 { |
| 103 | t.Fatalf("mixed-currency JSON should not emit a misleading total: %s", out.String()) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | func TestRunOutputSessionIDPreservesExistingFormats(t *testing.T) { |
| 108 | const raw = "20260723-120000.000000000-model" |
| 109 | identityKey := bytes.Repeat([]byte{0x41}, machineIdentityKeyBytes) |
| 110 | for _, format := range []runOutputFormat{runOutputText, runOutputJSON, runOutputStreamJSON} { |
| 111 | if got := runOutputSessionID(format, raw, nil); got != raw { |
| 112 | t.Fatalf("format %q session id = %q, want raw id %q", format, got, raw) |
| 113 | } |
| 114 | } |
| 115 | if got := runOutputSessionID(runOutputEventsJSONL, raw, identityKey); got != machineSessionIDWithKey(raw, identityKey) { |
| 116 | t.Fatalf("events-jsonl session id = %q, want machine id %q", got, machineSessionIDWithKey(raw, identityKey)) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func TestRunOutputStreamJSONEndsWithErrorResult(t *testing.T) { |
| 121 | var out bytes.Buffer |
| 122 | sink := newRunOutputSink(&out, runOutputStreamJSON) |
| 123 | sink.Emit(event.Event{Kind: event.Text, Text: "partial"}) |
| 124 | runErr := errors.New("provider failed") |
| 125 | if err := sink.Finalize("abc", time.Now(), runErr); err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | lines := strings.Split(strings.TrimSpace(out.String()), "\n") |
| 129 | if len(lines) != 2 { |
| 130 | t.Fatalf("stream lines = %d, want 2\n%s", len(lines), out.String()) |
| 131 | } |
| 132 | var wire map[string]any |
| 133 | if err := json.Unmarshal([]byte(lines[0]), &wire); err != nil || wire["kind"] != "text" { |
| 134 | t.Fatalf("wire event = %#v, err=%v", wire, err) |
| 135 | } |
| 136 | var result runResult |
| 137 | if err := json.Unmarshal([]byte(lines[1]), &result); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | if !result.IsError || result.Subtype != "error_during_execution" || result.Result != runErr.Error() { |
| 141 | t.Fatalf("error result = %+v", result) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | func TestRunOutputEventsJSONLIsStructuredAndRedacted(t *testing.T) { |
| 146 | var out bytes.Buffer |
| 147 | sink := newRunOutputSink(&out, runOutputEventsJSONL) |
| 148 | sink.Emit(event.Event{Kind: event.Text, Text: "PRIVATE ANSWER"}) |
| 149 | sink.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{ |
| 150 | ID: "PRIVATE TOOL ID", Name: "PRIVATE TOOL NAME", Args: `{"command":"PRIVATE COMMAND"}`, Output: "PRIVATE OUTPUT", Err: "PRIVATE ERROR", |
| 151 | }}) |
| 152 | sink.Emit(event.Event{Kind: event.ToolProgress, Tool: event.Tool{ID: "PRIVATE TOOL ID", Name: "PRIVATE TOOL NAME"}}) |
| 153 | sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{PromptTokens: 4, CompletionTokens: 2, Estimated: true}}) |
| 154 | if err := sink.Finalize("session-1", time.Now(), nil); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | lines := strings.Split(strings.TrimSpace(out.String()), "\n") |
| 158 | if len(lines) != 5 { |
| 159 | t.Fatalf("event lines = %d, output = %s", len(lines), out.String()) |
| 160 | } |
| 161 | var toolAliases []struct { |
| 162 | ToolID string `json:"tool_id"` |
| 163 | ToolName string `json:"tool_name"` |
| 164 | } |
| 165 | var sawEstimatedUsage bool |
| 166 | for i, line := range lines { |
| 167 | var payload map[string]any |
| 168 | if err := json.Unmarshal([]byte(line), &payload); err != nil { |
| 169 | t.Fatalf("line %d: %v", i, err) |
| 170 | } |
| 171 | if payload["schema_version"] != float64(machineSchemaVersion) || payload["sequence"] != float64(i+1) { |
| 172 | t.Fatalf("line %d envelope = %#v", i, payload) |
| 173 | } |
| 174 | if payload["kind"] == "tool_result" || payload["kind"] == "tool_progress" { |
| 175 | var aliases struct { |
| 176 | ToolID string `json:"tool_id"` |
| 177 | ToolName string `json:"tool_name"` |
| 178 | } |
| 179 | if err := json.Unmarshal([]byte(line), &aliases); err != nil { |
| 180 | t.Fatal(err) |
| 181 | } |
| 182 | toolAliases = append(toolAliases, aliases) |
| 183 | } |
| 184 | if payload["kind"] == "usage" { |
| 185 | usage, ok := payload["usage"].(map[string]any) |
| 186 | sawEstimatedUsage = ok && usage["estimated"] == true |
| 187 | } |
| 188 | } |
| 189 | if strings.Contains(out.String(), "PRIVATE") || !strings.Contains(out.String(), `"kind":"run_done"`) { |
| 190 | t.Fatalf("event stream was not redacted or terminated: %s", out.String()) |
| 191 | } |
| 192 | if len(toolAliases) != 2 || toolAliases[0].ToolID != "tool_1" || toolAliases[0].ToolName != "tool_name_1" || toolAliases[1] != toolAliases[0] { |
| 193 | t.Fatalf("tool aliases = %+v, want stable per-run opaque identities", toolAliases) |
| 194 | } |
| 195 | if !sawEstimatedUsage { |
| 196 | t.Fatalf("event stream lost estimated usage marker: %s", out.String()) |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | func TestEventsJSONLHasOneCanonicalFlag(t *testing.T) { |
| 201 | if _, err := parseRunOutputFormat("events-jsonl"); err == nil { |
| 202 | t.Fatal("events-jsonl must use the dedicated --events-jsonl flag") |
| 203 | } |
| 204 | var code int |
| 205 | stderr := captureStderr(t, func() { |
| 206 | code = runAgent([]string{"--events-jsonl", "--output-format", "json", "task"}, "dev") |
| 207 | }) |
| 208 | if code != 2 || !strings.Contains(stderr, "cannot be combined") { |
| 209 | t.Fatalf("exit=%d stderr=%q", code, stderr) |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | func TestRunOutputJSONClassifiesRecoveryPauseAsControlledOutcome(t *testing.T) { |
| 214 | var out bytes.Buffer |
| 215 | sink := newRunOutputSink(&out, runOutputJSON) |
| 216 | runErr := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "automatic recovery paused"}) |
| 217 | if err := sink.Finalize("abc", time.Now(), runErr); err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | var result runResult |
| 221 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | if result.IsError || result.Subtype != event.TurnOutcomeRecoveryPaused || result.Result != runErr.Error() || result.NumTurns != 1 { |
| 225 | t.Fatalf("recovery pause result = %+v", result) |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | func TestRunOutputEventsJSONLClassifiesRecoveryPauseAsControlledOutcome(t *testing.T) { |
| 230 | var out bytes.Buffer |
| 231 | sink := newRunOutputSink(&out, runOutputEventsJSONL) |
| 232 | runErr := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "automatic recovery paused"}) |
| 233 | if err := sink.Finalize("machine-session", time.Now(), runErr); err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | var result machineRunDone |
| 237 | if err := json.Unmarshal(out.Bytes(), &result); err != nil { |
| 238 | t.Fatal(err) |
| 239 | } |
| 240 | if !result.OK || result.NumTurns != 1 || result.SessionID != "machine-session" { |
| 241 | t.Fatalf("recovery pause result = %+v", result) |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | func TestClassifyRunCompletion(t *testing.T) { |
| 246 | pause := fmt.Errorf("wrapped: %w", &agent.RecoveryPauseError{Message: "paused"}) |
| 247 | if got := classifyRunCompletion(pause); got.outcome != event.TurnOutcomeRecoveryPaused || got.isError || got.exitCode != 0 { |
| 248 | t.Fatalf("pause completion = %+v", got) |
| 249 | } |
| 250 | if got := classifyRunCompletion(errors.New("provider failed")); got.outcome != "" || !got.isError || got.exitCode != 1 { |
| 251 | t.Fatalf("error completion = %+v", got) |
| 252 | } |
| 253 | if got := classifyRunCompletion(nil); got.outcome != "" || got.isError || got.exitCode != 0 { |
| 254 | t.Fatalf("success completion = %+v", got) |
| 255 | } |
| 256 | } |
| 257 |