| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "reflect" |
| 10 | "runtime" |
| 11 | "strings" |
| 12 | "sync/atomic" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "reasonix/internal/agent/testutil" |
| 17 | "reasonix/internal/event" |
| 18 | "reasonix/internal/provider" |
| 19 | "reasonix/internal/tool" |
| 20 | ) |
| 21 | |
| 22 | type toolCallReasoningRequiredProvider struct { |
| 23 | *testutil.MockProvider |
| 24 | } |
| 25 | |
| 26 | func (p toolCallReasoningRequiredProvider) RequiresToolCallReasoning() bool { return true } |
| 27 | |
| 28 | type configuredToolCallReasoningProvider struct { |
| 29 | *testutil.MockProvider |
| 30 | identity string |
| 31 | } |
| 32 | |
| 33 | type cancelMissingReasoningRetryProvider struct { |
| 34 | calls atomic.Int32 |
| 35 | retryUsageSent chan struct{} |
| 36 | } |
| 37 | |
| 38 | func (p *cancelMissingReasoningRetryProvider) Name() string { return "deepseek-cancel-retry" } |
| 39 | func (p *cancelMissingReasoningRetryProvider) RequiresToolCallReasoning() bool { |
| 40 | return true |
| 41 | } |
| 42 | |
| 43 | func (p *cancelMissingReasoningRetryProvider) Stream(ctx context.Context, _ provider.Request) (<-chan provider.Chunk, error) { |
| 44 | call := p.calls.Add(1) |
| 45 | ch := make(chan provider.Chunk) |
| 46 | go func() { |
| 47 | defer close(ch) |
| 48 | send := func(chunk provider.Chunk) bool { |
| 49 | select { |
| 50 | case <-ctx.Done(): |
| 51 | return false |
| 52 | case ch <- chunk: |
| 53 | return true |
| 54 | } |
| 55 | } |
| 56 | if call == 1 { |
| 57 | toolCall := provider.ToolCall{ID: "discarded", Name: "echo", Arguments: `{"text":"must not run"}`} |
| 58 | if !send(provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &toolCall}) { |
| 59 | return |
| 60 | } |
| 61 | if !send(provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12}}) { |
| 62 | return |
| 63 | } |
| 64 | send(provider.Chunk{Type: provider.ChunkDone}) |
| 65 | return |
| 66 | } |
| 67 | if !send(provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 1, TotalTokens: 11}}) { |
| 68 | return |
| 69 | } |
| 70 | close(p.retryUsageSent) |
| 71 | <-ctx.Done() |
| 72 | }() |
| 73 | return ch, nil |
| 74 | } |
| 75 | |
| 76 | func (p configuredToolCallReasoningProvider) RequiresToolCallReasoning() bool { return true } |
| 77 | func (p configuredToolCallReasoningProvider) MissingToolCallReasoningWarningIdentity() string { |
| 78 | return p.identity |
| 79 | } |
| 80 | |
| 81 | func echoRegistry() *tool.Registry { |
| 82 | reg := tool.NewRegistry() |
| 83 | reg.Add(echoTool{}) |
| 84 | return reg |
| 85 | } |
| 86 | |
| 87 | func TestRunPersistsUserCreatedAtWithoutSendingItToProvider(t *testing.T) { |
| 88 | const existingCreatedAt int64 = 1_718_000_000_000 |
| 89 | prov := testutil.NewMock("m", testutil.Turn{Text: "done"}) |
| 90 | session := NewSession("system") |
| 91 | session.Add(provider.Message{Role: provider.RoleUser, Content: "existing", CreatedAt: existingCreatedAt}) |
| 92 | agent := New(prov, tool.NewRegistry(), session, Options{}, event.Discard) |
| 93 | |
| 94 | if err := agent.Run(context.Background(), "new prompt"); err != nil { |
| 95 | t.Fatalf("Run: %v", err) |
| 96 | } |
| 97 | request := prov.LastRequest() |
| 98 | if request == nil { |
| 99 | t.Fatal("provider received no request") |
| 100 | } |
| 101 | for i, message := range request.Messages { |
| 102 | if message.CreatedAt != 0 { |
| 103 | t.Fatalf("provider message %d leaked createdAt %d", i, message.CreatedAt) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | messages := session.Snapshot() |
| 108 | if len(messages) < 3 || messages[1].CreatedAt != existingCreatedAt { |
| 109 | t.Fatalf("persisted existing timestamp changed: %+v", messages) |
| 110 | } |
| 111 | if messages[2].Role != provider.RoleUser || messages[2].CreatedAt <= 0 { |
| 112 | t.Fatalf("new user timestamp was not persisted: %+v", messages[2]) |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | func TestRunPersistsResponsesItemsAcrossSessionReload(t *testing.T) { |
| 117 | raw := json.RawMessage(`{"id":"ws_1","type":"web_search_call","status":"completed","action":{"type":"search","query":"latest"}}`) |
| 118 | prov := testutil.NewMock("deepseek-responses", testutil.Turn{Chunks: []provider.Chunk{ |
| 119 | {Type: provider.ChunkResponsesItem, ResponsesItem: raw}, |
| 120 | {Type: provider.ChunkText, Text: "answer"}, |
| 121 | {Type: provider.ChunkDone}, |
| 122 | }}) |
| 123 | session := NewSession("system") |
| 124 | agent := New(prov, tool.NewRegistry(), session, Options{}, event.Discard) |
| 125 | if err := agent.Run(context.Background(), "search"); err != nil { |
| 126 | t.Fatalf("Run: %v", err) |
| 127 | } |
| 128 | |
| 129 | messages := session.Snapshot() |
| 130 | assistant := messages[len(messages)-1] |
| 131 | if assistant.Role != provider.RoleAssistant || len(assistant.ResponsesItems) != 1 || string(assistant.ResponsesItems[0]) != string(raw) { |
| 132 | t.Fatalf("assistant Responses items = %#v, want persisted search item", assistant.ResponsesItems) |
| 133 | } |
| 134 | |
| 135 | path := filepath.Join(t.TempDir(), "responses-items.jsonl") |
| 136 | if err := session.Save(path); err != nil { |
| 137 | t.Fatalf("Save: %v", err) |
| 138 | } |
| 139 | loaded, err := LoadSession(path) |
| 140 | if err != nil { |
| 141 | t.Fatalf("LoadSession: %v", err) |
| 142 | } |
| 143 | loadedAssistant := loaded.Messages[len(loaded.Messages)-1] |
| 144 | if len(loadedAssistant.ResponsesItems) != 1 || string(loadedAssistant.ResponsesItems[0]) != string(raw) { |
| 145 | t.Fatalf("reloaded Responses items = %#v, want original item", loadedAssistant.ResponsesItems) |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // TestRunMultiToolRoundEmptyIDsSurvivePairing drives the real loop through a turn |
| 150 | // that fans out two tool calls carrying no id (a gateway that streams by index), |
| 151 | // then asserts both results still pair back after SanitizeToolPairing — the repair |
| 152 | // that runs on every send. Keying on tool_call_id alone collapsed them into one, |
| 153 | // dropping a result from the model's context on the very next turn. |
| 154 | func TestRunMultiToolRoundEmptyIDsSurvivePairing(t *testing.T) { |
| 155 | mp := testutil.NewMock("m", |
| 156 | testutil.Turn{ToolCalls: []provider.ToolCall{ |
| 157 | {ID: "", Name: "echo", Arguments: `{"text":"alpha"}`}, |
| 158 | {ID: "", Name: "echo", Arguments: `{"text":"beta"}`}, |
| 159 | }}, |
| 160 | testutil.Turn{Text: "done"}, |
| 161 | ) |
| 162 | a := New(mp, echoRegistry(), NewSession(""), Options{}, event.Discard) |
| 163 | if err := a.Run(context.Background(), "go"); err != nil { |
| 164 | t.Fatalf("Run: %v", err) |
| 165 | } |
| 166 | |
| 167 | repaired := provider.SanitizeToolPairing(a.Session().Messages) |
| 168 | var results []string |
| 169 | for _, m := range repaired { |
| 170 | if m.Role == provider.RoleTool { |
| 171 | results = append(results, m.Content) |
| 172 | } |
| 173 | } |
| 174 | if len(results) != 2 { |
| 175 | t.Fatalf("want 2 tool results after pairing, got %d: %v", len(results), results) |
| 176 | } |
| 177 | if results[0] == results[1] { |
| 178 | t.Fatalf("both results collapsed to %q — one was lost from the model's context", results[0]) |
| 179 | } |
| 180 | if !strings.Contains(results[0], "alpha") || !strings.Contains(results[1], "beta") { |
| 181 | t.Errorf("results lost their identity: %v", results) |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | func TestRunPersistsCumulativeAssistantWorkDuration(t *testing.T) { |
| 186 | mp := testutil.NewMock("m", |
| 187 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "call-1", Name: "echo", Arguments: `{"text":"hello"}`}}}, |
| 188 | testutil.Turn{Text: "done"}, |
| 189 | ) |
| 190 | a := New(mp, echoRegistry(), NewSession(""), Options{}, event.Discard) |
| 191 | if err := a.Run(context.Background(), "go"); err != nil { |
| 192 | t.Fatalf("Run: %v", err) |
| 193 | } |
| 194 | |
| 195 | var durations []int64 |
| 196 | for _, message := range a.Session().Messages { |
| 197 | if message.Role == provider.RoleAssistant { |
| 198 | durations = append(durations, message.WorkDurationMs) |
| 199 | } |
| 200 | } |
| 201 | if len(durations) != 2 { |
| 202 | t.Fatalf("assistant durations = %v, want two rounds", durations) |
| 203 | } |
| 204 | if durations[0] <= 0 || durations[1] < durations[0] { |
| 205 | t.Fatalf("assistant durations must be positive and cumulative: %v", durations) |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | // TestRunCancelledMidStreamLeavesResumableSession proves a turn cancelled before |
| 210 | // the model answered leaves the session well-formed: the user message stands, |
| 211 | // nothing dangling, and the repaired history is sendable as-is on resume. |
| 212 | func TestRunCancelledMidStreamLeavesResumableSession(t *testing.T) { |
| 213 | mp := testutil.NewMock("m", testutil.ErrorTurn(context.Canceled)) |
| 214 | a := New(mp, echoRegistry(), NewSession("sys"), Options{}, event.Discard) |
| 215 | |
| 216 | err := a.Run(context.Background(), "do the thing") |
| 217 | if !errors.Is(err, context.Canceled) { |
| 218 | t.Fatalf("Run should surface the cancellation, got %v", err) |
| 219 | } |
| 220 | |
| 221 | repaired := provider.SanitizeToolPairing(a.Session().Messages) |
| 222 | for i, m := range repaired { |
| 223 | if m.Role == provider.RoleTool { |
| 224 | t.Fatalf("a cancelled turn left a dangling tool message at %d: %+v", i, m) |
| 225 | } |
| 226 | } |
| 227 | last := repaired[len(repaired)-1] |
| 228 | if last.Role != provider.RoleUser || last.Content != "do the thing" { |
| 229 | t.Errorf("the pending user message should survive a cancel, got %+v", last) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | func TestRunRecoversInterruptedStreamAfterPartialText(t *testing.T) { |
| 234 | interrupted := &provider.StreamInterruptedError{Err: errors.New("deepseek-flash: read stream: unexpected EOF"), Reason: provider.StreamInterruptPrematureEOF} |
| 235 | mp := testutil.NewMock("m", |
| 236 | testutil.Turn{Text: "partial ", ChunkError: interrupted}, |
| 237 | testutil.Turn{Text: "continued"}, |
| 238 | ) |
| 239 | sink := &recordSink{} |
| 240 | a := New(mp, echoRegistry(), NewSession(""), Options{}, sink) |
| 241 | |
| 242 | if err := a.Run(context.Background(), "go"); err != nil { |
| 243 | t.Fatalf("Run should recover the interrupted stream, got %v", err) |
| 244 | } |
| 245 | if mp.CallCount() != 2 { |
| 246 | t.Fatalf("provider calls = %d, want 2", mp.CallCount()) |
| 247 | } |
| 248 | |
| 249 | reqs := mp.Requests() |
| 250 | if len(reqs) != 2 { |
| 251 | t.Fatalf("recorded requests = %d, want 2", len(reqs)) |
| 252 | } |
| 253 | // Codex-style: exact original request replay — no synthetic recovery user |
| 254 | // message, no partial assistant in the provider body. |
| 255 | if !providerRequestBodiesEqual(reqs[0], reqs[1]) { |
| 256 | t.Fatalf("retry must replay the identical provider request\nfirst=%+v\nsecond=%+v", reqs[0], reqs[1]) |
| 257 | } |
| 258 | for _, message := range reqs[1].Messages { |
| 259 | if message.LocalOnly || message.Content == "partial " { |
| 260 | t.Fatalf("partial assistant leaked into provider recovery request: %+v", reqs[1].Messages) |
| 261 | } |
| 262 | if strings.Contains(message.Content, "interrupted") && message.Role == provider.RoleUser { |
| 263 | t.Fatalf("synthetic stream recovery must not be injected: %+v", message) |
| 264 | } |
| 265 | } |
| 266 | // Successful recovery never persists a LocalOnly interrupted record. |
| 267 | for _, message := range a.Session().Messages { |
| 268 | if message.LocalOnly { |
| 269 | t.Fatalf("successful recovery must not leave LocalOnly interrupt records: %+v", message) |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | var streamed strings.Builder |
| 274 | for _, e := range sink.kinds(event.Text) { |
| 275 | streamed.WriteString(e.Text) |
| 276 | } |
| 277 | // Both attempts emit text to the sink; Desktop discards the first via |
| 278 | // stream_attempt. Agent still emits both for non-journal sinks. |
| 279 | if !strings.Contains(streamed.String(), "continued") { |
| 280 | t.Fatalf("streamed text = %q, want final continued answer", streamed.String()) |
| 281 | } |
| 282 | retries := sink.kinds(event.Retrying) |
| 283 | if len(retries) != 1 || retries[0].RetryAttempt != 1 || retries[0].RetryMax != maxStreamRecoveries || retries[0].RetryScope != event.RetryScopeStream { |
| 284 | t.Fatalf("retry events = %+v, want one stream recovery retry", retries) |
| 285 | } |
| 286 | attempts := sink.kinds(event.StreamAttempt) |
| 287 | if len(attempts) < 3 { |
| 288 | t.Fatalf("stream_attempt events = %d, want begin/discard/begin/commit at least", len(attempts)) |
| 289 | } |
| 290 | var sawDiscard, sawCommit bool |
| 291 | for _, e := range attempts { |
| 292 | if e.StreamAttempt.Action == event.StreamAttemptDiscard { |
| 293 | sawDiscard = true |
| 294 | if e.StreamAttempt.Reason != provider.StreamInterruptPrematureEOF { |
| 295 | t.Fatalf("discard reason = %q", e.StreamAttempt.Reason) |
| 296 | } |
| 297 | } |
| 298 | if e.StreamAttempt.Action == event.StreamAttemptCommit { |
| 299 | sawCommit = true |
| 300 | } |
| 301 | } |
| 302 | if !sawDiscard || !sawCommit { |
| 303 | t.Fatalf("stream attempts missing discard/commit: %+v", attempts) |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | func TestRunRecoversRepeatedInterruptedStreams(t *testing.T) { |
| 308 | interrupted := &provider.StreamInterruptedError{Err: errors.New("deepseek-flash: read stream: unexpected EOF")} |
| 309 | mp := testutil.NewMock("m", |
| 310 | testutil.Turn{Text: "first ", ChunkError: interrupted}, |
| 311 | testutil.Turn{Text: "second ", ChunkError: interrupted}, |
| 312 | testutil.Turn{Text: "done"}, |
| 313 | ) |
| 314 | sink := &recordSink{} |
| 315 | a := New(mp, echoRegistry(), NewSession(""), Options{}, sink) |
| 316 | |
| 317 | if err := a.Run(context.Background(), "go"); err != nil { |
| 318 | t.Fatalf("Run should recover repeated interrupted streams, got %v", err) |
| 319 | } |
| 320 | if mp.CallCount() != 3 { |
| 321 | t.Fatalf("provider calls = %d, want 3", mp.CallCount()) |
| 322 | } |
| 323 | reqs := mp.Requests() |
| 324 | if !providerRequestBodiesEqual(reqs[0], reqs[1]) || !providerRequestBodiesEqual(reqs[0], reqs[2]) { |
| 325 | t.Fatalf("all retries must replay the same frozen provider request") |
| 326 | } |
| 327 | |
| 328 | var streamed strings.Builder |
| 329 | for _, e := range sink.kinds(event.Text) { |
| 330 | streamed.WriteString(e.Text) |
| 331 | } |
| 332 | if !strings.Contains(streamed.String(), "done") { |
| 333 | t.Fatalf("streamed text = %q, want final done", streamed.String()) |
| 334 | } |
| 335 | retries := sink.kinds(event.Retrying) |
| 336 | if len(retries) != 2 || retries[0].RetryAttempt != 1 || retries[1].RetryAttempt != 2 { |
| 337 | t.Fatalf("retry events = %+v, want attempts 1 and 2", retries) |
| 338 | } |
| 339 | for _, retry := range retries { |
| 340 | if retry.RetryMax != maxStreamRecoveries || retry.RetryScope != event.RetryScopeStream { |
| 341 | t.Fatalf("retry = %+v, want max=%d scope=stream", retry, maxStreamRecoveries) |
| 342 | } |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | func TestRunRecoversInterruptedPartialToolCallWithoutExecutingIt(t *testing.T) { |
| 347 | interrupted := &provider.StreamInterruptedError{Err: errors.New("deepseek-flash: read stream: unexpected EOF")} |
| 348 | mp := testutil.NewMock("m", |
| 349 | testutil.Turn{Chunks: []provider.Chunk{ |
| 350 | {Type: provider.ChunkToolCallStart, ToolCall: &provider.ToolCall{ID: "c1", Name: "echo"}}, |
| 351 | {Type: provider.ChunkError, Err: interrupted}, |
| 352 | }}, |
| 353 | testutil.Turn{Text: "recovered"}, |
| 354 | ) |
| 355 | a := New(mp, echoRegistry(), NewSession(""), Options{}, event.Discard) |
| 356 | |
| 357 | if err := a.Run(context.Background(), "go"); err != nil { |
| 358 | t.Fatalf("Run should recover the interrupted tool-call stream, got %v", err) |
| 359 | } |
| 360 | |
| 361 | for _, m := range a.Session().Messages { |
| 362 | if m.Role == provider.RoleTool && !m.LocalOnly { |
| 363 | t.Fatalf("partial tool call should not have executed or produced a tool result: %+v", m) |
| 364 | } |
| 365 | if m.LocalOnly { |
| 366 | t.Fatalf("successful recovery must not leave LocalOnly interrupt: %+v", m) |
| 367 | } |
| 368 | } |
| 369 | reqs := mp.Requests() |
| 370 | if len(reqs) != 2 || !providerRequestBodiesEqual(reqs[0], reqs[1]) { |
| 371 | t.Fatalf("partial-tool interrupt must exact-replay without synthetic recovery") |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | func TestRunStreamRetryRequestCountIsLinearNotTriangular(t *testing.T) { |
| 376 | interrupted := &provider.StreamInterruptedError{Err: errors.New("eof"), Reason: provider.StreamInterruptPrematureEOF} |
| 377 | mp := testutil.NewMock("m", |
| 378 | testutil.Turn{Text: "a", Usage: &provider.Usage{PromptTokens: 30, CompletionTokens: 1, TotalTokens: 31, CacheMissTokens: 30}, ChunkError: interrupted}, |
| 379 | testutil.Turn{Text: "b", Usage: &provider.Usage{PromptTokens: 30, CompletionTokens: 1, TotalTokens: 31, CacheMissTokens: 30}, ChunkError: interrupted}, |
| 380 | testutil.Turn{Text: "ok", Usage: &provider.Usage{PromptTokens: 30, CompletionTokens: 2, TotalTokens: 32, CacheMissTokens: 30}}, |
| 381 | ) |
| 382 | sink := &recordSink{} |
| 383 | a := New(mp, echoRegistry(), NewSession(""), Options{}, sink) |
| 384 | if err := a.Run(context.Background(), "go"); err != nil { |
| 385 | t.Fatalf("Run: %v", err) |
| 386 | } |
| 387 | if mp.CallCount() != 3 { |
| 388 | t.Fatalf("provider calls = %d, want 3", mp.CallCount()) |
| 389 | } |
| 390 | usages := sink.kinds(event.Usage) |
| 391 | if len(usages) != 1 || usages[0].Usage == nil { |
| 392 | t.Fatalf("usage events = %d, want one aggregate", len(usages)) |
| 393 | } |
| 394 | u := usages[0].Usage |
| 395 | if u.RequestCount != 3 { |
| 396 | t.Fatalf("RequestCount = %d, want 3 (linear, not triangular 6)", u.RequestCount) |
| 397 | } |
| 398 | // Billable input is summed; context gauge uses ContextPromptTokens. |
| 399 | if u.PromptTokens != 90 { |
| 400 | t.Fatalf("PromptTokens = %d, want billable sum 90", u.PromptTokens) |
| 401 | } |
| 402 | if u.ContextPromptTokens != 30 { |
| 403 | t.Fatalf("ContextPromptTokens = %d, want latest 30", u.ContextPromptTokens) |
| 404 | } |
| 405 | if u.CacheHitTokens+u.CacheMissTokens != u.PromptTokens { |
| 406 | t.Fatalf("cache split %d+%d must align with PromptTokens %d", u.CacheHitTokens, u.CacheMissTokens, u.PromptTokens) |
| 407 | } |
| 408 | if u.CompletionTokens != 4 { |
| 409 | t.Fatalf("CompletionTokens = %d, want billable sum 4", u.CompletionTokens) |
| 410 | } |
| 411 | // ContextSnapshot and compaction use the latest full attempt shape. |
| 412 | if last := a.lastUsage.Load(); last == nil || last.PromptTokens != 30 { |
| 413 | t.Fatalf("lastUsage prompt = %+v, want latest attempt prompt 30", last) |
| 414 | } |
| 415 | } |
| 416 | |
| 417 | func TestRunExhaustedStreamRetriesPersistPendingLocalOnly(t *testing.T) { |
| 418 | interrupted := &provider.StreamInterruptedError{Err: errors.New("eof"), Reason: provider.StreamInterruptPrematureEOF} |
| 419 | turns := make([]testutil.Turn, 0, maxSamplingAttempts) |
| 420 | for i := 0; i < maxSamplingAttempts; i++ { |
| 421 | turns = append(turns, testutil.Turn{Text: "half", ChunkError: interrupted}) |
| 422 | } |
| 423 | mp := testutil.NewMock("m", turns...) |
| 424 | a := New(mp, echoRegistry(), NewSession(""), Options{}, event.Discard) |
| 425 | |
| 426 | err := a.Run(context.Background(), "go") |
| 427 | if !provider.IsStreamInterrupted(err) { |
| 428 | t.Fatalf("Run error = %v, want StreamInterruptedError after exhausting retries", err) |
| 429 | } |
| 430 | if mp.CallCount() != maxSamplingAttempts { |
| 431 | t.Fatalf("provider calls = %d, want %d", mp.CallCount(), maxSamplingAttempts) |
| 432 | } |
| 433 | var pending *provider.InterruptedTurnRecovery |
| 434 | var local provider.Message |
| 435 | for _, m := range a.Session().Messages { |
| 436 | if m.LocalOnly && m.InterruptedTurn != nil && m.InterruptedTurn.Pending { |
| 437 | pending = m.InterruptedTurn |
| 438 | local = m |
| 439 | } |
| 440 | } |
| 441 | if pending == nil || local.Content != "half" { |
| 442 | t.Fatalf("exhausted retries must leave one pending LocalOnly record: local=%+v pending=%+v", local, pending) |
| 443 | } |
| 444 | // No synthetic recovery user messages mid-turn. |
| 445 | for _, m := range a.Session().Messages { |
| 446 | if m.Role == provider.RoleUser && strings.Contains(m.Content, "previous assistant response was interrupted") { |
| 447 | t.Fatalf("must not inject synthetic stream recovery: %+v", m) |
| 448 | } |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | func TestRunCompleteUncommittedToolCallNeverExecutes(t *testing.T) { |
| 453 | // Full tool block arrived, but the stream was interrupted before a clean |
| 454 | // terminal — the call stays speculative and must never reach executeBatch. |
| 455 | interrupted := &provider.StreamInterruptedError{Err: errors.New("eof"), Reason: provider.StreamInterruptPrematureEOF} |
| 456 | writer := &countingWriterTool{} |
| 457 | reg := tool.NewRegistry() |
| 458 | reg.Add(writer) |
| 459 | mp := testutil.NewMock("m", |
| 460 | testutil.Turn{Chunks: []provider.Chunk{ |
| 461 | {Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "w1", Name: "write_file", Arguments: `{"path":"x.txt","content":"from-writer"}`}}, |
| 462 | {Type: provider.ChunkError, Err: interrupted}, |
| 463 | }}, |
| 464 | testutil.Turn{Text: "recovered without write"}, |
| 465 | ) |
| 466 | a := New(mp, reg, NewSession(""), Options{}, event.Discard) |
| 467 | if err := a.Run(context.Background(), "write it"); err != nil { |
| 468 | t.Fatalf("Run: %v", err) |
| 469 | } |
| 470 | if writer.calls.Load() != 0 { |
| 471 | t.Fatalf("writer executed %d times, want 0 (uncommitted tool call)", writer.calls.Load()) |
| 472 | } |
| 473 | } |
| 474 | |
| 475 | type countingWriterTool struct{ calls atomic.Int32 } |
| 476 | |
| 477 | func (c *countingWriterTool) Name() string { return "write_file" } |
| 478 | func (c *countingWriterTool) Description() string { return "count writes" } |
| 479 | func (c *countingWriterTool) Schema() json.RawMessage { |
| 480 | return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}}}`) |
| 481 | } |
| 482 | func (c *countingWriterTool) ReadOnly() bool { return false } |
| 483 | func (c *countingWriterTool) Execute(context.Context, json.RawMessage) (string, error) { |
| 484 | c.calls.Add(1) |
| 485 | return "wrote", nil |
| 486 | } |
| 487 | |
| 488 | // providerRequestBodiesEqual compares the provider-visible request surface |
| 489 | // (messages, tools order/bytes, temperature, token limit, response format). |
| 490 | func providerRequestBodiesEqual(a, b provider.Request) bool { |
| 491 | if a.MaxTokens != b.MaxTokens { |
| 492 | return false |
| 493 | } |
| 494 | if (a.Temperature == nil) != (b.Temperature == nil) { |
| 495 | return false |
| 496 | } |
| 497 | if a.Temperature != nil && b.Temperature != nil && *a.Temperature != *b.Temperature { |
| 498 | return false |
| 499 | } |
| 500 | if (a.ResponseFormat == nil) != (b.ResponseFormat == nil) { |
| 501 | return false |
| 502 | } |
| 503 | if a.ResponseFormat != nil && b.ResponseFormat != nil && a.ResponseFormat.Type != b.ResponseFormat.Type { |
| 504 | return false |
| 505 | } |
| 506 | if len(a.Messages) != len(b.Messages) || len(a.Tools) != len(b.Tools) { |
| 507 | return false |
| 508 | } |
| 509 | for i := range a.Messages { |
| 510 | am, bm := a.Messages[i], b.Messages[i] |
| 511 | if am.Role != bm.Role || am.Content != bm.Content || am.ReasoningContent != bm.ReasoningContent || |
| 512 | am.Name != bm.Name || am.ToolCallID != bm.ToolCallID || am.LocalOnly != bm.LocalOnly { |
| 513 | return false |
| 514 | } |
| 515 | if len(am.ToolCalls) != len(bm.ToolCalls) { |
| 516 | return false |
| 517 | } |
| 518 | for j := range am.ToolCalls { |
| 519 | if am.ToolCalls[j].ID != bm.ToolCalls[j].ID || am.ToolCalls[j].Name != bm.ToolCalls[j].Name || |
| 520 | am.ToolCalls[j].Arguments != bm.ToolCalls[j].Arguments { |
| 521 | return false |
| 522 | } |
| 523 | } |
| 524 | } |
| 525 | for i := range a.Tools { |
| 526 | if a.Tools[i].Name != b.Tools[i].Name || a.Tools[i].Description != b.Tools[i].Description || |
| 527 | string(a.Tools[i].Parameters) != string(b.Tools[i].Parameters) { |
| 528 | return false |
| 529 | } |
| 530 | } |
| 531 | return true |
| 532 | } |
| 533 | |
| 534 | func TestRunGenericStreamErrorPersistsLocalDisplayAndInjectsBoundedRecovery(t *testing.T) { |
| 535 | apiErr := errors.New("upstream reset") |
| 536 | mp := testutil.NewMock("m", |
| 537 | testutil.Turn{Reasoning: "private partial reasoning", Text: "visible partial", ChunkError: apiErr}, |
| 538 | testutil.Turn{Text: "continued safely"}, |
| 539 | ) |
| 540 | session := NewSession("system") |
| 541 | a := New(mp, echoRegistry(), session, Options{}, event.Discard) |
| 542 | |
| 543 | if err := a.Run(context.Background(), "change the file"); !errors.Is(err, apiErr) { |
| 544 | t.Fatalf("first Run error = %v, want %v", err, apiErr) |
| 545 | } |
| 546 | msgs := session.Snapshot() |
| 547 | last := msgs[len(msgs)-1] |
| 548 | if !last.LocalOnly || last.InterruptedTurn == nil || !last.InterruptedTurn.Pending { |
| 549 | t.Fatalf("terminal stream error did not leave pending local recovery: %+v", last) |
| 550 | } |
| 551 | if last.Content != "visible partial" || last.ReasoningContent != "private partial reasoning" { |
| 552 | t.Fatalf("local display lost streamed output: %+v", last) |
| 553 | } |
| 554 | |
| 555 | if err := a.Run(context.Background(), "continue"); err != nil { |
| 556 | t.Fatalf("second Run: %v", err) |
| 557 | } |
| 558 | req := mp.Requests()[1] |
| 559 | for _, message := range req.Messages { |
| 560 | if message.LocalOnly || strings.Contains(message.Content, "visible partial") || strings.Contains(message.ReasoningContent, "private partial reasoning") { |
| 561 | t.Fatalf("unsafe partial output leaked to provider: %+v", req.Messages) |
| 562 | } |
| 563 | } |
| 564 | lastUser := req.Messages[len(req.Messages)-1] |
| 565 | if lastUser.Role != provider.RoleUser || !strings.Contains(lastUser.Content, "<interrupted-turn-recovery>") || |
| 566 | !strings.Contains(lastUser.Content, "unsafe_partial_output: excluded") || !strings.HasSuffix(lastUser.Content, "continue") { |
| 567 | t.Fatalf("next user turn missing bounded recovery block: %+v", lastUser) |
| 568 | } |
| 569 | if got := StripTransientUserBlocks(lastUser.Content); got != "continue" { |
| 570 | t.Fatalf("recovery block leaked into user display: %q", got) |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | func TestRunRecoveryKeepsCompletedToolPairAndSummarizesChangedFile(t *testing.T) { |
| 575 | session := NewSession("system") |
| 576 | session.Add(provider.Message{Role: provider.RoleUser, Content: "update config"}) |
| 577 | session.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ |
| 578 | ID: "done-1", Name: "write_file", Arguments: `{"path":"config.json","content":"{}"}`, Added: 1, |
| 579 | }}}) |
| 580 | session.Add(provider.Message{Role: provider.RoleTool, ToolCallID: "done-1", Name: "write_file", Content: "wrote config.json"}) |
| 581 | session.Add(provider.Message{ |
| 582 | Role: provider.RoleTool, ToolCallID: provider.LocalOnlyToolID, Name: provider.LocalOnlyToolName, LocalOnly: true, |
| 583 | ReasoningContent: "unsafe partial reasoning", |
| 584 | InterruptedTurn: &provider.InterruptedTurnRecovery{ |
| 585 | Pending: true, |
| 586 | CompletedTools: []provider.InterruptedToolSummary{{ |
| 587 | ID: "done-1", Name: "write_file", Files: []string{"config.json"}, Added: 1, |
| 588 | }}, |
| 589 | InterruptedTools: []string{"bash"}, |
| 590 | DroppedPartialReasoning: true, |
| 591 | }, |
| 592 | }) |
| 593 | mp := testutil.NewMock("m", testutil.Turn{Text: "done"}) |
| 594 | a := New(mp, echoRegistry(), session, Options{}, event.Discard) |
| 595 | if err := a.Run(context.Background(), "continue"); err != nil { |
| 596 | t.Fatalf("Run: %v", err) |
| 597 | } |
| 598 | |
| 599 | req := mp.Requests()[0] |
| 600 | if len(req.Messages) != 5 { |
| 601 | t.Fatalf("provider request should contain system + user + complete pair + recovery user, got %+v", req.Messages) |
| 602 | } |
| 603 | if req.Messages[2].Role != provider.RoleAssistant || req.Messages[3].Role != provider.RoleTool { |
| 604 | t.Fatalf("completed tool pair was not replayed canonically: %+v", req.Messages) |
| 605 | } |
| 606 | last := req.Messages[len(req.Messages)-1] |
| 607 | for _, want := range []string{"write_file files=config.json diff=+1/-0", "interrupted_tools: bash", "inspect the current workspace", "continue"} { |
| 608 | if !strings.Contains(last.Content, want) { |
| 609 | t.Fatalf("recovery user message missing %q: %s", want, last.Content) |
| 610 | } |
| 611 | } |
| 612 | if strings.Contains(last.Content, "unsafe partial reasoning") { |
| 613 | t.Fatalf("raw partial reasoning leaked into recovery summary: %s", last.Content) |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | // TestRunWellFormedToolLoopRoundTrips is the happy-path baseline: a tool round |
| 618 | // then a final answer. The session must end with the assistant answer and pair |
| 619 | // cleanly (the repair is a no-op on well-formed histories). |
| 620 | func TestRunWellFormedToolLoopRoundTrips(t *testing.T) { |
| 621 | mp := testutil.NewMock("m", |
| 622 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 623 | testutil.Turn{Text: "all set"}, |
| 624 | ) |
| 625 | a := New(mp, echoRegistry(), NewSession(""), Options{}, event.Discard) |
| 626 | if err := a.Run(context.Background(), "go"); err != nil { |
| 627 | t.Fatalf("Run: %v", err) |
| 628 | } |
| 629 | |
| 630 | msgs := a.Session().Messages |
| 631 | last := msgs[len(msgs)-1] |
| 632 | if last.Role != provider.RoleAssistant || last.Content != "all set" { |
| 633 | t.Fatalf("final message should be the assistant answer, got %+v", last) |
| 634 | } |
| 635 | before := len(msgs) |
| 636 | if after := len(provider.SanitizeToolPairing(msgs)); after != before { |
| 637 | t.Errorf("repair mutated a well-formed session: %d -> %d", before, after) |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | // A provider without the DeepSeek tool-call reasoning policy must keep the |
| 642 | // ordinary two-call tool loop even when its tool-call turn has no reasoning. |
| 643 | func TestRunNonDeepSeekMissingToolCallReasoningDoesNotRetry(t *testing.T) { |
| 644 | mp := testutil.NewMock("openai", |
| 645 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 646 | testutil.Turn{Text: "all set"}, |
| 647 | ) |
| 648 | sink := &recordSink{} |
| 649 | a := New(mp, echoRegistry(), NewSession(""), Options{}, sink) |
| 650 | |
| 651 | if err := a.Run(context.Background(), "go"); err != nil { |
| 652 | t.Fatalf("Run: %v", err) |
| 653 | } |
| 654 | if got := mp.CallCount(); got != 2 { |
| 655 | t.Fatalf("provider calls = %d, want tool turn + final turn without recovery retry", got) |
| 656 | } |
| 657 | if got := len(sink.kinds(event.ToolDispatch)); got != 1 { |
| 658 | t.Fatalf("tool dispatches = %d, want one", got) |
| 659 | } |
| 660 | sink.mu.Lock() |
| 661 | recovery := append([]event.ProtocolRecoveryAudit(nil), sink.recovery...) |
| 662 | sink.mu.Unlock() |
| 663 | if len(recovery) != 0 { |
| 664 | t.Fatalf("non-DeepSeek provider emitted protocol recovery audits: %+v", recovery) |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | // A one-off missing reasoning_content response is replaced before any tool |
| 669 | // executes. The retry reuses identical input, its usage is accounted for, and |
| 670 | // no provider-protocol warning or duplicate tool card reaches the user. |
| 671 | func TestRunSilentlyRecoversMissingToolCallReasoning(t *testing.T) { |
| 672 | mp := testutil.NewMock("deepseek-proxy", |
| 673 | testutil.Turn{ |
| 674 | ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}, |
| 675 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12, CacheMissTokens: 10, FinishReason: "tool_calls"}, |
| 676 | }, |
| 677 | testutil.Turn{ |
| 678 | Reasoning: "retry reasoning", |
| 679 | ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}, |
| 680 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13, CacheHitTokens: 10, ReasoningTokens: 2, FinishReason: "tool_calls"}, |
| 681 | }, |
| 682 | testutil.Turn{Text: "done"}, |
| 683 | ) |
| 684 | sink := &recordSink{} |
| 685 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{}, sink) |
| 686 | |
| 687 | if err := a.Run(context.Background(), "go"); err != nil { |
| 688 | t.Fatalf("Run: %v", err) |
| 689 | } |
| 690 | var savedToolTurns int |
| 691 | var savedReasoning string |
| 692 | for _, m := range a.Session().Messages { |
| 693 | if m.Role == provider.RoleAssistant && len(m.ToolCalls) > 0 { |
| 694 | savedToolTurns++ |
| 695 | savedReasoning = m.ReasoningContent |
| 696 | } |
| 697 | } |
| 698 | if savedToolTurns != 1 || savedReasoning != "retry reasoning" { |
| 699 | t.Fatalf("saved tool turns = %d reasoning = %q, want one recovered turn: %+v", savedToolTurns, savedReasoning, a.Session().Messages) |
| 700 | } |
| 701 | if mp.CallCount() != 3 { |
| 702 | t.Fatalf("provider calls = %d, want malformed + retry + final", mp.CallCount()) |
| 703 | } |
| 704 | requests := mp.Requests() |
| 705 | if len(requests) < 2 || !reflect.DeepEqual(requests[0], requests[1]) { |
| 706 | t.Fatalf("protocol retry changed provider-visible request:\nfirst=%+v\nretry=%+v", requests[0], requests[1]) |
| 707 | } |
| 708 | for _, e := range sink.kinds(event.Notice) { |
| 709 | if strings.Contains(e.Text, "reasoning") || strings.Contains(e.Detail, "reasoning") { |
| 710 | t.Fatalf("provider protocol leaked into user notice: %+v", e) |
| 711 | } |
| 712 | } |
| 713 | if got := len(sink.kinds(event.ToolDispatch)); got != 1 { |
| 714 | t.Fatalf("tool dispatches = %d, want one adopted call", got) |
| 715 | } |
| 716 | usageEvents := sink.kinds(event.Usage) |
| 717 | if len(usageEvents) == 0 || usageEvents[0].Usage == nil || usageEvents[0].Usage.TotalTokens != 25 || usageEvents[0].Usage.CacheHitTokens != 10 || usageEvents[0].Usage.CacheMissTokens != 10 { |
| 718 | t.Fatalf("recovery usage was not merged truthfully: %+v", usageEvents) |
| 719 | } |
| 720 | if sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted) != 1 || sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryRecovered) != 1 { |
| 721 | t.Fatalf("unexpected recovery audit: %+v", sink.recovery) |
| 722 | } |
| 723 | } |
| 724 | |
| 725 | // An exact recovery replay may choose a normal final answer instead of |
| 726 | // repeating the original tool call. The replacement is authoritative because |
| 727 | // no tool has run yet: discard the speculative call, persist only the final |
| 728 | // response, and classify the outcome separately from recovered reasoning. |
| 729 | func TestMissingReasoningRecoveryAdoptsRetryWithoutToolCall(t *testing.T) { |
| 730 | mp := testutil.NewMock("deepseek-proxy", |
| 731 | testutil.Turn{ |
| 732 | ToolCalls: []provider.ToolCall{{ID: "discarded", Name: "echo", Arguments: `{"text":"must not run"}`}}, |
| 733 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12, FinishReason: "tool_calls"}, |
| 734 | }, |
| 735 | testutil.Turn{ |
| 736 | Text: "completed without a tool", |
| 737 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13, FinishReason: "stop"}, |
| 738 | }, |
| 739 | ) |
| 740 | sink := &recordSink{} |
| 741 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{}, sink) |
| 742 | |
| 743 | if err := a.Run(context.Background(), "go"); err != nil { |
| 744 | t.Fatalf("Run: %v", err) |
| 745 | } |
| 746 | if mp.CallCount() != 2 { |
| 747 | t.Fatalf("provider calls = %d, want malformed + replacement", mp.CallCount()) |
| 748 | } |
| 749 | var toolTurns, toolResults int |
| 750 | for _, message := range a.Session().Messages { |
| 751 | if message.Role == provider.RoleAssistant && len(message.ToolCalls) > 0 { |
| 752 | toolTurns++ |
| 753 | } |
| 754 | if message.Role == provider.RoleTool { |
| 755 | toolResults++ |
| 756 | } |
| 757 | } |
| 758 | if toolTurns != 0 || toolResults != 0 { |
| 759 | t.Fatalf("discarded tool response reached session: turns=%d results=%d session=%+v", toolTurns, toolResults, a.Session().Messages) |
| 760 | } |
| 761 | last := a.Session().Messages[len(a.Session().Messages)-1] |
| 762 | if last.Role != provider.RoleAssistant || last.Content != "completed without a tool" { |
| 763 | t.Fatalf("replacement response not adopted: %+v", last) |
| 764 | } |
| 765 | if got := len(sink.kinds(event.ToolDispatch)); got != 0 { |
| 766 | t.Fatalf("discarded tool dispatches = %d, want 0", got) |
| 767 | } |
| 768 | usageEvents := sink.kinds(event.Usage) |
| 769 | if len(usageEvents) == 0 || usageEvents[0].Usage == nil || usageEvents[0].Usage.TotalTokens != 25 { |
| 770 | t.Fatalf("replacement usage was not merged truthfully: %+v", usageEvents) |
| 771 | } |
| 772 | if sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted) != 1 || |
| 773 | sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryReplaced) != 1 || |
| 774 | sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryRecovered) != 0 || |
| 775 | sink.recoveryCount(event.ProtocolRecoveryMissingReasoningFallback) != 0 { |
| 776 | t.Fatalf("unexpected recovery classification: %+v", sink.recovery) |
| 777 | } |
| 778 | } |
| 779 | |
| 780 | func TestMissingReasoningRecoveryFailureFallsBackBeforeToolExecution(t *testing.T) { |
| 781 | mp := testutil.NewMock("deepseek-proxy", |
| 782 | testutil.Turn{ |
| 783 | ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}, |
| 784 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 2, TotalTokens: 12, FinishReason: "tool_calls"}, |
| 785 | }, |
| 786 | testutil.Turn{ |
| 787 | Usage: &provider.Usage{PromptTokens: 10, CompletionTokens: 1, TotalTokens: 11}, |
| 788 | ChunkError: errors.New("recovery stream failed"), |
| 789 | }, |
| 790 | testutil.Turn{Text: "done"}, |
| 791 | ) |
| 792 | sink := &recordSink{} |
| 793 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{}, sink) |
| 794 | |
| 795 | if err := a.Run(context.Background(), "go"); err != nil { |
| 796 | t.Fatalf("Run should keep the complete first response, got %v", err) |
| 797 | } |
| 798 | var toolResults int |
| 799 | for _, message := range a.Session().Messages { |
| 800 | if message.Role == provider.RoleTool && message.ToolCallID == "c1" { |
| 801 | toolResults++ |
| 802 | } |
| 803 | } |
| 804 | if toolResults != 1 { |
| 805 | t.Fatalf("tool results = %d, want the original call executed once", toolResults) |
| 806 | } |
| 807 | usageEvents := sink.kinds(event.Usage) |
| 808 | if len(usageEvents) == 0 || usageEvents[0].Usage == nil || usageEvents[0].Usage.TotalTokens != 23 { |
| 809 | t.Fatalf("failed recovery usage was not accounted for: %+v", usageEvents) |
| 810 | } |
| 811 | if sink.recoveryCount(event.ProtocolRecoveryMissingReasoningFallback) != 1 { |
| 812 | t.Fatalf("fallback audit missing: %+v", sink.recovery) |
| 813 | } |
| 814 | } |
| 815 | |
| 816 | func TestMissingReasoningRecoveryCancellationAccountsBothAttempts(t *testing.T) { |
| 817 | prov := &cancelMissingReasoningRetryProvider{retryUsageSent: make(chan struct{})} |
| 818 | sink := &recordSink{} |
| 819 | a := New(prov, echoRegistry(), NewSession(""), Options{}, sink) |
| 820 | ctx, cancel := context.WithCancel(context.Background()) |
| 821 | done := make(chan error, 1) |
| 822 | go func() { done <- a.Run(ctx, "go") }() |
| 823 | |
| 824 | select { |
| 825 | case <-prov.retryUsageSent: |
| 826 | cancel() |
| 827 | case <-time.After(time.Second): |
| 828 | cancel() |
| 829 | t.Fatal("timed out waiting for the recovery retry usage") |
| 830 | } |
| 831 | if err := <-done; !errors.Is(err, context.Canceled) { |
| 832 | t.Fatalf("Run error = %v, want context cancellation", err) |
| 833 | } |
| 834 | if got := prov.calls.Load(); got != 2 { |
| 835 | t.Fatalf("provider calls = %d, want malformed response plus recovery retry", got) |
| 836 | } |
| 837 | if got := len(sink.kinds(event.ToolDispatch)); got != 0 { |
| 838 | t.Fatalf("discarded tool dispatches = %d, want 0", got) |
| 839 | } |
| 840 | usages := sink.kinds(event.Usage) |
| 841 | if len(usages) != 1 || usages[0].Usage == nil || usages[0].Usage.TotalTokens != 23 || usages[0].Usage.FinishReason != "interrupted" { |
| 842 | t.Fatalf("recovery cancellation usage = %+v, want one merged interrupted total of 23", usages) |
| 843 | } |
| 844 | } |
| 845 | |
| 846 | func TestSetSessionRearmsInMemoryMissingReasoningRecovery(t *testing.T) { |
| 847 | mp := testutil.NewMock("deepseek-proxy", |
| 848 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 849 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1r", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 850 | testutil.Turn{Text: "done"}, |
| 851 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 852 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2r", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 853 | testutil.Turn{Text: "done again"}, |
| 854 | ) |
| 855 | sink := &recordSink{} |
| 856 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{}, sink) |
| 857 | |
| 858 | if err := a.Run(context.Background(), "go"); err != nil { |
| 859 | t.Fatalf("first Run: %v", err) |
| 860 | } |
| 861 | a.SetSession(NewSession("")) |
| 862 | if err := a.Run(context.Background(), "go"); err != nil { |
| 863 | t.Fatalf("second Run: %v", err) |
| 864 | } |
| 865 | if got := sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted); got != 2 { |
| 866 | t.Fatalf("recovery retries across two sessions = %d, want 2", got) |
| 867 | } |
| 868 | } |
| 869 | |
| 870 | // A shared state dir turns the old warning cooldown into a cross-process retry |
| 871 | // circuit breaker. The first process retries once; a fresh process immediately |
| 872 | // uses the empty-key fallback without doubling the request. |
| 873 | func TestMissingReasoningRecoveryRateLimitsAcrossProcesses(t *testing.T) { |
| 874 | stateDir := t.TempDir() |
| 875 | mp := testutil.NewMock("deepseek-proxy", |
| 876 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 877 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1r", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 878 | testutil.Turn{Text: "done"}, |
| 879 | ) |
| 880 | sink1 := &recordSink{} |
| 881 | a1 := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, sink1) |
| 882 | if err := a1.Run(context.Background(), "go"); err != nil { |
| 883 | t.Fatalf("first Run: %v", err) |
| 884 | } |
| 885 | if got := sink1.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted); got != 1 { |
| 886 | t.Fatalf("first process recovery retries = %d, want 1", got) |
| 887 | } |
| 888 | |
| 889 | mp2 := testutil.NewMock("deepseek-proxy", |
| 890 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 891 | testutil.Turn{Text: "done again"}, |
| 892 | ) |
| 893 | sink2 := &recordSink{} |
| 894 | a2 := New(toolCallReasoningRequiredProvider{mp2}, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, sink2) |
| 895 | if err := a2.Run(context.Background(), "go"); err != nil { |
| 896 | t.Fatalf("second process Run: %v", err) |
| 897 | } |
| 898 | if got := sink2.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted); got != 0 { |
| 899 | t.Fatalf("fresh process recovery retries = %d, want 0", got) |
| 900 | } |
| 901 | if got := sink2.recoveryCount(event.ProtocolRecoveryMissingReasoningRetrySuppressed); got != 1 { |
| 902 | t.Fatalf("fresh process suppressed retries = %d, want 1", got) |
| 903 | } |
| 904 | } |
| 905 | |
| 906 | func TestMissingReasoningRecoverySeparatesProviderConfigurations(t *testing.T) { |
| 907 | stateDir := t.TempDir() |
| 908 | retryCount := func(identity string) int { |
| 909 | mp := testutil.NewMock("deepseek-proxy", |
| 910 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 911 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1r", Name: "echo", Arguments: `{"text":"hi"}`}}}, |
| 912 | testutil.Turn{Text: "done"}, |
| 913 | ) |
| 914 | sink := &recordSink{} |
| 915 | a := New(configuredToolCallReasoningProvider{MockProvider: mp, identity: identity}, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, sink) |
| 916 | if err := a.Run(context.Background(), "go"); err != nil { |
| 917 | t.Fatalf("Run(%q): %v", identity, err) |
| 918 | } |
| 919 | return sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted) |
| 920 | } |
| 921 | if got := retryCount("openai\x00endpoint-a\x00deepseek-v4-pro"); got != 1 { |
| 922 | t.Fatalf("first configuration retries = %d, want 1", got) |
| 923 | } |
| 924 | if got := retryCount("openai\x00endpoint-a\x00deepseek-v4-pro"); got != 0 { |
| 925 | t.Fatalf("same configuration retries = %d, want 0", got) |
| 926 | } |
| 927 | if got := retryCount("openai\x00endpoint-b\x00deepseek-v4-pro"); got != 1 { |
| 928 | t.Fatalf("changed endpoint retries = %d, want 1", got) |
| 929 | } |
| 930 | if got := retryCount("openai\x00endpoint-a\x00deepseek-v4-flash"); got != 1 { |
| 931 | t.Fatalf("changed model retries = %d, want 1", got) |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | func TestThreeHealthyToolCallReasoningTurnsRearmFutureRegression(t *testing.T) { |
| 936 | stateDir := t.TempDir() |
| 937 | run := func(turns ...testutil.Turn) int { |
| 938 | mp := testutil.NewMock("deepseek-proxy", turns...) |
| 939 | sink := &recordSink{} |
| 940 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, sink) |
| 941 | if err := a.Run(context.Background(), "go"); err != nil { |
| 942 | t.Fatalf("Run: %v", err) |
| 943 | } |
| 944 | return sink.recoveryCount(event.ProtocolRecoveryMissingReasoningRetryAttempted) |
| 945 | } |
| 946 | missing := testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}}} |
| 947 | healthy := testutil.Turn{Reasoning: "call echo", ToolCalls: []provider.ToolCall{{ID: "c2", Name: "echo", Arguments: `{"text":"hi"}`}}} |
| 948 | if got := run(missing, missing, testutil.Turn{Text: "done"}); got != 1 { |
| 949 | t.Fatalf("first incident retries = %d, want 1", got) |
| 950 | } |
| 951 | for healthyTurn := 1; healthyTurn <= missingReasoningHealthyResolveStreak; healthyTurn++ { |
| 952 | if got := run(healthy, testutil.Turn{Text: "done"}); got != 0 { |
| 953 | t.Fatalf("healthy turn %d retries = %d, want 0", healthyTurn, got) |
| 954 | } |
| 955 | } |
| 956 | if got := run(missing, missing, testutil.Turn{Text: "done"}); got != 1 { |
| 957 | t.Fatalf("post-recovery regression retries = %d, want 1", got) |
| 958 | } |
| 959 | } |
| 960 | |
| 961 | func TestHealthyToolCallReasoningStreakWorksWithinOneAgentAndResetsOnMissing(t *testing.T) { |
| 962 | stateDir := t.TempDir() |
| 963 | prov := toolCallReasoningRequiredProvider{testutil.NewMock("deepseek-proxy")} |
| 964 | a := New(prov, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, event.Discard) |
| 965 | calls := []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}} |
| 966 | |
| 967 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || !retry { |
| 968 | t.Fatalf("initial observation = missing:%v retry:%v, want true/true", missing, retry) |
| 969 | } |
| 970 | for healthy := 1; healthy < missingReasoningHealthyResolveStreak; healthy++ { |
| 971 | a.observeMissingToolCallReasoning(calls, "healthy reasoning") |
| 972 | } |
| 973 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || retry { |
| 974 | t.Fatalf("missing reset = missing:%v retry:%v, want true/false", missing, retry) |
| 975 | } |
| 976 | for healthy := 1; healthy <= missingReasoningHealthyResolveStreak; healthy++ { |
| 977 | a.observeMissingToolCallReasoning(calls, "healthy reasoning") |
| 978 | } |
| 979 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || !retry { |
| 980 | t.Fatalf("post-recovery observation = missing:%v retry:%v, want true/true", missing, retry) |
| 981 | } |
| 982 | } |
| 983 | |
| 984 | func TestMissingReasoningRecoveryIOFailureStillSuppressesLocally(t *testing.T) { |
| 985 | statePath := filepath.Join(t.TempDir(), "not-a-directory") |
| 986 | if err := os.WriteFile(statePath, []byte("occupied"), 0o600); err != nil { |
| 987 | t.Fatal(err) |
| 988 | } |
| 989 | prov := toolCallReasoningRequiredProvider{testutil.NewMock("deepseek-proxy")} |
| 990 | a := New(prov, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: statePath}, event.Discard) |
| 991 | calls := []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}} |
| 992 | |
| 993 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || !retry { |
| 994 | t.Fatalf("initial observation = missing:%v retry:%v, want true/true", missing, retry) |
| 995 | } |
| 996 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || retry { |
| 997 | t.Fatalf("repeated observation = missing:%v retry:%v, want true/false", missing, retry) |
| 998 | } |
| 999 | } |
| 1000 | |
| 1001 | func TestHealthyToolCallReasoningRetriesTransientStateWriteFailure(t *testing.T) { |
| 1002 | if runtime.GOOS == "windows" { |
| 1003 | t.Skip("chmod permissions are not portable to Windows") |
| 1004 | } |
| 1005 | stateDir := t.TempDir() |
| 1006 | prov := toolCallReasoningRequiredProvider{testutil.NewMock("deepseek-proxy")} |
| 1007 | a := New(prov, echoRegistry(), NewSession(""), Options{MissingReasoningWarnStateDir: stateDir}, event.Discard) |
| 1008 | calls := []provider.ToolCall{{ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`}} |
| 1009 | |
| 1010 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || !retry { |
| 1011 | t.Fatalf("initial observation = missing:%v retry:%v, want true/true", missing, retry) |
| 1012 | } |
| 1013 | if err := os.Chmod(stateDir, 0o500); err != nil { |
| 1014 | t.Fatal(err) |
| 1015 | } |
| 1016 | permissionsRestored := false |
| 1017 | defer func() { |
| 1018 | if !permissionsRestored { |
| 1019 | _ = os.Chmod(stateDir, 0o700) |
| 1020 | } |
| 1021 | }() |
| 1022 | if missing, retry := a.observeMissingToolCallReasoning(calls, "healthy reasoning"); missing || retry { |
| 1023 | t.Fatalf("healthy observation = missing:%v retry:%v, want false/false", missing, retry) |
| 1024 | } |
| 1025 | if err := os.Chmod(stateDir, 0o700); err != nil { |
| 1026 | t.Fatal(err) |
| 1027 | } |
| 1028 | permissionsRestored = true |
| 1029 | for healthy := 0; healthy < missingReasoningHealthyResolveStreak-1; healthy++ { |
| 1030 | if missing, retry := a.observeMissingToolCallReasoning(calls, "healthy reasoning"); missing || retry { |
| 1031 | t.Fatalf("healthy recovery observation %d = missing:%v retry:%v, want false/false", healthy+1, missing, retry) |
| 1032 | } |
| 1033 | } |
| 1034 | |
| 1035 | if missing, retry := a.observeMissingToolCallReasoning(calls, ""); !missing || !retry { |
| 1036 | t.Fatalf("post-recovery observation = missing:%v retry:%v, want true/true", missing, retry) |
| 1037 | } |
| 1038 | } |
| 1039 | |
| 1040 | func TestRunPreservesOriginalRequiredToolCallReasoningAcrossHook(t *testing.T) { |
| 1041 | mp := testutil.NewMock("deepseek-proxy", |
| 1042 | testutil.Turn{ |
| 1043 | Reasoning: "original reasoning", |
| 1044 | ToolCalls: []provider.ToolCall{{ |
| 1045 | ID: "c1", Name: "echo", Arguments: `{"text":"hi"}`, |
| 1046 | }}, |
| 1047 | }, |
| 1048 | testutil.Turn{Text: "done"}, |
| 1049 | ) |
| 1050 | h := &stubHooks{hasPostLLM: true, postLLMOut: "translated display"} |
| 1051 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{Hooks: h}, event.Discard) |
| 1052 | |
| 1053 | if err := a.Run(context.Background(), "go"); err != nil { |
| 1054 | t.Fatalf("Run: %v", err) |
| 1055 | } |
| 1056 | reqs := mp.Requests() |
| 1057 | if len(reqs) != 2 { |
| 1058 | t.Fatalf("provider calls = %d, want 2", len(reqs)) |
| 1059 | } |
| 1060 | var toolCallAssistant provider.Message |
| 1061 | for _, m := range reqs[1].Messages { |
| 1062 | if m.Role == provider.RoleAssistant && len(m.ToolCalls) > 0 { |
| 1063 | toolCallAssistant = m |
| 1064 | break |
| 1065 | } |
| 1066 | } |
| 1067 | if toolCallAssistant.ReasoningContent != "original reasoning" { |
| 1068 | t.Fatalf("tool-call reasoning = %q, want original provider reasoning", toolCallAssistant.ReasoningContent) |
| 1069 | } |
| 1070 | if toolCallAssistant.ReasoningContent == "translated display" { |
| 1071 | t.Fatal("translated display text leaked into provider-visible tool-call reasoning") |
| 1072 | } |
| 1073 | } |
| 1074 | |
| 1075 | func TestRunStoresTransformedNonToolReasoningForToolCallOnlyProvider(t *testing.T) { |
| 1076 | mp := testutil.NewMock("deepseek-proxy", testutil.Turn{ |
| 1077 | Reasoning: "original reasoning", |
| 1078 | Text: "done", |
| 1079 | }) |
| 1080 | h := &stubHooks{hasPostLLM: true, postLLMOut: "translated display"} |
| 1081 | a := New(toolCallReasoningRequiredProvider{mp}, echoRegistry(), NewSession(""), Options{Hooks: h}, event.Discard) |
| 1082 | |
| 1083 | if err := a.Run(context.Background(), "go"); err != nil { |
| 1084 | t.Fatalf("Run: %v", err) |
| 1085 | } |
| 1086 | if got := assistantReasoning(a.session.Messages); got != "translated display" { |
| 1087 | t.Fatalf("stored non-tool reasoning = %q, want transformed display text", got) |
| 1088 | } |
| 1089 | } |
| 1090 |