| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | |
| 9 | "reasonix/internal/event" |
| 10 | "reasonix/internal/provider" |
| 11 | "reasonix/internal/tool" |
| 12 | ) |
| 13 | |
| 14 | func TestRunAcceptsReasoningOnlyFinalAnswer(t *testing.T) { |
| 15 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 16 | { |
| 17 | {Type: provider.ChunkReasoning, Text: "I should answer the user."}, |
| 18 | {Type: provider.ChunkDone}, |
| 19 | }, |
| 20 | }} |
| 21 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{}, event.Discard) |
| 22 | |
| 23 | if err := a.Run(context.Background(), "answer me"); err != nil { |
| 24 | t.Fatalf("Run: %v", err) |
| 25 | } |
| 26 | if prov.call != 1 { |
| 27 | t.Fatalf("provider calls = %d, want one clean reasoning-only completion", prov.call) |
| 28 | } |
| 29 | if got := lastAssistantContent(a.sess.conversation); got != "" { |
| 30 | t.Fatalf("last assistant content = %q, want empty content beside reasoning", got) |
| 31 | } |
| 32 | if sessionHasUserMessageContaining(a.sess.conversation, "visible answer") { |
| 33 | t.Fatal("must not inject a synthetic visible-answer retry") |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | func TestRunPrefixesReasoningLanguageOnReasoningOnlyCompletion(t *testing.T) { |
| 38 | prov := &mockProvider{name: "p", streams: [][]provider.Chunk{ |
| 39 | { |
| 40 | {Type: provider.ChunkReasoning, Text: "I should answer the user."}, |
| 41 | {Type: provider.ChunkDone}, |
| 42 | }, |
| 43 | }} |
| 44 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{ReasoningLanguage: "zh"}, event.Discard) |
| 45 | |
| 46 | if err := a.Run(context.Background(), "answer me"); err != nil { |
| 47 | t.Fatalf("Run: %v", err) |
| 48 | } |
| 49 | if len(prov.requests) != 1 { |
| 50 | t.Fatalf("provider requests = %d, want 1", len(prov.requests)) |
| 51 | } |
| 52 | got := lastUser(prov.requests[0]) |
| 53 | if !strings.HasPrefix(got, "<reasoning-language>") || !strings.Contains(got, "简体中文") { |
| 54 | t.Fatalf("last user = %q, want reasoning-language prefix", got) |
| 55 | } |
| 56 | if strings.Contains(got, "visible answer") { |
| 57 | t.Fatalf("reasoning-only completion must not receive a synthetic visible-answer retry: %q", got) |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | func TestRunRequiresVisibleFinalOnlyWhenExplicitlyRequested(t *testing.T) { |
| 62 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 63 | {{Type: provider.ChunkReasoning, Text: "thinking 1"}, {Type: provider.ChunkDone}}, |
| 64 | {{Type: provider.ChunkReasoning, Text: "thinking 2"}, {Type: provider.ChunkDone}}, |
| 65 | {{Type: provider.ChunkReasoning, Text: "thinking 3"}, {Type: provider.ChunkDone}}, |
| 66 | }} |
| 67 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{RequireVisibleFinal: true}, event.Discard) |
| 68 | |
| 69 | err := a.Run(context.Background(), "answer me") |
| 70 | if err == nil { |
| 71 | t.Fatal("expected repeated empty final answers to stop the run") |
| 72 | } |
| 73 | if !strings.Contains(err.Error(), "visible final answer") { |
| 74 | t.Fatalf("error = %v, want visible final answer", err) |
| 75 | } |
| 76 | if prov.call != 3 { |
| 77 | t.Fatalf("provider calls = %d, want three empty-answer attempts", prov.call) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func TestEmptyResponseLeavesRetryToUser(t *testing.T) { |
| 82 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 83 | {{Type: provider.ChunkDone}}, |
| 84 | {{Type: provider.ChunkText, Text: "visible reply"}, {Type: provider.ChunkDone}}, |
| 85 | }} |
| 86 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{}, event.Discard) |
| 87 | if err := a.Run(t.Context(), "answer me"); !errors.Is(err, provider.ErrEmptyResponse) { |
| 88 | t.Fatalf("err=%v", err) |
| 89 | } |
| 90 | if prov.call != 1 || lastAssistantContent(a.Session()) != "" { |
| 91 | t.Fatalf("calls=%d", prov.call) |
| 92 | } |
| 93 | if err := a.Run(t.Context(), "try again"); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | if prov.call != 2 || lastAssistantContent(a.Session()) != "visible reply" { |
| 97 | t.Fatalf("manual retry calls=%d", prov.call) |
| 98 | } |
| 99 | if sessionHasUserMessageContaining(a.Session(), "visible answer") { |
| 100 | t.Fatal("injected synthetic retry prompt") |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | func TestRunStopsOnZeroContentWithoutCommittingEmptyMessages(t *testing.T) { |
| 105 | turns := make([][]provider.Chunk, maxSamplingAttempts) |
| 106 | for i := range turns { |
| 107 | turns[i] = []provider.Chunk{{Type: provider.ChunkDone}} |
| 108 | } |
| 109 | prov := &scriptedProvider{name: "p", turns: turns} |
| 110 | sink := &recordSink{} |
| 111 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{}, sink) |
| 112 | |
| 113 | err := a.Run(context.Background(), "answer me") |
| 114 | if !errors.Is(err, provider.ErrEmptyResponse) { |
| 115 | t.Fatalf("Run error = %v, want ErrEmptyResponse", err) |
| 116 | } |
| 117 | if prov.call != 1 { |
| 118 | t.Fatalf("provider calls = %d, want 1", prov.call) |
| 119 | } |
| 120 | for _, message := range a.sess.conversation.Messages { |
| 121 | if message.Role == provider.RoleAssistant { |
| 122 | t.Fatalf("empty attempt committed assistant message: %+v", message) |
| 123 | } |
| 124 | if message.Role == provider.RoleUser && strings.Contains(message.Content, "visible answer") { |
| 125 | t.Fatalf("empty attempt injected synthetic user prompt: %q", message.Content) |
| 126 | } |
| 127 | } |
| 128 | retries := sink.kinds(event.Retrying) |
| 129 | if len(retries) != 0 { |
| 130 | t.Fatalf("unexpected retry events: %+v", retries) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | func lastAssistantContent(s *Session) string { |
| 135 | var out string |
| 136 | for _, m := range s.Messages { |
| 137 | if m.Role == provider.RoleAssistant { |
| 138 | out = m.Content |
| 139 | } |
| 140 | } |
| 141 | return out |
| 142 | } |
| 143 | |
| 144 | // deepseekThinkingProvider marks a scripted provider as DeepSeek thinking mode |
| 145 | // (provider.ToolCallReasoningPolicy) — the scope within which a reasoning-only |
| 146 | // finish_reason="stop" turn is accepted as a final answer. |
| 147 | type deepseekThinkingProvider struct{ *scriptedProvider } |
| 148 | |
| 149 | func (deepseekThinkingProvider) RequiresToolCallReasoning() bool { return true } |
| 150 | |
| 151 | func TestRunAcceptsReasoningOnlyFinalWhenModelStopped(t *testing.T) { |
| 152 | // DeepSeek thinking mode streams a long reasoning_content and then |
| 153 | // finishes with finish_reason="stop" but an empty content block. The |
| 154 | // model has explicitly signalled completion and its reasoning was |
| 155 | // streamed to the user, so the host must accept the turn instead of |
| 156 | // retrying and forcing another expensive thinking round. |
| 157 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 158 | { |
| 159 | {Type: provider.ChunkReasoning, Text: "The user asked a simple question; I have reasoned through it and the answer is ready."}, |
| 160 | {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "stop", TotalTokens: 10}}, |
| 161 | {Type: provider.ChunkDone}, |
| 162 | }, |
| 163 | }} |
| 164 | a := New(deepseekThinkingProvider{prov}, tool.NewRegistry(), NewSession(""), Options{}, event.Discard) |
| 165 | |
| 166 | if err := a.Run(context.Background(), "answer me"); err != nil { |
| 167 | t.Fatalf("Run: %v", err) |
| 168 | } |
| 169 | if prov.call != 1 { |
| 170 | t.Fatalf("provider calls = %d, want 1 (model signalled stop; no retry)", prov.call) |
| 171 | } |
| 172 | if sessionHasUserMessageContaining(a.sess.conversation, "visible answer") { |
| 173 | t.Fatal("must not inject a synthetic visible-answer retry when the model signalled stop") |
| 174 | } |
| 175 | if got := lastAssistantContent(a.sess.conversation); got != "" { |
| 176 | t.Fatalf("last assistant content = %q, want empty (answer lived in reasoning)", got) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | func TestRunAcceptsReasoningOnlyStopWithoutDeepSeekPolicy(t *testing.T) { |
| 181 | // Same chunk sequence as the accept test, but the provider does not |
| 182 | // declare DeepSeek thinking mode (ToolCallReasoningPolicy). The accept |
| 183 | // path must stay scoped to DeepSeek: local <think>-tag models keep the |
| 184 | // retry safety net that often recovers a visible answer on the second |
| 185 | // attempt, and a gateway that mislabels truncation as "stop" must not |
| 186 | // have a degenerate turn committed as the final answer. |
| 187 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 188 | { |
| 189 | {Type: provider.ChunkReasoning, Text: "thinking only, nothing visible"}, |
| 190 | {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "stop", TotalTokens: 10}}, |
| 191 | {Type: provider.ChunkDone}, |
| 192 | }, |
| 193 | }} |
| 194 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{}, event.Discard) |
| 195 | |
| 196 | if err := a.Run(context.Background(), "answer me"); err != nil { |
| 197 | t.Fatalf("Run: %v", err) |
| 198 | } |
| 199 | if prov.call != 1 { |
| 200 | t.Fatalf("provider calls = %d, want 1 for a reasoning-only clean terminal", prov.call) |
| 201 | } |
| 202 | if sessionHasUserMessageContaining(a.sess.conversation, "visible answer") { |
| 203 | t.Fatal("must not inject a synthetic visible-answer retry") |
| 204 | } |
| 205 | if got := lastAssistantContent(a.sess.conversation); got != "" { |
| 206 | t.Fatalf("last assistant content = %q, want empty content beside reasoning", got) |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | func BenchmarkHasVisibleFinalAnswer(b *testing.B) { |
| 211 | cases := []struct { |
| 212 | name string |
| 213 | text string |
| 214 | }{ |
| 215 | {"normal", "visible reply"}, |
| 216 | {"leading-space", strings.Repeat(" ", 256) + "visible reply"}, |
| 217 | {"all-space", strings.Repeat(" \n\t", 256)}, |
| 218 | } |
| 219 | for _, tc := range cases { |
| 220 | b.Run(tc.name, func(b *testing.B) { |
| 221 | b.ReportAllocs() |
| 222 | var got bool |
| 223 | for range b.N { |
| 224 | got = hasVisibleFinalAnswer(tc.text) |
| 225 | } |
| 226 | _ = got |
| 227 | }) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | func reasoningOnlyStop(text string) []provider.Chunk { |
| 232 | return []provider.Chunk{ |
| 233 | {Type: provider.ChunkReasoning, Text: text}, |
| 234 | {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "stop", TotalTokens: 10}}, |
| 235 | {Type: provider.ChunkDone}, |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | func countUserMessagesContaining(s *Session, needle string) int { |
| 240 | n := 0 |
| 241 | for _, message := range s.Messages { |
| 242 | if message.Role == provider.RoleUser && strings.Contains(message.Content, needle) { |
| 243 | n++ |
| 244 | } |
| 245 | } |
| 246 | return n |
| 247 | } |
| 248 | |
| 249 | func TestRunRetriesReasoningOnlyStopAfterToolRoundOnce(t *testing.T) { |
| 250 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 251 | {{Type: provider.ChunkReasoning, Text: "read the file"}, toolCallChunk("call-1", "read_file", `{}`), {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "tool_calls", TotalTokens: 10}}, {Type: provider.ChunkDone}}, |
| 252 | reasoningOnlyStop("I have the file contents; the answer is ready."), |
| 253 | {{Type: provider.ChunkText, Text: "The file says contents."}, {Type: provider.ChunkDone}}, |
| 254 | }} |
| 255 | reg := tool.NewRegistry() |
| 256 | reg.Add(fakeReadFileTool{}) |
| 257 | a := New(deepseekThinkingProvider{prov}, reg, NewSession(""), Options{}, event.Discard) |
| 258 | |
| 259 | if err := a.Run(context.Background(), "what does the file say"); err != nil { |
| 260 | t.Fatalf("Run: %v", err) |
| 261 | } |
| 262 | if prov.call != 3 { |
| 263 | t.Fatalf("provider calls = %d, want tool round + reasoning-only stop + one retry", prov.call) |
| 264 | } |
| 265 | if got := countUserMessagesContaining(a.sess.conversation, "visible answer"); got != 1 { |
| 266 | t.Fatalf("visible-answer retries = %d, want exactly 1", got) |
| 267 | } |
| 268 | if got := lastAssistantContent(a.sess.conversation); got != "The file says contents." { |
| 269 | t.Fatalf("last assistant content = %q, want the visible synthesis", got) |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | func TestRunAcceptsSecondReasoningOnlyStopAfterToolRound(t *testing.T) { |
| 274 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 275 | {{Type: provider.ChunkReasoning, Text: "read the file"}, toolCallChunk("call-1", "read_file", `{}`), {Type: provider.ChunkUsage, Usage: &provider.Usage{FinishReason: "tool_calls", TotalTokens: 10}}, {Type: provider.ChunkDone}}, |
| 276 | reasoningOnlyStop("done thinking"), |
| 277 | reasoningOnlyStop("still only thinking"), |
| 278 | }} |
| 279 | reg := tool.NewRegistry() |
| 280 | reg.Add(fakeReadFileTool{}) |
| 281 | a := New(deepseekThinkingProvider{prov}, reg, NewSession(""), Options{}, event.Discard) |
| 282 | |
| 283 | if err := a.Run(context.Background(), "what does the file say"); err != nil { |
| 284 | t.Fatalf("Run: %v", err) |
| 285 | } |
| 286 | if prov.call != 3 { |
| 287 | t.Fatalf("provider calls = %d, want a single bounded retry", prov.call) |
| 288 | } |
| 289 | if got := countUserMessagesContaining(a.sess.conversation, "visible answer"); got != 1 { |
| 290 | t.Fatalf("visible-answer retries = %d, want exactly 1", got) |
| 291 | } |
| 292 | } |
| 293 |