| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "reasonix/internal/event" |
| 11 | ) |
| 12 | |
| 13 | // The shape a reasoning-only turn takes on the Message event: Reasoning holds |
| 14 | // the whole turn, Text is empty (internal/event/event.go, Message). |
| 15 | // |
| 16 | // Observed against a local provider, where the turn's only output was a |
| 17 | // malformed tool call the model wrote as prose instead of emitting as a tool |
| 18 | // call. Headlessly that reached the caller as `result: ""` with |
| 19 | // `subtype: "success"` and exit 0. |
| 20 | var errRunFailedFixture = errors.New("provider failed") |
| 21 | |
| 22 | const reasoningOnlyTurn = `<|tool_call>call:bash{command:<|"|>mkdir -p docker<|"|>}<tool_call|>` |
| 23 | |
| 24 | func TestTextOutputFallsBackToReasoningWhenNoVisibleText(t *testing.T) { |
| 25 | var out bytes.Buffer |
| 26 | sink := newRunOutputSink(&out, runOutputText) |
| 27 | sink.Emit(event.Event{Kind: event.Reasoning, Text: reasoningOnlyTurn}) |
| 28 | sink.Emit(event.Event{Kind: event.Message, Reasoning: reasoningOnlyTurn}) |
| 29 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | if got := out.String(); got != reasoningOnlyTurn+"\n" { |
| 33 | t.Fatalf("text output = %q, want the reasoning-only turn rather than nothing", got) |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | func TestJSONResultCarriesReasoningAndSaysSo(t *testing.T) { |
| 38 | var out bytes.Buffer |
| 39 | sink := newRunOutputSink(&out, runOutputJSON) |
| 40 | sink.Emit(event.Event{Kind: event.Message, Reasoning: reasoningOnlyTurn}) |
| 41 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | var got runResult |
| 45 | if err := json.Unmarshal(out.Bytes(), &got); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | if got.Result != reasoningOnlyTurn { |
| 49 | t.Fatalf("result = %q, want the reasoning-only turn", got.Result) |
| 50 | } |
| 51 | if !got.ResultFromReasoning { |
| 52 | t.Fatal("result_from_reasoning = false; a caller cannot tell this from visible text the model never emitted") |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | // Visible text still wins, and the flag stays off, so nothing about an ordinary |
| 57 | // answer changes - including the case where a thinking model emits both. |
| 58 | func TestVisibleTextIsUnchangedByTheFallback(t *testing.T) { |
| 59 | for _, tc := range []struct { |
| 60 | name string |
| 61 | event event.Event |
| 62 | }{ |
| 63 | {"text only", event.Event{Kind: event.Message, Text: "final answer"}}, |
| 64 | {"text beside reasoning", event.Event{Kind: event.Message, Text: "final answer", Reasoning: "thinking"}}, |
| 65 | } { |
| 66 | t.Run(tc.name, func(t *testing.T) { |
| 67 | var out bytes.Buffer |
| 68 | sink := newRunOutputSink(&out, runOutputJSON) |
| 69 | sink.Emit(tc.event) |
| 70 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | var got runResult |
| 74 | if err := json.Unmarshal(out.Bytes(), &got); err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | if got.Result != "final answer" || got.ResultFromReasoning { |
| 78 | t.Fatalf("result = %q, from_reasoning = %v", got.Result, got.ResultFromReasoning) |
| 79 | } |
| 80 | if bytes.Contains(out.Bytes(), []byte("result_from_reasoning")) { |
| 81 | t.Fatalf("omitempty dropped: %s", out.Bytes()) |
| 82 | } |
| 83 | }) |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | // A later turn that answers normally must clear the flag the earlier one set, |
| 88 | // or a multi-turn run reports the wrong provenance for its final answer. |
| 89 | func TestLaterVisibleTurnClearsTheReasoningProvenance(t *testing.T) { |
| 90 | var out bytes.Buffer |
| 91 | sink := newRunOutputSink(&out, runOutputJSON) |
| 92 | sink.Emit(event.Event{Kind: event.Message, Reasoning: reasoningOnlyTurn}) |
| 93 | sink.Emit(event.Event{Kind: event.Message, Text: "final answer"}) |
| 94 | if err := sink.Finalize("session", time.Now(), nil); err != nil { |
| 95 | t.Fatal(err) |
| 96 | } |
| 97 | var got runResult |
| 98 | if err := json.Unmarshal(out.Bytes(), &got); err != nil { |
| 99 | t.Fatal(err) |
| 100 | } |
| 101 | if got.Result != "final answer" || got.ResultFromReasoning { |
| 102 | t.Fatalf("result = %q, from_reasoning = %v", got.Result, got.ResultFromReasoning) |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // An error result is the run's own message, never the model's reasoning. |
| 107 | func TestErrorResultIsNotMarkedAsReasoning(t *testing.T) { |
| 108 | var out bytes.Buffer |
| 109 | sink := newRunOutputSink(&out, runOutputJSON) |
| 110 | if err := sink.Finalize("session", time.Now(), errRunFailedFixture); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | var got runResult |
| 114 | if err := json.Unmarshal(out.Bytes(), &got); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | if got.Result != errRunFailedFixture.Error() || got.ResultFromReasoning { |
| 118 | t.Fatalf("result = %q, from_reasoning = %v", got.Result, got.ResultFromReasoning) |
| 119 | } |
| 120 | } |
| 121 |