| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "reasonix/internal/agent/testutil" |
| 11 | "reasonix/internal/event" |
| 12 | "reasonix/internal/provider" |
| 13 | "reasonix/internal/tool" |
| 14 | |
| 15 | _ "reasonix/internal/tool/builtin" |
| 16 | ) |
| 17 | |
| 18 | type stubBash struct{} |
| 19 | |
| 20 | func (stubBash) Name() string { return "bash" } |
| 21 | func (stubBash) Description() string { return "stub bash" } |
| 22 | func (stubBash) ReadOnly() bool { return false } |
| 23 | func (stubBash) Schema() json.RawMessage { |
| 24 | return json.RawMessage(`{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}`) |
| 25 | } |
| 26 | func (stubBash) Execute(context.Context, json.RawMessage) (string, error) { return "ok", nil } |
| 27 | |
| 28 | type stubWrite struct{} |
| 29 | |
| 30 | func (stubWrite) Name() string { return "write_file" } |
| 31 | func (stubWrite) Description() string { return "stub write" } |
| 32 | func (stubWrite) ReadOnly() bool { return false } |
| 33 | func (stubWrite) Schema() json.RawMessage { |
| 34 | return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}`) |
| 35 | } |
| 36 | func (stubWrite) Execute(context.Context, json.RawMessage) (string, error) { return "wrote", nil } |
| 37 | |
| 38 | // evidenceRegistry wires the real complete_step + todo_write builtins (the |
| 39 | // enforcement surface under test) alongside bash/write stubs that emit real |
| 40 | // receipts without touching the host — so the whole turn loop, ledger, gate, |
| 41 | // and host-advance run end to end. |
| 42 | func evidenceRegistry() *tool.Registry { |
| 43 | reg := tool.NewRegistry() |
| 44 | for _, bt := range tool.Builtins() { |
| 45 | if bt.Name() == "complete_step" || bt.Name() == "todo_write" { |
| 46 | reg.Add(bt) |
| 47 | } |
| 48 | } |
| 49 | reg.Add(stubBash{}) |
| 50 | reg.Add(stubWrite{}) |
| 51 | return reg |
| 52 | } |
| 53 | |
| 54 | func hostAdvances(sink *recordSink) int { |
| 55 | n := 0 |
| 56 | for _, e := range sink.kinds(event.ToolResult) { |
| 57 | if strings.HasPrefix(e.Tool.ID, "host-advance-") { |
| 58 | n++ |
| 59 | } |
| 60 | } |
| 61 | return n |
| 62 | } |
| 63 | |
| 64 | func readinessBlocked(err error) bool { |
| 65 | var readinessErr *FinalReadinessError |
| 66 | return errors.As(err, &readinessErr) |
| 67 | } |
| 68 | |
| 69 | // sessionContains reports whether any message body holds sub — used to assert a |
| 70 | // tool's own result text (a complete_step "signed off" or its rejection reason), |
| 71 | // since Run returns nil whether or not a tool call was rejected mid-turn. |
| 72 | func sessionContains(a *Agent, sub string) bool { |
| 73 | for _, m := range a.Session().Messages { |
| 74 | if strings.Contains(m.Content, sub) { |
| 75 | return true |
| 76 | } |
| 77 | } |
| 78 | return false |
| 79 | } |
| 80 | |
| 81 | // Serial plan: the model establishes the list once, then signs off each step |
| 82 | // with complete_step — the host advances the list (no per-step todo_write, so |
| 83 | // the #3909 batch-completion failure can't arise) and a cited command tolerates |
| 84 | // a cd-prefix drift. The final answer is allowed once every step is signed off. |
| 85 | func TestE2ESerialPlanHostAdvancesAndAllowsFinalAnswer(t *testing.T) { |
| 86 | mp := testutil.NewMock("m", |
| 87 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "t0", Name: "todo_write", |
| 88 | Arguments: `{"todos":[{"content":"test","status":"in_progress"},{"content":"vet","status":"pending"}]}`}}}, |
| 89 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "b1", Name: "bash", |
| 90 | Arguments: `{"command":"cd /repo && go test ./..."}`}}}, |
| 91 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 92 | Arguments: `{"step":"test","result":"tests pass","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`}}}, |
| 93 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "b2", Name: "bash", |
| 94 | Arguments: `{"command":"go vet ./..."}`}}}, |
| 95 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2", Name: "complete_step", |
| 96 | Arguments: `{"step":"vet","result":"vet passes","evidence":[{"kind":"verification","summary":"vet passes","command":"go vet ./..."}]}`}}}, |
| 97 | testutil.Turn{Text: "all done"}, |
| 98 | ) |
| 99 | sink := &recordSink{} |
| 100 | a := New(mp, evidenceRegistry(), NewSession("sys"), Options{}, sink) |
| 101 | |
| 102 | runErr := a.Run(context.Background(), "implement the plan") |
| 103 | if runErr != nil { |
| 104 | t.Fatalf("final answer blocked despite host-advanced completions: %v", runErr) |
| 105 | } |
| 106 | for i, td := range a.todoState { |
| 107 | if canonicalTodoStatus(td.Status) != "completed" { |
| 108 | t.Fatalf("canonical todo %d (%q) = %s, want completed", i+1, td.Content, td.Status) |
| 109 | } |
| 110 | } |
| 111 | if n := hostAdvances(sink); n < 2 { |
| 112 | t.Fatalf("host advanced %d times, want >=2 (one per complete_step)", n) |
| 113 | } |
| 114 | if readinessBlocked(runErr) { |
| 115 | t.Fatal("a correctly signed-off plan should not trip the readiness gate") |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // A command cited with a different string than it ran under (#2917: the model |
| 120 | // drops the cd-prefix) is still accepted via segment matching, in-turn. |
| 121 | func TestE2ECommandDriftAcceptedInTurn(t *testing.T) { |
| 122 | mp := testutil.NewMock("m", |
| 123 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "b1", Name: "bash", |
| 124 | Arguments: `{"command":"cd /Users/x/repo && git merge upstream/main --ff-only"}`}}}, |
| 125 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 126 | Arguments: `{"step":"sync","result":"synced","evidence":[{"kind":"verification","summary":"fast-forwarded","command":"git merge upstream/main --ff-only"}]}`}}}, |
| 127 | testutil.Turn{Text: "synced"}, |
| 128 | ) |
| 129 | a := New(mp, evidenceRegistry(), NewSession("sys"), Options{}, event.Discard) |
| 130 | if err := a.Run(context.Background(), "sync the branch"); err != nil { |
| 131 | t.Fatalf("Run: %v", err) |
| 132 | } |
| 133 | if !sessionContains(a, "signed off") { |
| 134 | t.Fatal("cd-prefixed command drift rejected a real verification") |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | // Cross-turn: a prior turn left an unfinished plan in the canonical state. A new |
| 139 | // turn that does work and prematurely claims "all done" without re-asserting the |
| 140 | // todos is blocked by the canonical fallback, then clears once both steps are |
| 141 | // actually signed off (host-advanced) — the loop that #2917 could not close. |
| 142 | func TestE2ECrossTurnCanonicalGateBlocksThenClears(t *testing.T) { |
| 143 | sess := NewSession("sys") |
| 144 | sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ |
| 145 | ID: "t0", Name: "todo_write", |
| 146 | Arguments: `{"todos":[{"content":"alpha","status":"in_progress"},{"content":"beta","status":"pending"}]}`}}}) |
| 147 | sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: "t0", Name: "todo_write", Content: "Todos updated"}) |
| 148 | |
| 149 | // The premature "all done" ends the first Run immediately (no readiness |
| 150 | // retries). A follow-up turn signs the steps off with complete_step (the |
| 151 | // cited diff paths are proven from the session history) and clears the gate. |
| 152 | mp := testutil.NewMock("m", |
| 153 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "w1", Name: "write_file", Arguments: `{"path":"alpha.go"}`}}}, |
| 154 | testutil.Turn{Text: "all done"}, |
| 155 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 156 | Arguments: `{"step":"alpha","result":"done","evidence":[{"kind":"diff","summary":"edited","paths":["alpha.go"]}]}`}}}, |
| 157 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2", Name: "complete_step", |
| 158 | Arguments: `{"step":"beta","result":"done","evidence":[{"kind":"manual","summary":"verified by inspection"}]}`}}}, |
| 159 | testutil.Turn{Text: "all done now"}, |
| 160 | ) |
| 161 | a := New(mp, evidenceRegistry(), sess, Options{}, event.Discard) |
| 162 | a.SetSession(sess) // rebuilds canonical {alpha in_progress, beta pending} |
| 163 | |
| 164 | firstErr := a.Run(context.Background(), "finish up") |
| 165 | if !readinessBlocked(firstErr) { |
| 166 | t.Fatalf("premature 'all done' error = %v, want FinalReadinessError from the cross-turn canonical gate", firstErr) |
| 167 | } |
| 168 | if err := a.Run(context.Background(), "finish up"); err != nil { |
| 169 | t.Fatalf("follow-up Run: %v", err) |
| 170 | } |
| 171 | for i, td := range a.todoState { |
| 172 | if canonicalTodoStatus(td.Status) != "completed" { |
| 173 | t.Fatalf("canonical todo %d (%q) = %s after sign-off, want completed", i+1, td.Content, td.Status) |
| 174 | } |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | func TestE2ECrossTurnPendingSignoffIsRejectedUntilCurrentAdvances(t *testing.T) { |
| 179 | sess := NewSession("sys") |
| 180 | sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ |
| 181 | ID: "t0", Name: "todo_write", |
| 182 | Arguments: `{"todos":[{"content":"alpha","status":"in_progress"},{"content":"beta","status":"pending"}]}`}}}) |
| 183 | sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: "t0", Name: "todo_write", Content: "Todos updated"}) |
| 184 | |
| 185 | mp := testutil.NewMock("m", |
| 186 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c0", Name: "complete_step", |
| 187 | Arguments: `{"step":"beta","result":"done","evidence":[{"kind":"manual","summary":"claimed"}]}`}}}, |
| 188 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 189 | Arguments: `{"step":"alpha","result":"done","evidence":[{"kind":"manual","summary":"checked"}]}`}}}, |
| 190 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c2", Name: "complete_step", |
| 191 | Arguments: `{"step":"beta","result":"done","evidence":[{"kind":"manual","summary":"checked"}]}`}}}, |
| 192 | testutil.Turn{Text: "all done"}, |
| 193 | ) |
| 194 | a := New(mp, evidenceRegistry(), sess, Options{}, event.Discard) |
| 195 | a.SetSession(sess) |
| 196 | |
| 197 | if err := a.Run(context.Background(), "continue"); err != nil { |
| 198 | t.Fatalf("Run: %v", err) |
| 199 | } |
| 200 | if !sessionContains(a, "only signs the current in_progress item") { |
| 201 | t.Fatal("cross-turn pending signoff was not rejected") |
| 202 | } |
| 203 | for i, td := range a.todoState { |
| 204 | if canonicalTodoStatus(td.Status) != "completed" { |
| 205 | t.Fatalf("canonical todo %d (%q) = %s, want completed", i+1, td.Content, td.Status) |
| 206 | } |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // Cross-turn diff evidence: a file edited in an earlier turn is signed off in a |
| 211 | // later turn whose per-turn ledger is empty. The session-history fallback must |
| 212 | // resolve the path receipt that the ledger no longer holds. |
| 213 | func TestE2ECrossTurnDiffEvidenceViaSessionFallback(t *testing.T) { |
| 214 | mp := testutil.NewMock("m", |
| 215 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "w1", Name: "write_file", Arguments: `{"path":"pkg/x.go"}`}}}, |
| 216 | testutil.Turn{Text: "edited x.go"}, |
| 217 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 218 | Arguments: `{"step":"edit x","result":"x updated","evidence":[{"kind":"diff","summary":"changed x","paths":["pkg/x.go"]}]}`}}}, |
| 219 | testutil.Turn{Text: "signed off"}, |
| 220 | ) |
| 221 | a := New(mp, evidenceRegistry(), NewSession("sys"), Options{}, event.Discard) |
| 222 | |
| 223 | if err := a.Run(context.Background(), "edit x.go"); err != nil { |
| 224 | t.Fatalf("turn 1: %v", err) |
| 225 | } |
| 226 | if err := a.Run(context.Background(), "now sign off that change"); err != nil { |
| 227 | t.Fatalf("turn 2: %v", err) |
| 228 | } |
| 229 | if !sessionContains(a, "signed off") { |
| 230 | t.Fatal("turn 2 rejected a cross-turn diff citation the session proves") |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | // A diff citation for a file no turn ever wrote stays rejected — the session |
| 235 | // fallback widens what counts as proof, it does not wave through fabrication. |
| 236 | func TestE2EUnbackedDiffEvidenceStillRejected(t *testing.T) { |
| 237 | mp := testutil.NewMock("m", |
| 238 | testutil.Turn{ToolCalls: []provider.ToolCall{{ID: "c1", Name: "complete_step", |
| 239 | Arguments: `{"step":"x","result":"y","evidence":[{"kind":"diff","summary":"claimed","paths":["never/written.go"]}]}`}}}, |
| 240 | testutil.Turn{Text: "done"}, |
| 241 | ) |
| 242 | a := New(mp, evidenceRegistry(), NewSession("sys"), Options{}, event.Discard) |
| 243 | |
| 244 | if err := a.Run(context.Background(), "sign off without doing the work"); err != nil { |
| 245 | t.Fatalf("Run: %v", err) |
| 246 | } |
| 247 | if !sessionContains(a, "no matching successful writer") { |
| 248 | t.Fatal("a diff citation for a never-written file was accepted") |
| 249 | } |
| 250 | if sessionContains(a, "signed off") { |
| 251 | t.Fatal("an unbacked diff citation was signed off") |
| 252 | } |
| 253 | } |
| 254 |