| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "reasonix/internal/event" |
| 11 | "reasonix/internal/evidence" |
| 12 | "reasonix/internal/instruction" |
| 13 | "reasonix/internal/provider" |
| 14 | "reasonix/internal/tool" |
| 15 | ) |
| 16 | |
| 17 | // scriptedProvider replays a distinct chunk set per Stream call, so a multi-turn |
| 18 | // Run() sees tool calls on turn 1 and a plain final answer on turn 2. |
| 19 | type scriptedProvider struct { |
| 20 | name string |
| 21 | turns [][]provider.Chunk |
| 22 | call int |
| 23 | requests []provider.Request |
| 24 | } |
| 25 | |
| 26 | func (s *scriptedProvider) Name() string { return s.name } |
| 27 | |
| 28 | func (s *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 29 | s.requests = append(s.requests, req) |
| 30 | i := s.call |
| 31 | if i >= len(s.turns) { |
| 32 | i = len(s.turns) - 1 |
| 33 | } |
| 34 | s.call++ |
| 35 | ch := make(chan provider.Chunk, len(s.turns[i])) |
| 36 | for _, c := range s.turns[i] { |
| 37 | ch <- c |
| 38 | } |
| 39 | close(ch) |
| 40 | return ch, nil |
| 41 | } |
| 42 | |
| 43 | func toolCallChunk(id, name, args string) provider.Chunk { |
| 44 | return provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: id, Name: name, Arguments: args}} |
| 45 | } |
| 46 | |
| 47 | func toolResult(s *Session, name string) string { |
| 48 | for _, m := range s.Messages { |
| 49 | if m.Role == provider.RoleTool && m.Name == name { |
| 50 | return m.Content |
| 51 | } |
| 52 | } |
| 53 | return "" |
| 54 | } |
| 55 | |
| 56 | func lastToolResult(s *Session, name string) string { |
| 57 | var result string |
| 58 | for _, m := range s.Messages { |
| 59 | if m.Role == provider.RoleTool && m.Name == name { |
| 60 | result = m.Content |
| 61 | } |
| 62 | } |
| 63 | return result |
| 64 | } |
| 65 | |
| 66 | func toolResultByID(s *Session, id string) string { |
| 67 | for _, m := range s.Messages { |
| 68 | if m.Role == provider.RoleTool && m.ToolCallID == id { |
| 69 | return m.Content |
| 70 | } |
| 71 | } |
| 72 | return "" |
| 73 | } |
| 74 | |
| 75 | func toolResults(s *Session, name string) []string { |
| 76 | var results []string |
| 77 | for _, m := range s.Messages { |
| 78 | if m.Role == provider.RoleTool && m.Name == name { |
| 79 | results = append(results, m.Content) |
| 80 | } |
| 81 | } |
| 82 | return results |
| 83 | } |
| 84 | |
| 85 | func sessionHasUserMessageContaining(s *Session, needle string) bool { |
| 86 | for _, m := range s.Messages { |
| 87 | if m.Role == provider.RoleUser && strings.Contains(provider.ModelMessages([]provider.Message{m})[0].Content, needle) { |
| 88 | return true |
| 89 | } |
| 90 | } |
| 91 | return false |
| 92 | } |
| 93 | |
| 94 | type readinessAuditSink struct { |
| 95 | events []evidence.ReadinessAudit |
| 96 | } |
| 97 | |
| 98 | func (s *readinessAuditSink) Emit(event.Event) {} |
| 99 | |
| 100 | func (s *readinessAuditSink) RecordReadinessAudit(a evidence.ReadinessAudit) { |
| 101 | s.events = append(s.events, a) |
| 102 | } |
| 103 | |
| 104 | // TestEvidenceFlowEndToEnd drives a full Run(): turn 1 runs bash then signs the |
| 105 | // step off citing that exact command; complete_step must see the host receipt |
| 106 | // recorded earlier in the same batch and report it host-verified. |
| 107 | func TestEvidenceFlowEndToEnd(t *testing.T) { |
| 108 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 109 | if !ok { |
| 110 | t.Fatal("complete_step builtin not registered") |
| 111 | } |
| 112 | reg := tool.NewRegistry() |
| 113 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 114 | reg.Add(completeStep) |
| 115 | |
| 116 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 117 | { |
| 118 | toolCallChunk("c1", "bash", `{"command":"go test ./..."}`), |
| 119 | toolCallChunk("c2", "complete_step", `{ |
| 120 | "step":"Run the suite", |
| 121 | "result":"tests pass", |
| 122 | "evidence":[{"kind":"verification","summary":"go test ./... passed","command":"go test ./..."}] |
| 123 | }`), |
| 124 | {Type: provider.ChunkDone}, |
| 125 | }, |
| 126 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 127 | }} |
| 128 | |
| 129 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 130 | if err := a.Run(context.Background(), "run the suite and sign the step off"); err != nil { |
| 131 | t.Fatalf("Run: %v", err) |
| 132 | } |
| 133 | |
| 134 | if got := toolResult(a.session, "complete_step"); !strings.Contains(got, "host-verified 1") { |
| 135 | t.Fatalf("complete_step result = %q, want it host-verified from the bash receipt", got) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | func TestDeliveryProfileEnforcesAcceptanceReviewVerificationAndSignoff(t *testing.T) { |
| 140 | reg := evidenceRegistry() |
| 141 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 142 | // Keep review available so this ordinary production change exercises the |
| 143 | // Medium-risk host-proof alternative instead of the minimal-registry bypass. |
| 144 | reg.Add(fakeTool{name: "review", readOnly: true}) |
| 145 | |
| 146 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 147 | {toolCallChunk("blocked-write", "write_file", `{"path":"main.go","content":"package main"}`), {Type: provider.ChunkDone}}, |
| 148 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Ship main","status":"in_progress","activeForm":"Shipping main"}]}`), {Type: provider.ChunkDone}}, |
| 149 | {toolCallChunk("write", "write_file", `{"path":"main.go","content":"package main"}`), {Type: provider.ChunkDone}}, |
| 150 | {toolCallChunk("review", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 151 | {toolCallChunk("verify", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 152 | {toolCallChunk("signoff", "complete_step", `{ |
| 153 | "step":"Ship main", |
| 154 | "result":"main is implemented and verified", |
| 155 | "evidence":[ |
| 156 | {"kind":"diff","summary":"main implementation added","paths":["main.go"]}, |
| 157 | {"kind":"verification","summary":"tests pass","command":"go test ./..."} |
| 158 | ] |
| 159 | }`), {Type: provider.ChunkDone}}, |
| 160 | {{Type: provider.ChunkText, Text: "delivered"}, {Type: provider.ChunkDone}}, |
| 161 | }} |
| 162 | |
| 163 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 164 | if err := a.Run(context.Background(), "implement main"); err != nil { |
| 165 | t.Fatalf("Run: %v", err) |
| 166 | } |
| 167 | if got := toolResult(a.session, "write_file"); !strings.Contains(got, "delivery-first mode requires acceptance criteria") { |
| 168 | t.Fatalf("first write result = %q, want delivery acceptance gate", got) |
| 169 | } |
| 170 | if !sessionHasUserMessageContaining(a.session, "<delivery-runtime>") { |
| 171 | t.Fatal("delivery runtime marker was not injected into the turn tail") |
| 172 | } |
| 173 | if got := lastToolResult(a.session, "complete_step"); !strings.Contains(got, "signed off") { |
| 174 | t.Fatalf("complete_step result = %q, want successful sign-off", got) |
| 175 | } |
| 176 | firstSystem := systemMessageContent(prov.requests[0]) |
| 177 | firstTools := prov.requests[0].Tools |
| 178 | for i, req := range prov.requests[1:] { |
| 179 | if got := systemMessageContent(req); got != firstSystem { |
| 180 | t.Fatalf("delivery request %d changed the cache-stable system prompt", i+2) |
| 181 | } |
| 182 | if !reflect.DeepEqual(req.Tools, firstTools) { |
| 183 | t.Fatalf("delivery request %d changed provider-visible tool schemas", i+2) |
| 184 | } |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | func systemMessageContent(req provider.Request) string { |
| 189 | for _, msg := range req.Messages { |
| 190 | if msg.Role == provider.RoleSystem { |
| 191 | return msg.Content |
| 192 | } |
| 193 | } |
| 194 | return "" |
| 195 | } |
| 196 | |
| 197 | func TestDeliveryProfileRequiresReviewBeforeFinalAnswer(t *testing.T) { |
| 198 | reg := evidenceRegistry() |
| 199 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 200 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 201 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Ship main","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 202 | {toolCallChunk("write", "write_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 203 | {toolCallChunk("verify", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 204 | {toolCallChunk("signoff", "complete_step", `{"step":"Ship main","result":"implemented","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 205 | {{Type: provider.ChunkText, Text: "done too early"}, {Type: provider.ChunkDone}}, |
| 206 | {toolCallChunk("review", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 207 | {toolCallChunk("renewed-signoff", "complete_step", `{"step":"Ship main","result":"implemented, reviewed, and verified","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 208 | {{Type: provider.ChunkText, Text: "done after review and signoff"}, {Type: provider.ChunkDone}}, |
| 209 | }} |
| 210 | sink := &readinessAuditSink{} |
| 211 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, sink) |
| 212 | ctx := deliveryGoalContext("goal-review", "implement main") |
| 213 | // The first final answer fails immediately (no readiness retries); the |
| 214 | // scoped follow-up adds the missing review and renews the sign-off. |
| 215 | if err := a.Run(ctx, "implement main"); !readinessBlocked(err) { |
| 216 | t.Fatalf("first Run err = %v, want FinalReadinessError for the missing review", err) |
| 217 | } |
| 218 | if len(sink.events) != 1 || sink.events[0].Result != evidence.ReadinessErrored || sink.events[0].MissingReview == 0 { |
| 219 | t.Fatalf("readiness audits = %+v, want one errored audit with missing review", sink.events) |
| 220 | } |
| 221 | if err := a.Run(ctx, "finish the goal"); err != nil { |
| 222 | t.Fatalf("follow-up Run: %v", err) |
| 223 | } |
| 224 | if len(sink.events) != 2 || sink.events[len(sink.events)-1].Result != evidence.ReadinessAllowed { |
| 225 | t.Fatalf("readiness audits = %+v, want a final allowed audit", sink.events) |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | func TestDeliveryProfileRejectsTextOnlyImplementationClaim(t *testing.T) { |
| 230 | reg := evidenceRegistry() |
| 231 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 232 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 233 | {{Type: provider.ChunkText, Text: "implemented"}, {Type: provider.ChunkDone}}, |
| 234 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Implement main","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 235 | {toolCallChunk("write", "write_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 236 | {toolCallChunk("review", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 237 | {toolCallChunk("verify", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 238 | {toolCallChunk("signoff", "complete_step", `{"step":"Implement main","result":"implemented","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 239 | {{Type: provider.ChunkText, Text: "implemented with evidence"}, {Type: provider.ChunkDone}}, |
| 240 | }} |
| 241 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 242 | err := a.Run(context.Background(), "implement main") |
| 243 | var readiness *FinalReadinessError |
| 244 | if !errors.As(err, &readiness) || !strings.Contains(readiness.Reason, "no successful mutation was observed") { |
| 245 | t.Fatalf("text-only implementation claim err = %v, want mutation readiness failure", err) |
| 246 | } |
| 247 | if prov.call != 1 { |
| 248 | t.Fatalf("provider calls = %d, want 1 (text-only claim rejected immediately, no retries)", prov.call) |
| 249 | } |
| 250 | } |
| 251 | |
| 252 | func TestDeliveryProfileCommandOnlyActionRequiresCriteriaAndSignoff(t *testing.T) { |
| 253 | reg := evidenceRegistry() |
| 254 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 255 | {toolCallChunk("blocked-test", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 256 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Run tests","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 257 | {toolCallChunk("verify", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 258 | {toolCallChunk("signoff", "complete_step", `{"step":"Run tests","result":"tests pass","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 259 | {{Type: provider.ChunkText, Text: "tests pass"}, {Type: provider.ChunkDone}}, |
| 260 | }} |
| 261 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 262 | if err := a.Run(context.Background(), "run tests"); err != nil { |
| 263 | t.Fatalf("Run: %v", err) |
| 264 | } |
| 265 | if got := toolResult(a.session, "bash"); !strings.Contains(got, "delivery-first mode requires acceptance criteria") { |
| 266 | t.Fatalf("first bash result = %q, want acceptance gate", got) |
| 267 | } |
| 268 | if got := lastToolResult(a.session, "complete_step"); !strings.Contains(got, "signed off") { |
| 269 | t.Fatalf("complete_step result = %q, want successful command-only sign-off", got) |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | func TestDeliveryProfileBlocksMixedVerificationBeforeItBecomesMutation(t *testing.T) { |
| 274 | reg := evidenceRegistry() |
| 275 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 276 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Check snake","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 277 | {toolCallChunk("mixed", "bash", `{"command":"python3 -c 'open(\"/tmp/snake_check.js\",\"w\").write(\"x\")' && node --check /tmp/snake_check.js"}`), {Type: provider.ChunkDone}}, |
| 278 | {toolCallChunk("safe", "bash", `{"command":"tail -n +2 snake.js | head -n 20 | node --check -"}`), {Type: provider.ChunkDone}}, |
| 279 | {toolCallChunk("signoff", "complete_step", `{"step":"Check snake","result":"syntax valid","evidence":[{"kind":"verification","summary":"syntax valid","command":"tail -n +2 snake.js | head -n 20 | node --check -"}]}`), {Type: provider.ChunkDone}}, |
| 280 | {{Type: provider.ChunkText, Text: "checked"}, {Type: provider.ChunkDone}}, |
| 281 | }} |
| 282 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 283 | if err := a.Run(context.Background(), "check the snake game"); err != nil { |
| 284 | t.Fatalf("Run: %v", err) |
| 285 | } |
| 286 | if got := toolResult(a.session, "bash"); !strings.Contains(got, "mixes a verification check") { |
| 287 | t.Fatalf("mixed command result = %q, want pre-execution split guidance", got) |
| 288 | } |
| 289 | if _, ok := a.evidence.LatestSuccessfulMutationIndex(); ok { |
| 290 | t.Fatal("blocked scratch-file verification must not become a successful mutation") |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | func TestDeliveryProfileExplainsMaskedVerifierExitBeforeExecution(t *testing.T) { |
| 295 | reg := evidenceRegistry() |
| 296 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 297 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Check snake","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 298 | {toolCallChunk("masked", "bash", `{"command":"tail -n +2 snake.js | head -n 20 | node --check -; echo \"EXIT: $?\""}`), {Type: provider.ChunkDone}}, |
| 299 | {toolCallChunk("safe", "bash", `{"command":"tail -n +2 snake.js | head -n 20 | node --check -"}`), {Type: provider.ChunkDone}}, |
| 300 | {toolCallChunk("signoff", "complete_step", `{"step":"Check snake","result":"syntax valid","evidence":[{"kind":"verification","summary":"syntax valid","command":"tail -n +2 snake.js | head -n 20 | node --check -"}]}`), {Type: provider.ChunkDone}}, |
| 301 | {{Type: provider.ChunkText, Text: "checked"}, {Type: provider.ChunkDone}}, |
| 302 | }} |
| 303 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 304 | if err := a.Run(context.Background(), "check the snake game"); err != nil { |
| 305 | t.Fatalf("Run: %v", err) |
| 306 | } |
| 307 | if got := toolResultByID(a.session, "masked"); !strings.Contains(got, "masks the verifier's exit status") { |
| 308 | t.Fatalf("masked command result = %q, want precise exit-status guidance", got) |
| 309 | } |
| 310 | if _, ok := a.evidence.LatestSuccessfulMutationIndex(); ok { |
| 311 | t.Fatal("blocked masked verifier must not become a successful mutation") |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | func TestDeliveryProfileBlocksOpaqueInlineInterpreterBeforeItBecomesMutation(t *testing.T) { |
| 316 | reg := evidenceRegistry() |
| 317 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 318 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Check snake","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 319 | {toolCallChunk("opaque", "bash", `{"command":"node -e 'require(\"fs\").readFileSync(\"snake.html\")'"}`), {Type: provider.ChunkDone}}, |
| 320 | {toolCallChunk("safe", "bash", `{"command":"tail -n +2 snake.js | head -n 20 | node --check -"}`), {Type: provider.ChunkDone}}, |
| 321 | {toolCallChunk("signoff", "complete_step", `{"step":"Check snake","result":"syntax valid","evidence":[{"kind":"verification","summary":"syntax valid","command":"tail -n +2 snake.js | head -n 20 | node --check -"}]}`), {Type: provider.ChunkDone}}, |
| 322 | {{Type: provider.ChunkText, Text: "checked"}, {Type: provider.ChunkDone}}, |
| 323 | }} |
| 324 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 325 | if err := a.Run(context.Background(), "check the snake game"); err != nil { |
| 326 | t.Fatalf("Run: %v", err) |
| 327 | } |
| 328 | if got := toolResultByID(a.session, "opaque"); !strings.Contains(got, "cannot audit inline interpreter source") { |
| 329 | t.Fatalf("opaque command result = %q, want pre-execution audit guidance", got) |
| 330 | } |
| 331 | if _, ok := a.evidence.LatestSuccessfulMutationIndex(); ok { |
| 332 | t.Fatal("blocked inline interpreter must not become a successful mutation") |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | func TestDeliveryProfileRequiresActiveTodoForLateMutation(t *testing.T) { |
| 337 | reg := evidenceRegistry() |
| 338 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 339 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 340 | {toolCallChunk("criteria", "todo_write", `{"todos":[{"content":"Ship main","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 341 | {toolCallChunk("write", "write_file", `{"path":"main.go","content":"package main"}`), {Type: provider.ChunkDone}}, |
| 342 | {toolCallChunk("review", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 343 | {toolCallChunk("verify", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 344 | {toolCallChunk("signoff", "complete_step", `{"step":"Ship main","result":"done","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 345 | {toolCallChunk("late-write", "write_file", `{"path":"main.go","content":"package main // late"}`), {Type: provider.ChunkDone}}, |
| 346 | {toolCallChunk("append-todo", "todo_write", `{"todos":[{"content":"Ship main","status":"completed"},{"content":"Apply review fix","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 347 | {toolCallChunk("retry-write", "write_file", `{"path":"main.go","content":"package main // reviewed"}`), {Type: provider.ChunkDone}}, |
| 348 | {toolCallChunk("review-2", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 349 | {toolCallChunk("verify-2", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 350 | {toolCallChunk("signoff-2", "complete_step", `{"step":"Apply review fix","result":"done","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 351 | {{Type: provider.ChunkText, Text: "delivered"}, {Type: provider.ChunkDone}}, |
| 352 | }} |
| 353 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 354 | if err := a.Run(context.Background(), "implement main and incorporate review fixes"); err != nil { |
| 355 | t.Fatalf("Run: %v", err) |
| 356 | } |
| 357 | if got := toolResultByID(a.session, "late-write"); !strings.Contains(got, "current in_progress todo") { |
| 358 | t.Fatalf("late mutation result = %q, want active-todo gate", got) |
| 359 | } |
| 360 | if got := toolResultByID(a.session, "retry-write"); strings.HasPrefix(got, "blocked:") || strings.HasPrefix(got, "error:") { |
| 361 | t.Fatalf("mutation after appended active todo should run, got %q", got) |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | func TestDeliveryProfileAllowsEvidenceBackedReadOnlyAnalysis(t *testing.T) { |
| 366 | reg := tool.NewRegistry() |
| 367 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 368 | prov := &scriptedProvider{name: "delivery", turns: [][]provider.Chunk{ |
| 369 | {toolCallChunk("read", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 370 | {{Type: provider.ChunkText, Text: "analysis"}, {Type: provider.ChunkDone}}, |
| 371 | }} |
| 372 | a := New(prov, reg, NewSession(""), Options{DeliveryProfile: true}, event.Discard) |
| 373 | if err := a.Run(context.Background(), "analyze main.go"); err != nil { |
| 374 | t.Fatalf("read-only analysis should not require mutation/sign-off: %v", err) |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | func TestEvidenceFlowEnforcesProjectChecksAfterWrite(t *testing.T) { |
| 379 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 380 | if !ok { |
| 381 | t.Fatal("complete_step builtin not registered") |
| 382 | } |
| 383 | reg := tool.NewRegistry() |
| 384 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 385 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 386 | reg.Add(completeStep) |
| 387 | |
| 388 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 389 | { |
| 390 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 391 | toolCallChunk("c2", "bash", `{"command":"go test ./..."}`), |
| 392 | toolCallChunk("c3", "complete_step", `{ |
| 393 | "step":"Edit code", |
| 394 | "result":"changed.go updated", |
| 395 | "evidence":[{"kind":"diff","summary":"updated code","paths":["changed.go"]}] |
| 396 | }`), |
| 397 | {Type: provider.ChunkDone}, |
| 398 | }, |
| 399 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 400 | }} |
| 401 | |
| 402 | a := New(prov, reg, NewSession(""), Options{ |
| 403 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 404 | }, event.Discard) |
| 405 | if err := a.Run(context.Background(), "edit and verify"); err != nil { |
| 406 | t.Fatalf("Run: %v", err) |
| 407 | } |
| 408 | |
| 409 | got := toolResult(a.session, "complete_step") |
| 410 | if !strings.Contains(got, "project checks 1") { |
| 411 | t.Fatalf("complete_step result = %q, want project check verified from same batch", got) |
| 412 | } |
| 413 | } |
| 414 | |
| 415 | func TestFinalReadinessAllowsFinalAnswerWithoutWriter(t *testing.T) { |
| 416 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 417 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 418 | }} |
| 419 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{ |
| 420 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 421 | }, event.Discard) |
| 422 | |
| 423 | if err := a.Run(context.Background(), "inspect only"); err != nil { |
| 424 | t.Fatalf("Run: %v", err) |
| 425 | } |
| 426 | if prov.call != 1 { |
| 427 | t.Fatalf("provider calls = %d, want 1", prov.call) |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | func TestFinalReadinessAllowsWriterWithoutChecksOrTodos(t *testing.T) { |
| 432 | reg := tool.NewRegistry() |
| 433 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 434 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 435 | { |
| 436 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 437 | {Type: provider.ChunkDone}, |
| 438 | }, |
| 439 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 440 | }} |
| 441 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 442 | |
| 443 | if err := a.Run(context.Background(), "simple edit"); err != nil { |
| 444 | t.Fatalf("Run: %v", err) |
| 445 | } |
| 446 | if prov.call != 2 { |
| 447 | t.Fatalf("provider calls = %d, want 2", prov.call) |
| 448 | } |
| 449 | } |
| 450 | |
| 451 | func TestFinalReadinessAuditSkipsWhenGateDoesNotApply(t *testing.T) { |
| 452 | t.Run("no writer", func(t *testing.T) { |
| 453 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 454 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 455 | }} |
| 456 | sink := &readinessAuditSink{} |
| 457 | a := New(prov, tool.NewRegistry(), NewSession(""), Options{ |
| 458 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 459 | }, sink) |
| 460 | |
| 461 | if err := a.Run(context.Background(), "inspect only"); err != nil { |
| 462 | t.Fatalf("Run: %v", err) |
| 463 | } |
| 464 | if len(sink.events) != 0 { |
| 465 | t.Fatalf("readiness audit events = %d, want 0: %+v", len(sink.events), sink.events) |
| 466 | } |
| 467 | }) |
| 468 | |
| 469 | t.Run("writer without checks or todo", func(t *testing.T) { |
| 470 | reg := tool.NewRegistry() |
| 471 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 472 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 473 | { |
| 474 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 475 | {Type: provider.ChunkDone}, |
| 476 | }, |
| 477 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 478 | }} |
| 479 | sink := &readinessAuditSink{} |
| 480 | a := New(prov, reg, NewSession(""), Options{}, sink) |
| 481 | |
| 482 | if err := a.Run(context.Background(), "simple edit"); err != nil { |
| 483 | t.Fatalf("Run: %v", err) |
| 484 | } |
| 485 | if len(sink.events) != 0 { |
| 486 | t.Fatalf("readiness audit events = %d, want 0: %+v", len(sink.events), sink.events) |
| 487 | } |
| 488 | }) |
| 489 | } |
| 490 | |
| 491 | func TestFinalReadinessBlocksUntilProjectCheckRunsAfterWriter(t *testing.T) { |
| 492 | reg := tool.NewRegistry() |
| 493 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 494 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 495 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 496 | { |
| 497 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 498 | {Type: provider.ChunkDone}, |
| 499 | }, |
| 500 | {{Type: provider.ChunkText, Text: "premature"}, {Type: provider.ChunkDone}}, |
| 501 | { |
| 502 | toolCallChunk("c2", "bash", `{"command":"go test ./..."}`), |
| 503 | {Type: provider.ChunkDone}, |
| 504 | }, |
| 505 | {{Type: provider.ChunkText, Text: "verified done"}, {Type: provider.ChunkDone}}, |
| 506 | }} |
| 507 | a := New(prov, reg, NewSession(""), Options{ |
| 508 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 509 | }, event.Discard) |
| 510 | ctx := deliveryGoalContext("goal-checks", "edit and finish") |
| 511 | |
| 512 | // The premature final answer fails immediately; the scoped follow-up runs |
| 513 | // the required project check after the preserved write and passes. |
| 514 | if err := a.Run(ctx, "edit and finish"); !readinessBlocked(err) { |
| 515 | t.Fatalf("premature Run err = %v, want FinalReadinessError", err) |
| 516 | } |
| 517 | if prov.call != 2 { |
| 518 | t.Fatalf("provider calls = %d, want writer turn + one blocked final answer (no retries)", prov.call) |
| 519 | } |
| 520 | if err := a.Run(ctx, "finish"); err != nil { |
| 521 | t.Fatalf("verified Run: %v", err) |
| 522 | } |
| 523 | if got := lastToolResult(a.session, "bash"); !strings.Contains(got, "bash done") { |
| 524 | t.Fatalf("bash tool result = %q, want command run after the block", got) |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | func TestFinalReadinessAuditRecordsBlockAndRecovery(t *testing.T) { |
| 529 | reg := tool.NewRegistry() |
| 530 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 531 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 532 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 533 | { |
| 534 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 535 | {Type: provider.ChunkDone}, |
| 536 | }, |
| 537 | {{Type: provider.ChunkText, Text: "premature"}, {Type: provider.ChunkDone}}, |
| 538 | { |
| 539 | toolCallChunk("c2", "bash", `{"command":"go test ./..."}`), |
| 540 | {Type: provider.ChunkDone}, |
| 541 | }, |
| 542 | {{Type: provider.ChunkText, Text: "verified done"}, {Type: provider.ChunkDone}}, |
| 543 | }} |
| 544 | sink := &readinessAuditSink{} |
| 545 | a := New(prov, reg, NewSession(""), Options{ |
| 546 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 547 | }, sink) |
| 548 | ctx := deliveryGoalContext("goal-audit", "edit and finish") |
| 549 | |
| 550 | if err := a.Run(ctx, "edit and finish"); !readinessBlocked(err) { |
| 551 | t.Fatalf("premature Run err = %v, want FinalReadinessError", err) |
| 552 | } |
| 553 | if len(sink.events) != 1 { |
| 554 | t.Fatalf("readiness audit events = %d, want 1: %+v", len(sink.events), sink.events) |
| 555 | } |
| 556 | blocked := sink.events[0] |
| 557 | if blocked.Result != evidence.ReadinessErrored || blocked.MissingProjectChecks != 1 || blocked.CommandMismatchMissing != 1 { |
| 558 | t.Fatalf("blocked audit = %+v, want missing project check command", blocked) |
| 559 | } |
| 560 | if err := a.Run(ctx, "finish"); err != nil { |
| 561 | t.Fatalf("verified Run: %v", err) |
| 562 | } |
| 563 | recovered := sink.events[len(sink.events)-1] |
| 564 | if recovered.Result != evidence.ReadinessAllowed || !recovered.Recovered { |
| 565 | t.Fatalf("recovery audit = %+v, want allowed recovered", recovered) |
| 566 | } |
| 567 | } |
| 568 | |
| 569 | func TestFinalReadinessRejectsProjectCheckBeforeWriter(t *testing.T) { |
| 570 | reg := tool.NewRegistry() |
| 571 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 572 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 573 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 574 | { |
| 575 | toolCallChunk("c1", "bash", `{"command":"go test ./..."}`), |
| 576 | toolCallChunk("c2", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 577 | {Type: provider.ChunkDone}, |
| 578 | }, |
| 579 | {{Type: provider.ChunkText, Text: "premature"}, {Type: provider.ChunkDone}}, |
| 580 | { |
| 581 | toolCallChunk("c3", "bash", `{"command":"go test ./..."}`), |
| 582 | {Type: provider.ChunkDone}, |
| 583 | }, |
| 584 | {{Type: provider.ChunkText, Text: "verified done"}, {Type: provider.ChunkDone}}, |
| 585 | }} |
| 586 | a := New(prov, reg, NewSession(""), Options{ |
| 587 | ProjectChecks: []instruction.VerifyCheck{{Command: "go test ./...", SourcePath: "AGENTS.md", Line: 3}}, |
| 588 | }, event.Discard) |
| 589 | ctx := deliveryGoalContext("goal-before-writer", "verify before edit, then finish") |
| 590 | |
| 591 | // The pre-writer check does not satisfy the after-writer requirement: the |
| 592 | // final answer fails, and the scoped follow-up reruns the check. |
| 593 | if err := a.Run(ctx, "verify before edit, then finish"); !readinessBlocked(err) { |
| 594 | t.Fatalf("premature Run err = %v, want FinalReadinessError", err) |
| 595 | } |
| 596 | if err := a.Run(ctx, "finish"); err != nil { |
| 597 | t.Fatalf("verified Run: %v", err) |
| 598 | } |
| 599 | if prov.call != 4 { |
| 600 | t.Fatalf("provider calls = %d, want pre-write check, writer, blocked final, post-write check", prov.call) |
| 601 | } |
| 602 | } |
| 603 | |
| 604 | func TestFinalReadinessRequiresCompleteStepAfterWriterWhenTodoSeen(t *testing.T) { |
| 605 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 606 | if !ok { |
| 607 | t.Fatal("todo_write builtin not registered") |
| 608 | } |
| 609 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 610 | if !ok { |
| 611 | t.Fatal("complete_step builtin not registered") |
| 612 | } |
| 613 | reg := tool.NewRegistry() |
| 614 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 615 | reg.Add(todoWrite) |
| 616 | reg.Add(completeStep) |
| 617 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 618 | { |
| 619 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 620 | toolCallChunk("c2", "todo_write", `{"todos":[{"content":"Edit code","status":"in_progress"}]}`), |
| 621 | {Type: provider.ChunkDone}, |
| 622 | }, |
| 623 | {{Type: provider.ChunkText, Text: "premature"}, {Type: provider.ChunkDone}}, |
| 624 | { |
| 625 | toolCallChunk("c3", "complete_step", `{ |
| 626 | "step":"Edit code", |
| 627 | "result":"changed.go updated", |
| 628 | "evidence":[{"kind":"diff","summary":"updated code","paths":["changed.go"]}] |
| 629 | }`), |
| 630 | toolCallChunk("c4", "todo_write", `{"todos":[{"content":"Edit code","status":"completed"}]}`), |
| 631 | {Type: provider.ChunkDone}, |
| 632 | }, |
| 633 | {{Type: provider.ChunkText, Text: "signed off done"}, {Type: provider.ChunkDone}}, |
| 634 | }} |
| 635 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 636 | ctx := deliveryGoalContext("goal-signoff", "edit with todo and finish") |
| 637 | |
| 638 | // The premature final answer fails immediately; the scoped follow-up signs |
| 639 | // the step off with complete_step and passes. |
| 640 | if err := a.Run(ctx, "edit with todo and finish"); !readinessBlocked(err) { |
| 641 | t.Fatalf("premature Run err = %v, want FinalReadinessError", err) |
| 642 | } |
| 643 | if err := a.Run(ctx, "finish"); err != nil { |
| 644 | t.Fatalf("signed-off Run: %v", err) |
| 645 | } |
| 646 | if got := lastToolResult(a.session, "complete_step"); !strings.Contains(got, "signed off") { |
| 647 | t.Fatalf("complete_step result = %q, want successful sign-off", got) |
| 648 | } |
| 649 | } |
| 650 | |
| 651 | func TestFinalReadinessStopsAfterFirstBlock(t *testing.T) { |
| 652 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 653 | if !ok { |
| 654 | t.Fatal("todo_write builtin not registered") |
| 655 | } |
| 656 | reg := tool.NewRegistry() |
| 657 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 658 | reg.Add(todoWrite) |
| 659 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 660 | { |
| 661 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 662 | toolCallChunk("c2", "todo_write", `{"todos":[{"content":"Edit code","status":"in_progress"}]}`), |
| 663 | {Type: provider.ChunkDone}, |
| 664 | }, |
| 665 | {{Type: provider.ChunkText, Text: "premature 1"}, {Type: provider.ChunkDone}}, |
| 666 | {{Type: provider.ChunkText, Text: "premature 2"}, {Type: provider.ChunkDone}}, |
| 667 | {{Type: provider.ChunkText, Text: "premature 3"}, {Type: provider.ChunkDone}}, |
| 668 | }} |
| 669 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 670 | |
| 671 | err := a.Run(context.Background(), "edit with todo and never sign off") |
| 672 | if err == nil { |
| 673 | t.Fatal("expected the first readiness block to stop the run") |
| 674 | } |
| 675 | if !strings.Contains(err.Error(), "final-answer readiness") { |
| 676 | t.Fatalf("error = %v, want final-answer readiness", err) |
| 677 | } |
| 678 | if prov.call != 2 { |
| 679 | t.Fatalf("provider calls = %d, want writer turn + one blocked final answer (no readiness retries)", prov.call) |
| 680 | } |
| 681 | } |
| 682 | |
| 683 | func TestFinalReadinessPermissionLoopGuardAllowsBlockedFinal(t *testing.T) { |
| 684 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 685 | if !ok { |
| 686 | t.Fatal("todo_write builtin not registered") |
| 687 | } |
| 688 | reg := tool.NewRegistry() |
| 689 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 690 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 691 | reg.Add(todoWrite) |
| 692 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 693 | { |
| 694 | toolCallChunk("w1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 695 | toolCallChunk("t1", "todo_write", `{"todos":[{"content":"Edit code","status":"in_progress"}]}`), |
| 696 | {Type: provider.ChunkDone}, |
| 697 | }, |
| 698 | {toolCallChunk("b1", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 699 | {toolCallChunk("b2", "bash", `{"command":"git status --short"}`), {Type: provider.ChunkDone}}, |
| 700 | {toolCallChunk("b3", "bash", `{"command":"ls -la"}`), {Type: provider.ChunkDone}}, |
| 701 | {{Type: provider.ChunkText, Text: "blocked by permission"}, {Type: provider.ChunkDone}}, |
| 702 | }} |
| 703 | sink, notices := noticeRecorder() |
| 704 | a := New(prov, reg, NewSession(""), Options{ |
| 705 | Gate: &stubGate{deny: map[string]bool{"bash": true}}, |
| 706 | }, sink) |
| 707 | |
| 708 | if err := a.Run(context.Background(), "edit with todo, then hit bash permission blocks"); err != nil { |
| 709 | t.Fatalf("Run: %v", err) |
| 710 | } |
| 711 | if prov.call != 5 { |
| 712 | t.Fatalf("provider calls = %d, want writer turn, three blocked bash calls, then final", prov.call) |
| 713 | } |
| 714 | if got := lastToolResult(a.session, "bash"); !strings.Contains(got, "[loop guard]") { |
| 715 | t.Fatalf("last bash result = %q, want permission loop guard", got) |
| 716 | } |
| 717 | if got := toolResults(a.session, "bash"); len(got) != stormBreakThreshold { |
| 718 | t.Fatalf("bash results = %d, want exactly %d blocked attempts", len(got), stormBreakThreshold) |
| 719 | } |
| 720 | if len(*notices) == 0 { |
| 721 | t.Fatal("loop guard should emit a user-facing notice") |
| 722 | } |
| 723 | } |
| 724 | |
| 725 | // TestFinalReadinessPermissionLoopGuardAllowsBlockedFinalForBatch pins the |
| 726 | // multi-call variant: the guard text lands on the batch's FIRST result, so any |
| 727 | // detection keyed to the latest tool message misses it. The loop-guard pass is |
| 728 | // host state and must let the model report the blocker regardless of where in |
| 729 | // the batch the guard text sits. |
| 730 | func TestFinalReadinessPermissionLoopGuardAllowsBlockedFinalForBatch(t *testing.T) { |
| 731 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 732 | if !ok { |
| 733 | t.Fatal("todo_write builtin not registered") |
| 734 | } |
| 735 | reg := tool.NewRegistry() |
| 736 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 737 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 738 | reg.Add(todoWrite) |
| 739 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 740 | { |
| 741 | toolCallChunk("w1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 742 | toolCallChunk("t1", "todo_write", `{"todos":[{"content":"Edit code","status":"in_progress"}]}`), |
| 743 | {Type: provider.ChunkDone}, |
| 744 | }, |
| 745 | { |
| 746 | toolCallChunk("b1a", "bash", `{"command":"go test ./..."}`), |
| 747 | toolCallChunk("b1b", "bash", `{"command":"go vet ./..."}`), |
| 748 | {Type: provider.ChunkDone}, |
| 749 | }, |
| 750 | { |
| 751 | toolCallChunk("b2a", "bash", `{"command":"git status --short"}`), |
| 752 | toolCallChunk("b2b", "bash", `{"command":"git diff --stat"}`), |
| 753 | {Type: provider.ChunkDone}, |
| 754 | }, |
| 755 | { |
| 756 | toolCallChunk("b3a", "bash", `{"command":"ls -la"}`), |
| 757 | toolCallChunk("b3b", "bash", `{"command":"pwd"}`), |
| 758 | {Type: provider.ChunkDone}, |
| 759 | }, |
| 760 | {{Type: provider.ChunkText, Text: "blocked by permission"}, {Type: provider.ChunkDone}}, |
| 761 | }} |
| 762 | a := New(prov, reg, NewSession(""), Options{ |
| 763 | Gate: &stubGate{deny: map[string]bool{"bash": true}}, |
| 764 | }, event.Discard) |
| 765 | |
| 766 | if err := a.Run(context.Background(), "edit with todo, then hit batched bash permission blocks"); err != nil { |
| 767 | t.Fatalf("Run: %v", err) |
| 768 | } |
| 769 | if prov.call != 5 { |
| 770 | t.Fatalf("provider calls = %d, want writer turn, three blocked batches, then final", prov.call) |
| 771 | } |
| 772 | results := toolResults(a.session, "bash") |
| 773 | if len(results) != 2*stormBreakThreshold { |
| 774 | t.Fatalf("bash results = %d, want %d blocked attempts across three batches", len(results), 2*stormBreakThreshold) |
| 775 | } |
| 776 | if !strings.Contains(results[len(results)-2], "[loop guard]") { |
| 777 | t.Fatalf("first result of the guarded batch should carry the loop guard, got: %q", results[len(results)-2]) |
| 778 | } |
| 779 | if strings.Contains(results[len(results)-1], "[loop guard]") { |
| 780 | t.Fatalf("last result of the guarded batch should stay untouched (the pass must not depend on it), got: %q", results[len(results)-1]) |
| 781 | } |
| 782 | } |
| 783 | |
| 784 | func TestTodoWriteOnlyTurnMayEndWithIncompleteTodos(t *testing.T) { |
| 785 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 786 | if !ok { |
| 787 | t.Fatal("todo_write builtin not registered") |
| 788 | } |
| 789 | reg := tool.NewRegistry() |
| 790 | reg.Add(todoWrite) |
| 791 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 792 | { |
| 793 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Draft plan","status":"in_progress"},{"content":"Implement","status":"pending"}]}`), |
| 794 | {Type: provider.ChunkDone}, |
| 795 | }, |
| 796 | {{Type: provider.ChunkText, Text: "here is the task list"}, {Type: provider.ChunkDone}}, |
| 797 | }} |
| 798 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 799 | |
| 800 | if err := a.Run(context.Background(), "create a todo list only"); err != nil { |
| 801 | t.Fatalf("Run: %v", err) |
| 802 | } |
| 803 | if prov.call != 2 { |
| 804 | t.Fatalf("provider calls = %d, want 2 without readiness retry", prov.call) |
| 805 | } |
| 806 | if got := lastToolResult(a.session, "todo_write"); !strings.Contains(got, "Todos updated") { |
| 807 | t.Fatalf("todo_write result = %q, want successful todo update", got) |
| 808 | } |
| 809 | } |
| 810 | |
| 811 | func TestReadOnlyContextAndTodoTurnMayEndWithIncompleteTodos(t *testing.T) { |
| 812 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 813 | if !ok { |
| 814 | t.Fatal("todo_write builtin not registered") |
| 815 | } |
| 816 | reg := tool.NewRegistry() |
| 817 | reg.Add(fakeTool{name: "read_file", readOnly: true}) |
| 818 | reg.Add(todoWrite) |
| 819 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 820 | { |
| 821 | toolCallChunk("c1", "read_file", `{"path":"README.md"}`), |
| 822 | toolCallChunk("c2", "todo_write", `{"todos":[{"content":"Draft plan","status":"in_progress"},{"content":"Implement","status":"pending"}]}`), |
| 823 | {Type: provider.ChunkDone}, |
| 824 | }, |
| 825 | {{Type: provider.ChunkText, Text: "I reviewed the context and wrote the list."}, {Type: provider.ChunkDone}}, |
| 826 | }} |
| 827 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 828 | |
| 829 | if err := a.Run(context.Background(), "read context and only draft a todo list"); err != nil { |
| 830 | t.Fatalf("Run: %v", err) |
| 831 | } |
| 832 | if prov.call != 2 { |
| 833 | t.Fatalf("provider calls = %d, want 2 without readiness retry", prov.call) |
| 834 | } |
| 835 | if got := lastToolResult(a.session, "todo_write"); !strings.Contains(got, "Todos updated") { |
| 836 | t.Fatalf("todo_write result = %q, want successful todo update", got) |
| 837 | } |
| 838 | } |
| 839 | |
| 840 | func TestFinalReadinessAuditRecordsTerminalError(t *testing.T) { |
| 841 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 842 | if !ok { |
| 843 | t.Fatal("todo_write builtin not registered") |
| 844 | } |
| 845 | reg := tool.NewRegistry() |
| 846 | reg.Add(fakeTool{name: "write_file", readOnly: false}) |
| 847 | reg.Add(todoWrite) |
| 848 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 849 | { |
| 850 | toolCallChunk("c1", "write_file", `{"path":"changed.go","content":"package main"}`), |
| 851 | toolCallChunk("c2", "todo_write", `{"todos":[{"content":"Edit code","status":"in_progress"}]}`), |
| 852 | {Type: provider.ChunkDone}, |
| 853 | }, |
| 854 | {{Type: provider.ChunkText, Text: "premature 1"}, {Type: provider.ChunkDone}}, |
| 855 | {{Type: provider.ChunkText, Text: "premature 2"}, {Type: provider.ChunkDone}}, |
| 856 | {{Type: provider.ChunkText, Text: "premature 3"}, {Type: provider.ChunkDone}}, |
| 857 | }} |
| 858 | sink := &readinessAuditSink{} |
| 859 | a := New(prov, reg, NewSession(""), Options{}, sink) |
| 860 | |
| 861 | err := a.Run(context.Background(), "edit with todo and never sign off") |
| 862 | if err == nil { |
| 863 | t.Fatal("expected the first readiness block to stop the run") |
| 864 | } |
| 865 | if len(sink.events) != 1 { |
| 866 | t.Fatalf("readiness audit events = %d, want 1 (no retries): %+v", len(sink.events), sink.events) |
| 867 | } |
| 868 | last := sink.events[len(sink.events)-1] |
| 869 | if last.Result != evidence.ReadinessErrored || last.IncompleteTodos == 0 { |
| 870 | t.Fatalf("terminal audit = %+v, want errored with incomplete todos", last) |
| 871 | } |
| 872 | } |
| 873 | |
| 874 | // TestEvidenceFlowRejectsUncitedCommand proves the loop rejects a sign-off whose |
| 875 | // cited command was never run: bash ran "go test", complete_step cites "go vet". |
| 876 | func TestEvidenceFlowRejectsUncitedCommand(t *testing.T) { |
| 877 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 878 | if !ok { |
| 879 | t.Fatal("complete_step builtin not registered") |
| 880 | } |
| 881 | reg := tool.NewRegistry() |
| 882 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 883 | reg.Add(completeStep) |
| 884 | |
| 885 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 886 | { |
| 887 | toolCallChunk("c1", "bash", `{"command":"go test ./..."}`), |
| 888 | toolCallChunk("c2", "complete_step", `{ |
| 889 | "step":"Vet the tree", |
| 890 | "result":"vet is clean", |
| 891 | "evidence":[{"kind":"verification","summary":"go vet passed","command":"go vet ./..."}] |
| 892 | }`), |
| 893 | {Type: provider.ChunkDone}, |
| 894 | }, |
| 895 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 896 | }} |
| 897 | |
| 898 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 899 | if err := a.Run(context.Background(), "vet the tree and sign off"); err != nil { |
| 900 | t.Fatalf("Run: %v", err) |
| 901 | } |
| 902 | |
| 903 | got := toolResult(a.session, "complete_step") |
| 904 | if !strings.Contains(got, "has no matching successful receipt") { |
| 905 | t.Fatalf("complete_step result = %q, want the uncited command rejected", got) |
| 906 | } |
| 907 | if strings.Contains(got, "host-verified") { |
| 908 | t.Fatalf("uncited command should not verify, got %q", got) |
| 909 | } |
| 910 | } |
| 911 | |
| 912 | func TestEvidenceFlowRejectsStepMissingFromTodoWrite(t *testing.T) { |
| 913 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 914 | if !ok { |
| 915 | t.Fatal("todo_write builtin not registered") |
| 916 | } |
| 917 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 918 | if !ok { |
| 919 | t.Fatal("complete_step builtin not registered") |
| 920 | } |
| 921 | reg := tool.NewRegistry() |
| 922 | reg.Add(todoWrite) |
| 923 | reg.Add(completeStep) |
| 924 | |
| 925 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 926 | { |
| 927 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Add parser","status":"in_progress"}]}`), |
| 928 | toolCallChunk("c2", "complete_step", `{ |
| 929 | "step":"Ship parser", |
| 930 | "result":"step is complete", |
| 931 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 932 | }`), |
| 933 | toolCallChunk("c3", "complete_step", `{ |
| 934 | "step":"Add parser", |
| 935 | "result":"parser added", |
| 936 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 937 | }`), |
| 938 | toolCallChunk("c4", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 939 | {Type: provider.ChunkDone}, |
| 940 | }, |
| 941 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 942 | }} |
| 943 | |
| 944 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 945 | if err := a.Run(context.Background(), "update todos then sign off the wrong step"); err != nil { |
| 946 | t.Fatalf("Run: %v", err) |
| 947 | } |
| 948 | |
| 949 | got := toolResult(a.session, "complete_step") |
| 950 | if !strings.Contains(got, "matching todo_write item") { |
| 951 | t.Fatalf("complete_step result = %q, want todo-backed rejection", got) |
| 952 | } |
| 953 | } |
| 954 | |
| 955 | func TestEvidenceFlowAcceptsTodoCompletionAfterCompleteStep(t *testing.T) { |
| 956 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 957 | if !ok { |
| 958 | t.Fatal("todo_write builtin not registered") |
| 959 | } |
| 960 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 961 | if !ok { |
| 962 | t.Fatal("complete_step builtin not registered") |
| 963 | } |
| 964 | reg := tool.NewRegistry() |
| 965 | reg.Add(todoWrite) |
| 966 | reg.Add(completeStep) |
| 967 | |
| 968 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 969 | { |
| 970 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Add parser","status":"in_progress"}]}`), |
| 971 | toolCallChunk("c2", "complete_step", `{ |
| 972 | "step":"Add parser", |
| 973 | "result":"parser added", |
| 974 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 975 | }`), |
| 976 | toolCallChunk("c3", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 977 | {Type: provider.ChunkDone}, |
| 978 | }, |
| 979 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 980 | }} |
| 981 | |
| 982 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 983 | if err := a.Run(context.Background(), "complete the todo with a sign-off first"); err != nil { |
| 984 | t.Fatalf("Run: %v", err) |
| 985 | } |
| 986 | |
| 987 | if got := lastToolResult(a.session, "todo_write"); !strings.Contains(got, "Todos updated") { |
| 988 | t.Fatalf("final todo_write result = %q, want update accepted", got) |
| 989 | } |
| 990 | } |
| 991 | |
| 992 | func TestEvidenceFlowRejectsTodoCompletionWithoutCompleteStep(t *testing.T) { |
| 993 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 994 | if !ok { |
| 995 | t.Fatal("todo_write builtin not registered") |
| 996 | } |
| 997 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 998 | if !ok { |
| 999 | t.Fatal("complete_step builtin not registered") |
| 1000 | } |
| 1001 | reg := tool.NewRegistry() |
| 1002 | reg.Add(todoWrite) |
| 1003 | reg.Add(completeStep) |
| 1004 | |
| 1005 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1006 | { |
| 1007 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Add parser","status":"in_progress"}]}`), |
| 1008 | toolCallChunk("c2", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 1009 | toolCallChunk("c3", "complete_step", `{ |
| 1010 | "step":"Add parser", |
| 1011 | "result":"parser added", |
| 1012 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1013 | }`), |
| 1014 | toolCallChunk("c4", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 1015 | {Type: provider.ChunkDone}, |
| 1016 | }, |
| 1017 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1018 | }} |
| 1019 | |
| 1020 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1021 | if err := a.Run(context.Background(), "complete the todo without a sign-off"); err != nil { |
| 1022 | t.Fatalf("Run: %v", err) |
| 1023 | } |
| 1024 | |
| 1025 | results := toolResults(a.session, "todo_write") |
| 1026 | if len(results) < 2 { |
| 1027 | t.Fatalf("todo_write results = %v, want the rejected completion result", results) |
| 1028 | } |
| 1029 | got := results[1] |
| 1030 | if !strings.Contains(got, "complete_step") { |
| 1031 | t.Fatalf("todo_write result = %q, want completion rejected until complete_step", got) |
| 1032 | } |
| 1033 | } |
| 1034 | |
| 1035 | func TestEvidenceFlowRecoversTodoCompletionAfterFailedCompleteStepWithProgress(t *testing.T) { |
| 1036 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 1037 | if !ok { |
| 1038 | t.Fatal("todo_write builtin not registered") |
| 1039 | } |
| 1040 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 1041 | if !ok { |
| 1042 | t.Fatal("complete_step builtin not registered") |
| 1043 | } |
| 1044 | reg := tool.NewRegistry() |
| 1045 | reg.Add(todoWrite) |
| 1046 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 1047 | reg.Add(completeStep) |
| 1048 | |
| 1049 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1050 | { |
| 1051 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Run project script","status":"in_progress"}]}`), |
| 1052 | toolCallChunk("c2", "bash", `{"command":"python \"script.py\""}`), |
| 1053 | toolCallChunk("c3", "complete_step", `{ |
| 1054 | "step":"Run project script", |
| 1055 | "result":"script ran", |
| 1056 | "evidence":[{"kind":"verification","summary":"script completed","command":"python other.py"}] |
| 1057 | }`), |
| 1058 | toolCallChunk("c4", "todo_write", `{"todos":[{"content":"Run project script","status":"completed"}]}`), |
| 1059 | {Type: provider.ChunkDone}, |
| 1060 | }, |
| 1061 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1062 | }} |
| 1063 | |
| 1064 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1065 | if err := a.Run(context.Background(), "recover after a failed complete_step"); err != nil { |
| 1066 | t.Fatalf("Run: %v", err) |
| 1067 | } |
| 1068 | |
| 1069 | stepResult := lastToolResult(a.session, "complete_step") |
| 1070 | if !strings.Contains(stepResult, "no matching successful receipt") { |
| 1071 | t.Fatalf("complete_step result = %q, want the sign-off attempt to fail first", stepResult) |
| 1072 | } |
| 1073 | if !strings.Contains(stepResult, `python \"script.py\"`) { |
| 1074 | t.Fatalf("complete_step result = %q, want the self-correction hint to include the real command", stepResult) |
| 1075 | } |
| 1076 | if strings.Contains(stepResult, "todo_write") { |
| 1077 | t.Fatalf("complete_step result = %q, want command hints without todo tool noise", stepResult) |
| 1078 | } |
| 1079 | if got := lastToolResult(a.session, "todo_write"); !strings.Contains(got, "1 completed") { |
| 1080 | t.Fatalf("todo_write result = %q, want completion recovery accepted", got) |
| 1081 | } |
| 1082 | } |
| 1083 | |
| 1084 | func TestEvidenceFlowRecoversAfterBatchTodoCompletionRejection(t *testing.T) { |
| 1085 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 1086 | if !ok { |
| 1087 | t.Fatal("todo_write builtin not registered") |
| 1088 | } |
| 1089 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 1090 | if !ok { |
| 1091 | t.Fatal("complete_step builtin not registered") |
| 1092 | } |
| 1093 | reg := tool.NewRegistry() |
| 1094 | reg.Add(todoWrite) |
| 1095 | reg.Add(completeStep) |
| 1096 | |
| 1097 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1098 | { |
| 1099 | toolCallChunk("c1", "todo_write", `{"todos":[ |
| 1100 | {"content":"Port entity imports","status":"in_progress"}, |
| 1101 | {"content":"Run build and tests","status":"pending"} |
| 1102 | ]}`), |
| 1103 | toolCallChunk("c2", "todo_write", `{"todos":[ |
| 1104 | {"content":"Port entity imports","status":"completed"}, |
| 1105 | {"content":"Run build and tests","status":"completed"} |
| 1106 | ]}`), |
| 1107 | {Type: provider.ChunkDone}, |
| 1108 | }, |
| 1109 | { |
| 1110 | toolCallChunk("c3", "complete_step", `{ |
| 1111 | "step":"Port entity imports", |
| 1112 | "result":"entity imports ported", |
| 1113 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1114 | }`), |
| 1115 | toolCallChunk("c4", "complete_step", `{ |
| 1116 | "step":"Run build and tests", |
| 1117 | "result":"build and tests ran", |
| 1118 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1119 | }`), |
| 1120 | {Type: provider.ChunkDone}, |
| 1121 | }, |
| 1122 | { |
| 1123 | toolCallChunk("c5", "complete_step", `{ |
| 1124 | "step":"Run build and tests", |
| 1125 | "result":"build and tests ran", |
| 1126 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1127 | }`), |
| 1128 | {Type: provider.ChunkDone}, |
| 1129 | }, |
| 1130 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1131 | }} |
| 1132 | |
| 1133 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1134 | if err := a.Run(context.Background(), "recover from a rejected batch todo update"); err != nil { |
| 1135 | t.Fatalf("Run: %v", err) |
| 1136 | } |
| 1137 | |
| 1138 | stepResults := toolResults(a.session, "complete_step") |
| 1139 | if len(stepResults) < 3 { |
| 1140 | t.Fatalf("complete_step results = %v, want blocked batch sign-off and a retry", stepResults) |
| 1141 | } |
| 1142 | if got := stepResults[1]; !strings.Contains(got, "only one successful complete_step") { |
| 1143 | t.Fatalf("second batched complete_step result = %q, want serial-signoff block", got) |
| 1144 | } |
| 1145 | if got := stepResults[2]; !strings.Contains(got, "signed off") { |
| 1146 | t.Fatalf("next-round complete_step result = %q, want successful sign-off", got) |
| 1147 | } |
| 1148 | for i, todo := range a.CanonicalTodoState() { |
| 1149 | if todo.Status != "completed" { |
| 1150 | t.Fatalf("canonical todo %d = %+v, want completed", i+1, todo) |
| 1151 | } |
| 1152 | } |
| 1153 | } |
| 1154 | |
| 1155 | func TestEvidenceFlowFailedCompleteStepDoesNotAuthorizeTodoCompletion(t *testing.T) { |
| 1156 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 1157 | if !ok { |
| 1158 | t.Fatal("todo_write builtin not registered") |
| 1159 | } |
| 1160 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 1161 | if !ok { |
| 1162 | t.Fatal("complete_step builtin not registered") |
| 1163 | } |
| 1164 | reg := tool.NewRegistry() |
| 1165 | reg.Add(todoWrite) |
| 1166 | reg.Add(completeStep) |
| 1167 | |
| 1168 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1169 | { |
| 1170 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Add parser","status":"in_progress"}]}`), |
| 1171 | toolCallChunk("c2", "complete_step", `{ |
| 1172 | "step":"Ship parser", |
| 1173 | "result":"parser shipped", |
| 1174 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1175 | }`), |
| 1176 | toolCallChunk("c3", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 1177 | toolCallChunk("c4", "complete_step", `{ |
| 1178 | "step":"Add parser", |
| 1179 | "result":"parser added", |
| 1180 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1181 | }`), |
| 1182 | toolCallChunk("c5", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 1183 | {Type: provider.ChunkDone}, |
| 1184 | }, |
| 1185 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1186 | }} |
| 1187 | |
| 1188 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1189 | if err := a.Run(context.Background(), "attempt completion after a failed sign-off"); err != nil { |
| 1190 | t.Fatalf("Run: %v", err) |
| 1191 | } |
| 1192 | |
| 1193 | results := toolResults(a.session, "todo_write") |
| 1194 | if len(results) < 2 { |
| 1195 | t.Fatalf("todo_write results = %v, want the rejected completion result", results) |
| 1196 | } |
| 1197 | got := results[1] |
| 1198 | if !strings.Contains(got, "complete_step") { |
| 1199 | t.Fatalf("todo_write result = %q, want failed complete_step not to authorize completion", got) |
| 1200 | } |
| 1201 | } |
| 1202 | |
| 1203 | func TestEvidenceFlowRejectsReplacedTodoAfterNumericCompleteStep(t *testing.T) { |
| 1204 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 1205 | if !ok { |
| 1206 | t.Fatal("todo_write builtin not registered") |
| 1207 | } |
| 1208 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 1209 | if !ok { |
| 1210 | t.Fatal("complete_step builtin not registered") |
| 1211 | } |
| 1212 | reg := tool.NewRegistry() |
| 1213 | reg.Add(todoWrite) |
| 1214 | reg.Add(completeStep) |
| 1215 | |
| 1216 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1217 | { |
| 1218 | toolCallChunk("c1", "todo_write", `{"todos":[{"content":"Add parser","status":"in_progress"}]}`), |
| 1219 | toolCallChunk("c2", "complete_step", `{ |
| 1220 | "step":"1", |
| 1221 | "result":"parser added", |
| 1222 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1223 | }`), |
| 1224 | toolCallChunk("c3", "todo_write", `{"todos":[{"content":"Ship parser","status":"completed"}]}`), |
| 1225 | toolCallChunk("c4", "todo_write", `{"todos":[{"content":"Add parser","status":"completed"}]}`), |
| 1226 | {Type: provider.ChunkDone}, |
| 1227 | }, |
| 1228 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1229 | }} |
| 1230 | |
| 1231 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1232 | if err := a.Run(context.Background(), "try to reuse a numeric sign-off for another todo"); err != nil { |
| 1233 | t.Fatalf("Run: %v", err) |
| 1234 | } |
| 1235 | |
| 1236 | results := toolResults(a.session, "todo_write") |
| 1237 | if len(results) < 2 { |
| 1238 | t.Fatalf("todo_write results = %v, want the rejected replacement result", results) |
| 1239 | } |
| 1240 | got := results[1] |
| 1241 | if !strings.Contains(got, "Ship parser") || !strings.Contains(got, "complete_step") { |
| 1242 | t.Fatalf("todo_write result = %q, want replaced todo rejected", got) |
| 1243 | } |
| 1244 | } |
| 1245 | |
| 1246 | func TestEvidenceFlowRejectsReorderedTodoAndRecoversSerially(t *testing.T) { |
| 1247 | todoWrite, ok := tool.LookupBuiltin("todo_write") |
| 1248 | if !ok { |
| 1249 | t.Fatal("todo_write builtin not registered") |
| 1250 | } |
| 1251 | completeStep, ok := tool.LookupBuiltin("complete_step") |
| 1252 | if !ok { |
| 1253 | t.Fatal("complete_step builtin not registered") |
| 1254 | } |
| 1255 | reg := tool.NewRegistry() |
| 1256 | reg.Add(todoWrite) |
| 1257 | reg.Add(completeStep) |
| 1258 | |
| 1259 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 1260 | { |
| 1261 | toolCallChunk("c1", "todo_write", `{"todos":[ |
| 1262 | {"content":"Add parser","status":"in_progress"}, |
| 1263 | {"content":"Write tests","status":"pending"} |
| 1264 | ]}`), |
| 1265 | toolCallChunk("c2", "complete_step", `{ |
| 1266 | "step":"1", |
| 1267 | "result":"parser added", |
| 1268 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1269 | }`), |
| 1270 | toolCallChunk("c3", "todo_write", `{"todos":[ |
| 1271 | {"content":"Write tests","status":"pending"}, |
| 1272 | {"content":"Add parser","status":"completed"} |
| 1273 | ]}`), |
| 1274 | {Type: provider.ChunkDone}, |
| 1275 | }, |
| 1276 | { |
| 1277 | toolCallChunk("c4", "complete_step", `{ |
| 1278 | "step":"Write tests", |
| 1279 | "result":"tests written", |
| 1280 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 1281 | }`), |
| 1282 | {Type: provider.ChunkDone}, |
| 1283 | }, |
| 1284 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 1285 | }} |
| 1286 | |
| 1287 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 1288 | if err := a.Run(context.Background(), "complete the signed todo after reordering it"); err != nil { |
| 1289 | t.Fatalf("Run: %v", err) |
| 1290 | } |
| 1291 | |
| 1292 | results := toolResults(a.session, "todo_write") |
| 1293 | if len(results) != 2 || !strings.Contains(results[1], "completed after unfinished") { |
| 1294 | t.Fatalf("reordered todo_write results = %v, want serial-order rejection", results) |
| 1295 | } |
| 1296 | for i, todo := range a.CanonicalTodoState() { |
| 1297 | if todo.Status != "completed" { |
| 1298 | t.Fatalf("canonical todo %d = %+v, want completed after serial recovery", i+1, todo) |
| 1299 | } |
| 1300 | } |
| 1301 | } |
| 1302 |