| 1 | package control |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | |
| 12 | "reasonix/internal/agent" |
| 13 | "reasonix/internal/event" |
| 14 | "reasonix/internal/evidence" |
| 15 | "reasonix/internal/goaleval" |
| 16 | "reasonix/internal/provider" |
| 17 | "reasonix/internal/store" |
| 18 | "reasonix/internal/tool" |
| 19 | ) |
| 20 | |
| 21 | // goalRuntimeController wires a controller whose goal turns carry no |
| 22 | // update_goal report, so the bounded evaluator decides every disposition. It |
| 23 | // returns the TurnDone/Notice channel for waiting. |
| 24 | func goalRuntimeController(t *testing.T, prov provider.Provider, eval goaleval.Evaluator) (*Controller, *agent.Agent, <-chan event.Event) { |
| 25 | t.Helper() |
| 26 | ag := agent.New(prov, goalRegistry(), agent.NewSession(""), agent.Options{}, event.Discard) |
| 27 | events := make(chan event.Event, 8) |
| 28 | c := New(Options{ |
| 29 | Runner: ag, |
| 30 | Executor: ag, |
| 31 | GoalEvaluator: eval, |
| 32 | Sink: event.FuncSink(func(e event.Event) { |
| 33 | if e.Kind == event.TurnDone || e.Kind == event.Notice { |
| 34 | events <- e |
| 35 | } |
| 36 | }), |
| 37 | }) |
| 38 | return c, ag, events |
| 39 | } |
| 40 | |
| 41 | // waitGoalTurnDone drains notices until the goal loop's TurnDone. |
| 42 | func waitGoalTurnDone(t *testing.T, events <-chan event.Event) { |
| 43 | t.Helper() |
| 44 | for e := range events { |
| 45 | if e.Kind == event.TurnDone { |
| 46 | return |
| 47 | } |
| 48 | } |
| 49 | t.Fatal("goal loop ended without TurnDone") |
| 50 | } |
| 51 | |
| 52 | // TestSimpleGoalWithoutReportCompletesViaEvaluator pins the acceptance |
| 53 | // criterion: a simple Q&A goal whose model never calls update_goal still ends |
| 54 | // on the first turn when the evaluator says complete. |
| 55 | func TestSimpleGoalWithoutReportCompletesViaEvaluator(t *testing.T) { |
| 56 | prov := &scriptedTurns{turns: [][]provider.Chunk{textTurn("Here is the answer.")}} |
| 57 | c, _, events := goalRuntimeController(t, prov, &fakeGoalEvaluator{outcome: goaleval.OutcomeComplete, reason: "the question is fully answered"}) |
| 58 | |
| 59 | c.Submit("/goal explain the cache behavior") |
| 60 | waitGoalTurnDone(t, events) |
| 61 | |
| 62 | if prov.call != 1 { |
| 63 | t.Fatalf("provider calls = %d, want 1 (evaluator decides on the first turn, no second round)", prov.call) |
| 64 | } |
| 65 | if got := c.GoalStatus(); got != GoalStatusComplete { |
| 66 | t.Fatalf("GoalStatus() = %q, want complete", got) |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | func TestGoalEvaluatorUsageCommitsBeforeFSMCompletion(t *testing.T) { |
| 71 | sink := NewGoalUsageTee(event.Discard) |
| 72 | mainProv := &scriptedTurns{turns: [][]provider.Chunk{textTurn("Here is the answer.")}} |
| 73 | evalProv := &scriptedTurns{turns: [][]provider.Chunk{{ |
| 74 | {Type: provider.ChunkText, Text: `{"outcome":"complete","reason":"done"}`}, |
| 75 | {Type: provider.ChunkUsage, Usage: &provider.Usage{PromptTokens: 60, CompletionTokens: 17, TotalTokens: 77}}, |
| 76 | {Type: provider.ChunkDone}, |
| 77 | }}} |
| 78 | executor := agent.New(mainProv, goalRegistry(), agent.NewSession(""), agent.Options{}, sink) |
| 79 | evaluator := goaleval.NewSessionWithSink(evalProv, nil, "test/evaluator", sink) |
| 80 | c := New(Options{Runner: executor, Executor: executor, GoalEvaluator: evaluator, Sink: sink}) |
| 81 | c.SetGoal("answer once") |
| 82 | if err := newTurnOrchestrator(c).runGoalLoopWithRawDisplay(context.Background(), "answer", "answer", ""); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | if c.GoalStatus() != GoalStatusComplete { |
| 86 | t.Fatalf("status = %q, want complete", c.GoalStatus()) |
| 87 | } |
| 88 | if got := c.GoalRuntime().TokensUsed; got != 77 { |
| 89 | t.Fatalf("evaluator usage = %d, want 77 committed before FSM completion", got) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | // TestEvaluatorOutcomesDriveFSM covers the evaluator verdict matrix. |
| 94 | func TestEvaluatorOutcomesDriveFSM(t *testing.T) { |
| 95 | cases := []struct { |
| 96 | name string |
| 97 | outcome goaleval.Outcome |
| 98 | wantStatus string |
| 99 | wantCause string |
| 100 | }{ |
| 101 | {"complete", goaleval.OutcomeComplete, GoalStatusComplete, ""}, |
| 102 | {"continue", goaleval.OutcomeContinue, GoalStatusRunning, ""}, |
| 103 | {"blocked", goaleval.OutcomeBlocked, GoalStatusBlocked, ""}, |
| 104 | {"uncertain fails closed", goaleval.OutcomeUncertain, GoalStatusBlocked, stopCauseEvaluator}, |
| 105 | } |
| 106 | for _, tc := range cases { |
| 107 | t.Run(tc.name, func(t *testing.T) { |
| 108 | prov := &scriptedTurns{turns: [][]provider.Chunk{textTurn("done.")}} |
| 109 | c, _, events := goalRuntimeController(t, prov, &fakeGoalEvaluator{outcome: tc.outcome, reason: "verdict"}) |
| 110 | c.Submit("/goal assess the impact") |
| 111 | waitGoalTurnDone(t, events) |
| 112 | if tc.wantStatus == GoalStatusRunning { |
| 113 | // continue keeps the loop going; without host-verifiable |
| 114 | // progress it eventually pauses on the no-progress gate rather |
| 115 | // than stopping at turn 1. |
| 116 | if got := c.GoalStatus(); got != GoalStatusBlocked { |
| 117 | t.Fatalf("GoalStatus() = %q, want the loop to keep going until a pause", got) |
| 118 | } |
| 119 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseNoProgress || rt.TurnsUsed < 2 { |
| 120 | t.Fatalf("runtime = %+v, want no-progress pause after multiple turns", rt) |
| 121 | } |
| 122 | return |
| 123 | } |
| 124 | if got := c.GoalStatus(); got != tc.wantStatus { |
| 125 | t.Fatalf("GoalStatus() = %q, want %q", got, tc.wantStatus) |
| 126 | } |
| 127 | if rt := c.GoalRuntime(); rt.StopCause != tc.wantCause { |
| 128 | t.Fatalf("StopCause = %q, want %q", rt.StopCause, tc.wantCause) |
| 129 | } |
| 130 | }) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | // TestEvaluatorErrorPausesFirstTurn pins fail-closed: an erroring evaluator |
| 135 | // pauses the goal on the first turn without looping to a fixed cap. |
| 136 | func TestEvaluatorErrorPausesFirstTurn(t *testing.T) { |
| 137 | prov := &scriptedTurns{turns: [][]provider.Chunk{textTurn("done.")}} |
| 138 | c, _, events := goalRuntimeController(t, prov, &fakeGoalEvaluator{err: context.DeadlineExceeded}) |
| 139 | c.Submit("/goal evaluate this") |
| 140 | waitGoalTurnDone(t, events) |
| 141 | if got := c.GoalStatus(); got != GoalStatusBlocked { |
| 142 | t.Fatalf("GoalStatus() = %q, want blocked (fail closed)", got) |
| 143 | } |
| 144 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseEvaluator || rt.TurnsUsed != 1 { |
| 145 | t.Fatalf("runtime = %+v, want evaluator pause after 1 turn", rt) |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // TestEvaluatorUnavailablePausesFirstTurn pins the no-evaluator configuration. |
| 150 | func TestEvaluatorUnavailablePausesFirstTurn(t *testing.T) { |
| 151 | prov := &scriptedTurns{turns: [][]provider.Chunk{textTurn("done.")}} |
| 152 | c, _, events := goalRuntimeController(t, prov, nil) |
| 153 | c.Submit("/goal evaluate this") |
| 154 | waitGoalTurnDone(t, events) |
| 155 | if got := c.GoalStatus(); got != GoalStatusBlocked { |
| 156 | t.Fatalf("GoalStatus() = %q, want blocked (evaluator unavailable)", got) |
| 157 | } |
| 158 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseEvaluator { |
| 159 | t.Fatalf("StopCause = %q, want %q", rt.StopCause, stopCauseEvaluator) |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | // TestEvaluatorCompleteStillGatedByReadiness: the evaluator's complete claim |
| 164 | // must pass host readiness — seeded incomplete todos keep the goal going. |
| 165 | func TestEvaluatorCompleteStillGatedByReadiness(t *testing.T) { |
| 166 | prov := &scriptedTurns{turns: [][]provider.Chunk{ |
| 167 | textTurn("done."), |
| 168 | textTurn("done again."), |
| 169 | }} |
| 170 | ag := agent.New(prov, goalRegistry(), agent.NewSession(""), agent.Options{}, event.Discard) |
| 171 | ag.SeedTodoState([]evidence.TodoItem{{Content: "Fix the parser", Status: "in_progress"}}) |
| 172 | done := make(chan event.Event, 1) |
| 173 | c := New(Options{ |
| 174 | Runner: ag, |
| 175 | Executor: ag, |
| 176 | GoalEvaluator: &fakeGoalEvaluator{outcome: goaleval.OutcomeComplete, reason: "all done"}, |
| 177 | Sink: event.FuncSink(func(e event.Event) { |
| 178 | if e.Kind == event.TurnDone { |
| 179 | done <- e |
| 180 | } |
| 181 | }), |
| 182 | }) |
| 183 | c.Submit("/goal fix everything") |
| 184 | <-done |
| 185 | // The evaluator's complete claim is rejected (incomplete todos) and the |
| 186 | // goal continues; without host-verifiable progress the no-progress gate |
| 187 | // pauses instead of looping forever. |
| 188 | if got := c.GoalStatus(); got != GoalStatusBlocked { |
| 189 | t.Fatalf("GoalStatus() = %q, want blocked (no-progress after rejected complete)", got) |
| 190 | } |
| 191 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseNoProgress { |
| 192 | t.Fatalf("StopCause = %q, want %q", rt.StopCause, stopCauseNoProgress) |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | // TestTurnTokenNoProgressPausesAndResumeExtendsBudget covers turn and |
| 197 | // no-progress budgets at the FSM level and the resume extension contract. |
| 198 | // Token hard limits no longer pause goals. |
| 199 | func TestTurnTokenNoProgressPausesAndResumeExtendsBudget(t *testing.T) { |
| 200 | newMachine := func() *goalMachine { |
| 201 | g := &goalMachine{goal: "fix the parser", status: GoalStatusRunning} |
| 202 | g.budgetClass = budgetClassWrite |
| 203 | g.turnsLimit = budgetQuota(budgetClassWrite) |
| 204 | g.tokensLimit = 0 |
| 205 | g.noProgressLimit = defaultNoProgressLimit |
| 206 | return g |
| 207 | } |
| 208 | in := func(report *goalTurnReport, before, after string) goalAdvanceInput { |
| 209 | return goalAdvanceInput{report: report, progressBefore: before, progressAfter: after} |
| 210 | } |
| 211 | |
| 212 | t.Run("turn budget pauses", func(t *testing.T) { |
| 213 | g := newMachine() |
| 214 | for i := 0; i < g.turnsLimit; i++ { |
| 215 | // Progress changes every turn so only the turn budget can fire. |
| 216 | res := g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "keep going"}, "s", "s"+fmt.Sprint(i))) |
| 217 | if res.cont && i == g.turnsLimit-1 { |
| 218 | t.Fatal("last turn must pause") |
| 219 | } |
| 220 | } |
| 221 | if g.status != GoalStatusBlocked || g.stopCause != stopCauseBudgetTurns { |
| 222 | t.Fatalf("machine = (%q, %q), want blocked+budget_turns", g.status, g.stopCause) |
| 223 | } |
| 224 | }) |
| 225 | |
| 226 | t.Run("token usage never pauses", func(t *testing.T) { |
| 227 | g := newMachine() |
| 228 | // Even a huge observational total must not stop the goal. |
| 229 | g.tokensUsed = 900_000 |
| 230 | res := g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "keep going"}, "s", "s2")) |
| 231 | if !res.cont { |
| 232 | t.Fatal("token usage must not pause the goal") |
| 233 | } |
| 234 | if g.stopCause != "" { |
| 235 | t.Fatalf("stop cause = %q, want empty", g.stopCause) |
| 236 | } |
| 237 | }) |
| 238 | |
| 239 | t.Run("no-progress pauses", func(t *testing.T) { |
| 240 | g := newMachine() |
| 241 | for i := 0; i < g.noProgressLimit; i++ { |
| 242 | g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "still working"}, "sig", "sig")) |
| 243 | } |
| 244 | if g.status != GoalStatusBlocked || g.stopCause != stopCauseNoProgress { |
| 245 | t.Fatalf("machine = (%q, %q), want blocked+no_progress", g.status, g.stopCause) |
| 246 | } |
| 247 | }) |
| 248 | |
| 249 | t.Run("host-verifiable progress resets no-progress", func(t *testing.T) { |
| 250 | g := newMachine() |
| 251 | for i := 0; i < g.noProgressLimit-1; i++ { |
| 252 | g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "still working"}, "sig", "sig")) |
| 253 | } |
| 254 | res := g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "progress!"}, "sig", "sig2")) |
| 255 | if !res.cont { |
| 256 | t.Fatalf("progress must reset the stall counter: %+v", g) |
| 257 | } |
| 258 | if g.noProgressTurns != 0 { |
| 259 | t.Fatalf("noProgressTurns = %d, want 0", g.noProgressTurns) |
| 260 | } |
| 261 | }) |
| 262 | |
| 263 | t.Run("resume extends turn budget once", func(t *testing.T) { |
| 264 | g := newMachine() |
| 265 | for i := 0; i < g.turnsLimit; i++ { |
| 266 | g.advance(in(&goalTurnReport{status: GoalStatusRunning, reason: "keep going"}, "s", "s")) |
| 267 | } |
| 268 | beforeLimit := g.turnsLimit |
| 269 | _, _, _, resumed, extended := g.resume(nil) |
| 270 | if !resumed || !extended { |
| 271 | t.Fatalf("resume = (%v, %v), want (true, true)", resumed, extended) |
| 272 | } |
| 273 | if g.turnsLimit != beforeLimit+budgetQuota(budgetClassWrite) { |
| 274 | t.Fatalf("turnsLimit = %d, want %d", g.turnsLimit, beforeLimit+budgetQuota(budgetClassWrite)) |
| 275 | } |
| 276 | if g.tokensLimit != 0 { |
| 277 | t.Fatalf("tokensLimit = %d, want 0 (no hard token ceiling)", g.tokensLimit) |
| 278 | } |
| 279 | if g.budgetExtensions != 1 || g.stopCause != "" || g.noProgressTurns != 0 { |
| 280 | t.Fatalf("machine after resume = %+v", g) |
| 281 | } |
| 282 | }) |
| 283 | |
| 284 | t.Run("manual pause resume does not extend", func(t *testing.T) { |
| 285 | g := newMachine() |
| 286 | g.pauseFor(stopCauseManual, "user paused", nil) |
| 287 | beforeLimit := g.turnsLimit |
| 288 | _, _, _, resumed, extended := g.resume(nil) |
| 289 | if !resumed || extended { |
| 290 | t.Fatalf("resume = (%v, %v), want (true, false)", resumed, extended) |
| 291 | } |
| 292 | if g.turnsLimit != beforeLimit { |
| 293 | t.Fatalf("turnsLimit changed on manual resume: %d", g.turnsLimit) |
| 294 | } |
| 295 | }) |
| 296 | } |
| 297 | |
| 298 | // TestGoalTurnRecorderProtocol covers idempotency, upgrades, terminal |
| 299 | // conflicts, and stale-epoch rejection. |
| 300 | func TestGoalTurnRecorderProtocol(t *testing.T) { |
| 301 | newRec := func(t *testing.T) (*goalMachine, *goalTurnRecorder) { |
| 302 | t.Helper() |
| 303 | g := &goalMachine{goal: "fix it", status: GoalStatusRunning} |
| 304 | g.scopeID = newGoalScopeID() |
| 305 | rec := g.newTurnRecorder(g.scopeID, g.continuationEpoch) |
| 306 | return g, rec |
| 307 | } |
| 308 | report := func(status, reason string) tool.GoalReport { |
| 309 | return tool.GoalReport{Status: status, Reason: reason, NextAction: ""} |
| 310 | } |
| 311 | |
| 312 | t.Run("idempotent same value", func(t *testing.T) { |
| 313 | _, rec := newRec(t) |
| 314 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "working")); err != nil { |
| 315 | t.Fatal(err) |
| 316 | } |
| 317 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "working")); err != nil { |
| 318 | t.Fatalf("identical repeat must be idempotent: %v", err) |
| 319 | } |
| 320 | if got := rec.validReport(rec.epoch); got == nil || got.status != GoalStatusRunning { |
| 321 | t.Fatalf("validReport = %+v", got) |
| 322 | } |
| 323 | }) |
| 324 | |
| 325 | t.Run("continue upgrades to complete", func(t *testing.T) { |
| 326 | _, rec := newRec(t) |
| 327 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "working")); err != nil { |
| 328 | t.Fatal(err) |
| 329 | } |
| 330 | if _, err := rec.RecordGoalReport(report(GoalStatusComplete, "")); err != nil { |
| 331 | t.Fatalf("continue → complete upgrade must be allowed: %v", err) |
| 332 | } |
| 333 | if got := rec.validReport(rec.epoch); got == nil || got.status != GoalStatusComplete { |
| 334 | t.Fatalf("validReport = %+v, want complete", got) |
| 335 | } |
| 336 | }) |
| 337 | |
| 338 | t.Run("terminal conflicts rejected", func(t *testing.T) { |
| 339 | _, rec := newRec(t) |
| 340 | if _, err := rec.RecordGoalReport(report(GoalStatusComplete, "")); err != nil { |
| 341 | t.Fatal(err) |
| 342 | } |
| 343 | if _, err := rec.RecordGoalReport(report(GoalStatusBlocked, "actually stuck")); err == nil { |
| 344 | t.Fatal("terminal complete must reject a later blocked report") |
| 345 | } |
| 346 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "just kidding")); err == nil { |
| 347 | t.Fatal("terminal complete must reject a later continue report") |
| 348 | } |
| 349 | }) |
| 350 | |
| 351 | t.Run("conflicting non-terminal rejected", func(t *testing.T) { |
| 352 | _, rec := newRec(t) |
| 353 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "doing A")); err != nil { |
| 354 | t.Fatal(err) |
| 355 | } |
| 356 | if _, err := rec.RecordGoalReport(report(GoalStatusRunning, "doing B")); err == nil { |
| 357 | t.Fatal("conflicting continue reports must be rejected") |
| 358 | } |
| 359 | }) |
| 360 | |
| 361 | t.Run("stale epoch invalidates report", func(t *testing.T) { |
| 362 | g, rec := newRec(t) |
| 363 | if _, err := rec.RecordGoalReport(report(GoalStatusComplete, "")); err != nil { |
| 364 | t.Fatal(err) |
| 365 | } |
| 366 | // The goal is replaced: epoch bumps, scope rotates. |
| 367 | g.set("replacement", GoalResearchAuto, "", nil) |
| 368 | if got := rec.validReport(rec.epoch); got != nil { |
| 369 | t.Fatalf("stale recorder report = %+v, want nil", got) |
| 370 | } |
| 371 | }) |
| 372 | |
| 373 | t.Run("late record after replacement rejected", func(t *testing.T) { |
| 374 | g, rec := newRec(t) |
| 375 | g.set("replacement", GoalResearchAuto, "", nil) |
| 376 | if _, err := rec.RecordGoalReport(report(GoalStatusComplete, "")); err == nil { |
| 377 | t.Fatal("late record on a replaced goal must be rejected") |
| 378 | } |
| 379 | }) |
| 380 | |
| 381 | t.Run("usage folds only for matching lifecycle", func(t *testing.T) { |
| 382 | g, rec := newRec(t) |
| 383 | rec.addUsage(150) |
| 384 | if g.tokensUsed != 150 { |
| 385 | t.Fatalf("tokensUsed = %d, want 150", g.tokensUsed) |
| 386 | } |
| 387 | g.set("replacement", GoalResearchAuto, "", nil) |
| 388 | rec.addUsage(50) |
| 389 | if g.tokensUsed != 0 { |
| 390 | t.Fatalf("stale usage folded into replacement goal: %d", g.tokensUsed) |
| 391 | } |
| 392 | }) |
| 393 | } |
| 394 | |
| 395 | // TestGoalUsageTeeAttributesScopedBillableCallsAndExcludesTitle covers the |
| 396 | // observational token accounting surface: executor/subagent-style usage counts, |
| 397 | // title generation does not. |
| 398 | func TestGoalUsageTeeAttributesScopedBillableCallsAndExcludesTitle(t *testing.T) { |
| 399 | tee := NewGoalUsageTee(event.Discard).(*goalUsageTee) |
| 400 | g := &goalMachine{goal: "ship it", status: GoalStatusRunning} |
| 401 | g.budgetClass = budgetClassWrite |
| 402 | g.turnsLimit = budgetQuota(budgetClassWrite) |
| 403 | g.tokensLimit = 0 |
| 404 | g.noProgressLimit = defaultNoProgressLimit |
| 405 | g.scopeID = newGoalScopeID() |
| 406 | rec := g.newTurnRecorder(g.scopeID, g.continuationEpoch) |
| 407 | tee.setActiveRecorder(rec) |
| 408 | |
| 409 | usage := func(tokens int) *provider.Usage { return &provider.Usage{TotalTokens: tokens} } |
| 410 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(100), UsageSource: event.UsageSourceExecutor}) |
| 411 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(200), UsageSource: event.UsageSourcePlanner}) |
| 412 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(300), UsageSource: event.UsageSourceSubagent}) |
| 413 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(400), UsageSource: event.UsageSourceCompaction}) |
| 414 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(500), UsageSource: event.UsageSourceRecoveryReviewer}) |
| 415 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(600), UsageSource: event.UsageSourceGoalEvaluator}) |
| 416 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(700), UsageSource: event.UsageSourceCapabilityRouter}) |
| 417 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(800), UsageSource: event.UsageSourceClassifier}) |
| 418 | // Title generation and unrelated background calls never count. |
| 419 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(900), UsageSource: event.UsageSourceTitle}) |
| 420 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(1000), UsageSource: event.UsageSourceTitle}) |
| 421 | |
| 422 | if rec.usageTokens() != 100+200+300+400+500+600+700+800 { |
| 423 | t.Fatalf("usageTokens = %d, want 3600", rec.usageTokens()) |
| 424 | } |
| 425 | if g.tokensUsed != 3600 { |
| 426 | t.Fatalf("live goal tokens = %d, want 3600", g.tokensUsed) |
| 427 | } |
| 428 | |
| 429 | // No active goal turn → nothing folds. |
| 430 | tee.setActiveRecorder(nil) |
| 431 | tee.Emit(event.Event{Kind: event.Usage, Usage: usage(50), UsageSource: event.UsageSourceExecutor}) |
| 432 | if rec.usageTokens() != 3600 { |
| 433 | t.Fatalf("usageTokens after span close = %d, want 3600", rec.usageTokens()) |
| 434 | } |
| 435 | } |
| 436 | |
| 437 | func TestBudgetClassForBareFaultIsWrite(t *testing.T) { |
| 438 | // User-reported Chinese bare fault → write turn quota (20), no token ceiling. |
| 439 | class := budgetClassFor("数据模型管理器又出现历史 BUG 了……", GoalResearchAuto) |
| 440 | if class != budgetClassWrite { |
| 441 | t.Fatalf("budget class = %q, want write", class) |
| 442 | } |
| 443 | if turns := budgetQuota(class); turns != 20 { |
| 444 | t.Fatalf("write turn quota = %d, want 20", turns) |
| 445 | } |
| 446 | // Consultative / diagnostic fault statements stay simple. |
| 447 | for _, goal := range []string{ |
| 448 | "为什么会出现这个 BUG？", |
| 449 | "只分析原因，不要修改代码。", |
| 450 | "诊断数据库连接失败原因。", |
| 451 | "复现并定位问题，但不要修复。", |
| 452 | } { |
| 453 | if got := budgetClassFor(goal, GoalResearchAuto); got != budgetClassSimple { |
| 454 | t.Errorf("budgetClassFor(%q) = %q, want simple", goal, got) |
| 455 | } |
| 456 | } |
| 457 | // Explicit mutation verbs remain write. |
| 458 | if got := budgetClassFor("fix the crash in settings", GoalResearchAuto); got != budgetClassWrite { |
| 459 | t.Fatalf("explicit fix class = %q, want write", got) |
| 460 | } |
| 461 | } |
| 462 | |
| 463 | func TestGoalLegacyBudgetTokensSidecarAutoResumes(t *testing.T) { |
| 464 | dir := t.TempDir() |
| 465 | path := filepath.Join(dir, "session.jsonl") |
| 466 | // Old sidecar: paused solely because of the removed token hard limit. |
| 467 | state := goalState{ |
| 468 | Goal: "应用打开设置时崩溃", |
| 469 | Status: GoalStatusBlocked, |
| 470 | StopCause: stopCauseBudgetTokens, |
| 471 | Block: "token budget exhausted (0/200000 tokens used)", |
| 472 | BudgetClass: budgetClassWrite, |
| 473 | TurnsUsed: 1, |
| 474 | TurnsLimit: 20, |
| 475 | TokensUsed: 214_000, |
| 476 | TokensLimit: 200_000, |
| 477 | BudgetExtensions: 0, |
| 478 | NoProgressLimit: defaultNoProgressLimit, |
| 479 | Todos: []evidence.TodoItem{{ |
| 480 | Content: "verify the repaired model mapping", Status: "in_progress", |
| 481 | }}, |
| 482 | } |
| 483 | raw, err := json.Marshal(state) |
| 484 | if err != nil { |
| 485 | t.Fatal(err) |
| 486 | } |
| 487 | if err := os.WriteFile(store.SessionGoalState(path), raw, 0o600); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | g := &goalMachine{} |
| 491 | migPath, migData, migrated := g.restoreFromState(path) |
| 492 | if !migrated { |
| 493 | t.Fatal("legacy budget_tokens pause must migrate") |
| 494 | } |
| 495 | if g.status != GoalStatusRunning || g.stopCause != "" { |
| 496 | t.Fatalf("status/stopCause = %q/%q, want running/empty", g.status, g.stopCause) |
| 497 | } |
| 498 | if g.block != "" { |
| 499 | t.Fatalf("block = %q, want empty after legacy token pause migration", g.block) |
| 500 | } |
| 501 | if g.tokensUsed != 214_000 { |
| 502 | t.Fatalf("tokensUsed = %d, want preserved 214000", g.tokensUsed) |
| 503 | } |
| 504 | if g.tokensLimit != 0 { |
| 505 | t.Fatalf("tokensLimit = %d, want 0", g.tokensLimit) |
| 506 | } |
| 507 | if g.turnsUsed != 1 || g.turnsLimit != 20 { |
| 508 | t.Fatalf("turns = %d/%d, want 1/20", g.turnsUsed, g.turnsLimit) |
| 509 | } |
| 510 | if err := g.writeStateErr(migPath, migData); err != nil { |
| 511 | t.Fatal(err) |
| 512 | } |
| 513 | var migratedState goalState |
| 514 | if err := json.Unmarshal(migData, &migratedState); err != nil { |
| 515 | t.Fatal(err) |
| 516 | } |
| 517 | if len(migratedState.Todos) != 1 || migratedState.Todos[0].Content != "verify the repaired model mapping" { |
| 518 | t.Fatalf("migration lost persisted todos: %+v", migratedState.Todos) |
| 519 | } |
| 520 | // Second load must stay running without re-entering the legacy pause. |
| 521 | g2 := &goalMachine{} |
| 522 | if _, _, migrated2 := g2.restoreFromState(path); migrated2 { |
| 523 | t.Fatal("normalized sidecar migrated a second time") |
| 524 | } |
| 525 | if g2.status != GoalStatusRunning || g2.stopCause != "" { |
| 526 | t.Fatalf("second load = %q/%q, want running/empty", g2.status, g2.stopCause) |
| 527 | } |
| 528 | } |
| 529 | |
| 530 | func TestGoalLargeTokenUsageDoesNotExhaustBudget(t *testing.T) { |
| 531 | g := &goalMachine{ |
| 532 | goal: "ship", status: GoalStatusRunning, |
| 533 | budgetClass: budgetClassSimple, turnsLimit: 10, tokensUsed: 900_000, tokensLimit: 0, |
| 534 | noProgressLimit: defaultNoProgressLimit, |
| 535 | } |
| 536 | if g.budgetExhausted() { |
| 537 | t.Fatal("budgetExhausted must ignore tokensUsed") |
| 538 | } |
| 539 | res := g.advance(goalAdvanceInput{ |
| 540 | report: &goalTurnReport{status: GoalStatusRunning, reason: "progress"}, |
| 541 | progressBefore: "a", |
| 542 | progressAfter: "b", |
| 543 | }) |
| 544 | if !res.cont { |
| 545 | t.Fatal("goal with large tokensUsed must continue while turns remain") |
| 546 | } |
| 547 | } |
| 548 | |
| 549 | // TestGoalUsageTotalTokensFallback checks the prompt+completion fallback when |
| 550 | // TotalTokens is missing (never double-counting cache hit/miss). |
| 551 | func TestGoalUsageTotalTokensFallback(t *testing.T) { |
| 552 | u := &provider.Usage{PromptTokens: 100, CompletionTokens: 20, CacheHitTokens: 90} |
| 553 | if got := usageTotalTokens(u); got != 120 { |
| 554 | t.Fatalf("fallback = %d, want 120 (prompt+completion, no cache double count)", got) |
| 555 | } |
| 556 | u.TotalTokens = 200 |
| 557 | if got := usageTotalTokens(u); got != 200 { |
| 558 | t.Fatalf("TotalTokens preferred = %d, want 200", got) |
| 559 | } |
| 560 | } |
| 561 | |
| 562 | // TestGoalSidecarCompatRestoresOldAndNewFields pins the compatibility contract: |
| 563 | // an old sidecar without the budget fields restores with re-derived defaults, |
| 564 | // and a new sidecar's pause (blocked + stopCause) survives a controller rebuild |
| 565 | // without failing open. |
| 566 | func TestGoalSidecarCompatRestoresOldAndNewFields(t *testing.T) { |
| 567 | t.Run("old sidecar restores with defaults", func(t *testing.T) { |
| 568 | dir := t.TempDir() |
| 569 | path := filepath.Join(dir, "session.jsonl") |
| 570 | // Old sidecar: only goal/status/turns — no budget fields. |
| 571 | data := []byte(`{"goal":"legacy goal","status":"running","turns":3}`) |
| 572 | if err := os.WriteFile(store.SessionGoalState(path), data, 0o600); err != nil { |
| 573 | t.Fatal(err) |
| 574 | } |
| 575 | exec := agent.New(nil, nil, agent.NewSession("sys"), agent.Options{}, event.Discard) |
| 576 | c := New(Options{Executor: exec, SessionDir: dir, Label: "test"}) |
| 577 | c.Resume(agent.NewSession("sys"), path) |
| 578 | rt := c.GoalRuntime() |
| 579 | if rt.TurnsUsed != 3 { |
| 580 | t.Fatalf("TurnsUsed = %d, want 3 (legacy Turns carried over)", rt.TurnsUsed) |
| 581 | } |
| 582 | if rt.TokensUsed != 0 { |
| 583 | t.Fatalf("TokensUsed = %d, want 0 (no legacy token record)", rt.TokensUsed) |
| 584 | } |
| 585 | if rt.TurnsLimit == 0 { |
| 586 | t.Fatalf("turn limit not re-derived: %+v", rt) |
| 587 | } |
| 588 | if rt.TokensLimit != 0 { |
| 589 | t.Fatalf("TokensLimit = %d, want 0 (no hard token limit)", rt.TokensLimit) |
| 590 | } |
| 591 | }) |
| 592 | |
| 593 | t.Run("new sidecar pause survives rebuild", func(t *testing.T) { |
| 594 | dir := t.TempDir() |
| 595 | path := filepath.Join(dir, "session.jsonl") |
| 596 | exec := agent.New(nil, nil, agent.NewSession("sys"), agent.Options{}, event.Discard) |
| 597 | c := New(Options{Executor: exec, SessionDir: dir, SessionPath: path, Label: "test"}) |
| 598 | c.SetGoal("ship the release") |
| 599 | c.goals.pauseFor(stopCauseBudgetTurns, "turn budget exhausted", nil) |
| 600 | statePath, data, ok := c.goals.buildStateLocked(nil) |
| 601 | if !ok { |
| 602 | t.Fatal("no persisted state") |
| 603 | } |
| 604 | if err := os.WriteFile(statePath, data, 0o600); err != nil { |
| 605 | t.Fatal(err) |
| 606 | } |
| 607 | |
| 608 | freshExec := agent.New(nil, nil, agent.NewSession("sys"), agent.Options{}, event.Discard) |
| 609 | fresh := New(Options{Executor: freshExec, SessionDir: dir, Label: "fresh"}) |
| 610 | fresh.Resume(agent.NewSession("sys"), path) |
| 611 | if fresh.GoalStatus() != GoalStatusBlocked { |
| 612 | t.Fatalf("restored status = %q, want blocked (safe pause never fails open)", fresh.GoalStatus()) |
| 613 | } |
| 614 | if rt := fresh.GoalRuntime(); rt.StopCause != stopCauseBudgetTurns { |
| 615 | t.Fatalf("restored stop cause = %q, want %q", rt.StopCause, stopCauseBudgetTurns) |
| 616 | } |
| 617 | // Resuming a budget-paused goal extends the budget. |
| 618 | if !fresh.ResumeGoal() { |
| 619 | t.Fatal("resume rejected the restored paused goal") |
| 620 | } |
| 621 | if rt := fresh.GoalRuntime(); rt.BudgetExtensions != 1 { |
| 622 | t.Fatalf("budget extensions = %d, want 1 after resuming a budget pause", rt.BudgetExtensions) |
| 623 | } |
| 624 | }) |
| 625 | } |
| 626 | |
| 627 | // TestGoalPauseResumeCommands covers the /goal pause and /goal resume CLI |
| 628 | // surface plus the runtime view. |
| 629 | func TestGoalPauseResumeCommands(t *testing.T) { |
| 630 | cmd, ok := ParseGoalCommand("/goal pause") |
| 631 | if !ok || cmd.Action != GoalCommandPause { |
| 632 | t.Fatalf("ParseGoalCommand(/goal pause) = %+v", cmd) |
| 633 | } |
| 634 | cmd, ok = ParseGoalCommand("/goal resume") |
| 635 | if !ok || cmd.Action != GoalCommandResume { |
| 636 | t.Fatalf("ParseGoalCommand(/goal resume) = %+v", cmd) |
| 637 | } |
| 638 | cmd, ok = ParseGoalCommand("/goal") |
| 639 | if !ok || cmd.Action != GoalCommandStatus { |
| 640 | t.Fatalf("ParseGoalCommand(/goal) = %+v", cmd) |
| 641 | } |
| 642 | |
| 643 | c := New(Options{Sink: event.Discard}) |
| 644 | if c.PauseGoal() { |
| 645 | t.Fatal("PauseGoal without a goal must return false") |
| 646 | } |
| 647 | c.SetGoal("long-running research") |
| 648 | if !c.PauseGoal() { |
| 649 | t.Fatal("PauseGoal on a running goal must return true") |
| 650 | } |
| 651 | if got := c.GoalStatus(); got != GoalStatusBlocked { |
| 652 | t.Fatalf("GoalStatus() = %q, want blocked", got) |
| 653 | } |
| 654 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseManual { |
| 655 | t.Fatalf("StopCause = %q, want manual", rt.StopCause) |
| 656 | } |
| 657 | // The goal text and budget survive the pause. |
| 658 | if got := c.Goal(); got != "long-running research" { |
| 659 | t.Fatalf("Goal() = %q, want preserved", got) |
| 660 | } |
| 661 | if !c.ResumeGoal() { |
| 662 | t.Fatal("ResumeGoal on a manually paused goal must return true") |
| 663 | } |
| 664 | if got := c.GoalStatus(); got != GoalStatusRunning { |
| 665 | t.Fatalf("GoalStatus() after resume = %q, want running", got) |
| 666 | } |
| 667 | if rt := c.GoalRuntime(); rt.StopCause != "" { |
| 668 | t.Fatalf("StopCause after resume = %q, want cleared", rt.StopCause) |
| 669 | } |
| 670 | } |
| 671 | |
| 672 | // TestGoalRuntimeViewPopulatesFromController covers the runtime view surface |
| 673 | // the CLI and desktop read. |
| 674 | func TestGoalRuntimeViewPopulatesFromController(t *testing.T) { |
| 675 | c := New(Options{Sink: event.Discard}) |
| 676 | c.SetGoal("finish the migration") |
| 677 | rt := c.GoalRuntime() |
| 678 | if rt.TurnsUsed != 0 || rt.TurnsLimit == 0 || rt.NoProgressLimit == 0 { |
| 679 | t.Fatalf("runtime view = %+v, want derived turn budget defaults", rt) |
| 680 | } |
| 681 | if rt.TokensLimit != 0 { |
| 682 | t.Fatalf("TokensLimit = %d, want 0 (no hard token limit)", rt.TokensLimit) |
| 683 | } |
| 684 | } |
| 685 | |
| 686 | // TestFooterTextDoesNotDriveGoalState pins the acceptance criterion: a |
| 687 | // historical [goal:complete] footer in the latest answer never influences the |
| 688 | // FSM — only the structured tool report does. |
| 689 | func TestFooterTextDoesNotDriveGoalState(t *testing.T) { |
| 690 | prov := &scriptedTurns{turns: [][]provider.Chunk{textTurn("All done.\n\n[goal:complete]")}} |
| 691 | c, _, events := goalRuntimeController(t, prov, &fakeGoalEvaluator{outcome: goaleval.OutcomeContinue, reason: "work is ongoing"}) |
| 692 | c.Submit("/goal migrate the storage") |
| 693 | waitGoalTurnDone(t, events) |
| 694 | // The footer alone must never complete the goal: the evaluator's continue |
| 695 | // keeps it going until the no-progress gate pauses it. |
| 696 | if got := c.GoalStatus(); got == GoalStatusComplete { |
| 697 | t.Fatal("a [goal:complete] footer must not complete the goal") |
| 698 | } |
| 699 | if rt := c.GoalRuntime(); rt.StopCause != stopCauseNoProgress { |
| 700 | t.Fatalf("runtime = %+v, want no-progress pause (footer ignored)", rt) |
| 701 | } |
| 702 | c.ClearGoal() |
| 703 | for _, m := range c.History() { |
| 704 | if m.Role == provider.RoleUser && strings.Contains(m.Content, "update_goal") { |
| 705 | return |
| 706 | } |
| 707 | } |
| 708 | t.Fatal("goal prompt should instruct the update_goal protocol") |
| 709 | } |
| 710 | |
| 711 | // minimalFakeTool is a no-op tool for delivery-flow tests. |
| 712 | type minimalFakeTool struct { |
| 713 | name string |
| 714 | readOnly bool |
| 715 | } |
| 716 | |
| 717 | func (f minimalFakeTool) Name() string { return f.name } |
| 718 | func (f minimalFakeTool) Description() string { return "" } |
| 719 | func (f minimalFakeTool) Schema() json.RawMessage { return json.RawMessage(`{"type":"object"}`) } |
| 720 | func (f minimalFakeTool) ReadOnly() bool { return f.readOnly } |
| 721 | func (f minimalFakeTool) Execute(context.Context, json.RawMessage) (string, error) { |
| 722 | return f.name + " done", nil |
| 723 | } |
| 724 | |
| 725 | // TestGoalDeliveryWorkflowCompletesAfterVerifiedSignoff covers the |
| 726 | // Goal + Delivery combination: the model works (edit → verify → review → |
| 727 | // complete_step), reports complete via update_goal, and the goal completes — |
| 728 | // no user-facing recovery card. |
| 729 | func TestGoalDeliveryWorkflowCompletesAfterVerifiedSignoff(t *testing.T) { |
| 730 | todoWrite, _ := tool.LookupBuiltin("todo_write") |
| 731 | completeStep, _ := tool.LookupBuiltin("complete_step") |
| 732 | reg := goalRegistry() |
| 733 | reg.Add(todoWrite) |
| 734 | reg.Add(completeStep) |
| 735 | reg.Add(minimalFakeTool{name: "write_file"}) |
| 736 | reg.Add(minimalFakeTool{name: "read_file", readOnly: true}) |
| 737 | reg.Add(minimalFakeTool{name: "bash"}) |
| 738 | |
| 739 | prov := &scriptedTurns{turns: flattenTurns( |
| 740 | [][]provider.Chunk{ |
| 741 | {toolCallChunk("t0", "todo_write", `{"todos":[{"content":"Ship main","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 742 | {toolCallChunk("w1", "write_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 743 | {toolCallChunk("rv", "read_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 744 | {toolCallChunk("vf", "bash", `{"command":"go test ./..."}`), {Type: provider.ChunkDone}}, |
| 745 | {toolCallChunk("sg", "complete_step", `{"step":"Ship main","result":"implemented","evidence":[{"kind":"verification","summary":"tests pass","command":"go test ./..."}]}`), {Type: provider.ChunkDone}}, |
| 746 | {toolCallChunk("ug", "update_goal", `{"status":"complete","reason":""}`), {Type: provider.ChunkDone}}, |
| 747 | textTurn("Ship main delivered."), |
| 748 | }, |
| 749 | )} |
| 750 | ag := agent.New(prov, reg, agent.NewSession(""), agent.Options{DeliveryProfile: true}, event.Discard) |
| 751 | done := make(chan event.Event, 1) |
| 752 | var doneReadiness *event.FinalReadiness |
| 753 | c := New(Options{ |
| 754 | Runner: ag, |
| 755 | Executor: ag, |
| 756 | Sink: event.FuncSink(func(e event.Event) { |
| 757 | if e.Kind == event.TurnDone { |
| 758 | doneReadiness = e.Readiness |
| 759 | done <- e |
| 760 | } |
| 761 | }), |
| 762 | }) |
| 763 | c.Submit("/goal implement main") |
| 764 | <-done |
| 765 | |
| 766 | if got := c.GoalStatus(); got != GoalStatusComplete { |
| 767 | t.Fatalf("GoalStatus() = %q, want complete after verified sign-off", got) |
| 768 | } |
| 769 | if doneReadiness != nil { |
| 770 | t.Fatalf("TurnDone.Readiness = %+v, want nil (Goal absorbs readiness; no recovery card)", doneReadiness) |
| 771 | } |
| 772 | if got := c.Goal(); got != "" { |
| 773 | t.Fatalf("completed goal should be cleared, got %q", got) |
| 774 | } |
| 775 | } |
| 776 | |
| 777 | // TestPlainDeliveryReadinessFailureSurfacesRecoveryCardWithoutRetries covers |
| 778 | // the plain (non-Goal) Delivery combination: readiness failure ends the run on |
| 779 | // the first final answer, surfaces the recovery card, and never auto-continues. |
| 780 | func TestPlainDeliveryReadinessFailureSurfacesRecoveryCardWithoutRetries(t *testing.T) { |
| 781 | todoWrite, _ := tool.LookupBuiltin("todo_write") |
| 782 | reg := tool.NewRegistry() |
| 783 | reg.Add(todoWrite) |
| 784 | reg.Add(minimalFakeTool{name: "write_file"}) |
| 785 | prov := &scriptedTurns{turns: [][]provider.Chunk{ |
| 786 | {toolCallChunk("w1", "write_file", `{"path":"main.go"}`), {Type: provider.ChunkDone}}, |
| 787 | {toolCallChunk("t0", "todo_write", `{"todos":[{"content":"Ship main","status":"in_progress"}]}`), {Type: provider.ChunkDone}}, |
| 788 | textTurn("premature final"), |
| 789 | textTurn("extra turn that must never run"), |
| 790 | }} |
| 791 | ag := agent.New(prov, reg, agent.NewSession(""), agent.Options{DeliveryProfile: true}, event.Discard) |
| 792 | done := make(chan event.Event, 1) |
| 793 | c := New(Options{ |
| 794 | Runner: ag, |
| 795 | Executor: ag, |
| 796 | Sink: event.FuncSink(func(e event.Event) { |
| 797 | if e.Kind == event.TurnDone { |
| 798 | done <- e |
| 799 | } |
| 800 | }), |
| 801 | }) |
| 802 | |
| 803 | c.Submit("implement main") |
| 804 | ev := <-done |
| 805 | if ev.Readiness == nil || len(ev.Readiness.Missing) == 0 { |
| 806 | t.Fatalf("TurnDone.Readiness = %+v, want missing requirements for the recovery card", ev.Readiness) |
| 807 | } |
| 808 | if prov.call != 3 { |
| 809 | t.Fatalf("provider calls = %d, want 3 (work turn + final answer, no readiness retries)", prov.call) |
| 810 | } |
| 811 | if got := c.GoalStatus(); got != GoalStatusStopped { |
| 812 | t.Fatalf("GoalStatus() = %q, want stopped (no goal involved)", got) |
| 813 | } |
| 814 | } |
| 815 |