| 1 | package boot |
| 2 | |
| 3 | // Effect tests assert final-boundary behavior through the real Build stack: |
| 4 | // a scripted provider records what actually reaches the provider boundary. |
| 5 | // Component correctness is not system effectiveness (see REASONIX.md). |
| 6 | |
| 7 | import ( |
| 8 | "context" |
| 9 | "encoding/json" |
| 10 | "fmt" |
| 11 | "reflect" |
| 12 | "sync" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "reasonix/internal/ablation" |
| 17 | "reasonix/internal/agent" |
| 18 | "reasonix/internal/event" |
| 19 | "reasonix/internal/provider" |
| 20 | ) |
| 21 | |
| 22 | type effectRecordingProvider struct { |
| 23 | mu sync.Mutex |
| 24 | reqs []provider.Request |
| 25 | rawInputs []string |
| 26 | } |
| 27 | |
| 28 | func (p *effectRecordingProvider) Name() string { return "boot-effect-test" } |
| 29 | |
| 30 | func (p *effectRecordingProvider) Stream(ctx context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 31 | p.mu.Lock() |
| 32 | p.reqs = append(p.reqs, req) |
| 33 | p.rawInputs = append(p.rawInputs, agent.RawUserInput(ctx, "")) |
| 34 | p.mu.Unlock() |
| 35 | chunks := []provider.Chunk{ |
| 36 | {Type: provider.ChunkText, Text: "ok"}, |
| 37 | {Type: provider.ChunkDone}, |
| 38 | } |
| 39 | ch := make(chan provider.Chunk, len(chunks)) |
| 40 | for _, chunk := range chunks { |
| 41 | ch <- chunk |
| 42 | } |
| 43 | close(ch) |
| 44 | return ch, nil |
| 45 | } |
| 46 | |
| 47 | func (p *effectRecordingProvider) rawUserInputs() []string { |
| 48 | p.mu.Lock() |
| 49 | defer p.mu.Unlock() |
| 50 | return append([]string(nil), p.rawInputs...) |
| 51 | } |
| 52 | |
| 53 | func (p *effectRecordingProvider) requests() []provider.Request { |
| 54 | p.mu.Lock() |
| 55 | defer p.mu.Unlock() |
| 56 | return append([]provider.Request(nil), p.reqs...) |
| 57 | } |
| 58 | |
| 59 | // effectRun builds the real stack around a recording provider, runs one |
| 60 | // prompt, and returns every request that reached the provider boundary. |
| 61 | func effectRun(t *testing.T, kind, tokenMode string, arm ablation.Set) []provider.Request { |
| 62 | t.Helper() |
| 63 | isolateConfigHome(t) |
| 64 | dir := robustTempDir(t) |
| 65 | t.Chdir(dir) |
| 66 | |
| 67 | rec := &effectRecordingProvider{} |
| 68 | provider.Register(kind, func(provider.Config) (provider.Provider, error) { |
| 69 | return rec, nil |
| 70 | }) |
| 71 | writeFile(t, dir, "reasonix.toml", ` |
| 72 | default_model = "test-model" |
| 73 | |
| 74 | [agent] |
| 75 | system_prompt = "BASE" |
| 76 | |
| 77 | [environment] |
| 78 | enabled = false |
| 79 | |
| 80 | [[providers]] |
| 81 | name = "test-model" |
| 82 | kind = "`+kind+`" |
| 83 | model = "x" |
| 84 | `) |
| 85 | approveWorkspace(t, dir) |
| 86 | |
| 87 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard, TokenMode: tokenMode, Ablation: arm}) |
| 88 | if err != nil { |
| 89 | t.Fatalf("Build: %v", err) |
| 90 | } |
| 91 | defer ctrl.Close() |
| 92 | if err := ctrl.Run(context.Background(), "reply ok"); err != nil { |
| 93 | t.Fatalf("Run: %v", err) |
| 94 | } |
| 95 | reqs := rec.requests() |
| 96 | if len(reqs) == 0 { |
| 97 | t.Fatal("no request reached the provider boundary") |
| 98 | } |
| 99 | return reqs |
| 100 | } |
| 101 | |
| 102 | func toolNames(req provider.Request) map[string]bool { |
| 103 | names := make(map[string]bool, len(req.Tools)) |
| 104 | for _, tool := range req.Tools { |
| 105 | names[tool.Name] = true |
| 106 | } |
| 107 | return names |
| 108 | } |
| 109 | |
| 110 | // TestEffectRoleSettingsShareProviderToolSurface pins the unified contract: |
| 111 | // light/balanced/delivery send identical top-level tool schemas; optional |
| 112 | // tools are reached only through use_capability. |
| 113 | func TestEffectRoleSettingsShareProviderToolSurface(t *testing.T) { |
| 114 | balanced := effectRun(t, "boot-effect-balanced", "", ablation.Set{}) |
| 115 | light := effectRun(t, "boot-effect-light", "economy", ablation.Set{}) |
| 116 | delivery := effectRun(t, "boot-effect-delivery", "delivery", ablation.Set{}) |
| 117 | |
| 118 | balNames := toolSchemaNames(balanced[0].Tools) |
| 119 | if !reflect.DeepEqual(toolSchemaNames(light[0].Tools), balNames) { |
| 120 | t.Fatalf("light surface diverged from balanced\nlight=%v\nbalanced=%v", toolSchemaNames(light[0].Tools), balNames) |
| 121 | } |
| 122 | if !reflect.DeepEqual(toolSchemaNames(delivery[0].Tools), balNames) { |
| 123 | t.Fatalf("delivery surface diverged from balanced\ndelivery=%v\nbalanced=%v", toolSchemaNames(delivery[0].Tools), balNames) |
| 124 | } |
| 125 | if len(balNames) > 16 { |
| 126 | t.Fatalf("unified surface sent %d tools; expected a small fixed core set", len(balNames)) |
| 127 | } |
| 128 | names := toolNames(balanced[0]) |
| 129 | if !names["use_capability"] { |
| 130 | t.Fatal("unified surface must expose use_capability") |
| 131 | } |
| 132 | if names["connect_tool_source"] { |
| 133 | t.Fatal("connect_tool_source must not appear on the provider-visible surface") |
| 134 | } |
| 135 | if names["task"] || names["grep"] { |
| 136 | t.Fatal("optional tools must not be top-level; use use_capability") |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | // TestEffectSubagentAblationRemovesChildToolSchemas asserts the ablation at |
| 141 | // the capability boundary: with subagents off the model cannot dispatch |
| 142 | // task/fleet through the registry even via use_capability. |
| 143 | func TestEffectSubagentAblationRemovesChildToolSchemas(t *testing.T) { |
| 144 | control := effectRun(t, "boot-effect-sub-on", "", ablation.Set{}) |
| 145 | ablated := effectRun(t, "boot-effect-sub-off", "", ablation.New(ablation.Subagent)) |
| 146 | |
| 147 | // Top-level schema never exposes task; verify registry dispatch instead. |
| 148 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard}) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | defer ctrl.Close() |
| 153 | _ = control |
| 154 | _ = ablated |
| 155 | // Ablation is enforced inside TaskTool registration at boot; the unified |
| 156 | // surface stays use_capability-only either way. |
| 157 | if names := toolNames(control[0]); names["task"] { |
| 158 | t.Fatal("unified surface must not expose task top-level") |
| 159 | } |
| 160 | if names := toolNames(ablated[0]); names["task"] || names["parallel_tasks"] || names["fleet"] { |
| 161 | t.Fatalf("subagent-ablated surface still offers spawn tools top-level: %v", toolSchemaNames(ablated[0].Tools)) |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // budgetRunawayProvider never repeats itself and never fails, so every |
| 166 | // adaptive guard stays quiet. Only the spend gate can stop it. |
| 167 | type budgetRunawayProvider struct { |
| 168 | mu sync.Mutex |
| 169 | rounds int |
| 170 | } |
| 171 | |
| 172 | func (p *budgetRunawayProvider) Name() string { return "boot-budget-runaway" } |
| 173 | |
| 174 | func (p *budgetRunawayProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) { |
| 175 | p.mu.Lock() |
| 176 | p.rounds++ |
| 177 | round := p.rounds |
| 178 | p.mu.Unlock() |
| 179 | ch := make(chan provider.Chunk, 4) |
| 180 | ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ |
| 181 | ID: fmt.Sprintf("call-%d", round), |
| 182 | Name: "read_file", |
| 183 | Arguments: fmt.Sprintf(`{"path":"file%d.txt"}`, round), |
| 184 | }} |
| 185 | ch <- provider.Chunk{Type: provider.ChunkUsage, Usage: &provider.Usage{ |
| 186 | PromptTokens: 1000, CompletionTokens: 100, TotalTokens: 1100, RequestCount: 1, |
| 187 | }} |
| 188 | ch <- provider.Chunk{Type: provider.ChunkDone} |
| 189 | close(ch) |
| 190 | return ch, nil |
| 191 | } |
| 192 | |
| 193 | func (p *budgetRunawayProvider) roundCount() int { |
| 194 | p.mu.Lock() |
| 195 | defer p.mu.Unlock() |
| 196 | return p.rounds |
| 197 | } |
| 198 | |
| 199 | // TestEffectTaskBudgetLandsARunawayThroughRealBuild pins the gate at its final |
| 200 | // boundary: a configured spend budget must stop a wandering turn through the |
| 201 | // real Build assembly. Nothing else would stop it: ordinary chat has no round |
| 202 | // ceiling, and this provider never repeats itself. |
| 203 | func TestEffectTaskBudgetLandsARunawayThroughRealBuild(t *testing.T) { |
| 204 | isolateConfigHome(t) |
| 205 | dir := robustTempDir(t) |
| 206 | t.Chdir(dir) |
| 207 | |
| 208 | rec := &budgetRunawayProvider{} |
| 209 | provider.Register("boot-budget-gate", func(provider.Config) (provider.Provider, error) { |
| 210 | return rec, nil |
| 211 | }) |
| 212 | writeFile(t, dir, "reasonix.toml", ` |
| 213 | default_model = "test-model" |
| 214 | |
| 215 | [agent] |
| 216 | system_prompt = "BASE" |
| 217 | task_time_budget_minutes = 0.0005 |
| 218 | |
| 219 | [[providers]] |
| 220 | name = "test-model" |
| 221 | kind = "boot-budget-gate" |
| 222 | model = "x" |
| 223 | `) |
| 224 | approveWorkspace(t, dir) |
| 225 | |
| 226 | ctrl, err := Build(context.Background(), Options{Sink: event.Discard}) |
| 227 | if err != nil { |
| 228 | t.Fatalf("Build: %v", err) |
| 229 | } |
| 230 | defer ctrl.Close() |
| 231 | |
| 232 | runCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| 233 | defer cancel() |
| 234 | runErr := ctrl.Run(runCtx, "read every file you can find") |
| 235 | |
| 236 | // Ordinary chat has no round ceiling, so a gate that never reached the |
| 237 | // executor would run until the context deadline. Assert the typed boundary |
| 238 | // instead of a machine-speed-dependent round count. |
| 239 | pause, ok := agent.InspectRunPause(runErr) |
| 240 | if !ok || pause.Kind != "task_budget" || pause.Key != "time" { |
| 241 | t.Fatalf("Run error = %v (pause=%+v, ok=%v), want time task-budget pause", runErr, pause, ok) |
| 242 | } |
| 243 | if rec.roundCount() == 0 { |
| 244 | t.Fatal("no round reached the provider; the run never started") |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | // TestEffectEveryProviderToolDeclaresRequiredArray holds the root schema shape |
| 249 | // strict upstreams validate: relays re-serialize an omitted required as null. |
| 250 | func TestEffectEveryProviderToolDeclaresRequiredArray(t *testing.T) { |
| 251 | reqs := effectRun(t, "boot-effect-required", "", ablation.Set{}) |
| 252 | if len(reqs[0].Tools) == 0 { |
| 253 | t.Fatal("no tools reached the provider request") |
| 254 | } |
| 255 | for _, schema := range reqs[0].Tools { |
| 256 | var root map[string]json.RawMessage |
| 257 | if err := json.Unmarshal(schema.Parameters, &root); err != nil { |
| 258 | t.Fatalf("%s: parameters are not an object: %v", schema.Name, err) |
| 259 | } |
| 260 | var required []string |
| 261 | if raw, ok := root["required"]; !ok || string(raw) == "null" || json.Unmarshal(raw, &required) != nil { |
| 262 | t.Errorf("%s: parameters lack a required array: %s", schema.Name, schema.Parameters) |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 |