| 1 | package capability |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | |
| 7 | "reasonix/internal/config" |
| 8 | "reasonix/internal/plugin" |
| 9 | "reasonix/internal/tool" |
| 10 | ) |
| 11 | |
| 12 | func boolPtr(b bool) *bool { return &b } |
| 13 | |
| 14 | func TestLoadCachedToolsForSpecsHonorsSchemaCacheKey(t *testing.T) { |
| 15 | t.Setenv("REASONIX_CACHE_HOME", t.TempDir()) |
| 16 | fresh := plugin.Spec{Name: "gh", Type: "stdio", Command: "gh-mcp"} |
| 17 | if err := plugin.SaveCachedSchema("gh", plugin.CachedSchema{ |
| 18 | CacheKey: plugin.SchemaCacheKey(fresh), |
| 19 | Tools: []plugin.CachedTool{{Name: "search_issues", Description: "search", ReadOnly: true}}, |
| 20 | }); err != nil { |
| 21 | t.Fatal(err) |
| 22 | } |
| 23 | stale := plugin.Spec{Name: "old", Type: "stdio", Command: "old-mcp"} |
| 24 | if err := plugin.SaveCachedSchema("old", plugin.CachedSchema{ |
| 25 | CacheKey: "some-other-cache-key", |
| 26 | Tools: []plugin.CachedTool{{Name: "do_thing"}}, |
| 27 | }); err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | |
| 31 | cached, keyOK := LoadCachedToolsForSpecs([]plugin.Spec{fresh, stale, {Name: "absent"}}, plugin.HostProfileCore) |
| 32 | if len(cached["gh"]) != 1 || !keyOK["gh"] { |
| 33 | t.Fatalf("fresh cache: tools=%v keyOK=%v", cached["gh"], keyOK["gh"]) |
| 34 | } |
| 35 | if len(cached["old"]) != 1 || keyOK["old"] { |
| 36 | t.Fatalf("stale cache must load with keyOK=false: tools=%v keyOK=%v", cached["old"], keyOK["old"]) |
| 37 | } |
| 38 | if _, ok := cached["absent"]; ok { |
| 39 | t.Fatal("server without cache must be absent") |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func TestBuildCatalogSurfacesCachedToolsForAutoStartFalse(t *testing.T) { |
| 44 | cached := map[string][]plugin.CachedTool{ |
| 45 | "gh": {{Name: "search_issues", Description: "search", ReadOnly: true}}, |
| 46 | "old": {{Name: "do_thing"}}, |
| 47 | } |
| 48 | keyOK := map[string]bool{"gh": true, "old": false} |
| 49 | cat := BuildCatalog(CatalogOptions{ |
| 50 | Plugins: []config.PluginEntry{ |
| 51 | {Name: "gh", AutoStart: boolPtr(false)}, |
| 52 | {Name: "old", AutoStart: boolPtr(false)}, |
| 53 | }, |
| 54 | CachedTools: cached, |
| 55 | CacheKeyOK: keyOK, |
| 56 | }) |
| 57 | byID := map[string]Entry{} |
| 58 | for _, e := range cat.Entries { |
| 59 | byID[e.ID] = e |
| 60 | } |
| 61 | toolEntry, ok := byID["mcp-tool:gh/search_issues"] |
| 62 | if !ok { |
| 63 | t.Fatalf("cached tool missing from catalog: %v", cat.Entries) |
| 64 | } |
| 65 | if !toolEntry.ReadOnly || toolEntry.ToolName == "" { |
| 66 | t.Fatalf("cached tool entry lost metadata: %+v", toolEntry) |
| 67 | } |
| 68 | if server := byID["mcp-server:old"]; server.Status != StatusStale { |
| 69 | t.Fatalf("cache-key-mismatched schema should mark the server stale, got %q", server.Status) |
| 70 | } |
| 71 | if staleTool, ok := byID["mcp-tool:old/do_thing"]; !ok { |
| 72 | t.Fatal("stale cached tools should still appear as candidates") |
| 73 | } else if staleTool.Status != StatusStale { |
| 74 | t.Fatalf("stale server's cached tools must inherit stale, got %q", staleTool.Status) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | func TestRecordRouterUsageAccumulates(t *testing.T) { |
| 79 | a := &Audit{} |
| 80 | a.RecordRouterUsage(100, 20, 0.005, 340) |
| 81 | a.RecordRouterUsage(50, 10, 0.002, 160) |
| 82 | snap := a.Snapshot() |
| 83 | if snap.RouterPromptTokens != 150 || snap.RouterCompletionTokens != 30 { |
| 84 | t.Fatalf("token counters: prompt=%d completion=%d", snap.RouterPromptTokens, snap.RouterCompletionTokens) |
| 85 | } |
| 86 | if snap.RouterCost < 0.0069 || snap.RouterCost > 0.0071 { |
| 87 | t.Fatalf("cost = %v", snap.RouterCost) |
| 88 | } |
| 89 | if snap.RouterLatencyMs != 500 { |
| 90 | t.Fatalf("latency = %v", snap.RouterLatencyMs) |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | func TestAuditRecordsDecisionFunnelAndDecline(t *testing.T) { |
| 95 | a := &Audit{} |
| 96 | a.RecordDecision(RouteDecision{Candidates: []RouteCandidate{ |
| 97 | {Policy: AutoUseRequire}, |
| 98 | {Policy: AutoUsePrefer}, |
| 99 | {Policy: AutoUseSuggest}, |
| 100 | }}) |
| 101 | a.RecordDecline() |
| 102 | snap := a.Snapshot() |
| 103 | if snap.RoutedCandidates != 3 || snap.RoutedRequire != 1 || snap.RoutedPrefer != 1 || snap.RoutedSuggest != 1 || snap.Declines != 1 { |
| 104 | t.Fatalf("decision funnel audit: candidates=%d require=%d prefer=%d suggest=%d declines=%d", |
| 105 | snap.RoutedCandidates, snap.RoutedRequire, snap.RoutedPrefer, snap.RoutedSuggest, snap.Declines) |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | func TestDeliveryRouteRenderKeepsCapabilityIDAndProxyInstruction(t *testing.T) { |
| 110 | entry := Entry{ |
| 111 | ID: "mcp-tool:gh/search_issues", Kind: KindMCPTool, Name: "gh/search_issues", |
| 112 | Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "gh", |
| 113 | } |
| 114 | d := RouteDecision{ClosedLoop: true, Candidates: []RouteCandidate{{Entry: entry, Policy: AutoUsePrefer, Reason: "matches task"}}} |
| 115 | out := RenderTransientBlock(d) |
| 116 | if !strings.Contains(out, "mcp-tool:gh/search_issues") { |
| 117 | t.Fatalf("delivery render must keep the concrete capability id:\n%s", out) |
| 118 | } |
| 119 | if !strings.Contains(out, `use_capability(action="call", capability_id="mcp-tool:gh/search_issues"`) { |
| 120 | t.Fatalf("delivery render must instruct the proxy call:\n%s", out) |
| 121 | } |
| 122 | if strings.Contains(out, "connect_tool_source") { |
| 123 | t.Fatalf("connect_tool_source is not registered in Delivery:\n%s", out) |
| 124 | } |
| 125 | // Server entries direct the model to connect-and-list via the same proxy. |
| 126 | server := Entry{ID: "mcp-server:gh", Kind: KindMCPServer, Name: "gh", Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "gh"} |
| 127 | out = RenderTransientBlock(RouteDecision{ClosedLoop: true, Candidates: []RouteCandidate{{Entry: server, Policy: AutoUseSuggest, Reason: "r"}}}) |
| 128 | if !strings.Contains(out, `use_capability(action="call", capability_id="mcp-server:gh")`) || !strings.Contains(out, "list its tools") { |
| 129 | t.Fatalf("server candidate must instruct connect-and-list:\n%s", out) |
| 130 | } |
| 131 | // Non-delivery keeps the historical connect_tool_source instruction. |
| 132 | d.ClosedLoop = false |
| 133 | out = RenderTransientBlock(d) |
| 134 | if !strings.Contains(out, "connect_tool_source") { |
| 135 | t.Fatalf("non-delivery render lost connect_tool_source:\n%s", out) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | func TestCapabilityProxyRouteRenderKeepsConcreteMCPIDs(t *testing.T) { |
| 140 | for _, entry := range []Entry{ |
| 141 | {ID: "mcp-tool:gh/search_issues", Kind: KindMCPTool, Name: "gh/search_issues", Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "gh"}, |
| 142 | {ID: "mcp-server:gh", Kind: KindMCPServer, Name: "gh", Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "gh"}, |
| 143 | } { |
| 144 | out := RenderTransientBlock(RouteDecision{ |
| 145 | CapabilityProxy: true, |
| 146 | Candidates: []RouteCandidate{{Entry: entry, Policy: AutoUsePrefer, Reason: "matches task"}}, |
| 147 | }) |
| 148 | if !strings.Contains(out, "- "+entry.ID+" ") { |
| 149 | t.Fatalf("capability proxy route must lead with the concrete id %q:\n%s", entry.ID, out) |
| 150 | } |
| 151 | if strings.Contains(out, "source:mcp/gh") { |
| 152 | t.Fatalf("capability proxy route rewrote %q to an unusable source target:\n%s", entry.ID, out) |
| 153 | } |
| 154 | if !strings.Contains(out, `use_capability(action="call", capability_id="`+entry.ID+`"`) { |
| 155 | t.Fatalf("capability proxy route lost the concrete call instruction for %q:\n%s", entry.ID, out) |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // CapabilityProxy replaces the connector for every capability kind: |
| 160 | // connect_tool_source is unregistered, so a configured skill must route |
| 161 | // through the stable proxy with its concrete id. |
| 162 | skill := Entry{ID: "skill:review", Kind: KindSkill, Name: "review", Status: StatusConfigured, ConnectSource: "skills"} |
| 163 | out := RenderTransientBlock(RouteDecision{ |
| 164 | CapabilityProxy: true, |
| 165 | Candidates: []RouteCandidate{{Entry: skill, Policy: AutoUseSuggest, Reason: "matches task"}}, |
| 166 | }) |
| 167 | if !strings.Contains(out, "- skill:review ") || !strings.Contains(out, `use_capability(action="call", capability_id="skill:review"`) { |
| 168 | t.Fatalf("capability proxy route lost the concrete skill call instruction:\n%s", out) |
| 169 | } |
| 170 | if strings.Contains(out, "connect_tool_source") || strings.Contains(out, "source:skills") { |
| 171 | t.Fatalf("capability proxy route emitted the retired skill connector:\n%s", out) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | func TestMCPServerEntriesPropagatesFailureToCachedTools(t *testing.T) { |
| 176 | entries := MCPServerEntries(CatalogOptions{ |
| 177 | Plugins: []config.PluginEntry{{Name: "github", Type: "http", URL: "https://example.test/mcp"}}, |
| 178 | Failed: map[string]string{"github": "http 401"}, |
| 179 | CachedTools: map[string][]plugin.CachedTool{ |
| 180 | "github": {{Name: "search_issues", Description: "search issues", ReadOnly: true}}, |
| 181 | }, |
| 182 | }) |
| 183 | |
| 184 | for _, entry := range entries { |
| 185 | if entry.ID == "mcp-tool:github/search_issues" { |
| 186 | if entry.Status != StatusFailed { |
| 187 | t.Fatalf("cached tool status = %q, want %q", entry.Status, StatusFailed) |
| 188 | } |
| 189 | return |
| 190 | } |
| 191 | } |
| 192 | t.Fatal("cached MCP tool entry not found") |
| 193 | } |
| 194 | |
| 195 | func TestBuildCatalogUnavailableServerOverridesRegistryCachedTool(t *testing.T) { |
| 196 | const server = "github" |
| 197 | tests := []struct { |
| 198 | name string |
| 199 | failed map[string]string |
| 200 | disabled map[string]bool |
| 201 | want Status |
| 202 | reason string |
| 203 | }{ |
| 204 | {name: "failed", failed: map[string]string{server: "http 401"}, want: StatusFailed, reason: "http 401"}, |
| 205 | {name: "disabled", disabled: map[string]bool{server: true}, want: StatusDisabled}, |
| 206 | } |
| 207 | for _, tc := range tests { |
| 208 | t.Run(tc.name, func(t *testing.T) { |
| 209 | cat := BuildCatalog(CatalogOptions{ |
| 210 | Tools: []tool.ContractEntry{{ |
| 211 | Name: plugin.ModelToolName(server, "search_issues"), |
| 212 | Description: "search issues", |
| 213 | ReadOnly: true, |
| 214 | }}, |
| 215 | Plugins: []config.PluginEntry{{Name: server, Type: "http", URL: "https://example.test/mcp"}}, |
| 216 | Failed: tc.failed, |
| 217 | Disabled: tc.disabled, |
| 218 | CachedTools: map[string][]plugin.CachedTool{ |
| 219 | server: {{Name: "search_issues", Description: "search issues", ReadOnly: true}}, |
| 220 | }, |
| 221 | }) |
| 222 | |
| 223 | entry, ok := cat.Lookup("mcp-tool:github/search_issues") |
| 224 | if !ok { |
| 225 | t.Fatal("registry-backed cached MCP tool missing from catalog") |
| 226 | } |
| 227 | if entry.Status != tc.want || entry.FailureReason != tc.reason { |
| 228 | t.Fatalf("registry-backed cached tool = %+v, want status=%q reason=%q", entry, tc.want, tc.reason) |
| 229 | } |
| 230 | if decision := Route("查一下 GitHub issue", cat.Entries); len(decision.Candidates) != 0 { |
| 231 | t.Fatalf("unavailable registry-backed cached MCP tool was routed: %+v", decision.Candidates) |
| 232 | } |
| 233 | }) |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | func TestOrdinaryRouteRenderDeduplicatesCollapsedMCPSourceLines(t *testing.T) { |
| 238 | candidates := []RouteCandidate{ |
| 239 | { |
| 240 | Entry: Entry{ |
| 241 | ID: "mcp-tool:search/search", Kind: KindMCPTool, Name: "search/search", |
| 242 | Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "search", |
| 243 | }, |
| 244 | Policy: AutoUsePrefer, Reason: "the task appears to need fresh external data", |
| 245 | }, |
| 246 | { |
| 247 | Entry: Entry{ |
| 248 | ID: "mcp-tool:search/fetch", Kind: KindMCPTool, Name: "search/fetch", |
| 249 | Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "search", |
| 250 | }, |
| 251 | Policy: AutoUsePrefer, Reason: "the task appears to need fresh external data", |
| 252 | }, |
| 253 | { |
| 254 | Entry: Entry{ |
| 255 | ID: "mcp-tool:docs/read", Kind: KindMCPTool, Name: "docs/read", |
| 256 | Status: StatusConfigured, ConnectSource: "mcp", ConnectName: "docs", |
| 257 | }, |
| 258 | Policy: AutoUsePrefer, Reason: "the task appears to need fresh external data", |
| 259 | }, |
| 260 | } |
| 261 | |
| 262 | out := RenderTransientBlock(RouteDecision{Candidates: candidates}) |
| 263 | if got := strings.Count(out, "- source:mcp/search "); got != 1 { |
| 264 | t.Fatalf("collapsed MCP source rendered %d times, want 1:\n%s", got, out) |
| 265 | } |
| 266 | if got := strings.Count(out, "- source:mcp/docs "); got != 1 { |
| 267 | t.Fatalf("independent MCP source rendered %d times, want 1:\n%s", got, out) |
| 268 | } |
| 269 | |
| 270 | for _, decision := range []RouteDecision{ |
| 271 | {ClosedLoop: true, Candidates: candidates}, |
| 272 | {CapabilityProxy: true, Candidates: candidates}, |
| 273 | } { |
| 274 | proxyOut := RenderTransientBlock(decision) |
| 275 | for _, candidate := range candidates { |
| 276 | if !strings.Contains(proxyOut, "- "+candidate.Entry.ID+" ") { |
| 277 | t.Fatalf("proxy route lost concrete capability %q:\n%s", candidate.Entry.ID, proxyOut) |
| 278 | } |
| 279 | } |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | func TestCatalogKeepsProxyToolsAfterConnect(t *testing.T) { |
| 284 | proxy := map[string][]plugin.CachedTool{ |
| 285 | "gh": {{Name: "search_issues", Description: "search", ReadOnly: true}}, |
| 286 | } |
| 287 | cat := BuildCatalog(CatalogOptions{ |
| 288 | Plugins: []config.PluginEntry{{Name: "gh", AutoStart: boolPtr(false)}}, |
| 289 | Connected: map[string]bool{"gh": true}, // server is ready now |
| 290 | ProxyTools: proxy, |
| 291 | }) |
| 292 | byID := map[string]Entry{} |
| 293 | for _, e := range cat.Entries { |
| 294 | byID[e.ID] = e |
| 295 | } |
| 296 | toolEntry, ok := byID["mcp-tool:gh/search_issues"] |
| 297 | if !ok { |
| 298 | t.Fatalf("proxy-connected tool vanished from catalog: %+v", cat.Entries) |
| 299 | } |
| 300 | if toolEntry.Status != StatusReady { |
| 301 | t.Fatalf("proxy-connected tool should be ready, got %q", toolEntry.Status) |
| 302 | } |
| 303 | // When the same server's tools are already on the registry, no duplicates. |
| 304 | cat = BuildCatalog(CatalogOptions{ |
| 305 | Tools: []tool.ContractEntry{{Name: plugin.ModelToolName("gh", "search_issues")}}, |
| 306 | Plugins: []config.PluginEntry{{Name: "gh", AutoStart: boolPtr(false)}}, |
| 307 | Connected: map[string]bool{"gh": true}, |
| 308 | ProxyTools: proxy, |
| 309 | }) |
| 310 | count := 0 |
| 311 | for _, e := range cat.Entries { |
| 312 | if e.ID == "mcp-tool:gh/search_issues" { |
| 313 | count++ |
| 314 | } |
| 315 | } |
| 316 | // The registry's own ToolEntries contribution is the single source here; |
| 317 | // the proxy snapshot must not add a duplicate. |
| 318 | if count != 1 { |
| 319 | t.Fatalf("registry-backed server should have exactly one catalog entry, got %d", count) |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | func TestCatalogDoesNotRouteProxyToolsAfterFailure(t *testing.T) { |
| 324 | cat := BuildCatalog(CatalogOptions{ |
| 325 | Plugins: []config.PluginEntry{{Name: "gh", AutoStart: boolPtr(false)}}, |
| 326 | Failed: map[string]string{"gh": "connection reset"}, |
| 327 | Connected: map[string]bool{"gh": true}, |
| 328 | CachedTools: map[string][]plugin.CachedTool{"gh": {{Name: "search_issues"}}}, |
| 329 | ProxyTools: map[string][]plugin.CachedTool{"gh": {{Name: "search_issues"}}}, |
| 330 | }) |
| 331 | entry, ok := cat.Lookup("mcp-tool:gh/search_issues") |
| 332 | if !ok || entry.Status != StatusFailed { |
| 333 | t.Fatalf("failed server proxy tool = (%+v, %v), want failed catalog entry", entry, ok) |
| 334 | } |
| 335 | } |
| 336 |