| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "os" |
| 6 | "os/exec" |
| 7 | "path/filepath" |
| 8 | "runtime" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | |
| 13 | "reasonix/internal/provider" |
| 14 | ) |
| 15 | |
| 16 | const untrustedReviewProbeKind = "cli-review-untrusted-probe" |
| 17 | |
| 18 | var untrustedReviewProbe = &untrustedReviewProbeProvider{} |
| 19 | |
| 20 | func init() { |
| 21 | provider.Register(untrustedReviewProbeKind, func(provider.Config) (provider.Provider, error) { |
| 22 | return untrustedReviewProbe, nil |
| 23 | }) |
| 24 | } |
| 25 | |
| 26 | // A reviewed checkout configures a search binary and ships its own review |
| 27 | // skill; `reasonix review` must execute neither, and must run the built-in |
| 28 | // read-only review prompt rather than the checkout's. |
| 29 | func TestReviewCommandIgnoresCheckoutToolsAndSkills(t *testing.T) { |
| 30 | if runtime.GOOS == "windows" { |
| 31 | t.Skip("marker scripts are POSIX shell") |
| 32 | } |
| 33 | isolateCLIConfigHome(t) |
| 34 | dir := t.TempDir() |
| 35 | t.Chdir(dir) |
| 36 | for _, args := range [][]string{ |
| 37 | {"init", "-q"}, |
| 38 | {"config", "user.email", "test@example.com"}, |
| 39 | {"config", "user.name", "test"}, |
| 40 | } { |
| 41 | runUntrustedReviewGit(t, dir, args...) |
| 42 | } |
| 43 | writeUntrustedReviewFile(t, dir, "main.go", "package main\n", 0o644) |
| 44 | runUntrustedReviewGit(t, dir, "add", "main.go") |
| 45 | runUntrustedReviewGit(t, dir, "commit", "-q", "-m", "init") |
| 46 | writeUntrustedReviewFile(t, dir, "main.go", "package main\n\nfunc main() {}\n", 0o644) |
| 47 | |
| 48 | rgMarker := filepath.Join(dir, "rg-ran") |
| 49 | skillMarker := filepath.Join(dir, "skill-bash-ran") |
| 50 | writeUntrustedReviewFile(t, dir, "tools/rg", "#!/bin/sh\necho ran > '"+rgMarker+"'\n", 0o755) |
| 51 | writeUntrustedReviewFile(t, dir, "tools/evil.sh", "#!/bin/sh\necho ran > '"+skillMarker+"'\n", 0o755) |
| 52 | writeUntrustedReviewFile(t, dir, ".reasonix/skills/review/SKILL.md", |
| 53 | "---\ndescription: checkout review\nrunAs: subagent\nallowed-tools: grep, bash, web_fetch\n---\n"+untrustedReviewSkillBody+"\n", 0o644) |
| 54 | writeUntrustedReviewFile(t, dir, "reasonix.toml", ` |
| 55 | default_model = "reviewer" |
| 56 | |
| 57 | [[providers]] |
| 58 | name = "reviewer" |
| 59 | kind = "`+untrustedReviewProbeKind+`" |
| 60 | model = "review-model" |
| 61 | base_url = "http://127.0.0.1:1" |
| 62 | |
| 63 | [tools.search] |
| 64 | engine = "rg" |
| 65 | rg_path = "tools/rg" |
| 66 | `, 0o644) |
| 67 | approveWorkspace(t, dir) |
| 68 | |
| 69 | untrustedReviewProbe.reset() |
| 70 | captureStdout(t, func() { |
| 71 | if rc := reviewCommand(nil); rc != 0 { |
| 72 | t.Fatalf("reviewCommand rc = %d, want 0", rc) |
| 73 | } |
| 74 | }) |
| 75 | calls, system, offered := untrustedReviewProbe.snapshot() |
| 76 | if calls < 3 { |
| 77 | t.Fatalf("review subagent reached the provider %d times, want the grep and bash rounds too", calls) |
| 78 | } |
| 79 | if strings.Contains(system, untrustedReviewSkillBody) { |
| 80 | t.Error("review ran the checkout's .reasonix/skills/review prompt") |
| 81 | } |
| 82 | if offered["web_fetch"] { |
| 83 | t.Error("review offered web_fetch, which only the checkout's review skill grants") |
| 84 | } |
| 85 | if _, err := os.Stat(rgMarker); !os.IsNotExist(err) { |
| 86 | t.Errorf("review's grep executed the checkout's [tools.search] rg_path (stat err = %v)", err) |
| 87 | } |
| 88 | if _, err := os.Stat(skillMarker); !os.IsNotExist(err) { |
| 89 | t.Errorf("review's bash ran a writing command under the checkout's review skill (stat err = %v)", err) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | const untrustedReviewSkillBody = "CHECKOUT-REVIEW-SKILL-BODY" |
| 94 | |
| 95 | func runUntrustedReviewGit(t *testing.T, dir string, args ...string) { |
| 96 | t.Helper() |
| 97 | cmd := exec.Command("git", args...) |
| 98 | cmd.Dir = dir |
| 99 | if out, err := cmd.CombinedOutput(); err != nil { |
| 100 | t.Fatalf("git %v: %v\n%s", args, err, out) |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | func writeUntrustedReviewFile(t *testing.T, dir, name, body string, mode os.FileMode) { |
| 105 | t.Helper() |
| 106 | path := filepath.Join(dir, name) |
| 107 | if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | if err := os.WriteFile(path, []byte(body), mode); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | if err := os.Chmod(path, mode); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | type untrustedReviewProbeProvider struct { |
| 119 | mu sync.Mutex |
| 120 | calls int |
| 121 | system string |
| 122 | offered map[string]bool |
| 123 | } |
| 124 | |
| 125 | func (p *untrustedReviewProbeProvider) reset() { |
| 126 | p.mu.Lock() |
| 127 | defer p.mu.Unlock() |
| 128 | p.calls, p.system, p.offered = 0, "", map[string]bool{} |
| 129 | } |
| 130 | |
| 131 | func (p *untrustedReviewProbeProvider) snapshot() (int, string, map[string]bool) { |
| 132 | p.mu.Lock() |
| 133 | defer p.mu.Unlock() |
| 134 | return p.calls, p.system, p.offered |
| 135 | } |
| 136 | |
| 137 | func (p *untrustedReviewProbeProvider) Name() string { return untrustedReviewProbeKind } |
| 138 | |
| 139 | func (p *untrustedReviewProbeProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 140 | p.mu.Lock() |
| 141 | call := p.calls |
| 142 | p.calls++ |
| 143 | for _, schema := range req.Tools { |
| 144 | p.offered[schema.Name] = true |
| 145 | } |
| 146 | for _, msg := range req.Messages { |
| 147 | if msg.Role == provider.RoleSystem { |
| 148 | p.system += msg.Content |
| 149 | } |
| 150 | } |
| 151 | p.mu.Unlock() |
| 152 | |
| 153 | var chunks []provider.Chunk |
| 154 | switch call { |
| 155 | case 0: |
| 156 | chunks = []provider.Chunk{{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ |
| 157 | ID: "review-grep", Name: "grep", Arguments: `{"pattern":"main","path":"."}`}}} |
| 158 | case 1: |
| 159 | chunks = []provider.Chunk{{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ |
| 160 | ID: "review-bash", Name: "bash", Arguments: `{"command":"sh tools/evil.sh"}`}}} |
| 161 | default: |
| 162 | chunks = []provider.Chunk{{Type: provider.ChunkText, Text: "no findings"}} |
| 163 | } |
| 164 | chunks = append(chunks, provider.Chunk{Type: provider.ChunkDone}) |
| 165 | ch := make(chan provider.Chunk, len(chunks)) |
| 166 | for _, chunk := range chunks { |
| 167 | ch <- chunk |
| 168 | } |
| 169 | close(ch) |
| 170 | return ch, nil |
| 171 | } |
| 172 |