| 1 | // Guards that stand between the model and the user's machine, over the real |
| 2 | // MCP server with the fake backend (nothing reaches osascript or the desktop): |
| 3 | // - app_script policy: shell escapes refused, named apps go through the |
| 4 | // consent ledger (System Events and its processes included); |
| 5 | // - irreversible-action confirmation: pay/buy/order/send/transfer/delete |
| 6 | // controls need a per-call user confirmation that no app grant covers. |
| 7 | import { hostKeysLine, attest, attestParams } from "./fixtures/host-decision.mjs"; |
| 8 | import { test, before, after } from "node:test"; |
| 9 | import assert from "node:assert/strict"; |
| 10 | import { spawn } from "node:child_process"; |
| 11 | import fs from "node:fs"; |
| 12 | import os from "node:os"; |
| 13 | import path from "node:path"; |
| 14 | import url from "node:url"; |
| 15 | import { checkAppScript, appScriptMode } from "../src/app-script-policy.mjs"; |
| 16 | import { helperStaleness, newerVersion } from "../src/app-socket.mjs"; |
| 17 | |
| 18 | const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); |
| 19 | const ROOT = path.resolve(__dirname, ".."); |
| 20 | const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-guard-state-")); |
| 21 | const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-guard-rec-")); |
| 22 | const work = fs.mkdtempSync(path.join(os.tmpdir(), "cu-guard-")); |
| 23 | const callsFile = path.join(work, "calls.jsonl"); |
| 24 | const controlFile = path.join(work, "control.json"); |
| 25 | |
| 26 | const EXTRA = [ |
| 27 | { index: 9, path: [0, 3], windowIndex: 0, role: "AXButton", label: "Place order", position: { x: 200, y: 20 }, size: { w: 80, h: 30 } }, |
| 28 | { index: 10, path: [0, 4], windowIndex: 0, role: "AXButton", label: "Delete", position: { x: 300, y: 20 }, size: { w: 60, h: 30 } }, |
| 29 | { index: 11, path: [0, 5], windowIndex: 0, role: "AXTextField", label: "Send to", position: { x: 10, y: 100 }, size: { w: 150, h: 25 } }, |
| 30 | ]; |
| 31 | |
| 32 | let server; |
| 33 | let buf = ""; |
| 34 | const pending = new Map(); |
| 35 | let nextId = 1; |
| 36 | |
| 37 | function rpc(method, params) { |
| 38 | const id = nextId++; |
| 39 | return new Promise((resolve, reject) => { |
| 40 | const t = setTimeout(() => { pending.delete(id); reject(new Error(`timeout: ${method}`)); }, 30_000); |
| 41 | pending.set(id, (msg) => { clearTimeout(t); resolve(msg); }); |
| 42 | server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: attestParams(method, params) }) + "\n"); |
| 43 | }); |
| 44 | } |
| 45 | async function tool(name, args = {}) { |
| 46 | const res = await rpc("tools/call", { name, arguments: args }); |
| 47 | assert.ok(res.result, `${name}: protocol error ${JSON.stringify(res.error ?? {})}`); |
| 48 | return JSON.parse(res.result.content[0].text); |
| 49 | } |
| 50 | const calls = (method) => fs.existsSync(callsFile) |
| 51 | ? fs.readFileSync(callsFile, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((c) => c.method === method) |
| 52 | : []; |
| 53 | const answerResolve = (element) => fs.writeFileSync(controlFile, JSON.stringify({ found: true, element })); |
| 54 | |
| 55 | before(async () => { |
| 56 | server = spawn(process.execPath, [path.join(ROOT, "mcp", "server.mjs")], { |
| 57 | env: { |
| 58 | ...process.env, |
| 59 | CODEWHALE_CU_APP: "off", |
| 60 | CODEWHALE_CU_APP_SCRIPT: "", |
| 61 | CODEWHALE_CU_STATE_DIR: stateDir, |
| 62 | CODEWHALE_CU_RECORDINGS_DIR: recDir, |
| 63 | CODEWHALE_CU_TEST_BACKEND: path.join(__dirname, "fixtures", "fake-backend.mjs"), |
| 64 | FAKE_BACKEND_CALLS: callsFile, |
| 65 | FAKE_BACKEND_CONTROL: controlFile, |
| 66 | FAKE_BACKEND_EXTRA_ELEMENTS: JSON.stringify(EXTRA), |
| 67 | }, |
| 68 | stdio: ["pipe", "pipe", "pipe"], |
| 69 | }); |
| 70 | server.stdin.write(hostKeysLine()); |
| 71 | server.stdout.setEncoding("utf8"); |
| 72 | server.stdout.on("data", (d) => { |
| 73 | buf += d; |
| 74 | let i; |
| 75 | while ((i = buf.indexOf("\n")) !== -1) { |
| 76 | const line = buf.slice(0, i).trim(); |
| 77 | buf = buf.slice(i + 1); |
| 78 | if (!line) continue; |
| 79 | try { |
| 80 | const msg = JSON.parse(line); |
| 81 | if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); } |
| 82 | } catch {} |
| 83 | } |
| 84 | }); |
| 85 | await rpc("initialize", { protocolVersion: "2025-06-18" }); |
| 86 | assert.equal((await tool("consent", { action: "allow", app: "FakeApp" })).ok, true); |
| 87 | }); |
| 88 | |
| 89 | after(() => { |
| 90 | try { server.stdin.end(); } catch {} |
| 91 | server?.kill("SIGTERM"); |
| 92 | for (const d of [stateDir, recDir, work]) fs.rmSync(d, { recursive: true, force: true }); |
| 93 | }); |
| 94 | |
| 95 | // ---- app_script policy (unit) ---- |
| 96 | |
| 97 | test("app_script policy refuses shell escapes in AppleScript and JXA", () => { |
| 98 | for (const [script, language] of [ |
| 99 | ['do shell script "id"', "applescript"], |
| 100 | ['do shell ¬\n script "id"', "applescript"], |
| 101 | ['tell application "Terminal" to do script "id"', "applescript"], |
| 102 | ['«event sysoexec» "id"', "applescript"], |
| 103 | ['use framework "Foundation"\ncurrent application\'s NSTask\'s new()', "applescript"], |
| 104 | ['run script "do shell" & " script \\"id\\""', "applescript"], |
| 105 | ['tell application "System Events" to keystroke "id"', "applescript"], |
| 106 | ['var a = Application.currentApplication(); a.includeStandardAdditions = true; a.doShellScript("id")', "javascript"], |
| 107 | ['var a = Application.currentApplication(); a["do" + "ShellScript"]("id")', "javascript"], |
| 108 | ['var k = "doShell" + "Script"; var o = {[k]: 1}', "javascript"], |
| 109 | ['ObjC.import("Foundation"); $.NSTask.alloc.init', "javascript"], |
| 110 | ['[].constructor.constructor("return 1")()', "javascript"], |
| 111 | ['Reflect.get(Application.currentApplication(), "x")', "javascript"], |
| 112 | ['Application("iTerm2").createWindowWithDefaultProfile()', "javascript"], |
| 113 | ]) { |
| 114 | const r = checkAppScript(script, language); |
| 115 | assert.ok(r.refused, `must refuse: ${script}`); |
| 116 | } |
| 117 | }); |
| 118 | |
| 119 | test("app_script policy names every target app and refuses targets it cannot read", () => { |
| 120 | assert.deepEqual(checkAppScript('return "whole computer"').targets, [{ bundle_id: "com.apple.osascript" }]); |
| 121 | assert.deepEqual(checkAppScript('tell application "Finder" to get name of every window').targets, [{ name: "Finder" }]); |
| 122 | assert.deepEqual(checkAppScript('tell application id "com.apple.Safari" to get URL of front document').targets, [{ bundle_id: "com.apple.Safari" }]); |
| 123 | const se = checkAppScript('tell application "System Events" to tell process "Safari" to click button 1 of window 1'); |
| 124 | assert.equal(se.refused, null); |
| 125 | assert.deepEqual(se.targets, [{ name: "System Events" }, { name: "Safari" }]); |
| 126 | const jxa = checkAppScript('Application("System Events").processes.byName("Safari").windows[0].name()', "javascript"); |
| 127 | assert.equal(jxa.refused, null); |
| 128 | assert.deepEqual(jxa.targets, [{ name: "System Events" }, { name: "Safari" }]); |
| 129 | assert.equal(checkAppScript("set p to path to application support folder from user domain").refused, null); |
| 130 | assert.equal(checkAppScript('Application("Finder").windows.at(0).name()', "javascript").refused, null); |
| 131 | for (const [script, language] of [ |
| 132 | ['tell application ("Term" & "inal") to activate', "applescript"], |
| 133 | ['tell application "System Events" to tell (first process whose frontmost is true) to click button 1', "applescript"], |
| 134 | ['tell application "System Events" to click button 1 of window 1 of process 1', "applescript"], |
| 135 | ['var n = "Fin" + "der"; Application(n).activate()', "javascript"], |
| 136 | ['Application("System Events").processes.whose({frontmost: true})[0].name()', "javascript"], |
| 137 | ]) assert.ok(checkAppScript(script, language).refused, `must refuse: ${script}`); |
| 138 | }); |
| 139 | |
| 140 | test("app_script with no named app is refused or needs osascript consent", () => { |
| 141 | assert.ok(checkAppScript('read POSIX file "/etc/hosts"').refused); |
| 142 | for (const script of ['set f to open for access POSIX file "/tmp/x" with write permission', 'write "x" to file "Macintosh HD:tmp:x"', 'open location "https://example.com"', 'mount volume "smb://host/share"', 'system attribute "HOME"']) { |
| 143 | assert.ok(checkAppScript(script).refused, script); |
| 144 | } |
| 145 | const bare = checkAppScript("return (current date) as string"); |
| 146 | assert.equal(bare.refused, null); |
| 147 | assert.deepEqual(bare.targets, [{ bundle_id: "com.apple.osascript" }]); |
| 148 | assert.deepEqual(checkAppScript('"ok".toUpperCase()', "javascript").targets, [{ bundle_id: "com.apple.osascript" }]); |
| 149 | }); |
| 150 | |
| 151 | test("Finder open of a denied app is refused", () => { |
| 152 | for (const [script, language] of [ |
| 153 | ['tell application "Finder" to open POSIX file "/Applications/Terminal.app"', "applescript"], |
| 154 | ['tell application "Finder" to open file "run.command" of desktop', "applescript"], |
| 155 | ['tell application "Finder" to open application file "Terminal.app" of folder "Applications" of startup disk', "applescript"], |
| 156 | ['Application("Finder").open(Path("/Applications/Terminal.app"))', "javascript"], |
| 157 | ['Application("Finder").launch()', "javascript"], |
| 158 | ]) { |
| 159 | assert.match(checkAppScript(script, language).refused ?? "", /opening files or applications|file paths/, script); |
| 160 | } |
| 161 | }); |
| 162 | |
| 163 | test("app_script policy modes: off refuses everything, unknown fails closed, unrestricted keeps targets", () => { |
| 164 | assert.equal(appScriptMode({}), "apps"); |
| 165 | assert.equal(appScriptMode({ CODEWHALE_CU_APP_SCRIPT: "nonsense" }), "off"); |
| 166 | assert.ok(checkAppScript("return 1", "applescript", { CODEWHALE_CU_APP_SCRIPT: "off" }).refused); |
| 167 | const open = checkAppScript('tell application "Mail" to do shell script "id"', "applescript", { CODEWHALE_CU_APP_SCRIPT: "unrestricted" }); |
| 168 | assert.equal(open.refused, null); |
| 169 | assert.deepEqual(open.targets, [{ name: "Mail" }]); |
| 170 | }); |
| 171 | |
| 172 | // ---- app_script policy (server) ---- |
| 173 | |
| 174 | test("E7: shell escapes are refused by the server before any dispatch", async () => { |
| 175 | const before = calls("app_script").length; |
| 176 | for (const [script, language] of [['do shell script "id"', undefined], ['Application.currentApplication().doShellScript("id")', "javascript"]]) { |
| 177 | const r = await tool("app_script", { script, ...(language ? { language } : {}) }); |
| 178 | assert.equal(r.ok, false); |
| 179 | assert.equal(r.error.code, "script_refused", JSON.stringify(r)); |
| 180 | } |
| 181 | assert.equal(calls("app_script").length, before, "nothing reached the backend"); |
| 182 | }); |
| 183 | |
| 184 | test("E6: an app reached through System Events goes through the ledger, and a denied one is refused", async () => { |
| 185 | const script = 'tell application "System Events" to tell process "Vault" to get name of window 1'; |
| 186 | const first = await tool("app_script", { script }); |
| 187 | assert.equal(first.error?.code, "consent_required", JSON.stringify(first)); |
| 188 | assert.match(first.error.message, /System Events/); |
| 189 | assert.equal((await tool("consent", { action: "allow", app: "System Events" })).ok, true); |
| 190 | assert.equal((await tool("consent", { action: "deny", app: "Vault" })).ok, true); |
| 191 | const denied = await tool("app_script", { script }); |
| 192 | assert.equal(denied.error?.code, "app_denied", JSON.stringify(denied)); |
| 193 | assert.equal(calls("app_script").length, 0, "no refused script was dispatched"); |
| 194 | const ok = await tool("app_script", { script: 'tell application "System Events" to get name of every process' }); |
| 195 | assert.equal(ok.ok, true, JSON.stringify(ok)); |
| 196 | assert.equal(calls("app_script").length, 1); |
| 197 | }); |
| 198 | |
| 199 | // ---- irreversible-action confirmation ---- |
| 200 | |
| 201 | test("E3: a Place order click needs a per-call confirmation that no app grant covers", async () => { |
| 202 | const state = await tool("get_app_state", {}); |
| 203 | const target = { type: "element", state_id: state.state_id, index: 9 }; |
| 204 | answerResolve({ role: "AXButton", label: "Place order", position: { x: 200, y: 20 }, size: { w: 80, h: 30 } }); |
| 205 | const clicksBefore = calls("left_click").length; |
| 206 | const refused = await tool("click", { target }); |
| 207 | assert.equal(refused.error?.code, "confirmation_required", JSON.stringify(refused)); |
| 208 | assert.equal(refused.confirm.label, "Place order"); |
| 209 | assert.match(refused.confirm.token, /^confirm-[0-9a-f]+$/); |
| 210 | assert.equal(calls("left_click").length, clicksBefore, "the refused click was never dispatched"); |
| 211 | // Repeating without confirmation hands back the same pending token. |
| 212 | assert.equal((await tool("click", { target })).confirm.token, refused.confirm.token); |
| 213 | // An app-level allow is not a confirmation. |
| 214 | assert.equal((await tool("consent", { action: "allow", app: "FakeApp" })).ok, true); |
| 215 | assert.equal((await tool("click", { target })).error?.code, "confirmation_required"); |
| 216 | assert.equal((await tool("consent", { action: "allow", confirm: "confirm-000" })).error?.code, "confirmation_unknown"); |
| 217 | const confirmed = await tool("consent", { action: "allow", confirm: refused.confirm.token }); |
| 218 | assert.equal(confirmed.ok, true, JSON.stringify(confirmed)); |
| 219 | assert.equal(confirmed.confirmed.label, "Place order"); |
| 220 | // A different call is not admitted by that confirmation. |
| 221 | const other = await tool("click", { target, clicks: 2 }); |
| 222 | assert.equal(other.error?.code, "confirmation_required"); |
| 223 | const ok = await tool("click", { target }); |
| 224 | assert.equal(ok.ok, true, JSON.stringify(ok)); |
| 225 | assert.equal(calls("left_click").length, clicksBefore + 1); |
| 226 | // Single use: the identical call asks again. |
| 227 | assert.equal((await tool("click", { target })).error?.code, "confirmation_required"); |
| 228 | assert.equal((await tool("consent", { action: "allow", confirm: refused.confirm.token })).error?.code, "confirmation_unknown"); |
| 229 | }); |
| 230 | |
| 231 | test("E5: delete through perform_action, a coordinate click or a menu is gated; a Send-to text field is not", async () => { |
| 232 | const state = await tool("get_app_state", {}); |
| 233 | answerResolve({ role: "AXButton", label: "Delete", position: { x: 300, y: 20 }, size: { w: 60, h: 30 } }); |
| 234 | const pressed = await tool("perform_action", { target: { type: "element", state_id: state.state_id, index: 10 }, action: "AXPress" }); |
| 235 | assert.equal(pressed.error?.code, "confirmation_required", JSON.stringify(pressed)); |
| 236 | const keyed = await tool("key", { text: "space", target: { type: "element", state_id: state.state_id, index: 10 } }); |
| 237 | assert.equal(keyed.error?.code, "confirmation_required", JSON.stringify(keyed)); |
| 238 | const coord = await tool("click", { target: { type: "coordinate", space: "screen", x: 320, y: 30 } }); |
| 239 | assert.equal(coord.error?.code, "confirmation_required", JSON.stringify(coord)); |
| 240 | assert.equal(coord.confirm.label, "Delete"); |
| 241 | await tool("open_application", { name: "FakeApp" }); |
| 242 | const menu = await tool("invoke_menu", { path: ["Edit", "Delete"] }); |
| 243 | assert.equal(menu.error?.code, "confirmation_required", JSON.stringify(menu)); |
| 244 | const save = await tool("invoke_menu", { path: ["File", "Save"] }); |
| 245 | assert.notEqual(save.error?.code, "confirmation_required"); |
| 246 | answerResolve({ role: "AXTextField", label: "Send to", position: { x: 10, y: 100 }, size: { w: 150, h: 25 } }); |
| 247 | const field = await tool("click", { target: { type: "element", state_id: state.state_id, index: 11 } }); |
| 248 | assert.notEqual(field.error?.code, "confirmation_required", JSON.stringify(field)); |
| 249 | fs.rmSync(controlFile, { force: true }); |
| 250 | }); |
| 251 | |
| 252 | // ---- helper staleness (K6) ---- |
| 253 | |
| 254 | test("a consent decision is never a run_actions step or a replayed trajectory step", async () => { |
| 255 | // Batched: refused before any step runs, so the grant is not recorded. |
| 256 | const batched = await tool("run_actions", { steps: [ |
| 257 | { tool: "consent", arguments: { action: "allow", app: "BatchedApp" } }, |
| 258 | { tool: "screenshot", arguments: {} }, |
| 259 | ] }); |
| 260 | assert.equal(batched.error?.code, "bad_args", JSON.stringify(batched)); |
| 261 | assert.match(batched.error.message, /consent decisions cannot be a run_actions step/); |
| 262 | const status = await tool("consent", { action: "status" }); |
| 263 | assert.ok(!JSON.stringify(status).includes("BatchedApp"), "the batched allow was not recorded"); |
| 264 | // Replayed: a recorded allow/revoke stops the replay instead of re-deciding. |
| 265 | await tool("trajectory", { action: "start" }); |
| 266 | assert.equal((await tool("consent", { action: "allow", app: "ReplayApp" })).ok, true); |
| 267 | assert.equal((await tool("consent", { action: "revoke", app: "ReplayApp" })).ok, true); |
| 268 | const stopped = await tool("trajectory", { action: "stop" }); |
| 269 | const dry = await tool("trajectory", { action: "replay", id: path.basename(stopped.file), dry_run: true }); |
| 270 | assert.deepEqual(dry.not_replayable, [0, 1], JSON.stringify(dry)); |
| 271 | const replay = await tool("trajectory", { action: "replay", id: path.basename(stopped.file) }); |
| 272 | assert.equal(replay.results[0].code, "not_replayable", JSON.stringify(replay)); |
| 273 | assert.ok(!JSON.stringify(await tool("consent", { action: "status" })).includes("ReplayApp"), "the replay did not re-grant ReplayApp"); |
| 274 | }); |
| 275 | |
| 276 | test("D2: a helper newer than the bundled plugin is not stale; an older one is", () => { |
| 277 | assert.equal(newerVersion("0.11.3", "0.11.2"), true); |
| 278 | assert.equal(newerVersion("0.11.10", "0.11.9"), true); |
| 279 | assert.equal(helperStaleness("0.11.3", "0.11.2").stale, false, "notarized 0.11.3 beside the 0.11.2 built-in"); |
| 280 | assert.equal(helperStaleness("0.11.2", "0.11.2").stale, false); |
| 281 | const old = helperStaleness("0.11.2", "0.11.3"); |
| 282 | assert.equal(old.stale, true); |
| 283 | assert.match(old.note, /restart/); |
| 284 | assert.equal(helperStaleness("garbage", "0.11.3").stale, false, "an unreadable version is not reported as stale"); |
| 285 | }); |
| 286 |