| 1 | // One request handler for every out-of-process runner of the backends: the |
| 2 | // ssh remote agent (one request per process) and the desktop app daemon (one |
| 3 | // long-lived process). Only tools in ALLOWED execute, so neither the ssh |
| 4 | // transport nor the app socket can ever become a generic shell. |
| 5 | import url from "node:url"; |
| 6 | import { exec } from "./remote-runtime.mjs"; |
| 7 | import { withSignal, throwIfAborted } from "./exec.mjs"; |
| 8 | import { checkAppScript } from "./app-script-policy.mjs"; |
| 9 | import * as consent from "./consent.mjs"; |
| 10 | |
| 11 | export const ALLOWED = new Set([ |
| 12 | "preview", "platform", "probe", "list_displays", "switch_display", "list_apps", "list_sessions", "list_windows", |
| 13 | "open_application", "kill_app", "set_window_frame", "get_app_state", "resolve_element", "screenshot", "zoom", |
| 14 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 15 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 16 | "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", |
| 17 | "type", "key", "hold_key", "set_value", "focus", "get_value", "select_text", "perform_action", "invoke_menu", |
| 18 | "read_clipboard", "write_clipboard", "cursor_position", |
| 19 | "recordingStart", "recordingStop", "recordingStatus", "recordingList", |
| 20 | "app_script", |
| 21 | ]); |
| 22 | |
| 23 | /** app_script runs osascript where this handler executes. The remote agent is |
| 24 | * the exception that proves the transport rule: it must never become a shell, |
| 25 | * so scripting is honored on the local computer (computerId "local") only. */ |
| 26 | const LOCAL_ONLY_TOOLS = new Set(["app_script"]); |
| 27 | |
| 28 | const backends = new Map(); |
| 29 | const heldPointers = new Map(); |
| 30 | const INPUT_MUTATIONS = new Set([ |
| 31 | "open_application", "left_click", "double_click", "triple_click", "right_click", "middle_click", "mouse_move", |
| 32 | "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "set_value", "focus", "select_text", "perform_action", "invoke_menu", |
| 33 | ]); |
| 34 | let queue = Promise.resolve(); |
| 35 | |
| 36 | async function backend(computerId, sessionId, persistentInputOwner) { |
| 37 | const key = `${computerId}:${sessionId}`; |
| 38 | if (!backends.has(key)) { |
| 39 | // Same test hook as src/transport.mjs, so the out-of-process route can be |
| 40 | // driven end to end against a recording backend (never set in production). |
| 41 | const test = process.env.CODEWHALE_CU_TEST_BACKEND; |
| 42 | const mod = await import(test ? url.pathToFileURL(test).href : `./backends/${process.platform}.mjs`); |
| 43 | backends.set(key, mod.create({ exec: { ...exec, persistentInputOwner }, computer: { id: computerId, transport: "local", platform: process.platform } })); |
| 44 | } |
| 45 | return backends.get(key); |
| 46 | } |
| 47 | |
| 48 | const sessions = new Map(); |
| 49 | let controlMode = "ready"; |
| 50 | let controlGeneration = 0; |
| 51 | let cleanupPending = false; |
| 52 | |
| 53 | /** Human-facing state contains app identity and action names, never task text. */ |
| 54 | export function controlStatus() { |
| 55 | return { mode: controlMode, cleanupPending, sessions: [...sessions.values()] |
| 56 | .filter((s) => !s.closed && s.target) |
| 57 | .map((s) => ({ target: s.target, mode: s.mode, action: s.action ?? null })) }; |
| 58 | } |
| 59 | |
| 60 | // Called only by the launcher's inherited control channel, never an MCP tool. |
| 61 | // Abort before queuing cleanup so even a held gesture yields to the person. |
| 62 | export async function setControlMode(mode) { |
| 63 | if (!["ready", "paused", "stopped"].includes(mode)) throw new Error("Unknown control mode"); |
| 64 | if (mode === "ready") { |
| 65 | if (cleanupPending) throw new Error("Input is still being released; try again in a moment."); |
| 66 | controlMode = mode; |
| 67 | return controlStatus(); |
| 68 | } |
| 69 | controlMode = mode; |
| 70 | const generation = ++controlGeneration; |
| 71 | cleanupPending = true; |
| 72 | const results = await Promise.allSettled([...sessions.keys()].map((key) => { |
| 73 | const colon = key.indexOf(":"); |
| 74 | return releaseSessionInput(key.slice(colon + 1), key.slice(0, colon), { close: mode === "stopped" }); |
| 75 | })); |
| 76 | const failure = results.find((r) => r.status === "rejected"); |
| 77 | // A cleanup failure stays blocked. Resume cannot hide owned input. |
| 78 | if (failure) throw failure.reason; |
| 79 | if (generation === controlGeneration) cleanupPending = false; |
| 80 | return controlStatus(); |
| 81 | } |
| 82 | |
| 83 | function enqueue(fn) { |
| 84 | const next = queue.then(fn); |
| 85 | queue = next.catch(() => {}); |
| 86 | return next; |
| 87 | } |
| 88 | |
| 89 | /** Cancel this host's work, then release only input held by its backend. */ |
| 90 | export function releaseSessionInput(sessionId, computerId = "local", { close = false } = {}) { |
| 91 | const key = `${computerId}:${sessionId}`; |
| 92 | let session = sessions.get(key); |
| 93 | if (!session) { |
| 94 | if (!close) return Promise.resolve(); |
| 95 | session = { requests: new Set(), closed: true, touched: Date.now() }; |
| 96 | sessions.set(key, session); |
| 97 | } |
| 98 | if (close) session.closed = true; |
| 99 | for (const controller of session.requests) controller.abort(); |
| 100 | return enqueue(async () => { |
| 101 | try { |
| 102 | await withSignal(null, () => backends.get(key)?.releaseInput?.()); |
| 103 | if (heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 104 | if (close) await withSignal(null, () => backends.get(key)?.closeSession?.()); |
| 105 | if (close) backends.delete(key); |
| 106 | } finally { session.touched = Date.now(); } |
| 107 | }); |
| 108 | } |
| 109 | |
| 110 | export function closeSession(sessionId, computerId = "local") { |
| 111 | return releaseSessionInput(sessionId, computerId, { close: true }); |
| 112 | } |
| 113 | |
| 114 | /** |
| 115 | * A freshly granted session owner supersedes a closed-session tombstone: |
| 116 | * without this, a daemon that re-leases a session id (the old owner socket |
| 117 | * died with the previous daemon) would keep aborting every request on it. |
| 118 | * In-flight requests from the dead owner stay aborted; the tombstone is the |
| 119 | * only thing removed. |
| 120 | */ |
| 121 | export function reopenSession(sessionId, computerId = "local") { |
| 122 | sessions.delete(`${computerId}:${sessionId}`); |
| 123 | } |
| 124 | |
| 125 | export function closeAllSessions() { |
| 126 | return Promise.allSettled([...sessions.keys()].map((key) => { |
| 127 | const colon = key.indexOf(":"); |
| 128 | return closeSession(key.slice(colon + 1), key.slice(0, colon)); |
| 129 | })); |
| 130 | } |
| 131 | |
| 132 | /** |
| 133 | * Content-free session registry view for agents: which sessions are live, |
| 134 | * what each is bound to, what it is doing right now, and whether any session |
| 135 | * holds a pointer. App identity and action names only — task text never |
| 136 | * reaches this process, so none can leak. Read-only by construction. |
| 137 | */ |
| 138 | export function summarizeSessions() { |
| 139 | const live = [...sessions.entries()].filter(([, s]) => !s.closed); |
| 140 | return { |
| 141 | control: controlMode, |
| 142 | count: live.length, |
| 143 | sessions: live.map(([key, s]) => { |
| 144 | const computerId = key.slice(0, key.indexOf(":")); |
| 145 | return { |
| 146 | target: s.target ?? null, |
| 147 | mode: s.mode ?? null, |
| 148 | action: s.action ?? null, |
| 149 | ageSec: Math.max(0, Math.round((Date.now() - s.touched) / 1000)), |
| 150 | inputHeld: heldPointers.get(computerId) === key, |
| 151 | }; |
| 152 | }), |
| 153 | }; |
| 154 | } |
| 155 | |
| 156 | /** App identities a request names directly, for the consent ledger. */ |
| 157 | function namedApps(tool, args) { |
| 158 | if (tool !== "open_application" && tool !== "kill_app") return []; |
| 159 | const ref = {}; |
| 160 | if (typeof args.bundle_id === "string" && args.bundle_id.trim()) ref.bundle_id = args.bundle_id.trim(); |
| 161 | if (typeof args.name === "string" && args.name.trim()) ref.name = args.name.trim(); |
| 162 | if (Number.isInteger(args.pid) && args.pid > 0) ref.pid = args.pid; |
| 163 | return Object.keys(ref).length ? [ref] : []; |
| 164 | } |
| 165 | |
| 166 | /** |
| 167 | * The runner's own policy, applied whoever sent the request: the MCP server, |
| 168 | * or any other process that reaches this socket. A script must pass the |
| 169 | * app_script policy, and no request may touch an app the user denied. |
| 170 | * (Allows are the MCP server's to check; a deny needs no signature.) |
| 171 | */ |
| 172 | function policyRefusal(tool, args, computerId) { |
| 173 | let targets = namedApps(tool, args); |
| 174 | if (tool === "app_script") { |
| 175 | const policy = checkAppScript(args.script, args.language); |
| 176 | if (policy.refused) return { code: "script_refused", message: `app_script refused: ${policy.refused}` }; |
| 177 | targets = policy.targets; |
| 178 | } |
| 179 | for (const ref of targets) { |
| 180 | if (consent.decisionFor(computerId, consent.appKeys(ref)).state === "denied") { |
| 181 | return { code: "app_denied", message: `the user denied access to ${ref.name ?? ref.bundle_id ?? `pid ${ref.pid}`} on this computer` }; |
| 182 | } |
| 183 | } |
| 184 | return null; |
| 185 | } |
| 186 | |
| 187 | /** |
| 188 | * Execute one {tool, args} request on this machine's backend. Never throws: |
| 189 | * every outcome is a receipt object with `ok`. |
| 190 | */ |
| 191 | export async function handle(req, { computerId = "local", sessionId = "direct", signal, persistentInputOwner = false } = {}) { |
| 192 | const tool = req?.tool; |
| 193 | if (!ALLOWED.has(tool)) { |
| 194 | return { ok: false, error: { code: "tool_not_allowed", message: `tool "${tool}" is not in the remote allow-list` } }; |
| 195 | } |
| 196 | if (LOCAL_ONLY_TOOLS.has(tool) && computerId !== "local") { |
| 197 | return { ok: false, error: { code: "unsupported_on_transport", message: `"${tool}" runs on the local computer only — a remote agent stays a computer-use channel, never a shell` } }; |
| 198 | } |
| 199 | if (tool === "platform") return { ok: true, platform: process.platform }; |
| 200 | const refusal = policyRefusal(tool, req.args ?? {}, computerId); |
| 201 | if (refusal) return { ok: false, platform: process.platform, tool, error: refusal }; |
| 202 | if (controlMode !== "ready") return { ok: false, error: { code: `control_${controlMode}`, message: `Computer Use is ${controlMode} by the user. Wait for them to resume it in the menu bar.` } }; |
| 203 | const generation = controlGeneration; |
| 204 | const key = `${computerId}:${sessionId}`; |
| 205 | let session = sessions.get(key); |
| 206 | if (!session) { |
| 207 | // Retain closed-session tombstones briefly, and bound abandoned sessions |
| 208 | // when a host is killed without a graceful MCP disconnect. |
| 209 | for (const [id, old] of sessions) { |
| 210 | if (old.requests.size || Date.now() - old.touched <= (old.closed ? 300_000 : 3_600_000)) continue; |
| 211 | if (backends.has(id)) { |
| 212 | const colon = id.indexOf(":"); |
| 213 | try { await closeSession(id.slice(colon + 1), id.slice(0, colon)); } |
| 214 | catch (err) { return { ok: false, error: { code: "input_release_failed", message: String(err?.message ?? err) } }; } |
| 215 | } else sessions.delete(id); |
| 216 | } |
| 217 | if ([...sessions.values()].filter((entry) => !entry.closed).length >= 256) return { ok: false, error: { code: "session_limit", message: "Too many active computer sessions; close unused hosts or restart the helper." } }; |
| 218 | session = { requests: new Set(), closed: false, touched: Date.now() }; |
| 219 | sessions.set(key, session); |
| 220 | } |
| 221 | const controller = new AbortController(); |
| 222 | const abort = () => controller.abort(); |
| 223 | signal?.addEventListener("abort", abort, { once: true }); |
| 224 | if (signal?.aborted || session.closed) controller.abort(); |
| 225 | session.requests.add(controller); |
| 226 | // One desktop can execute only one input gesture at a time. The queue spans |
| 227 | // sockets and sessions; a disconnected/cancelled request is checked again |
| 228 | // when its slot arrives, before it can post any input. |
| 229 | try { |
| 230 | return await enqueue(() => withSignal(controller.signal, async () => { |
| 231 | throwIfAborted(); |
| 232 | if (generation !== controlGeneration || controlMode !== "ready") throw Object.assign(new Error("Computer control was interrupted by the user."), { code: "cancelled" }); |
| 233 | const instance = await backend(computerId, sessionId, persistentInputOwner); |
| 234 | throwIfAborted(); |
| 235 | session.action = tool; |
| 236 | if (tool === "open_application") { session.target = null; session.mode = null; } |
| 237 | if (INPUT_MUTATIONS.has(tool) && heldPointers.has(computerId) && heldPointers.get(computerId) !== key) { |
| 238 | return { ok: false, error: { code: "input_busy", message: "Another computer session owns a held pointer; release it or close that session before sending input." } }; |
| 239 | } |
| 240 | const fn = instance[tool]; |
| 241 | if (typeof fn !== "function") { |
| 242 | return { ok: false, error: { code: "unsupported_on_platform", message: `"${tool}" is not implemented on ${process.platform}` } }; |
| 243 | } |
| 244 | // Preserve ownership across calls, not just during the serialized |
| 245 | // request. Another host's click/up must not release this host's press. |
| 246 | if (tool === "left_mouse_down") heldPointers.set(computerId, key); |
| 247 | let data; |
| 248 | try { data = await fn(req.args ?? {}); throwIfAborted(); } |
| 249 | catch (error) { |
| 250 | if (["left_mouse_down", "left_mouse_up", "mouse_move"].includes(tool) && heldPointers.get(computerId) === key) { |
| 251 | await withSignal(null, () => instance.releaseInput?.()); |
| 252 | if (heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 253 | } |
| 254 | throw error; |
| 255 | } |
| 256 | if (tool === "left_mouse_up" && heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 257 | if (tool === "open_application" && data?.resolved) { |
| 258 | session.target = { name: String(data.resolved.name ?? "Application").slice(0, 128), pid: data.resolved.pid }; |
| 259 | session.mode = data.shared_pointer || data.activate ? "foreground" : "background"; |
| 260 | } |
| 261 | return { ok: true, platform: process.platform, tool, data }; |
| 262 | })); |
| 263 | } catch (err) { |
| 264 | return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err), ...(err?.inputMayHaveBeenSent || err?.requestDispatched ? { request_dispatched: true } : {}) } }; |
| 265 | } finally { |
| 266 | signal?.removeEventListener("abort", abort); |
| 267 | session.requests.delete(controller); |
| 268 | session.action = null; |
| 269 | session.touched = Date.now(); |
| 270 | } |
| 271 | } |
| 272 |