| 1 | // codewhale-cu tool schemas — single source of truth for tools/list. |
| 2 | // Every action/observation tool accepts an optional `computer` id; supplying it |
| 3 | // switches the active computer first (switch-by-use is the default model). |
| 4 | const computerParam = { |
| 5 | type: "string", |
| 6 | description: "Computer id to act on. Defaults to the active computer. Providing a different registered id switches to it first (sticky).", |
| 7 | }; |
| 8 | |
| 9 | const strategyParam = { |
| 10 | enum: ["auto", "a11y", "event", "app"], |
| 11 | description: "macOS auto (default): element targets press that exact revalidated element and fail closed, with no coordinate fallback; coordinate targets hit-test the point for an accessibility press, including focus of a field that is not AXPressable. a11y: require an accessibility press or focus and fail closed otherwise. app: if accessibility cannot act, send the click as a window-routed event to the bound app's window. event: skip the accessibility hit-test and send the window-routed click directly. On macOS the user's cursor is never moved; raw pointer input needs activate:true because the window route briefly makes the app key. Other platforms use raw events. action_sent confirms dispatch, not the effect; observe again before deciding another action.", |
| 12 | }; |
| 13 | |
| 14 | const elementTargetSchema = { |
| 15 | type: "object", |
| 16 | description: "Element target: the flat index from the latest get_app_state on this computer. state_id is optional — supply it only to pin a specific earlier observation.", |
| 17 | required: ["type", "index"], |
| 18 | properties: { |
| 19 | type: { const: "element" }, |
| 20 | state_id: { type: "string" }, |
| 21 | index: { type: "integer", minimum: 0 }, |
| 22 | }, |
| 23 | additionalProperties: false, |
| 24 | }; |
| 25 | |
| 26 | const targetSchema = { |
| 27 | oneOf: [ |
| 28 | elementTargetSchema, |
| 29 | { |
| 30 | type: "object", |
| 31 | description: "Pixel coordinates in the latest returned raster (screenshot or zoom) for this computer.", |
| 32 | required: ["type", "x", "y"], |
| 33 | properties: { |
| 34 | type: { const: "coordinate" }, |
| 35 | x: { type: "integer" }, |
| 36 | y: { type: "integer" }, |
| 37 | space: { enum: ["raster", "screen"], description: "raster (default): pixels in the latest screenshot/OCR/zoom. screen: absolute screen points; do not convert them yourself." }, |
| 38 | }, |
| 39 | additionalProperties: false, |
| 40 | }, |
| 41 | ], |
| 42 | }; |
| 43 | |
| 44 | export const TOOLS = [ |
| 45 | { name: "preview", description: "macOS: show or hide the nonactivating app preview with the drawn agent cursor. On by default while an app is bound — each action updates the captured window and cursor without moving the real pointer. Set enabled:false to mute it for the session.", inputSchema: { type: "object", properties: { enabled: { type: "boolean" }, computer: computerParam }, additionalProperties: false } }, |
| 46 | // ---- computers (switching is a default) ---- |
| 47 | { |
| 48 | name: "computer", description: "The computer registry. action list | switch | register | spawn | remove. switch/register/spawn/remove take `id`; register also takes transport (local|ssh|hdc) plus host/port/user/knownHosts/target/installAgent; spawn takes transport (docker) plus optional image/label and creates a task-owned disposable desktop that remove or session end destroys. Prefer a spawned computer for work that does not need the user's own session. Every other tool also accepts `computer` to switch stickily on use.", |
| 49 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["list", "switch", "register", "spawn", "remove"] }, id: { type: "string", description: "Short id for the registered computer (letters, digits, dot, dash)" }, transport: { enum: ["local", "ssh", "hdc", "docker"] }, label: { type: "string" }, image: { type: "string", description: "spawn/docker: image to run (default the plugin's Linux desktop image)" }, host: { type: "string", description: "ssh: hostname" }, port: { type: "integer", description: "ssh: port (default 22)" }, user: { type: "string", description: "ssh: user" }, knownHosts: { type: "string", description: "ssh: absolute path to the existing trusted host-key file" }, target: { type: "string", description: "hdc: target key (omit for the only connected device)" }, installAgent: { type: "boolean", description: "ssh: push the remote agent before first use (default true)" } }, additionalProperties: false }, |
| 50 | }, |
| 51 | { |
| 52 | name: "computer_list", |
| 53 | description: "List registered computers (local, ssh, docker, harmony/hdc) and which one is active. Every other tool acts on the active computer unless given `computer`.", |
| 54 | inputSchema: { type: "object", properties: {}, additionalProperties: false }, |
| 55 | }, |
| 56 | { |
| 57 | name: "computer_switch", |
| 58 | description: "Switch the active computer. Subsequent tools act on it by default.", |
| 59 | inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string", description: "Registered computer id (see computer_list)" } }, additionalProperties: false }, |
| 60 | }, |
| 61 | { |
| 62 | name: "computer_register", |
| 63 | description: "Register or update a computer. transport=local (this machine), ssh (runs the bundled remote agent over ssh; agent is pushed automatically), hdc (HarmonyOS device via hdc).", |
| 64 | inputSchema: { |
| 65 | type: "object", |
| 66 | required: ["computer", "transport"], |
| 67 | properties: { |
| 68 | computer: { type: "string", description: "Short id for the computer (letters, digits, dot, dash)" }, |
| 69 | transport: { enum: ["local", "ssh", "hdc"] }, |
| 70 | label: { type: "string" }, |
| 71 | host: { type: "string", description: "ssh: hostname" }, |
| 72 | port: { type: "integer", description: "ssh: port (default 22)" }, |
| 73 | user: { type: "string", description: "ssh: user" }, |
| 74 | knownHosts: { type: "string", description: "ssh: absolute path to the existing trusted host-key file" }, |
| 75 | target: { type: "string", description: "hdc: target key (omit for the only connected device)" }, |
| 76 | installAgent: { type: "boolean", description: "ssh: push the remote agent before first use (default true)" }, |
| 77 | }, |
| 78 | additionalProperties: false, |
| 79 | }, |
| 80 | }, |
| 81 | { |
| 82 | name: "computer_spawn", |
| 83 | description: "Spawn a task-owned disposable computer. transport=docker provisions an isolated Linux desktop container registered under `computer`; every other tool works on it unchanged. The spawned computer is destroyed by computer_remove or when the session ends. Prefer it over local when the task does not need the user's own session.", |
| 84 | inputSchema: { |
| 85 | type: "object", |
| 86 | required: ["computer", "transport"], |
| 87 | properties: { |
| 88 | computer: { type: "string", description: "Short id for the spawned computer (letters, digits, dot, dash)" }, |
| 89 | transport: { enum: ["docker"] }, |
| 90 | image: { type: "string", description: "docker image (default the plugin's Linux desktop image)" }, |
| 91 | label: { type: "string" }, |
| 92 | }, |
| 93 | additionalProperties: false, |
| 94 | }, |
| 95 | }, |
| 96 | { |
| 97 | name: "computer_remove", |
| 98 | description: "Remove a registered computer. 'local' cannot be removed.", |
| 99 | inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string" } }, additionalProperties: false }, |
| 100 | }, |
| 101 | { |
| 102 | name: "consent", |
| 103 | description: "Per-app consent on the local computer. Any call that targets an app — open_application, an app_ref, an element, or an action on the bound app — refuses consent_required until the user decides; record their answer here. action status | allow | deny | revoke. app is a name or bundle id (or pid:/number for a pid); scope 'foreground' is the separate darwin decision for foreground control (open_application activate:true). Decisions apply to this session; remember:true persists them.", |
| 104 | inputSchema: { |
| 105 | type: "object", |
| 106 | required: ["action"], |
| 107 | properties: { |
| 108 | action: { enum: ["status", "allow", "deny", "revoke"] }, |
| 109 | app: { type: "string", description: "App identity: name ('Safari'), bundle id ('com.apple.Safari'), or pid ('pid:1234')" }, |
| 110 | name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, |
| 111 | scope: { enum: ["app", "foreground"], description: "app (default): consent to use one application. foreground: consent to foreground control and key focus (darwin activate:true)" }, |
| 112 | remember: { type: "boolean", description: "Persist the decision across sessions (default: this session only)" }, |
| 113 | confirm: { type: "string", description: "allow only: the token from a confirmation_required refusal. Record it only after the user approved that exact action (pay, buy, send, transfer, delete) in their own words; it admits one identical call." }, |
| 114 | computer: computerParam, |
| 115 | }, |
| 116 | additionalProperties: false, |
| 117 | }, |
| 118 | }, |
| 119 | { |
| 120 | name: "consent_status", |
| 121 | description: "List recorded app-consent decisions for a computer (persisted and this session's) plus the foreground decision.", |
| 122 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 123 | }, |
| 124 | { |
| 125 | name: "consent_allow", |
| 126 | description: "Record an allow decision: app (name/bundle_id/pid/app string) or scope:'foreground'. remember:true persists it.", |
| 127 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] }, remember: { type: "boolean" }, confirm: { type: "string", description: "Token from a confirmation_required refusal, recorded only after the user approved that exact action." } }, additionalProperties: false }, |
| 128 | }, |
| 129 | { |
| 130 | name: "consent_deny", |
| 131 | description: "Record a deny decision: app (name/bundle_id/pid/app string) or scope:'foreground'. remember:true persists it.", |
| 132 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] }, remember: { type: "boolean" } }, additionalProperties: false }, |
| 133 | }, |
| 134 | { |
| 135 | name: "consent_revoke", |
| 136 | description: "Remove recorded decisions for an app or scope:'foreground' (session and persisted).", |
| 137 | inputSchema: { type: "object", properties: { computer: computerParam, app: { type: "string" }, name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, scope: { enum: ["app", "foreground"] } }, additionalProperties: false }, |
| 138 | }, |
| 139 | // ---- observe & resolve ---- |
| 140 | { |
| 141 | name: "request_access", |
| 142 | description: "Probe permissions and capabilities of a computer (accessibility, screen capture, recording, missing tools). Call once when readiness is unknown or a permission failure is explicitly named.", |
| 143 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 144 | }, |
| 145 | { |
| 146 | name: "list_displays", |
| 147 | description: "List displays/panels with geometry and pixel scale.", |
| 148 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 149 | }, |
| 150 | { |
| 151 | name: "switch_display", |
| 152 | description: "Set which display subsequent screenshots/recordings capture on this computer.", |
| 153 | inputSchema: { type: "object", required: ["index"], properties: { index: { type: "integer", minimum: 1 }, computer: computerParam }, additionalProperties: false }, |
| 154 | }, |
| 155 | { |
| 156 | name: "list_apps", |
| 157 | description: "List running applications (name, pid, bundle id, frontmost). Defaults to regular user-facing apps; pass all:true to include background agents and helpers (menu-bar extras, XPC services, CLI processes); pass installed:true for the installed catalog of openable apps (running or not, with a running flag) — that scan takes a moment.", |
| 158 | inputSchema: { type: "object", properties: { all: { type: "boolean", description: "Include accessory/background processes, not just regular apps. Use when looking for a menu-bar or helper process; keep the default for picking an app to control." }, installed: { type: "boolean", description: "List installed apps (openable, running or not) from the standard Applications folders instead of running processes." }, computer: computerParam }, additionalProperties: false }, |
| 159 | }, |
| 160 | { |
| 161 | name: "list_windows", |
| 162 | description: "List application windows. On macOS, app_ref selects the app; omission follows the app selected by open_application, or the frontmost app before a selection. Other platforms list all windows and reject app_ref selectors.", |
| 163 | inputSchema: { |
| 164 | type: "object", |
| 165 | properties: { |
| 166 | app_ref: { |
| 167 | type: "object", |
| 168 | properties: { |
| 169 | pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" }, |
| 170 | }, |
| 171 | additionalProperties: false, |
| 172 | }, |
| 173 | computer: computerParam, |
| 174 | }, |
| 175 | additionalProperties: false, |
| 176 | }, |
| 177 | }, |
| 178 | { |
| 179 | name: "wait_for", |
| 180 | description: "Poll this computer's accessibility state until elements matching query/role appear (state:\"present\", default) or until none remain (state:\"absent\"). Returns the matched elements bound to a fresh state_id, ready to target. Prefer this over a get_app_state/wait loop after actions that load, animate or dismiss UI.", |
| 181 | inputSchema: { |
| 182 | type: "object", |
| 183 | properties: { |
| 184 | query: { type: "string", description: "Case-insensitive substring over label, value and role. At least one of query/role is required." }, |
| 185 | role: { type: "string", description: "Exact accessibility role, e.g. AXButton, AXTextField." }, |
| 186 | state: { enum: ["present", "absent"], default: "present", description: "present: wait until a match exists. absent: wait until no match remains (dialogs dismissed, loading finished)." }, |
| 187 | timeout: { type: "number", minimum: 0.5, maximum: 60, description: "Seconds to poll before giving up; default 10." }, |
| 188 | interval: { type: "integer", minimum: 100, maximum: 5000, description: "Milliseconds between observations; default 400." }, |
| 189 | limit: { type: "integer", minimum: 1, maximum: 100, description: "Max matched elements to return; default 20." }, |
| 190 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "Same selector rules as get_app_state; omission follows the app selected by open_application." }, |
| 191 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app." }, |
| 192 | computer: computerParam, |
| 193 | }, |
| 194 | additionalProperties: false, |
| 195 | }, |
| 196 | }, |
| 197 | { |
| 198 | name: "get_app_state", |
| 199 | description: "Read an application's text, controls, actions and layout without requiring vision. The default summary keeps app content and top-level menus; full adds nested menus and tree structure. Act on observed elements with {type:'element', index} and refresh after UI changes. Missing labels or values are unknown, not an invitation to guess; request a screenshot only when useful.", |
| 200 | inputSchema: { |
| 201 | type: "object", |
| 202 | properties: { |
| 203 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "macOS accepts PID, name and bundle identity. Linux accepts only a unique exact AT-SPI app name. Windows accepts only a unique exact window title in name (from list_windows.title). HarmonyOS rejects explicit app selectors." }, |
| 204 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app. Other platforms reject this selector." }, |
| 205 | detail: { enum: ["summary", "compact", "full"], default: "summary", description: "Summary is the concise default (controls, values, actions, layout). compact is smaller: same indices, shorter labels, no nested menus. full includes nested menus and tree paths." }, |
| 206 | query: { type: "string", description: "Case-insensitive substring over label, value and role. Use this instead of downloading the whole tree." }, |
| 207 | role: { type: "string", description: "Exact accessibility role filter, e.g. AXButton, AXTextField." }, |
| 208 | limit: { type: "integer", minimum: 1, maximum: 200, description: "Max elements to return after filtering. Prefer this over a second unfiltered dump." }, |
| 209 | offset: { type: "integer", minimum: 0, description: "Skip this many matching elements (pagination)." }, |
| 210 | include_ocr: { type: "boolean", default: false, description: "On macOS, also recognize visible text locally from the selected app window. Requires Screen Recording permission. Returns text, confidence and raster coordinate targets for UI that accessibility cannot read; no vision model is required. Do not combine with compact unless you need the blocks." }, |
| 211 | ocr_region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points. When include_ocr is true, recognize only this rect instead of the whole window." }, |
| 212 | computer: computerParam, |
| 213 | }, |
| 214 | additionalProperties: false, |
| 215 | }, |
| 216 | }, |
| 217 | { |
| 218 | name: "screenshot", |
| 219 | description: "Capture the screen (all or one display, optional region) as PNG/JPEG. The receipt carries raster geometry; later coordinate targets refer to this raster.", |
| 220 | inputSchema: { |
| 221 | type: "object", |
| 222 | properties: { |
| 223 | app_ref: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" } }, description: "macOS: capture this app window even when it is in the background." }, |
| 224 | display: { type: ["integer", "string"], description: "Display index or 'all'" }, |
| 225 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" }, |
| 226 | path: { type: "string", description: "Optional absolute .png/.jpg/.jpeg path inside the recordings directory. Omit to use a generated name there." }, |
| 227 | computer: computerParam, |
| 228 | }, |
| 229 | additionalProperties: false, |
| 230 | }, |
| 231 | }, |
| 232 | { |
| 233 | name: "zoom", |
| 234 | description: "Close-up crop of the latest screenshot. Choose points from the returned child raster only.", |
| 235 | inputSchema: { |
| 236 | type: "object", |
| 237 | required: ["region"], |
| 238 | properties: { |
| 239 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in last-raster pixels" }, |
| 240 | path: { type: "string", description: "Optional absolute .png/.jpg/.jpeg path inside the recordings directory. Omit to use a generated name there." }, |
| 241 | computer: computerParam, |
| 242 | }, |
| 243 | additionalProperties: false, |
| 244 | }, |
| 245 | }, |
| 246 | { |
| 247 | name: "cursor_position", |
| 248 | description: "Read the current pointer position in screen points.", |
| 249 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 250 | }, |
| 251 | { |
| 252 | name: "list_sessions", |
| 253 | description: "List the live computer sessions on this machine: bound target, delivery mode, current action, idle age, and whether any session currently holds a pointer. Read-only and content-free (no task text is ever recorded). Use it to see who else — another model or agent — is driving the computer before you act.", |
| 254 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 255 | }, |
| 256 | { |
| 257 | name: "kill_app", |
| 258 | description: "Quit a running application by exact name, bundle_id or pid. Refuses when several running applications match (pass pid) and never terminates the Computer Use helper itself. force:true force-quits an unresponsive app — unsaved work is discarded.", |
| 259 | inputSchema: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" }, force: { type: "boolean", description: "force-quit when the graceful quit does not complete" }, computer: computerParam }, additionalProperties: false }, |
| 260 | }, |
| 261 | { |
| 262 | name: "browser", |
| 263 | description: "Drive a Chromium-family browser over the DevTools protocol — exact element addressing instead of pixel clicking, in a self-owned profile (the user's own browser is never touched). Actions: start {url?} | status | navigate {url} | click {selector | point} | type {text, selector?, enter?} | screenshot {full?} | stop. Elements are CSS selectors; coordinates are page-viewport pixels from screenshot (never screen points). One tab per session; the last session out closes the browser.", |
| 264 | inputSchema: { |
| 265 | type: "object", required: ["action"], |
| 266 | properties: { |
| 267 | action: { enum: ["start", "status", "navigate", "click", "type", "screenshot", "stop"] }, |
| 268 | url: { type: "string", description: "http(s):// or about:blank (start, navigate)" }, |
| 269 | tab: { type: "string", description: "attach mode only (start): a tab id from status to work in — a person's tab is used only when named" }, |
| 270 | selector: { type: "string", description: "CSS selector (click, or type focus)" }, |
| 271 | point: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false, description: "page-viewport pixels — the browser screenshot space, never screen points" }, |
| 272 | text: { type: "string", description: "text to insert (type)" }, |
| 273 | enter: { type: "boolean", description: "press Enter after typing" }, |
| 274 | full: { type: "boolean", description: "capture the full page, not just the viewport" }, |
| 275 | computer: computerParam, |
| 276 | }, |
| 277 | additionalProperties: false, |
| 278 | }, |
| 279 | }, |
| 280 | { |
| 281 | name: "browser_start", |
| 282 | description: "Launch or reuse the self-owned Chromium profile and open this session's tab. The user's own browser is never touched. On a Codewhale Computer (attach mode) it attaches to the computer's shared browser instead; `tab` picks one of its tabs.", |
| 283 | inputSchema: { type: "object", properties: { url: { type: "string", description: "optional http(s) URL to open" }, tab: { type: "string", description: "attach mode: tab id from browser_status" }, computer: computerParam }, additionalProperties: false }, |
| 284 | }, |
| 285 | { |
| 286 | name: "browser_status", |
| 287 | description: "Read the browser session: running, tabs, and the active tab's url/title.", |
| 288 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 289 | }, |
| 290 | { |
| 291 | name: "browser_navigate", |
| 292 | description: "Navigate this session's tab to an http(s) or about:blank URL and wait for load.", |
| 293 | inputSchema: { type: "object", required: ["url"], properties: { url: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 294 | }, |
| 295 | { |
| 296 | name: "browser_click", |
| 297 | description: "Click in the page: a CSS selector's box center, or a page-viewport point.", |
| 298 | inputSchema: { type: "object", properties: { selector: { type: "string" }, point: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false }, computer: computerParam }, additionalProperties: false }, |
| 299 | }, |
| 300 | { |
| 301 | name: "browser_type", |
| 302 | description: "Insert text into the page (optionally focusing a CSS selector first); enter:true presses Enter.", |
| 303 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, selector: { type: "string" }, enter: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 304 | }, |
| 305 | { |
| 306 | name: "browser_screenshot", |
| 307 | description: "Capture the page (viewport, or the full page with full:true) as a PNG in the recordings dir.", |
| 308 | inputSchema: { type: "object", properties: { full: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 309 | }, |
| 310 | { |
| 311 | name: "browser_stop", |
| 312 | description: "Close this session's tab; the shared browser closes when no tabs remain.", |
| 313 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 314 | }, |
| 315 | { |
| 316 | name: "trajectory", |
| 317 | description: "Record this session's tool calls to a local JSONL and replay them later. Actions: start | stop | status (file, turns, recent files) | replay {id?, dry_run?} — replay re-enters the normal tool pipeline, so permissions, grants and the kill switch still apply, and it stops at the first refusal. Off unless started; entered text (typed text, set values, clipboard writes) is redacted and those steps are not replayable; files are owner-only and stay in the recordings dir on this machine.", |
| 318 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["start", "stop", "status", "replay"] }, id: { type: "string", description: "traj-*.jsonl name from status; defaults to the most recent" }, dry_run: { type: "boolean", description: "list what replay would do without executing anything" }, computer: computerParam }, additionalProperties: false }, |
| 319 | }, |
| 320 | { |
| 321 | name: "trajectory_start", |
| 322 | description: "Start recording this session's tool calls to a local JSONL.", |
| 323 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 324 | }, |
| 325 | { |
| 326 | name: "trajectory_stop", |
| 327 | description: "Stop recording and report the file and turn count.", |
| 328 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 329 | }, |
| 330 | { |
| 331 | name: "trajectory_status", |
| 332 | description: "Report whether a trajectory is recording, the file, and recent trajectories.", |
| 333 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 334 | }, |
| 335 | { |
| 336 | name: "trajectory_replay", |
| 337 | description: "Replay a recorded trajectory through the normal tool pipeline, stopping at the first refusal.", |
| 338 | inputSchema: { type: "object", properties: { id: { type: "string" }, dry_run: { type: "boolean" }, computer: computerParam }, additionalProperties: false }, |
| 339 | }, |
| 340 | { |
| 341 | name: "set_window_frame", |
| 342 | description: "Move or resize one window by exact geometry and read the result back. frame is in screen points, the same space list_windows reports: {x,y,w,h}. window_id is the zero-based window index from list_windows. Some windows refuse (fullscreen, tiled); the receipt carries the app's own before/after readback and `verified`.", |
| 343 | inputSchema: { type: "object", required: ["window_id", "frame"], properties: { app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "defaults to the bound app" }, window_id: { type: "integer", minimum: 0, description: "zero-based window index from list_windows" }, frame: { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, w: { type: "number" }, h: { type: "number" } }, required: ["x", "y", "w", "h"], additionalProperties: false }, computer: computerParam }, additionalProperties: false }, |
| 344 | }, |
| 345 | { |
| 346 | name: "open_application", |
| 347 | description: "Launch or activate an application. Copy user-provided names character-for-character; never translate, normalize, or strip suffixes. On macOS prefer bundle_id when known.", |
| 348 | inputSchema: { |
| 349 | type: "object", |
| 350 | properties: { |
| 351 | name: { type: "string" }, bundle_id: { type: "string" }, url: { type: "string" }, |
| 352 | pid: { type: "integer", description: "Bind to this exact process. Use when two processes share a bundle id (list_apps shows both); it takes precedence over name and bundle_id and never launches anything." }, |
| 353 | activate: { type: "boolean", description: "Bring to foreground; defaults to false — background is the default on every platform. On macOS false keeps process-bound keyboard/accessibility control and refuses raw pointer gestures (they would borrow key focus); on Windows it launches the app minimized; on Linux it restores the previously focused window after launch. True selects foreground control and requires the separate foreground consent — pointer input still goes to the app's window, never the user's cursor; use only when the user has authorized exclusive desktop use. Neither mode is an isolated computer." }, |
| 354 | computer: computerParam, |
| 355 | }, |
| 356 | additionalProperties: false, |
| 357 | }, |
| 358 | }, |
| 359 | // ---- pointer ---- |
| 360 | { |
| 361 | name: "click", description: "Click a target: `button` left/right/middle (left default) and `clicks` 1..3 (left only). Element targets press that exact accessibility element; coordinate targets need a fresh raster. The per-action names (left_click, double_click, right_click, middle_click, triple_click) stay callable as aliases.", |
| 362 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, button: { enum: ["left", "right", "middle"], default: "left" }, clicks: { type: "integer", minimum: 1, maximum: 3, default: 1 }, strategy: strategyParam, computer: computerParam }, additionalProperties: false }, |
| 363 | }, |
| 364 | { |
| 365 | name: "pointer", description: "Raw pointer primitives: action \"move\" (hover without clicking), \"down\" (press and hold), \"up\" (release; target optional — releases at the last point). On macOS these drive the Codewhale pointer, never the user\'s cursor: move is a window-routed hover, and down/move/up buffer a drag that reaches the window on up. They need activate:true.", |
| 366 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["move", "down", "up"] }, target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 367 | }, |
| 368 | { |
| 369 | name: "left_click", description: "Left-click a coordinate (pixels in the latest raster) or perform an element's press action. macOS background mode uses accessibility and refuses fallbacks that require keyboard focus.", |
| 370 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, strategy: strategyParam, computer: computerParam }, additionalProperties: false }, |
| 371 | }, |
| 372 | { |
| 373 | name: "double_click", description: "Double-click a target.", |
| 374 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 375 | }, |
| 376 | { |
| 377 | name: "triple_click", description: "Triple-click a target (e.g. select a paragraph).", |
| 378 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 379 | }, |
| 380 | { |
| 381 | name: "right_click", description: "Right-click a target (context menu).", |
| 382 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 383 | }, |
| 384 | { |
| 385 | name: "middle_click", description: "Middle-click a target.", |
| 386 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 387 | }, |
| 388 | { |
| 389 | name: "mouse_move", description: "Move the pointer without clicking (hover).", |
| 390 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 391 | }, |
| 392 | { |
| 393 | name: "left_click_drag", description: "Press at from_target, move in steps, release at to. macOS requires explicit foreground control; background mode refuses because window-routed events still take keyboard focus.", |
| 394 | inputSchema: { type: "object", required: ["from_target", "to"], properties: { from_target: targetSchema, to: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 395 | }, |
| 396 | { |
| 397 | name: "left_mouse_down", description: "Press and hold the left button at a target. Release with left_mouse_up.", |
| 398 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 399 | }, |
| 400 | { |
| 401 | name: "left_mouse_up", description: "Release the left button pressed by left_mouse_down. An optional target releases at that point instead of where the button went down.", |
| 402 | inputSchema: { type: "object", properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 403 | }, |
| 404 | { |
| 405 | name: "scroll", description: "Scroll up/down/left/right at a target. macOS background mode uses accessibility scrollbars; amount counts native increments or 5% normalized steps, named in the receipt. It refuses wheel-event fallbacks that take focus. Other raw routes use lines/notches. Prefer an observed scroll-area element.", |
| 406 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, direction: { enum: ["up", "down", "left", "right"] }, amount: { type: "integer", minimum: 1, maximum: 100 }, computer: computerParam }, additionalProperties: false }, |
| 407 | }, |
| 408 | // ---- text & keyboard ---- |
| 409 | { |
| 410 | name: "type", description: "Type unicode text into the focused control. Newlines in `text` are Return/Enter key presses, not literal characters — never put \\n in a composer by hoping it will send. Focus the field first (click, focus, or set_value), or pass an element `target` to focus it in the same call. On macOS the receipt carries `verified:true` only when the focused control's value actually reflects the typed text; on `verified:false` the text may have gone nowhere — observe again before relying on it.", |
| 411 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, press_enter: { type: "boolean", description: "After typing, press Return/Enter once. Prefer this to putting a newline in `text` when you want to send." }, target: { ...elementTargetSchema, description: "Element target from get_app_state; it is accessibility-focused first, then the text is typed. Element targets only." }, computer: computerParam }, additionalProperties: false }, |
| 412 | }, |
| 413 | { |
| 414 | name: "key", description: "Press a named key or chord. Examples: return, enter, backspace, tab, escape, cmd+c (macOS), ctrl+c (Linux/Windows). This is the key-press tool; type() cannot send modifiers or Return by itself except via newlines/press_enter. macOS background mode refuses modified or window-targeted keys that need keyboard focus; prefer invoke_menu. Repeat with `repeat`. Pass an element `target` to accessibility-focus it first. `duration` holds the key instead of tapping (hold_key semantics) and cannot be combined with repeat or target.", |
| 415 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, repeat: { type: "integer", minimum: 1, maximum: 100 }, duration: { type: "number", minimum: 0.05, maximum: 30, description: "Hold the key for this many seconds instead of tapping." }, target: { ...elementTargetSchema, description: "Element target from get_app_state; it is accessibility-focused first, then the key is sent. Element targets only." }, computer: computerParam }, additionalProperties: false }, |
| 416 | }, |
| 417 | { |
| 418 | name: "hold_key", description: "Hold a key for `duration` seconds (0.05..30).", |
| 419 | inputSchema: { type: "object", required: ["text", "duration"], properties: { text: { type: "string" }, duration: { type: "number", minimum: 0.05, maximum: 30 }, computer: computerParam }, additionalProperties: false }, |
| 420 | }, |
| 421 | { |
| 422 | name: "set_value", description: "Set an editable element's value with readback verification. On macOS native controls use AXValue; web-area replacement requires foreground control and refuses in background mode. Prefer browser control for web fields. Element targets only.", |
| 423 | inputSchema: { type: "object", required: ["target", "value"], properties: { target: elementTargetSchema, value: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 424 | }, |
| 425 | { |
| 426 | name: "focus", description: "Focus an observed element through the accessibility layer (background-safe). Prefer this before type() on composers that ignore AXPress.", |
| 427 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 428 | }, |
| 429 | { |
| 430 | name: "get_value", description: "Read the live accessibility value of an observed element (text fields, sliders). Prefer this over dumping the whole tree.", |
| 431 | inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false }, |
| 432 | }, |
| 433 | { |
| 434 | name: "find_elements", description: "Search the latest get_app_state (or take a fresh one) for elements matching query/role without returning the full dump.", |
| 435 | inputSchema: { |
| 436 | type: "object", |
| 437 | properties: { |
| 438 | query: { type: "string" }, |
| 439 | role: { type: "string" }, |
| 440 | state_id: { type: "string", description: "Reuse a previous observation; omit to observe now." }, |
| 441 | limit: { type: "integer", minimum: 1, maximum: 100 }, |
| 442 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false }, |
| 443 | computer: computerParam, |
| 444 | }, |
| 445 | additionalProperties: false, |
| 446 | }, |
| 447 | }, |
| 448 | { |
| 449 | name: "run_actions", description: "Run up to 8 computer-use tools in order on this computer. Stops on the first failure. Each step is {tool, arguments}. Use for click→type→key(return)→get_value without extra round trips.", |
| 450 | inputSchema: { |
| 451 | type: "object", |
| 452 | required: ["steps"], |
| 453 | properties: { |
| 454 | steps: { |
| 455 | type: "array", |
| 456 | minItems: 1, |
| 457 | maxItems: 8, |
| 458 | items: { |
| 459 | type: "object", |
| 460 | required: ["tool"], |
| 461 | properties: { |
| 462 | tool: { type: "string" }, |
| 463 | arguments: { type: "object" }, |
| 464 | }, |
| 465 | additionalProperties: false, |
| 466 | }, |
| 467 | }, |
| 468 | computer: computerParam, |
| 469 | }, |
| 470 | additionalProperties: false, |
| 471 | }, |
| 472 | }, |
| 473 | { |
| 474 | name: "select_text", description: "Select a text range [start, length] in an element, or place the caret when the range is omitted. Element targets only.", |
| 475 | inputSchema: { type: "object", required: ["target"], properties: { target: elementTargetSchema, text_range: { type: "array", items: { type: "integer" }, minItems: 2, maxItems: 2 }, computer: computerParam }, additionalProperties: false }, |
| 476 | }, |
| 477 | { |
| 478 | name: "perform_action", description: "Invoke a named accessibility action on an element (e.g. AXPress on macOS, Invoke on Windows/UIA, click on harmony). Only actions the element advertises. Element targets only.", |
| 479 | inputSchema: { type: "object", required: ["target", "action"], properties: { target: elementTargetSchema, action: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 480 | }, |
| 481 | { |
| 482 | name: "invoke_menu", description: "macOS: invoke an application menu item by title path (e.g. [\"File\",\"New\"]). Runs through accessibility with no focus lease and no key events — prefer this over cmd-key chords for app commands (New, Save, Quit and menu-only actions). App-level commands work without a key window; window-targeted items (Close) can validate against the app's key window and may no-op in the background — prefer the window's close-button element for those. Acts on the app bound with open_application. Verify the effect (list_windows / get_app_state) before reporting success.", |
| 483 | inputSchema: { |
| 484 | type: "object", required: ["path"], |
| 485 | properties: { |
| 486 | path: { type: "array", minItems: 1, maxItems: 3, items: { type: "string", minLength: 1 }, description: "Menu titles from the menu bar inward, e.g. [\"File\",\"Close Window\"]. Exact titles as shown, including an ellipsis when the app shows one. Application menus (the second menu bar group named after the app) work too." }, |
| 487 | computer: computerParam, |
| 488 | }, |
| 489 | additionalProperties: false, |
| 490 | }, |
| 491 | }, |
| 492 | // ---- clipboard / runtime ---- |
| 493 | { |
| 494 | name: "clipboard", description: "Read or write the system clipboard as UTF-8 text: action \"read\" or \"write\" (write requires text). This is the user's real clipboard — restore it when a round-trip is needed.", |
| 495 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["read", "write"] }, text: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 496 | }, |
| 497 | { |
| 498 | name: "read_clipboard", description: "Read the system clipboard as UTF-8 text.", |
| 499 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 500 | }, |
| 501 | { |
| 502 | name: "write_clipboard", description: "Write UTF-8 text to the system clipboard.", |
| 503 | inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 504 | }, |
| 505 | // ---- recording ---- |
| 506 | { |
| 507 | name: "recording", description: "Screen recordings: action start | stop | status | list. `start` accepts display/fps/region/app_ref/window_id/durationSec/intervalMs; stop/status take the recording `id`; list reports what exists. Darwin records through ScreenCaptureKit; other platforms state their own limits in the receipt.", |
| 508 | inputSchema: { type: "object", required: ["action"], properties: { action: { enum: ["start", "stop", "status", "list"] }, id: { type: "string", description: "Recording id for stop/status" }, display: { type: "integer" }, fps: { type: "number" }, region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4 }, app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false }, window_id: { type: "integer" }, durationSec: { type: "number" }, intervalMs: { type: "integer" }, computer: computerParam }, additionalProperties: false }, |
| 509 | }, |
| 510 | { |
| 511 | name: "recording_start", |
| 512 | description: "Start screen recording on a computer (mp4/mov). Darwin: ScreenCaptureKit via the native helper (timed or until recording_stop; honors region, no recorder overlay, stops on session exit). Pass app_ref to record only the selected app's window rect — captured at start and not tracked across moves. Linux and Windows: unavailable pending session-owned recorder cleanup; use screenshots. HarmonyOS: snapshot-series muxed with ffmpeg.", |
| 513 | inputSchema: { |
| 514 | type: "object", |
| 515 | properties: { |
| 516 | display: { type: ["integer", "string"] }, |
| 517 | fps: { type: "integer", minimum: 1, maximum: 60, description: "Linux/Windows/harmony-series only" }, |
| 518 | region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" }, |
| 519 | app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false, description: "macOS only: record the rect this app's window occupies at start. Omission follows the app selected by open_application." }, |
| 520 | window_id: { type: "integer", description: "macOS only: zero-based window index within the app; requires or implies app_ref." }, |
| 521 | durationSec: { type: "number", minimum: 1, maximum: 7200, description: "macOS only: auto-stop after N seconds" }, |
| 522 | intervalMs: { type: "integer", minimum: 150, maximum: 5000, description: "harmony snapshot-series frame interval" }, |
| 523 | computer: computerParam, |
| 524 | }, |
| 525 | additionalProperties: false, |
| 526 | }, |
| 527 | }, |
| 528 | { |
| 529 | name: "recording_stop", |
| 530 | description: "Stop a running recording and finalize the file.", |
| 531 | inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 532 | }, |
| 533 | { |
| 534 | name: "recording_status", |
| 535 | description: "Status of one recording (running, bytes so far).", |
| 536 | inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false }, |
| 537 | }, |
| 538 | { |
| 539 | name: "recording_list", |
| 540 | description: "List recordings and screenshots saved on a computer.", |
| 541 | inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false }, |
| 542 | }, |
| 543 | // ---- programmatic interface ---- |
| 544 | { |
| 545 | name: "app_script", |
| 546 | description: "macOS, local computer only: run an AppleScript or JXA (JavaScript for Automation) script through osascript — the programmatic interface inside apps that have a scripting dictionary (Finder, Mail, Safari, Calendar, Notes, Reminders, Music, System Events and most native apps). Prefer this over clicking when the app exposes one: deterministic, returns values, needs no Accessibility grant and never touches the pointer. The receipt carries stdout as `result`; a non-zero exit fails `script_error` with stderr, a user-declined consent fails `automation_denied` (the fix is System Settings → Privacy & Security → Automation, not a retry). Refused on ssh/hdc computers (`unsupported_on_transport`) — the remote channel stays computer-use only, never a shell. Not a shell locally either: shell escapes (do shell script, doShellScript), the ObjC bridge, dynamic code and terminal apps fail `script_refused`, and every app the script names needs the user's consent like any other target.", |
| 547 | inputSchema: { |
| 548 | type: "object", required: ["script"], |
| 549 | properties: { |
| 550 | script: { type: "string", minLength: 1, description: "Script source. For app arguments use `on run argv` in JXA or read them inside the script; keep scripts single-purpose." }, |
| 551 | language: { enum: ["applescript", "javascript"], description: "applescript (default) or javascript for JXA" }, |
| 552 | timeout: { type: "number", minimum: 1, maximum: 120, description: "Seconds before the script is killed; default 30." }, |
| 553 | computer: computerParam, |
| 554 | }, |
| 555 | additionalProperties: false, |
| 556 | }, |
| 557 | }, |
| 558 | // ---- kill switch ---- |
| 559 | { |
| 560 | name: "stop_computer_control", |
| 561 | description: "Kill switch: refuse all further computer-use actions for the rest of the session. Read-only probes stay available.", |
| 562 | inputSchema: { type: "object", properties: { reason: { type: "string" } }, additionalProperties: false }, |
| 563 | }, |
| 564 | { |
| 565 | name: "wait", |
| 566 | description: "Pause before the next observation (0..30s). Use after actions that animate or load.", |
| 567 | inputSchema: { type: "object", properties: { seconds: { type: "number", minimum: 0, maximum: 30 } }, additionalProperties: false }, |
| 568 | }, |
| 569 | ]; |
| 570 | |
| 571 | export const TOOL_NAMES = new Set(TOOLS.map((t) => t.name)); |
| 572 | |
| 573 | /** Required argument names per tool, straight from each inputSchema. */ |
| 574 | export const REQUIRED_ARGS = new Map(TOOLS.map((t) => [t.name, t.inputSchema.required ?? []])); |
| 575 | |
| 576 | /** Tools whose target must be an observed element — a coordinate reaches the |
| 577 | * backend unresolvable and fails opaquely, so refuse it at the boundary. */ |
| 578 | export const ELEMENT_ONLY_TARGET = new Set(["set_value", "select_text", "perform_action"]); |
| 579 | |
| 580 | /** Tools that never touch a computer (available even after kill switch). */ |
| 581 | export const READ_ONLY_TOOLS = new Set([ |
| 582 | "computer_list", "stop_computer_control", "wait", "request_access", "recording_list", "recording_status", |
| 583 | "find_elements", "get_value", "list_sessions", "browser_status", "trajectory_status", "trajectory_start", "trajectory_stop", |
| 584 | "consent_status", |
| 585 | ]); |
| 586 | |
| 587 | /** Tools dispatchable to a remote agent over ssh (allow-list must match agent.mjs). */ |
| 588 | export const REMOTE_TOOLS = new Set([ |
| 589 | "preview", "probe", "list_displays", "switch_display", "list_apps", "list_sessions", "list_windows", |
| 590 | "open_application", "kill_app", "set_window_frame", "get_app_state", "resolve_element", "screenshot", "zoom", |
| 591 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 592 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 593 | "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", |
| 594 | "type", "key", "hold_key", "set_value", "focus", "get_value", "select_text", "perform_action", "invoke_menu", |
| 595 | "read_clipboard", "write_clipboard", "cursor_position", |
| 596 | "recordingStart", "recordingStop", "recordingStatus", "recordingList", |
| 597 | "app_script", |
| 598 | ]); |
| 599 | |
| 600 | /** Map public tool name -> backend method name. */ |
| 601 | export const BACKEND_METHOD = Object.fromEntries( |
| 602 | TOOLS.filter((t) => !["computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove", "consent_status", "consent_allow", "consent_deny", "consent_revoke", "stop_computer_control", "wait", "wait_for", "find_elements", "run_actions", "trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay"].includes(t.name)) |
| 603 | .map((t) => [t.name, { |
| 604 | request_access: "probe", |
| 605 | recording_start: "recordingStart", |
| 606 | recording_stop: "recordingStop", |
| 607 | recording_status: "recordingStatus", |
| 608 | recording_list: "recordingList", |
| 609 | }[t.name] ?? t.name]), |
| 610 | ); |
| 611 | |
| 612 | /** |
| 613 | * Wire-name expansion for merged tools, used by capability grants: naming a |
| 614 | * merged tool admits every action it can dispatch to. |
| 615 | */ |
| 616 | export const MERGED_EXPANSION = { |
| 617 | click: ["left_click", "double_click", "triple_click", "right_click", "middle_click"], |
| 618 | pointer: ["mouse_move", "left_mouse_down", "left_mouse_up"], |
| 619 | clipboard: ["read_clipboard", "write_clipboard"], |
| 620 | recording: ["recording_start", "recording_stop", "recording_status", "recording_list"], |
| 621 | computer: ["computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove"], |
| 622 | consent: ["consent_status", "consent_allow", "consent_deny", "consent_revoke"], |
| 623 | key: ["key", "hold_key"], |
| 624 | browser: ["browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop"], |
| 625 | trajectory: ["trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay"], |
| 626 | }; |
| 627 | |
| 628 | /** |
| 629 | * Parse CODEWHALE_CU_GRANT — "read-only", or a comma list of tool names — |
| 630 | * into a wire-name set. The grant is fixed when the server starts (there is |
| 631 | * no tool that can widen it) and it is enforced twice: here, so the model |
| 632 | * never sees or reaches an ungranted tool, and at the app daemon, so a |
| 633 | * narrowed server cannot smuggle one through. Returns null when unset. |
| 634 | */ |
| 635 | export function parseGrant(value) { |
| 636 | if (value == null || (typeof value === "string" && !value.trim())) return null; |
| 637 | const out = new Set(); |
| 638 | for (const raw of String(value).split(",")) { |
| 639 | const name = raw.trim(); |
| 640 | if (!name) continue; |
| 641 | if (name === "read-only") { for (const tool of OBSERVATION_TOOLS) out.add(tool); continue; } |
| 642 | if (MERGED_EXPANSION[name]) { for (const tool of MERGED_EXPANSION[name]) out.add(tool); continue; } |
| 643 | out.add(name); |
| 644 | } |
| 645 | return out.size ? out : null; |
| 646 | } |
| 647 | |
| 648 | // ---------- MCP tool annotations ---------- |
| 649 | // Host-facing hints for approval and sandbox policy (MCP spec `annotations`). |
| 650 | // Hints describe the tool's design; they are not runtime gates. Observation |
| 651 | // tools read local state; `openWorld` is true when a tool acts on applications |
| 652 | // or computers outside this process; `destructive` marks tools that change what |
| 653 | // the user sees or holds (input, clipboard, registrations). |
| 654 | const READ_ONLY_ANNOTATION = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }; |
| 655 | const INPUT_ANNOTATION = { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }; |
| 656 | const TOOL_ANNOTATIONS = { |
| 657 | // Observation — reads only. |
| 658 | request_access: READ_ONLY_ANNOTATION, computer_list: READ_ONLY_ANNOTATION, list_displays: READ_ONLY_ANNOTATION, |
| 659 | list_apps: READ_ONLY_ANNOTATION, list_windows: READ_ONLY_ANNOTATION, wait_for: READ_ONLY_ANNOTATION, |
| 660 | list_sessions: READ_ONLY_ANNOTATION, |
| 661 | kill_app: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 662 | set_window_frame: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 663 | get_app_state: READ_ONLY_ANNOTATION, find_elements: READ_ONLY_ANNOTATION, get_value: READ_ONLY_ANNOTATION, |
| 664 | screenshot: READ_ONLY_ANNOTATION, zoom: READ_ONLY_ANNOTATION, cursor_position: READ_ONLY_ANNOTATION, |
| 665 | read_clipboard: READ_ONLY_ANNOTATION, recording_list: READ_ONLY_ANNOTATION, recording_status: READ_ONLY_ANNOTATION, |
| 666 | wait: READ_ONLY_ANNOTATION, |
| 667 | // Session controls — local state, not the user's apps. |
| 668 | preview: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 669 | stop_computer_control: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 670 | switch_display: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 671 | recording_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 672 | recording_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 673 | // Computer registry — touches other machines. |
| 674 | computer_switch: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 675 | computer_register: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 676 | computer_spawn: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 677 | computer_remove: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 678 | // Consent — the user's own decision record, not an action on apps. |
| 679 | consent: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 680 | consent_status: READ_ONLY_ANNOTATION, |
| 681 | consent_allow: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 682 | consent_deny: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 683 | consent_revoke: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 684 | open_application: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 685 | // Input — changes what the user sees. |
| 686 | left_click: INPUT_ANNOTATION, double_click: INPUT_ANNOTATION, triple_click: INPUT_ANNOTATION, |
| 687 | right_click: INPUT_ANNOTATION, middle_click: INPUT_ANNOTATION, left_click_drag: INPUT_ANNOTATION, |
| 688 | left_mouse_down: INPUT_ANNOTATION, left_mouse_up: INPUT_ANNOTATION, |
| 689 | type: INPUT_ANNOTATION, key: INPUT_ANNOTATION, hold_key: INPUT_ANNOTATION, invoke_menu: INPUT_ANNOTATION, |
| 690 | perform_action: INPUT_ANNOTATION, run_actions: INPUT_ANNOTATION, |
| 691 | // Scripting — acts on apps through their own dictionaries, not through input. |
| 692 | app_script: INPUT_ANNOTATION, |
| 693 | mouse_move: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 694 | scroll: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 695 | set_value: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 696 | focus: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 697 | select_text: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 698 | write_clipboard: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, |
| 699 | // Merged surface (aliases keep the wire names above callable). |
| 700 | click: INPUT_ANNOTATION, |
| 701 | pointer: INPUT_ANNOTATION, |
| 702 | browser: INPUT_ANNOTATION, |
| 703 | trajectory: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 704 | trajectory_status: READ_ONLY_ANNOTATION, |
| 705 | trajectory_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 706 | trajectory_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, |
| 707 | trajectory_replay: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true }, |
| 708 | browser_status: READ_ONLY_ANNOTATION, |
| 709 | browser_screenshot: READ_ONLY_ANNOTATION, |
| 710 | browser_start: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 711 | browser_stop: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true }, |
| 712 | browser_navigate: INPUT_ANNOTATION, |
| 713 | browser_click: INPUT_ANNOTATION, |
| 714 | browser_type: INPUT_ANNOTATION, |
| 715 | clipboard: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false }, |
| 716 | recording: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, |
| 717 | computer: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: true }, |
| 718 | }; |
| 719 | for (const tool of TOOLS) { |
| 720 | tool.annotations = TOOL_ANNOTATIONS[tool.name] ?? INPUT_ANNOTATION; |
| 721 | } |
| 722 | |
| 723 | /** |
| 724 | * Tools that only observe, straight from their annotations. This is the |
| 725 | * "read-only" capability grant — distinct from READ_ONLY_TOOLS (the smaller |
| 726 | * post-kill-switch set that also drives the safety valve). |
| 727 | */ |
| 728 | export const OBSERVATION_TOOLS = new Set(TOOLS.filter((t) => t.annotations.readOnlyHint === true).map((t) => t.name)); |
| 729 | |
| 730 | /** |
| 731 | * Tools refused with `computer_busy_human_driving` while a person holds the |
| 732 | * control lease (src/lease.mjs). Derived fail-closed: every tool that is not |
| 733 | * an observation and acts on the world is gated unless it is listed here as |
| 734 | * session bookkeeping. run_actions and trajectory_replay are gated per step |
| 735 | * (they re-enter callTool); browser_stop only detaches in attach mode. |
| 736 | */ |
| 737 | const LEASE_EXEMPT = new Set([ |
| 738 | "computer", "computer_switch", "computer_register", "computer_spawn", "computer_remove", |
| 739 | "trajectory_replay", "run_actions", "browser_stop", |
| 740 | ]); |
| 741 | export const LEASE_GATED_TOOLS = new Set(TOOLS.filter((t) => |
| 742 | !OBSERVATION_TOOLS.has(t.name) && !READ_ONLY_TOOLS.has(t.name) && !LEASE_EXEMPT.has(t.name) |
| 743 | && (t.annotations.openWorldHint === true || t.annotations.destructiveHint === true)).map((t) => t.name)); |
| 744 | |
| 745 | /** |
| 746 | * Merged-away names. They stay callable as aliases (receipts, pinned hosts and |
| 747 | * existing tests keep working) but never appear in tools/list — the advertised |
| 748 | * surface is what costs every session context. |
| 749 | */ |
| 750 | const HIDDEN_FROM_LIST = new Set([ |
| 751 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 752 | "mouse_move", "left_mouse_down", "left_mouse_up", |
| 753 | "read_clipboard", "write_clipboard", |
| 754 | "recording_start", "recording_stop", "recording_status", "recording_list", |
| 755 | "computer_list", "computer_switch", "computer_register", "computer_spawn", "computer_remove", |
| 756 | "consent_status", "consent_allow", "consent_deny", "consent_revoke", |
| 757 | "hold_key", |
| 758 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 759 | "trajectory_start", "trajectory_stop", "trajectory_status", "trajectory_replay", |
| 760 | ]); |
| 761 | for (const tool of TOOLS) { |
| 762 | if (HIDDEN_FROM_LIST.has(tool.name)) tool.hidden = true; |
| 763 | } |
| 764 | |
| 765 | /** |
| 766 | * Expand a merged, advertised tool into the wire tool it dispatches to. |
| 767 | * Runs before every gate in the dispatcher (required args, kill switch, |
| 768 | * routing), so a merged call can never bypass one; validation that the wire |
| 769 | * schema cannot express (which action, what each action needs) lives here and |
| 770 | * fails as bad_args with the requested name. Unknown names pass through |
| 771 | * unchanged — the alias surface is the rest of TOOLS. |
| 772 | */ |
| 773 | export function resolveTool(name, args = {}) { |
| 774 | const bad = (message) => Object.assign(new Error(message), { code: "bad_args" }); |
| 775 | switch (name) { |
| 776 | case "click": { |
| 777 | const button = args.button ?? "left"; |
| 778 | const clicks = args.clicks ?? 1; |
| 779 | const rest = { ...args }; |
| 780 | delete rest.button; |
| 781 | delete rest.clicks; |
| 782 | const wire = button === "left" && clicks === 1 ? "left_click" |
| 783 | : button === "left" && clicks === 2 ? "double_click" |
| 784 | : button === "left" && clicks === 3 ? "triple_click" |
| 785 | : button === "right" && clicks === 1 ? "right_click" |
| 786 | : button === "middle" && clicks === 1 ? "middle_click" |
| 787 | : null; |
| 788 | if (!wire) throw bad(`click supports left with 1-3 clicks, right x1 or middle x1 (got ${JSON.stringify(button)} x${clicks})`); |
| 789 | if (button !== "left") delete rest.strategy; // strategy is an a11y-left-click concept |
| 790 | return { name: wire, args: rest }; |
| 791 | } |
| 792 | case "pointer": { |
| 793 | const rest = { ...args }; |
| 794 | delete rest.action; |
| 795 | const wire = { move: "mouse_move", down: "left_mouse_down", up: "left_mouse_up" }[args.action]; |
| 796 | if (!wire) throw bad(`pointer action must be "move", "down" or "up" (got ${JSON.stringify(args.action)})`); |
| 797 | return { name: wire, args: rest }; |
| 798 | } |
| 799 | case "clipboard": { |
| 800 | const rest = { ...args }; |
| 801 | delete rest.action; |
| 802 | if (args.action === "read") return { name: "read_clipboard", args: { computer: rest.computer } }; |
| 803 | if (args.action === "write") { |
| 804 | if (typeof rest.text !== "string") throw bad("clipboard action \"write\" requires text"); |
| 805 | return { name: "write_clipboard", args: { text: rest.text, computer: rest.computer } }; |
| 806 | } |
| 807 | throw bad(`clipboard action must be "read" or "write" (got ${JSON.stringify(args.action)})`); |
| 808 | } |
| 809 | case "recording": { |
| 810 | const rest = { ...args }; |
| 811 | delete rest.action; |
| 812 | const wire = { start: "recording_start", stop: "recording_stop", status: "recording_status", list: "recording_list" }[args.action]; |
| 813 | if (!wire) throw bad(`recording action must be start, stop, status or list (got ${JSON.stringify(args.action)})`); |
| 814 | if ((args.action === "stop" || args.action === "status") && rest.id == null) throw bad(`recording action "${args.action}" requires id`); |
| 815 | return { name: wire, args: rest }; |
| 816 | } |
| 817 | case "computer": { |
| 818 | const rest = { ...args }; |
| 819 | delete rest.action; |
| 820 | const wire = { list: "computer_list", switch: "computer_switch", register: "computer_register", spawn: "computer_spawn", remove: "computer_remove" }[args.action]; |
| 821 | if (!wire) throw bad(`computer action must be list, switch, register, spawn or remove (got ${JSON.stringify(args.action)})`); |
| 822 | if (args.action === "list") return { name: wire, args: {} }; |
| 823 | if (rest.id == null) throw bad(`computer action "${args.action}" requires id`); |
| 824 | const id = rest.id; |
| 825 | delete rest.id; |
| 826 | return { name: wire, args: { ...rest, computer: id } }; |
| 827 | } |
| 828 | case "consent": { |
| 829 | const rest = { ...args }; |
| 830 | delete rest.action; |
| 831 | const wire = { status: "consent_status", allow: "consent_allow", deny: "consent_deny", revoke: "consent_revoke" }[args.action]; |
| 832 | if (!wire) throw bad(`consent action must be status, allow, deny or revoke (got ${JSON.stringify(args.action)})`); |
| 833 | if (args.action === "status") return { name: wire, args: { computer: rest.computer } }; |
| 834 | const foreground = rest.scope === "foreground"; |
| 835 | const confirming = args.action === "allow" && typeof rest.confirm === "string"; |
| 836 | if (!foreground && !confirming && rest.app == null && rest.name == null && rest.bundle_id == null && rest.pid == null) { |
| 837 | throw bad(`consent action "${args.action}" needs an app (name, bundle_id, pid or app string) — or scope:"foreground" for the shared-pointer decision`); |
| 838 | } |
| 839 | return { name: wire, args: rest }; |
| 840 | } |
| 841 | case "key": { |
| 842 | if (args.duration == null) return { name, args }; |
| 843 | const { duration, repeat, target, ...rest } = args; |
| 844 | if (repeat != null || target != null) throw bad("key with duration holds the key — repeat and target cannot be combined with it"); |
| 845 | if (!Number.isFinite(duration) || duration < 0.05 || duration > 30) throw bad("duration must be 0.05..30 seconds"); |
| 846 | return { name: "hold_key", args: { ...rest, duration } }; |
| 847 | } |
| 848 | case "browser": { |
| 849 | const rest = { ...args }; |
| 850 | delete rest.action; |
| 851 | switch (args.action) { |
| 852 | case "start": |
| 853 | if (rest.url != null && typeof rest.url !== "string") throw bad("browser start url must be a string"); |
| 854 | return { name: "browser_start", args: rest }; |
| 855 | case "status": return { name: "browser_status", args: rest }; |
| 856 | case "navigate": |
| 857 | if (typeof rest.url !== "string" || !rest.url.trim()) throw bad('browser action "navigate" requires url'); |
| 858 | return { name: "browser_navigate", args: rest }; |
| 859 | case "click": { |
| 860 | const hasSelector = typeof rest.selector === "string" && rest.selector.trim(); |
| 861 | const hasPoint = rest.point != null && Number.isFinite(rest.point?.x) && Number.isFinite(rest.point?.y); |
| 862 | if (hasSelector && hasPoint) throw bad('browser action "click" takes selector or point, not both — pick one target'); |
| 863 | if (!hasSelector && !hasPoint) throw bad('browser action "click" needs selector (CSS) or point {x,y}'); |
| 864 | if (!hasSelector) delete rest.selector; |
| 865 | if (!hasPoint) delete rest.point; |
| 866 | return { name: "browser_click", args: rest }; |
| 867 | } |
| 868 | case "type": |
| 869 | if (typeof rest.text !== "string" || !rest.text.length) throw bad('browser action "type" requires text'); |
| 870 | return { name: "browser_type", args: rest }; |
| 871 | case "screenshot": return { name: "browser_screenshot", args: rest }; |
| 872 | case "stop": return { name: "browser_stop", args: rest }; |
| 873 | default: |
| 874 | throw bad(`browser action must be start, status, navigate, click, type, screenshot or stop (got ${JSON.stringify(args.action)})`); |
| 875 | } |
| 876 | } |
| 877 | case "trajectory": { |
| 878 | const rest = { ...args }; |
| 879 | delete rest.action; |
| 880 | switch (args.action) { |
| 881 | case "start": return { name: "trajectory_start", args: rest }; |
| 882 | case "stop": return { name: "trajectory_stop", args: rest }; |
| 883 | case "status": return { name: "trajectory_status", args: rest }; |
| 884 | case "replay": return { name: "trajectory_replay", args: rest }; |
| 885 | default: |
| 886 | throw bad(`trajectory action must be start, stop, status or replay (got ${JSON.stringify(args.action)})`); |
| 887 | } |
| 888 | } |
| 889 | default: |
| 890 | return { name, args }; |
| 891 | } |
| 892 | } |
| 893 |