| 1 | // macOS backend. Zero third-party dependencies: |
| 2 | // - observation and input: native Accessibility and CoreGraphics APIs |
| 3 | // - stills: /usr/sbin/screencapture |
| 4 | // - video: ScreenCaptureKit in the signed helper (macOS 13+, no overlay) |
| 5 | // - crop: sips - clipboard: pbcopy/pbpaste |
| 6 | // Helper requests travel as one JSON argument without shell interpolation. |
| 7 | import fs from "node:fs"; |
| 8 | import os from "node:os"; |
| 9 | import path from "node:path"; |
| 10 | import crypto from "node:crypto"; |
| 11 | import { fileURLToPath } from "node:url"; |
| 12 | import { spawn } from "node:child_process"; |
| 13 | import { run, runOk, ExecError, tryJson, have, withSignal, wait, throwIfAborted, currentSignal } from "../exec.mjs"; |
| 14 | import { stateDir } from "../registry.mjs"; |
| 15 | import { createBrowser } from "../browser-cdp.mjs"; |
| 16 | import { recordingsDir, recordingsOutputPath } from "../recordings.mjs"; |
| 17 | |
| 18 | /** Base64 expands 3 bytes to 4, padded to a multiple of 4. */ |
| 19 | const encodedSize = (bytes) => Math.ceil(bytes / 3) * 4; |
| 20 | |
| 21 | /** |
| 22 | * Largest base64 payload a single JSON-RPC message may carry. Stdio hosts cap |
| 23 | * what a server may write between message boundaries (Claude Code disconnects |
| 24 | * at 16MB) and model image APIs cap well below that. Mirrors |
| 25 | * CODEWHALE_CU_MAX_IMAGE_BYTES in mcp/server.mjs, which keeps the hard guard. |
| 26 | */ |
| 27 | const rasterByteBudget = () => (Number(process.env.CODEWHALE_CU_MAX_IMAGE_BYTES) > 0 |
| 28 | ? Number(process.env.CODEWHALE_CU_MAX_IMAGE_BYTES) |
| 29 | : 5_000_000); |
| 30 | |
| 31 | /** |
| 32 | * Pixel dimensions from a PNG IHDR or a JPEG frame header, reading only the |
| 33 | * bytes that carry them rather than the whole raster. |
| 34 | */ |
| 35 | function imagePixels(file) { |
| 36 | const fd = fs.openSync(file, "r"); |
| 37 | try { |
| 38 | const head = Buffer.alloc(24); |
| 39 | fs.readSync(fd, head, 0, 24, 0); |
| 40 | if (head[0] === 0x89 && head.toString("ascii", 1, 4) === "PNG") { |
| 41 | return { w: head.readUInt32BE(16), h: head.readUInt32BE(20) }; |
| 42 | } |
| 43 | if (head[0] !== 0xff || head[1] !== 0xd8) throw new ExecError(`unrecognized raster format: ${file}`); |
| 44 | // Walk JPEG segments to the frame header. SOF0/1/2/3/5..7/9..11/13..15 |
| 45 | // carry the dimensions; DHT/DQT and the rest are skipped by their length. |
| 46 | const size = fs.fstatSync(fd).size; |
| 47 | const seg = Buffer.alloc(9); |
| 48 | for (let at = 2; at + 9 <= size; ) { |
| 49 | fs.readSync(fd, seg, 0, 9, at); |
| 50 | if (seg[0] !== 0xff) throw new ExecError(`malformed JPEG at byte ${at}: ${file}`); |
| 51 | const marker = seg[1]; |
| 52 | if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { |
| 53 | return { w: seg.readUInt16BE(7), h: seg.readUInt16BE(5) }; |
| 54 | } |
| 55 | if (marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd9)) { at += 2; continue; } |
| 56 | at += 2 + seg.readUInt16BE(2); |
| 57 | } |
| 58 | throw new ExecError(`JPEG carries no frame header: ${file}`); |
| 59 | } finally { |
| 60 | fs.closeSync(fd); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | /** |
| 65 | * Shrink a raster until it fits the single-message budget. |
| 66 | * |
| 67 | * A 5K display captures to ~22MB of PNG, which is ~29MB of base64 — past every |
| 68 | * host limit, so the alternative is handing back a receipt with no picture and |
| 69 | * a screenshot tool that never shows anything. Downscaling here, before the |
| 70 | * caller reads the PNG header, keeps coordinates exact by construction: |
| 71 | * `pixels` comes from the header, `points` stays in screen points, and `scale` |
| 72 | * is derived from the two, so raster-to-point conversion follows automatically. |
| 73 | * |
| 74 | * PNG bytes track pixel count, so the long edge shrinks by the square root of |
| 75 | * the overshoot. The estimate is verified rather than trusted — screen content |
| 76 | * compresses unevenly — and gives up rather than shrinking past legibility. |
| 77 | */ |
| 78 | async function fitRasterToBudget(file) { |
| 79 | for (let attempt = 0; attempt < 4; attempt += 1) { |
| 80 | const budget = rasterByteBudget(); |
| 81 | const size = fs.statSync(file).size; |
| 82 | if (encodedSize(size) <= budget) return; |
| 83 | const { w, h } = imagePixels(file); |
| 84 | const longest = Math.max(w, h); |
| 85 | if (longest <= 640) return; |
| 86 | const overshoot = encodedSize(size) / budget; |
| 87 | const target = Math.max(640, Math.floor((longest / Math.sqrt(overshoot)) * 0.9)); |
| 88 | if (target >= longest) return; |
| 89 | await runOk("sips", ["-Z", String(target), file], { timeoutMs: 20_000 }); |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | const KEY_CODES = { |
| 94 | return: 36, enter: 36, tab: 48, space: 49, escape: 53, esc: 53, delete: 51, |
| 95 | backspace: 51, forwarddelete: 117, home: 115, end: 119, pageup: 116, pagedown: 121, |
| 96 | left: 123, right: 124, down: 125, up: 126, clear: 71, capslock: 57, f1: 122, |
| 97 | f2: 120, f3: 99, f4: 118, f5: 96, f6: 97, f7: 98, f8: 100, f9: 101, f10: 109, |
| 98 | f11: 103, f12: 111, volumeup: 72, volumedown: 73, mute: 74, help: 114, |
| 99 | a: 0, s: 1, d: 2, f: 3, h: 4, g: 5, z: 6, x: 7, c: 8, v: 9, b: 11, q: 12, |
| 100 | w: 13, e: 14, r: 15, y: 16, t: 17, "1": 18, "2": 19, "3": 20, "4": 21, |
| 101 | "5": 23, "6": 22, "7": 26, "8": 28, "9": 25, "0": 29, "-": 27, "=": 24, |
| 102 | "[": 33, "]": 30, "\\": 42, ";": 41, "'": 39, ",": 43, ".": 47, "/": 44, |
| 103 | o: 31, u: 32, i: 34, p: 35, l: 37, j: 38, k: 40, n: 45, m: 46, |
| 104 | }; |
| 105 | const MODIFIERS = { |
| 106 | cmd: 1 << 20, command: 1 << 20, win: 1 << 20, meta: 1 << 20, |
| 107 | shift: 1 << 17, ctrl: 1 << 18, control: 1 << 18, alt: 1 << 19, opt: 1 << 19, option: 1 << 19, |
| 108 | fn: 1 << 23, function: 1 << 23, |
| 109 | }; |
| 110 | // CGEventType values (CGEventTypes.h). The dragged codes are easy to get |
| 111 | // wrong: 6 is LeftMouseDragged and 7 is RightMouseDragged, so a left drag sent |
| 112 | // as 7 is delivered as a right-button drag and no view ever sees it. |
| 113 | const MOUSE = { |
| 114 | left: { down: 1, up: 2, dragged: 6 }, |
| 115 | right: { down: 3, up: 4, dragged: 7 }, |
| 116 | middle: { down: 25, up: 26, dragged: 27 }, |
| 117 | }; |
| 118 | const MOUSE_MOVED = 5; |
| 119 | |
| 120 | /** |
| 121 | * Native refusals carry exception reasons; these map to stable codes so |
| 122 | * receipts and callers can branch without parsing prose. Unknown reasons stay |
| 123 | * uncoded (the message is the contract there). |
| 124 | */ |
| 125 | export function nativeErrorCode(message) { |
| 126 | const m = String(message ?? ""); |
| 127 | if (/^background_focus_required:/.test(m)) return "background_focus_required"; |
| 128 | if (/^user_busy:/.test(m)) return "user_busy"; |
| 129 | if (/ambiguous/i.test(m)) return "window_ambiguous"; |
| 130 | if (/not capturable/i.test(m)) return "window_not_capturable"; |
| 131 | if (/several running applications match/i.test(m)) return "ambiguous_application"; |
| 132 | if (/cannot be terminated by this plugin/i.test(m)) return "protected_application"; |
| 133 | if (/refused the window frame change/i.test(m)) return "frame_refused"; |
| 134 | if (/no accessibility geometry/i.test(m)) return "window_target_not_found"; |
| 135 | if (/application not found|no running application/i.test(m)) return "app_not_found"; |
| 136 | return null; |
| 137 | } |
| 138 | |
| 139 | /** |
| 140 | * Choose the menu element for an exact title: a menu bar item at level 0, an |
| 141 | * open menu's item below it. Exact match only — a fuzzy match would activate |
| 142 | * the wrong command, and menu titles are stable enough to state precisely. |
| 143 | * Exported for tests; the walk itself is native. |
| 144 | */ |
| 145 | export function pickMenuElement(elements, label, menuBar) { |
| 146 | const role = menuBar ? "AXMenuBarItem" : "AXMenuItem"; |
| 147 | return elements.find((el) => el?.label === label && el?.role === role) ?? null; |
| 148 | } |
| 149 | |
| 150 | /** |
| 151 | * Regular apps are what "open an app" means; accessories and daemons answer |
| 152 | * menu-bar and background questions. Keep the signal, drop the XPC soup. |
| 153 | * A helper that predates the activation_policy field returns the list whole. |
| 154 | */ |
| 155 | export function selectApps(apps, all) { |
| 156 | if (all || !apps.some((a) => a.activation_policy)) return apps; |
| 157 | return apps.filter((a) => a.activation_policy === "regular" || a.frontmost === true); |
| 158 | } |
| 159 | |
| 160 | /** |
| 161 | * Front-lease interference verdict (SHA-6643 slice 1). The native helper |
| 162 | * reports the borrow window (lease_ms) and the HID idle clock around it |
| 163 | * (idle_before_s/idle_after_s); synthesized events do not tick that clock, |
| 164 | * so a clock that fails to advance across the window means hardware input |
| 165 | * arrived mid-lease. Null when the reply carries no accounting (no borrow, |
| 166 | * or a helper that predates it) — receipts stay quiet then. |
| 167 | */ |
| 168 | const LEASE_IDLE_EPSILON_S = 0.25; |
| 169 | export function leaseVerdict(r) { |
| 170 | if (r?.front_lease !== true) return null; |
| 171 | const leaseMs = r.lease_ms, before = r.idle_before_s, after = r.idle_after_s; |
| 172 | if (![leaseMs, before, after].every(Number.isFinite)) return null; |
| 173 | return after < before + leaseMs / 1000 - LEASE_IDLE_EPSILON_S; |
| 174 | } |
| 175 | |
| 176 | /** |
| 177 | * Threads the interference accounting from a native lease reply into a |
| 178 | * model-facing receipt. Call sites that build receipts field-by-field |
| 179 | * spread this; verbatim flows (type) already carry it via native(). |
| 180 | */ |
| 181 | export function leaseAccounting(r) { |
| 182 | if (r?.front_lease !== true) return {}; |
| 183 | const out = {}; |
| 184 | for (const k of ["lease_ms", "idle_before_s", "idle_after_s", "yield_ms"]) { |
| 185 | if (Number.isFinite(r[k])) out[k] = r[k]; |
| 186 | } |
| 187 | if (typeof r.user_input_during_lease === "boolean") out.user_input_during_lease = r.user_input_during_lease; |
| 188 | return out; |
| 189 | } |
| 190 | |
| 191 | export function create({ exec }) { |
| 192 | const runL = (cmd, args, opts) => exec.run(cmd, args, opts); |
| 193 | // The preview panel is on by default: while a session is bound to an app, |
| 194 | // every action updates the floating capture and its drawn cursor so the |
| 195 | // person can watch without the real pointer moving. `preview(enabled:false)` |
| 196 | // mutes it for the session. |
| 197 | // The preview panel is live while a session is bound: after the first |
| 198 | // successful capture a timer keeps refreshing it, so the person watches the |
| 199 | // app instead of a frozen still. CODEWHALE_CU_PREVIEW_REFRESH_MS=0 disables |
| 200 | // the loop (tests, headless); the floor keeps a hostile value tolerable. |
| 201 | const state = { activeDisplay: 1, lastRaster: null, inputApp: null, foregroundInput: false, previewEnabled: true, pointer: null, heldDrag: null }; |
| 202 | // Shared-surface politeness: front leases, real-pointer gestures, |
| 203 | // foreground keys and activations wait for a gap in the user's hardware |
| 204 | // input rather than interleave with their typing. The helper reads the |
| 205 | // same HID idle clock it uses for interference accounting. gap<=0 turns |
| 206 | // the wait off entirely; wait_ms bounds it and refuses user_busy if the |
| 207 | // person is still active. Successful waits report yield_ms in the receipt. |
| 208 | const yieldArgs = { |
| 209 | yield_gap_ms: Number(process.env.CODEWHALE_CU_YIELD_GAP_MS ?? 450), |
| 210 | yield_wait_ms: Number(process.env.CODEWHALE_CU_YIELD_WAIT_MS ?? 2500), |
| 211 | }; |
| 212 | let previewLoop = null; |
| 213 | let previewBusy = false; |
| 214 | function stopPreviewLoop() { if (previewLoop) { clearInterval(previewLoop); previewLoop = null; } } |
| 215 | // A hide must not race an in-flight capture: its late preview_notify would |
| 216 | // re-show a panel that was just dismissed. |
| 217 | async function quiescePreview() { for (let i = 0; i < 20 && previewBusy; i++) await wait(25); } |
| 218 | const browser = createBrowser(); |
| 219 | function startPreviewLoop() { |
| 220 | if (previewLoop) return; |
| 221 | const ms = Number(process.env.CODEWHALE_CU_PREVIEW_REFRESH_MS ?? 1000); |
| 222 | if (!Number.isFinite(ms) || ms <= 0) return; |
| 223 | previewLoop = setInterval(() => { |
| 224 | if (previewBusy || !state.previewEnabled || !state.inputApp) return; |
| 225 | previewBusy = true; |
| 226 | updatePreview(false).catch(() => {}).finally(() => { previewBusy = false; }); |
| 227 | }, Math.max(50, ms)); |
| 228 | previewLoop.unref?.(); |
| 229 | } |
| 230 | |
| 231 | async function nativeHelper() { |
| 232 | let helper = process.env.CODEWHALE_CU_APP_BUNDLE |
| 233 | ? path.join(process.env.CODEWHALE_CU_APP_BUNDLE, "Contents", "MacOS", "accessibility") : null; |
| 234 | if (!helper || !fs.existsSync(helper)) { |
| 235 | const packaged = fileURLToPath(new URL("../../bin/darwin/accessibility", import.meta.url)); |
| 236 | if (fs.existsSync(packaged)) helper = packaged; |
| 237 | } |
| 238 | // A source checkout (plugin installs in other hosts) self-compiles an |
| 239 | // unsigned helper, which has no TCC grant. Prefer the installed app's |
| 240 | // signed helper so accessibility and screen-recording grants carry over. |
| 241 | if (!helper || !fs.existsSync(helper)) { |
| 242 | const installed = path.join(os.homedir(), "Applications", "Codewhale Computer Use.app", "Contents", "MacOS", "accessibility"); |
| 243 | if (fs.existsSync(installed)) helper = installed; |
| 244 | } |
| 245 | if (!helper || !fs.existsSync(helper)) { |
| 246 | const source = fileURLToPath(new URL("./darwin-accessibility.m", import.meta.url)); |
| 247 | const hash = crypto.createHash("sha256").update(fs.readFileSync(source)).update(fs.readFileSync(new URL("./darwin-recording.h", import.meta.url))).update(fs.readFileSync(new URL("./darwin-ocr.h", import.meta.url))).digest("hex").slice(0, 16); |
| 248 | const dir = path.join(os.homedir(), ".codewhale-cu", "bin"); |
| 249 | fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); |
| 250 | helper = path.join(dir, `accessibility-${hash}`); |
| 251 | if (!fs.existsSync(helper)) { |
| 252 | const tmp = `${helper}-${process.pid}`; |
| 253 | const r = await runL("clang", ["-fobjc-arc", "-Os", "-framework", "Cocoa", "-framework", "ApplicationServices", "-framework", "ScreenCaptureKit", "-framework", "AVFoundation", "-framework", "CoreMedia", "-framework", "Vision", source, "-o", tmp], { timeoutMs: 60_000 }); |
| 254 | if (r.code !== 0) throw new ExecError(`native accessibility helper needs a built app or Xcode Command Line Tools: ${r.stderr}`, r); |
| 255 | fs.renameSync(tmp, helper); |
| 256 | } |
| 257 | } |
| 258 | return helper; |
| 259 | } |
| 260 | |
| 261 | function requireFocusControl() { |
| 262 | if (!state.foregroundInput) throw Object.assign(new ExecError("This action would take keyboard focus and was not sent in background mode. Use an accessibility action, browser control, or a separate computer."), { code: "background_focus_required" }); |
| 263 | } |
| 264 | |
| 265 | async function native(tool, args = {}) { |
| 266 | // Window-addressed events still borrow keyboard focus. Block before even |
| 267 | // starting an older installed helper, including the app-scoped fallback. |
| 268 | if (["bg_pointer", "bg_key"].includes(tool)) requireFocusControl(); |
| 269 | // Every resolved target (element center or screen point) is where the |
| 270 | // action lands; tracking it here means the preview cursor follows element |
| 271 | // actions, not just raw pointer events. |
| 272 | if (tool === "type" && !state.foregroundInput && (await native("input_capabilities"))?.background_focus_guard !== 1) { |
| 273 | throw Object.assign(new ExecError("Update the Computer Use helper before background typing; this helper may borrow keyboard focus."), { code: "app_upgrade_required" }); |
| 274 | } |
| 275 | const t = args?.target; |
| 276 | if (t && Number.isFinite(t.x) && Number.isFinite(t.y)) state.pointer = { x: t.x, y: t.y }; |
| 277 | if (tool === "bg_pointer") { |
| 278 | const last = [...(args.steps ?? [])].reverse().find((s) => Number.isFinite(s?.x) && Number.isFinite(s?.y)); |
| 279 | if (last) state.pointer = { x: last.x, y: last.y }; |
| 280 | } |
| 281 | const helper = await nativeHelper(); |
| 282 | const r = await runL(helper, [JSON.stringify({ tool, args: { ...args, ...yieldArgs, input_app_ref: state.inputApp, foreground_input: state.foregroundInput, owner_pipe: true } })], { timeoutMs: 20_000, ownerPipe: true }); |
| 283 | if (r.aborted || r.timedOut || r.code !== 0) { |
| 284 | const error = new ExecError(r.aborted ? "computer request cancelled" : r.timedOut ? "native accessibility helper timed out" : r.stderr.trim() || "native accessibility helper failed", r); |
| 285 | if (r.aborted) error.code = "cancelled"; |
| 286 | else error.code = nativeErrorCode(error.message) ?? undefined; |
| 287 | // A deterministic native refusal sent no input. A killed/timed-out |
| 288 | // helper may have posted the press before losing its response. |
| 289 | const postsPress = (tool === "key_event" && args.down) || ["type", "perform_action", "click_element", "scroll_element", "set_value", "focus_element", "select_text", "bg_pointer", "bg_key"].includes(tool) || (tool === "hit_test" && args.perform); |
| 290 | error.inputMayHaveBeenSent = postsPress && r.spawned === true && (r.aborted || r.timedOut); |
| 291 | if (error.inputMayHaveBeenSent) error.message += "; input may already have been sent — observe the target before doing anything else"; |
| 292 | throw error; |
| 293 | } |
| 294 | const result = tryJson(r.stdout, null); |
| 295 | const interference = leaseVerdict(result); |
| 296 | if (interference !== null) result.user_input_during_lease = interference; |
| 297 | if (state.previewEnabled && state.inputApp && ["type", "key_event", "bg_pointer", "bg_key", "set_value", "select_text", "perform_action", "hit_test", "click_element", "scroll_element", "focus_element"].includes(tool)) { |
| 298 | try { await updatePreview(); } catch (error) { result.preview_error = error.message; } |
| 299 | } |
| 300 | return result; |
| 301 | } |
| 302 | |
| 303 | async function requireBackgroundActions() { |
| 304 | if ((await native("input_capabilities"))?.background_actions !== 1) { |
| 305 | throw Object.assign(new ExecError("Update the Computer Use helper to use background focus, selection, context menus and scrolling."), { code: "app_upgrade_required" }); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | function assertBoundElement(target) { |
| 310 | if (!state.inputApp || target.app_ref?.pid !== state.inputApp.pid) throw new ExecError("element does not belong to the bound application — open_application and observe again"); |
| 311 | if (!Array.isArray(target.path) || !Number.isInteger(target.windowIndex) || !target.role) throw new ExecError("element has no resolved accessibility identity"); |
| 312 | } |
| 313 | |
| 314 | async function nativeLease(tool, args) { |
| 315 | if (!exec.runInputLease) throw new ExecError("This executor cannot safely own held input; update Computer Use"); |
| 316 | if ((await native("input_capabilities"))?.input_lease !== 1) throw new ExecError("The native helper needs an update for disconnect-safe held input"); |
| 317 | const helper = await nativeHelper(); |
| 318 | try { |
| 319 | return await exec.runInputLease(helper, [JSON.stringify({ tool, args: { ...args, ...yieldArgs, input_app_ref: state.inputApp, foreground_input: state.foregroundInput, owner_pipe: true, input_lease: true } })]); |
| 320 | } catch (error) { |
| 321 | error.code = nativeErrorCode(error.message) ?? error.code; |
| 322 | throw error; |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | async function updatePreview(show = false) { |
| 327 | const win = await native("window_info", { app_ref: state.inputApp }); |
| 328 | const dir = path.join(stateDir(), "preview"); |
| 329 | fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); |
| 330 | const temp = path.join(dir, "next.png"), file = path.join(dir, "latest.png"); |
| 331 | const r = await runL("screencapture", ["-x", "-o", "-l", String(win.window_id), "-t", "png", temp], { timeoutMs: 8000 }); |
| 332 | if (r.code !== 0) throw new ExecError(`background preview capture failed: ${r.stderr}`); |
| 333 | fs.renameSync(temp, file); |
| 334 | const p = state.pointer; |
| 335 | // The user's own hardware cursor goes on the preview too, so the panel |
| 336 | // shows both pointers in the same window-relative space. |
| 337 | let userCursor = null; |
| 338 | try { userCursor = await native("cursor_position"); } catch {} |
| 339 | await native("preview_notify", { enabled: true, show, title: `Codewhale · ${win.name} · ${state.foregroundInput ? "Shared desktop control" : "Background app control"}`, x: p ? (p.x-win.points.x)/win.points.w : -1, y: p ? (p.y-win.points.y)/win.points.h : -1, |
| 340 | user_x: userCursor && Number.isFinite(userCursor.x) ? (userCursor.x-win.points.x)/win.points.w : -1, |
| 341 | user_y: userCursor && Number.isFinite(userCursor.y) ? (userCursor.y-win.points.y)/win.points.h : -1 }); |
| 342 | // Any successful capture (bind, explicit preview, action refresh) starts |
| 343 | // the live refresh; the tick itself re-enters this function as a no-op. |
| 344 | if (state.previewEnabled && state.inputApp) startPreviewLoop(); |
| 345 | return { enabled: true, file, app: state.inputApp, pointer: p }; |
| 346 | } |
| 347 | |
| 348 | // ---------- pointer input ---------- |
| 349 | // Our qualified raw pointer path uses the shared event tap, which moves |
| 350 | // the user's real cursor. Process/window-directed mouse delivery has not |
| 351 | // passed the independent fixture. So the pointer path is: |
| 352 | // 1. accessibility action on the element under the point (quiet, exact), |
| 353 | // 2. otherwise refuse in background mode. Explicit foreground control |
| 354 | // permits a global gesture only when the bound application owns the |
| 355 | // window under the point. Restoring the cursor is not isolation. |
| 356 | // Every receipt says which of the two happened. |
| 357 | function mouseName(button) { return { left: "left", right: "right", middle: "middle" }[button] ?? "left"; } |
| 358 | |
| 359 | function assertInScreen(x, y) { |
| 360 | if (!Number.isFinite(x) || !Number.isFinite(y)) throw new ExecError("coordinates must be finite numbers"); |
| 361 | } |
| 362 | |
| 363 | function buttonCode(button) { return button === "middle" ? 2 : button === "right" ? 1 : 0; } |
| 364 | |
| 365 | /** |
| 366 | * Every pointer gesture — click, hover, drag, wheel — goes to a window of the |
| 367 | * bound application as window-routed event records. The user's hardware |
| 368 | * cursor is never posted to, warped or held: there is no shared-pointer |
| 369 | * route to fall back to. Window ownership is enforced inside the helper (the |
| 370 | * records are addressed to a window id of the bound app), so a covering |
| 371 | * window cannot receive them. |
| 372 | */ |
| 373 | async function windowPointer(steps, extra = {}) { |
| 374 | requireFocusControl(); |
| 375 | if ((await native("input_capabilities"))?.window_record !== 1) { |
| 376 | throw Object.assign(new ExecError("Pointer input needs the window-routed pointer, which this helper cannot resolve; update Computer Use or use an accessibility action. The user's cursor is never used instead."), { code: "bg_dispatch_unavailable" }); |
| 377 | } |
| 378 | const r = await native("bg_pointer", { steps, ...extra }); |
| 379 | return { action_sent: true, strategy: "window-record", input_scope: "application-window", pointer_moved: false, |
| 380 | front_lease: r.front_lease === true, window: r.window ?? null, ...leaseAccounting(r), |
| 381 | ...(typeof r.front_restored === "boolean" ? { front_restored: r.front_restored } : {}), |
| 382 | ...(r.menu_lease_held ? { menu_lease_held: true } : {}) }; |
| 383 | } |
| 384 | |
| 385 | function clickSteps(button, x, y, clicks) { |
| 386 | const m = MOUSE[button] ?? MOUSE.left; |
| 387 | const b = buttonCode(button); |
| 388 | const steps = [{ type: MOUSE_MOVED, x, y, button: b, clickState: 0 }]; |
| 389 | for (let i = 1; i <= clicks; i++) { |
| 390 | steps.push({ type: m.down, x, y, button: b, clickState: i }); |
| 391 | steps.push({ type: m.up, x, y, button: b, clickState: i }); |
| 392 | } |
| 393 | return steps; |
| 394 | } |
| 395 | |
| 396 | /** |
| 397 | * Coordinate pointer click. A left single click is first hit-tested against |
| 398 | * the bound application's accessibility tree: when the point names a |
| 399 | * pressable element we perform its semantic action, which needs no pointer |
| 400 | * and no foreground. strategy="a11y" requires that and fails closed; |
| 401 | * strategy="event" goes straight to the guarded global gesture. |
| 402 | */ |
| 403 | async function pointerClick(button, x, y, clicks, strategy = "auto") { |
| 404 | assertInScreen(x, y); |
| 405 | if (!["auto", "a11y", "event", "app"].includes(strategy)) throw new ExecError(`strategy must be auto, a11y, app or event (got ${JSON.stringify(strategy)})`); |
| 406 | let a11yReason = null; |
| 407 | if (strategy !== "event" && ["left", "right"].includes(button) && clicks === 1) { |
| 408 | if (button === "right") await requireBackgroundActions(); |
| 409 | const hit = await native("hit_test", { x, y, perform: true, ...(button === "right" ? { operation: "context" } : {}) }); |
| 410 | if (hit?.action_sent) { |
| 411 | return { action_sent: true, strategy: "a11y", action: hit.action, pointer_moved: false, at: { x, y }, button, clicks, |
| 412 | element: { role: hit.element?.role ?? null, label: hit.element?.label ?? null } }; |
| 413 | } |
| 414 | a11yReason = hit?.reason ?? "not_found"; |
| 415 | if (strategy === "a11y") { |
| 416 | throw new ExecError(`no supported accessibility click at (${x}, ${y}) in the bound application (${a11yReason}) — observe the available actions, use strategy "app" for a window-routed pointer click, or a separate computer`); |
| 417 | } |
| 418 | } else if (strategy === "a11y") { |
| 419 | throw new ExecError(`strategy "a11y" is only available for a left single click on this backend; ${mouseName(button)} x${clicks} has no accessibility equivalent`); |
| 420 | } |
| 421 | // Whatever the strategy, a raw click is a window-routed record: "app" and |
| 422 | // "event" only choose whether the accessibility hit-test runs first. |
| 423 | const r = await windowPointer(clickSteps(button, x, y, clicks), |
| 424 | a11yReason === "web_popup_requires_real_click" ? { menu_poll_ms: 6000 } : {}); |
| 425 | return { ...r, at: { x, y }, button, clicks, ...(a11yReason ? { a11y_reason: a11yReason } : {}) }; |
| 426 | } |
| 427 | |
| 428 | async function withPressedKey(code, flags, action) { |
| 429 | const lease = await nativeLease("key_event", { code, flags, down: true }); |
| 430 | try { |
| 431 | await action(); |
| 432 | // The acknowledgement carries the yield_ms the helper waited for a |
| 433 | // hardware-input gap before posting the press. |
| 434 | return lease.receipt; |
| 435 | } finally { |
| 436 | await withSignal(null, () => lease.release()); |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | function parseChord(text) { |
| 441 | const parts = String(text).split("+").map((s) => s.trim().toLowerCase()).filter(Boolean); |
| 442 | if (!parts.length) throw new ExecError("empty key text"); |
| 443 | let flags = 0; |
| 444 | let key = null; |
| 445 | for (const p of parts) { |
| 446 | if (MODIFIERS[p] != null) flags |= MODIFIERS[p]; |
| 447 | else if (KEY_CODES[p] != null) { if (key) throw new ExecError(`multiple non-modifier keys in "${text}"`); key = p; } |
| 448 | else throw new ExecError(`unknown key "${p}" (supported: ${Object.keys(KEY_CODES).join(", ")} + modifiers cmd/ctrl/alt/shift/fn)`); |
| 449 | } |
| 450 | if (key == null) throw new ExecError(`no non-modifier key in "${text}"`); |
| 451 | return { flags, code: KEY_CODES[key], key }; |
| 452 | } |
| 453 | |
| 454 | // ---------- displays ---------- |
| 455 | async function displayInfo() { return native("displays"); } |
| 456 | |
| 457 | // ---------- screenshots ---------- |
| 458 | async function screenshot({ display, region, app_ref, window_id, path: outPath } = {}) { |
| 459 | // Once an app is selected, ordinary observations follow it behind the |
| 460 | // user's work. An explicit display/region remains a deliberate desktop capture. |
| 461 | if (app_ref === undefined && display === undefined && region === undefined) app_ref = state.inputApp ?? undefined; |
| 462 | const dir = recordingsDir(); |
| 463 | fs.mkdirSync(dir, { recursive: true }); |
| 464 | // JPEG, not PNG. A screen is photographic content — gradients, wallpaper, |
| 465 | // antialiased text — and lossless compression of it is enormous: the same |
| 466 | // 5760x3240 frame is 21.8MB as PNG and 2.1MB as JPEG, at full resolution |
| 467 | // and with terminal text still crisp. PNG stays available by asking for a |
| 468 | // `.png` path, which is what a pixel-exact comparison wants. |
| 469 | const file = recordingsOutputPath(outPath) ?? path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.jpg`); |
| 470 | const args = ["-x", "-t", /\.png$/i.test(file) ? "png" : "jpg"]; |
| 471 | const disp = display ?? state.activeDisplay; |
| 472 | // An explicit app reference resolves first and alone: nothing may run |
| 473 | // before it and redirect the capture to another target. |
| 474 | const window = app_ref !== undefined ? await native("window_info", { app_ref, window_id }) : null; |
| 475 | if (window && region) throw new ExecError("choose app_ref or region, not both"); |
| 476 | // On the display path, resolve displays before capturing so an unknown |
| 477 | // index is a clean error instead of a raster silently labelled with another |
| 478 | // display's geometry — list_displays reports `index` and `id` separately, |
| 479 | // and a caller passing the id would otherwise get points and scale that |
| 480 | // mis-target every later coordinate. A window capture ignores `display`. |
| 481 | let displays = null; |
| 482 | if (!window) { |
| 483 | displays = await displayInfo(); |
| 484 | if (disp != null && disp !== "all" && !displays.some((x) => x.index === disp)) { |
| 485 | throw new ExecError(`no display ${disp}; have [${displays.map((x) => x.index).join(", ")}] — screenshot takes the display index from list_displays, not its id`); |
| 486 | } |
| 487 | } |
| 488 | if (window) args.push("-o", "-l", String(window.window_id)); |
| 489 | else if (disp && disp !== "all") args.push("-D", String(disp)); |
| 490 | if (region) { |
| 491 | if (!region.every((n) => Number.isFinite(n) && n >= 0) || region.length !== 4) { |
| 492 | throw new ExecError("region must be [x, y, w, h] in screen points"); |
| 493 | } |
| 494 | args.push("-R", region.join(",")); |
| 495 | } |
| 496 | args.push(file); |
| 497 | const r = await runL("screencapture", args, { timeoutMs: 20_000 }); |
| 498 | if (r.code !== 0) throw new ExecError(`screencapture exited ${r.code}: ${r.stderr.trim().slice(0, 300)}`, r); |
| 499 | await fitRasterToBudget(file); |
| 500 | const stat = fs.statSync(file); |
| 501 | displays ??= await displayInfo(); |
| 502 | const d = displays.find((x) => x.index === (disp === "all" ? 1 : disp)) ?? displays[0]; |
| 503 | const scale = d?.scale ?? 1; |
| 504 | state.lastRaster = { |
| 505 | file, |
| 506 | ...(window ? { app_ref, window_index: window_id ?? 0 } : {}), |
| 507 | bytes: stat.size, |
| 508 | display: disp ?? 1, |
| 509 | // Region and window rasters describe that rect, not the whole display. |
| 510 | // The PNG header is the pixel ground truth; scale is derived from |
| 511 | // pixels/points below so Retina and mixed-DPI stay exact. |
| 512 | points: window?.points ?? (region ? { x: region[0], y: region[1], w: region[2], h: region[3] } : d?.points ?? null), |
| 513 | pixels: imagePixels(file), |
| 514 | scale: d?.scale ?? 1, |
| 515 | capturedAt: new Date().toISOString(), |
| 516 | }; |
| 517 | if (state.lastRaster.points?.w) state.lastRaster.scale = state.lastRaster.pixels.w / state.lastRaster.points.w; |
| 518 | return { ...state.lastRaster, path: file }; |
| 519 | } |
| 520 | |
| 521 | // Always crops the last raster this backend captured; a caller-named |
| 522 | // source file is not accepted. |
| 523 | async function zoom({ region, path: outPath }) { |
| 524 | // Validate the caller's output path before anything else runs. |
| 525 | const explicitOut = recordingsOutputPath(outPath); |
| 526 | if (!state.lastRaster) throw new ExecError("no screenshot taken yet on this computer — call screenshot first"); |
| 527 | const [x, y, w, h] = region; |
| 528 | if (![x, y, w, h].every((n) => Number.isInteger(n) && n >= 0) || !w || !h || x + w > state.lastRaster.pixels.w || y + h > state.lastRaster.pixels.h) throw new ExecError("region must be [x, y, w, h] in last-raster pixels"); |
| 529 | const src = state.lastRaster.file; |
| 530 | const dir = recordingsDir(); |
| 531 | fs.mkdirSync(dir, { recursive: true }); |
| 532 | const out = explicitOut ?? path.join(dir, `zoom-${crypto.randomBytes(4).toString("hex")}.png`); |
| 533 | await runOk("sips", ["-s", "format", "png", "-c", String(Math.round(h)), String(Math.round(w)), "--cropOffset", String(Math.round(y)), String(Math.round(x)), src, "--out", out], { timeoutMs: 15_000 }); |
| 534 | const parent = state.lastRaster; |
| 535 | state.lastRaster = { file: out, bytes: fs.statSync(out).size, source: src, region, |
| 536 | points: { x: (parent.points?.x ?? 0) + x / parent.scale, y: (parent.points?.y ?? 0) + y / parent.scale, w: w / parent.scale, h: h / parent.scale }, |
| 537 | pixels: { w, h }, scale: parent.scale, capturedAt: new Date().toISOString() }; |
| 538 | return state.lastRaster; |
| 539 | } |
| 540 | |
| 541 | // ---------- recording ---------- |
| 542 | const rec = new Map(); // Includes starting children so session close owns them too. |
| 543 | |
| 544 | function requestRecordingStop(r) { |
| 545 | if (r.child.exitCode == null && r.child.signalCode == null) { |
| 546 | r.child.stdin.end(); |
| 547 | r.child.kill("SIGINT"); |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | async function waitForRecordingStop(r, timeoutMs) { |
| 552 | let timer; |
| 553 | try { |
| 554 | return await Promise.race([r.completion, new Promise(resolve => { |
| 555 | timer = setTimeout(async () => { |
| 556 | r.child.kill("SIGKILL"); |
| 557 | let reapTimer; |
| 558 | const terminated = await Promise.race([r.completion.then(() => true), new Promise(done => { reapTimer = setTimeout(() => done(false), 500); })]); |
| 559 | clearTimeout(reapTimer); |
| 560 | resolve({ code: -1, terminated, error: terminated ? "screen recorder finalization timed out; partial file retained" : "screen recorder could not be terminated; recording ownership retained for retry" }); |
| 561 | }, timeoutMs); |
| 562 | })]); |
| 563 | } finally { clearTimeout(timer); } |
| 564 | } |
| 565 | |
| 566 | async function recordingStart({ display, durationSec, region, app_ref, window_id } = {}) { |
| 567 | const dir = recordingsDir(); |
| 568 | fs.mkdirSync(dir, { recursive: true }); |
| 569 | const id = crypto.randomBytes(4).toString("hex"); |
| 570 | const file = path.join(dir, `rec-${id}.mov`); |
| 571 | const displays = await displayInfo(); |
| 572 | // app_ref scopes the recording to the app's window rect: resolved once at |
| 573 | // start through the same window_info the AX path uses, so a background |
| 574 | // window records behind the user's work. The rect is fixed at start — |
| 575 | // it does not track later moves or resizes. |
| 576 | let window = null; |
| 577 | if (app_ref !== undefined || window_id != null) { |
| 578 | window = await native("window_info", { app_ref: app_ref === undefined ? state.inputApp ?? undefined : app_ref, window_id }); |
| 579 | if (!window?.points || !(window.points.w > 0) || !(window.points.h > 0)) throw new ExecError("the selected application has no capturable window — call list_windows"); |
| 580 | if (region) throw new ExecError("choose app_ref or region, not both"); |
| 581 | region = [window.points.x, window.points.y, window.points.w, window.points.h]; |
| 582 | } |
| 583 | let disp = display ?? state.activeDisplay; |
| 584 | if (window && display == null) { |
| 585 | const cx = region[0] + region[2] / 2, cy = region[1] + region[3] / 2; |
| 586 | const host = displays.find(d => d.points && cx >= d.points.x && cx < d.points.x + d.points.w && cy >= d.points.y && cy < d.points.y + d.points.h); |
| 587 | if (host) disp = host.index; |
| 588 | } |
| 589 | const selected = displays.find(d => d.index === disp); |
| 590 | if (!selected) throw new ExecError("choose one available display for recording"); |
| 591 | if (durationSec != null && (!Number.isFinite(durationSec) || durationSec <= 0)) throw new ExecError("durationSec must be positive"); |
| 592 | const capabilities = await native("input_capabilities"); |
| 593 | if (capabilities?.record_owner_pipe !== 1) throw new ExecError("native screen recorder cannot own its client lifetime; update Computer Use before recording"); |
| 594 | const helper = await nativeHelper(); |
| 595 | throwIfAborted(); |
| 596 | const child = spawn(helper, [JSON.stringify({ tool: "record", args: { file, displayID: selected.id, region, durationSec, owner_pipe: true } })], { stdio: ["pipe", "pipe", "pipe"] }); |
| 597 | child.stdin.on("error", () => {}); |
| 598 | const startedAt = new Date().toISOString(); |
| 599 | let stderr = "", output = "", ready = false; |
| 600 | const completion = new Promise(resolve => { |
| 601 | child.once("error", error => resolve({ code: -1, error: error.message })); |
| 602 | child.once("close", code => resolve({ code, error: stderr.trim() })); |
| 603 | }); |
| 604 | child.stderr.on("data", chunk => { stderr = (stderr + chunk).slice(-4000); }); |
| 605 | const recording = { child, completion, pid: child.pid, file, startedAt, mode: "ScreenCaptureKit", display: disp }; |
| 606 | rec.set(id, recording); |
| 607 | const signal = currentSignal(); |
| 608 | let timer, abort; |
| 609 | try { |
| 610 | await new Promise((resolve, reject) => { |
| 611 | abort = () => { requestRecordingStop(recording); reject(Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" })); }; |
| 612 | signal?.addEventListener("abort", abort, { once: true }); |
| 613 | if (signal?.aborted) { abort(); return; } |
| 614 | timer = setTimeout(() => reject(new ExecError("screen recorder startup timed out")), 20_000); |
| 615 | child.stdout.on("data", chunk => { |
| 616 | output += chunk; |
| 617 | let i; |
| 618 | while ((i = output.indexOf("\n")) >= 0) { |
| 619 | const line = output.slice(0, i); output = output.slice(i + 1); |
| 620 | try { if (JSON.parse(line).ready) { ready = true; resolve(); } } catch {} |
| 621 | } |
| 622 | }); |
| 623 | completion.then(result => { if (!ready) reject(new ExecError(result.error || "screen recorder exited before capture started")); }); |
| 624 | }); |
| 625 | throwIfAborted(); |
| 626 | return { id, pid: child.pid, file, display: disp, durationSec: durationSec ?? null, region: region ?? null, fps: 30, mode: "ScreenCaptureKit", startedAt, |
| 627 | ...(window ? { window: { id: window.window_id ?? null, name: window.name ?? null }, note: "Recording the window's rect as it was at start; it does not track moves or resizes." } : {}) }; |
| 628 | } catch (error) { |
| 629 | requestRecordingStop(recording); |
| 630 | const result = await waitForRecordingStop(recording, 2_000); |
| 631 | if (result.terminated !== false) rec.delete(id); |
| 632 | throw error; |
| 633 | } finally { |
| 634 | clearTimeout(timer); |
| 635 | signal?.removeEventListener("abort", abort); |
| 636 | } |
| 637 | } |
| 638 | |
| 639 | async function recordingStop({ id }) { |
| 640 | const r = rec.get(id); |
| 641 | if (!r) throw new ExecError(`unknown or already-finished recording "${id}"`); |
| 642 | requestRecordingStop(r); |
| 643 | const result = await waitForRecordingStop(r, 20_000); |
| 644 | if (result.code !== 0) throw new ExecError(result.error || "screen recorder failed; partial file retained"); |
| 645 | const size = fs.existsSync(r.file) ? fs.statSync(r.file).size : 0; |
| 646 | if (!size) throw new ExecError("screen recorder produced no video"); |
| 647 | rec.delete(id); |
| 648 | return { id, file: r.file, mp4: null, bytes: size, mode: r.mode, startedAt: r.startedAt, stoppedAt: new Date().toISOString() }; |
| 649 | } |
| 650 | |
| 651 | async function closeSession() { |
| 652 | // The preview this session showed must not outlive the session; a panel |
| 653 | // from a dead session has no owner to refresh or hide it. |
| 654 | stopPreviewLoop(); |
| 655 | await quiescePreview(); |
| 656 | if (state.previewEnabled && state.inputApp) { |
| 657 | try { await native("preview_notify", { enabled: false }); } catch { /* hiding is best-effort */ } |
| 658 | } |
| 659 | state.previewEnabled = false; |
| 660 | await browser.close().catch(() => {}); |
| 661 | const owned = [...rec.entries()]; |
| 662 | for (const [, recording] of owned) requestRecordingStop(recording); |
| 663 | const results = await Promise.all(owned.map(async ([id, recording]) => { |
| 664 | const result = await waitForRecordingStop(recording, 2_000); |
| 665 | if (result.terminated !== false) rec.delete(id); |
| 666 | return result; |
| 667 | })); |
| 668 | const failed = results.find(result => result.code !== 0); |
| 669 | if (failed) throw new ExecError(failed.error || "screen recorder failed; partial file retained"); |
| 670 | } |
| 671 | |
| 672 | async function recordingStatus({ id }) { |
| 673 | const r = rec.get(id); |
| 674 | if (!r) return { id, running: false }; |
| 675 | const alive = r.child.exitCode == null && r.child.signalCode == null; |
| 676 | return { id, running: alive, pid: r.pid, file: r.file, bytes: fs.existsSync(r.file) ? fs.statSync(r.file).size : 0, startedAt: r.startedAt }; |
| 677 | } |
| 678 | |
| 679 | async function recordingList() { |
| 680 | const dir = recordingsDir(); |
| 681 | const out = []; |
| 682 | for (const f of fs.existsSync(dir) ? fs.readdirSync(dir) : []) { |
| 683 | const full = path.join(dir, f); |
| 684 | const st = fs.statSync(full); |
| 685 | if (st.isFile() && /\.(mov|mp4|png|jpe?g)$/i.test(f)) out.push({ file: full, bytes: st.size, modifiedAt: st.mtime.toISOString() }); |
| 686 | } |
| 687 | out.sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)); |
| 688 | return { dir, recordings: out.slice(0, 50), running: [...rec.keys()] }; |
| 689 | } |
| 690 | |
| 691 | // ---------- apps / windows ---------- |
| 692 | async function listApps(args = {}) { |
| 693 | if (args?.installed === true) { |
| 694 | const r = await native("installed_apps", {}); |
| 695 | const apps = Array.isArray(r?.apps) ? r.apps : []; |
| 696 | return { |
| 697 | apps, |
| 698 | total: apps.length, |
| 699 | installed: true, |
| 700 | note: "Installed catalog from /Applications, /System/Applications and ~/Applications; running flags reflect this moment. This scan takes a moment.", |
| 701 | }; |
| 702 | } |
| 703 | const r = await native("list_apps"); |
| 704 | const apps = Array.isArray(r?.apps) ? r.apps : []; |
| 705 | const shown = selectApps(apps, args?.all === true); |
| 706 | return { |
| 707 | apps: shown, |
| 708 | total: apps.length, |
| 709 | filtered: args?.all === true ? "all" : "regular", |
| 710 | ...(shown.length !== apps.length ? { note: "Regular (user-facing) apps only — pass all:true to include menu-bar helpers and background processes." } : {}), |
| 711 | }; |
| 712 | } |
| 713 | |
| 714 | async function listWindows({ app_ref } = {}) { return native("list_windows", { app_ref: app_ref === undefined ? state.inputApp ?? undefined : app_ref }); } |
| 715 | |
| 716 | async function openApplication({ name, bundle_id: bid, pid, url: urlArg, activate = false } = {}) { |
| 717 | if (!name && !bid && !pid) throw new ExecError("open_application needs name, bundle_id or pid"); |
| 718 | // Failed selection must not leave an earlier app armed for foreground |
| 719 | // input, nor a buffered drag aimed at the previous binding. |
| 720 | state.foregroundInput = false; |
| 721 | state.inputApp = null; |
| 722 | state.heldDrag = null; |
| 723 | // pid is the most specific identity and the only one that separates two |
| 724 | // processes of the same bundle (e.g. a second Chrome on its own profile), |
| 725 | // so it wins when given. |
| 726 | const find = {}; |
| 727 | if (pid) find.pid = pid; else if (bid) find.bundle_id = bid; else find.name = String(name).replace(/\.app$/, ""); |
| 728 | let p; |
| 729 | let launched = false; |
| 730 | // Binding an already-running app must not ask LaunchServices to reopen |
| 731 | // it: reopen can raise windows even with open -g on some applications. |
| 732 | if (!urlArg) { |
| 733 | try { p = await native("app_info", { app_ref: find, activate }); } |
| 734 | catch (error) { |
| 735 | if (!error.message.includes("application not found")) throw error; |
| 736 | } |
| 737 | } |
| 738 | if (!p?.found) { |
| 739 | if (!name && !bid) throw Object.assign(new ExecError(`no running application with pid ${pid}; call list_apps for the current processes`), { code: "app_not_found" }); |
| 740 | const args = []; |
| 741 | if (urlArg) args.push(urlArg); |
| 742 | if (bid) args.unshift("-b", bid); else args.unshift("-a", name); |
| 743 | if (!activate) args.unshift("-g"); |
| 744 | const r = await runL("open", args, { timeoutMs: 25_000 }); |
| 745 | if (r.code !== 0) { |
| 746 | const stderr = (r.stderr ?? "").trim(); |
| 747 | // A name or bundle id that resolves nowhere is a stable refusal code, |
| 748 | // not a generic opener failure — agents branch on the code. |
| 749 | const code = /Unable to find application|failed while trying to determine the application/i.test(stderr) |
| 750 | ? "app_not_found" : undefined; |
| 751 | throw Object.assign(new ExecError(`open failed: ${stderr.slice(0, 200)}`), { code }); |
| 752 | } |
| 753 | launched = true; |
| 754 | await new Promise((res) => setTimeout(res, 600)); |
| 755 | p = await native("app_info", { app_ref: find, activate }); |
| 756 | } |
| 757 | if (activate && p?.frontmost === false) throw Object.assign(new ExecError("The selected application did not become frontmost; no input mode was enabled. Continue with background control or wait for the user."), { code: "activation_not_confirmed" }); |
| 758 | if (p?.bundle_id === "net.codewhale.computer-use") throw Object.assign(new ExecError("The Computer Use setup and safety controls belong to the user and cannot be operated by this plugin."), { code: "protected_application" }); |
| 759 | // A bare executable has no bundle id; carrying an empty one would make the |
| 760 | // identity unmatchable. |
| 761 | state.inputApp = { pid: p.pid, ...(p.bundle_id ? { bundle_id: p.bundle_id } : {}), ...(p.name ? { name: p.name } : {}) }; |
| 762 | state.foregroundInput = !!activate; |
| 763 | // Surface the watch panel on bind; a capture failure (e.g. missing Screen |
| 764 | // Recording) must never block the bind itself. The first successful |
| 765 | // capture also starts the refresh loop so the panel stays live while bound. |
| 766 | if (state.previewEnabled) { |
| 767 | previewBusy = true; |
| 768 | updatePreview(true).catch(() => {}).finally(() => { previewBusy = false; }); |
| 769 | } |
| 770 | return { launched, activate, keyboard_delivery: activate ? "foreground-guarded" : "process", input_scope: activate ? "shared-desktop" : "application", shared_pointer: false, pointer_route: "window-record", isolated_desktop: false, url: urlArg ?? null, resolved: p?.found ? { name: p.name, pid: p.pid, bundle_id: p.bundle_id, frontmost: p.frontmost } : null, |
| 771 | ...(Number.isFinite(p?.yield_ms) && p.yield_ms > 0 ? { yield_ms: p.yield_ms } : {}) }; |
| 772 | } |
| 773 | |
| 774 | /** |
| 775 | * Menu items by title path, through accessibility only: no key events, no |
| 776 | * focus lease. Menus expose items only while open, so each level is pressed |
| 777 | * and the next is polled for. Exact titles; an ellipsis is part of the title. |
| 778 | */ |
| 779 | async function invokeMenu(menuPath) { |
| 780 | if (!state.inputApp) throw new ExecError("open_application first — invoke_menu acts on the bound application"); |
| 781 | if (!Array.isArray(menuPath) || menuPath.length < 1 || menuPath.length > 3 || menuPath.some((s) => typeof s !== "string" || !s.trim())) { |
| 782 | throw new ExecError('invoke_menu needs path: 1..3 non-empty menu titles, e.g. ["File","New"]'); |
| 783 | } |
| 784 | const titles = menuPath.map((s) => s.trim()); |
| 785 | const app_ref = state.inputApp; |
| 786 | const pressed = []; |
| 787 | for (let level = 0; level < titles.length; level++) { |
| 788 | const found = await findMenuItem(app_ref, titles[level], level === 0); |
| 789 | if (!found) { |
| 790 | throw Object.assign(new ExecError(`menu item "${titles[level]}" not found ${pressed.length ? `under ${pressed.join(" ▸ ")}` : "on the menu bar"} — menus expose items only while open; check the exact title with get_app_state (an ellipsis is part of the title)`), { code: "menu_item_not_found" }); |
| 791 | } |
| 792 | if (found.enabled === false) { |
| 793 | throw Object.assign(new ExecError(`menu item "${titles[level]}" is present but disabled right now — the app validates it against its current state (in background mode that is often a missing key window for window-targeted commands like Close). Use an element action on the window's own control instead of pressing a disabled item.`), { code: "menu_item_disabled" }); |
| 794 | } |
| 795 | const target = { app_ref, windowIndex: found.windowIndex ?? 0, path: found.path, role: found.role, label: found.label }; |
| 796 | assertBoundElement(target); |
| 797 | const action = found.role === "AXMenuItem" && (found.actions ?? []).includes("AXPick") ? "AXPick" : "AXPress"; |
| 798 | await native("perform_action", { target, action }); |
| 799 | pressed.push(titles[level]); |
| 800 | if (level < titles.length - 1) await wait(140); |
| 801 | } |
| 802 | return { action_sent: true, strategy: "a11y", route: "accessibility", delivery: "background", menu: pressed, front_lease: false, |
| 803 | note: "Menu activation used accessibility only — no key events or focus lease. Verify the app effect (list_windows / get_app_state) before reporting success." }; |
| 804 | } |
| 805 | |
| 806 | /** Poll for the exact menu element; opens and submenu population are async. */ |
| 807 | async function findMenuItem(app_ref, label, menuBar) { |
| 808 | const deadline = Date.now() + 4_000; |
| 809 | for (;;) { |
| 810 | const obs = await native("get_app_state", { app_ref, detail: "full" }); |
| 811 | const hit = pickMenuElement(obs?.elements ?? [], label, menuBar); |
| 812 | if (hit) return hit; |
| 813 | if (Date.now() >= deadline) return null; |
| 814 | await wait(120); |
| 815 | } |
| 816 | } |
| 817 | |
| 818 | // ---------- clipboard / cursor / waits ---------- |
| 819 | async function readClipboard() { |
| 820 | const r = await runL("pbpaste", [], { timeoutMs: 5_000, maxBuffer: 4 * 1024 * 1024 }); |
| 821 | return { text: r.stdout, encoding: "utf8" }; |
| 822 | } |
| 823 | async function writeClipboard({ text }) { |
| 824 | const child = spawn("pbcopy", [], { stdio: ["pipe", "ignore", "ignore"] }); |
| 825 | child.stdin.end(String(text ?? "")); |
| 826 | await new Promise((res, rej) => { child.on("close", res); child.on("error", rej); }); |
| 827 | return { written: String(text ?? "").length }; |
| 828 | } |
| 829 | async function cursorPosition() { return native("cursor_position"); } |
| 830 | |
| 831 | // ---------- app scripting ---------- |
| 832 | // The programmatic interface into apps that ship a scripting dictionary: |
| 833 | // osascript runs AppleScript (default) or JXA. It never moves the pointer, |
| 834 | // needs no Accessibility grant, and returns values instead of "sent" |
| 835 | // receipts — which is why it ranks above clicking wherever a dictionary |
| 836 | // exists. The script travels as one argv entry; no shell ever parses it. |
| 837 | async function appScript({ script, language = "applescript", timeout } = {}) { |
| 838 | if (typeof script !== "string" || !script.trim()) { |
| 839 | throw Object.assign(new ExecError("app_script needs a non-empty script string"), { code: "bad_args" }); |
| 840 | } |
| 841 | const lang = language === "javascript" ? ["-l", "JavaScript"] : language === "applescript" ? [] : null; |
| 842 | if (!lang) throw Object.assign(new ExecError('app_script language must be "applescript" or "javascript"'), { code: "bad_args" }); |
| 843 | const timeoutMs = Math.min(Math.max(Number(timeout) > 0 ? Number(timeout) : 30, 1), 120) * 1000; |
| 844 | const r = await runL("osascript", [...lang, "-e", script], { timeoutMs, maxBuffer: 8 * 1024 * 1024 }); |
| 845 | if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" }); |
| 846 | if (r.timedOut) throw Object.assign(new ExecError(`app_script timed out after ${Math.round(timeoutMs / 1000)}s — the script or a consent dialog was still open`, r), { code: "script_timeout" }); |
| 847 | if (r.code !== 0) { |
| 848 | const stderr = (r.stderr || r.stdout || "").trim(); |
| 849 | if (/-1743|not authorized to send apple events|not permitted/i.test(stderr)) { |
| 850 | throw Object.assign(new ExecError(`${stderr} — Automation consent was refused or is missing; allow the responsible app to control the target in System Settings → Privacy & Security → Automation`, r), { code: "automation_denied" }); |
| 851 | } |
| 852 | if (/\(-?128\)|user canceled/i.test(stderr)) { |
| 853 | throw Object.assign(new ExecError(stderr || "the script was cancelled by the user", r), { code: "script_cancelled" }); |
| 854 | } |
| 855 | throw Object.assign(new ExecError(stderr || `osascript exited ${r.code}`, r), { code: "script_error" }); |
| 856 | } |
| 857 | return { language, result: r.stdout.trim(), stderr: r.stderr.trim() || null }; |
| 858 | } |
| 859 | |
| 860 | // ---------- probe ---------- |
| 861 | async function probe() { |
| 862 | const caps = { screenshot: true, recording: true, accessibility_tree: true, clipboard: true, displays: true, app_script: true }; |
| 863 | const perms = {}; |
| 864 | try { |
| 865 | const ax = await native("permissions"); |
| 866 | perms.accessibility = ax.trusted ? "granted" : "denied"; |
| 867 | } catch { perms.accessibility = "denied_or_unavailable"; } |
| 868 | caps.accessibility_tree = perms.accessibility === "granted"; |
| 869 | caps.raw_input = caps.accessibility_tree; |
| 870 | try { |
| 871 | const t = os.tmpdir() + `/cu-probe-${crypto.randomBytes(3).toString("hex")}.png`; |
| 872 | const r = await runL("screencapture", ["-x", "-R0,0,2,2", "-t", "png", t], { timeoutMs: 8_000 }); |
| 873 | perms.screen_capture = r.code === 0 ? "ok" : "failed"; |
| 874 | try { fs.rmSync(t, { force: true }); } catch {} |
| 875 | } catch { perms.screen_capture = "failed"; } |
| 876 | caps.screenshot = perms.screen_capture === "ok"; |
| 877 | caps.recording = caps.screenshot; |
| 878 | return { platform: "darwin", capabilities: caps, permissions: perms, note: "macOS does not expose Screen-Recording TCC state to CLI; a black/empty screenshot means Screen Recording permission is missing. Background mode (open_application activate:false) uses process-bound keyboard events and accessibility actions; shared pointer gestures are refused. Foreground control (activate:true) uses the shared desktop and requires exclusive use. Neither mode is an isolated desktop. App-specific behavior still requires verification." }; |
| 879 | } |
| 880 | |
| 881 | return { |
| 882 | platform: "darwin", |
| 883 | probe, |
| 884 | list_displays: displayInfo, |
| 885 | async switch_display({ index }) { |
| 886 | const ds = await displayInfo(); |
| 887 | if (!ds.some((d) => d.index === index)) throw new ExecError(`no display ${index}; have [${ds.map((d) => d.index).join(", ")}]`); |
| 888 | state.activeDisplay = index; |
| 889 | return { activeDisplay: index }; |
| 890 | }, |
| 891 | list_apps: listApps, |
| 892 | set_window_frame: async ({ app_ref, window_id, frame } = {}) => { |
| 893 | if (!frame || !Number.isFinite(frame.x) || !Number.isFinite(frame.y) || !Number.isFinite(frame.w) || !Number.isFinite(frame.h) || frame.w <= 0 || frame.h <= 0) { |
| 894 | throw Object.assign(new ExecError("set_window_frame needs frame {x,y,w,h} with positive w/h"), { code: "bad_args" }); |
| 895 | } |
| 896 | if (!Number.isSafeInteger(window_id) || window_id < 0) { |
| 897 | throw Object.assign(new ExecError("set_window_frame needs window_id (a non-negative window index from list_windows)"), { code: "bad_args" }); |
| 898 | } |
| 899 | const r = await native("set_window_frame", { app_ref, window_id, frame }); |
| 900 | return { ...r, verified: r?.verified === true, note: r?.note ?? "the after frame is the app's own readback; cross-check with list_windows before relying on it" }; |
| 901 | }, |
| 902 | list_windows: listWindows, |
| 903 | open_application: openApplication, |
| 904 | get_app_state: async ({ app_ref, detail, depth, window_id, include_ocr = false, ocr_region } = {}) => { |
| 905 | const t0 = Date.now(); |
| 906 | const t = await native("get_app_state", { app_ref: app_ref === undefined ? state.inputApp ?? undefined : app_ref, detail, window_id }); |
| 907 | if (process.env.CODEWHALE_CU_DEBUG_OBSERVE) console.error(`observe ${Date.now() - t0}ms elements=${t.elements?.length} truncated=${t.truncated}`); |
| 908 | if (!t.found) throw new ExecError("application not found — call list_apps for exact names/pids"); |
| 909 | if (include_ocr) { |
| 910 | // Resolve once through AX, then capture only that exact application's |
| 911 | // selected window. A changing foreground cannot redirect this image. |
| 912 | let raster; |
| 913 | try { |
| 914 | if (!Number.isSafeInteger(t.pid) || t.pid <= 0) throw new ExecError("The observed application did not provide an exact process identity for OCR"); |
| 915 | if ((await native("input_capabilities"))?.window_ocr !== 1) throw new ExecError("The native helper needs an update for selected-window text recognition"); |
| 916 | // PNG here, against the JPEG default: this raster is fed to text |
| 917 | // recognition, not to a viewer, and lossless glyph edges are what |
| 918 | // Vision reads. A single window is small enough that the size the |
| 919 | // JPEG default exists to solve does not arise. |
| 920 | const ocrDir = path.join(recordingsDir(), "captures"); |
| 921 | fs.mkdirSync(ocrDir, { recursive: true }); |
| 922 | raster = await screenshot({ |
| 923 | ...(ocr_region |
| 924 | ? { region: ocr_region } |
| 925 | : { app_ref: { pid: t.pid, ...(t.bundle_id ? { bundle_id: t.bundle_id } : {}) }, window_id }), |
| 926 | path: path.join(ocrDir, `ocr-${crypto.randomBytes(4).toString("hex")}.png`), |
| 927 | }); |
| 928 | const ocr = await native("recognize_text", { file: raster.file }); |
| 929 | if (ocr?.status === "ok" && ocr.pixels?.w === raster.pixels.w && ocr.pixels?.h === raster.pixels.h && Array.isArray(ocr.blocks)) { |
| 930 | t.ocr = { ...ocr, raster, blocks: ocr.blocks.map(block => ({ ...block, target: { |
| 931 | type: "coordinate", x: Math.floor(block.bounds.x + block.bounds.w / 2), y: Math.floor(block.bounds.y + block.bounds.h / 2), |
| 932 | } })) }; |
| 933 | } else { |
| 934 | t.ocr = { status: "unavailable", engine: "apple_vision", reason: ocr?.reason ?? "The native OCR helper needs an update or returned mismatched image dimensions", blocks: [], raster }; |
| 935 | } |
| 936 | } catch (error) { |
| 937 | throwIfAborted(); |
| 938 | if (error.code === "cancelled") throw error; |
| 939 | t.ocr = { status: "unavailable", engine: "apple_vision", reason: error.message, blocks: [], ...(raster ? { raster } : {}) }; |
| 940 | } |
| 941 | } |
| 942 | return t; |
| 943 | }, |
| 944 | resolve_element: async ({ app_ref, windowIndex, path: pathArr } = {}) => { |
| 945 | const r = await native("resolve_element", { app_ref, windowIndex: windowIndex ?? 0, path: pathArr ?? [] }); |
| 946 | return { found: !!r?.found, element: r?.element ?? null, reason: r?.reason ?? null }; |
| 947 | }, |
| 948 | preview: async ({ enabled = true } = {}) => { |
| 949 | state.previewEnabled = enabled; |
| 950 | if (!enabled) { stopPreviewLoop(); await quiescePreview(); await native("preview_notify", { enabled: false }); return { enabled: false }; } |
| 951 | if (!state.inputApp) throw new ExecError("open_application first to choose the preview app"); |
| 952 | return updatePreview(true); |
| 953 | }, |
| 954 | screenshot, |
| 955 | zoom, |
| 956 | left_click: async ({ target, strategy = "auto" } = {}) => { |
| 957 | if (target?.type !== "element" || strategy === "event" || strategy === "app") return pointerClick("left", target?.x, target?.y, 1, strategy); |
| 958 | if (!["auto", "a11y"].includes(strategy)) throw new ExecError(`strategy must be auto, a11y, app or event (got ${JSON.stringify(strategy)})`); |
| 959 | try { |
| 960 | assertBoundElement(target); |
| 961 | if ((await native("input_capabilities"))?.element_identity !== 1) throw new ExecError("native helper needs an update for element identity validation"); |
| 962 | const semantic = ["AXTextField", "AXTextArea", "AXComboBox", "AXRow", "AXCell", "AXMenuItem"].includes(target.role); |
| 963 | if (semantic) await requireBackgroundActions(); |
| 964 | const receipt = await native(semantic ? "click_element" : "perform_action", { target, action: "AXPress" }); |
| 965 | if (!receipt?.action_sent) throw new ExecError("element press was not acknowledged"); |
| 966 | return { ...receipt, action: receipt.action ?? "AXPress", strategy: "a11y", pointer_moved: false, |
| 967 | element: { role: target.role, label: target.label ?? null }, verified: receipt.verified ?? false, verification_required: "observation" }; |
| 968 | } catch (error) { |
| 969 | // An AX frame can cover other controls. Never turn a refused or |
| 970 | // ambiguous element press into another element's press or a raw click. |
| 971 | error.message += ' — no coordinate fallback was sent; take a fresh screenshot or OCR observation and choose an advertised action or a separate computer'; |
| 972 | throw error; |
| 973 | } |
| 974 | }, |
| 975 | double_click: ({ target } = {}) => pointerClick("left", target?.x, target?.y, 2), |
| 976 | triple_click: ({ target } = {}) => pointerClick("left", target?.x, target?.y, 3), |
| 977 | right_click: async ({ target } = {}) => { |
| 978 | if (target?.type !== "element") return pointerClick("right", target?.x, target?.y, 1); |
| 979 | assertBoundElement(target); |
| 980 | await requireBackgroundActions(); |
| 981 | return native("click_element", { target, context: true }); |
| 982 | }, |
| 983 | middle_click: ({ target } = {}) => pointerClick("middle", target?.x, target?.y, 1), |
| 984 | // Hover moves only the Codewhale pointer: a mouse-moved record to the |
| 985 | // window under it. While a button is held the point joins the drag path, |
| 986 | // and the whole drag is delivered to the window on left_mouse_up. |
| 987 | mouse_move: async ({ target } = {}) => { |
| 988 | assertInScreen(target?.x, target?.y); |
| 989 | if (state.heldDrag) { |
| 990 | if (state.heldDrag.path.length >= 64) throw new ExecError("a held drag takes at most 64 intermediate points; release it with left_mouse_up"); |
| 991 | state.heldDrag.path.push({ x: target.x, y: target.y }); |
| 992 | state.pointer = { x: target.x, y: target.y }; |
| 993 | return { action_sent: false, deferred: true, strategy: "window-record", at: state.pointer, pointer_moved: false, |
| 994 | note: "the button is held on the Codewhale pointer; the drag reaches the window on left_mouse_up" }; |
| 995 | } |
| 996 | const r = await windowPointer([{ type: MOUSE_MOVED, x: target.x, y: target.y, button: 0, clickState: 0 }]); |
| 997 | return { ...r, at: { x: target.x, y: target.y } }; |
| 998 | }, |
| 999 | left_mouse_down: async ({ target } = {}) => { |
| 1000 | assertInScreen(target?.x, target?.y); |
| 1001 | requireFocusControl(); |
| 1002 | if (state.heldDrag) throw new ExecError("this session already holds the left pointer button; release it first"); |
| 1003 | if ((await native("input_capabilities"))?.window_record !== 1) { |
| 1004 | throw Object.assign(new ExecError("Pointer input needs the window-routed pointer, which this helper cannot resolve; update Computer Use. The user's cursor is never used instead."), { code: "bg_dispatch_unavailable" }); |
| 1005 | } |
| 1006 | state.heldDrag = { from: { x: target.x, y: target.y }, path: [], app: state.inputApp }; |
| 1007 | state.pointer = { x: target.x, y: target.y }; |
| 1008 | return { action_sent: false, deferred: true, strategy: "window-record", at: state.pointer, pointer_moved: false, |
| 1009 | note: "the button is held on the Codewhale pointer; the press reaches the window with the rest of the drag on left_mouse_up" }; |
| 1010 | }, |
| 1011 | left_mouse_up: async ({ target } = {}) => { |
| 1012 | const held = state.heldDrag; |
| 1013 | if (!held) throw new ExecError("no agent pointer button is held by this session"); |
| 1014 | state.heldDrag = null; |
| 1015 | const loc = target ?? state.pointer ?? held.from; |
| 1016 | assertInScreen(loc.x, loc.y); |
| 1017 | if (held.app?.pid !== state.inputApp?.pid) throw new ExecError("the bound application changed while the button was held; nothing was sent"); |
| 1018 | const { from } = held; |
| 1019 | const steps = [ |
| 1020 | { type: MOUSE_MOVED, x: from.x, y: from.y, button: 0, clickState: 0 }, |
| 1021 | { type: MOUSE.left.down, x: from.x, y: from.y, button: 0, clickState: 1, delayMs: 60 }, |
| 1022 | ]; |
| 1023 | let last = from; |
| 1024 | for (const p of [...held.path, loc]) { |
| 1025 | const n = Math.max(1, Math.min(12, Math.ceil(Math.hypot(p.x - last.x, p.y - last.y) / 20))); |
| 1026 | for (let i = 1; i <= n; i++) steps.push({ type: MOUSE.left.dragged, x: last.x + ((p.x - last.x) * i) / n, y: last.y + ((p.y - last.y) * i) / n, button: 0, clickState: 1, delayMs: 30 }); |
| 1027 | last = p; |
| 1028 | } |
| 1029 | steps.push({ type: MOUSE.left.up, x: loc.x, y: loc.y, button: 0, clickState: 1, delayMs: 80 }); |
| 1030 | const r = await windowPointer(steps); |
| 1031 | state.pointer = { x: loc.x, y: loc.y }; |
| 1032 | return { ...r, from, to: state.pointer, at: state.pointer }; |
| 1033 | }, |
| 1034 | left_click_drag: async ({ from_target: from, to } = {}) => { |
| 1035 | assertInScreen(from?.x, from?.y); assertInScreen(to?.x, to?.y); |
| 1036 | const steps = [ |
| 1037 | { type: MOUSE_MOVED, x: from.x, y: from.y, button: 0, clickState: 0 }, |
| 1038 | { type: MOUSE.left.down, x: from.x, y: from.y, button: 0, clickState: 1, delayMs: 60 }, |
| 1039 | ]; |
| 1040 | const n = 12; |
| 1041 | for (let i = 1; i <= n; i++) { |
| 1042 | steps.push({ type: MOUSE.left.dragged, x: from.x + ((to.x - from.x) * i) / n, y: from.y + ((to.y - from.y) * i) / n, button: 0, clickState: 1, delayMs: 45 }); |
| 1043 | } |
| 1044 | steps.push({ type: MOUSE.left.up, x: to.x, y: to.y, button: 0, clickState: 1, delayMs: 80 }); |
| 1045 | return { ...(await windowPointer(steps)), from, to }; |
| 1046 | }, |
| 1047 | scroll: async ({ target, direction = "down", amount = 5 } = {}) => { |
| 1048 | assertInScreen(target?.x, target?.y); |
| 1049 | if (!state.foregroundInput) { |
| 1050 | await requireBackgroundActions(); |
| 1051 | if (target.type === "element") { |
| 1052 | assertBoundElement(target); |
| 1053 | return native("scroll_element", { target, direction, amount }); |
| 1054 | } |
| 1055 | const receipt = await native("hit_test", { x: target.x, y: target.y, perform: true, direction, amount, |
| 1056 | operation: ["left", "right"].includes(direction) ? "scroll-horizontal" : "scroll-vertical" }); |
| 1057 | if (receipt?.action_sent) return receipt; |
| 1058 | if ((await native("input_capabilities"))?.window_record !== 1) { |
| 1059 | throw Object.assign(new ExecError(`No background scrollbar at this point (${receipt?.reason ?? "not_found"}); choose an observed scroll area or a separate computer.`), { code: "background_scroll_unavailable" }); |
| 1060 | } |
| 1061 | } |
| 1062 | // No AX scrollbar here (overlay scrollers, web pages), or foreground |
| 1063 | // control: wheel records reach the view through the window route. |
| 1064 | const dx = direction === "left" ? amount : direction === "right" ? -amount : 0; |
| 1065 | const dy = direction === "up" ? amount : direction === "down" ? -amount : 0; |
| 1066 | const notches = Math.max(1, Math.min(100, Math.round(amount))); |
| 1067 | const steps = []; |
| 1068 | for (let i = 0; i < notches; i++) steps.push({ scroll: [Math.sign(dx), Math.sign(dy)], x: target.x, y: target.y, delayMs: 15 }); |
| 1069 | return { ...(await windowPointer(steps)), direction, amount, verified: false, verification_required: "observation" }; |
| 1070 | }, |
| 1071 | type: (args = {}) => native("type", args), |
| 1072 | key: async ({ text, repeat = 1, target } = {}) => { |
| 1073 | const { flags, code, key } = parseChord(text); |
| 1074 | const n = Math.max(1, Math.min(100, repeat)); |
| 1075 | // Modified and window-targeted keys need a key window. Background |
| 1076 | // mode must never make one by borrowing the user's keyboard focus. |
| 1077 | if (flags !== 0 || target != null) requireFocusControl(); |
| 1078 | let yieldMs = 0; |
| 1079 | for (let i = 0; i < n; i++) { |
| 1080 | const press = await withPressedKey(code, flags, () => {}); |
| 1081 | if (Number.isFinite(press?.yield_ms)) yieldMs = Math.max(yieldMs, press.yield_ms); |
| 1082 | if (i < n - 1) await wait(30); |
| 1083 | } |
| 1084 | return { action_sent: true, key, code, keyboard_delivery: state.foregroundInput ? "foreground-guarded" : "process", repeat: n, |
| 1085 | ...(yieldMs > 0 ? { yield_ms: yieldMs } : {}), |
| 1086 | ...(flags !== 0 && !state.foregroundInput ? { note: "process delivery (no focus lease was taken); menu key equivalents can be dropped without a key window. Verify the effect before retrying, or use invoke_menu for app menu commands." } : {}) }; |
| 1087 | }, |
| 1088 | hold_key: async ({ text, duration } = {}) => { |
| 1089 | const { flags, code, key } = parseChord(text); |
| 1090 | if (flags !== 0) requireFocusControl(); |
| 1091 | const d = Math.max(0.05, Math.min(30, Number(duration) || 1)); |
| 1092 | const press = await withPressedKey(code, flags, () => wait(d * 1000)); |
| 1093 | return { action_sent: true, key, keyboard_delivery: state.foregroundInput ? "foreground-guarded" : "process", heldSec: d, |
| 1094 | ...(Number.isFinite(press?.yield_ms) && press.yield_ms > 0 ? { yield_ms: press.yield_ms } : {}) }; |
| 1095 | }, |
| 1096 | set_value: async (args = {}) => { |
| 1097 | if (args.target?.type !== "element") throw new ExecError("set_value needs an element target — {type:'element',index} from get_app_state"); |
| 1098 | try { |
| 1099 | return await native("set_value", args); |
| 1100 | } catch (error) { |
| 1101 | // Web text controls ignore AXValue writes, so the native side refuses |
| 1102 | // before dispatch. The replacement path is focus + select-all + type |
| 1103 | // with a read-back verify — the same shape kimi-cu uses, with the |
| 1104 | // value proven rather than asserted. |
| 1105 | if (!/web area/i.test(error.message)) throw error; |
| 1106 | if (args.target?.type !== "element") throw error; |
| 1107 | requireFocusControl(); |
| 1108 | const value = String(args.value ?? ""); |
| 1109 | await native("focus_element", { target: args.target }); |
| 1110 | // cmd+a through the record channel: menu key equivalents only |
| 1111 | // validate against a key window, which the lease provides. bg_key |
| 1112 | // posts a complete press (down and up); the `down` field is unused. |
| 1113 | await native("bg_key", { code: 0, flags: 1 << 20 }); |
| 1114 | await new Promise((r) => setTimeout(r, 60)); |
| 1115 | await native("type", { text: value }); |
| 1116 | const back = await native("get_value", { target: args.target }); |
| 1117 | const verified = back?.value === value; |
| 1118 | return { action_sent: true, strategy: "focus-type-replace", role: back?.role ?? null, |
| 1119 | after: back?.value ?? null, verified, |
| 1120 | ...(verified ? {} : { note: "replacement did not verify against the control's own value; observe before relying on it" }) }; |
| 1121 | } |
| 1122 | }, |
| 1123 | focus: (args = {}) => native("focus_element", args), |
| 1124 | get_value: (args = {}) => native("get_value", args), |
| 1125 | select_text: async (args = {}) => { |
| 1126 | if (args.target?.type !== "element") throw new ExecError("select_text needs an element target — {type:'element',index} from get_app_state"); |
| 1127 | return native("select_text", args); |
| 1128 | }, |
| 1129 | perform_action: async (args = {}) => { |
| 1130 | if (args.target?.type !== "element") throw new ExecError("perform_action needs an element target — {type:'element',index} from get_app_state"); |
| 1131 | return native("perform_action", args); |
| 1132 | }, |
| 1133 | invoke_menu: async ({ path: menuPath } = {}) => invokeMenu(menuPath), |
| 1134 | app_script: appScript, |
| 1135 | read_clipboard: readClipboard, |
| 1136 | write_clipboard: writeClipboard, |
| 1137 | cursor_position: cursorPosition, |
| 1138 | recordingStart, |
| 1139 | recordingStop, |
| 1140 | recordingStatus, |
| 1141 | recordingList, |
| 1142 | closeSession, |
| 1143 | list_sessions: async () => ({ |
| 1144 | via: "direct", |
| 1145 | count: 1, |
| 1146 | sessions: [{ |
| 1147 | target: state.inputApp ? { pid: state.inputApp.pid, ...(state.inputApp.bundle_id ? { bundle_id: state.inputApp.bundle_id } : {}), ...(state.inputApp.name ? { name: state.inputApp.name } : {}) } : null, |
| 1148 | mode: state.foregroundInput ? "foreground" : "background", |
| 1149 | action: null, |
| 1150 | ageSec: 0, |
| 1151 | inputHeld: !!state.heldDrag, |
| 1152 | }], |
| 1153 | }), |
| 1154 | kill_app: async (args = {}) => { |
| 1155 | const { name, bundle_id, pid, force } = args; |
| 1156 | if (!name && !bundle_id && pid == null) throw Object.assign(new ExecError("kill_app needs name, bundle_id or pid"), { code: "bad_args" }); |
| 1157 | return native("kill_app", { name, bundle_id, pid, force: force === true }); |
| 1158 | }, |
| 1159 | browser_start: browser.start, |
| 1160 | browser_status: browser.status, |
| 1161 | browser_navigate: browser.navigate, |
| 1162 | browser_click: browser.click, |
| 1163 | browser_type: browser.type, |
| 1164 | browser_screenshot: browser.screenshot, |
| 1165 | browser_stop: browser.stop, |
| 1166 | // A held drag is buffered, not held on any real button: nothing to release. |
| 1167 | releaseInput: async () => { state.heldDrag = null; }, |
| 1168 | }; |
| 1169 | } |
| 1170 | |
| 1171 | export default { create }; |
| 1172 |