返回 CodeWhale
linux.mjs
根目录 / crates / tui / plugins / computer-use / src / backends / linux.mjs
1 // Linux backend — X11 first (xdotool/wmctrl/scrot/xclip), Wayland where the
2 // right tools exist (grim/wtype/ydotool/wf-recorder/wl-clipboard). The
3 // accessibility tree comes from AT-SPI via python3+pyatspi when installed.
4 // Everything probes at call time and fails closed with the missing tool named.
5 import fs from "node:fs";
6 import os from "node:os";
7 import path from "node:path";
8 import crypto from "node:crypto";
9 import { spawn } from "node:child_process";
10 import { run as nativeRun, runOk, ExecError, tryJson, have as nativeHave, withSignal, throwIfAborted, wait } from "../exec.mjs";
11 import { pngSize } from "../png-size.mjs";
12 import { createBrowser } from "../browser-cdp.mjs";
13 import { recordingsDir, recordingsOutputPath } from "../recordings.mjs";
14
15 const XKEYS = {
16 return: "Return", enter: "Return", tab: "Tab", escape: "Escape", esc: "Escape",
17 space: "space", backspace: "BackSpace", delete: "Delete", home: "Home", end: "End",
18 pageup: "Page_Up", pagedown: "Page_Down", left: "Left", right: "Right", up: "Up",
19 down: "Down", capslock: "Caps_Lock", menu: "Menu", print: "Print",
20 };
21
22 function spawnDetached(cmd, args, stdinText = "", quiet = true) {
23 const child = spawn(cmd, args, {
24 stdio: stdinText ? ["pipe", "ignore", quiet ? "ignore" : "pipe"] : ["ignore", "ignore", quiet ? "ignore" : "pipe"],
25 detached: true,
26 });
27 if (stdinText) child.stdin.end(stdinText);
28 child.unref();
29 return child;
30 }
31
32 /** Coordinate clicks on this backend are always raw pointer events; strategy="a11y" must fail closed rather than silently degrade. */
33 function assertEventStrategy(strategy) {
34 if (strategy != null && strategy !== "auto" && strategy !== "event") {
35 throw new ExecError(`strategy "${strategy}" is macOS-only; this backend dispatches coordinate clicks as raw pointer events — use an element target for a semantic action`);
36 }
37 }
38
39 function appName(ref) {
40 if (ref === undefined) return "";
41 if (!ref || typeof ref !== "object" || Array.isArray(ref) || Object.keys(ref).length !== 1 || typeof ref.name !== "string" || !ref.name.trim()) {
42 throw Object.assign(new ExecError("Linux accessibility targeting supports only a nonblank app_ref.name; PID and bundle selectors are unavailable"), { code: "unsupported_selector" });
43 }
44 return ref.name;
45 }
46
47 function rejectWindowSelectors(args) {
48 if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw Object.assign(new ExecError("Linux window listing and screenshots do not support app_ref or window_id selectors"), { code: "unsupported_selector" });
49 }
50
51 function assertAppRootWindow(index, windowId) {
52 if (windowId !== undefined || (index !== undefined && index !== 0)) throw Object.assign(new ExecError("Linux accessibility paths start at the app root; window selectors are unavailable"), { code: "unsupported_selector" });
53 }
54
55 function outputPath(file) {
56 if (typeof file !== "string" || !path.isAbsolute(file) || file.includes("\0")) throw new ExecError("output path must be an absolute filename");
57 return file;
58 }
59
60 export function create({ exec } = {}) {
61 const run = exec?.run ?? nativeRun;
62 const have = exec?.have ?? nativeHave;
63 const browser = createBrowser({ platform: "linux" });
64 function requireInputOwner() {
65 if (exec?.persistentInputOwner !== true) throw Object.assign(new ExecError(
66 "This held-input gesture requires a connected Codewhale Computer Use desktop helper so a disconnected client cannot leave keys or buttons pressed. Start the helper and reconnect before retrying."
67 ), { code: "input_owner_required" });
68 }
69
70 const tools = {};
71 let session = null; // "x11" | "wayland"
72 let probed = false;
73 let lastRaster = null;
74 let mouseHeld = false;
75 const heldKeys = new Set();
76
77 async function releaseMouse() {
78 if (!mouseHeld) return;
79 await withSignal(null, () => session === "x11" ? xdotool(["mouseup", "1"], { timeoutMs: 2_000 }) : ydotool(["click", "0x80"], { timeoutMs: 2_000 }));
80 mouseHeld = false;
81 }
82
83 async function releaseKey(key) {
84 if (!heldKeys.has(key)) return;
85 await withSignal(null, () => xdotool(["keyup", key], { timeoutMs: 2_000 }));
86 heldKeys.delete(key);
87 }
88
89 async function releaseInput() {
90 await releaseMouse();
91 for (const key of heldKeys) await releaseKey(key);
92 }
93
94 async function probeSession() {
95 if (probed) return session;
96 throwIfAborted();
97 const wayland = !!(process.env.WAYLAND_DISPLAY || process.env.XDG_SESSION_TYPE === "wayland");
98 const x11 = !!(process.env.DISPLAY || process.env.XDG_SESSION_TYPE === "x11");
99 session = wayland && !x11 ? "wayland" : x11 ? "x11" : null;
100 if (session === null) {
101 const e = new ExecError("no X11 ($DISPLAY) or Wayland ($WAYLAND_DISPLAY) session visible to this process — set DISPLAY or run inside the desktop session");
102 e.code = "no_session";
103 throw e;
104 }
105 for (const t of ["xdotool", "wmctrl", "scrot", "import", "grim", "slurp", "wtype", "ydotool", "wf-recorder", "ffmpeg", "xclip", "xsel", "wl-copy", "wl-paste", "python3", "xrandr", "swaymsg", "hyprctl"]) {
106 tools[t] = await have(t);
107 }
108 tools.pyatspi = tools.python3 && (await run("python3", ["-c", "import pyatspi"], { timeoutMs: 10_000 })).code === 0;
109 throwIfAborted();
110 probed = true;
111 return session;
112 }
113
114 function need(tool, purpose) {
115 if (!tools[tool]) throw new ExecError(`linux backend needs "${tool}" for ${purpose} — install it and retry`);
116 }
117
118 async function shotTool() {
119 if (session === "wayland") { need("grim", "screenshots on Wayland"); return { cmd: "grim", base: [] }; }
120 if (session === "x11") {
121 if (tools.scrot) return { cmd: "scrot", base: ["-z"] };
122 need("import", "screenshots on X11 (imagemagick)");
123 return { cmd: "import", base: ["-window", "root"] };
124 }
125 throw new ExecError("no X11 ($DISPLAY) or Wayland ($WAYLAND_DISPLAY) session visible to this process");
126 }
127
128 /** Capture a PNG to `file`, optionally cropped to region [x,y,w,h] points. */
129 async function takeShot(file, region) {
130 outputPath(file);
131 const { cmd, base } = await shotTool();
132 let args = [...base];
133 if (cmd === "grim") {
134 if (region) args.push("-g", `${Math.round(region[0])},${Math.round(region[1])} ${Math.round(region[2])}x${Math.round(region[3])}`);
135 args.push(file);
136 } else if (cmd === "scrot") {
137 if (region) args.push("-a", `${Math.round(region[0])},${Math.round(region[1])},${Math.round(region[2])},${Math.round(region[3])}`);
138 args.push(file);
139 } else {
140 if (region) args.push("-crop", `${Math.round(region[2])}x${Math.round(region[3])}+${Math.round(region[0])}+${Math.round(region[1])}`);
141 args.push(file);
142 }
143 const r = await run(cmd, args, { timeoutMs: 10_000 });
144 if (r.code !== 0) throw new ExecError(`${cmd} exited ${r.code}: ${r.stderr.trim().slice(0, 300)}`, r);
145 }
146
147 async function xdotool(args, opts = {}) {
148 need("xdotool", "input on X11");
149 throwIfAborted();
150 const r = await run("xdotool", args, opts);
151 throwIfAborted();
152 if (r.code !== 0) throw new ExecError(`xdotool ${args[0]} exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
153 return r.stdout.trim();
154 }
155
156 async function ydotool(args, opts = {}) {
157 need("ydotool", "input on Wayland (ydotool needs its daemon running: sudo ydotoold)");
158 throwIfAborted();
159 const r = await run("ydotool", args, opts);
160 throwIfAborted();
161 if (r.code !== 0) throw new ExecError(`ydotool exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
162 return r.stdout.trim();
163 }
164
165 function xdotoolKey(text) {
166 return String(text).split("+").map((p) => {
167 const k = p.trim().toLowerCase();
168 if (XKEYS[k]) return XKEYS[k];
169 if (/^f\d{1,2}$/.test(k)) return k.toUpperCase();
170 return p.trim(); // pass through names already in xdotool form
171 }).join("+");
172 }
173
174 async function waylandKey(text, { repeat = 1, holdMs = 0 } = {}) {
175 need("wtype", "key presses on Wayland");
176 const parts = String(text).split("+").map((part) => part.trim().toLowerCase());
177 const aliases = { control: "ctrl", meta: "logo", cmd: "logo", super: "logo" };
178 const modifiers = new Set(["ctrl", "alt", "shift", "logo", "win", "altgr", "capslock"]);
179 const rawKey = parts.pop();
180 const key = xdotoolKey(rawKey);
181 const mods = parts.map((part) => aliases[part] ?? part);
182 if (!key || mods.some((mod) => !modifiers.has(mod))) throw new ExecError(`unknown key combination "${text}"`);
183 const modKey = aliases[rawKey] ?? rawKey;
184 const onlyModifier = modifiers.has(modKey);
185 const args = mods.flatMap((mod) => ["-M", mod]);
186 for (let i = 0; i < repeat; i++) {
187 args.push(onlyModifier ? "-M" : "-P", onlyModifier ? modKey : key);
188 if (holdMs) args.push("-s", String(holdMs));
189 args.push(onlyModifier ? "-m" : "-p", onlyModifier ? modKey : key);
190 }
191 args.push(...mods.reverse().flatMap((mod) => ["-m", mod]));
192 // wtype owns a temporary Wayland keyboard; the compositor releases its
193 // keys on process exit, including cancellation. Keep the complete gesture
194 // in one process (https://github.com/atx/wtype#usage).
195 throwIfAborted();
196 const result = await run("wtype", args, { timeoutMs: Math.max(10_000, holdMs + 8_000) });
197 throwIfAborted();
198 if (result.code !== 0) throw new ExecError(`wtype exited ${result.code}: ${result.stderr.trim().slice(0, 200)}`, result);
199 }
200
201 function assertNum(v, name) {
202 const n = Number(v);
203 if (!Number.isFinite(n)) throw new ExecError(`${name} must be a finite number`);
204 return n;
205 }
206
207 // ---------- AT-SPI tree ----------
208 const PYATSPI_APP = `def resolve_app(desktop, app_name):
209 apps = [desktop.getChildAtIndex(i) for i in range(desktop.childCount)]
210 if app_name:
211 matches = [app for app in apps if app and app_name.casefold() == (app.name or "").casefold()]
212 return matches[0] if len(matches) == 1 else None
213 return next((app for app in apps if app and app.childCount), None)
214 `;
215 const PYATSPI_WALK = `import json, sys, pyatspi
216 ${PYATSPI_APP}
217 app_name = sys.argv[1] if len(sys.argv) > 1 else None
218 depth_max = int(sys.argv[2]) if len(sys.argv) > 2 else 8
219 max_el = int(sys.argv[3]) if len(sys.argv) > 3 else 400
220 desktop = pyatspi.Registry.getDesktop(0)
221 root = resolve_app(desktop, app_name)
222 if root is None:
223 print(json.dumps({"found": False}))
224 sys.exit(0)
225 els = []
226 truncated = False
227 def info(e, path):
228 ext = None
229 try: ext = e.queryComponent().getExtents(pyatspi.DESKTOP_COORDS)
230 except Exception: pass
231 txt = None
232 try:
233 q = e.queryText()
234 n = min(120, q.characterCount)
235 if n > 0: txt = q.getText(0, n)
236 except Exception: pass
237 acts = []
238 try:
239 a = e.queryAction()
240 acts = [a.getName(i) for i in range(a.nActions)]
241 except Exception: pass
242 els.append({"index": len(els), "path": path, "role": e.getRoleName() if e.getRoleName() else None,
243 "label": e.name or None, "value": txt,
244 "position": {"x": ext.x, "y": ext.y} if ext else None,
245 "size": {"w": ext.width, "h": ext.height} if ext else None,
246 "actions": acts})
247 def walk(e, path, d):
248 global truncated
249 if len(els) >= max_el or d > depth_max:
250 truncated = True
251 return
252 try: info(e, path)
253 except Exception: return
254 for i in range(e.childCount):
255 try: c = e.getChildAtIndex(i)
256 except Exception: continue
257 if c: walk(c, path + [i], d + 1)
258 walk(root, [], 0)
259 print(json.dumps({"found": True, "name": root.name, "elements": els, "truncated": truncated}))`;
260
261 async function atspiResolve(target, pythonBody, extraArg = null) {
262 const name = appName(target.app_ref);
263 assertAppRootWindow(target.windowIndex, target.window_id);
264 await probeSession();
265 need("python3", "semantic element actions (AT-SPI)");
266 const script = `import json, sys, pyatspi
267 ${PYATSPI_APP}
268 desktop = pyatspi.Registry.getDesktop(0)
269 app_name = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] else None
270 target_path = json.loads(sys.argv[2])
271 extra = sys.argv[3] if len(sys.argv) > 3 else None
272 node = resolve_app(desktop, app_name)
273 if node is None:
274 print(json.dumps({"ok": False, "code": "app_not_found"}))
275 sys.exit(0)
276 found = None
277 stack = [(node, [])]
278 while stack:
279 n, p = stack.pop(0)
280 if p == target_path:
281 found = n
282 break
283 if len(p) > 12: continue
284 try:
285 for i in range(n.childCount):
286 c = n.getChildAtIndex(i)
287 if c: stack.append((c, p + [i]))
288 except Exception: pass
289 if found is None:
290 print(json.dumps({"ok": False, "code": "element_stale"}))
291 sys.exit(0)
292 try:
293 ${pythonBody}
294 except Exception as e:
295 print(json.dumps({"ok": False, "code": str(e)}))`;
296 const argv = ["-c", script, name, JSON.stringify(target.path ?? [])];
297 if (extraArg != null) argv.push(String(extraArg));
298 const r = await run("python3", argv, { timeoutMs: 30_000 });
299 const out = tryJson((r.stdout.trim().split("\n").pop() ?? ""), null);
300 if (!out) throw new ExecError(`AT-SPI action failed: ${(r.stderr || r.stdout).slice(0, 250)}`, r);
301 return out;
302 }
303
304 // ---------- input helpers ----------
305 function clickButton(button, clicks) {
306 if (session === "x11") {
307 const args = ["click"];
308 if (clicks > 1) args.push("--repeat", String(clicks), "--delay", "80");
309 args.push(String(button));
310 return xdotool(args);
311 }
312 // ydotool click mask: down|up|count nibble (0xC0 = left click once, +1 per extra click;
313 // 0x04 bit selects right button, 0x02 middle).
314 const count = Math.max(1, Math.min(3, clicks));
315 const code = button === 3 ? 0xc0 + count + 0x04 : button === 2 ? 0xc0 + count + 0x02 : 0xc0 + count - 1;
316 return ydotool(["click", "0x" + code.toString(16)]);
317 }
318
319 return {
320 platform: "linux",
321 releaseInput,
322 browser_start: browser.start,
323 browser_status: browser.status,
324 browser_navigate: browser.navigate,
325 browser_click: browser.click,
326 browser_type: browser.type,
327 browser_screenshot: browser.screenshot,
328 browser_stop: browser.stop,
329 closeSession: async () => { await browser.close().catch(() => {}); },
330 probe: async () => {
331 const s = await probeSession();
332 const caps = {
333 screenshot: !!((session === "wayland" && tools.grim) || (session === "x11" && (tools.scrot || tools.import))),
334 clipboard: !!(tools.xclip || tools.xsel || (tools["wl-copy"] && tools["wl-paste"])),
335 recording: false,
336 accessibility_tree: tools.pyatspi,
337 held_input: exec?.persistentInputOwner === true && !!(session === "x11" ? tools.xdotool : tools.wtype && tools.ydotool),
338 };
339 const missing = [];
340 if (exec?.persistentInputOwner !== true) missing.push("connected Computer Use desktop helper (held keys, held buttons and drag)");
341 if (session === "x11" && !tools.xdotool) missing.push("xdotool (input)");
342 if (session === "wayland" && !tools.ydotool) missing.push("ydotool+ydotoold (mouse input)");
343 if (session === "wayland" && !tools.grim) missing.push("grim (screenshots)");
344 if (session === "x11" && !tools.scrot && !tools.import) missing.push("scrot or imagemagick (screenshots)");
345 missing.push("session-owned recording (unavailable in this version; use screenshots)");
346 if (!tools.pyatspi) missing.push("python3-pyatspi (accessibility tree)");
347 // Real permission probes, not just `have()`: each check is bounded to 10s.
348 const permissions = { input: "failed", screen_capture: "failed", accessibility: "unavailable" };
349 if (session === "x11" && tools.xdotool) {
350 const r = await run("xdotool", ["getdisplaygeometry"], { timeoutMs: 10_000 });
351 permissions.input = r.code === 0 ? "ok" : "failed";
352 } else if (session === "wayland" && tools.ydotool) {
353 permissions.input = "unavailable"; // ydotool can't be probed without moving the pointer
354 }
355 try {
356 const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-probe-"));
357 try {
358 await takeShot(path.join(dir, "probe.png"), [0, 0, 2, 2]);
359 permissions.screen_capture = "ok";
360 } finally {
361 try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
362 }
363 } catch {}
364 if (tools.pyatspi) {
365 const r = await run("python3", ["-c", "import pyatspi; pyatspi.Registry.getDesktop(0).childCount"], { timeoutMs: 10_000 });
366 permissions.accessibility = r.code === 0 ? "ok" : "failed";
367 }
368 if (permissions.screen_capture === "failed" || permissions.input === "failed") {
369 const bad = [];
370 if (permissions.input === "failed") bad.push(`input (${session === "wayland" ? "ydotool" : "xdotool getdisplaygeometry"})`);
371 if (permissions.screen_capture === "failed") bad.push(`screen_capture (${session === "wayland" ? "grim" : "scrot/import"} probe shot)`);
372 const e = new ExecError(`permission checks failed: ${bad.join("; ")} — permissions ${JSON.stringify(permissions)}`);
373 e.code = "permissions_denied";
374 throw e;
375 }
376 return { platform: "linux", session: s, capabilities: caps, permissions, missing, note: "Every capability probes at call time and fails closed naming the missing tool." };
377 },
378 list_displays: async () => {
379 await probeSession();
380 if (session === "x11" && tools.xrandr) {
381 const r = await runOk("xrandr", ["--query"], { timeoutMs: 15_000 });
382 const displays = [];
383 let i = 1;
384 for (const m of r.stdout.matchAll(/^(\S+) connected (?:primary )?(\d+)x(\d+)\+(\d+)\+(\d+)/gm)) {
385 displays.push({ index: i++, name: m[1], points: { x: Number(m[4]), y: Number(m[5]), w: Number(m[2]), h: Number(m[3]) }, pixels: { w: Number(m[2]), h: Number(m[3]) }, scale: 1, main: /primary/.test(m[0]) || i === 1 });
386 }
387 if (displays.length) return displays;
388 }
389 if (session === "wayland" && tools.swaymsg) {
390 const r = await run("swaymsg", ["-t", "get_outputs", "-r"], { timeoutMs: 15_000 });
391 const outs = tryJson(r.stdout, []);
392 if (Array.isArray(outs) && outs.length) {
393 return outs.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.rect?.x, y: o.rect?.y, w: o.rect?.width, h: o.rect?.height }, pixels: { w: o.current_mode?.width, h: o.current_mode?.height }, scale: o.scale ?? 1, main: i === 0 }));
394 }
395 }
396 if (session === "wayland" && tools.hyprctl) {
397 const r = await run("hyprctl", ["-j", "monitors"], { timeoutMs: 15_000 });
398 const ms = tryJson(r.stdout, []);
399 if (Array.isArray(ms) && ms.length) {
400 return ms.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.x, y: o.y, w: o.width, h: o.height }, pixels: { w: o.width, h: o.height }, scale: o.scale ?? 1, main: !!o.main || i === 0 }));
401 }
402 }
403 throw new ExecError("display enumeration needs xrandr (X11) or swaymsg/hyprctl (Wayland) — install one and retry");
404 },
405 switch_display: async ({ index }) => ({ activeDisplay: index ?? 1, note: "linux screenshots grab the compositor's virtual screen; per-display selection applies only where the shot tool supports it" }),
406 list_apps: async () => {
407 await probeSession();
408 if (session === "x11" && tools.wmctrl) {
409 const r = await runOk("wmctrl", ["-lx"], { timeoutMs: 15_000 });
410 const seen = new Map();
411 for (const line of r.stdout.split("\n")) {
412 const parts = line.split(/\s+/);
413 const wmClass = parts[2];
414 if (wmClass) seen.set(wmClass, { name: wmClass.split(".")[0], wm_class: wmClass });
415 }
416 return { apps: [...seen.values()] };
417 }
418 if (session === "wayland" && (tools.swaymsg || tools.hyprctl)) {
419 const w = await this.list_windows();
420 const seen = new Map();
421 for (const win of w.windows) {
422 const cls = win.wm_class || win.app_id;
423 if (cls) seen.set(cls, { name: cls, wm_class: cls });
424 }
425 return { apps: [...seen.values()] };
426 }
427 throw new ExecError("list_apps needs wmctrl (X11) or swaymsg/hyprctl (Wayland)");
428 },
429 list_windows: async (args = {}) => {
430 rejectWindowSelectors(args);
431 await probeSession();
432 if (session === "x11" && tools.wmctrl) {
433 const r = await runOk("wmctrl", ["-lGx"], { timeoutMs: 15_000 });
434 const windows = [];
435 for (const line of r.stdout.split("\n")) {
436 const m = /^(\S+)\s+(-?\d+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/.exec(line);
437 if (m) windows.push({ id: m[1], desktop: m[2], position: { x: Number(m[3]), y: Number(m[4]) }, size: { w: Number(m[5]), h: Number(m[6]) }, wm_class: m[7], title: m[8] });
438 }
439 return { windows };
440 }
441 if (session === "wayland" && tools.swaymsg) {
442 const r = await run("swaymsg", ["-t", "get_tree", "-r"], { timeoutMs: 15_000 });
443 const windows = [];
444 const walk = (n) => {
445 if (n.type === "con" && n.name) windows.push({ id: String(n.id), title: n.name, wm_class: n.app_id ?? null, position: { x: n.rect?.x, y: n.rect?.y }, size: { w: n.rect?.width, h: n.rect?.height }, focused: !!n.focused });
446 (n.nodes ?? []).forEach(walk);
447 (n.floating_nodes ?? []).forEach(walk);
448 };
449 walk(tryJson(r.stdout, {}));
450 return { windows };
451 }
452 if (session === "wayland" && tools.hyprctl) {
453 const r = await run("hyprctl", ["-j", "clients"], { timeoutMs: 15_000 });
454 const clients = tryJson(r.stdout, []);
455 return { windows: clients.map((c) => ({ id: String(c.address), title: c.title, wm_class: c.class, position: { x: c.at?.[0], y: c.at?.[1] }, size: { w: c.size?.[0], h: c.size?.[1] }, focused: !!c.focused })) };
456 }
457 throw new ExecError("list_windows needs wmctrl (X11), swaymsg (sway) or hyprctl (hyprland)");
458 },
459 open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => {
460 const target = name ?? bid;
461 if (!target || !/^[A-Za-z0-9][A-Za-z0-9 ._-]*$/.test(target)) throw new ExecError("open_application needs a plain executable/desktop name");
462 // activate defaults to background: on X11 a new window grabs focus, so
463 // remember the active window and hand focus back after the launch.
464 let prevWindow = null;
465 if (activate !== true) {
466 try {
467 await probeSession();
468 if (session === "x11" && tools.xdotool) {
469 const active = await run("xdotool", ["getactivewindow"], { timeoutMs: 3_000 });
470 if (active.code === 0 && /^\d+$/.test(active.stdout.trim())) prevWindow = active.stdout.trim();
471 }
472 } catch { /* no session/tools — the launch itself is still fine */ }
473 }
474 spawnDetached(target, urlArg ? [urlArg] : [], "", true);
475 await new Promise((r) => setTimeout(r, 500));
476 let focusRestored = false;
477 if (prevWindow) {
478 try {
479 focusRestored = (await run("xdotool", ["windowactivate", prevWindow], { timeoutMs: 3_000 })).code === 0;
480 } catch { /* best-effort */ }
481 }
482 return { launched: true, name: target, url: urlArg ?? null, activate: activate === true, ...(activate === true ? {} : { focus_restored: focusRestored }) };
483 },
484 get_app_state: async ({ app_ref, window_id } = {}) => {
485 const name = appName(app_ref);
486 if (window_id !== undefined) throw Object.assign(new ExecError("Linux app-state window_id selection is unavailable"), { code: "unsupported_selector" });
487 const t = await run("python3", ["-c", PYATSPI_WALK, name, "10", "500"], { timeoutMs: 45_000 }).then((r) =>
488 tryJson((r.stdout.trim().split("\n").pop() ?? ""), null));
489 if (!t) throw new ExecError("AT-SPI walk failed — is python3-pyatspi installed and the desktop running an accessibility bus (AT_SPI_BUS)?");
490 if (!t.found) throw new ExecError("application not found or name is ambiguous in the AT-SPI tree — use a unique exact app_ref.name");
491 return t;
492 },
493 screenshot: async (args = {}) => {
494 rejectWindowSelectors(args);
495 const { display, region } = args;
496 const outPath = recordingsOutputPath(args.path);
497 await probeSession();
498 const dir = recordingsDir();
499 fs.mkdirSync(dir, { recursive: true });
500 const file = outPath ?? path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
501 await takeShot(file, region);
502 const dims = pngSize(file);
503 lastRaster = {
504 file,
505 bytes: fs.statSync(file).size,
506 // Region rasters describe the region; full shots get geometry from the
507 // PNG itself (Linux shots are always scale 1: points == pixels).
508 points: region ? { x: region[0], y: region[1], w: region[2], h: region[3] } : dims ? { x: 0, y: 0, w: dims.w, h: dims.h } : null,
509 pixels: dims ?? (region ? { w: Math.round(region[2]), h: Math.round(region[3]) } : null),
510 scale: 1,
511 capturedAt: new Date().toISOString(),
512 };
513 return { ...lastRaster };
514 },
515 resolve_element: async ({ app_ref, windowIndex, window_id, path: pathArr } = {}) => {
516 const name = appName(app_ref);
517 assertAppRootWindow(windowIndex, window_id);
518 await probeSession();
519 need("python3", "element resolution (AT-SPI)");
520 const script = `import json, sys, pyatspi
521 ${PYATSPI_APP}
522 desktop = pyatspi.Registry.getDesktop(0)
523 app_name = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] else None
524 target_path = json.loads(sys.argv[2])
525 root = resolve_app(desktop, app_name)
526 if root is None:
527 print(json.dumps({"found": False, "element": None, "reason": "app_not_found"}))
528 sys.exit(0)
529 node = root
530 ok = True
531 for k in target_path:
532 found = None
533 try:
534 if k < node.childCount:
535 found = node.getChildAtIndex(k)
536 except Exception:
537 found = None
538 if found is None:
539 ok = False
540 break
541 node = found
542 if not ok:
543 print(json.dumps({"found": True, "element": None, "reason": "element_stale"}))
544 sys.exit(0)
545 ext = None
546 try: ext = node.queryComponent().getExtents(pyatspi.DESKTOP_COORDS)
547 except Exception: pass
548 print(json.dumps({"found": True, "reason": None, "element": {
549 "role": node.getRoleName() or None, "label": node.name or None,
550 "position": {"x": ext.x, "y": ext.y} if ext else None,
551 "size": {"w": ext.width, "h": ext.height} if ext else None}}))`;
552 const r = await run("python3", ["-c", script, name, JSON.stringify(pathArr ?? [])], { timeoutMs: 30_000 });
553 const out = tryJson(r.stdout.trim().split("\n").pop() ?? "", null);
554 if (!out) throw new ExecError(`AT-SPI resolve failed: ${(r.stderr || r.stdout).slice(0, 250)}`, r);
555 return out;
556 },
557 // Always crops the last raster this backend captured; a caller-named
558 // source file is not accepted.
559 zoom: async ({ region, path: outPath }) => {
560 // Validate the caller's output path before anything else runs.
561 const explicitOut = recordingsOutputPath(outPath);
562 need("ffmpeg", "zoom/crop");
563 const src = lastRaster?.file;
564 if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
565 const out = outputPath(explicitOut ?? path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`));
566 await runOk("ffmpeg", ["-y", "-loglevel", "error", "-i", src, "-vf", `crop=${Math.round(region[2])}:${Math.round(region[3])}:${Math.round(region[0])}:${Math.round(region[1])}`, out], { timeoutMs: 20_000 });
567 return { file: out, bytes: fs.statSync(out).size, region, source: src };
568 },
569 left_click: ({ target, strategy }) => { assertNum(target.x, "x"); assertNum(target.y, "y"); assertEventStrategy(strategy); return inputChain(target.x, target.y, () => clickButton(1, 1)); },
570 double_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 2)),
571 triple_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 3)),
572 right_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(3, 1)),
573 middle_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(2, 1)),
574 mouse_move: ({ target }) => inputMove(target.x, target.y),
575 left_click_drag: async ({ from_target: from, to }) => {
576 requireInputOwner();
577 await inputMove(from.x, from.y);
578 throwIfAborted();
579 mouseHeld = true;
580 try {
581 if (session === "x11") await xdotool(["mousedown", "1"]);
582 else await ydotool(["click", "0x40"]);
583 for (let i = 1; i <= 10; i++) {
584 await wait(20);
585 await inputMove(from.x + ((to.x - from.x) * i) / 10, from.y + ((to.y - from.y) * i) / 10);
586 }
587 } finally { await releaseMouse(); }
588 return { action_sent: true, from, to };
589 },
590 left_mouse_down: async ({ target } = {}) => {
591 requireInputOwner();
592 await probeSession();
593 if (target) await inputMove(target.x, target.y);
594 throwIfAborted();
595 mouseHeld = true;
596 try {
597 if (session === "x11") await xdotool(["mousedown", "1"]);
598 else await ydotool(["click", "0x40"]);
599 } catch (err) { await releaseMouse(); throw err; }
600 return { action_sent: true };
601 },
602 left_mouse_up: async () => {
603 if (!mouseHeld) throw Object.assign(new ExecError("no agent pointer press to release"), { code: "input_not_held" });
604 await releaseMouse();
605 return { action_sent: true };
606 },
607 scroll: async ({ target, direction = "down", amount = 3 }) => {
608 await inputMove(target.x, target.y);
609 if (session === "x11") {
610 const buttons = { down: 5, up: 4, right: 7, left: 6 };
611 await xdotool(["click", "--repeat", String(Math.max(1, Math.min(30, amount))), "--delay", "60", String(buttons[direction] ?? 5)]);
612 return { action_sent: true, direction, amount };
613 }
614 // Wayland: synthesize wheel via ydotool is not wired in this build — honest refusal.
615 throw new ExecError('scroll on Wayland is not available in this build; use swipe-style drags or run an X11/XWayland window. (Roadmap: ydotool wheel events.)');
616 },
617 type: async ({ text }) => {
618 if (!text) return { action_sent: false, note: "empty text" };
619 await probeSession();
620 if (session === "x11") {
621 // xdotool `type` remaps a spare keycode for characters absent from the
622 // current keymap. Two failure modes follow: a cased letter produces a
623 // single-symbol key whose XKB level 0 is the lowercase form (Ü → ü),
624 // and consecutive remaps inside one `type` call race the X server's
625 // keymap-change propagation, so non-ASCII chars intermittently drop or
626 // arrive mangled (héllo → hllo, 日本 → 本). Route every non-ASCII char
627 // through `key U<hex>` — one synchronous remap+press+restore per char —
628 // adding Shift only when the char is cased-uppercase, and batch ASCII
629 // runs through `type` as before.
630 let runText = "";
631 const chunks = [];
632 for (const ch of String(text)) {
633 if (ch.codePointAt(0) > 127) {
634 if (runText) { chunks.push(runText); runText = ""; }
635 chunks.push(ch);
636 } else runText += ch;
637 }
638 if (runText) chunks.push(runText);
639 for (const chunk of chunks) {
640 // Supplementary-plane chars are one code point but length 2; test
641 // the code point, not the string length.
642 if (chunk.codePointAt(0) > 127) {
643 const hex = chunk.codePointAt(0).toString(16).toUpperCase().padStart(4, "0");
644 const shift = chunk !== chunk.toLowerCase() ? "shift+" : "";
645 await xdotool(["key", `${shift}U${hex}`]);
646 // Each temp remap restores the keymap as soon as the event is
647 // queued; a lagging app can then read the press against the
648 // restored map and drop it. A short settle narrows that window.
649 // Under heavy host saturation XTEST drops remain possible — that
650 // residual is documented in the suite's known_limitations.
651 await new Promise((r) => setTimeout(r, 30));
652 } else {
653 await xdotool(["type", "--delay", "12", "--", chunk]);
654 }
655 }
656 return { action_sent: true, chars: text.length };
657 }
658 need("wtype", "typing on Wayland");
659 const r = await run("wtype", ["--", String(text)], { timeoutMs: 15_000 });
660 if (r.code !== 0) throw new ExecError(`wtype failed: ${r.stderr.slice(0, 200)}`, r);
661 return { action_sent: true, chars: text.length };
662 },
663 key: async ({ text, repeat = 1 }) => {
664 await probeSession();
665 const k = xdotoolKey(text);
666 const n = Math.max(1, Math.min(100, Number(repeat) || 1));
667 if (session === "x11") {
668 throwIfAborted();
669 heldKeys.add(k);
670 try {
671 await xdotool(["key", "--repeat", String(n), "--delay", "60", k]);
672 heldKeys.delete(k);
673 } finally { await releaseKey(k); }
674 } else await waylandKey(text, { repeat: n });
675 return { action_sent: true, key: k };
676 },
677 hold_key: async ({ text, duration }) => {
678 requireInputOwner();
679 await probeSession();
680 const k = xdotoolKey(text);
681 const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
682 if (session === "x11") {
683 throwIfAborted();
684 heldKeys.add(k);
685 try {
686 await xdotool(["keydown", k]);
687 await wait(d * 1000);
688 } finally { await releaseKey(k); }
689 } else await waylandKey(text, { holdMs: Math.round(d * 1000) });
690 return { action_sent: true, key: k, heldSec: d };
691 },
692 set_value: async ({ target, value }) => {
693 // Select a supported interface before sending input. A refused write or
694 // failed readback must never trigger a second, ambiguously applied edit.
695 const out = await atspiResolve(target, ` state = found.getState()
696 if not state.contains(pyatspi.STATE_ENABLED):
697 raise RuntimeError("element_disabled")
698 try:
699 editor = found.queryEditableText()
700 except NotImplementedError:
701 editor = None
702 if editor is not None:
703 if not state.contains(pyatspi.STATE_EDITABLE):
704 raise RuntimeError("element_read_only")
705 if not editor.setTextContents(extra):
706 raise RuntimeError("value_rejected")
707 text = found.queryText()
708 after = text.getText(0, text.characterCount)
709 if after != extra:
710 raise RuntimeError("value_verification_failed")
711 else:
712 import math
713 desired = float(extra)
714 if not math.isfinite(desired):
715 raise RuntimeError("invalid_value")
716 numeric = found.queryValue()
717 numeric.currentValue = desired
718 after = numeric.currentValue
719 if after != desired:
720 raise RuntimeError("value_verification_failed")
721 print(json.dumps({"ok": True, "after": after}))`, String(value));
722 if (!out.ok) throw new ExecError(`set_value failed: ${out.code}`);
723 return { action_sent: true, strategy: "a11y", verified: true, after: out.after };
724 },
725 select_text: async () => { throw new ExecError("select_text is not implemented on the linux backend — fail-closed"); },
726 perform_action: async ({ target, action }) => {
727 const body = ` a = found.queryAction()
728 names = [a.getName(i) for i in range(a.nActions)]
729 want = (extra or "click").lower()
730 match = next((n for n in names if n.lower() == want), None)
731 if match is None and want == "click":
732 match = next((n for n in names if n.lower() in ("click", "press", "activate")), None)
733 if match is None:
734 print(json.dumps({"ok": False, "code": "action_not_found: " + ",".join(names)}))
735 else:
736 a.doAction(names.index(match))
737 print(json.dumps({"ok": True, "sent": True}))`;
738 const out = await atspiResolve(target, body, String(action));
739 if (!out.ok) throw new ExecError(`perform_action failed: ${out.code}`);
740 return { action_sent: true, strategy: "a11y", action };
741 },
742 read_clipboard: async () => {
743 await probeSession();
744 const cmd = session === "x11"
745 ? (tools.xclip ? ["xclip", "-selection", "clipboard", "-o"] : ["xsel", "--clipboard", "--output"])
746 : ["wl-paste"];
747 need(cmd[0], "clipboard read");
748 const r = await run(cmd[0], cmd.slice(1), { timeoutMs: 10_000 });
749 if (r.code !== 0) throw new ExecError("clipboard read failed", r);
750 return { text: r.stdout, encoding: "utf8" };
751 },
752 write_clipboard: async ({ text }) => {
753 await probeSession();
754 const cmd = session === "x11"
755 ? (tools.xclip ? ["xclip", "-selection", "clipboard"] : ["xsel", "--clipboard", "--input"])
756 : ["wl-copy"];
757 need(cmd[0], "clipboard write");
758 spawnDetached(cmd[0], cmd.slice(1), String(text ?? ""), true);
759 return { written: String(text ?? "").length };
760 },
761 cursor_position: async () => {
762 await probeSession();
763 if (session === "x11") {
764 const out = await xdotool(["getmouselocation"]);
765 const m = /x:(-?\d+)\s+y:(-?\d+)/.exec(out);
766 if (!m) throw new ExecError(`could not parse xdotool getmouselocation output: ${out}`);
767 return { x: Number(m[1]), y: Number(m[2]) };
768 }
769 throw new ExecError("cursor position needs an X11 session in this build");
770 },
771 recordingStart: async () => {
772 throw Object.assign(new ExecError("Recording is unavailable on this platform until the recorder has session-owned cleanup. Use screenshots instead."), { code: "owned_recording_unavailable" });
773 },
774 recordingStop: async ({ id }) => { throw new ExecError(`unknown recording "${id}"`); },
775 recordingStatus: ({ id }) => ({ id, running: false }),
776 recordingList: async () => {
777 const dir = recordingsDir();
778 const out = fs.existsSync(dir)
779 ? fs.readdirSync(dir).filter((f) => /\.(mp4|mkv|png)$/i.test(f)).map((f) => {
780 const st = fs.statSync(path.join(dir, f));
781 return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
782 }).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
783 : [];
784 return { dir, recordings: out, running: [] };
785 },
786 };
787
788 async function inputChain(x, y, act) {
789 await inputMove(x, y);
790 await act();
791 return { action_sent: true, at: { x: Number(x), y: Number(y) } };
792 }
793
794 async function inputMove(x, y) {
795 await probeSession();
796 const nx = Math.round(assertNum(x, "x"));
797 const ny = Math.round(assertNum(y, "y"));
798 if (session === "x11") await xdotool(["mousemove", "--sync", String(nx), String(ny)]);
799 else await ydotool(["moveto", String(nx), String(ny)]);
800 }
801 }
802
803 export default { create };
804
804 lines Plain Text