返回 CodeWhale
win32.mjs
根目录 / crates / tui / plugins / computer-use / src / backends / win32.mjs
1 // Windows backend. Every operation is one PowerShell invocation whose script
2 // travels as a base64 -EncodedCommand, so tool arguments never become shell
3 // syntax. Screenshots + UIA accessibility come from .NET; raw pointer and
4 // keyboard events come from user32 P/Invoke (SendInput/mouse_event).
5 // Recording stays unavailable until its native process has session-owned cleanup.
6 import fs from "node:fs";
7 import path from "node:path";
8 import crypto from "node:crypto";
9 import { run, ExecError, tryJson, withSignal, throwIfAborted } from "../exec.mjs";
10 import { createBrowser } from "../browser-cdp.mjs";
11 import { recordingsDir, recordingsOutputPath } from "../recordings.mjs";
12
13 const USER32 = `
14 using System;
15 using System.Runtime.InteropServices;
16 public static class User32 {
17 [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
18 [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, UIntPtr dwExtraInfo);
19 [DllImport("user32.dll", SetLastError = true)] private static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize);
20 [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT lpPoint);
21 [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
22 [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
23 [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
24 [StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
25 [StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
26 [StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; }
27 [StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public INPUTUNION u; }
28 public const uint MOVED = 0x0001, LEFTDOWN = 0x0002, LEFTUP = 0x0004, RIGHTDOWN = 0x0008, RIGHTUP = 0x0010,
29 MIDDLEDOWN = 0x0020, MIDDLEUP = 0x0040, WHEEL = 0x0800, HWHEEL = 0x1000, ABSOLUTE = 0x8000, VIRTUALKEY = 0x4000, KEYUP = 0x0002, UNICODE = 0x0004;
30 public static INPUT MouseInput(uint flags, int x, int y, uint data) {
31 return new INPUT { type = 0, u = new INPUTUNION { mi = new MOUSEINPUT { dx = x, dy = y, mouseData = data, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } };
32 }
33 public static INPUT KeyInput(ushort vk, ushort scan, uint flags) {
34 return new INPUT { type = 1, u = new INPUTUNION { ki = new KEYBDINPUT { wVk = vk, wScan = scan, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } };
35 }
36 public sealed class InputDeliveryException : InvalidOperationException {
37 public readonly uint Inserted;
38 public InputDeliveryException(uint inserted, int expected, int error)
39 : base(String.Format("SendInput inserted {0} of {1} events (Win32 error {2})", inserted, expected, error)) { Inserted = inserted; }
40 }
41 public static void SendChecked(INPUT[] inputs) {
42 uint sent = SendInput((uint)inputs.Length, inputs, Marshal.SizeOf(typeof(INPUT)));
43 if (sent != (uint)inputs.Length) {
44 int error = Marshal.GetLastWin32Error();
45 throw new InputDeliveryException(sent, inputs.Length, error);
46 }
47 }
48 public static void SendKey(ushort vk, uint flags) {
49 SendChecked(new INPUT[] { KeyInput(vk, 0, flags) });
50 }
51 public static void SendString(string text) {
52 // UTF-16 code units, including both halves of surrogate pairs. UNICODE
53 // requires wVk=0 and may only be combined with KEYUP, never SCANCODE.
54 foreach (char c in text) {
55 INPUT up = KeyInput(0, c, UNICODE | KEYUP);
56 try { SendChecked(new INPUT[] { KeyInput(0, c, UNICODE), up }); }
57 catch (InputDeliveryException failure) {
58 // A partial pair may leave only our down event inserted. Release it
59 // once, never replay text, and still report the original failure.
60 if (failure.Inserted == 1) {
61 try { SendChecked(new INPUT[] { up }); }
62 catch (Exception cleanup) {
63 throw new AggregateException("Unicode input failed: " + failure.Message + "; key-up recovery failed: " + cleanup.Message, failure, cleanup);
64 }
65 }
66 throw;
67 }
68 }
69 }
70 }`;
71
72 // Windows virtual-key codes for named keys.
73 const VK = {
74 return: 0x0d, enter: 0x0d, tab: 0x09, escape: 0x1b, esc: 0x1b, space: 0x20,
75 backspace: 0x08, delete: 0x2e, home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22,
76 left: 0x25, up: 0x26, right: 0x27, down: 0x28, capslock: 0x14, insert: 0x2d,
77 f1: 0x70, f2: 0x71, f3: 0x72, f4: 0x73, f5: 0x74, f6: 0x75, f7: 0x76, f8: 0x77,
78 f9: 0x78, f10: 0x79, f11: 0x7a, f12: 0x7b,
79 };
80 const MODVK = { ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b };
81
82 // Every action runs in a fresh powershell.exe process, so a bootstrap process
83 // can never register the User32 type for later spawns. Each User32-backed
84 // invocation therefore carries its own type definition via this prelude.
85 // Compilation, conversion and native-call exceptions must stop before success.
86 const USER32_PRELUDE = `$ErrorActionPreference = 'Stop';\nAdd-Type -TypeDefinition @'\n${USER32}\n'@ -ErrorAction Stop;`;
87
88 // Initialize UIA proxies from a typed CLR frame. The legacy proxy loader
89 // walks ReflectedType on its call stack; PowerShell dynamic frames can be null
90 // and leave standard controls exposed as plain panes without action patterns.
91 const UIA_PRELUDE = `Add-Type -AssemblyName UIAutomationClient;
92 Add-Type -AssemblyName UIAutomationTypes;
93 Add-Type -ReferencedAssemblies ([System.Windows.Automation.AutomationElement].Assembly.Location) -TypeDefinition @'
94 using System.Windows.Automation;
95 public static class CUAutomationProviders {
96 public static void Register() {
97 var assembly = typeof(AutomationElement).Assembly.GetName();
98 assembly.Name = "UIAutomationClientsideProviders";
99 // Legacy .NET clears its one-time default-proxy flag before walking a
100 // dynamic PowerShell stack, which can throw NullReferenceException.
101 // Retry only registration (no UI action); a second failure propagates.
102 try { ClientSettings.RegisterClientSideProviderAssembly(assembly); }
103 catch (System.NullReferenceException) { ClientSettings.RegisterClientSideProviderAssembly(assembly); }
104 }
105 }
106 '@;
107 [CUAutomationProviders]::Register();`;
108
109 /** Coordinate clicks on this backend are always raw pointer events; strategy="a11y" must fail closed rather than silently degrade. */
110 function assertEventStrategy(strategy) {
111 if (strategy != null && strategy !== "auto" && strategy !== "event") {
112 throw new ExecError(`strategy "${strategy}" is macOS-only; this backend dispatches coordinate clicks as raw pointer events — use an element target for a semantic action`);
113 }
114 }
115
116 function unsupportedSelector(message) {
117 return Object.assign(new ExecError(message), { code: "unsupported_selector" });
118 }
119
120 // A cached child path alone is unsafe: sibling/window order can change.
121 // Runtime IDs bind both the root and leaf to the observation that supplied them.
122 function elementScript(target) {
123 const ids = [target?.runtime_id, target?.window_runtime_id];
124 if (ids.some(id => !Array.isArray(id) || !id.length || !id.every(Number.isInteger))
125 || !Array.isArray(target?.path) || target.path[0] !== 0 || !target.path.every(i => Number.isInteger(i) && i >= 0)
126 || (target.windowIndex != null && target.windowIndex !== 0) || Object.hasOwn(target, "window_id")) {
127 throw unsupportedSelector("Windows semantic actions require a fresh observed element with window and element runtime identities");
128 }
129 const encoded = Buffer.from(JSON.stringify(target), "utf16le").toString("base64");
130 return `${UIA_PRELUDE}
131 $target = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')) | ConvertFrom-Json;
132 $root = [System.Windows.Automation.AutomationElement]::RootElement;
133 $windows = @($root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition));
134 $matches = @($windows | Where-Object { ($_.GetRuntimeId() -join ',') -ceq ($target.window_runtime_id -join ',') });
135 if ($matches.Count -ne 1) { throw 'element_stale: observed window no longer exists' }
136 $cur = $matches[0];
137 if ($target.app_ref.name -and $cur.Current.Name -cne $target.app_ref.name) { throw 'element_stale: window identity changed' }
138 if ($target.app_ref.pid -and $cur.Current.ProcessId -ne $target.app_ref.pid) { throw 'element_stale: window process changed' }
139 for ($step = 1; $step -lt $target.path.Count; $step++) {
140 $kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
141 $i = $target.path[$step];
142 if ($i -ge $kids.Count) { throw 'element_stale: observed path changed' }
143 $cur = $kids[$i];
144 }
145 if (($cur.GetRuntimeId() -join ',') -cne ($target.runtime_id -join ',')) { throw 'element_stale: observed element was replaced' }
146 if ($target.role -and ($cur.Current.ControlType.ProgrammaticName -replace '^ControlType\\.', '') -cne $target.role) { throw 'element_stale: role changed' }
147 if ($null -ne $target.label -and $cur.Current.Name -cne $target.label) { throw 'element_stale: label changed' }
148 if (-not $cur.Current.IsEnabled) { throw 'element_disabled: observed element is disabled' }
149 `;
150 }
151
152 export function create(opts = {}) {
153 // Allow tests (and other embedders) to inject a runner so no real
154 // powershell.exe is spawned. Production uses the imported runner.
155 const injectedRun = opts.exec && typeof opts.exec.run === "function" ? opts.exec.run : null;
156 const runner = injectedRun ?? run;
157 const browser = createBrowser({ platform: "win32" });
158
159 function requireInputOwner() {
160 if (opts.exec?.persistentInputOwner !== true) throw Object.assign(new ExecError(
161 "This held-input gesture requires a connected Codewhale Computer Use desktop helper so a disconnected client cannot leave keys or buttons pressed. Start the helper and reconnect before retrying."
162 ), { code: "input_owner_required" });
163 }
164
165 async function ps(script, o = {}) {
166 throwIfAborted();
167 const encoded = Buffer.from(`$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';\n[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding($false);\n${script}`, "utf16le").toString("base64");
168 return runner("powershell.exe", ["-NoProfile", "-NonInteractive", "-EncodedCommand", encoded], {
169 timeoutMs: o.timeoutMs ?? 25_000,
170 maxBuffer: 32 * 1024 * 1024,
171 });
172 }
173
174 /** ps() but truthful: timeout, nonzero exit, and spawn failure all throw. */
175 async function psOk(script, o = {}) {
176 const r = await ps(script, o);
177 if (r.aborted) throw Object.assign(new ExecError("computer request cancelled", r), { code: "cancelled" });
178 if (r.timedOut) throw new ExecError(`powershell timed out after ${o.timeoutMs ?? 25_000}ms`, r);
179 if (r.code !== 0) {
180 const raw = (r.stderr || r.stdout).trim();
181 // EncodedCommand serializes errors as CLIXML; surface the error strings,
182 // not a truncated XML/progress header that conceals the actual failure.
183 const messages = [...raw.matchAll(/<S S="Error">([\s\S]*?)<\/S>/g)].map(m => m[1]
184 .replace(/_x([0-9A-Fa-f]{4})_/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
185 .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&"));
186 throw new ExecError(`powershell.exe exited ${r.code}: ${(messages.join("") || raw).slice(0, 1600)}`, r);
187 }
188 return r;
189 }
190
191 async function psJson(script, o = {}) {
192 const r = await psOk(script, o);
193 const out = r.stdout.trim();
194 const j = tryJson(out, null);
195 if (!j) throw new ExecError(`powershell did not return JSON: ${(r.stderr || out).trim().slice(0, 300)}`, r);
196 return j;
197 }
198
199 let lastRaster = null;
200 let activeDisplay = null;
201 const heldButtons = new Set();
202 const heldKeys = new Set();
203
204 async function releaseInput({ buttons = [...heldButtons], keys = [...heldKeys] } = {}) {
205 buttons = buttons.filter((button) => heldButtons.has(button));
206 keys = keys.filter((key) => heldKeys.has(key));
207 if (!buttons.length && !keys.length) return;
208 const releases = [
209 ...buttons.map((button) => `[User32]::mouse_event([User32]::${button}UP, 0, 0, 0, [UIntPtr]::Zero);`),
210 ...keys.reverse().map((vk) => `[User32]::SendKey(${vk}, 2);`),
211 ];
212 await withSignal(null, () => withUser32(releases.join("\n"), { timeoutMs: 2_000 }));
213 for (const button of buttons) heldButtons.delete(button);
214 for (const key of keys) heldKeys.delete(key);
215 }
216
217 async function withKeys(keys, action) {
218 throwIfAborted();
219 for (const code of keys) heldKeys.add(code);
220 try {
221 const result = await action();
222 for (const code of keys) heldKeys.delete(code);
223 return result;
224 } catch (failure) {
225 try { await releaseInput({ buttons: [], keys }); }
226 catch (cleanup) {
227 throw Object.assign(new ExecError(`Keyboard input failed: ${failure.message}; release failed: ${cleanup.message}`, failure.result), {
228 code: failure.code, cause: failure, cleanupError: cleanup,
229 });
230 }
231 throw failure;
232 }
233 }
234
235 function keyChord(text) {
236 const parts = String(text).split("+").map((part) => part.trim().toLowerCase());
237 const key = parts.pop();
238 const mods = parts.map((part) => MODVK[part]);
239 const vk = VK[key] ?? MODVK[key] ?? (/^[a-z0-9]$/.test(key) ? key.toUpperCase().charCodeAt(0) : null);
240 if (vk == null || mods.some((mod) => mod == null)) throw new ExecError(`unknown key combination "${text}"`);
241 return { key, vk, mods: [...new Set(mods)] };
242 }
243
244 /** Self-contained User32 invocation: prelude + script, fails truthfully. */
245 async function withUser32(script, opts) {
246 return psOk(`${USER32_PRELUDE}\n${script}`, opts);
247 }
248
249 return {
250 platform: "win32",
251 releaseInput,
252 browser_start: browser.start,
253 browser_status: browser.status,
254 browser_navigate: browser.navigate,
255 browser_click: browser.click,
256 browser_type: browser.type,
257 browser_screenshot: browser.screenshot,
258 browser_stop: browser.stop,
259 closeSession: async () => { await browser.close().catch(() => {}); },
260 probe: async () => {
261 const psOk = await ps("Write-Output 'ok'").then((r) => r.code === 0).catch(() => false);
262 return {
263 platform: "win32",
264 powershell: psOk,
265 capabilities: { screenshot: psOk, accessibility_tree: psOk, clipboard: psOk, recording: false, raw_input: psOk, held_input: psOk && opts.exec?.persistentInputOwner === true },
266 note: "Recording is unavailable until session-owned cleanup is implemented; use screenshots. UIA accessibility works without extra installs. Held keys, held buttons and drag require a connected Computer Use desktop helper.",
267 };
268 },
269 list_displays: async () => {
270 const d = await psJson(`Add-Type -AssemblyName System.Windows.Forms;
271 $arr = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object { [pscustomobject]@{ name = $_.DeviceName; primary = $_.Primary; x = $_.Bounds.X; y = $_.Bounds.Y; w = $_.Bounds.Width; h = $_.Bounds.Height } });
272 @{ displays = $arr } | ConvertTo-Json -Depth 4 -Compress;`, { timeoutMs: 15_000 });
273 return d.displays.map((x, i) => ({ index: i + 1, name: x.name, points: { x: x.x, y: x.y, w: x.w, h: x.h }, pixels: { w: x.w, h: x.h }, scale: 1, main: !!x.primary }));
274 },
275 async switch_display({ index = 1 }) {
276 const displays = await this.list_displays();
277 if (!displays.some(d => d.index === index)) throw new ExecError("display index is out of range");
278 activeDisplay = index;
279 return { activeDisplay };
280 },
281 list_apps: async () => {
282 const j = await psJson(`Add-Type -AssemblyName System.Windows.Forms;
283 $out = Get-Process | Where-Object { $_.MainWindowTitle } | ForEach-Object { [pscustomobject]@{ name = $_.ProcessName; pid2 = $_.Id; title = $_.MainWindowTitle } } | ConvertTo-Json -Compress;
284 if (-not $out) { $out = '[]' }
285 Write-Output ('{"apps": ' + $out + '}');`);
286 return { apps: (Array.isArray(j.apps) ? j.apps : [j.apps]).map((a) => ({ name: a.name, pid: a.pid2, title: a.title })) };
287 },
288 list_windows: async (args = {}) => {
289 if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows list_windows does not support app_ref or window_id; omit them to list all windows");
290 const j = await psJson(`$ErrorActionPreference = 'Stop';
291 Add-Type -AssemblyName System.Windows.Forms;
292 Add-Type -TypeDefinition @'
293 using System;
294 using System.Text;
295 using System.Collections.Generic;
296 using System.Runtime.InteropServices;
297 public static class WinEnum {
298 [DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
299 [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
300 [DllImport("user32.dll")] static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
301 [DllImport("user32.dll")] static extern int GetWindowTextLength(IntPtr hWnd);
302 [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
303 [DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
304 [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
305 delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
306 public static List<string> List() {
307 var result = new List<string>();
308 EnumWindows((h, l) => {
309 if (!IsWindowVisible(h)) return true;
310 int len = GetWindowTextLength(h);
311 var sb = new StringBuilder(len + 1);
312 GetWindowText(h, sb, sb.Capacity);
313 uint pid; GetWindowThreadProcessId(h, out pid);
314 RECT r; GetWindowRect(h, out r);
315 if (sb.Length > 0 && r.Right > r.Left)
316 result.Add(pid + "|" + r.Left + "," + r.Top + "," + (r.Right - r.Left) + "," + (r.Bottom - r.Top) + "|" + sb.ToString());
317 return true;
318 }, IntPtr.Zero);
319 return result;
320 }
321 }
322 '@;
323 $json = [WinEnum]::List() | ForEach-Object { $p = $_.Split('|', 2); $parts = $p[1].Split('|', 2); [pscustomobject]@{ pid2 = [int]$p[0]; geom = $parts[0]; title = $parts[1] } } | ConvertTo-Json -Compress;
324 if (-not $json) { $json = '[]' }
325 Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 });
326 return {
327 windows: (Array.isArray(j.windows) ? j.windows : [j.windows]).map((w) => {
328 const g = String(w.geom).split(",").map(Number);
329 return { pid: w.pid2, title: w.title, position: { x: g[0], y: g[1] }, size: { w: g[2], h: g[3] } };
330 }),
331 };
332 },
333 open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => {
334 const target = name ?? bid;
335 if (typeof target !== "string" || !/^[A-Za-z0-9][A-Za-z0-9 .:_-]*$/.test(target)) throw new ExecError("open_application needs a plain app or executable name");
336 let argumentsScript = "";
337 if (urlArg != null) {
338 if (typeof urlArg !== "string" || !URL.canParse(urlArg) || /[\0\r\n]/.test(urlArg)) throw new ExecError("open_application url must be an absolute URL");
339 // Start-Process joins ArgumentList into a Windows command line. Quote
340 // one argument there, and transport that string as data into PowerShell.
341 const quoted = '"' + urlArg.replace(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/g, '$1$1') + '"';
342 const encoded = Buffer.from(quoted, "utf16le").toString("base64");
343 argumentsScript = `$launchArg = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${encoded}')); `;
344 }
345 // activate defaults to background on every platform: a minimized
346 // launch leaves the user's foreground window alone. Windows input is
347 // still shared-surface — this only controls the launch, not input.
348 const windowStyle = activate === true ? "" : " -WindowStyle Minimized";
349 const r = await psOk(`${argumentsScript}Start-Process -FilePath "${target}"${windowStyle}${urlArg != null ? " -ArgumentList $launchArg" : ""}; Write-Output '{"launched": true}'`, { timeoutMs: 20_000 });
350 if (r.code !== 0) throw new ExecError(`Start-Process failed: ${r.stderr.trim().slice(0, 200)}`, r);
351 return { launched: true, name: target, url: urlArg ?? null, activate: activate === true };
352 },
353 get_app_state: async (args = {}) => {
354 if (Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows get_app_state does not support window_id");
355 const { app_ref, detail } = args;
356 if (Object.hasOwn(args, "app_ref") && (!app_ref || typeof app_ref !== "object" || Array.isArray(app_ref)
357 || Object.keys(app_ref).length !== 1 || !Object.hasOwn(app_ref, "name") || typeof app_ref.name !== "string" || !app_ref.name.trim())) {
358 throw unsupportedSelector("Windows get_app_state supports only app_ref: { name: exact window title }; PID, bundle_id and other references are unsupported");
359 }
360 const filter = Buffer.from(app_ref?.name ?? "", "utf16le").toString("base64");
361 const maxEls = detail === "full" ? 800 : 400;
362 const j = await psJson(`${UIA_PRELUDE}
363 $max = ${maxEls};
364 $filter = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${filter}'));
365 $root = [System.Windows.Automation.AutomationElement]::RootElement;
366 $els = New-Object System.Collections.ArrayList;
367 $found = $false; $truncated = $false; $appName = $null;
368 $targets = @($root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition));
369 if ($filter) {
370 $targets = @($targets | Where-Object { [string]::Equals($_.Current.Name, $filter, [StringComparison]::OrdinalIgnoreCase) });
371 if ($targets.Count -gt 1) { throw 'More than one application window has this exact name' }
372 }
373 foreach ($t in $targets) {
374 $nm = $t.Current.Name;
375 $found = $true; $appName = $nm;
376 $stack = New-Object System.Collections.Stack;
377 $windowId = @($t.GetRuntimeId());
378 $stack.Push(@($t, @(0)));
379 while ($stack.Count -gt 0) {
380 $entry = $stack.Pop(); $cur = $entry[0]; $path = $entry[1];
381 if ($els.Count -ge $max) { $truncated = $true; break }
382 $rect = $cur.Current.BoundingRectangle;
383 $acts = @();
384 try { $acts = @($cur.GetSupportedPatterns() | ForEach-Object { $_.ProgrammaticName -replace 'PatternIdentifiers\\.Pattern$','' -replace 'Pattern$','' }) } catch {}
385 $value = ''; $vp = $null;
386 if (-not $cur.Current.IsPassword -and $cur.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $value = [string]$vp.Current.Value }
387 [void]$els.Add([pscustomobject]@{ index = $els.Count; path = @($path); runtime_id = @($cur.GetRuntimeId()); window_runtime_id = $windowId; role = [string]$cur.Current.ControlType.ProgrammaticName; label = [string]$cur.Current.Name; value = $value.Substring(0, [Math]::Min(120, $value.Length)); enabled = $cur.Current.IsEnabled;
388 x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height; actions = $acts });
389 $kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
390 for ($i = $kids.Count - 1; $i -ge 0; $i--) { $stack.Push(@($kids[$i], ($path + $i))) }
391 }
392 break;
393 }
394 $result = [pscustomobject]@{ found = $found; name = $appName; truncated = $truncated; elements = @($els | ForEach-Object { [pscustomobject]@{ index = $_.index; path = @($_.path); runtime_id = $_.runtime_id; window_runtime_id = $_.window_runtime_id; role = ($_.role -replace 'ControlType.',''); label = $_.label; value = $_.value; enabled = $_.enabled; position = [pscustomobject]@{ x = $_.x; y = $_.y }; size = [pscustomobject]@{ w = $_.w; h = $_.h }; actions = $_.actions } }) };
395 Write-Output ($result | ConvertTo-Json -Depth 6 -Compress);`, { timeoutMs: 60_000 });
396 if (!j.found) throw new ExecError("application window not found in UIA tree — pass app_ref.name as the exact window title from list_windows or list_apps.title");
397 return j;
398 },
399 screenshot: async (args = {}) => {
400 if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw unsupportedSelector("Windows screenshot does not support app_ref or window_id; omit them for a desktop screenshot");
401 const { display = activeDisplay, region } = args;
402 const outPath = recordingsOutputPath(args.path);
403 if (display != null && (!Number.isInteger(display) || display < 1)) throw new ExecError("display index must be a positive integer");
404 if (region != null && (!Array.isArray(region) || region.length !== 4 || !region.every(Number.isInteger) || region[2] <= 0 || region[3] <= 0)) throw new ExecError("region must be integer [x,y,width,height] with positive size");
405 const dir = recordingsDir();
406 fs.mkdirSync(dir, { recursive: true });
407 const file = path.resolve(outPath ?? path.join(dir, `shot-${crypto.randomBytes(6).toString("hex")}.png`));
408 const meta = await psJson(`Add-Type -AssemblyName System.Windows.Forms; Add-Type -AssemblyName System.Drawing;
409 $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen;
410 ${display == null ? "" : `$screens = [System.Windows.Forms.Screen]::AllScreens; if (${display} -gt $screens.Count) { throw 'display index is out of range' }; $bounds = $screens[${display - 1}].Bounds;`}
411 ${region == null ? "" : `$crop = New-Object System.Drawing.Rectangle(${region.join(",")}); if (-not $bounds.Contains($crop)) { throw 'region is outside capture bounds' }; $bounds = $crop;`}
412 $bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height);
413 try {
414 $g = [System.Drawing.Graphics]::FromImage($bmp);
415 try { $g.CopyFromScreen($bounds.X, $bounds.Y, 0, 0, $bounds.Size); } finally { $g.Dispose(); }
416 $bmp.Save('${file.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png);
417 } finally { $bmp.Dispose(); }
418 @{ x = $bounds.X; y = $bounds.Y; w = $bounds.Width; h = $bounds.Height } | ConvertTo-Json -Compress;`, { timeoutMs: 30_000 });
419 if (!fs.existsSync(file) || ![meta.x, meta.y, meta.w, meta.h].every(Number.isFinite) || meta.w <= 0 || meta.h <= 0) throw new ExecError("screenshot did not return a valid raster");
420 lastRaster = { file, bytes: fs.statSync(file).size, points: { x: meta.x, y: meta.y, w: meta.w, h: meta.h }, pixels: { w: meta.w, h: meta.h }, scale: 1, capturedAt: new Date().toISOString() };
421 return { ...lastRaster };
422 },
423 // Always crops the last raster this backend captured; a caller-named
424 // source file is not accepted.
425 zoom: async ({ region, path: outPath }) => {
426 // Validate the caller's output path before anything else runs.
427 const explicitOut = recordingsOutputPath(outPath);
428 const src = lastRaster?.file;
429 if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
430 const out = explicitOut ?? path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`);
431 const script = `Add-Type -AssemblyName System.Drawing;
432 $img = [System.Drawing.Image]::FromFile('${src.replace(/'/g, "''")}');
433 $rect = New-Object System.Drawing.Rectangle(${Math.round(region[0])}, ${Math.round(region[1])}, ${Math.round(region[2])}, ${Math.round(region[3])});
434 $bmp = New-Object System.Drawing.Bitmap($rect.Width, $rect.Height);
435 $g = [System.Drawing.Graphics]::FromImage($bmp);
436 $g.DrawImage($img, (New-Object System.Drawing.Rectangle(0, 0, $rect.Width, $rect.Height)), $rect, [System.Drawing.GraphicsUnit]::Pixel);
437 $g.Dispose();
438 $bmp.Save('${out.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png);
439 $bmp.Dispose(); $img.Dispose();
440 Write-Output '{"ok": true}';`;
441 const r = await psOk(script, { timeoutMs: 20_000 });
442 if (r.code !== 0 || !fs.existsSync(out)) throw new ExecError(`zoom failed: ${(r.stderr || "").slice(0, 250)}`, r);
443 return { file: out, bytes: fs.statSync(out).size, region, source: src };
444 },
445 left_click: ({ target, strategy }) => { assertEventStrategy(strategy); return clickAt(0, target.x, target.y, 1); },
446 double_click: ({ target }) => clickAt(0, target.x, target.y, 2),
447 triple_click: ({ target }) => clickAt(0, target.x, target.y, 3),
448 right_click: ({ target }) => clickAt(1, target.x, target.y, 1),
449 middle_click: ({ target }) => clickAt(2, target.x, target.y, 1),
450 mouse_move: async ({ target }) => {
451 await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null; Write-Output '{"ok": true}'`);
452 return { action_sent: true, at: { x: target.x, y: target.y } };
453 },
454 left_click_drag: async ({ from_target: from, to }) => {
455 requireInputOwner();
456 throwIfAborted();
457 heldButtons.add("LEFT");
458 try {
459 await withUser32(`[User32]::SetCursorPos(${Math.round(from.x)}, ${Math.round(from.y)}) | Out-Null;
460 Start-Sleep -Milliseconds 80;
461 [User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero);
462 Start-Sleep -Milliseconds 80;
463 [User32]::SetCursorPos(${Math.round(to.x)}, ${Math.round(to.y)}) | Out-Null;
464 Start-Sleep -Milliseconds 80;
465 [User32]::mouse_event([User32]::LEFTUP, 0, 0, 0, [UIntPtr]::Zero);
466 Write-Output '{"ok": true}';`, { timeoutMs: 20_000 });
467 heldButtons.delete("LEFT");
468 return { action_sent: true, from, to };
469 } finally { await releaseInput({ buttons: ["LEFT"], keys: [] }); }
470 },
471 left_mouse_down: async ({ target }) => {
472 requireInputOwner();
473 // Ternary must select ONLY the optional move prefix; the LEFTDOWN press
474 // always runs, so a targeted press both moves and presses.
475 const move = target ? `[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;\n` : "";
476 throwIfAborted();
477 heldButtons.add("LEFT");
478 try { await withUser32(`${move}[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`); }
479 catch (err) { await releaseInput({ buttons: ["LEFT"], keys: [] }); throw err; }
480 return { action_sent: true };
481 },
482 left_mouse_up: async () => {
483 if (!heldButtons.has("LEFT")) throw Object.assign(new ExecError("no agent pointer press to release"), { code: "input_not_held" });
484 await releaseInput({ buttons: ["LEFT"], keys: [] });
485 return { action_sent: true };
486 },
487 scroll: async ({ target, direction = "down", amount = 3 }) => {
488 const notches = Math.max(1, Math.min(30, amount));
489 // Preserve signed wheel deltas as DWORD bits before PowerShell converts
490 // the argument: its signed 0xFFFFFFFF literal cannot mask negatives.
491 const data = (direction === "down" ? -1 : direction === "up" ? 1 : 0) * notches * 120;
492 const hdata = (direction === "right" ? 1 : direction === "left" ? -1 : 0) * notches * 120;
493 await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;
494 Start-Sleep -Milliseconds 60;
495 [User32]::mouse_event([User32]::WHEEL, 0, 0, ${data >>> 0}, [UIntPtr]::Zero);
496 [User32]::mouse_event([User32]::HWHEEL, 0, 0, ${hdata >>> 0}, [UIntPtr]::Zero);
497 Write-Output '{"ok": true}';`);
498 return { action_sent: true, direction, amount };
499 },
500 type: async ({ text }) => {
501 if (!text) return { action_sent: false, note: "empty text" };
502 const b64 = Buffer.from(String(text), "utf16le").toString("base64");
503 const script = `$text = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'));
504 [User32]::SendString($text);
505 Write-Output ('{"ok": true, "chars": ' + $text.Length + '}');`;
506 await withUser32(script, { timeoutMs: Math.max(20_000, text.length * 60) });
507 return { action_sent: true, chars: text.length, strategy: "unicode-sendinput" };
508 },
509 key: async ({ text, repeat = 1 }) => {
510 const { key, vk, mods } = keyChord(text);
511 return withKeys([...mods, vk], async () => {
512 const n = Math.max(1, Math.min(100, repeat));
513 await withUser32(`$seq = @();
514 ${mods.map((m) => `$seq += [User32]::KeyInput(${m}, 0, 0);`).join("\n")}
515 for ($i = 0; $i -lt ${n}; $i++) { $seq += [User32]::KeyInput(${vk}, 0, 0); $seq += [User32]::KeyInput(${vk}, 0, 2); }
516 ${[...mods].reverse().map((m) => `$seq += [User32]::KeyInput(${m}, 0, 2);`).join("\n")}
517 [User32]::SendChecked([User32+INPUT[]]$seq);
518 Write-Output '{"ok": true}';`);
519 return { action_sent: true, key, repeat: n };
520 });
521 },
522 hold_key: async ({ text, duration }) => {
523 requireInputOwner();
524 const { key, vk, mods } = keyChord(text);
525 const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
526 const keys = [...mods, vk];
527 const event = (code, flags) => `[User32]::SendKey(${code}, ${flags});`;
528 return withKeys(keys, async () => {
529 await withUser32(`${keys.map((code) => event(code, 0)).join("\n")}
530 Start-Sleep -Milliseconds ${Math.round(d * 1000)};
531 ${[...keys].reverse().map((code) => event(code, 2)).join("\n")}
532 Write-Output '{"ok": true}';`, { timeoutMs: Math.max(10_000, d * 1000 + 8000) });
533 return { action_sent: true, key, heldSec: d };
534 });
535 },
536 set_value: async ({ target, value }) => {
537 const resolve = elementScript(target);
538 const b64 = Buffer.from(String(value ?? ""), "utf16le").toString("base64");
539 const j = await psJson(`${resolve}
540 $val = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'));
541 $vp = $cur.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern);
542 if ($vp.Current.IsReadOnly) { throw 'element_read_only' }
543 $vp.SetValue($val);
544 @{ ok = $true; verified = ($vp.Current.Value -ceq $val) } | ConvertTo-Json -Compress;`, { timeoutMs: 45_000 });
545 if (!j.ok) throw new ExecError("set_value failed");
546 return { action_sent: true, strategy: "a11y", verified: j.verified === true };
547 },
548 select_text: async () => { throw new ExecError("select_text is not implemented on the win32 backend yet — fail-closed"); },
549 perform_action: async ({ target, action = "Invoke" }) => {
550 const resolve = elementScript(target);
551 const actions = { invoke: ["Invoke", "Invoke"], click: ["Invoke", "Invoke"], toggle: ["Toggle", "Toggle"], expand: ["ExpandCollapse", "Expand"], expandcollapse: ["ExpandCollapse", "Expand"], collapse: ["ExpandCollapse", "Collapse"], select: ["SelectionItem", "Select"], selectionitem: ["SelectionItem", "Select"] };
552 const chosen = actions[String(action).toLowerCase()];
553 if (!chosen) throw new ExecError("unsupported UIA action");
554 await psOk(`${resolve}
555 $pattern = $cur.GetCurrentPattern([System.Windows.Automation.${chosen[0]}Pattern]::Pattern);
556 $pattern.${chosen[1]}();`, { timeoutMs: 45_000 });
557 return { action_sent: true, strategy: "a11y", action };
558 },
559 read_clipboard: async () => {
560 const j = await psJson(`$t = Get-Clipboard -Raw -ErrorAction SilentlyContinue;
561 @{ text = [string]$t } | ConvertTo-Json -Compress;`, { timeoutMs: 10_000 });
562 return { text: j.text ?? "", encoding: "utf8" };
563 },
564 write_clipboard: async ({ text }) => {
565 const b64 = Buffer.from(String(text ?? ""), "utf16le").toString("base64");
566 await psOk(`Set-Clipboard -Value ([System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}')));
567 Write-Output '{"ok": true}';`, { timeoutMs: 10_000 });
568 return { written: String(text ?? "").length };
569 },
570 cursor_position: async () => {
571 const j = await psJson(`${USER32_PRELUDE}
572 $p = New-Object User32+POINT;
573 [void][User32]::GetCursorPos([ref]$p);
574 Write-Output ('{"x": ' + $p.X + ', "y": ' + $p.Y + '}');`);
575 return { x: j.x, y: j.y };
576 },
577 recordingStart: async () => {
578 throw Object.assign(new ExecError("Recording is unavailable on this platform until the recorder has session-owned cleanup. Use screenshots instead."), { code: "owned_recording_unavailable" });
579 },
580 recordingStop: async ({ id }) => { throw new ExecError(`unknown recording "${id}"`); },
581 recordingStatus: ({ id }) => ({ id, running: false }),
582 recordingList: async () => {
583 const dir = recordingsDir();
584 const out = fs.existsSync(dir)
585 ? fs.readdirSync(dir).filter((f) => /\.mp4$/i.test(f)).map((f) => {
586 const st = fs.statSync(path.join(dir, f));
587 return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
588 }).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
589 : [];
590 return { dir, recordings: out, running: [] };
591 },
592 };
593
594 async function clickAt(button, x, y, clicks) {
595 if (!Number.isFinite(Number(x)) || !Number.isFinite(Number(y))) throw new ExecError("coordinates must be finite numbers");
596 const flags = button === 1 ? "RIGHTDOWN, RIGHTUP" : button === 2 ? "MIDDLEDOWN, MIDDLEUP" : "LEFTDOWN, LEFTUP";
597 const seq = [];
598 for (let i = 0; i < clicks; i++) seq.push(`[User32]::mouse_event([User32]::${flags.split(",")[0].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 40; [User32]::mouse_event([User32]::${flags.split(",")[1].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 60;`);
599 const held = button === 1 ? "RIGHT" : button === 2 ? "MIDDLE" : "LEFT";
600 throwIfAborted();
601 heldButtons.add(held);
602 try {
603 await withUser32(`[User32]::SetCursorPos(${Math.round(x)}, ${Math.round(y)}) | Out-Null;
604 Start-Sleep -Milliseconds 60;
605 ${seq.join("\n")}
606 Write-Output '{"ok": true}';`, { timeoutMs: 20_000 });
607 heldButtons.delete(held);
608 return { action_sent: true, at: { x: Number(x), y: Number(y) }, button, clicks };
609 } finally { await releaseInput({ buttons: [held], keys: [] }); }
610 }
611 }
612
613 export default { create };
614
614 lines Plain Text