| 1 | #import <Cocoa/Cocoa.h> |
| 2 | #import <ApplicationServices/ApplicationServices.h> |
| 3 | #import <dlfcn.h> |
| 4 | #include <unistd.h> |
| 5 | #include <signal.h> |
| 6 | #include <poll.h> |
| 7 | #include <fcntl.h> |
| 8 | #include <sys/file.h> |
| 9 | #include <sys/stat.h> |
| 10 | #import "darwin-recording.h" |
| 11 | #import "darwin-ocr.h" |
| 12 | |
| 13 | static volatile sig_atomic_t cuCancelled = 0; |
| 14 | static BOOL cuOwnerPipe = NO; |
| 15 | static NSDictionary *cuLeaseKey = nil; |
| 16 | static pid_t cuLeasePid = 0; |
| 17 | static NSRunningApplication *cuLeaseApp = nil; |
| 18 | #ifdef CU_TEST |
| 19 | static NSString *cuTestLockDir = nil; |
| 20 | static NSString *cuTestReleaseFile = nil; |
| 21 | static CGEventFlags cuTestInheritedTextFlags = 0; |
| 22 | static NSNumber *cuTestIdleSeconds = nil; |
| 23 | #endif |
| 24 | static void cuCancel(int signum) { cuCancelled = 1; } |
| 25 | static void cuCheckCancelled(void) { |
| 26 | if(cuOwnerPipe) { struct pollfd fd={STDIN_FILENO,POLLHUP,0}; if(poll(&fd,1,0)>0 && (fd.revents&POLLHUP)) cuCancelled=1; } |
| 27 | if(cuCancelled) @throw [NSException exceptionWithName:@"cancelled" reason:@"computer request cancelled" userInfo:nil]; |
| 28 | } |
| 29 | static void cuLockInput(void) { |
| 30 | // One physical desktop, including separately launched direct MCP hosts. |
| 31 | // The kernel releases this lock if the native owner itself crashes. |
| 32 | NSString *dir=[NSHomeDirectory() stringByAppendingPathComponent:@".codewhale-cu"]; |
| 33 | #ifdef CU_TEST |
| 34 | if(cuTestLockDir) dir=cuTestLockDir; |
| 35 | #endif |
| 36 | [NSFileManager.defaultManager createDirectoryAtPath:dir withIntermediateDirectories:YES attributes:@{NSFilePosixPermissions:@0700} error:nil]; |
| 37 | int fd=open([[dir stringByAppendingPathComponent:@"input.lock"] fileSystemRepresentation],O_CREAT|O_RDWR|O_NOFOLLOW|O_CLOEXEC,0600); |
| 38 | if(fd<0) @throw [NSException exceptionWithName:@"input_lock" reason:[NSString stringWithFormat:@"cannot open Computer Use input ownership lock: %s",strerror(errno)] userInfo:nil]; |
| 39 | struct stat st; |
| 40 | if(fd<0 || fstat(fd,&st)!=0 || !S_ISREG(st.st_mode) || st.st_uid!=getuid() || flock(fd,LOCK_EX|LOCK_NB)!=0) { |
| 41 | if(fd>=0) close(fd); |
| 42 | @throw [NSException exceptionWithName:@"input_busy" reason:@"another Computer Use session owns held input; release its key or pointer before sending input" userInfo:nil]; |
| 43 | } |
| 44 | } |
| 45 | static BOOL cuFrontmostIsPid(pid_t pid); |
| 46 | static void cuRequireForeground(NSRunningApplication *expected) { |
| 47 | NSRunningApplication *actual=NSWorkspace.sharedWorkspace.frontmostApplication; |
| 48 | if(!cuFrontmostIsPid(expected.processIdentifier)) |
| 49 | @throw [NSException exceptionWithName:@"focus" reason:[NSString stringWithFormat:@"foreground changed to %@ (pid %d); expected %@ (pid %d). No key-down or text was sent to the new foreground application.",actual.localizedName?:@"unknown application",actual.processIdentifier,expected.localizedName?:@"bound application",expected.processIdentifier] userInfo:nil]; |
| 50 | } |
| 51 | static id cuPostKey(NSDictionary *args, pid_t destination) { |
| 52 | CGEventRef event=CGEventCreateKeyboardEvent(NULL,[args[@"code"] unsignedShortValue],[args[@"down"] boolValue]); |
| 53 | CGEventSetFlags(event,[args[@"flags"] unsignedLongLongValue]); |
| 54 | if([args[@"foreground_input"] boolValue]) CGEventPost(kCGHIDEventTap,event); |
| 55 | else CGEventPostToPid(destination,event); |
| 56 | CFRelease(event); |
| 57 | return @{@"action_sent":@YES}; |
| 58 | } |
| 59 | static void cuPrint(id result) { |
| 60 | NSData *data=[NSJSONSerialization dataWithJSONObject:result options:NSJSONWritingFragmentsAllowed error:nil]; |
| 61 | puts([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding].UTF8String); fflush(stdout); |
| 62 | } |
| 63 | static void cuReleaseLease(void) { |
| 64 | #ifdef CU_TEST |
| 65 | if(cuTestReleaseFile) { [@"released" writeToFile:cuTestReleaseFile atomically:YES encoding:NSUTF8StringEncoding error:nil]; cuTestReleaseFile=nil; } |
| 66 | #endif |
| 67 | if(cuLeaseKey) { |
| 68 | NSMutableDictionary *up=[cuLeaseKey mutableCopy]; up[@"down"]=@NO; |
| 69 | if([up[@"foreground_input"] boolValue] || !cuLeaseApp.terminated) cuPostKey(up,cuLeasePid); |
| 70 | cuLeaseKey=nil; cuLeaseApp=nil; |
| 71 | } |
| 72 | } |
| 73 | // A held lease is only ever a key: the pointer is never held on the user's |
| 74 | // cursor. Any line (or EOF) from the owner releases it. |
| 75 | static void cuWaitForLease(void) { |
| 76 | @try { |
| 77 | while(!cuCancelled) { |
| 78 | struct pollfd fd={STDIN_FILENO,POLLIN|POLLHUP,0}; |
| 79 | int ready=poll(&fd,1,100); |
| 80 | if(ready<=0) continue; |
| 81 | char byte; ssize_t n=read(STDIN_FILENO,&byte,1); |
| 82 | if(n<=0 || byte=='\n') break; |
| 83 | } |
| 84 | } @finally { cuReleaseLease(); } |
| 85 | } |
| 86 | |
| 87 | /** |
| 88 | * Window-routed events require exclusive foreground control. Addressing a |
| 89 | * window avoids cursor movement but its key-window lease redirects the |
| 90 | * person's keyboard. It must never be used as a background fallback. |
| 91 | */ |
| 92 | static void cuRequireFocusControl(NSDictionary *args) { |
| 93 | if(![args[@"foreground_input"] boolValue]) |
| 94 | @throw [NSException exceptionWithName:@"background_focus_required" reason:@"background_focus_required: this action requires keyboard focus; no input was sent. Use accessibility, browser control, or a separate computer." userInfo:nil]; |
| 95 | } |
| 96 | typedef OSStatus (*cuGetFrontFn)(ProcessSerialNumber *); |
| 97 | typedef OSStatus (*cuGetPSNFn)(pid_t, ProcessSerialNumber *); |
| 98 | typedef OSStatus (*cuSetFrontFn)(ProcessSerialNumber *, uint32_t, uint32_t); |
| 99 | typedef OSStatus (*cuPostRecordFn)(ProcessSerialNumber *, const void *); |
| 100 | typedef void (*cuSetWinLocFn)(CGEventRef, CGPoint); |
| 101 | static BOOL axActivate(pid_t pid); |
| 102 | static BOOL cuBgResolved = NO; |
| 103 | static cuGetFrontFn cuGetFront; |
| 104 | static cuGetPSNFn cuGetPSN; |
| 105 | static cuSetFrontFn cuSetFront; |
| 106 | static cuPostRecordFn cuPostRecord; |
| 107 | static cuSetWinLocFn cuSetWinLoc; |
| 108 | static BOOL cuResolveBgPointer(void) { |
| 109 | if(cuBgResolved) return cuGetFront && cuGetPSN && cuSetFront && cuPostRecord && cuSetWinLoc; |
| 110 | cuBgResolved = YES; |
| 111 | void *sl = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY); |
| 112 | void *hs = dlopen("/System/Library/Frameworks/ApplicationServices.framework/Frameworks/HIServices.framework/HIServices", RTLD_LAZY); |
| 113 | if(sl) { |
| 114 | cuGetFront = (cuGetFrontFn)dlsym(sl, "_SLPSGetFrontProcess"); |
| 115 | cuSetFront = (cuSetFrontFn)dlsym(sl, "SLPSSetFrontProcessWithOptions"); |
| 116 | cuPostRecord = (cuPostRecordFn)dlsym(sl, "SLPSPostEventRecordTo"); |
| 117 | cuSetWinLoc = (cuSetWinLocFn)dlsym(sl, "CGEventSetWindowLocation"); |
| 118 | } |
| 119 | if(hs) cuGetPSN = (cuGetPSNFn)dlsym(hs, "GetProcessForPID"); |
| 120 | return cuGetFront && cuGetPSN && cuSetFront && cuPostRecord && cuSetWinLoc; |
| 121 | } |
| 122 | /** Smallest layer-0 window of pid containing p (a sheet beats its parent). */ |
| 123 | static BOOL cuWindowAtPointForPid(pid_t pid, CGPoint p, uint32_t *outWin, CGRect *outFrame) { |
| 124 | NSArray *windows = CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionAll, kCGNullWindowID)); |
| 125 | double bestArea = 0; |
| 126 | BOOL found = NO; |
| 127 | for(NSDictionary *w in windows) { |
| 128 | if([w[(__bridge NSString *)kCGWindowOwnerPID] intValue] != pid) continue; |
| 129 | if([w[(__bridge NSString *)kCGWindowLayer] intValue] != 0) continue; |
| 130 | NSNumber *alpha = w[(__bridge NSString *)kCGWindowAlpha]; |
| 131 | if(alpha && [alpha doubleValue] <= 0) continue; |
| 132 | CGRect b; |
| 133 | if(!CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds], &b)) continue; |
| 134 | if(b.size.width < 1 || b.size.height < 1 || !CGRectContainsPoint(b, p)) continue; |
| 135 | double area = b.size.width * b.size.height; |
| 136 | if(!found || area < bestArea) { |
| 137 | found = YES; bestArea = area; |
| 138 | *outWin = [w[(__bridge NSString *)kCGWindowNumber] unsignedIntValue]; |
| 139 | *outFrame = b; |
| 140 | } |
| 141 | } |
| 142 | return found; |
| 143 | } |
| 144 | /** |
| 145 | * State for a window-routed action. Measured on macOS 26.1: the record |
| 146 | * channel only delivers to views when the target window is key, and the |
| 147 | * front-process lease (kCPSNoWindows-style options, no windows raised) is |
| 148 | * what makes it key — the window-focus record alone makes it main, which |
| 149 | * leaves events arriving at the process and swallowed by first-mouse |
| 150 | * semantics. The lease is taken for every gesture and restored in @finally; |
| 151 | * menus opened during it are held across calls (watchdog-capped) because a |
| 152 | * menu closes the moment the lease ends. Chromium rebuilds its AX tree |
| 153 | * lazily across the first transitions, so observes poll through the rebuild. |
| 154 | */ |
| 155 | typedef struct { ProcessSerialNumber frontPSN, targetPSN, postPSN; pid_t frontPid; pid_t targetPid; pid_t postPid; uint32_t postWin; BOOL swapped; double t0, idleBefore, idleAfter, leaseMs, yieldMs; } cuBgLease; |
| 156 | static BOOL axActivate(pid_t pid); |
| 157 | static BOOL cuFrontmostIsPid(pid_t pid); |
| 158 | static id attr(AXUIElementRef el, NSString *name); |
| 159 | static void axPrepare(AXUIElementRef app); |
| 160 | /** |
| 161 | * The frame and owner pid of a CGWindow by number — no pid filter, because |
| 162 | * the window an action is routed to may belong to a hosting service. |
| 163 | */ |
| 164 | static BOOL cuWindowInfoForNumber(uint32_t winNum, CGRect *outFrame, pid_t *outPid) { |
| 165 | NSArray *windows = CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionAll, kCGNullWindowID)); |
| 166 | for(NSDictionary *w in windows) { |
| 167 | if([w[(__bridge NSString *)kCGWindowNumber] unsignedIntValue] != winNum) continue; |
| 168 | if(outPid) *outPid = [w[(__bridge NSString *)kCGWindowOwnerPID] intValue]; |
| 169 | if(outFrame) CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds], outFrame); |
| 170 | return YES; |
| 171 | } |
| 172 | return NO; |
| 173 | } |
| 174 | /** |
| 175 | * Service-hosted UI (openAndSavePanelService panels and similar XPC surfaces) |
| 176 | * presents as TWO co-located windows at the same frame: a proxy owned by the |
| 177 | * client app and the real window owned by the service. Records addressed to |
| 178 | * the client die in its event queue — the key-equivalent and text handlers |
| 179 | * run in the service's AppKit. A foreign window only takes over routing when |
| 180 | * its owner is an XPC service: two ordinary apps can share a frame (two |
| 181 | * maximized windows), and that must never redirect the user's input. |
| 182 | */ |
| 183 | static BOOL cuHostedWindowAtSameFrame(pid_t excludePid, CGRect f, uint32_t *outWin, pid_t *outPid) { |
| 184 | NSArray *windows = CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionAll, kCGNullWindowID)); |
| 185 | for(NSDictionary *w in windows) { |
| 186 | pid_t owner = [w[(__bridge NSString *)kCGWindowOwnerPID] intValue]; |
| 187 | if(owner == excludePid || [w[(__bridge NSString *)kCGWindowLayer] intValue] != 0) continue; |
| 188 | CGRect b; |
| 189 | if(!CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds], &b)) continue; |
| 190 | if(fabs(b.origin.x - f.origin.x) > 2 || fabs(b.origin.y - f.origin.y) > 2) continue; |
| 191 | if(fabs(b.size.width - f.size.width) > 2 || fabs(b.size.height - f.size.height) > 2) continue; |
| 192 | NSRunningApplication *ra = [NSRunningApplication runningApplicationWithProcessIdentifier:owner]; |
| 193 | if(![ra.bundleIdentifier containsString:@".xpc."]) continue; |
| 194 | *outWin = [w[(__bridge NSString *)kCGWindowNumber] unsignedIntValue]; |
| 195 | if(outPid) *outPid = owner; |
| 196 | return YES; |
| 197 | } |
| 198 | return NO; |
| 199 | } |
| 200 | // User-activity yield: before a moment that touches shared input — a front |
| 201 | // lease, a real-pointer gesture, foreground keys — wait for a gap in the |
| 202 | // person's hardware input rather than cutting between their keystrokes. |
| 203 | // kCGAnyInputEventType only counts hardware events (verified 2026-09-17: |
| 204 | // our posted events never tick it), so it is exactly "is the human using |
| 205 | // the machine right now". yield_gap_ms is the quiet window we wait for (0 |
| 206 | // disables); yield_wait_ms bounds the wait. A busy desktop refuses before |
| 207 | // input is sent; a deadline is not permission to interrupt the person. |
| 208 | // Returns milliseconds yielded, 0 when the surface was already free. |
| 209 | static double cuYieldToUser(NSDictionary *args) { |
| 210 | double gap=[args[@"yield_gap_ms"] doubleValue], wait=[args[@"yield_wait_ms"] doubleValue]; |
| 211 | if(gap<=0 || wait<=0) return 0; |
| 212 | NSTimeInterval start=NSProcessInfo.processInfo.systemUptime; |
| 213 | while(YES) { |
| 214 | cuCheckCancelled(); |
| 215 | double idle=CGEventSourceSecondsSinceLastEventType(kCGEventSourceStateHIDSystemState,kCGAnyInputEventType); |
| 216 | #ifdef CU_TEST |
| 217 | if(cuTestIdleSeconds) idle=cuTestIdleSeconds.doubleValue; |
| 218 | #endif |
| 219 | double elapsed=(NSProcessInfo.processInfo.systemUptime-start)*1000.0; |
| 220 | if(isfinite(idle) && idle>=0 && idle*1000.0>=gap) return elapsed; |
| 221 | if(elapsed>=wait) |
| 222 | @throw [NSException exceptionWithName:@"user_busy" reason:@"user_busy: no quiet input window became available; no input was sent. Wait for the user to finish before trying again." userInfo:nil]; |
| 223 | usleep((useconds_t)(fmin(40.0,wait-elapsed)*1000.0)); |
| 224 | } |
| 225 | } |
| 226 | static BOOL cuBgLeaseBegin(NSRunningApplication *inputApp, uint32_t winNum, BOOL swap, NSDictionary *args, cuBgLease *lease, NSString **why) { |
| 227 | cuRequireFocusControl(args); |
| 228 | lease->targetPid = inputApp.processIdentifier; |
| 229 | lease->swapped = NO; |
| 230 | lease->t0 = lease->idleBefore = lease->idleAfter = lease->leaseMs = lease->yieldMs = 0; |
| 231 | if(cuGetPSN(lease->targetPid, &lease->targetPSN) != 0) { |
| 232 | *why = @"could not resolve the process serial number for a window-routed action; no input was sent"; |
| 233 | return NO; |
| 234 | } |
| 235 | cuCheckCancelled(); |
| 236 | if(swap && !cuFrontmostIsPid(lease->targetPid)) { |
| 237 | // An already-frontmost target needs no swap: the record already addresses |
| 238 | // the window, and setting an app front of itself changes nothing. Skipping |
| 239 | // it keeps receipts truthful (front_lease:false) and avoids a restore |
| 240 | // attempt for focus that was never borrowed. |
| 241 | // Yield to the person first: borrowing the front while they are |
| 242 | // mid-keystroke can redirect their input into our window. The wait is |
| 243 | // reported as yield_ms, not hidden, and runs before t0 so the borrow |
| 244 | // window measures only the time focus was actually held. |
| 245 | lease->yieldMs = cuYieldToUser(args); |
| 246 | NSRunningApplication *frontApp = NSWorkspace.sharedWorkspace.frontmostApplication; |
| 247 | lease->frontPid = frontApp.processIdentifier; |
| 248 | if(cuGetFront(&lease->frontPSN) != 0 || cuSetFront(&lease->targetPSN, 0, 0x400) != 0) { |
| 249 | *why = @"the window server refused the background focus lease; no input was sent"; |
| 250 | return NO; |
| 251 | } |
| 252 | lease->swapped = YES; |
| 253 | // Interference accounting (SHA-6643): the borrow window and the HID idle |
| 254 | // clock around it. Synthesized events do not tick this clock (verified |
| 255 | // 2026-09-17: a posted key leaves it advancing), so a clock that fails to |
| 256 | // advance across the window means hardware input arrived mid-lease. |
| 257 | lease->t0 = CFAbsoluteTimeGetCurrent(); |
| 258 | lease->idleBefore = CGEventSourceSecondsSinceLastEventType(kCGEventSourceStateHIDSystemState, kCGAnyInputEventType); |
| 259 | } |
| 260 | // Records are addressed to the window's real owner, which is not always |
| 261 | // the bound app: a service-hosted panel (openAndSavePanelService) leaves |
| 262 | // a same-frame proxy in the client while its handlers run in the service. |
| 263 | // Events posted to the client's PSN die in its queue. The front lease |
| 264 | // stays on the client — that is what makes its panels key — while the |
| 265 | // focus record and event records go to the owning process. |
| 266 | lease->postPid = lease->targetPid; |
| 267 | lease->postPSN = lease->targetPSN; |
| 268 | lease->postWin = winNum; |
| 269 | { |
| 270 | CGRect wf = CGRectZero; |
| 271 | pid_t owner = 0; |
| 272 | if(cuWindowInfoForNumber(winNum, &wf, &owner)) { |
| 273 | pid_t real = 0; uint32_t realWin = 0; |
| 274 | if(owner != lease->targetPid) { |
| 275 | NSRunningApplication *ra = [NSRunningApplication runningApplicationWithProcessIdentifier:owner]; |
| 276 | if([ra.bundleIdentifier containsString:@".xpc."]) { real = owner; realWin = winNum; } |
| 277 | } else cuHostedWindowAtSameFrame(lease->targetPid, wf, &realWin, &real); |
| 278 | if(real && cuGetPSN(real, &lease->postPSN) == 0) { lease->postPid = real; lease->postWin = realWin; } |
| 279 | else { lease->postPSN = lease->targetPSN; } |
| 280 | } |
| 281 | } |
| 282 | uint8_t rec[0xf8]; |
| 283 | memset(rec, 0, sizeof(rec)); |
| 284 | rec[0x24] = 0xf8; rec[0x28] = 0x0d; |
| 285 | rec[0x5c] = (lease->postWin >> 24) & 0xff; rec[0x5d] = (lease->postWin >> 16) & 0xff; |
| 286 | rec[0x5e] = (lease->postWin >> 8) & 0xff; rec[0x5f] = lease->postWin & 0xff; |
| 287 | rec[0xaa] = 0x01; |
| 288 | cuPostRecord(&lease->postPSN, rec); |
| 289 | usleep(30000); |
| 290 | return YES; |
| 291 | } |
| 292 | static void cuLeaseAccounting(NSMutableDictionary *receipt, cuBgLease *lease) { |
| 293 | // No numbers when nothing was borrowed, or when the lease is still held |
| 294 | // across calls (menu path): a zero window would claim an instant lease. |
| 295 | // Millisecond precision: these ride every lease receipt and its trajectory. |
| 296 | if(!lease->swapped || lease->leaseMs <= 0) return; |
| 297 | if(lease->yieldMs > 0) receipt[@"yield_ms"] = @(round(lease->yieldMs)); |
| 298 | receipt[@"lease_ms"] = @(round(lease->leaseMs * 1000.0) / 1000.0); |
| 299 | receipt[@"idle_before_s"] = @(round(lease->idleBefore * 1000.0) / 1000.0); |
| 300 | receipt[@"idle_after_s"] = @(round(lease->idleAfter * 1000.0) / 1000.0); |
| 301 | } |
| 302 | static BOOL cuBgLeaseEnd(cuBgLease *lease) { |
| 303 | if(!lease->swapped) return YES; // nothing was borrowed |
| 304 | cuSetFront(&lease->frontPSN, 0, 0x400); |
| 305 | // NSWorkspace's frontmost view is stale in a one-shot helper; the SLS |
| 306 | // front-process read is authoritative. Re-assert through AX while the |
| 307 | // lease is still visible. The outcome is reported, not assumed: a failed |
| 308 | // restore means the person's next keystrokes land in the wrong app. |
| 309 | for(int i = 0; i < 20; i++) { |
| 310 | if(!cuFrontmostIsPid(lease->targetPid)) break; |
| 311 | axActivate(lease->frontPid); |
| 312 | usleep(50000); |
| 313 | } |
| 314 | // Measured after restore: the borrow window runs take -> handed back, and |
| 315 | // input that lands during a struggling restore counts as mid-lease. |
| 316 | lease->idleAfter = CGEventSourceSecondsSinceLastEventType(kCGEventSourceStateHIDSystemState, kCGAnyInputEventType); |
| 317 | lease->leaseMs = (CFAbsoluteTimeGetCurrent() - lease->t0) * 1000.0; |
| 318 | return !cuFrontmostIsPid(lease->targetPid); |
| 319 | } |
| 320 | static BOOL cuFrontmostIsPid(pid_t pid) { |
| 321 | ProcessSerialNumber front, want; |
| 322 | if(cuResolveBgPointer() && cuGetFront(&front) == 0 && cuGetPSN(pid, &want) == 0) |
| 323 | return front.highLongOfPSN == want.highLongOfPSN && front.lowLongOfPSN == want.lowLongOfPSN; |
| 324 | [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.02]]; |
| 325 | return NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier == pid; |
| 326 | } |
| 327 | /** |
| 328 | * A menu opened under a front-process lease closes the moment the lease |
| 329 | * ends, so for those (Chromium) the lease is held in a state file across |
| 330 | * calls and given back by the next raw-input call or a 6 s watchdog — |
| 331 | * and only while the target is still frontmost: the user taking another |
| 332 | * app in the meantime is a choice, never something to yank back. |
| 333 | */ |
| 334 | static NSString *cuFrontLeaseFile(void) { |
| 335 | return [NSHomeDirectory() stringByAppendingPathComponent:@".codewhale-cu/front-lease.json"]; |
| 336 | } |
| 337 | static void cuFrontLeaseRestoreIfHeld(void) { |
| 338 | NSString *file = cuFrontLeaseFile(); |
| 339 | NSData *data = [NSData dataWithContentsOfFile:file]; |
| 340 | if(!data) return; |
| 341 | NSDictionary *held = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; |
| 342 | if(![held isKindOfClass:NSDictionary.class]) return; |
| 343 | [NSFileManager.defaultManager removeItemAtPath:file error:nil]; |
| 344 | if(!cuResolveBgPointer()) return; |
| 345 | pid_t targetPid = [held[@"targetPid"] intValue]; |
| 346 | if(!cuFrontmostIsPid(targetPid)) return; |
| 347 | ProcessSerialNumber psn = { (UInt32)[held[@"frontHi"] unsignedIntValue], (UInt32)[held[@"frontLo"] unsignedIntValue] }; |
| 348 | cuSetFront(&psn, 0, 0x400); |
| 349 | for(int i = 0; i < 10; i++) { |
| 350 | if(!cuFrontmostIsPid(targetPid)) break; |
| 351 | axActivate([held[@"frontPid"] intValue]); |
| 352 | usleep(50000); |
| 353 | } |
| 354 | } |
| 355 | static BOOL cuFrontLeaseHeldForPid(pid_t pid) { |
| 356 | NSData *data = [NSData dataWithContentsOfFile:cuFrontLeaseFile()]; |
| 357 | if(!data) return NO; |
| 358 | NSDictionary *held = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; |
| 359 | return [held isKindOfClass:NSDictionary.class] && [held[@"targetPid"] intValue] == pid; |
| 360 | } |
| 361 | static BOOL cuMenuOpenForApp(pid_t pid) { |
| 362 | AXUIElementRef app = AXUIElementCreateApplication(pid); |
| 363 | AXUIElementSetMessagingTimeout(app, 1.0); |
| 364 | axPrepare(app); |
| 365 | // Chromium vends an open select popup inside the window's subtree rather |
| 366 | // than as an app-level AXMenu, so both shapes are searched, bounded. |
| 367 | int budget = 400; |
| 368 | NSMutableArray *stack = [NSMutableArray array]; |
| 369 | [stack addObjectsFromArray:attr(app, @"AXChildren") ?: @[]]; |
| 370 | [stack addObjectsFromArray:attr(app, @"AXWindows") ?: @[]]; |
| 371 | BOOL open = NO; |
| 372 | while(stack.count && budget-- > 0 && !open) { |
| 373 | id node = stack.lastObject; |
| 374 | [stack removeLastObject]; |
| 375 | NSString *role = attr((__bridge AXUIElementRef)node, @"AXRole"); |
| 376 | if([role isEqual:@"AXMenu"]) { open = YES; break; } |
| 377 | if([role isEqual:@"AXMenuBar"] || [role isEqual:@"AXMenuBarItem"]) continue; |
| 378 | [stack addObjectsFromArray:attr((__bridge AXUIElementRef)node, @"AXChildren") ?: @[]]; |
| 379 | } |
| 380 | CFRelease(app); |
| 381 | return open; |
| 382 | } |
| 383 | static void cuFrontLeaseHold(cuBgLease *lease) { |
| 384 | NSString *token = NSUUID.UUID.UUIDString; |
| 385 | // Menu items take seconds to reappear in Chromium's rebuilt tree; the |
| 386 | // observe+pick must fit inside the hold. |
| 387 | NSNumber *deadline = @((long long)([NSDate new].timeIntervalSince1970 * 1000) + 15000); |
| 388 | NSDictionary *state = @{ @"token": token, @"targetPid": @(lease->targetPid), @"frontPid": @(lease->frontPid), |
| 389 | @"frontHi": @(lease->frontPSN.highLongOfPSN), @"frontLo": @(lease->frontPSN.lowLongOfPSN), |
| 390 | @"deadline": deadline }; |
| 391 | NSString *file = cuFrontLeaseFile(); |
| 392 | [NSFileManager.defaultManager createDirectoryAtPath:file.stringByDeletingLastPathComponent withIntermediateDirectories:YES attributes:@{ NSFilePosixPermissions: @0700 } error:nil]; |
| 393 | NSData *data = [NSJSONSerialization dataWithJSONObject:state options:0 error:nil]; |
| 394 | if(!data || ![data writeToFile:file atomically:YES]) { cuBgLeaseEnd(lease); return; } |
| 395 | NSDictionary *req = @{ @"tool": @"front_lease_watchdog", @"args": @{ @"token": token, @"deadline": deadline } }; |
| 396 | NSData *reqData = [NSJSONSerialization dataWithJSONObject:req options:0 error:nil]; |
| 397 | if(reqData) { |
| 398 | NSTask *watch = [NSTask new]; |
| 399 | watch.executableURL = [NSURL fileURLWithPath:NSProcessInfo.processInfo.arguments[0]]; |
| 400 | watch.arguments = @[ [[NSString alloc] initWithData:reqData encoding:NSUTF8StringEncoding] ]; |
| 401 | watch.standardInput = NSFileHandle.fileHandleWithNullDevice; |
| 402 | watch.standardOutput = NSFileHandle.fileHandleWithNullDevice; |
| 403 | watch.standardError = NSFileHandle.fileHandleWithNullDevice; |
| 404 | [watch launchAndReturnError:nil]; |
| 405 | } |
| 406 | } |
| 407 | /** Field-set + record post shared by mouse, wheel and keyboard events. */ |
| 408 | static void cuPostEventRecord(cuBgLease *lease, CGEventRef e, CGPoint winLoc) { |
| 409 | CGEventSetIntegerValueField(e, 0, 3); |
| 410 | CGEventSetIntegerValueField(e, 7, 3); |
| 411 | CGEventSetIntegerValueField(e, 0x28, lease->postPid); |
| 412 | CGEventSetIntegerValueField(e, 0x33, lease->postWin); |
| 413 | CGEventSetIntegerValueField(e, 0x5b, lease->postWin); |
| 414 | CGEventSetIntegerValueField(e, 0x5c, lease->postWin); |
| 415 | cuSetWinLoc(e, winLoc); |
| 416 | void *record = *(void **)((char *)e + 0x18); |
| 417 | if(record) cuPostRecord(&lease->postPSN, record); else CGEventPostToPid(lease->postPid, e); |
| 418 | } |
| 419 | static id attr(AXUIElementRef el, NSString *name); |
| 420 | static void axPrepare(AXUIElementRef app); |
| 421 | /** The CGWindowNumber and frame of the layer-0 window of pid whose frame matches an AX window rect. */ |
| 422 | static BOOL cuWindowNumberForFrame(pid_t pid, CGRect axFrame, uint32_t *outWin) { |
| 423 | NSArray *windows = CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionAll, kCGNullWindowID)); |
| 424 | for(NSDictionary *w in windows) { |
| 425 | if([w[(__bridge NSString *)kCGWindowOwnerPID] intValue] != pid) continue; |
| 426 | if([w[(__bridge NSString *)kCGWindowLayer] intValue] != 0) continue; |
| 427 | CGRect b; |
| 428 | if(!CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds], &b)) continue; |
| 429 | if(fabs(b.origin.x - axFrame.origin.x) > 2 || fabs(b.origin.y - axFrame.origin.y) > 2) continue; |
| 430 | if(fabs(b.size.width - axFrame.size.width) > 2 || fabs(b.size.height - axFrame.size.height) > 2) continue; |
| 431 | *outWin = [w[(__bridge NSString *)kCGWindowNumber] unsignedIntValue]; |
| 432 | return YES; |
| 433 | } |
| 434 | return NO; |
| 435 | } |
| 436 | static NSDictionary *cuBgPointer(NSRunningApplication *inputApp, NSDictionary *args) { |
| 437 | if(!cuResolveBgPointer()) |
| 438 | @throw [NSException exceptionWithName:@"bg_dispatch_unavailable" reason:@"window-routed background pointer is unavailable: SLPSPostEventRecordTo/CGEventSetWindowLocation not resolvable via SkyLight; no input was sent" userInfo:nil]; |
| 439 | NSArray *steps = args[@"steps"]; |
| 440 | if(![steps isKindOfClass:NSArray.class] || !steps.count || steps.count > 400) |
| 441 | @throw [NSException exceptionWithName:@"args" reason:@"bg_pointer needs 1..400 steps" userInfo:nil]; |
| 442 | CGPoint anchor = CGPointZero; |
| 443 | BOOL haveAnchor = NO; |
| 444 | for(NSDictionary *step in steps) { |
| 445 | // A scroll step carries its point alongside the deltas. |
| 446 | if([step[@"x"] isKindOfClass:NSNumber.class] && [step[@"y"] isKindOfClass:NSNumber.class]) { |
| 447 | anchor = CGPointMake([step[@"x"] doubleValue], [step[@"y"] doubleValue]); |
| 448 | haveAnchor = YES; break; |
| 449 | } |
| 450 | } |
| 451 | if(!haveAnchor) @throw [NSException exceptionWithName:@"args" reason:@"bg_pointer steps carry no point" userInfo:nil]; |
| 452 | uint32_t winNum = 0; |
| 453 | CGRect frame = CGRectZero; |
| 454 | if(!cuWindowAtPointForPid(inputApp.processIdentifier, anchor, &winNum, &frame)) |
| 455 | @throw [NSException exceptionWithName:@"window" reason:@"no window of the bound application covers the start point; no input was sent" userInfo:nil]; |
| 456 | cuBgLease lease; |
| 457 | NSString *why = nil; |
| 458 | // Measured on macOS 26.1: view-level mouse delivery requires the window to |
| 459 | // be key, and only the front-process lease makes it key. The window-focus |
| 460 | // record alone makes it main — events reach the process and are swallowed. |
| 461 | if(!cuBgLeaseBegin(inputApp, winNum, YES, args, &lease, &why)) |
| 462 | @throw [NSException exceptionWithName:@"bg_dispatch_unavailable" reason:why userInfo:nil]; |
| 463 | BOOL menuLeaseHeld = NO; |
| 464 | BOOL restored = YES; |
| 465 | @try { |
| 466 | for(NSDictionary *step in steps) { |
| 467 | cuCheckCancelled(); |
| 468 | if([step[@"scroll"] isKindOfClass:NSArray.class]) { |
| 469 | NSArray *d = step[@"scroll"]; |
| 470 | // Pixel units: Chromium ignores line-unit wheel events entirely |
| 471 | // (measured). One notch ≈ one line ≈ 40 px. |
| 472 | CGEventRef wheel = CGEventCreateScrollWheelEvent(NULL, kCGScrollEventUnitPixel, 2, [d[1] intValue] * 40, [d[0] intValue] * 40); |
| 473 | cuPostEventRecord(&lease, wheel, CGPointMake(anchor.x - frame.origin.x, anchor.y - frame.origin.y)); |
| 474 | CFRelease(wheel); |
| 475 | } else { |
| 476 | CGPoint p = CGPointMake([step[@"x"] doubleValue], [step[@"y"] doubleValue]); |
| 477 | int type = [step[@"type"] intValue], button = [step[@"button"] intValue]; |
| 478 | CGEventRef e = CGEventCreateMouseEvent(NULL, (CGEventType)type, p, (CGMouseButton)button); |
| 479 | CGEventSetIntegerValueField(e, kCGMouseEventClickState, [step[@"clickState"] longLongValue]); |
| 480 | BOOL pressed = type == kCGEventLeftMouseDown || type == kCGEventRightMouseDown || type == kCGEventOtherMouseDown |
| 481 | || type == kCGEventLeftMouseDragged || type == kCGEventRightMouseDragged || type == kCGEventOtherMouseDragged; |
| 482 | CGEventSetDoubleValueField(e, 2, pressed ? 1.0 : 0.0); |
| 483 | cuPostEventRecord(&lease, e, CGPointMake(p.x - frame.origin.x, p.y - frame.origin.y)); |
| 484 | CFRelease(e); |
| 485 | } |
| 486 | usleep((useconds_t)([step[@"delayMs"] intValue] ?: 20) * 1000); |
| 487 | } |
| 488 | // A menu opened under a front lease dies when the lease ends; menus |
| 489 | // animate in after the mouse-up, so a click-ending gesture polls briefly. |
| 490 | // A web popup needs the poll to ride out Chromium's post-activation AX |
| 491 | // rebuild (seconds), which the caller asks for with menu_poll_ms. |
| 492 | if(lease.swapped) { |
| 493 | int lastType = -1; |
| 494 | for(NSDictionary *step in [steps reverseObjectEnumerator]) { |
| 495 | if([step[@"type"] isKindOfClass:NSNumber.class]) { lastType = [step[@"type"] intValue]; break; } |
| 496 | } |
| 497 | BOOL clickEnded = lastType == kCGEventLeftMouseUp || lastType == kCGEventRightMouseUp || lastType == kCGEventOtherMouseUp; |
| 498 | NSInteger pollMs = MAX(0, MIN(10000, [args[@"menu_poll_ms"] integerValue] ?: 720)); |
| 499 | menuLeaseHeld = cuMenuOpenForApp(inputApp.processIdentifier); |
| 500 | if(!menuLeaseHeld && clickEnded) { |
| 501 | for(NSInteger waited = 0; waited < pollMs && !menuLeaseHeld; waited += 60) { |
| 502 | usleep(60000); |
| 503 | menuLeaseHeld = cuMenuOpenForApp(inputApp.processIdentifier); |
| 504 | } |
| 505 | } |
| 506 | } |
| 507 | } @finally { |
| 508 | if(menuLeaseHeld) cuFrontLeaseHold(&lease); |
| 509 | else restored = cuBgLeaseEnd(&lease); |
| 510 | } |
| 511 | NSMutableDictionary *receipt = [@{@"action_sent":@YES, @"strategy":@"window-record", @"pointer_moved":@NO, |
| 512 | @"front_lease":@(lease.swapped), @"window":@{@"id":@(lease.postWin)}} mutableCopy]; |
| 513 | if(lease.postPid != lease.targetPid) receipt[@"window_owner_pid"] = @(lease.postPid); |
| 514 | if(lease.swapped) receipt[@"front_restored"] = @(restored); |
| 515 | cuLeaseAccounting(receipt, &lease); |
| 516 | if(menuLeaseHeld) receipt[@"menu_lease_held"] = @YES; |
| 517 | return receipt; |
| 518 | } |
| 519 | static id attr(AXUIElementRef el, NSString *name) { |
| 520 | #ifdef CU_TEST |
| 521 | // The observation fixture exercises the real walker without reading a GUI. |
| 522 | if([(__bridge id)el isKindOfClass:NSDictionary.class]) return ((__bridge NSDictionary *)el)[name]; |
| 523 | #endif |
| 524 | CFTypeRef out = NULL; |
| 525 | AXError e = AXUIElementCopyAttributeValue(el, (__bridge CFStringRef)name, &out); |
| 526 | return e == kAXErrorSuccess ? CFBridgingRelease(out) : nil; |
| 527 | } |
| 528 | /** |
| 529 | * Opt the target into full accessibility. Chromium-family apps (Chrome, |
| 530 | * Electron) and WebKit content do not vend AXWebArea descendants until an |
| 531 | * assistive client sets AXEnhancedUserInterface; Firefox-style engines gate |
| 532 | * behind AXManualAccessibility. Without this an observe returns the chrome |
| 533 | * of the window — menu bar, toolbar, tab strip — and no page content at all. |
| 534 | * Apps that do not know these attributes simply refuse the write. |
| 535 | */ |
| 536 | static void axPrepare(AXUIElementRef app) { |
| 537 | #ifdef CU_TEST |
| 538 | if([(__bridge id)app isKindOfClass:NSDictionary.class]) return; |
| 539 | #endif |
| 540 | AXUIElementSetAttributeValue(app,(__bridge CFStringRef)@"AXEnhancedUserInterface",kCFBooleanTrue); |
| 541 | AXUIElementSetAttributeValue(app,(__bridge CFStringRef)@"AXManualAccessibility",kCFBooleanTrue); |
| 542 | } |
| 543 | static NSDictionary *geometry(id v, BOOL size) { |
| 544 | if (!v || CFGetTypeID((__bridge CFTypeRef)v) != AXValueGetTypeID()) return nil; |
| 545 | if (size) { CGSize s; if (AXValueGetValue((__bridge AXValueRef)v,kAXValueCGSizeType,&s)) return @{ @"w":@(s.width), @"h":@(s.height) }; } |
| 546 | else { CGPoint p; if (AXValueGetValue((__bridge AXValueRef)v,kAXValueCGPointType,&p)) return @{ @"x":@(p.x), @"y":@(p.y) }; } |
| 547 | return nil; |
| 548 | } |
| 549 | static NSDictionary *info(AXUIElementRef el, NSInteger index, NSInteger win, NSArray *path) { |
| 550 | NSMutableDictionary *d = [@{@"index":@(index), @"windowIndex":@(win), @"path":path} mutableCopy]; |
| 551 | for (NSString *key in @[@"role",@"subrole",@"value",@"enabled",@"focused"]) { |
| 552 | NSDictionary *names = @{@"role":@"AXRole",@"subrole":@"AXSubrole",@"value":@"AXValue",@"enabled":@"AXEnabled",@"focused":@"AXFocused"}; |
| 553 | id v = attr(el,names[key]); |
| 554 | if ([v isKindOfClass:NSString.class]) d[key] = [v length]>12000 ? [v substringToIndex:12000] : v; |
| 555 | else if ([v isKindOfClass:NSNumber.class]) d[key] = v; |
| 556 | } |
| 557 | id label = attr(el,@"AXTitle"); |
| 558 | if (![label isKindOfClass:NSString.class] || ![label length]) label = attr(el,@"AXDescription"); |
| 559 | if ([label isKindOfClass:NSString.class]) d[@"label"] = label; |
| 560 | id p=geometry(attr(el,@"AXPosition"),NO), s=geometry(attr(el,@"AXSize"),YES); |
| 561 | if(p) d[@"position"]=p; if(s) d[@"size"]=s; |
| 562 | CFArrayRef actions=NULL; |
| 563 | #ifdef CU_TEST |
| 564 | if([(__bridge id)el isKindOfClass:NSDictionary.class]) d[@"actions"]=attr(el,@"actions")?:@[]; |
| 565 | else |
| 566 | #endif |
| 567 | if(AXUIElementCopyActionNames(el,&actions)==kAXErrorSuccess) d[@"actions"]=CFBridgingRelease(actions); |
| 568 | else d[@"actions"]=@[]; |
| 569 | return d; |
| 570 | } |
| 571 | static void cuValidateElementIdentity(AXUIElementRef el, NSDictionary *target) { |
| 572 | NSDictionary *current=info(el,0,0,@[]); |
| 573 | for(NSString *key in @[@"role",@"label"]) { |
| 574 | id expected=target[key]?:NSNull.null, actual=current[key]?:NSNull.null; |
| 575 | if(![expected isEqual:actual]) @throw [NSException exceptionWithName:@"stale" reason:[NSString stringWithFormat:@"element changed %@; observe again before acting",key] userInfo:nil]; |
| 576 | } |
| 577 | } |
| 578 | static void walk(AXUIElementRef el, NSInteger win, NSArray *path, NSInteger depth, NSInteger limit, NSInteger max, BOOL depthIsTruncation, NSMutableArray *out, BOOL *truncated) { |
| 579 | // Breadth first keeps a long file listing from hiding its dialog buttons. |
| 580 | NSMutableArray *queue=[NSMutableArray arrayWithObject:@{@"el":(__bridge id)el,@"path":path,@"depth":@(depth)}]; |
| 581 | for(NSUInteger cursor=0;cursor<queue.count;cursor++) { |
| 582 | if(out.count>=max){ *truncated=YES; break; } |
| 583 | NSDictionary *item=queue[cursor]; |
| 584 | AXUIElementRef current=(__bridge AXUIElementRef)item[@"el"]; |
| 585 | NSArray *currentPath=item[@"path"]; |
| 586 | NSInteger currentDepth=[item[@"depth"] integerValue]; |
| 587 | [out addObject:info(current,out.count,win,currentPath)]; |
| 588 | NSArray *kids=attr(current,@"AXChildren"); |
| 589 | if(currentDepth>=limit){ if(kids.count && depthIsTruncation) *truncated=YES; continue; } |
| 590 | for(NSUInteger i=0;i<kids.count;i++) { |
| 591 | if(queue.count-cursor>=(NSUInteger)max){ *truncated=YES; break; } |
| 592 | [queue addObject:@{@"el":kids[i],@"path":[currentPath arrayByAddingObject:@(i)],@"depth":@(currentDepth+1)}]; |
| 593 | } |
| 594 | } |
| 595 | } |
| 596 | static NSArray *observeElements(AXUIElementRef app, NSArray *ws, NSDictionary *args, BOOL listWindows, BOOL *truncated) { |
| 597 | NSMutableArray *out=[NSMutableArray array]; |
| 598 | BOOL full=[args[@"detail"] isEqual:@"full"]; |
| 599 | // Web content nests deep: a browser form's controls commonly sit 12+ levels |
| 600 | // under the window, and a real page holds hundreds of controls. Depth and |
| 601 | // budget must cover that or every browser observe is silently headless. |
| 602 | // A filtered observe (query/role) is a targeted search, not a page dump, so |
| 603 | // it earns the deep budget — a control past the summary depth must still be |
| 604 | // findable. |
| 605 | BOOL deep=full||args[@"query"]||args[@"role"]; |
| 606 | NSInteger limit=deep?24:16, max=deep?1600:900; |
| 607 | if(!listWindows && !args[@"window_id"]) { |
| 608 | // Open popup menus remain useful. Hidden menu-bar descendants belong in |
| 609 | // the full view; summary reserves their budget for the app's actual UI. |
| 610 | NSInteger menuMax=max/4; |
| 611 | NSArray *children=attr(app,@"AXChildren"); |
| 612 | for(NSUInteger i=0;i<children.count;i++) { |
| 613 | NSString *role=attr((__bridge AXUIElementRef)children[i],@"AXRole"); |
| 614 | if([role isEqual:@"AXMenu"]) walk((__bridge AXUIElementRef)children[i],-2,@[@(i)],0,limit,menuMax/2,YES,out,truncated); |
| 615 | } |
| 616 | id menu=attr(app,@"AXMenuBar"); |
| 617 | if(menu) walk((__bridge AXUIElementRef)menu,-1,@[],0,full?limit:1,menuMax,full,out,truncated); |
| 618 | } |
| 619 | for(NSUInteger i=0;i<ws.count;i++) { |
| 620 | if(args[@"window_id"] && i!=[args[@"window_id"] unsignedIntegerValue]) continue; |
| 621 | if(listWindows){ NSMutableDictionary *d=[info((__bridge AXUIElementRef)ws[i],i,i,@[]) mutableCopy]; d[@"title"]=d[@"label"]?:@""; [out addObject:d]; } |
| 622 | else walk((__bridge AXUIElementRef)ws[i],i,@[],0,limit,max,YES,out,truncated); |
| 623 | } |
| 624 | return out; |
| 625 | } |
| 626 | /** |
| 627 | * An AXWebArea whose recorded path produced no descendants is a page the walk |
| 628 | * could not see into — typically Chromium still assembling its accessibility |
| 629 | * subtree right after AXEnhancedUserInterface was set. Worth one re-observe |
| 630 | * after a short settle rather than reporting an empty page. |
| 631 | */ |
| 632 | static BOOL hasOrphanWebArea(NSArray *out) { |
| 633 | for(NSDictionary *d in out) { |
| 634 | if(![d[@"role"] isEqual:@"AXWebArea"]) continue; |
| 635 | NSArray *p=d[@"path"]; NSInteger w=[d[@"windowIndex"] integerValue]; |
| 636 | BOOL kids=NO; |
| 637 | for(NSDictionary *e in out) { |
| 638 | if(e==d || [e[@"windowIndex"] integerValue]!=w) continue; |
| 639 | NSArray *q=e[@"path"]; |
| 640 | if(q.count<=p.count) continue; |
| 641 | BOOL prefix=YES; |
| 642 | for(NSUInteger i=0;i<p.count;i++) if(![q[i] isEqual:p[i]]) { prefix=NO; break; } |
| 643 | if(prefix) { kids=YES; break; } |
| 644 | } |
| 645 | if(!kids) return YES; |
| 646 | } |
| 647 | return NO; |
| 648 | } |
| 649 | static BOOL cuFrame(AXUIElementRef el, CGRect *out) { |
| 650 | NSDictionary *p=geometry(attr(el,@"AXPosition"),NO), *z=geometry(attr(el,@"AXSize"),YES); |
| 651 | if(!p || !z) return NO; |
| 652 | *out=CGRectMake([p[@"x"] doubleValue],[p[@"y"] doubleValue],[z[@"w"] doubleValue],[z[@"h"] doubleValue]); |
| 653 | return YES; |
| 654 | } |
| 655 | /** |
| 656 | * The CGWindow that should receive input for an element. An AXSheet ancestor |
| 657 | * is preferred over the app AXWindow: a hosted panel (openAndSavePanelService) |
| 658 | * is bridged into the client's tree as a sheet but is its own real window, |
| 659 | * owned by the service. The window lookup prefers the client pid, then a |
| 660 | * co-located XPC-service window — the service's window is the one whose |
| 661 | * handlers actually consume events. outRole names the resolved ancestor. |
| 662 | */ |
| 663 | static BOOL cuElementWindow(id element, pid_t preferPid, uint32_t *outWin, pid_t *outOwner, NSString **outRole) { |
| 664 | id sheet = nil, win = nil; |
| 665 | id node = element; |
| 666 | for(int depth = 0; node && depth < 64; depth++) { |
| 667 | NSString *r = attr((__bridge AXUIElementRef)node, @"AXRole"); |
| 668 | if([r isEqual:@"AXSheet"] && !sheet) sheet = node; |
| 669 | else if([r isEqual:@"AXWindow"]) { win = node; break; } |
| 670 | else if([r isEqual:@"AXApplication"]) break; |
| 671 | id parent = attr((__bridge AXUIElementRef)node, @"AXParent"); |
| 672 | if(!parent || CFEqual((__bridge CFTypeRef)parent, (__bridge CFTypeRef)node)) break; |
| 673 | node = parent; |
| 674 | } |
| 675 | for(id cand in @[sheet ?: [NSNull null], win ?: [NSNull null]]) { |
| 676 | if(cand == [NSNull null]) continue; |
| 677 | CGRect f; |
| 678 | if(!cuFrame((__bridge AXUIElementRef)cand, &f)) continue; |
| 679 | uint32_t w = 0; pid_t o = 0; |
| 680 | if(cuWindowNumberForFrame(preferPid, f, &w)) { |
| 681 | o = preferPid; |
| 682 | uint32_t fw = 0; pid_t fo = 0; |
| 683 | if(cuHostedWindowAtSameFrame(preferPid, f, &fw, &fo)) { w = fw; o = fo; } |
| 684 | } else if(!cuHostedWindowAtSameFrame(preferPid, f, &w, &o)) continue; |
| 685 | *outWin = w; |
| 686 | if(outOwner) *outOwner = o; |
| 687 | if(outRole) *outRole = attr((__bridge AXUIElementRef)cand, @"AXRole"); |
| 688 | return YES; |
| 689 | } |
| 690 | return NO; |
| 691 | } |
| 692 | static NSDictionary *capturableWindow(NSArray *windows, pid_t pid, NSString *name, CGRect preferred) { |
| 693 | NSDictionary *matched=nil; |
| 694 | for(NSDictionary *w in windows) { |
| 695 | if([w[(__bridge NSString *)kCGWindowOwnerPID] intValue]!=pid || [w[(__bridge NSString *)kCGWindowLayer] intValue]!=0) continue; |
| 696 | CGRect b; if(!CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds],&b) || b.size.width<1 || b.size.height<1) continue; |
| 697 | if(fabs(b.origin.x-preferred.origin.x)>1 || fabs(b.origin.y-preferred.origin.y)>1 || fabs(b.size.width-preferred.size.width)>1 || fabs(b.size.height-preferred.size.height)>1) continue; |
| 698 | if(matched) @throw [NSException exceptionWithName:@"window" reason:@"the selected window is ambiguous; observe the app windows again" userInfo:nil]; |
| 699 | matched=@{@"window_id":w[(__bridge NSString *)kCGWindowNumber],@"name":name?:@"App",@"points":@{@"x":@(b.origin.x),@"y":@(b.origin.y),@"w":@(b.size.width),@"h":@(b.size.height)}}; |
| 700 | } |
| 701 | if(!matched) @throw [NSException exceptionWithName:@"window" reason:@"the selected app window is not capturable; observe the app windows again" userInfo:nil]; |
| 702 | return matched; |
| 703 | } |
| 704 | static NSArray *cuActions(AXUIElementRef el) { |
| 705 | CFArrayRef names=NULL; |
| 706 | #ifdef CU_TEST |
| 707 | if([(__bridge id)el isKindOfClass:NSDictionary.class]) return attr(el,@"actions")?:@[]; |
| 708 | #endif |
| 709 | return AXUIElementCopyActionNames(el,&names)==kAXErrorSuccess?CFBridgingRelease(names):@[]; |
| 710 | } |
| 711 | static BOOL cuSettable(AXUIElementRef el, NSString *name) { |
| 712 | #ifdef CU_TEST |
| 713 | if([(__bridge id)el isKindOfClass:NSDictionary.class]) return [attr(el,@"settable") containsObject:name]; |
| 714 | #endif |
| 715 | Boolean settable=false; |
| 716 | return AXUIElementIsAttributeSettable(el,(__bridge CFStringRef)name,&settable)==kAXErrorSuccess && settable; |
| 717 | } |
| 718 | static BOOL axHasWebAncestor(AXUIElementRef el); |
| 719 | static NSString *cuClickAction(AXUIElementRef el, BOOL context) { |
| 720 | id enabled=attr(el,@"AXEnabled"); |
| 721 | if([enabled isKindOfClass:NSNumber.class] && ![enabled boolValue]) return nil; |
| 722 | NSArray *actions=cuActions(el); |
| 723 | if(context) return [actions containsObject:@"AXShowMenu"]?@"AXShowMenu":nil; |
| 724 | NSString *role=attr(el,@"AXRole"); |
| 725 | // A web popup button's AXPress does not open the native menu — only a real |
| 726 | // mouse event does. Reporting it unpressable routes the click through the |
| 727 | // window-record path, which opens the menu the page actually shows. |
| 728 | if([@[@"AXMenuButton",@"AXPopUpButton"] containsObject:role] && axHasWebAncestor(el)) return nil; |
| 729 | // Pressing a text field is toolkit-dependent; focus its insertion point directly. |
| 730 | if([@[@"AXTextField",@"AXTextArea",@"AXComboBox"] containsObject:role] && cuSettable(el,@"AXFocused")) return @"AXFocused"; |
| 731 | if([actions containsObject:@"AXPress"]) return @"AXPress"; |
| 732 | if([role isEqual:@"AXMenuItem"] && [actions containsObject:@"AXPick"]) return @"AXPick"; |
| 733 | if([@[@"AXRow",@"AXCell"] containsObject:role] && cuSettable(el,@"AXSelected")) return @"AXSelected"; |
| 734 | // Focusing is the missing middle between "not pressable" and a real click, |
| 735 | // but only for explicit text-entry roles. Anything else — a span, a group, |
| 736 | // Chromium's page-content container, Qt composers — is better served by the |
| 737 | // window-record click the caller falls back to: focusing a container is not |
| 738 | // a click, and reporting one would swallow the press. |
| 739 | if(cuSettable(el,@"AXFocused") && [@[@"AXTextField",@"AXTextArea",@"AXComboBox",@"AXSearchField",@"AXSecureTextField"] containsObject:role]) return @"AXFocused"; |
| 740 | return nil; |
| 741 | } |
| 742 | static NSDictionary *cuClick(AXUIElementRef el, BOOL context) { |
| 743 | // A control whose rendered frame is empty is one a user could not click: |
| 744 | // virtualized lists and collapsed regions vend elements that do not exist on |
| 745 | // screen. Pressing one either does nothing or toggles a row the caller |
| 746 | // cannot see. Refuse with the recovery spelled out. |
| 747 | NSDictionary *sz=geometry(attr(el,@"AXSize"),YES); |
| 748 | if(sz && ([sz[@"w"] doubleValue]<=0 || [sz[@"h"] doubleValue]<=0)) |
| 749 | @throw [NSException exceptionWithName:@"degenerate_frame" reason:@"target element has a degenerate frame (zero size); it is hidden or collapsed in a virtualized container — scroll it into view and observe again before clicking" userInfo:nil]; |
| 750 | NSString *action=cuClickAction(el,context); |
| 751 | if(!action) @throw [NSException exceptionWithName:@"background_action_unavailable" reason:@"this control has no supported accessibility click; observe its advertised actions or use a separate computer" userInfo:nil]; |
| 752 | cuCheckCancelled(); |
| 753 | BOOL attribute=[action isEqual:@"AXFocused"] || [action isEqual:@"AXSelected"]; |
| 754 | AXError error=attribute?AXUIElementSetAttributeValue(el,(__bridge CFStringRef)action,kCFBooleanTrue):AXUIElementPerformAction(el,(__bridge CFStringRef)action); |
| 755 | if(error!=kAXErrorSuccess) @throw [NSException exceptionWithName:@"action" reason:[NSString stringWithFormat:@"accessibility %@ failed: %d; no pointer fallback was sent",action,error] userInfo:nil]; |
| 756 | return @{@"action_sent":@YES,@"strategy":@"a11y",@"action":action,@"pointer_moved":@NO, |
| 757 | @"verified":@(attribute && [attr(el,action) boolValue])}; |
| 758 | } |
| 759 | static id cuScrollBar(AXUIElementRef el, BOOL horizontal) { |
| 760 | id enabled=attr(el,@"AXEnabled"); |
| 761 | if([enabled isKindOfClass:NSNumber.class] && ![enabled boolValue]) return nil; |
| 762 | return attr(el,horizontal?@"AXHorizontalScrollBar":@"AXVerticalScrollBar"); |
| 763 | } |
| 764 | static NSDictionary *cuScroll(AXUIElementRef el, NSDictionary *args) { |
| 765 | BOOL horizontal=[@[@"left",@"right"] containsObject:args[@"direction"]]; |
| 766 | id bar=nil; |
| 767 | for(id cur=(__bridge id)el;cur && !bar;) { |
| 768 | AXUIElementRef node=(__bridge AXUIElementRef)cur; |
| 769 | bar=cuScrollBar(node,horizontal); |
| 770 | if([attr(node,@"AXRole") isEqual:@"AXWindow"]) break; |
| 771 | cur=attr(node,@"AXParent"); |
| 772 | } |
| 773 | if(!bar) @throw [NSException exceptionWithName:@"background_scroll_unavailable" reason:@"no accessibility scrollbar at this target; choose an observed scroll area or a separate computer" userInfo:nil]; |
| 774 | AXUIElementRef control=(__bridge AXUIElementRef)bar; |
| 775 | BOOL forward=[@[@"down",@"right"] containsObject:args[@"direction"]]; |
| 776 | NSString *action=forward?@"AXIncrement":@"AXDecrement"; |
| 777 | NSInteger count=MAX(1,MIN(100,[args[@"amount"] integerValue])); |
| 778 | id before=attr(control,@"AXValue"); |
| 779 | BOOL advertised=[cuActions(control) containsObject:action]; |
| 780 | // Native scrollbars commonly expose a normalized value instead of actions. |
| 781 | // Report that unit explicitly: it is not a claim about a toolkit's line size. |
| 782 | BOOL normalized=!advertised && [before isKindOfClass:NSNumber.class] && [before doubleValue]>=0 && [before doubleValue]<=1 && cuSettable(control,@"AXValue"); |
| 783 | if(!advertised && !normalized) @throw [NSException exceptionWithName:@"background_scroll_unavailable" reason:@"the accessibility scrollbar has no supported action or writable normalized value" userInfo:nil]; |
| 784 | for(NSInteger i=0;i<(advertised?count:1);i++) { |
| 785 | cuCheckCancelled(); |
| 786 | NSNumber *value=@(MAX(0,MIN(1,[before doubleValue]+(forward?1:-1)*0.05*count))); |
| 787 | AXError error=advertised?AXUIElementPerformAction(control,(__bridge CFStringRef)action):AXUIElementSetAttributeValue(control,kAXValueAttribute,(__bridge CFTypeRef)value); |
| 788 | if(error!=kAXErrorSuccess) @throw [NSException exceptionWithName:@"action" reason:[NSString stringWithFormat:@"accessibility scroll failed: %d; no pointer fallback was sent",error] userInfo:nil]; |
| 789 | } |
| 790 | id after=attr(control,@"AXValue"); |
| 791 | return @{@"action_sent":@YES,@"strategy":@"a11y",@"pointer_moved":@NO,@"action":advertised?action:@"AXValue", |
| 792 | @"unit":advertised?@"accessibility_increment":@"normalized_scrollbar",@"before":before?:NSNull.null,@"after":after?:NSNull.null, |
| 793 | @"verified":@(before && after && ![before isEqual:after])}; |
| 794 | } |
| 795 | /** |
| 796 | * Smallest pressable element whose frame contains p. |
| 797 | * |
| 798 | * AXUIElementCopyElementAtPosition is the first resolver, but several toolkits |
| 799 | * (Chromium's browser process among them) answer it with the window rather |
| 800 | * than the control the user sees, so a coordinate would silently degrade to a |
| 801 | * raw event. Searching the subtree geometrically recovers the real target; |
| 802 | * "smallest containing" is what picks the button instead of its group. Bounded |
| 803 | * so a huge tree cannot stall an action. |
| 804 | */ |
| 805 | static void cuSearch(AXUIElementRef el, CGPoint p, int depth, int *budget, id *best, double *bestArea, NSString *operation) { |
| 806 | if(depth>24 || (*budget)--<=0) return; |
| 807 | CGRect frame; |
| 808 | if(cuFrame(el,&frame)) { |
| 809 | // Children are laid out inside their parent (and clipped when they are |
| 810 | // not), so a frame that misses the point prunes the whole subtree. |
| 811 | if(!CGRectContainsPoint(frame,p)) return; |
| 812 | double area=frame.size.width*frame.size.height; |
| 813 | BOOL suitable=[operation hasPrefix:@"scroll"]?cuScrollBar(el,[operation isEqual:@"scroll-horizontal"])!=nil:cuClickAction(el,[operation isEqual:@"context"])!=nil; |
| 814 | if(suitable && (!*best || area<=*bestArea)) { *best=(__bridge id)el; *bestArea=area; } |
| 815 | } |
| 816 | for(id kid in attr(el,@"AXChildren")) cuSearch((__bridge AXUIElementRef)kid,p,depth+1,budget,best,bestArea,operation); |
| 817 | } |
| 818 | /** |
| 819 | * Every key an app_ref supplies must match. Matching any one of them would let |
| 820 | * {pid, bundle_id} land on a *different* process of the same bundle — the |
| 821 | * user's own browser instead of the one the agent opened — and then type into |
| 822 | * their window. Identity here is a conjunction, deliberately. |
| 823 | */ |
| 824 | static BOOL matchesName(NSString *have, NSString *want) { |
| 825 | return have && [have caseInsensitiveCompare:want]==NSOrderedSame; |
| 826 | } |
| 827 | /** |
| 828 | * Bring an application forward. -[NSRunningApplication activateWithOptions:] |
| 829 | * is ignored on macOS 14+ when the caller is not itself frontmost, which a |
| 830 | * background helper never is. The WindowServer's own front-process channel |
| 831 | * (0x200 = raising) works from any TCC-trusted process; setting AXFrontmost |
| 832 | * through the Accessibility grant is the fallback. |
| 833 | */ |
| 834 | static BOOL axActivate(pid_t pid) { |
| 835 | if(cuResolveBgPointer()) { |
| 836 | ProcessSerialNumber psn; |
| 837 | if(cuGetPSN(pid,&psn)==0 && cuSetFront(&psn,0,0x200)==0) return YES; |
| 838 | } |
| 839 | AXUIElementRef app=AXUIElementCreateApplication(pid); |
| 840 | AXError e=AXUIElementSetAttributeValue(app,kAXFrontmostAttribute,kCFBooleanTrue); |
| 841 | CFRelease(app); |
| 842 | return e==kAXErrorSuccess; |
| 843 | } |
| 844 | static NSRunningApplication *resolve(NSDictionary *ref) { |
| 845 | // Only omission selects the frontmost app. An explicit but malformed |
| 846 | // identity must never redirect observation or input to the user's app. |
| 847 | if(!ref) return NSWorkspace.sharedWorkspace.frontmostApplication; |
| 848 | if(![ref isKindOfClass:NSDictionary.class] || !ref.count) return nil; |
| 849 | for(id key in ref) { |
| 850 | id value=ref[key]; |
| 851 | if([key isEqual:@"pid"]) { |
| 852 | if(![value isKindOfClass:NSNumber.class] || CFGetTypeID((__bridge CFTypeRef)value)==CFBooleanGetTypeID() |
| 853 | || [value doubleValue]<=0 || [value doubleValue]>INT_MAX || [value doubleValue]!=[value intValue]) return nil; |
| 854 | } else if([key isEqual:@"name"] || [key isEqual:@"bundle_id"]) { |
| 855 | if(![value isKindOfClass:NSString.class] || ![value stringByTrimmingCharactersInSet:NSCharacterSet.whitespaceAndNewlineCharacterSet].length) return nil; |
| 856 | } else return nil; |
| 857 | } |
| 858 | NSString *bundle=ref[@"bundle_id"], *name=ref[@"name"]; |
| 859 | // A name or bundle shared by several running processes (two instances of |
| 860 | // one browser, a helper and its app) must not resolve to whichever happens |
| 861 | // to be listed first: that sends observation and input to the wrong window. |
| 862 | // Refuse like kill_app does; the observed pid disambiguates. |
| 863 | NSMutableArray<NSRunningApplication *> *hits=[NSMutableArray array]; |
| 864 | for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) { |
| 865 | if(ref[@"pid"] && a.processIdentifier!=[ref[@"pid"] intValue]) continue; |
| 866 | if(bundle && !matchesName(a.bundleIdentifier,bundle)) continue; |
| 867 | if(name && !matchesName(a.localizedName,name)) continue; |
| 868 | [hits addObject:a]; |
| 869 | } |
| 870 | if(hits.count>1) { |
| 871 | NSMutableArray *desc=[NSMutableArray array]; |
| 872 | for(NSRunningApplication *a in hits) [desc addObject:[NSString stringWithFormat:@"%@ (pid %d)",a.localizedName?:@"?",a.processIdentifier]]; |
| 873 | @throw [NSException exceptionWithName:@"app" reason:[NSString stringWithFormat:@"several running applications match (%@); pass pid to choose one",[desc componentsJoinedByString:@", "]] userInfo:nil]; |
| 874 | } |
| 875 | return hits.firstObject; |
| 876 | } |
| 877 | static CGEventRef textEvent(NSString *text, BOOL down) { |
| 878 | UniChar *chars=calloc(text.length,sizeof(UniChar)); [text getCharacters:chars range:NSMakeRange(0,text.length)]; |
| 879 | CGEventRef event=CGEventCreateKeyboardEvent(NULL,0,down); |
| 880 | #ifdef CU_TEST |
| 881 | // Simulate physical modifier state without posting a system key event. |
| 882 | CGEventSetFlags(event,cuTestInheritedTextFlags); |
| 883 | #endif |
| 884 | // Literal text must not inherit the user's held Command/Control/Option/Shift. |
| 885 | CGEventSetFlags(event,0); |
| 886 | CGEventKeyboardSetUnicodeString(event,text.length,chars); free(chars); return event; |
| 887 | } |
| 888 | static BOOL cuTextRole(NSString *role) { |
| 889 | return [@[@"AXTextField",@"AXTextArea",@"AXComboBox",@"AXSearchField",@"AXSecureTextField",@"AXWebArea"] containsObject:role]; |
| 890 | } |
| 891 | /** |
| 892 | * Whether an element lives inside a browser/webview subtree. Chromium accepts |
| 893 | * AXSelectedText and AXValue writes on web controls and then ignores them — |
| 894 | * or, worse, a numeric control coerces the write to empty. Web elements get |
| 895 | * real keystrokes (after AXFocused) instead of semantic writes. |
| 896 | */ |
| 897 | static BOOL axHasWebAncestor(AXUIElementRef el) { |
| 898 | // `node` stays an `id` so ARC keeps each ancestor alive through the walk — |
| 899 | // a raw AXUIElementRef would dangle the moment `parent` is reassigned. |
| 900 | id node=(__bridge id)el; |
| 901 | for(int depth=0;node && depth<64;depth++) { |
| 902 | NSString *role=attr((__bridge AXUIElementRef)node,@"AXRole"); |
| 903 | if([role isEqual:@"AXWebArea"]) return YES; |
| 904 | id parent=attr((__bridge AXUIElementRef)node,@"AXParent"); |
| 905 | if(!parent || CFEqual((CFTypeRef)parent,(CFTypeRef)node)) break; |
| 906 | node=parent; |
| 907 | } |
| 908 | return NO; |
| 909 | } |
| 910 | // Without a readable selection range only an exact append can be verified. |
| 911 | // Matching length or an already-present suffix is not evidence of delivery. |
| 912 | static BOOL cuTypeVerified(NSString *before, NSString *after, NSString *text) { |
| 913 | return before && after && [after isEqual:[before stringByAppendingString:text]]; |
| 914 | } |
| 915 | static id cuFocusedElement(pid_t pid) { |
| 916 | AXUIElementRef appEl=AXUIElementCreateApplication(pid); |
| 917 | AXUIElementSetMessagingTimeout(appEl,2.0); |
| 918 | axPrepare(appEl); |
| 919 | id focused=attr(appEl,@"AXFocusedUIElement"); |
| 920 | CFRelease(appEl); |
| 921 | return focused; |
| 922 | } |
| 923 | /** |
| 924 | * Resolve a {windowIndex, path} element target inside the app's AX tree. |
| 925 | * Returns nil when the window or any path step no longer exists — the same |
| 926 | * staleness contract as the element-action tools. |
| 927 | */ |
| 928 | static id cuResolvePathTarget(pid_t pid, NSDictionary *t) { |
| 929 | AXUIElementRef appEl = AXUIElementCreateApplication(pid); |
| 930 | AXUIElementSetMessagingTimeout(appEl, 2.0); |
| 931 | axPrepare(appEl); |
| 932 | NSArray *ws = attr(appEl, @"AXWindows") ?: @[]; |
| 933 | NSInteger wi = [t[@"windowIndex"] integerValue]; |
| 934 | id el = wi == -1 ? attr(appEl, @"AXMenuBar") : wi == -2 ? (__bridge id)appEl : (wi >= 0 && wi < ws.count ? ws[wi] : nil); |
| 935 | for(NSNumber *i in t[@"path"] ?: @[]) { |
| 936 | NSArray *kids = el ? attr((__bridge AXUIElementRef)el, @"AXChildren") : nil; |
| 937 | if(i.unsignedIntegerValue >= kids.count) { el = nil; break; } |
| 938 | el = kids[i.unsignedIntegerValue]; |
| 939 | } |
| 940 | CFRelease(appEl); |
| 941 | return el; |
| 942 | } |
| 943 | /** |
| 944 | * Type into whatever holds focus in the bound app, then prove it landed. |
| 945 | * Dispatch succeeding is not delivery (a process with no text receiver drops |
| 946 | * the events silently), so the receipt reports `verified` from the focused |
| 947 | * control's own value. Failure to verify is reported, not thrown — the events |
| 948 | * already went out. The one throw is before any event is posted: a focused |
| 949 | * element that is clearly not a text control. |
| 950 | */ |
| 951 | static NSDictionary *cuType(NSDictionary *args, NSRunningApplication *inputApp, id focused, BOOL simulated) { |
| 952 | NSString *text=args[@"text"]; |
| 953 | if(![text isKindOfClass:NSString.class]) @throw [NSException exceptionWithName:@"text" reason:@"text must be a string" userInfo:nil]; |
| 954 | NSString *role=focused?attr((__bridge AXUIElementRef)focused,@"AXRole"):nil; |
| 955 | BOOL secure=[role isEqual:@"AXSecureTextField"]; |
| 956 | NSString *before=nil; |
| 957 | // A secure field's value is never read; it verifies as unverifiable. |
| 958 | if(focused && !secure) { id v=attr((__bridge AXUIElementRef)focused,@"AXValue"); if([v isKindOfClass:NSString.class]) before=v; } |
| 959 | // Fail closed only on strong evidence: something holds focus and it is |
| 960 | // clearly not text. No focused element at all still receives the events — |
| 961 | // some apps take process-directed keys without reporting AX focus. |
| 962 | if(focused && !cuTextRole(role) && !before) |
| 963 | @throw [NSException exceptionWithName:@"focus" reason:[NSString stringWithFormat:@"focused element is a %@, not a text control — click or focus a text field first",role?:@"unknown element"] userInfo:nil]; |
| 964 | NSString *expected=nil; |
| 965 | if(before && focused) { |
| 966 | id range=attr((__bridge AXUIElementRef)focused,@"AXSelectedTextRange"); CFRange selected; |
| 967 | if(range && CFGetTypeID((__bridge CFTypeRef)range)==AXValueGetTypeID() && AXValueGetValue((__bridge AXValueRef)range,kAXValueCFRangeType,&selected) |
| 968 | && selected.location>=0 && selected.length>=0 && selected.location<=before.length && selected.length<=before.length-selected.location) |
| 969 | expected=[before stringByReplacingCharactersInRange:NSMakeRange(selected.location,selected.length) withString:text]; |
| 970 | } |
| 971 | // delivery:"events" forces the real keystroke stream. Service-backed |
| 972 | // fields (a hosted panel's path box) accept an AXSelectedText write and |
| 973 | // then overwrite it from their own model — the write verifies at read |
| 974 | // time and reverts on commit. Real key events reach the field's editor. |
| 975 | BOOL semantic=!simulated && focused && ![args[@"foreground_input"] boolValue] && ![args[@"delivery"] isEqual:@"events"] && cuSettable((__bridge AXUIElementRef)focused,@"AXSelectedText") |
| 976 | && !axHasWebAncestor((__bridge AXUIElementRef)focused); |
| 977 | if(semantic) { |
| 978 | cuCheckCancelled(); |
| 979 | AXError error=AXUIElementSetAttributeValue((__bridge AXUIElementRef)focused,kAXSelectedTextAttribute,(__bridge CFStringRef)text); |
| 980 | if(error!=kAXErrorSuccess) @throw [NSException exceptionWithName:@"action" reason:[NSString stringWithFormat:@"accessibility text insertion failed: %d; observe before retrying; no keyboard fallback was sent",error] userInfo:nil]; |
| 981 | } |
| 982 | // One grapheme per event, the way a keyboard delivers them. Batching |
| 983 | // several into one CGEventKeyboardSetUnicodeString is faster but Electron |
| 984 | // apps coalesce the pending payload and keep only the final batch, so a |
| 985 | // typed string silently arrives truncated to its tail. |
| 986 | // |
| 987 | // Astral graphemes (surrogate pairs, emoji, flags, ZWJ sequences) are |
| 988 | // dropped by the WindowServer's key translation whenever the target window |
| 989 | // is not front-and-visible — measured: occluded Chrome keeps every BMP |
| 990 | // grapheme and loses 🐳. The window-record channel delivers the real event |
| 991 | // instead. The trigger is the text, not an occlusion guess: if the string |
| 992 | // carries any multi-unit grapheme, the whole stream rides the record |
| 993 | // channel under one lease. Background mode refuses that lease before input; |
| 994 | // it cannot promise reliable astral text delivery through a focus swap. |
| 995 | uint32_t typeWin = 0; |
| 996 | CGRect typeFrame = CGRectZero; |
| 997 | pid_t typeOwner = 0; |
| 998 | BOOL needsRecord = NO; |
| 999 | if(!simulated && !semantic && focused && ![args[@"foreground_input"] boolValue]) { |
| 1000 | for(NSUInteger i = 0; i < text.length && !needsRecord && cuResolveBgPointer();) { |
| 1001 | NSRange r = [text rangeOfComposedCharacterSequencesForRange:NSMakeRange(i, 1)]; |
| 1002 | if(r.length > 1) needsRecord = YES; |
| 1003 | i = NSMaxRange(r); |
| 1004 | } |
| 1005 | // The focused element may live in a hosted panel whose window belongs to |
| 1006 | // a service, not the app — resolve its real window and owner either way, |
| 1007 | // so process-posted text also reaches the right queue. |
| 1008 | if(cuElementWindow(focused, inputApp.processIdentifier, &typeWin, &typeOwner, nil) |
| 1009 | && cuWindowInfoForNumber(typeWin, &typeFrame, nil)) { |
| 1010 | // A service-owned window's queue is where the panel's field lives; |
| 1011 | // process posting to the app would drop the text. The record route |
| 1012 | // under a lease delivers it and supplies key status. |
| 1013 | if(typeOwner && typeOwner != inputApp.processIdentifier) needsRecord = YES; |
| 1014 | } |
| 1015 | } |
| 1016 | cuBgLease lease = {0}; |
| 1017 | BOOL leasing = NO; |
| 1018 | BOOL typeRestored = YES; |
| 1019 | if(needsRecord && typeWin && cuResolveBgPointer()) { |
| 1020 | NSString *why = nil; |
| 1021 | leasing = cuBgLeaseBegin(inputApp, typeWin, YES, args, &lease, &why); |
| 1022 | } |
| 1023 | // Foreground keystrokes share the user's keyboard: wait for a hardware- |
| 1024 | // input gap once before the stream, not per grapheme — the stream itself |
| 1025 | // is already paced like a fast typist. |
| 1026 | double yieldMs=(!simulated && !semantic && [args[@"foreground_input"] boolValue]) ? cuYieldToUser(args) : 0; |
| 1027 | @try { |
| 1028 | for(NSUInteger i=0;!semantic && i<text.length && !cuCancelled;) { |
| 1029 | if([args[@"foreground_input"] boolValue]) cuRequireForeground(inputApp); |
| 1030 | cuCheckCancelled(); |
| 1031 | NSRange range=[text rangeOfComposedCharacterSequencesForRange:NSMakeRange(i,1)]; |
| 1032 | NSString *chunk=[text substringWithRange:range]; |
| 1033 | if(!simulated) for(int down=1;down>=0;down--) { |
| 1034 | CGEventRef event=textEvent(chunk,down); |
| 1035 | if([args[@"foreground_input"] boolValue]) CGEventPost(kCGHIDEventTap,event); |
| 1036 | else if(leasing) cuPostEventRecord(&lease, event, CGPointMake(CGRectGetMidX(typeFrame) - typeFrame.origin.x, CGRectGetMidY(typeFrame) - typeFrame.origin.y)); |
| 1037 | else CGEventPostToPid(typeOwner ? typeOwner : inputApp.processIdentifier,event); |
| 1038 | CFRelease(event); |
| 1039 | } |
| 1040 | i=NSMaxRange(range); usleep(10000); |
| 1041 | } |
| 1042 | } @finally { if(leasing) typeRestored = cuBgLeaseEnd(&lease); } |
| 1043 | if(cuCancelled) @throw [NSException exceptionWithName:@"cancelled" reason:@"computer request cancelled" userInfo:nil]; |
| 1044 | NSString *after=nil; |
| 1045 | if(focused && !secure) { |
| 1046 | #ifdef CU_TEST |
| 1047 | if(simulated) { NSString *s=((NSMutableDictionary *)focused)[@"after"]; if(s) ((NSMutableDictionary *)focused)[@"AXValue"]=s; } |
| 1048 | else |
| 1049 | #endif |
| 1050 | usleep(80000); |
| 1051 | id v=attr((__bridge AXUIElementRef)focused,@"AXValue"); |
| 1052 | if([v isKindOfClass:NSString.class]) after=v; |
| 1053 | } |
| 1054 | BOOL verified=expected?[after isEqual:expected]:cuTypeVerified(before,after,text); |
| 1055 | NSMutableDictionary *receipt=[@{@"action_sent":@YES,@"chars":@(text.length),@"strategy":semantic?@"a11y-selected-text":@"unicode-events", |
| 1056 | @"keyboard_delivery":semantic?@"accessibility":[args[@"foreground_input"] boolValue]?@"foreground-guarded":leasing?@"window-record":@"process", |
| 1057 | @"verified":@(verified),@"focused_role":role?:[NSNull null]} mutableCopy]; |
| 1058 | if(leasing) receipt[@"window_focused"]=@YES; |
| 1059 | if(lease.swapped) receipt[@"front_restored"]=@(typeRestored); |
| 1060 | if(yieldMs>0) receipt[@"yield_ms"]=@(round(yieldMs)); |
| 1061 | cuLeaseAccounting(receipt,&lease); |
| 1062 | if(!verified) receipt[@"verification_required"]=@"screenshot"; |
| 1063 | return receipt; |
| 1064 | } |
| 1065 | static NSDictionary *windowAtPoint(NSArray *windows, CGPoint p) { |
| 1066 | NSMutableArray *skipped=[NSMutableArray array]; |
| 1067 | for(NSDictionary *w in windows) { // front to back |
| 1068 | CGRect b; |
| 1069 | if(!CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)w[(__bridge NSString *)kCGWindowBounds],&b)) continue; |
| 1070 | if(!CGRectContainsPoint(b,p)) continue; |
| 1071 | pid_t owner=[w[(__bridge NSString *)kCGWindowOwnerPID] intValue]; |
| 1072 | NSString *name=w[(__bridge NSString *)kCGWindowOwnerName]?:@""; |
| 1073 | NSNumber *alpha=w[(__bridge NSString *)kCGWindowAlpha], *layer=w[(__bridge NSString *)kCGWindowLayer]?:@0; |
| 1074 | // Visible floating windows occlude input just like normal windows. |
| 1075 | if(alpha && [alpha doubleValue]<=0) { [skipped addObject:@{@"owner":name,@"why":@"transparent"}]; continue; } |
| 1076 | return @{@"found":@YES,@"owner_pid":@(owner),@"owner_name":name, |
| 1077 | @"window_id":w[(__bridge NSString *)kCGWindowNumber]?:@0,@"layer":layer, |
| 1078 | @"skipped":skipped}; |
| 1079 | } |
| 1080 | return @{@"found":@NO,@"skipped":skipped}; |
| 1081 | } |
| 1082 | |
| 1083 | static id execute(NSDictionary *p) { |
| 1084 | NSString *tool=p[@"tool"]; NSDictionary *args=p[@"args"]?:@{}; |
| 1085 | // The user's hardware cursor is never driven: there is no route that posts |
| 1086 | // mouse events to the HID tap, warps the cursor or holds its buttons. |
| 1087 | if([@[@"pointer_sequence",@"release_input"] containsObject:tool]) |
| 1088 | @throw [NSException exceptionWithName:@"real_pointer_refused" reason:@"real_pointer_refused: Computer Use never drives the user's cursor; pointer input goes to the bound app's window (bg_pointer)" userInfo:nil]; |
| 1089 | if([@[@"bg_key",@"bg_pointer"] containsObject:tool]) cuRequireFocusControl(args); |
| 1090 | cuOwnerPipe=[args[@"owner_pipe"] boolValue]; |
| 1091 | BOOL mutates=[@[@"type",@"key_event",@"bg_key",@"mouse_event",@"scroll",@"bg_pointer",@"set_value",@"focus_element",@"select_text",@"perform_action",@"click_element",@"scroll_element"] containsObject:tool] |
| 1092 | || ([tool isEqual:@"hit_test"] && [args[@"perform"] boolValue]) |
| 1093 | || ([tool isEqual:@"app_info"] && [args[@"activate"] boolValue]); |
| 1094 | if([tool isEqual:@"key_event"] && ![args[@"down"] boolValue] && [args[@"owned_release"] boolValue]) { |
| 1095 | if(!AXIsProcessTrusted()) @throw [NSException exceptionWithName:@"permission" reason:@"Accessibility permission is missing" userInfo:nil]; |
| 1096 | cuLockInput(); |
| 1097 | // Release a confirmed/ambiguous press even if its original app has exited. |
| 1098 | return cuPostKey(args,[args[@"input_app_ref"][@"pid"] intValue]); |
| 1099 | } |
| 1100 | cuCheckCancelled(); |
| 1101 | if([tool isEqual:@"input_capabilities"]) return @{@"input_lease":@1,@"owner_pipe":@YES,@"record_owner_pipe":@1,@"window_ocr":@1,@"element_identity":@1,@"background_actions":@1,@"background_focus_guard":@1,@"window_record":@(cuResolveBgPointer()?1:0)}; |
| 1102 | if([tool isEqual:@"front_lease_watchdog"]) { |
| 1103 | long long remaining = [args[@"deadline"] longLongValue] - (long long)([NSDate new].timeIntervalSince1970 * 1000); |
| 1104 | if(remaining > 0 && remaining < 30000) usleep((useconds_t)remaining * 1000); |
| 1105 | NSData *data = [NSData dataWithContentsOfFile:cuFrontLeaseFile()]; |
| 1106 | NSDictionary *held = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil; |
| 1107 | if([held isKindOfClass:NSDictionary.class] && [held[@"token"] isEqual:args[@"token"]]) cuFrontLeaseRestoreIfHeld(); |
| 1108 | return @{@"ok":@YES}; |
| 1109 | } |
| 1110 | if([tool isEqual:@"record"]) return cuRecord(args); |
| 1111 | if([tool isEqual:@"recognize_text"]) return cuRecognizeText(args[@"file"]); |
| 1112 | #ifdef CU_TEST |
| 1113 | if([tool isEqual:@"inspect_focus_control"]) { cuRequireFocusControl(args); return @{@"allowed":@YES}; } |
| 1114 | if([tool isEqual:@"inspect_user_yield"]) { |
| 1115 | cuTestIdleSeconds=args[@"idle_seconds"]; |
| 1116 | if([args[@"cancelled"] boolValue]) cuCancelled=1; |
| 1117 | return @{@"yield_ms":@(cuYieldToUser(args))}; |
| 1118 | } |
| 1119 | if([tool isEqual:@"inspect_click_action"]) return @{@"action":cuClickAction((__bridge AXUIElementRef)args[@"element"],[args[@"context"] boolValue])?:NSNull.null}; |
| 1120 | if([tool isEqual:@"inspect_element_identity"]) { |
| 1121 | cuValidateElementIdentity((__bridge AXUIElementRef)args[@"element"],args[@"target"]); |
| 1122 | return @{@"identity_matches":@YES}; |
| 1123 | } |
| 1124 | if([tool isEqual:@"inspect_window_at_point"]) return windowAtPoint(args[@"windows"],CGPointMake([args[@"x"] doubleValue],[args[@"y"] doubleValue])); |
| 1125 | if([tool isEqual:@"inspect_window_match"]) { |
| 1126 | NSDictionary *b=args[@"bounds"]; |
| 1127 | CGRect bounds=CGRectMake([b[@"x"] doubleValue],[b[@"y"] doubleValue],[b[@"w"] doubleValue],[b[@"h"] doubleValue]); |
| 1128 | return capturableWindow(args[@"windows"],[args[@"pid"] intValue],@"Fixture",bounds); |
| 1129 | } |
| 1130 | if([tool isEqual:@"inspect_observation"]) { |
| 1131 | BOOL truncated=NO; |
| 1132 | NSArray *elements=observeElements((__bridge AXUIElementRef)args[@"app"],args[@"windows"]?:@[],args,NO,&truncated); |
| 1133 | return @{@"elements":elements,@"truncated":@(truncated)}; |
| 1134 | } |
| 1135 | if([tool isEqual:@"test_input_lease"]) { |
| 1136 | cuTestLockDir=args[@"lock_dir"]; cuLockInput(); |
| 1137 | cuTestReleaseFile=args[@"release_file"]; |
| 1138 | if([args[@"work_ms"] intValue]>0) { |
| 1139 | cuPrint(@{@"action_sent":@YES,@"input_lease":@YES}); |
| 1140 | for(int elapsed=0;elapsed<[args[@"work_ms"] intValue];elapsed+=20) { cuCheckCancelled(); usleep(20000); } |
| 1141 | } |
| 1142 | return @{@"action_sent":@YES}; |
| 1143 | } |
| 1144 | if([tool isEqual:@"inspect_text_event"]) { |
| 1145 | cuTestInheritedTextFlags=[args[@"inherited_flags"] unsignedLongLongValue]; |
| 1146 | CGEventRef event=textEvent(args[@"text"],YES); UniChar chars[4096]; UniCharCount length=0; |
| 1147 | CGEventKeyboardGetUnicodeString(event,4096,&length,chars); CGEventFlags flags=CGEventGetFlags(event); CFRelease(event); |
| 1148 | return @{@"text":[NSString stringWithCharacters:chars length:length],@"flags":@(flags)}; |
| 1149 | } |
| 1150 | // Drives the real typing logic against a fixture focused element instead of |
| 1151 | // a live app: `after` is the value the element reports once the text lands, |
| 1152 | // which a fixture omits to model an app that swallows the events. |
| 1153 | if([tool isEqual:@"inspect_type"]) { |
| 1154 | id fixture=args[@"focused"]; |
| 1155 | return cuType(args, nil, [fixture isKindOfClass:NSDictionary.class]?[fixture mutableCopy]:nil, YES); |
| 1156 | } |
| 1157 | #endif |
| 1158 | if([tool isEqual:@"permissions"]) return @{@"trusted":@(AXIsProcessTrusted())}; |
| 1159 | if([tool isEqual:@"list_apps"]) { |
| 1160 | NSMutableArray *apps=[NSMutableArray array]; |
| 1161 | for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) { |
| 1162 | NSInteger policy = a.activationPolicy; |
| 1163 | NSString *policyName = (policy >= 0 && policy <= 2) ? @[@"regular",@"accessory",@"prohibited"][policy] : @"unknown"; |
| 1164 | [apps addObject:@{@"name":a.localizedName?:@"",@"pid":@(a.processIdentifier),@"bundle_id":a.bundleIdentifier?:@"",@"frontmost":@(a.active),@"hidden":@(a.hidden),@"activation_policy":policyName}]; |
| 1165 | } |
| 1166 | return @{@"apps":apps}; |
| 1167 | } |
| 1168 | if([tool isEqual:@"displays"]) { |
| 1169 | uint32_t n=0; CGGetActiveDisplayList(0,NULL,&n); CGDirectDisplayID ids[n]; CGGetActiveDisplayList(n,ids,&n); |
| 1170 | NSMutableArray *out=[NSMutableArray array]; |
| 1171 | for(uint32_t i=0;i<n;i++){ CGRect b=CGDisplayBounds(ids[i]); CGDisplayModeRef mode=CGDisplayCopyDisplayMode(ids[i]); |
| 1172 | size_t w=CGDisplayModeGetPixelWidth(mode),h=CGDisplayModeGetPixelHeight(mode); CGDisplayModeRelease(mode); |
| 1173 | [out addObject:@{@"index":@(i+1),@"id":@(ids[i]),@"main":@(ids[i]==CGMainDisplayID()),@"points":@{@"x":@(b.origin.x),@"y":@(b.origin.y),@"w":@(b.size.width),@"h":@(b.size.height)},@"pixels":@{@"w":@(w),@"h":@(h)},@"scale":@(w/b.size.width)}]; } |
| 1174 | return out; |
| 1175 | } |
| 1176 | if([tool isEqual:@"preview_notify"]) { |
| 1177 | [[NSDistributedNotificationCenter defaultCenter] postNotificationName:@"net.codewhale.computer-use.preview" object:nil userInfo:args deliverImmediately:YES]; |
| 1178 | return @{@"updated":@YES}; |
| 1179 | } |
| 1180 | if([tool isEqual:@"window_info"]) { |
| 1181 | NSRunningApplication *a=resolve(args[@"app_ref"]?:args[@"input_app_ref"]); |
| 1182 | if(!a) @throw [NSException exceptionWithName:@"app" reason:@"application not found" userInfo:nil]; |
| 1183 | AXUIElementRef ax=AXUIElementCreateApplication(a.processIdentifier); |
| 1184 | axPrepare(ax); |
| 1185 | NSArray *axWindows=attr(ax,@"AXWindows"); |
| 1186 | NSInteger index=[args[@"window_id"] integerValue]; |
| 1187 | CGRect preferred; |
| 1188 | BOOL integerIndex=!args[@"window_id"] || ([args[@"window_id"] isKindOfClass:NSNumber.class] && [args[@"window_id"] doubleValue]==index); |
| 1189 | BOOL valid=integerIndex && index>=0 && index<axWindows.count && cuFrame((__bridge AXUIElementRef)axWindows[index],&preferred); |
| 1190 | CFRelease(ax); |
| 1191 | if(!valid) @throw [NSException exceptionWithName:@"window" reason:@"the selected app window has no accessibility geometry; call list_windows for a valid window index" userInfo:nil]; |
| 1192 | NSArray *windows=CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionAll,kCGNullWindowID)); |
| 1193 | return capturableWindow(windows,a.processIdentifier,a.localizedName,preferred); |
| 1194 | } |
| 1195 | // Which application owns the point a pointer event would land on. A global |
| 1196 | // pointer event goes to whatever is on top, so this is what stops a click |
| 1197 | // meant for the agent's app from landing in the user's window. |
| 1198 | if([tool isEqual:@"window_at_point"]) { |
| 1199 | CGPoint p=CGPointMake([args[@"x"] doubleValue],[args[@"y"] doubleValue]); |
| 1200 | NSArray *windows=CFBridgingRelease(CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly|kCGWindowListExcludeDesktopElements,kCGNullWindowID)); |
| 1201 | return windowAtPoint(windows,p); |
| 1202 | } |
| 1203 | if([tool isEqual:@"app_info"]) { |
| 1204 | NSRunningApplication *a=resolve(args[@"app_ref"]); |
| 1205 | if(!a) @throw [NSException exceptionWithName:@"app" reason:@"application not found" userInfo:nil]; |
| 1206 | if([a.bundleIdentifier isEqual:@"net.codewhale.computer-use"] && [args[@"activate"] boolValue]) @throw [NSException exceptionWithName:@"protected" reason:@"Computer Use safety controls belong to the user." userInfo:nil]; |
| 1207 | double yieldMs=0; |
| 1208 | if([args[@"activate"] boolValue]) { |
| 1209 | cuLockInput(); |
| 1210 | // Activation takes the person's foreground — wait for a hardware- |
| 1211 | // input gap first so a mid-type activation cannot swallow their |
| 1212 | // next keystrokes. |
| 1213 | yieldMs=cuYieldToUser(args); |
| 1214 | } |
| 1215 | cuCheckCancelled(); |
| 1216 | if([args[@"activate"] boolValue] && !axActivate(a.processIdentifier)) [a activateWithOptions:0]; |
| 1217 | if([args[@"activate"] boolValue]) for(int i=0;i<120;i++) { |
| 1218 | cuCheckCancelled(); |
| 1219 | // NSWorkspace only refreshes its frontmost view through run-loop |
| 1220 | // notifications; a one-shot helper that never services them reads a |
| 1221 | // stale answer for seconds and misreports a working activation. |
| 1222 | [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.025]]; |
| 1223 | if(NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier==a.processIdentifier) break; |
| 1224 | } |
| 1225 | NSMutableDictionary *info=[@{@"found":@YES,@"name":a.localizedName?:@"",@"pid":@(a.processIdentifier),@"bundle_id":a.bundleIdentifier?:@"",@"frontmost":@(a.active)} mutableCopy]; |
| 1226 | if(yieldMs>0) info[@"yield_ms"]=@(round(yieldMs)); |
| 1227 | return info; |
| 1228 | } |
| 1229 | if([tool isEqual:@"kill_app"]) { |
| 1230 | NSString *bundle=args[@"bundle_id"], *name=args[@"name"]; NSNumber *pidNum=args[@"pid"]; |
| 1231 | if(!bundle && !name && !pidNum) @throw [NSException exceptionWithName:@"args" reason:@"kill_app needs name, bundle_id or pid" userInfo:nil]; |
| 1232 | if(pidNum && (![pidNum isKindOfClass:NSNumber.class] || [pidNum doubleValue]<=0 || [pidNum doubleValue]>INT_MAX || [pidNum doubleValue]!=[pidNum intValue])) @throw [NSException exceptionWithName:@"args" reason:@"kill_app pid must be a positive integer" userInfo:nil]; |
| 1233 | // A name that matches two running apps must not guess which one to end. |
| 1234 | NSMutableArray *hits=[NSMutableArray array]; |
| 1235 | for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) { |
| 1236 | if(pidNum && a.processIdentifier!=[pidNum intValue]) continue; |
| 1237 | if(bundle && !matchesName(a.bundleIdentifier?:@"",bundle)) continue; |
| 1238 | if(name && !matchesName(a.localizedName?:@"",name)) continue; |
| 1239 | [hits addObject:a]; |
| 1240 | } |
| 1241 | if(!hits.count) @throw [NSException exceptionWithName:@"app" reason:@"application not found" userInfo:nil]; |
| 1242 | if(hits.count>1) { |
| 1243 | NSMutableArray *desc=[NSMutableArray array]; |
| 1244 | for(NSRunningApplication *a in hits) [desc addObject:[NSString stringWithFormat:@"%@ (pid %d)",a.localizedName?:@"?",a.processIdentifier]]; |
| 1245 | @throw [NSException exceptionWithName:@"app" reason:[NSString stringWithFormat:@"several running applications match (%@); pass pid to choose one",[desc componentsJoinedByString:@", "]] userInfo:nil]; |
| 1246 | } |
| 1247 | NSRunningApplication *target=hits[0]; |
| 1248 | pid_t tp=target.processIdentifier; |
| 1249 | // The helper, its host (the daemon or MCP server), and the app bundle that |
| 1250 | // owns this process must never be terminable through the agent surface. |
| 1251 | if([(target.bundleIdentifier?:@"") isEqual:@"net.codewhale.computer-use"] || tp==getpid() || tp==getppid()) |
| 1252 | @throw [NSException exceptionWithName:@"protected" reason:@"the Computer Use helper and its host cannot be terminated by this plugin" userInfo:nil]; |
| 1253 | [target terminate]; |
| 1254 | for(int i=0;i<60 && !target.isTerminated;i++) { cuCheckCancelled(); [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; } |
| 1255 | BOOL forced=NO; |
| 1256 | if(!target.isTerminated && [args[@"force"] boolValue]) { |
| 1257 | [target forceTerminate]; forced=YES; |
| 1258 | for(int i=0;i<40 && !target.isTerminated;i++) { cuCheckCancelled(); [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; } |
| 1259 | } |
| 1260 | return @{@"killed":@(target.isTerminated),@"pid":@(tp),@"name":target.localizedName?:@"",@"force_used":@(forced)}; |
| 1261 | } |
| 1262 | if([tool isEqual:@"installed_apps"]) { |
| 1263 | // Installed catalog: the apps a person could open, running or not. Root + |
| 1264 | // one level of subdirectories (e.g. /Applications/Utilities); bundle |
| 1265 | // identity comes from the bundle itself, never from the directory name. |
| 1266 | NSMutableDictionary *running=@{}.mutableCopy; |
| 1267 | for(NSRunningApplication *a in NSWorkspace.sharedWorkspace.runningApplications) { |
| 1268 | if(a.bundleIdentifier) running[a.bundleIdentifier]=@(a.processIdentifier); |
| 1269 | } |
| 1270 | NSMutableDictionary *byId=@{}.mutableCopy; |
| 1271 | NSFileManager *fm=NSFileManager.defaultManager; |
| 1272 | NSArray *roots=@[@"/Applications", @"/System/Applications", [NSHomeDirectory() stringByAppendingPathComponent:@"Applications"]]; |
| 1273 | for(NSString *root in roots) { |
| 1274 | cuCheckCancelled(); |
| 1275 | NSString *top=[root stringByResolvingSymlinksInPath]; |
| 1276 | NSMutableArray *dirs=[NSMutableArray array]; [dirs addObject:top]; |
| 1277 | for(NSString *sub in ([fm contentsOfDirectoryAtPath:top error:nil]?:@[])) { |
| 1278 | if([sub hasPrefix:@"."]||[sub hasSuffix:@".app"]) continue; |
| 1279 | NSString *p=[top stringByAppendingPathComponent:sub]; |
| 1280 | BOOL isDir=NO; |
| 1281 | if([fm fileExistsAtPath:p isDirectory:&isDir] && isDir) [dirs addObject:p]; |
| 1282 | } |
| 1283 | for(NSString *dir in dirs) { |
| 1284 | for(NSString *item in ([fm contentsOfDirectoryAtPath:dir error:nil]?:@[])) { |
| 1285 | if(![item hasSuffix:@".app"]) continue; |
| 1286 | NSString *p=[dir stringByAppendingPathComponent:item]; |
| 1287 | NSBundle *b=[NSBundle bundleWithPath:p]; |
| 1288 | NSString *bid=b.bundleIdentifier; |
| 1289 | if(!bid || byId[bid]) continue; |
| 1290 | NSString *name=[b objectForInfoDictionaryKey:@"CFBundleDisplayName"]; |
| 1291 | if(!name.length) name=[b objectForInfoDictionaryKey:@"CFBundleName"]; |
| 1292 | if(!name.length) name=[item stringByDeletingPathExtension]; |
| 1293 | byId[bid]=@{@"name":name,@"bundle_id":bid,@"path":p}; |
| 1294 | } |
| 1295 | } |
| 1296 | } |
| 1297 | NSMutableArray *out=[NSMutableArray array]; |
| 1298 | for(NSString *bid in byId) { |
| 1299 | NSMutableDictionary *e=[byId[bid] mutableCopy]; |
| 1300 | NSNumber *pid=running[bid]; |
| 1301 | e[@"running"]=(pid!=nil)?@YES:@NO; |
| 1302 | if(pid) e[@"pid"]=pid; |
| 1303 | [out addObject:e]; |
| 1304 | } |
| 1305 | [out sortUsingComparator:^NSComparisonResult(NSDictionary *a, NSDictionary *b){ return [a[@"name"] localizedCaseInsensitiveCompare:b[@"name"]]; }]; |
| 1306 | return @{@"apps":out,@"count":@(out.count)}; |
| 1307 | } |
| 1308 | if([tool isEqual:@"set_window_frame"]) { |
| 1309 | NSRunningApplication *a=resolve(args[@"app_ref"]); |
| 1310 | if(!a) @throw [NSException exceptionWithName:@"app" reason:@"application not found" userInfo:nil]; |
| 1311 | NSDictionary *frame=args[@"frame"]; |
| 1312 | double fx=NAN,fy=NAN,fw=NAN,fh=NAN; |
| 1313 | if([frame isKindOfClass:NSDictionary.class]) { |
| 1314 | fx=[frame[@"x"] doubleValue]; fy=[frame[@"y"] doubleValue]; |
| 1315 | fw=[frame[@"w"] doubleValue]; fh=[frame[@"h"] doubleValue]; |
| 1316 | } |
| 1317 | if(!isfinite(fx)||!isfinite(fy)||!isfinite(fw)||!isfinite(fh)||fw<=0||fh<=0) |
| 1318 | @throw [NSException exceptionWithName:@"args" reason:@"set_window_frame needs frame {x,y,w,h} with positive w/h" userInfo:nil]; |
| 1319 | NSNumber *idxNum=args[@"window_id"]; |
| 1320 | if(![idxNum isKindOfClass:NSNumber.class] || [idxNum doubleValue]!=[idxNum intValue] || [idxNum intValue]<0) |
| 1321 | @throw [NSException exceptionWithName:@"args" reason:@"set_window_frame needs window_id (a non-negative window index from list_windows)" userInfo:nil]; |
| 1322 | AXUIElementRef app=AXUIElementCreateApplication(a.processIdentifier); |
| 1323 | axPrepare(app); |
| 1324 | NSArray *windows=attr(app,@"AXWindows"); |
| 1325 | NSInteger idx=[idxNum intValue]; |
| 1326 | if(idx>=(NSInteger)windows.count) { CFRelease(app); @throw [NSException exceptionWithName:@"window" reason:@"window_id is out of range; call list_windows for valid indices" userInfo:nil]; } |
| 1327 | AXUIElementRef win=(__bridge AXUIElementRef)windows[idx]; |
| 1328 | CGRect before=CGRectNull; cuFrame(win,&before); |
| 1329 | cuCheckCancelled(); |
| 1330 | CGPoint p=CGPointMake(fx,fy); CGSize z=CGSizeMake(fw,fh); |
| 1331 | AXValueRef pos=AXValueCreate(kAXValueCGPointType,&p), size=AXValueCreate(kAXValueCGSizeType,&z); |
| 1332 | AXError pe=AXUIElementSetAttributeValue(win,(__bridge CFStringRef)@"AXPosition",pos); |
| 1333 | AXError se=AXUIElementSetAttributeValue(win,(__bridge CFStringRef)@"AXSize",size); |
| 1334 | // Some apps re-anchor a window's origin when its size changes; re-assert |
| 1335 | // the position once after the size has had a run-loop turn to settle. |
| 1336 | if(pe==kAXErrorSuccess) { |
| 1337 | [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; |
| 1338 | AXError pe2=AXUIElementSetAttributeValue(win,(__bridge CFStringRef)@"AXPosition",pos); |
| 1339 | if(pe2!=kAXErrorSuccess) pe=pe2; |
| 1340 | } |
| 1341 | if(pos) CFRelease(pos); if(size) CFRelease(size); |
| 1342 | if(pe!=kAXErrorSuccess && se!=kAXErrorSuccess) { |
| 1343 | CFRelease(app); |
| 1344 | @throw [NSException exceptionWithName:@"window" reason:@"the app refused the window frame change (it may be fullscreen, tiled or non-resizable)" userInfo:nil]; |
| 1345 | } |
| 1346 | // Apps apply frame changes over a few run-loop turns; verify by reading the |
| 1347 | // window's own geometry back, not by trusting the set call. |
| 1348 | CGRect after=before; |
| 1349 | for(int i=0;i<40;i++) { |
| 1350 | cuCheckCancelled(); |
| 1351 | [NSRunLoop.currentRunLoop runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; |
| 1352 | cuFrame(win,&after); |
| 1353 | if(fabs(after.origin.x-fx)<1 && fabs(after.origin.y-fy)<1 && fabs(after.size.width-fw)<1 && fabs(after.size.height-fh)<1) break; |
| 1354 | } |
| 1355 | CFRelease(app); |
| 1356 | BOOL verified = fabs(after.origin.x-fx)<1 && fabs(after.origin.y-fy)<1 && fabs(after.size.width-fw)<1 && fabs(after.size.height-fh)<1; |
| 1357 | NSMutableDictionary *done=[@{@"action_sent":@YES,@"window_id":@(idx), |
| 1358 | @"before":@{@"x":@(before.origin.x),@"y":@(before.origin.y),@"w":@(before.size.width),@"h":@(before.size.height)}, |
| 1359 | @"after":@{@"x":@(after.origin.x),@"y":@(after.origin.y),@"w":@(after.size.width),@"h":@(after.size.height)}, |
| 1360 | @"verified":@(verified)} mutableCopy]; |
| 1361 | if(pe!=kAXErrorSuccess || se!=kAXErrorSuccess) { |
| 1362 | done[@"ax_errors"]=@{@"position":@(pe),@"size":@(se)}; |
| 1363 | done[@"note"]=@"the app constrained or refused part of the frame (minimum sizes and fixed-size windows are common); the after readback is what actually happened"; |
| 1364 | } |
| 1365 | return done; |
| 1366 | } |
| 1367 | NSRunningApplication *inputApp=nil; |
| 1368 | if([@[@"type",@"key_event",@"bg_key",@"mouse_event",@"scroll",@"hit_test",@"bg_pointer"] containsObject:tool]) { |
| 1369 | if(![args[@"input_app_ref"] isKindOfClass:NSDictionary.class]) @throw [NSException exceptionWithName:@"focus" reason:@"open_application first to bind the input destination" userInfo:nil]; |
| 1370 | inputApp=resolve(args[@"input_app_ref"]); |
| 1371 | if(!inputApp || inputApp.terminated) @throw [NSException exceptionWithName:@"focus" reason:@"input application is no longer running; open_application again" userInfo:nil]; |
| 1372 | if([inputApp.bundleIdentifier isEqual:@"net.codewhale.computer-use"]) @throw [NSException exceptionWithName:@"protected" reason:@"Computer Use safety controls belong to the user." userInfo:nil]; |
| 1373 | } |
| 1374 | // A held menu lease is given back before fresh raw input or an explicit |
| 1375 | // activation; AX element actions (the pick itself) leave it alone. |
| 1376 | if([@[@"bg_pointer",@"type",@"key_event",@"bg_key"] containsObject:tool] |
| 1377 | || ([tool isEqual:@"app_info"] && [args[@"activate"] boolValue])) |
| 1378 | cuFrontLeaseRestoreIfHeld(); |
| 1379 | if(mutates) { cuCheckCancelled(); cuLockInput(); } |
| 1380 | if(!AXIsProcessTrusted()) @throw [NSException exceptionWithName:@"permission" reason:@"Accessibility permission is missing for Codewhale Computer Use (or the direct host)." userInfo:nil]; |
| 1381 | if([tool isEqual:@"type"]) { |
| 1382 | id focused = cuFocusedElement(inputApp.processIdentifier); |
| 1383 | NSDictionary *tt = args[@"target"]; |
| 1384 | // An explicit element target overrides app-level focus for delivery |
| 1385 | // routing — a hosted panel's field is never the app's AXFocusedUIElement. |
| 1386 | if([tt isKindOfClass:NSDictionary.class]) { |
| 1387 | id el = cuResolvePathTarget(inputApp.processIdentifier, tt); |
| 1388 | if(!el) @throw [NSException exceptionWithName:@"stale" reason:@"element is no longer available; observe again" userInfo:nil]; |
| 1389 | focused = el; |
| 1390 | } |
| 1391 | return cuType(args, inputApp, focused, NO); |
| 1392 | } |
| 1393 | if([tool isEqual:@"key_event"]) { |
| 1394 | double yieldMs=0; |
| 1395 | if([args[@"foreground_input"] boolValue] && [args[@"down"] boolValue]) { |
| 1396 | cuRequireForeground(inputApp); |
| 1397 | // Real hardware taps land between the person's keystrokes unless we |
| 1398 | // wait for a gap first; the up event is part of our own press and |
| 1399 | // must not wait. |
| 1400 | yieldMs=cuYieldToUser(args); |
| 1401 | cuRequireForeground(inputApp); |
| 1402 | } |
| 1403 | cuCheckCancelled(); |
| 1404 | id result=cuPostKey(args,inputApp.processIdentifier); |
| 1405 | if(yieldMs>0) { NSMutableDictionary *r=[result mutableCopy]; r[@"yield_ms"]=@(round(yieldMs)); result=r; } |
| 1406 | if([args[@"input_lease"] boolValue] && [args[@"down"] boolValue]) { cuLeaseKey=args; cuLeasePid=inputApp.processIdentifier; cuLeaseApp=inputApp; } |
| 1407 | return result; |
| 1408 | } |
| 1409 | // A key chord with modifier flags through the window-record channel: menu |
| 1410 | // key equivalents (cmd+a, cmd+shift+g) only validate against a key window, |
| 1411 | // which the lease provides. Used by background select-all/replace flows. |
| 1412 | if([tool isEqual:@"bg_key"]) { |
| 1413 | if(!cuResolveBgPointer()) |
| 1414 | @throw [NSException exceptionWithName:@"bg_dispatch_unavailable" reason:@"window-routed background keys are unavailable; no input was sent" userInfo:nil]; |
| 1415 | // The window to post into: an explicit element target's window when the |
| 1416 | // caller names one (a panel's field, say), otherwise the focused |
| 1417 | // element's. An app-level AXFocusedUIElement always reports the app's own |
| 1418 | // window focus, which misses hosted panels entirely. |
| 1419 | id anchor = nil; |
| 1420 | NSDictionary *t = args[@"target"]; |
| 1421 | if([t isKindOfClass:NSDictionary.class]) { |
| 1422 | anchor = cuResolvePathTarget(inputApp.processIdentifier, t); |
| 1423 | if(!anchor) @throw [NSException exceptionWithName:@"stale" reason:@"element is no longer available; observe again" userInfo:nil]; |
| 1424 | } else anchor = cuFocusedElement(inputApp.processIdentifier); |
| 1425 | uint32_t keyWin = 0; |
| 1426 | CGRect keyFrame = CGRectZero; |
| 1427 | pid_t keyOwner = 0; |
| 1428 | NSString *winRole = nil; |
| 1429 | if(anchor) cuElementWindow(anchor, inputApp.processIdentifier, &keyWin, &keyOwner, &winRole) |
| 1430 | && cuWindowInfoForNumber(keyWin, &keyFrame, nil); |
| 1431 | if(!keyWin) @throw [NSException exceptionWithName:@"focus" reason:@"no focused window for a window-routed key; focus a control first" userInfo:nil]; |
| 1432 | cuBgLease lease; |
| 1433 | NSString *why = nil; |
| 1434 | if(!cuBgLeaseBegin(inputApp, keyWin, YES, args, &lease, &why)) |
| 1435 | @throw [NSException exceptionWithName:@"bg_dispatch_unavailable" reason:why userInfo:nil]; |
| 1436 | BOOL keyRestored = YES; |
| 1437 | @try { |
| 1438 | for(int down = 1; down >= 0; down--) { |
| 1439 | CGEventRef event = CGEventCreateKeyboardEvent(NULL, [args[@"code"] unsignedShortValue], down ? true : false); |
| 1440 | CGEventSetFlags(event, [args[@"flags"] unsignedLongLongValue]); |
| 1441 | cuPostEventRecord(&lease, event, CGPointMake(CGRectGetMidX(keyFrame) - keyFrame.origin.x, CGRectGetMidY(keyFrame) - keyFrame.origin.y)); |
| 1442 | CFRelease(event); |
| 1443 | usleep(30000); |
| 1444 | } |
| 1445 | } @finally { keyRestored = cuBgLeaseEnd(&lease); } |
| 1446 | NSMutableDictionary *receipt = [@{@"action_sent":@YES, @"strategy":@"window-record", @"keyboard_delivery":@"window-record", @"front_lease":@(lease.swapped)} mutableCopy]; |
| 1447 | if(winRole) receipt[@"window_role"] = winRole; |
| 1448 | if(lease.postPid != lease.targetPid) receipt[@"window_owner_pid"] = @(lease.postPid); |
| 1449 | if(lease.swapped) receipt[@"front_restored"] = @(keyRestored); |
| 1450 | cuLeaseAccounting(receipt, &lease); |
| 1451 | return receipt; |
| 1452 | } |
| 1453 | if([tool isEqual:@"mouse_event"]) { |
| 1454 | CGPoint p=CGPointMake([args[@"x"] doubleValue],[args[@"y"] doubleValue]); |
| 1455 | CGEventRef event=CGEventCreateMouseEvent(NULL,[args[@"type"] unsignedIntValue],p,[args[@"button"] unsignedIntValue]); |
| 1456 | CGEventSetIntegerValueField(event,kCGMouseEventClickState,[args[@"clickState"] longLongValue]); |
| 1457 | // Address the window as well as the process using public event fields. |
| 1458 | // Dispatch is not delivery: a toolkit may still discard these events. |
| 1459 | // This primitive needs effect readback before a caller can rely on it. |
| 1460 | if([args[@"windowNumber"] longLongValue]>0) { |
| 1461 | CGEventSetIntegerValueField(event,kCGMouseEventWindowUnderMousePointer,[args[@"windowNumber"] longLongValue]); |
| 1462 | CGEventSetIntegerValueField(event,kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent,[args[@"windowNumber"] longLongValue]); |
| 1463 | } |
| 1464 | cuCheckCancelled(); |
| 1465 | CGEventPostToPid(inputApp.processIdentifier,event); CFRelease(event); return @{@"action_sent":@YES}; |
| 1466 | } |
| 1467 | // Accessibility-first coordinate action: resolve the point against the bound |
| 1468 | // application's AX tree and press the element it names. Callers fall back to |
| 1469 | // raw CGEvents when this reports found=NO, so it must fail closed rather than |
| 1470 | // guess: a point owned by another process, or a point that only lands on a |
| 1471 | // container, is not a press. |
| 1472 | if([tool isEqual:@"hit_test"]) { |
| 1473 | CGPoint p=CGPointMake([args[@"x"] doubleValue],[args[@"y"] doubleValue]); |
| 1474 | AXUIElementRef appEl=AXUIElementCreateApplication(inputApp.processIdentifier); |
| 1475 | AXUIElementSetMessagingTimeout(appEl,2.0); |
| 1476 | axPrepare(appEl); |
| 1477 | AXUIElementRef raw=NULL; |
| 1478 | AXError err=AXUIElementCopyElementAtPosition(appEl,(float)p.x,(float)p.y,&raw); |
| 1479 | id hit=nil; |
| 1480 | if(err==kAXErrorSuccess && raw) { |
| 1481 | pid_t owner=0; |
| 1482 | if(AXUIElementGetPid(raw,&owner)==kAXErrorSuccess && owner==inputApp.processIdentifier) hit=CFBridgingRelease(raw); |
| 1483 | else CFRelease(raw); // Another app may cover a background window. Search only our own tree below. |
| 1484 | } |
| 1485 | |
| 1486 | id chosen=nil; |
| 1487 | BOOL insideSheet=NO; |
| 1488 | NSString *operation=args[@"operation"]?:@"click"; |
| 1489 | BOOL scrolling=[operation hasPrefix:@"scroll"]; |
| 1490 | // 1. The element under the point, or the nearest ancestor that can be |
| 1491 | // pressed — a label inside a button is the common case. |
| 1492 | for(id cur=hit; cur && !chosen;) { |
| 1493 | AXUIElementRef el=(__bridge AXUIElementRef)cur; |
| 1494 | id role=attr(el,@"AXRole"); |
| 1495 | if([role isEqual:@"AXSheet"]) insideSheet=YES; |
| 1496 | if([role isEqual:@"AXWindow"] || [role isEqual:@"AXApplication"]) break; |
| 1497 | if(scrolling?cuScrollBar(el,[operation isEqual:@"scroll-horizontal"])!=nil:cuClickAction(el,[operation isEqual:@"context"])!=nil) { chosen=cur; break; } |
| 1498 | cur=attr(el,@"AXParent"); |
| 1499 | } |
| 1500 | // 2. Otherwise search downward for the smallest control covering the point. |
| 1501 | if(!chosen) { |
| 1502 | int budget=1500; double area=0; id best=nil; |
| 1503 | if(hit) cuSearch((__bridge AXUIElementRef)hit,p,0,&budget,&best,&area,operation); |
| 1504 | else for(id w in attr(appEl,@"AXWindows")) { |
| 1505 | CGRect frame; |
| 1506 | if(!cuFrame((__bridge AXUIElementRef)w,&frame) || !CGRectContainsPoint(frame,p)) continue; |
| 1507 | // A sheet owns the window's interaction, even when the sheet does not cover p. |
| 1508 | NSArray *sheets=attr((__bridge AXUIElementRef)w,@"AXSheets"); |
| 1509 | if(sheets.count) { |
| 1510 | for(id sheet in sheets) cuSearch((__bridge AXUIElementRef)sheet,p,0,&budget,&best,&area,operation); |
| 1511 | insideSheet=YES; |
| 1512 | } else cuSearch((__bridge AXUIElementRef)w,p,0,&budget,&best,&area,operation); |
| 1513 | break; // Never click through another window of the same app. |
| 1514 | } |
| 1515 | chosen=best; |
| 1516 | } |
| 1517 | CFRelease(appEl); |
| 1518 | if(!chosen) { |
| 1519 | // A web popup button reports unpressable on purpose (its menu only opens |
| 1520 | // from a real mouse event); name it so the caller can poll for the menu |
| 1521 | // through Chromium's post-activation AX rebuild. |
| 1522 | if(hit) { |
| 1523 | for(id cur=hit; cur;) { |
| 1524 | AXUIElementRef el=(__bridge AXUIElementRef)cur; |
| 1525 | id role=attr(el,@"AXRole"); |
| 1526 | if([role isEqual:@"AXWindow"] || [role isEqual:@"AXApplication"]) break; |
| 1527 | if([@[@"AXMenuButton",@"AXPopUpButton"] containsObject:role] && axHasWebAncestor(el)) |
| 1528 | return @{@"found":@NO,@"reason":@"web_popup_requires_real_click"}; |
| 1529 | cur=attr(el,@"AXParent"); |
| 1530 | } |
| 1531 | } |
| 1532 | return @{@"found":@NO,@"reason":hit?@"no_pressable_element_at_point":@"no_element_at_point"}; |
| 1533 | } |
| 1534 | |
| 1535 | // A press invokes the control's action directly, which would sail straight |
| 1536 | // past a window-modal sheet that a real click cannot cross. Refuse instead: |
| 1537 | // the caller must deal with the sheet. |
| 1538 | if(!insideSheet) { |
| 1539 | id owner=chosen; |
| 1540 | for(int up=0; up<12 && owner; up++) { |
| 1541 | AXUIElementRef el=(__bridge AXUIElementRef)owner; |
| 1542 | id role=attr(el,@"AXRole"); |
| 1543 | if([role isEqual:@"AXSheet"]) { insideSheet=YES; break; } |
| 1544 | if([role isEqual:@"AXWindow"]) { |
| 1545 | for(id kid in attr(el,@"AXChildren")) { |
| 1546 | if([attr((__bridge AXUIElementRef)kid,@"AXRole") isEqual:@"AXSheet"]) |
| 1547 | return @{@"found":@NO,@"reason":@"window_blocked_by_modal_sheet"}; |
| 1548 | } |
| 1549 | break; |
| 1550 | } |
| 1551 | owner=attr(el,@"AXParent"); |
| 1552 | } |
| 1553 | } |
| 1554 | |
| 1555 | NSDictionary *element=info((__bridge AXUIElementRef)chosen,0,0,@[]); |
| 1556 | if(![args[@"perform"] boolValue]) return @{@"found":@YES,@"element":element,@"action_sent":@NO}; |
| 1557 | cuCheckCancelled(); |
| 1558 | NSMutableDictionary *receipt=[(scrolling?cuScroll((__bridge AXUIElementRef)chosen,args):cuClick((__bridge AXUIElementRef)chosen,[operation isEqual:@"context"])) mutableCopy]; |
| 1559 | receipt[@"found"]=@YES; receipt[@"element"]=element; return receipt; |
| 1560 | } |
| 1561 | /** |
| 1562 | * Pointer gestures are window-routed event records addressed to a window of |
| 1563 | * the bound app (cuBgPointer). The cursor the user holds is never moved. |
| 1564 | */ |
| 1565 | if([tool isEqual:@"bg_pointer"]) return cuBgPointer(inputApp, args); |
| 1566 | if([tool isEqual:@"scroll"]) { |
| 1567 | cuCheckCancelled(); |
| 1568 | CGEventRef event=CGEventCreateScrollWheelEvent(NULL,kCGScrollEventUnitLine,2,[args[@"dy"] intValue],[args[@"dx"] intValue]); CGEventPostToPid(inputApp.processIdentifier,event); CFRelease(event); return @{@"action_sent":@YES}; |
| 1569 | } |
| 1570 | if([tool isEqual:@"cursor_position"]) { |
| 1571 | CGEventRef event=CGEventCreate(NULL); CGPoint p=CGEventGetLocation(event); CFRelease(event); return @{@"x":@(p.x),@"y":@(p.y)}; |
| 1572 | } |
| 1573 | NSRunningApplication *a=resolve(args[@"app_ref"]?:args[@"target"][@"app_ref"]); |
| 1574 | if(!a) @throw [NSException exceptionWithName:@"app" reason:@"application not found" userInfo:nil]; |
| 1575 | if(mutates && [a.bundleIdentifier isEqual:@"net.codewhale.computer-use"]) @throw [NSException exceptionWithName:@"protected" reason:@"Computer Use safety controls belong to the user." userInfo:nil]; |
| 1576 | AXUIElementRef app=AXUIElementCreateApplication(a.processIdentifier); |
| 1577 | AXUIElementSetMessagingTimeout(app,2.0); |
| 1578 | axPrepare(app); |
| 1579 | @try { |
| 1580 | NSArray *ws=attr(app,@"AXWindows")?:@[]; |
| 1581 | NSDictionary *identity=@{@"found":@YES,@"name":a.localizedName?:@"",@"pid":@(a.processIdentifier),@"bundle_id":a.bundleIdentifier?:@"",@"frontmost":@(a.active)}; |
| 1582 | if([tool isEqual:@"get_app_state"] || [tool isEqual:@"list_windows"]) { |
| 1583 | BOOL truncated=NO; |
| 1584 | BOOL list=[tool isEqual:@"list_windows"]; |
| 1585 | NSArray *out=observeElements(app,ws,args,list,&truncated); |
| 1586 | // A window that vends no descendants at all is not a page — it is an app |
| 1587 | // whose content tree is mid-rebuild (Chromium tears down and rebuilds |
| 1588 | // its accessibility tree across activation transitions, which takes |
| 1589 | // seconds). Poll briefly rather than returning an empty UI. A filtered |
| 1590 | // observe (query/role) legitimately matches nothing, so only unfiltered |
| 1591 | // observes earn the wait. |
| 1592 | // While a menu lease is held for this app, the thing being waited on is |
| 1593 | // the open menu's items: they reappear only when Chromium's rebuilt |
| 1594 | // tree re-vends them, so poll until an in-window AXMenuItem exists. |
| 1595 | BOOL leaseHeld = cuFrontLeaseHeldForPid(a.processIdentifier); |
| 1596 | int maxAttempts = leaseHeld ? 20 : 8; |
| 1597 | for(int attempt=0; !list && attempt<maxAttempts; attempt++) { |
| 1598 | BOOL empty = hasOrphanWebArea(out); |
| 1599 | if(!empty && !args[@"query"] && !args[@"role"]) { |
| 1600 | if(leaseHeld) { |
| 1601 | empty = YES; |
| 1602 | for(NSDictionary *d in out) { |
| 1603 | if([d[@"role"] isEqual:@"AXMenuItem"] && [d[@"windowIndex"] integerValue] >= 0) { empty = NO; break; } |
| 1604 | } |
| 1605 | } else if(ws.count) { |
| 1606 | BOOL anyContent = NO; |
| 1607 | for(NSDictionary *d in out) { |
| 1608 | if([d[@"windowIndex"] integerValue] >= 0 && [d[@"path"] count] > 0) { anyContent = YES; break; } |
| 1609 | } |
| 1610 | empty = !anyContent; |
| 1611 | } |
| 1612 | } |
| 1613 | if(!empty) break; |
| 1614 | usleep(300000); |
| 1615 | ws=attr(app,@"AXWindows")?:ws; |
| 1616 | truncated=NO; |
| 1617 | out=observeElements(app,ws,args,list,&truncated); |
| 1618 | } |
| 1619 | NSMutableDictionary *d=[identity mutableCopy]; d[list?@"windows":@"elements"]=out; d[@"truncated"]=@(truncated); return d; |
| 1620 | } |
| 1621 | if([tool isEqual:@"resolve_element"]) { |
| 1622 | NSInteger wi=[args[@"windowIndex"] integerValue]; |
| 1623 | id el=wi==-1?attr(app,@"AXMenuBar"):wi==-2?(__bridge id)app:(wi>=0 && wi<ws.count?ws[wi]:nil); |
| 1624 | if(!el) return @{@"found":@NO,@"element":[NSNull null],@"reason":@"window_not_found"}; |
| 1625 | for(NSNumber *i in args[@"path"]?:@[]) { NSArray *kids=attr((__bridge AXUIElementRef)el,@"AXChildren"); if(i.unsignedIntegerValue>=kids.count) return @{@"found":@NO,@"element":[NSNull null],@"reason":@"path_not_found"}; el=kids[i.unsignedIntegerValue]; } |
| 1626 | return @{@"found":@YES,@"element":info((__bridge AXUIElementRef)el,0,wi,args[@"path"]?:@[]),@"reason":[NSNull null]}; |
| 1627 | } |
| 1628 | NSDictionary *t=args[@"target"]; NSInteger wi=[t[@"windowIndex"] integerValue]; |
| 1629 | id el=wi==-1?attr(app,@"AXMenuBar"):wi==-2?(__bridge id)app:(wi>=0 && wi<ws.count?ws[wi]:nil); |
| 1630 | if(!el) @throw [NSException exceptionWithName:@"stale" reason:@"window is no longer available; observe again" userInfo:nil]; |
| 1631 | for(NSNumber *i in t[@"path"]) { NSArray *kids=attr((__bridge AXUIElementRef)el,@"AXChildren"); if(i.unsignedIntegerValue>=kids.count) @throw [NSException exceptionWithName:@"stale" reason:@"element is no longer available; observe again" userInfo:nil]; el=kids[i.unsignedIntegerValue]; } |
| 1632 | if(wi>=0) { |
| 1633 | NSArray *sheets=attr((__bridge AXUIElementRef)ws[wi],@"AXSheets"); |
| 1634 | if(sheets.count) { |
| 1635 | BOOL inside=NO; id ancestor=el; |
| 1636 | for(int depth=0;ancestor && depth<64;depth++) { |
| 1637 | for(id sheet in sheets) if(CFEqual((__bridge CFTypeRef)ancestor,(__bridge CFTypeRef)sheet)) inside=YES; |
| 1638 | if(inside) break; |
| 1639 | ancestor=attr((__bridge AXUIElementRef)ancestor,@"AXParent"); |
| 1640 | } |
| 1641 | if(!inside) @throw [NSException exceptionWithName:@"modal" reason:@"window blocked by modal sheet; observe and handle the dialog first" userInfo:nil]; |
| 1642 | } |
| 1643 | } |
| 1644 | cuCheckCancelled(); |
| 1645 | if([t[@"type"] isEqual:@"element"]) cuValidateElementIdentity((__bridge AXUIElementRef)el,t); |
| 1646 | if([tool isEqual:@"click_element"]) return cuClick((__bridge AXUIElementRef)el,[args[@"context"] boolValue]); |
| 1647 | if([tool isEqual:@"scroll_element"]) return cuScroll((__bridge AXUIElementRef)el,args); |
| 1648 | AXError e=kAXErrorFailure; |
| 1649 | if([tool isEqual:@"get_value"]) { |
| 1650 | id v=attr((__bridge AXUIElementRef)el,@"AXValue"); |
| 1651 | return @{@"ok":@YES,@"strategy":@"a11y",@"value":v?:[NSNull null],@"role":attr((__bridge AXUIElementRef)el,@"AXRole")?:[NSNull null]}; |
| 1652 | } |
| 1653 | if([tool isEqual:@"set_value"]) { |
| 1654 | NSString *role=attr((__bridge AXUIElementRef)el,@"AXRole"); |
| 1655 | if(axHasWebAncestor((__bridge AXUIElementRef)el)) |
| 1656 | @throw [NSException exceptionWithName:@"value" reason:@"this element lives in a web area, which ignores background AXValue writes (numeric controls may even coerce them to empty). No write was sent — focus the element and use type instead, then verify with get_value." userInfo:nil]; |
| 1657 | BOOL numeric=[@[@"AXIncrementor",@"AXSlider",@"AXStepper",@"AXValueIndicator",@"AXProgressIndicator"] containsObject:role]; |
| 1658 | id value=args[@"value"]; |
| 1659 | if(numeric) { |
| 1660 | // These controls type AXValue as a number. Writing a string is the |
| 1661 | // classic "clears the field instead of setting it" bug — a web |
| 1662 | // incrementor may coerce "" over "150" and report success. |
| 1663 | if(![value isKindOfClass:NSNumber.class] || CFGetTypeID((__bridge CFTypeRef)value)==CFBooleanGetTypeID()) { |
| 1664 | NSString *s=[value isKindOfClass:NSString.class]?value:[value description]; |
| 1665 | static NSNumberFormatter *fmt=nil; |
| 1666 | if(!fmt) { fmt=[NSNumberFormatter new]; fmt.locale=[NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]; fmt.numberStyle=NSNumberFormatterDecimalStyle; } |
| 1667 | NSNumber *n=[fmt numberFromString:s]; |
| 1668 | if(!n) @throw [NSException exceptionWithName:@"value" reason:[NSString stringWithFormat:@"%@ takes a numeric AXValue; %@ does not parse — focus the control and type instead",role?:@"this control",s] userInfo:nil]; |
| 1669 | value=n; |
| 1670 | } |
| 1671 | } |
| 1672 | e=AXUIElementSetAttributeValue((__bridge AXUIElementRef)el,kAXValueAttribute,(__bridge CFTypeRef)value); |
| 1673 | if(e==kAXErrorSuccess) { |
| 1674 | usleep(60000); |
| 1675 | id after=attr((__bridge AXUIElementRef)el,@"AXValue"); |
| 1676 | BOOL verified=NO; |
| 1677 | if(numeric) verified=[after isKindOfClass:NSNumber.class] && fabs([after doubleValue]-[value doubleValue])<1e-6; |
| 1678 | else verified=[after isKindOfClass:NSString.class] && [after isEqual:value]; |
| 1679 | NSMutableDictionary *done=[@{@"action_sent":@YES,@"strategy":@"a11y",@"role":role?:[NSNull null],@"after":after?:[NSNull null],@"verified":@(verified)} mutableCopy]; |
| 1680 | if(!verified) done[@"note"]=@"AXValue write did not verify: the control kept its own value (Electron/web text elements and numeric steppers commonly ignore background AXValue writes). Focus the element and use type instead, then verify with get_value."; |
| 1681 | return done; |
| 1682 | } |
| 1683 | } |
| 1684 | else if([tool isEqual:@"focus_element"]) e=AXUIElementSetAttributeValue((__bridge AXUIElementRef)el,kAXFocusedAttribute,kCFBooleanTrue); |
| 1685 | else if([tool isEqual:@"select_text"]){ NSArray *r=args[@"text_range"]?:@[@0,@0]; if(r.count!=2 || [r[0] longValue]<0 || [r[1] longValue]<0) @throw [NSException exceptionWithName:@"range" reason:@"text_range must be [start, length], both nonnegative" userInfo:nil]; CFRange range=CFRangeMake([r[0] longValue],[r[1] longValue]); AXValueRef v=AXValueCreate(kAXValueCFRangeType,&range); e=AXUIElementSetAttributeValue((__bridge AXUIElementRef)el,kAXSelectedTextRangeAttribute,v); CFRelease(v); } |
| 1686 | else if([tool isEqual:@"perform_action"]){ CFArrayRef actions=NULL; AXUIElementCopyActionNames((__bridge AXUIElementRef)el,&actions); NSArray *names=CFBridgingRelease(actions); if(![names containsObject:args[@"action"]]) @throw [NSException exceptionWithName:@"action" reason:@"action is not advertised by this element" userInfo:nil]; cuCheckCancelled(); e=AXUIElementPerformAction((__bridge AXUIElementRef)el,(__bridge CFStringRef)args[@"action"]); } |
| 1687 | if(e!=kAXErrorSuccess) @throw [NSException exceptionWithName:@"action" reason:[NSString stringWithFormat:@"accessibility action failed: %d",e] userInfo:nil]; |
| 1688 | NSMutableDictionary *done=[@{@"action_sent":@YES,@"strategy":@"a11y"} mutableCopy]; |
| 1689 | if([tool isEqual:@"focus_element"]) done[@"focused"]=@YES; |
| 1690 | return done; |
| 1691 | } @finally { CFRelease(app); } |
| 1692 | } |
| 1693 | int main(int argc, const char **argv){ @autoreleasepool { |
| 1694 | signal(SIGTERM,cuCancel); signal(SIGINT,cuCancel); signal(SIGPIPE,SIG_IGN); |
| 1695 | @try { if(argc!=2) @throw [NSException exceptionWithName:@"args" reason:@"expected one JSON argument" userInfo:nil]; |
| 1696 | NSError *error=nil; id p=[NSJSONSerialization JSONObjectWithData:[[NSString stringWithUTF8String:argv[1]] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:&error]; |
| 1697 | if(![p isKindOfClass:NSDictionary.class]) @throw [NSException exceptionWithName:@"json" reason:@"invalid request" userInfo:nil]; |
| 1698 | id result=execute(p); |
| 1699 | if([p[@"args"][@"input_lease"] boolValue]) { NSMutableDictionary *ack=[result mutableCopy]; ack[@"input_lease"]=@YES; result=ack; } |
| 1700 | NSData *data=[NSJSONSerialization dataWithJSONObject:result options:NSJSONWritingFragmentsAllowed error:&error]; |
| 1701 | if(!data) @throw [NSException exceptionWithName:@"json" reason:error.localizedDescription userInfo:nil]; |
| 1702 | puts([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding].UTF8String); fflush(stdout); |
| 1703 | if([p[@"args"][@"input_lease"] boolValue]) cuWaitForLease(); |
| 1704 | return 0; |
| 1705 | } @catch(NSException *e){ cuReleaseLease(); fprintf(stderr,"%s\n",e.reason.UTF8String); return 1; } |
| 1706 | } } |
| 1707 |