| 1 | import { test } from 'node:test'; |
| 2 | import assert from 'node:assert/strict'; |
| 3 | import fs from 'node:fs'; |
| 4 | import os from 'node:os'; |
| 5 | import path from 'node:path'; |
| 6 | import { spawn, spawnSync } from 'node:child_process'; |
| 7 | import zlib from 'node:zlib'; |
| 8 | import crypto from 'node:crypto'; |
| 9 | import { create, leaseVerdict, leaseAccounting } from '../src/backends/darwin.mjs'; |
| 10 | import { withSignal, currentSignal, runInputLease } from '../src/exec.mjs'; |
| 11 | |
| 12 | // Simulate the native lease process: it owns cleanup itself, keeping the |
| 13 | // original app identity even if the JS backend is rebound before release. |
| 14 | function leaseExecutor(run) { |
| 15 | return { async run(cmd,args,opts) { |
| 16 | const result=await run(cmd,args,opts); |
| 17 | if(args[0]?.startsWith('{') && JSON.parse(args[0]).tool==='input_capabilities' && result.code===0 && JSON.parse(result.stdout).input_lease===undefined) return {code:0,stderr:'',stdout:JSON.stringify({input_lease:1,background_focus_guard:1})}; |
| 18 | return result; |
| 19 | }, async runInputLease(cmd, argv) { |
| 20 | const request = JSON.parse(argv[0]); |
| 21 | const release = async ({point} = {}) => { |
| 22 | const args = request.args; |
| 23 | const native = request.tool === 'key_event' |
| 24 | ? {tool:'key_event',args:{...args,down:false,owned_release:true}} |
| 25 | : {tool:'release_input',args:{...args,point:point??args.steps.at(-1),button:0}}; |
| 26 | await withSignal(null,()=>run(cmd,[JSON.stringify(native)],{timeoutMs:20000,ownerPipe:true})); |
| 27 | }; |
| 28 | const result = await run(cmd,argv,{timeoutMs:20000,ownerPipe:true}); |
| 29 | if (result.code !== 0 || result.aborted || result.timedOut) { |
| 30 | if(result.spawned && (result.aborted || result.timedOut)) await release(); |
| 31 | throw Object.assign(new Error(result.aborted?'computer request cancelled':result.timedOut?'native accessibility helper timed out':result.stderr),{code:result.aborted?'cancelled':'native_error'}); |
| 32 | } |
| 33 | return {receipt:JSON.parse(result.stdout),release,async send({point}) { |
| 34 | const result=await run(cmd,[JSON.stringify({tool:'pointer_sequence',args:{...request.args,steps:[{type:6,...point,button:0}]}})],{timeoutMs:20000,ownerPipe:true}); |
| 35 | return JSON.parse(result.stdout); |
| 36 | }}; |
| 37 | }}; |
| 38 | } |
| 39 | |
| 40 | test('native summary keeps text and top-level menus without spending the UI budget on hidden menu trees', {skip:process.platform!=='darwin'}, t=>{ |
| 41 | const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cu-native-observation-'));t.after(()=>fs.rmSync(dir,{recursive:true,force:true})); |
| 42 | const binary=path.join(dir,'native'); |
| 43 | const build=spawnSync('clang',['-DCU_TEST=1','-fobjc-arc','-Os','-framework','Cocoa','-framework','ApplicationServices','-framework','ScreenCaptureKit','-framework','AVFoundation','-framework','CoreMedia','-framework','Vision','src/backends/darwin-accessibility.m','-o',binary],{encoding:'utf8'}); |
| 44 | assert.equal(build.status,0,build.stderr); |
| 45 | |
| 46 | for (const tool of ['bg_key','bg_pointer','inspect_focus_control']) { |
| 47 | const r=spawnSync(binary,[JSON.stringify({tool,args:{app_scoped:true,foreground_input:false}})],{encoding:'utf8'}); |
| 48 | assert.equal(r.status,1); |
| 49 | assert.match(r.stderr,/background_focus_required/); |
| 50 | } |
| 51 | const consented=spawnSync(binary,[JSON.stringify({tool:'inspect_focus_control',args:{foreground_input:true}})],{encoding:'utf8'}); |
| 52 | assert.equal(consented.status,0,consented.stderr); |
| 53 | |
| 54 | for(const [element,context,action] of [ |
| 55 | [{AXRole:'AXTextField',actions:['AXPress'],settable:['AXFocused']},false,'AXFocused'], |
| 56 | [{AXRole:'AXRow',settable:['AXSelected']},false,'AXSelected'], |
| 57 | [{AXRole:'AXMenuItem',actions:['AXPick']},false,'AXPick'], |
| 58 | [{AXRole:'AXButton',actions:['AXShowMenu']},true,'AXShowMenu'], |
| 59 | [{AXRole:'AXButton',AXEnabled:false,actions:['AXPress']},false,null], |
| 60 | // Focusing anything but a text-entry role is not a click; the caller |
| 61 | // falls back to real delivery. |
| 62 | [{AXRole:'AXGroup',settable:['AXFocused']},false,null], |
| 63 | [{AXRole:'AXSearchField',settable:['AXFocused']},false,'AXFocused'], |
| 64 | [{AXRole:'AXGroup',settable:['AXFocused','AXSelectedText']},false,null], |
| 65 | ]) { |
| 66 | const r=spawnSync(binary,[JSON.stringify({tool:'inspect_click_action',args:{element,context}})],{encoding:'utf8'}); |
| 67 | assert.equal(r.status,0,r.stderr); |
| 68 | assert.equal(JSON.parse(r.stdout).action,action); |
| 69 | } |
| 70 | const node=(role,label,children=[])=>({AXRole:role,AXTitle:label,AXChildren:children,actions:['AXPress']}); |
| 71 | const app={AXMenuBar:node('AXMenuBar','Menu bar',[node('AXMenuBarItem','File',[node('AXMenu','File menu',Array.from({length:150},(_,i)=>node('AXMenuItem',`Command ${i}`)))])]),AXChildren:[node('AXMenu','Popup',[node('AXMenuItem','Choose')])]}; |
| 72 | const windows=[node('AXWindow','Fixture',Array.from({length:350},(_,i)=>({...node('AXTextField',`Field ${i}`),AXValue:`Value ${i}`,AXFocused:i===349})))]; |
| 73 | const observe=(detail)=>{ |
| 74 | const r=spawnSync(binary,[JSON.stringify({tool:'inspect_observation',args:{app,windows,detail}})],{encoding:'utf8'}); |
| 75 | assert.equal(r.status,0,r.stderr);return JSON.parse(r.stdout); |
| 76 | }; |
| 77 | for(const detail of [undefined,'summary','compact']) { |
| 78 | const state=observe(detail); |
| 79 | assert.equal(state.truncated,false,'intentional menu summarization is not a truncated observation'); |
| 80 | assert.equal(state.elements.length,355); |
| 81 | assert.ok(state.elements.some(e=>e.label==='File')); |
| 82 | assert.ok(state.elements.some(e=>e.label==='Choose'),'open popup actions remain observable'); |
| 83 | assert.ok(!state.elements.some(e=>e.label==='Command 0')); |
| 84 | const field=state.elements.find(e=>e.label==='Field 349'); |
| 85 | assert.equal(field.value,'Value 349');assert.equal(field.focused,true); |
| 86 | assert.deepEqual(field.path,[349]);assert.equal(field.windowIndex,0); |
| 87 | } |
| 88 | const full=observe('full'); |
| 89 | assert.equal(full.truncated,false); |
| 90 | assert.equal(full.elements.find(e=>e.label==='Command 149').windowIndex,-1); |
| 91 | assert.deepEqual(full.elements.find(e=>e.label==='Command 149').path,[0,0,149]); |
| 92 | assert.ok(full.elements.some(e=>e.label==='Field 349')); |
| 93 | const identity=(element,target)=>spawnSync(binary,[JSON.stringify({tool:'inspect_element_identity',args:{element,target}})],{encoding:'utf8'}); |
| 94 | const target={role:'AXMenuItem',label:'Files'}; |
| 95 | assert.equal(identity(node('AXMenuItem','Files'),target).status,0); |
| 96 | for(const element of [node('AXMenuItem','Delete'),node('AXButton','Files'),node('AXMenuItem','')]) { |
| 97 | const refused=identity(element,target); |
| 98 | assert.equal(refused.status,1);assert.match(refused.stderr,/element changed (role|label)/); |
| 99 | } |
| 100 | assert.equal(identity(node('AXMenuItem','Files'),{role:'AXMenuItem'}).status,1,'an unlabeled element replaced by a labeled one is stale too'); |
| 101 | }); |
| 102 | |
| 103 | test('native Unicode encoding round-trips through the actual CoreGraphics event', {skip:process.platform!=='darwin'}, t=>{ |
| 104 | const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cu-native-test-'));t.after(()=>fs.rmSync(dir,{recursive:true,force:true})); |
| 105 | const binary=path.join(dir,'native'); |
| 106 | const build=spawnSync('clang',['-DCU_TEST=1','-fobjc-arc','-Os','-framework','Cocoa','-framework','ApplicationServices','-framework','ScreenCaptureKit','-framework','AVFoundation','-framework','CoreMedia','-framework','Vision','src/backends/darwin-accessibility.m','-o',binary],{encoding:'utf8'}); |
| 107 | assert.equal(build.status,0,build.stderr); |
| 108 | for(const text of ['Hello 世界 🐋','quote " slash \\ newline\n','e\u0301 👨👩👧👦']){ |
| 109 | const r=spawnSync(binary,[JSON.stringify({tool:'inspect_text_event',args:{text}})],{encoding:'utf8'}); |
| 110 | assert.equal(r.status,0,r.stderr);assert.equal(JSON.parse(r.stdout).text,text); |
| 111 | assert.equal(JSON.parse(r.stdout).flags,0,'literal text carries no physical modifiers'); |
| 112 | for (const inherited_flags of [1<<17,1<<18,1<<19,1<<20,(1<<17)|(1<<20)]) { |
| 113 | const inherited=spawnSync(binary,[JSON.stringify({tool:'inspect_text_event',args:{text,inherited_flags}})],{encoding:'utf8'}); |
| 114 | assert.equal(inherited.status,0,inherited.stderr); |
| 115 | assert.deepEqual(JSON.parse(inherited.stdout),{text,flags:0}); |
| 116 | } |
| 117 | } |
| 118 | // No mode reaches the user's cursor: the HID pointer route and its held |
| 119 | // button release are refused even under explicit foreground control. |
| 120 | for (const tool of ['pointer_sequence','release_input']) for (const foreground_input of [false,true]) { |
| 121 | const pointer=spawnSync(binary,[JSON.stringify({tool,args:{foreground_input,steps:[{type:5,x:1,y:1}],point:{x:1,y:1},button:0}})],{encoding:'utf8'}); |
| 122 | assert.equal(pointer.status,1); |
| 123 | assert.match(pointer.stderr,/real_pointer_refused/); |
| 124 | } |
| 125 | }); |
| 126 | |
| 127 | test('native window matching refuses another process, mismatched geometry and ambiguous captures', {skip:process.platform!=='darwin'}, t=>{ |
| 128 | const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cu-native-window-'));t.after(()=>fs.rmSync(dir,{recursive:true,force:true})); |
| 129 | const binary=path.join(dir,'native'); |
| 130 | const build=spawnSync('clang',['-DCU_TEST=1','-fobjc-arc','-Os','-framework','Cocoa','-framework','ApplicationServices','-framework','ScreenCaptureKit','-framework','AVFoundation','-framework','CoreMedia','-framework','Vision','src/backends/darwin-accessibility.m','-o',binary],{encoding:'utf8'}); |
| 131 | assert.equal(build.status,0,build.stderr); |
| 132 | const selected={kCGWindowOwnerPID:123,kCGWindowLayer:0,kCGWindowNumber:42,kCGWindowBounds:{X:10,Y:20,Width:400,Height:300}}; |
| 133 | const query=windows=>spawnSync(binary,[JSON.stringify({tool:'inspect_window_match',args:{pid:123,bounds:{x:10,y:20,w:400,h:300},windows}})],{encoding:'utf8'}); |
| 134 | const wrongProcess={...selected,kCGWindowOwnerPID:999,kCGWindowNumber:43}; |
| 135 | const wrongSize={...selected,kCGWindowNumber:44,kCGWindowBounds:{...selected.kCGWindowBounds,Width:800}}; |
| 136 | const found=query([wrongProcess,wrongSize,selected]); |
| 137 | assert.equal(found.status,0,found.stderr);assert.equal(JSON.parse(found.stdout).window_id,42); |
| 138 | const missing=query([wrongProcess,wrongSize]); |
| 139 | assert.equal(missing.status,1);assert.match(missing.stderr,/selected app window is not capturable/); |
| 140 | const ambiguous=query([selected,{...selected,kCGWindowNumber:45}]); |
| 141 | assert.equal(ambiguous.status,1);assert.match(ambiguous.stderr,/selected window is ambiguous/); |
| 142 | const owner=windows=>{ |
| 143 | const result=spawnSync(binary,[JSON.stringify({tool:'inspect_window_at_point',args:{x:50,y:50,input_app_ref:{pid:123},windows}})],{encoding:'utf8'}); |
| 144 | assert.equal(result.status,0,result.stderr);return JSON.parse(result.stdout); |
| 145 | }; |
| 146 | for(const layer of [0,3,8,25,1000]) { |
| 147 | const floating={...wrongProcess,kCGWindowLayer:layer,kCGWindowAlpha:1}; |
| 148 | assert.equal(owner([floating,selected]).owner_pid,999,'visible foreign windows own the point at every layer'); |
| 149 | assert.equal(owner([{...floating,kCGWindowAlpha:0.001},selected]).owner_pid,999,'faint foreign windows remain occluders'); |
| 150 | assert.equal(owner([{...floating,kCGWindowAlpha:0},selected]).owner_pid,123,'transparent overlays do not intercept'); |
| 151 | } |
| 152 | assert.equal(owner([{...selected,kCGWindowLayer:8}]).owner_pid,123,'the bound app can target its own floating panels'); |
| 153 | |
| 154 | }); |
| 155 | |
| 156 | test('native owner pipe survives forced MCP exit, releases promptly and excludes competing input', {skip:process.platform!=='darwin'}, async t=>{ |
| 157 | const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cu-native-owner-')); t.after(()=>fs.rmSync(dir,{recursive:true,force:true})); |
| 158 | const binary=path.join(dir,'native'); |
| 159 | const build=spawnSync('clang',['-DCU_TEST=1','-fobjc-arc','-Os','-framework','Cocoa','-framework','ApplicationServices','-framework','ScreenCaptureKit','-framework','AVFoundation','-framework','CoreMedia','-framework','Vision','src/backends/darwin-accessibility.m','-o',binary],{encoding:'utf8'}); |
| 160 | assert.equal(build.status,0,build.stderr); |
| 161 | for(const workMs of [0,5000]) { |
| 162 | const releaseFile=path.join(dir,`released-${workMs}`); |
| 163 | const args={owner_pipe:true,input_lease:true,lock_dir:dir,release_file:releaseFile,work_ms:workMs}; |
| 164 | const request=JSON.stringify({tool:'test_input_lease',args}); |
| 165 | const parent=spawn(process.execPath,['--input-type=module','-e',` |
| 166 | import {runInputLease} from ${JSON.stringify(new URL('../src/exec.mjs',import.meta.url).href)}; |
| 167 | await runInputLease(process.argv[1],[process.argv[2]]); |
| 168 | console.log('ready'); setInterval(()=>{},1000); |
| 169 | `,binary,request],{stdio:['ignore','pipe','pipe']}); |
| 170 | t.after(()=>{if(parent.exitCode===null)parent.kill('SIGKILL');}); |
| 171 | await new Promise((resolve,reject)=>{parent.stdout.once('data',resolve);parent.once('exit',code=>reject(new Error(`parent exited before ready: ${code}`)));}); |
| 172 | const competing=spawnSync(binary,[JSON.stringify({tool:'test_input_lease',args:{lock_dir:dir}})],{encoding:'utf8'}); |
| 173 | assert.equal(competing.status,1); |
| 174 | assert.match(competing.stderr,/another Computer Use session owns held input/); |
| 175 | assert.ok(!fs.existsSync(releaseFile)); |
| 176 | const exited=new Promise(resolve=>parent.once('exit',resolve));parent.kill('SIGKILL');await exited; |
| 177 | const deadline=Date.now()+2000; |
| 178 | while((!fs.existsSync(releaseFile)||fs.readFileSync(releaseFile,'utf8')!=='released')&&Date.now()<deadline)await new Promise(resolve=>setTimeout(resolve,20)); |
| 179 | assert.equal(fs.readFileSync(releaseFile,'utf8'),'released','native cleanup ran on owner pipe EOF'); |
| 180 | const successor=await runInputLease(binary,[JSON.stringify({tool:'test_input_lease',args:{...args,work_ms:0}})]);await successor.release(); |
| 181 | } |
| 182 | }); |
| 183 | test('macOS backend binds native input to the opened process and reports denied permissions honestly', async t=>{ |
| 184 | const bundle=fs.mkdtempSync(path.join(os.tmpdir(),'cu-bundle-test-'));const old=process.env.CODEWHALE_CU_APP_BUNDLE; |
| 185 | t.after(()=>{if(old===undefined)delete process.env.CODEWHALE_CU_APP_BUNDLE;else process.env.CODEWHALE_CU_APP_BUNDLE=old;fs.rmSync(bundle,{recursive:true,force:true});}); |
| 186 | fs.mkdirSync(path.join(bundle,'Contents','MacOS'),{recursive:true});fs.writeFileSync(path.join(bundle,'Contents','MacOS','accessibility'),'');process.env.CODEWHALE_CU_APP_BUNDLE=bundle; |
| 187 | const calls=[]; |
| 188 | const backend=create({exec:leaseExecutor(async (cmd,args,opts)=>{ |
| 189 | assert.ok(Number.isFinite(opts.timeoutMs)); |
| 190 | if(cmd==='open')return {code:0,stdout:'',stderr:''}; |
| 191 | if(cmd==='screencapture')return {code:1,stdout:'',stderr:'denied'}; |
| 192 | const request=JSON.parse(args[0]);calls.push(request); |
| 193 | return {code:0,stderr:'',stdout:JSON.stringify(request.tool==='app_info'?{found:true,pid:123,bundle_id:'test.app'}:request.tool==='permissions'?{trusted:false}:{action_sent:true})}; |
| 194 | })}); |
| 195 | await backend.open_application({name:'TextEdit',activate:false});await backend.key({text:'return'});await backend.type({text:'Hello 世界 🐋'}); |
| 196 | const events=calls.filter(c=>c.tool==='key_event');assert.equal(events.length,2);assert.equal(events[0].args.code,36);assert.equal(events[0].args.flags,0);assert.equal(events[0].args.input_app_ref.pid,123);assert.equal(events[1].args.down,false); |
| 197 | assert.equal(calls.find(c=>c.tool==='type').args.text,'Hello 世界 🐋'); |
| 198 | const probe=await backend.probe();assert.equal(probe.permissions.accessibility,'denied');assert.equal(probe.capabilities.raw_input,false);assert.equal(probe.capabilities.screenshot,false); |
| 199 | }); |
| 200 | |
| 201 | test('macOS background binding avoids reopen and delivers a held drag at the agent pointer, not the user pointer', async t=>{ |
| 202 | const bundle=fs.mkdtempSync(path.join(os.tmpdir(),'cu-quiet-test-'));const old=process.env.CODEWHALE_CU_APP_BUNDLE; |
| 203 | t.after(()=>{if(old===undefined)delete process.env.CODEWHALE_CU_APP_BUNDLE;else process.env.CODEWHALE_CU_APP_BUNDLE=old;fs.rmSync(bundle,{recursive:true,force:true});}); |
| 204 | fs.mkdirSync(path.join(bundle,'Contents','MacOS'),{recursive:true});fs.writeFileSync(path.join(bundle,'Contents','MacOS','accessibility'),'');process.env.CODEWHALE_CU_APP_BUNDLE=bundle; |
| 205 | const calls=[]; |
| 206 | const backend=create({exec:leaseExecutor(async (cmd,args)=>{ |
| 207 | assert.notEqual(cmd,'open','binding a running app must not reopen its windows'); |
| 208 | const request=JSON.parse(args[0]);calls.push(request); |
| 209 | assert.notEqual(request.tool,'cursor_position','release must not sample the physical pointer'); |
| 210 | const body=request.tool==='app_info'?{found:true,pid:123,bundle_id:'test.app'} |
| 211 | :request.tool==='input_capabilities'?{input_lease:1,window_record:1,background_focus_guard:1} |
| 212 | :{action_sent:true}; |
| 213 | return {code:0,stderr:'',stdout:JSON.stringify(body)}; |
| 214 | })}); |
| 215 | await backend.open_application({name:'TextEdit'}); |
| 216 | assert.equal(calls[0].args.activate,false); |
| 217 | await assert.rejects(backend.left_mouse_up({}),/no agent pointer/); |
| 218 | await backend.open_application({name:'TextEdit',activate:true}); |
| 219 | await backend.left_mouse_down({target:{x:100,y:200}}); |
| 220 | await backend.mouse_move({target:{x:140,y:250}}); |
| 221 | assert.ok(!calls.some(c=>c.tool==='bg_pointer'),'a held button is buffered, not pressed on any real pointer'); |
| 222 | const up=await backend.left_mouse_up({}); |
| 223 | assert.equal(up.pointer_moved,false); |
| 224 | const drag=calls.findLast(c=>c.tool==='bg_pointer'); |
| 225 | assert.equal(calls.filter(c=>c.tool==='bg_pointer').length,1,'the whole drag is one delivery'); |
| 226 | assert.deepEqual([drag.args.steps[1].type,drag.args.steps[1].x,drag.args.steps[1].y],[1,100,200],'pressed where the agent pointer went down'); |
| 227 | assert.ok(drag.args.steps.some(s=>s.x===140&&s.y===250&&s.type===6),'the drag passes the hovered point'); |
| 228 | assert.deepEqual([drag.args.steps.at(-1).type,drag.args.steps.at(-1).x,drag.args.steps.at(-1).y],[2,140,250],'released at the agent pointer'); |
| 229 | assert.equal(drag.args.input_app_ref.pid,123); |
| 230 | assert.ok(!calls.some(c=>['pointer_sequence','release_input','window_at_point'].includes(c.tool))); |
| 231 | assert.ok(!calls.some(c=>c.tool==='preview_notify'),'background actions do not open preview'); |
| 232 | }); |
| 233 | |
| 234 | /** Backend wired to a scripted native helper; returns the requests it made. */ |
| 235 | function stubBackend(t, reply) { |
| 236 | const bundle = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-hit-test-')); |
| 237 | const old = process.env.CODEWHALE_CU_APP_BUNDLE; |
| 238 | t.after(() => { if (old === undefined) delete process.env.CODEWHALE_CU_APP_BUNDLE; else process.env.CODEWHALE_CU_APP_BUNDLE = old; fs.rmSync(bundle, { recursive: true, force: true }); }); |
| 239 | fs.mkdirSync(path.join(bundle, 'Contents', 'MacOS'), { recursive: true }); |
| 240 | fs.writeFileSync(path.join(bundle, 'Contents', 'MacOS', 'accessibility'), ''); |
| 241 | process.env.CODEWHALE_CU_APP_BUNDLE = bundle; |
| 242 | const calls = []; |
| 243 | const backend = create({ exec: leaseExecutor(async (cmd, args) => { |
| 244 | const request = JSON.parse(args[0]); |
| 245 | calls.push(request); |
| 246 | const custom = reply(request); |
| 247 | if (custom?.nativeResult) return custom.nativeResult; |
| 248 | const body = request.tool === 'app_info' ? custom ?? { found: true, pid: 321, bundle_id: 'test.app' } |
| 249 | : custom |
| 250 | ?? (request.tool === 'window_at_point' ? { found: true, owner_pid: 321, owner_name: 'TextEdit', window_id: 9, layer: 0 } |
| 251 | : { action_sent: true, restored: true }); |
| 252 | return { code: 0, stderr: '', stdout: JSON.stringify(body) }; |
| 253 | }) }); |
| 254 | return { backend, calls }; |
| 255 | } |
| 256 | |
| 257 | const PRESSABLE = { found: true, element: { role: 'AXButton', label: 'Tab B', actions: ['AXPress'] }, action: 'AXPress', action_sent: true }; |
| 258 | |
| 259 | test('busy native input keeps its typed refusal through ordinary and held-input routes', async t => { |
| 260 | const { backend } = stubBackend(t, r => { |
| 261 | if (r.tool === 'input_capabilities') return { input_lease: 1, background_focus_guard: 1 }; |
| 262 | if (r.tool === 'type' || r.tool === 'key_event') return { nativeResult: { |
| 263 | code: 1, stdout: '', stderr: 'user_busy: no quiet input window became available; no input was sent.', |
| 264 | } }; |
| 265 | return null; |
| 266 | }); |
| 267 | await backend.open_application({ name: 'Fixture' }); |
| 268 | await assert.rejects(backend.type({ text: 'hello' }), { code: 'user_busy' }); |
| 269 | await assert.rejects(backend.key({ text: 'a' }), { code: 'user_busy' }); |
| 270 | }); |
| 271 | const NOT_PRESSABLE = { found: false, reason: 'no_pressable_element' }; |
| 272 | const FILES_TARGET = { type:'element', app_ref:{pid:321,bundle_id:'test.app'}, windowIndex:0, path:[0,4,2], |
| 273 | role:'AXMenuItem', label:'Files', x:1607, y:692 }; |
| 274 | |
| 275 | test('macOS background scroll and context menus keep the selected element without a global gesture', async t => { |
| 276 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,background_actions:1}:null); |
| 277 | await backend.open_application({name:'Fixture'}); |
| 278 | const area={...FILES_TARGET,role:'AXScrollArea'}; |
| 279 | await backend.scroll({target:area,direction:'down',amount:4}); |
| 280 | assert.deepEqual(calls.find(r=>r.tool==='scroll_element').args.target,area); |
| 281 | await backend.right_click({target:FILES_TARGET}); |
| 282 | assert.equal(calls.find(r=>r.tool==='click_element').args.context,true); |
| 283 | assert.ok(!calls.some(r=>['pointer_sequence','window_at_point','mouse_event'].includes(r.tool))); |
| 284 | }); |
| 285 | |
| 286 | test('macOS scroll cannot retry an ambiguous semantic dispatch or downgrade an old helper', async t => { |
| 287 | let supported=false; |
| 288 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,background_actions:supported?1:0} |
| 289 | :r.tool==='scroll_element'?{nativeResult:{code:null,spawned:true,timedOut:true,stdout:'',stderr:''}}:null); |
| 290 | await backend.open_application({name:'Fixture'}); |
| 291 | await assert.rejects(backend.scroll({target:FILES_TARGET}),e=>e.code==='app_upgrade_required'); |
| 292 | assert.equal(calls.filter(r=>r.tool==='scroll_element').length,0); |
| 293 | supported=true; |
| 294 | await assert.rejects(backend.scroll({target:FILES_TARGET}),e=>e.inputMayHaveBeenSent===true); |
| 295 | assert.equal(calls.filter(r=>r.tool==='scroll_element').length,1); |
| 296 | assert.ok(!calls.some(r=>r.tool==='pointer_sequence')); |
| 297 | }); |
| 298 | |
| 299 | test('macOS unqualified observations stay bound; malformed explicit targets never select the foreground', async t => { |
| 300 | const {backend,calls}=stubBackend(t, r=>r.tool==='get_app_state'?{found:true,pid:321,elements:[]}:null); |
| 301 | await backend.open_application({name:'Fixture'}); |
| 302 | for(const tool of ['get_app_state','list_windows']) { |
| 303 | await backend[tool]({}); |
| 304 | assert.equal(calls.at(-1).args.app_ref.pid,321); |
| 305 | await backend[tool]({app_ref:null}); |
| 306 | assert.equal(calls.at(-1).args.app_ref,null); |
| 307 | } |
| 308 | }); |
| 309 | |
| 310 | test('macOS failed activation cannot leave a previous shared-desktop binding armed', async t => { |
| 311 | let frontmost=true; |
| 312 | const {backend,calls}=stubBackend(t,r=>r.tool==='app_info'?{found:true,pid:321,bundle_id:'test.app',frontmost}:null); |
| 313 | await backend.open_application({name:'Fixture',activate:true}); |
| 314 | frontmost=false; |
| 315 | await assert.rejects(backend.open_application({name:'Fixture',activate:true}),e=>e.code==='activation_not_confirmed'); |
| 316 | await assert.rejects(backend.mouse_move({target:{x:10,y:10}}),e=>e.code==='background_focus_required'); |
| 317 | assert.ok(!calls.some(r=>['pointer_sequence','bg_pointer'].includes(r.tool))); |
| 318 | }); |
| 319 | |
| 320 | test('macOS lease verdict flags hardware input inside the borrow window only', () => { |
| 321 | const base = { front_lease: true, lease_ms: 120, idle_before_s: 5.0 }; |
| 322 | assert.equal(leaseVerdict({ ...base, idle_after_s: 5.12 }), false); |
| 323 | assert.equal(leaseVerdict({ ...base, idle_after_s: 0.05 }), true); |
| 324 | assert.equal(leaseVerdict({ ...base, idle_after_s: 5.12 - 0.24 }), false); |
| 325 | assert.equal(leaseVerdict({ front_lease: false, lease_ms: 120, idle_before_s: 5, idle_after_s: 0.01 }), null); |
| 326 | assert.equal(leaseVerdict({ front_lease: true, lease_ms: 120 }), null); |
| 327 | assert.equal(leaseVerdict(null), null); |
| 328 | const threaded = leaseAccounting({ ...base, idle_after_s: 0.05, user_input_during_lease: true }); |
| 329 | assert.deepEqual(threaded, { lease_ms: 120, idle_before_s: 5.0, idle_after_s: 0.05, user_input_during_lease: true }); |
| 330 | assert.deepEqual(leaseAccounting({ front_lease: true }), {}); |
| 331 | assert.deepEqual(leaseAccounting({ front_lease: false }), {}); |
| 332 | }); |
| 333 | |
| 334 | test('background key focus is refused even when an older helper advertises window records', async t => { |
| 335 | const { backend, calls } = stubBackend(t, r => r.tool === 'input_capabilities' ? { input_lease: 1, window_record: 1 } : null); |
| 336 | for (const args of [{text:'cmd+w'}, {text:'return',target:{type:'element',index:1}}]) { |
| 337 | await assert.rejects(backend.key(args), {code:'background_focus_required'}); |
| 338 | } |
| 339 | assert.ok(!calls.some(r => ['bg_key','key_event'].includes(r.tool))); |
| 340 | }); |
| 341 | |
| 342 | test('macOS open_application reports launched only when it actually launched the app', async t => { |
| 343 | const bundle=fs.mkdtempSync(path.join(os.tmpdir(),'cu-launch-flag-')); |
| 344 | const old=process.env.CODEWHALE_CU_APP_BUNDLE; |
| 345 | t.after(()=>{ if(old===undefined) delete process.env.CODEWHALE_CU_APP_BUNDLE; else process.env.CODEWHALE_CU_APP_BUNDLE=old; fs.rmSync(bundle,{recursive:true,force:true}); }); |
| 346 | fs.mkdirSync(path.join(bundle,'Contents','MacOS'),{recursive:true}); |
| 347 | fs.writeFileSync(path.join(bundle,'Contents','MacOS','accessibility'),''); |
| 348 | process.env.CODEWHALE_CU_APP_BUNDLE=bundle; |
| 349 | let running=true; |
| 350 | const opens=[]; |
| 351 | const backend=create({exec:{run:async(cmd,args)=>{ |
| 352 | if(cmd==='open'){ opens.push(args); running=true; return {code:0,stderr:'',stdout:''}; } |
| 353 | const request=JSON.parse(args[0]); |
| 354 | if(request.tool==='app_info'){ |
| 355 | if(!running) return {code:1,stderr:'application not found',stdout:''}; |
| 356 | return {code:0,stderr:'',stdout:JSON.stringify({found:true,pid:321,name:'Fixture',bundle_id:'test.app',frontmost:false})}; |
| 357 | } |
| 358 | return {code:0,stderr:'',stdout:JSON.stringify({action_sent:true})}; |
| 359 | }}}); |
| 360 | const rebound=await backend.open_application({name:'Fixture'}); |
| 361 | assert.equal(rebound.launched,false); |
| 362 | assert.equal(opens.length,0); |
| 363 | running=false; |
| 364 | const fresh=await backend.open_application({name:'Fixture'}); |
| 365 | assert.equal(fresh.launched,true); |
| 366 | assert.equal(opens.length,1); |
| 367 | assert.ok(opens[0].includes('-g'),'background launch stays in the background'); |
| 368 | }); |
| 369 | |
| 370 | test('macOS app-scoped fallback also refuses with an older helper', async t => { |
| 371 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1}:r.tool==='hit_test'?NOT_PRESSABLE:null); |
| 372 | await backend.open_application({name:'Fixture'}); |
| 373 | await assert.rejects(backend.left_click({target:{x:70,y:80},strategy:'app'}),{code:'background_focus_required'}); |
| 374 | assert.ok(!calls.some(r=>['pointer_sequence','bg_pointer'].includes(r.tool))); |
| 375 | }); |
| 376 | |
| 377 | test('macOS background control never escalates an unavailable semantic action to shared pointer input', async t => { |
| 378 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1,background_actions:1,background_focus_guard:1}:r.tool==='hit_test'?NOT_PRESSABLE:null); |
| 379 | const binding=await backend.open_application({name:'Fixture'}); |
| 380 | assert.equal(binding.input_scope,'application'); |
| 381 | assert.equal(binding.shared_pointer,false); |
| 382 | assert.equal(binding.isolated_desktop,false); |
| 383 | const target={x:70,y:80}; |
| 384 | for(const [tool,args] of [ |
| 385 | ['left_click',{target}], ['left_click',{target,strategy:'event'}], |
| 386 | ['double_click',{target}], ['triple_click',{target}], ['right_click',{target}], |
| 387 | ['middle_click',{target}], ['mouse_move',{target}], ['left_mouse_down',{target}], |
| 388 | ['left_click_drag',{from_target:target,to:{x:90,y:100}}], ['scroll',{target}], |
| 389 | ]) await assert.rejects(backend[tool](args),error=>error.code===(tool==='scroll'?'background_scroll_unavailable':'background_focus_required')); |
| 390 | assert.ok(!calls.some(r=>['pointer_sequence','bg_pointer','release_input','window_at_point'].includes(r.tool))); |
| 391 | await backend.type({text:'Background typing'}); |
| 392 | const typed=calls.filter(r=>r.tool==='type'); |
| 393 | assert.equal(typed.length,1); |
| 394 | assert.equal(typed[0].args.foreground_input,false); |
| 395 | }); |
| 396 | |
| 397 | test('macOS foreground binding never shares the pointer, and returning to background drops a buffered drag', async t => { |
| 398 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,window_record:1,background_actions:1}:null); |
| 399 | const binding=await backend.open_application({name:'Fixture',activate:true}); |
| 400 | assert.equal(binding.input_scope,'shared-desktop'); |
| 401 | assert.equal(binding.shared_pointer,false); |
| 402 | assert.equal(binding.pointer_route,'window-record'); |
| 403 | assert.equal(binding.isolated_desktop,false); |
| 404 | await backend.left_mouse_down({target:{x:70,y:80}}); |
| 405 | await backend.open_application({name:'Fixture',activate:false}); |
| 406 | await assert.rejects(backend.mouse_move({target:{x:90,y:100}}),error=>error.code==='background_focus_required'); |
| 407 | await assert.rejects(backend.left_mouse_up({}),/no agent pointer button is held/); |
| 408 | await backend.releaseInput(); |
| 409 | assert.ok(!calls.some(r=>['bg_pointer','pointer_sequence','release_input'].includes(r.tool))); |
| 410 | }); |
| 411 | |
| 412 | test('macOS element click preserves the observed path despite an oversized frame center', async t => { |
| 413 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1,background_actions:1,background_focus_guard:1}:r.tool==='hit_test'?PRESSABLE:null); |
| 414 | await backend.open_application({name:'Fixture'}); |
| 415 | const receipt=await backend.left_click({target:FILES_TARGET}); |
| 416 | assert.equal(receipt.action_sent,true); |
| 417 | assert.equal(receipt.element.label,'Files'); |
| 418 | assert.equal(receipt.verified,false); |
| 419 | assert.equal(receipt.verification_required,'observation'); |
| 420 | const presses=calls.filter(r=>r.tool==='click_element'); |
| 421 | assert.equal(presses.length,1,'one dispatch, even when the app does not report a changed state'); |
| 422 | assert.deepEqual(presses[0].args.target,FILES_TARGET); |
| 423 | assert.equal(presses[0].args.action,'AXPress'); |
| 424 | assert.ok(!calls.some(r=>['hit_test','pointer_sequence','window_at_point'].includes(r.tool)),'the center never selects another control'); |
| 425 | }); |
| 426 | |
| 427 | for(const reason of ['action is not advertised by this element','element changed label; observe again','window blocked by modal sheet']) |
| 428 | test(`macOS element click does not fall back after ${reason}`, async t => { |
| 429 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1,background_actions:1,background_focus_guard:1} |
| 430 | :r.tool==='click_element'?{nativeResult:{code:1,stdout:'',stderr:reason}}:null); |
| 431 | await backend.open_application({name:'Fixture'}); |
| 432 | await assert.rejects(backend.left_click({target:FILES_TARGET,strategy:'a11y'}),error=>error.message.includes(reason)&&/fresh screenshot or OCR/.test(error.message)); |
| 433 | assert.equal(calls.filter(r=>r.tool==='click_element').length,1); |
| 434 | assert.ok(!calls.some(r=>['hit_test','pointer_sequence'].includes(r.tool))); |
| 435 | }); |
| 436 | |
| 437 | test('macOS ambiguous element press is never retried or converted to pointer input', async t => { |
| 438 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,element_identity:1,background_actions:1,background_focus_guard:1} |
| 439 | :r.tool==='click_element'?{nativeResult:{code:null,spawned:true,timedOut:true,stdout:'',stderr:''}}:null); |
| 440 | await backend.open_application({name:'Fixture'}); |
| 441 | await assert.rejects(backend.left_click({target:FILES_TARGET}),error=>error.inputMayHaveBeenSent===true&&/timed out/.test(error.message)); |
| 442 | assert.equal(calls.filter(r=>r.tool==='click_element').length,1); |
| 443 | assert.ok(!calls.some(r=>['hit_test','pointer_sequence'].includes(r.tool))); |
| 444 | }); |
| 445 | |
| 446 | test('macOS element clicks refuse missing identity, another bound app and an old helper before dispatch', async t => { |
| 447 | const {backend,calls}=stubBackend(t,()=>null); |
| 448 | await backend.open_application({name:'Fixture'}); |
| 449 | await assert.rejects(backend.left_click({target:{...FILES_TARGET,path:undefined}}),/no resolved accessibility identity/); |
| 450 | await assert.rejects(backend.left_click({target:{...FILES_TARGET,app_ref:{pid:999}}}),/bound application/); |
| 451 | await assert.rejects(backend.left_click({target:FILES_TARGET}),/helper needs an update/); |
| 452 | assert.ok(!calls.some(r=>['perform_action','hit_test','pointer_sequence'].includes(r.tool))); |
| 453 | }); |
| 454 | |
| 455 | test('macOS explicit event selection skips the tree and stays window-routed', async t => { |
| 456 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,window_record:1,background_actions:1}:null); |
| 457 | await backend.open_application({name:'Fixture',activate:true}); |
| 458 | const receipt=await backend.left_click({target:FILES_TARGET,strategy:'event'}); |
| 459 | assert.equal(receipt.strategy,'window-record'); |
| 460 | assert.equal(receipt.pointer_moved,false); |
| 461 | const seq=calls.find(r=>r.tool==='bg_pointer'); |
| 462 | assert.deepEqual([seq.args.steps[1].x,seq.args.steps[1].y],[1607,692]); |
| 463 | assert.ok(!calls.some(r=>['perform_action','hit_test','pointer_sequence','window_at_point'].includes(r.tool))); |
| 464 | }); |
| 465 | |
| 466 | test('macOS refuses an old native helper before any held input is dispatched', async t => { |
| 467 | const {backend,calls}=stubBackend(t,request=>request.tool==='input_capabilities'?{input_lease:0}:null); |
| 468 | await backend.open_application({name:'Fixture',activate:true}); |
| 469 | await assert.rejects(backend.key({text:'cmd+n'}),/helper needs an update/); |
| 470 | await assert.rejects(backend.left_mouse_down({target:{x:70,y:80}}),{code:'bg_dispatch_unavailable'}); |
| 471 | assert.ok(!calls.some(request=>['key_event','pointer_sequence','bg_pointer','release_input'].includes(request.tool))); |
| 472 | }); |
| 473 | |
| 474 | test('macOS a buffered drag is never delivered to a different binding', async t => { |
| 475 | const {backend,calls}=stubBackend(t,request=>request.tool==='app_info'?{found:true,pid:request.args.app_ref.name==='First'?321:654,bundle_id:'test.app'}:request.tool==='input_capabilities'?{input_lease:1,window_record:1}:null); |
| 476 | await backend.open_application({name:'First',activate:true}); |
| 477 | await backend.left_mouse_down({target:{x:70,y:80}}); |
| 478 | await backend.open_application({name:'Second',activate:true}); |
| 479 | await assert.rejects(backend.left_mouse_up({}),/no agent pointer button is held/); |
| 480 | assert.ok(!calls.some(request=>['bg_pointer','release_input'].includes(request.tool))); |
| 481 | }); |
| 482 | |
| 483 | test('macOS cancellation releases a held key without replaying it', async t => { |
| 484 | const controller = new AbortController(); |
| 485 | const { backend, calls } = stubBackend(t, request => { |
| 486 | if (request.tool === 'key_event' && request.args.down) controller.abort(); |
| 487 | if (request.tool === 'key_event' && !request.args.down) assert.equal(currentSignal(), null); |
| 488 | }); |
| 489 | await backend.open_application({name:'Fixture',activate:true}); |
| 490 | await assert.rejects(withSignal(controller.signal, () => backend.hold_key({text:'shift+a', duration:30})), /cancelled/); |
| 491 | assert.deepEqual(calls.filter(r=>r.tool==='key_event').map(r=>r.args.down), [true,false]); |
| 492 | }); |
| 493 | |
| 494 | test('macOS session cleanup has no real button to release', async t => { |
| 495 | const { backend, calls } = stubBackend(t, r=>r.tool==='input_capabilities'?{input_lease:1,window_record:1,background_actions:1}:null); |
| 496 | await backend.open_application({name:'Fixture',activate:true}); |
| 497 | await backend.releaseInput(); |
| 498 | await backend.left_mouse_down({target:{x:70,y:80}}); |
| 499 | await withSignal(AbortSignal.abort(), () => backend.releaseInput()); |
| 500 | await assert.rejects(backend.left_mouse_up({}),/no agent pointer button is held/); |
| 501 | assert.ok(!calls.some(r=>['release_input','bg_pointer','pointer_sequence'].includes(r.tool))); |
| 502 | }); |
| 503 | |
| 504 | test('macOS foreground delivery requires explicit activation and resets on background binding', async t => { |
| 505 | const { backend, calls } = stubBackend(t, () => null); |
| 506 | await backend.open_application({name:'Fixture',activate:true}); |
| 507 | assert.equal((await backend.key({text:'return'})).keyboard_delivery,'foreground-guarded'); |
| 508 | assert.ok(calls.filter(r=>r.tool==='key_event').every(r=>r.args.foreground_input)); |
| 509 | await backend.open_application({name:'Fixture',activate:false}); |
| 510 | assert.equal((await backend.key({text:'return'})).keyboard_delivery,'process'); |
| 511 | assert.equal(calls.at(-1).args.foreground_input,false); |
| 512 | }); |
| 513 | |
| 514 | test('background modifier holds refuse while plain process keys remain available', async t => { |
| 515 | const {backend,calls}=stubBackend(t,()=>null); |
| 516 | await backend.open_application({name:'Fixture'}); |
| 517 | await assert.rejects(backend.hold_key({text:'cmd+a',duration:0.05}),{code:'background_focus_required'}); |
| 518 | assert.ok(!calls.some(c=>['bg_key','key_event'].includes(c.tool))); |
| 519 | assert.equal((await backend.key({text:'tab'})).keyboard_delivery,'process'); |
| 520 | assert.ok(calls.filter(c=>c.tool==='key_event').every(c=>c.args.foreground_input===false)); |
| 521 | }); |
| 522 | |
| 523 | test('background pointer fallbacks refuse without dispatch or moving focus', async t => { |
| 524 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities' |
| 525 | ?{input_lease:1,window_record:1,background_actions:1}:r.tool==='hit_test'?NOT_PRESSABLE:null); |
| 526 | await backend.open_application({name:'Fixture'}); |
| 527 | for (const call of [ |
| 528 | ()=>backend.left_click({target:{x:20,y:20}}), |
| 529 | ()=>backend.left_click({target:{x:20,y:20},strategy:'app'}), |
| 530 | ()=>backend.left_click_drag({from_target:{x:20,y:20},to:{x:30,y:30}}), |
| 531 | ()=>backend.scroll({target:{x:20,y:20},direction:'down'}), |
| 532 | ]) await assert.rejects(call(),{code:'background_focus_required'}); |
| 533 | assert.ok(!calls.some(c=>['bg_pointer','pointer_sequence'].includes(c.tool))); |
| 534 | }); |
| 535 | |
| 536 | test('background web value replacement refuses before focusing or selecting text', async t => { |
| 537 | const {backend,calls}=stubBackend(t,r=>r.tool==='set_value'?{nativeResult:{code:1,stderr:'web area refuses direct AXValue'}}:null); |
| 538 | await assert.rejects(backend.set_value({target:{type:'element',index:1},value:'replacement'}),{code:'background_focus_required'}); |
| 539 | assert.ok(!calls.some(c=>['focus_element','bg_key','type'].includes(c.tool))); |
| 540 | }); |
| 541 | |
| 542 | test('background typing refuses a helper without the native focus guard', async t => { |
| 543 | const {backend,calls}=stubBackend(t,r=>r.tool==='input_capabilities'?{input_lease:1,window_record:1}:null); |
| 544 | await assert.rejects(backend.type({text:'Hello 🐋'}),{code:'app_upgrade_required'}); |
| 545 | assert.ok(!calls.some(c=>c.tool==='type')); |
| 546 | }); |
| 547 | |
| 548 | test('macOS input handlers refuse a missing target without dereferencing it', async t => { |
| 549 | const {backend,calls}=stubBackend(t,()=>null); |
| 550 | await backend.open_application({name:'Fixture'}); |
| 551 | for (const call of [ |
| 552 | ()=>backend.left_mouse_down({}), |
| 553 | ()=>backend.left_mouse_down(), |
| 554 | ()=>backend.mouse_move({}), |
| 555 | ()=>backend.left_click({}), |
| 556 | ()=>backend.scroll({}), |
| 557 | ()=>backend.select_text({}), |
| 558 | ()=>backend.set_value({value:'x'}), |
| 559 | ()=>backend.perform_action({action:'AXPress'}), |
| 560 | ]) { |
| 561 | await assert.rejects(call, error=>!(error instanceof TypeError)); |
| 562 | } |
| 563 | assert.ok(!calls.some(c=>['pointer_sequence','bg_pointer','select_text','set_value','perform_action'].includes(c.tool))); |
| 564 | }); |
| 565 | |
| 566 | test('macOS foreground refusal never sends an unowned global key-up', async t => { |
| 567 | const { backend, calls } = stubBackend(t, request => request.tool === 'key_event' && request.args.down |
| 568 | ? { nativeResult: { code: 1, spawned: true, stdout: '', stderr: 'foreground changed to Mail (pid 999); expected Fixture (pid 321)' } } : null); |
| 569 | await backend.open_application({name:'Fixture',activate:true}); |
| 570 | await assert.rejects(backend.key({text:'cmd+n'}), /foreground changed to Mail \(pid 999\)/); |
| 571 | await assert.rejects(backend.hold_key({text:'shift+a',duration:30}), /expected Fixture/); |
| 572 | assert.deepEqual(calls.filter(r=>r.tool==='key_event').map(r=>r.args.down), [true,true]); |
| 573 | }); |
| 574 | |
| 575 | test('macOS cancellation before child spawn does not release keys it never pressed', async t => { |
| 576 | const { backend, calls } = stubBackend(t, request => request.tool === 'key_event' && request.args.down |
| 577 | ? { nativeResult: { code: -1, spawned: false, aborted: true, stdout: '', stderr: '' } } : null); |
| 578 | await backend.open_application({name:'Fixture',activate:true}); |
| 579 | await assert.rejects(backend.hold_key({text:'shift+a',duration:30}), /cancelled/); |
| 580 | assert.deepEqual(calls.filter(r=>r.tool==='key_event').map(r=>r.args.down), [true]); |
| 581 | }); |
| 582 | |
| 583 | for (const failure of ['aborted','timedOut']) test(`macOS ${failure} after child spawn releases an ambiguous key press`, async t => { |
| 584 | const { backend, calls } = stubBackend(t, request => { |
| 585 | if(request.tool==='key_event' && request.args.down) return { nativeResult: { code: null, spawned: true, [failure]: true, stdout: '', stderr: '' } }; |
| 586 | if(request.tool==='key_event' && !request.args.down) assert.equal(currentSignal(),null); |
| 587 | }); |
| 588 | await backend.open_application({name:'Fixture',activate:true}); |
| 589 | await assert.rejects(backend.key({text:'cmd+n'}), /cancelled|timed out/); |
| 590 | const events=calls.filter(r=>r.tool==='key_event'); |
| 591 | assert.deepEqual(events.map(r=>r.args.down), [true,false]); |
| 592 | assert.equal(events[1].args.owned_release,true); |
| 593 | }); |
| 594 | |
| 595 | test('macOS a cancelled drag delivery leaves nothing held', async t => { |
| 596 | const { backend, calls } = stubBackend(t, request => request.tool==='bg_pointer' |
| 597 | ? {nativeResult:{code:null,spawned:true,aborted:true,stdout:'',stderr:''}} |
| 598 | : request.tool==='input_capabilities' ? {input_lease:1,window_record:1} : null); |
| 599 | await backend.open_application({name:'Fixture',activate:true}); |
| 600 | await backend.left_mouse_down({target:{x:70,y:80}}); |
| 601 | await assert.rejects(backend.left_mouse_up({target:{x:120,y:80}}), /cancelled/); |
| 602 | await assert.rejects(backend.left_mouse_up({}), /no agent pointer button is held/); |
| 603 | assert.equal(calls.filter(r=>r.tool==='bg_pointer').length,1); |
| 604 | assert.ok(!calls.some(r=>['pointer_sequence','release_input'].includes(r.tool))); |
| 605 | }); |
| 606 | |
| 607 | test('macOS coordinate left_click prefers the accessibility element under the point', async (t) => { |
| 608 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'hit_test' ? PRESSABLE : null)); |
| 609 | await backend.open_application({ name: 'TextEdit' }); |
| 610 | const receipt = await backend.left_click({ target: { x: 220, y: 180 } }); |
| 611 | assert.equal(receipt.strategy, 'a11y'); |
| 612 | assert.equal(receipt.action, 'AXPress'); |
| 613 | assert.equal(receipt.element.label, 'Tab B'); |
| 614 | const hit = calls.find((c) => c.tool === 'hit_test'); |
| 615 | assert.deepEqual([hit.args.x, hit.args.y, hit.args.perform], [220, 180, true]); |
| 616 | assert.equal(hit.args.input_app_ref.pid, 321); |
| 617 | assert.ok(!calls.some((c) => c.tool === 'mouse_event'), 'a semantic press must not also post raw pointer events'); |
| 618 | }); |
| 619 | |
| 620 | test('macOS coordinate left_click falls back to a window-routed click when no element is pressable', async (t) => { |
| 621 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'input_capabilities' ? {input_lease:1,background_actions:1,window_record:1} : r.tool === 'hit_test' ? NOT_PRESSABLE : null)); |
| 622 | await backend.open_application({ name: 'TextEdit', activate:true }); |
| 623 | const receipt = await backend.left_click({ target: { x: 40, y: 90 } }); |
| 624 | assert.equal(receipt.strategy, 'window-record'); |
| 625 | assert.equal(receipt.pointer_moved, false, 'the user cursor never moves'); |
| 626 | assert.equal(receipt.a11y_reason, 'no_pressable_element'); |
| 627 | const seq = calls.find((c) => c.tool === 'bg_pointer'); |
| 628 | assert.deepEqual(seq.args.steps.map((s) => s.type), [5, 1, 2], 'move, down, up in one gesture'); |
| 629 | assert.deepEqual([seq.args.steps[1].x, seq.args.steps[1].y, seq.args.steps[1].clickState], [40, 90, 1]); |
| 630 | assert.ok(!calls.some((c) => ['pointer_sequence', 'window_at_point'].includes(c.tool))); |
| 631 | }); |
| 632 | |
| 633 | test('macOS click strategies: event skips the tree and a11y fails closed', async (t) => { |
| 634 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'input_capabilities' ? {input_lease:1,background_actions:1,window_record:1} : r.tool === 'hit_test' ? NOT_PRESSABLE : null)); |
| 635 | await backend.open_application({ name: 'TextEdit', activate:true }); |
| 636 | |
| 637 | const forced = await backend.left_click({ target: { x: 10, y: 20 }, strategy: 'event' }); |
| 638 | assert.equal(forced.strategy, 'window-record'); |
| 639 | assert.ok(!calls.some((c) => c.tool === 'hit_test'), 'strategy=event never hit-tests'); |
| 640 | |
| 641 | await assert.rejects(backend.left_click({ target: { x: 10, y: 20 }, strategy: 'a11y' }), /no supported accessibility click/); |
| 642 | await assert.rejects(backend.left_click({ target: { x: 10, y: 20 }, strategy: 'sideways' }), /strategy must be auto, a11y, app or event/); |
| 643 | |
| 644 | calls.length = 0; |
| 645 | const dbl = await backend.double_click({ target: { x: 10, y: 20 } }); |
| 646 | assert.equal(dbl.strategy, 'window-record'); |
| 647 | assert.deepEqual(calls.find((c) => c.tool === 'bg_pointer').args.steps.map((s) => s.clickState), [0, 1, 1, 2, 2]); |
| 648 | assert.equal((await backend.right_click({ target: { x: 10, y: 20 } })).strategy, 'window-record'); |
| 649 | assert.equal(calls.find(c => c.tool === 'hit_test').args.operation, 'context'); |
| 650 | }); |
| 651 | |
| 652 | test('macOS drag and scroll travel as one window-routed gesture that never moves the cursor', async (t) => { |
| 653 | const { backend, calls } = stubBackend(t, r=>r.tool==='input_capabilities'?{input_lease:1,window_record:1,background_actions:1}:null); |
| 654 | await backend.open_application({ name: 'TextEdit', activate:true }); |
| 655 | |
| 656 | const drag = await backend.left_click_drag({ from_target: { x: 10, y: 10 }, to: { x: 110, y: 10 } }); |
| 657 | assert.equal(drag.pointer_moved, false); |
| 658 | const dragSeq = calls.find((c) => c.tool === 'bg_pointer'); |
| 659 | assert.equal(dragSeq.args.steps.at(-1).type, 2, 'released at the destination'); |
| 660 | assert.deepEqual([dragSeq.args.steps.at(-1).x, dragSeq.args.steps.at(-1).y], [110, 10]); |
| 661 | |
| 662 | calls.length = 0; |
| 663 | const scrolled = await backend.scroll({ target: { x: 10, y: 10 }, direction: 'down', amount: 3 }); |
| 664 | assert.equal(scrolled.pointer_moved, false); |
| 665 | const scrollSeq = calls.find((c) => c.tool === 'bg_pointer'); |
| 666 | const notches = scrollSeq.args.steps.filter((s) => s.scroll); |
| 667 | assert.equal(notches.length, 3, 'one notch per unit of amount, like a real wheel'); |
| 668 | assert.deepEqual(notches[0].scroll, [0, -1]); |
| 669 | assert.deepEqual([notches[0].x, notches[0].y], [10, 10]); |
| 670 | assert.ok(!calls.some((c) => c.tool === 'pointer_sequence')); |
| 671 | }); |
| 672 | |
| 673 | test('native hit_test fails closed without a bound application', { skip: process.platform !== 'darwin' }, (t) => { |
| 674 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-hit-native-')); |
| 675 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 676 | const binary = path.join(dir, 'native'); |
| 677 | const build = spawnSync('clang', ['-DCU_TEST=1', '-fobjc-arc', '-Os', '-framework', 'Cocoa', '-framework', 'ApplicationServices', '-framework', 'ScreenCaptureKit', '-framework', 'AVFoundation', '-framework', 'CoreMedia', '-framework', 'Vision', 'src/backends/darwin-accessibility.m', '-o', binary], { encoding: 'utf8' }); |
| 678 | assert.equal(build.status, 0, build.stderr); |
| 679 | const r = spawnSync(binary, [JSON.stringify({ tool: 'hit_test', args: { x: 1, y: 1, perform: true } })], { encoding: 'utf8' }); |
| 680 | assert.equal(r.status, 1); |
| 681 | assert.match(r.stderr, /open_application first/); |
| 682 | }); |
| 683 | |
| 684 | test('native type verifies delivery against the focused control and fails closed on a non-text focus', { skip: process.platform !== 'darwin' }, (t) => { |
| 685 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-type-native-')); |
| 686 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 687 | const binary = path.join(dir, 'native'); |
| 688 | const build = spawnSync('clang', ['-DCU_TEST=1', '-ftrivial-auto-var-init=pattern', '-fobjc-arc', '-Os', '-framework', 'Cocoa', '-framework', 'ApplicationServices', '-framework', 'ScreenCaptureKit', '-framework', 'AVFoundation', '-framework', 'CoreMedia', '-framework', 'Vision', 'src/backends/darwin-accessibility.m', '-o', binary], { encoding: 'utf8' }); |
| 689 | assert.equal(build.status, 0, build.stderr); |
| 690 | const type = (args) => spawnSync(binary, [JSON.stringify({ tool: 'inspect_type', args })], { encoding: 'utf8' }); |
| 691 | |
| 692 | let r = type({ text: 'world', focused: { AXRole: 'AXTextField', AXValue: 'Hello ', after: 'Hello world' } }); |
| 693 | assert.equal(r.status, 0, r.stderr); |
| 694 | let receipt = JSON.parse(r.stdout); |
| 695 | assert.equal(receipt.action_sent, true); |
| 696 | assert.equal(receipt.chars, 5); |
| 697 | assert.equal(receipt.strategy, 'unicode-events'); |
| 698 | assert.equal(receipt.verified, true, 'suffix match confirms delivery'); |
| 699 | assert.equal(receipt.focused_role, 'AXTextField'); |
| 700 | assert.ok(!('verification_required' in receipt)); |
| 701 | |
| 702 | r = type({ text: 'their', focused: { AXRole: 'AXTextArea', AXValue: 'I love ', after: 'I love thier' } }); |
| 703 | assert.equal(JSON.parse(r.stdout).verified, false, 'matching length does not establish that the requested text landed'); |
| 704 | r = type({ text: 'world', focused: { AXRole: 'AXTextField', AXValue: 'Hello world', after: 'Hello world' } }); |
| 705 | assert.equal(JSON.parse(r.stdout).verified, false, 'an existing suffix must not verify a dropped insertion'); |
| 706 | |
| 707 | r = type({ text: 'x', focused: { AXRole: 'AXButton', AXTitle: 'Save' } }); |
| 708 | assert.equal(r.status, 1, 'a clearly non-text focus refuses before any event is posted'); |
| 709 | assert.match(r.stderr, /focused element is a AXButton, not a text control/); |
| 710 | |
| 711 | for (const focused of [null, { AXRole: 'AXWebArea' }, { AXRole: 'AXSecureTextField' }, { AXRole: 'AXTextField', AXValue: 'abc' }]) { |
| 712 | r = type({ text: 'hi', focused }); |
| 713 | assert.equal(r.status, 0, r.stderr); |
| 714 | receipt = JSON.parse(r.stdout); |
| 715 | assert.equal(receipt.action_sent, true, 'unverifiable readbacks still report dispatch'); |
| 716 | assert.equal(receipt.verified, false); |
| 717 | assert.equal(receipt.verification_required, 'screenshot'); |
| 718 | } |
| 719 | const unleased = JSON.parse(type({ text: 'hi', focused: null }).stdout); |
| 720 | assert.equal(unleased.focused_role, null); |
| 721 | for (const field of ['front_restored', 'lease_ms', 'idle_before_s', 'idle_after_s']) { |
| 722 | assert.ok(!(field in unleased), `${field} must not be invented for typing without a lease`); |
| 723 | } |
| 724 | |
| 725 | // Exercise the real wait with a deterministic HID clock, without posting |
| 726 | // input. Reuse this native build; cancellation and a busy deadline refuse. |
| 727 | const yieldToUser = (args) => spawnSync(binary, [JSON.stringify({ tool: 'inspect_user_yield', |
| 728 | args: { yield_gap_ms: 450, yield_wait_ms: 60, ...args } })], { encoding: 'utf8' }); |
| 729 | const idle = yieldToUser({ idle_seconds: 1 }); |
| 730 | assert.equal(idle.status, 0, idle.stderr); |
| 731 | assert.ok(Number.isFinite(JSON.parse(idle.stdout).yield_ms)); |
| 732 | for (const idle_seconds of [0, -1]) { |
| 733 | const busy = yieldToUser({ idle_seconds }); |
| 734 | assert.equal(busy.status, 1, 'busy or unavailable HID clock must refuse'); |
| 735 | assert.match(busy.stderr, /^user_busy:.*no input was sent/); |
| 736 | assert.equal(busy.stdout, ''); |
| 737 | } |
| 738 | const cancelled = yieldToUser({ idle_seconds: 0, cancelled: true }); |
| 739 | assert.equal(cancelled.status, 1); |
| 740 | assert.match(cancelled.stderr, /computer request cancelled/); |
| 741 | assert.equal(JSON.parse(yieldToUser({ idle_seconds: 0, yield_gap_ms: 0 }).stdout).yield_ms, 0); |
| 742 | |
| 743 | }); |
| 744 | |
| 745 | test('macOS type passes the native verification receipt through untouched', async (t) => { |
| 746 | const nativeReceipt = { action_sent: true, chars: 5, strategy: 'unicode-events', keyboard_delivery: 'process', verified: false, focused_role: null, verification_required: 'screenshot' }; |
| 747 | const { backend } = stubBackend(t, (r) => (r.tool === 'type' ? nativeReceipt : null)); |
| 748 | await backend.open_application({ name: 'TextEdit' }); |
| 749 | const { preview_error, ...receipt } = await backend.type({ text: 'hello' }); |
| 750 | assert.deepEqual(receipt, nativeReceipt); |
| 751 | assert.ok(preview_error, 'preview refresh failure is reported, not swallowed'); |
| 752 | }); |
| 753 | |
| 754 | // ---------- dogfood 2026-09-17: app_not_found, live preview, sessions, kill ---------- |
| 755 | |
| 756 | const nap = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); |
| 757 | |
| 758 | function withEnv(t, vars) { |
| 759 | const old = {}; |
| 760 | for (const [k, v] of Object.entries(vars)) { |
| 761 | old[k] = process.env[k]; |
| 762 | if (v === undefined) delete process.env[k]; else process.env[k] = v; |
| 763 | } |
| 764 | t.after(() => { for (const [k, v] of Object.entries(old)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; } }); |
| 765 | } |
| 766 | |
| 767 | function fakeBundle(t) { |
| 768 | const bundle = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-fake-bundle-')); |
| 769 | t.after(() => fs.rmSync(bundle, { recursive: true, force: true })); |
| 770 | fs.mkdirSync(path.join(bundle, 'Contents', 'MacOS'), { recursive: true }); |
| 771 | fs.writeFileSync(path.join(bundle, 'Contents', 'MacOS', 'accessibility'), ''); |
| 772 | return bundle; |
| 773 | } |
| 774 | |
| 775 | test('open_application on an unknown name, bundle or pid fails as app_not_found', async (t) => { |
| 776 | withEnv(t, { CODEWHALE_CU_APP_BUNDLE: fakeBundle(t) }); |
| 777 | const backend = create({ exec: { |
| 778 | async run(cmd, args) { |
| 779 | if (cmd === 'open') { |
| 780 | const stderr = args.includes('-b') |
| 781 | ? 'LSCopyApplicationURLsForBundleIdentifier() failed while trying to determine the application with bundle identifier com.nonexistent.app.' |
| 782 | : "Unable to find application named 'NoSuchAppZZZ'"; |
| 783 | return { code: 1, stderr, stdout: '' }; |
| 784 | } |
| 785 | const request = JSON.parse(args[0]); |
| 786 | if (request.tool === 'app_info') { |
| 787 | const pid = request.args?.app_ref?.pid; |
| 788 | return pid |
| 789 | ? { code: 1, stderr: `no running application with pid ${pid}`, stdout: '' } |
| 790 | : { code: 1, stderr: 'application not found', stdout: '' }; |
| 791 | } |
| 792 | return { code: 0, stderr: '', stdout: '{}' }; |
| 793 | }, |
| 794 | async runInputLease() { throw new Error('not used'); }, |
| 795 | } }); |
| 796 | await assert.rejects(backend.open_application({ name: 'NoSuchAppZZZ' }), |
| 797 | (e) => e.code === 'app_not_found' && /open failed: Unable to find application/.test(e.message), |
| 798 | 'a name that resolves nowhere is app_not_found, not tool_error'); |
| 799 | await assert.rejects(backend.open_application({ bundle_id: 'com.nonexistent.app' }), |
| 800 | (e) => e.code === 'app_not_found', 'a bundle id that resolves nowhere is app_not_found'); |
| 801 | await assert.rejects(backend.open_application({ pid: 999999 }), |
| 802 | (e) => e.code === 'app_not_found', 'a dead pid is app_not_found'); |
| 803 | }); |
| 804 | |
| 805 | test('preview goes live after a real capture; mute and session close tear it down', async (t) => { |
| 806 | withEnv(t, { |
| 807 | CODEWHALE_CU_APP_BUNDLE: fakeBundle(t), |
| 808 | CODEWHALE_CU_STATE_DIR: fs.mkdtempSync(path.join(os.tmpdir(), 'cu-preview-state-')), |
| 809 | CODEWHALE_CU_PREVIEW_REFRESH_MS: '60', |
| 810 | }); |
| 811 | const calls = []; |
| 812 | const backend = create({ exec: { |
| 813 | async run(cmd, args) { |
| 814 | if (cmd === 'screencapture') { |
| 815 | fs.writeFileSync(args[args.length - 1], 'png'); |
| 816 | calls.push({ tool: 'screencapture' }); |
| 817 | return { code: 0, stderr: '', stdout: '' }; |
| 818 | } |
| 819 | const request = JSON.parse(args[0]); |
| 820 | calls.push(request); |
| 821 | const body = request.tool === 'app_info' ? { found: true, pid: 123, bundle_id: 'test.app', name: 'TextEdit' } |
| 822 | : request.tool === 'window_info' ? { window_id: 9, name: 'TextEdit', points: { x: 0, y: 0, w: 100, h: 100 } } |
| 823 | : request.tool === 'cursor_position' ? { x: 1, y: 2 } |
| 824 | : { updated: true }; |
| 825 | return { code: 0, stderr: '', stdout: JSON.stringify(body) }; |
| 826 | }, |
| 827 | async runInputLease() { throw new Error('not used'); }, |
| 828 | } }); |
| 829 | const captures = () => calls.filter((c) => c.tool === 'window_info').length; |
| 830 | |
| 831 | await backend.open_application({ name: 'TextEdit' }); |
| 832 | await nap(400); |
| 833 | const live = captures(); |
| 834 | assert.ok(live >= 2, `the panel refreshes on a timer while bound (${live} captures)`); |
| 835 | assert.ok(calls.some((c) => c.tool === 'preview_notify' && c.args?.enabled === true), 'binding shows the panel'); |
| 836 | |
| 837 | await backend.preview({ enabled: false }); |
| 838 | assert.ok(calls.some((c) => c.tool === 'preview_notify' && c.args?.enabled === false), 'mute hides the panel'); |
| 839 | const muted = captures(); |
| 840 | await nap(300); |
| 841 | assert.equal(captures(), muted, 'muting stops the refresh loop'); |
| 842 | |
| 843 | await backend.preview({ enabled: true }); |
| 844 | await nap(300); |
| 845 | assert.ok(captures() > muted, 're-enabling restarts the loop'); |
| 846 | |
| 847 | await backend.closeSession(); |
| 848 | assert.equal(calls.at(-1).tool, 'preview_notify'); |
| 849 | assert.equal(calls.at(-1).args.enabled, false, 'session close hides the panel it showed'); |
| 850 | const closed = captures(); |
| 851 | await nap(300); |
| 852 | assert.equal(captures(), closed, 'session close stops the loop'); |
| 853 | }); |
| 854 | |
| 855 | test('CODEWHALE_CU_PREVIEW_REFRESH_MS=0 keeps the panel a single frame', async (t) => { |
| 856 | withEnv(t, { |
| 857 | CODEWHALE_CU_APP_BUNDLE: fakeBundle(t), |
| 858 | CODEWHALE_CU_STATE_DIR: fs.mkdtempSync(path.join(os.tmpdir(), 'cu-preview-off-')), |
| 859 | CODEWHALE_CU_PREVIEW_REFRESH_MS: '0', |
| 860 | }); |
| 861 | const calls = []; |
| 862 | const backend = create({ exec: { |
| 863 | async run(cmd, args) { |
| 864 | if (cmd === 'screencapture') { fs.writeFileSync(args[args.length - 1], 'png'); return { code: 0, stderr: '', stdout: '' }; } |
| 865 | const request = JSON.parse(args[0]); |
| 866 | calls.push(request); |
| 867 | const body = request.tool === 'app_info' ? { found: true, pid: 123, bundle_id: 'test.app', name: 'TextEdit' } |
| 868 | : request.tool === 'window_info' ? { window_id: 9, name: 'TextEdit', points: { x: 0, y: 0, w: 10, h: 10 } } |
| 869 | : request.tool === 'cursor_position' ? { x: 0, y: 0 } |
| 870 | : { updated: true }; |
| 871 | return { code: 0, stderr: '', stdout: JSON.stringify(body) }; |
| 872 | }, |
| 873 | async runInputLease() { throw new Error('not used'); }, |
| 874 | } }); |
| 875 | await backend.open_application({ name: 'TextEdit' }); |
| 876 | await nap(350); |
| 877 | assert.equal(calls.filter((c) => c.tool === 'window_info').length, 1, 'exactly the bind capture, no timer'); |
| 878 | }); |
| 879 | |
| 880 | test('list_sessions in direct mode reports this process as the only session', async (t) => { |
| 881 | const { backend } = stubBackend(t, (r) => (r.tool === 'app_info' ? { found: true, pid: 321, bundle_id: 'test.app', name: 'TextEdit' } : null)); |
| 882 | assert.equal((await backend.list_sessions()).sessions[0].target, null, 'unbound direct session has no target'); |
| 883 | await backend.open_application({ name: 'TextEdit' }); |
| 884 | const s = await backend.list_sessions(); |
| 885 | assert.equal(s.via, 'direct'); |
| 886 | assert.equal(s.count, 1); |
| 887 | assert.deepEqual(s.sessions[0].target, { pid: 321, bundle_id: 'test.app', name: 'TextEdit' }); |
| 888 | assert.equal(s.sessions[0].mode, 'background'); |
| 889 | assert.equal(s.sessions[0].inputHeld, false); |
| 890 | }); |
| 891 | |
| 892 | test('list_apps {installed:true} returns the installed catalog with running flags', async (t) => { |
| 893 | const catalog = { apps: [{ name: 'Safari', bundle_id: 'com.apple.Safari', path: '/Applications/Safari.app', running: true, pid: 42 }, { name: 'Calculator', bundle_id: 'com.apple.calculator', path: '/System/Applications/Calculator.app', running: false }], count: 2 }; |
| 894 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'installed_apps' ? catalog : null)); |
| 895 | const r = await backend.list_apps({ installed: true }); |
| 896 | assert.equal(r.installed, true); |
| 897 | assert.equal(r.apps.length, 2); |
| 898 | assert.equal(r.apps[0].running, true); |
| 899 | assert.equal(r.apps[1].running, false); |
| 900 | assert.match(r.note, /takes a moment/); |
| 901 | assert.ok(calls.some((c) => c.tool === 'installed_apps')); |
| 902 | assert.ok(!calls.some((c) => c.tool === 'list_apps'), 'the running-process list is not consulted'); |
| 903 | }); |
| 904 | |
| 905 | test('set_window_frame validates geometry and the window index, then passes the readback through', async (t) => { |
| 906 | const receipt = { action_sent: true, window_id: 0, before: { x: 0, y: 0, w: 100, h: 100 }, after: { x: 40, y: 40, w: 300, h: 200 }, verified: true }; |
| 907 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'set_window_frame' ? receipt : null)); |
| 908 | await assert.rejects(backend.set_window_frame({ window_id: 0, frame: { x: 1, y: 2, w: 0, h: 5 } }), (e) => e.code === 'bad_args'); |
| 909 | await assert.rejects(backend.set_window_frame({ window_id: -1, frame: { x: 1, y: 2, w: 3, h: 5 } }), (e) => e.code === 'bad_args'); |
| 910 | await assert.rejects(backend.set_window_frame({ window_id: 0, frame: { x: 1, y: 2, w: Number.NaN, h: 5 } }), (e) => e.code === 'bad_args'); |
| 911 | const r = await backend.set_window_frame({ window_id: 0, frame: { x: 40, y: 40, w: 300, h: 200 } }); |
| 912 | assert.equal(r.verified, true); |
| 913 | assert.deepEqual(r.after, { x: 40, y: 40, w: 300, h: 200 }); |
| 914 | const sent = calls.filter((c) => c.tool === 'set_window_frame').at(-1); |
| 915 | assert.deepEqual(sent.args.frame, { x: 40, y: 40, w: 300, h: 200 }); |
| 916 | assert.equal(sent.args.window_id, 0); |
| 917 | }); |
| 918 | |
| 919 | test('kill_app validates its identity client-side and passes the native receipt through', async (t) => { |
| 920 | const receipt = { killed: true, pid: 321, name: 'TextEdit', force_used: false }; |
| 921 | const { backend, calls } = stubBackend(t, (r) => (r.tool === 'kill_app' ? receipt : null)); |
| 922 | await assert.rejects(backend.kill_app({}), (e) => e.code === 'bad_args', 'an identity is required'); |
| 923 | assert.deepEqual(await backend.kill_app({ pid: 321 }), receipt); |
| 924 | const sent = calls.filter((c) => c.tool === 'kill_app').at(-1); |
| 925 | assert.equal(sent.args.pid, 321); |
| 926 | assert.equal(sent.args.force, false, 'force defaults to a graceful quit'); |
| 927 | await backend.kill_app({ name: 'TextEdit', force: true }); |
| 928 | assert.equal(calls.filter((c) => c.tool === 'kill_app').at(-1).args.force, true, 'force passes through'); |
| 929 | }); |
| 930 | |
| 931 | // A 1x1 PNG is enough: screenshot reads its IHDR for the pixel ground truth. |
| 932 | const PNG_1X1 = Buffer.from( |
| 933 | 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', |
| 934 | 'base64', |
| 935 | ); |
| 936 | |
| 937 | // list_displays reports `index` and `id` separately (a lone main display is |
| 938 | // commonly index 1 / id 3). Passing the id where the index belongs must be a |
| 939 | // clean refusal, never a capture labelled with another display's geometry: |
| 940 | // those points/scale feed every later coordinate target. |
| 941 | function displayBackend(t, { displays }) { |
| 942 | const bundle = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-display-test-')); |
| 943 | const old = process.env.CODEWHALE_CU_APP_BUNDLE; |
| 944 | t.after(() => { if (old === undefined) delete process.env.CODEWHALE_CU_APP_BUNDLE; else process.env.CODEWHALE_CU_APP_BUNDLE = old; fs.rmSync(bundle, { recursive: true, force: true }); }); |
| 945 | fs.mkdirSync(path.join(bundle, 'Contents', 'MacOS'), { recursive: true }); |
| 946 | fs.writeFileSync(path.join(bundle, 'Contents', 'MacOS', 'accessibility'), ''); |
| 947 | process.env.CODEWHALE_CU_APP_BUNDLE = bundle; |
| 948 | const captures = []; |
| 949 | const backend = create({ exec: leaseExecutor(async (cmd, args) => { |
| 950 | if (cmd === 'screencapture') { |
| 951 | captures.push(args); |
| 952 | fs.writeFileSync(args.at(-1), PNG_1X1); |
| 953 | return { code: 0, stdout: '', stderr: '' }; |
| 954 | } |
| 955 | const request = JSON.parse(args[0]); |
| 956 | return { code: 0, stderr: '', stdout: JSON.stringify(request.tool === 'displays' ? displays : { action_sent: true }) }; |
| 957 | }) }); |
| 958 | return { backend, captures }; |
| 959 | } |
| 960 | |
| 961 | test('macOS screenshot rejects a display id used as an index instead of mislabelling the raster', async (t) => { |
| 962 | const displays = [{ index: 1, id: 3, main: true, points: { x: 0, y: 0, w: 2880, h: 1620 }, scale: 2 }]; |
| 963 | const { backend, captures } = displayBackend(t, { displays }); |
| 964 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-display-shot-')); |
| 965 | // Caller-chosen capture paths must sit in the recordings directory. |
| 966 | const oldRec = process.env.CODEWHALE_CU_RECORDINGS_DIR; process.env.CODEWHALE_CU_RECORDINGS_DIR = dir; |
| 967 | t.after(() => { if (oldRec === undefined) delete process.env.CODEWHALE_CU_RECORDINGS_DIR; else process.env.CODEWHALE_CU_RECORDINGS_DIR = oldRec; }); |
| 968 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 969 | |
| 970 | await assert.rejects( |
| 971 | backend.screenshot({ display: 3, path: path.join(dir, 'a.png') }), |
| 972 | (err) => /no display 3/.test(err.message) && /index/.test(err.message), |
| 973 | ); |
| 974 | assert.equal(captures.length, 0, 'a rejected display must not reach screencapture'); |
| 975 | |
| 976 | const ok = await backend.screenshot({ display: 1, path: path.join(dir, 'b.png') }); |
| 977 | assert.equal(ok.display, 1); |
| 978 | assert.deepEqual(ok.points, displays[0].points); |
| 979 | assert.deepEqual(captures.at(-1).slice(0, 5), ['-x', '-t', 'png', '-D', '1']); |
| 980 | }); |
| 981 | |
| 982 | // A 5K display captures to ~22MB of PNG, which is ~29MB of base64 — past the |
| 983 | // 16MB a stdio host will accept in one message. That once dropped the whole |
| 984 | // transport and every other tool with it. The raster is shrunk to fit instead, |
| 985 | // and the geometry must follow it: `pixels` is the PNG's, `points` stays in |
| 986 | // screen points, and `scale` is derived from the two, so raster-to-point |
| 987 | // conversion stays exact at the smaller size. |
| 988 | // A real PNG of `w`x`h` random pixels — incompressible, like a busy screen. |
| 989 | function noisePng(w, h) { |
| 990 | const chunk = (type, body) => { |
| 991 | const len = Buffer.alloc(4); len.writeUInt32BE(body.length); |
| 992 | const tag = Buffer.concat([Buffer.from(type, 'ascii'), body]); |
| 993 | const crc = Buffer.alloc(4); crc.writeUInt32BE(zlib.crc32 ? zlib.crc32(tag) : crc32(tag)); |
| 994 | return Buffer.concat([len, tag, crc]); |
| 995 | }; |
| 996 | const ihdr = Buffer.alloc(13); |
| 997 | ihdr.writeUInt32BE(w, 0); ihdr.writeUInt32BE(h, 4); |
| 998 | ihdr[8] = 8; ihdr[9] = 2; // 8-bit RGB |
| 999 | const rows = Buffer.alloc(h * (1 + w * 3)); |
| 1000 | crypto.randomFillSync(rows); |
| 1001 | for (let y = 0; y < h; y += 1) rows[y * (1 + w * 3)] = 0; // filter: none |
| 1002 | return Buffer.concat([ |
| 1003 | Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), |
| 1004 | chunk('IHDR', ihdr), chunk('IDAT', zlib.deflateSync(rows)), chunk('IEND', Buffer.alloc(0)), |
| 1005 | ]); |
| 1006 | } |
| 1007 | function crc32(buf) { |
| 1008 | let c = ~0; |
| 1009 | for (const b of buf) { c ^= b; for (let k = 0; k < 8; k += 1) c = (c >>> 1) ^ (0xedb88320 & -(c & 1)); } |
| 1010 | return (~c) >>> 0; |
| 1011 | } |
| 1012 | |
| 1013 | // A 5K display captures to ~22MB of PNG, which is ~29MB of base64 — past the |
| 1014 | // 16MB a stdio host will accept in one message. That once dropped the whole |
| 1015 | // transport and every other tool with it. The raster is shrunk to fit instead, |
| 1016 | // and the geometry must follow it: `pixels` is the PNG's, `points` stays in |
| 1017 | // screen points, and `scale` is derived from the two, so raster-to-point |
| 1018 | // conversion stays exact at the smaller size. |
| 1019 | // This integration test exercises the real macOS sips resizer. |
| 1020 | test('macOS screenshot shrinks an over-budget raster and keeps its geometry exact', { skip: process.platform !== 'darwin' }, async (t) => { |
| 1021 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-budget-shot-')); |
| 1022 | // Caller-chosen capture paths must sit in the recordings directory. |
| 1023 | const oldRec = process.env.CODEWHALE_CU_RECORDINGS_DIR; process.env.CODEWHALE_CU_RECORDINGS_DIR = dir; |
| 1024 | t.after(() => { if (oldRec === undefined) delete process.env.CODEWHALE_CU_RECORDINGS_DIR; else process.env.CODEWHALE_CU_RECORDINGS_DIR = oldRec; }); |
| 1025 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 1026 | const big = path.join(dir, 'big.png'); |
| 1027 | fs.writeFileSync(big, noisePng(1600, 900)); |
| 1028 | |
| 1029 | const oldBudget = process.env.CODEWHALE_CU_MAX_IMAGE_BYTES; |
| 1030 | process.env.CODEWHALE_CU_MAX_IMAGE_BYTES = '1500000'; |
| 1031 | t.after(() => { if (oldBudget === undefined) delete process.env.CODEWHALE_CU_MAX_IMAGE_BYTES; else process.env.CODEWHALE_CU_MAX_IMAGE_BYTES = oldBudget; }); |
| 1032 | assert.ok(Math.ceil(fs.statSync(big).size / 3) * 4 > 1_500_000, 'fixture must start over budget'); |
| 1033 | |
| 1034 | const displays = [{ index: 1, id: 3, main: true, points: { x: 0, y: 0, w: 800, h: 450 }, scale: 2 }]; |
| 1035 | const bundle = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-budget-bundle-')); |
| 1036 | const oldBundle = process.env.CODEWHALE_CU_APP_BUNDLE; |
| 1037 | t.after(() => { if (oldBundle === undefined) delete process.env.CODEWHALE_CU_APP_BUNDLE; else process.env.CODEWHALE_CU_APP_BUNDLE = oldBundle; fs.rmSync(bundle, { recursive: true, force: true }); }); |
| 1038 | fs.mkdirSync(path.join(bundle, 'Contents', 'MacOS'), { recursive: true }); |
| 1039 | fs.writeFileSync(path.join(bundle, 'Contents', 'MacOS', 'accessibility'), ''); |
| 1040 | process.env.CODEWHALE_CU_APP_BUNDLE = bundle; |
| 1041 | |
| 1042 | const backend = create({ exec: leaseExecutor(async (cmd, args) => { |
| 1043 | if (cmd === 'screencapture') { |
| 1044 | fs.copyFileSync(big, args.at(-1)); |
| 1045 | return { code: 0, stdout: '', stderr: '' }; |
| 1046 | } |
| 1047 | const request = JSON.parse(args[0]); |
| 1048 | return { code: 0, stderr: '', stdout: JSON.stringify(request.tool === 'displays' ? displays : { action_sent: true }) }; |
| 1049 | }) }); |
| 1050 | |
| 1051 | const out = path.join(dir, 'shot.png'); |
| 1052 | const shot = await backend.screenshot({ display: 1, path: out }); |
| 1053 | |
| 1054 | assert.ok(Math.ceil(fs.statSync(out).size / 3) * 4 <= 1_500_000, 'raster still over budget'); |
| 1055 | assert.ok(shot.pixels.w < 1600, 'an over-budget raster must actually shrink'); |
| 1056 | |
| 1057 | // The invariant that matters: the raster centre still resolves to the centre |
| 1058 | // of the display in screen points. |
| 1059 | assert.equal(shot.points.w, 800, 'points stay in screen points'); |
| 1060 | const centreX = Math.round((shot.pixels.w / 2) / shot.scale) + (shot.points.x ?? 0); |
| 1061 | assert.ok(Math.abs(centreX - 400) <= 2, `raster centre maps to ${centreX}, expected ~400`); |
| 1062 | }); |
| 1063 | |
| 1064 | // A screen is photographic content, and lossless compression of it is enormous: |
| 1065 | // the same 5760x3240 frame measures 21.8MB as PNG and 2.1MB as JPEG at full |
| 1066 | // resolution. The PNG default is what made a single screenshot exceed the |
| 1067 | // 16MB a stdio host accepts in one message and drop the whole session. |
| 1068 | test('macOS screenshot captures JPEG by default and honours an explicit .png path', async (t) => { |
| 1069 | const displays = [{ index: 1, id: 3, main: true, points: { x: 0, y: 0, w: 2880, h: 1620 }, scale: 2 }]; |
| 1070 | const { backend, captures } = displayBackend(t, { displays }); |
| 1071 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-format-')); |
| 1072 | // Caller-chosen capture paths must sit in the recordings directory. |
| 1073 | const oldRec = process.env.CODEWHALE_CU_RECORDINGS_DIR; process.env.CODEWHALE_CU_RECORDINGS_DIR = dir; |
| 1074 | t.after(() => { if (oldRec === undefined) delete process.env.CODEWHALE_CU_RECORDINGS_DIR; else process.env.CODEWHALE_CU_RECORDINGS_DIR = oldRec; }); |
| 1075 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 1076 | |
| 1077 | const dflt = await backend.screenshot({ display: 1 }); |
| 1078 | assert.deepEqual(captures.at(-1).slice(0, 3), ['-x', '-t', 'jpg'], 'the default capture is JPEG'); |
| 1079 | assert.match(dflt.path, /\.jpg$/); |
| 1080 | |
| 1081 | await backend.screenshot({ display: 1, path: path.join(dir, 'exact.png') }); |
| 1082 | assert.deepEqual(captures.at(-1).slice(0, 3), ['-x', '-t', 'png'], 'an explicit .png path stays lossless'); |
| 1083 | |
| 1084 | await backend.screenshot({ display: 1, path: path.join(dir, 'shot.jpeg') }); |
| 1085 | assert.deepEqual(captures.at(-1).slice(0, 3), ['-x', '-t', 'jpg']); |
| 1086 | |
| 1087 | await assert.rejects( |
| 1088 | backend.screenshot({ display: 1, path: path.join(dir, 'shot.gif') }), |
| 1089 | /must end in \.png, \.jpg or \.jpeg/, |
| 1090 | ); |
| 1091 | }); |
| 1092 | |
| 1093 | // Dimensions come from the raster's own header. A JPEG carries them in a frame |
| 1094 | // marker rather than at a fixed offset, and reading the wrong bytes would |
| 1095 | // mis-scale every coordinate target derived from the capture. |
| 1096 | test('macOS raster dimensions are read from both PNG and JPEG headers', async (t) => { |
| 1097 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-dims-')); |
| 1098 | // Caller-chosen capture paths must sit in the recordings directory. |
| 1099 | const oldRec = process.env.CODEWHALE_CU_RECORDINGS_DIR; process.env.CODEWHALE_CU_RECORDINGS_DIR = dir; |
| 1100 | t.after(() => { if (oldRec === undefined) delete process.env.CODEWHALE_CU_RECORDINGS_DIR; else process.env.CODEWHALE_CU_RECORDINGS_DIR = oldRec; }); |
| 1101 | t.after(() => fs.rmSync(dir, { recursive: true, force: true })); |
| 1102 | const png = path.join(dir, 'a.png'); |
| 1103 | fs.writeFileSync(png, noisePng(321, 123)); |
| 1104 | const jpg = path.join(dir, 'a.jpg'); |
| 1105 | spawnSync('sips', ['-s', 'format', 'jpeg', png, '--out', jpg], { stdio: 'ignore' }); |
| 1106 | if (!fs.existsSync(jpg)) { t.skip('sips unavailable'); return; } |
| 1107 | |
| 1108 | const displays = [{ index: 1, id: 3, main: true, points: { x: 0, y: 0, w: 321, h: 123 }, scale: 1 }]; |
| 1109 | for (const [fixture, name] of [[png, 'shot.png'], [jpg, 'shot.jpg']]) { |
| 1110 | const bundle = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-dims-bundle-')); |
| 1111 | const old = process.env.CODEWHALE_CU_APP_BUNDLE; |
| 1112 | fs.mkdirSync(path.join(bundle, 'Contents', 'MacOS'), { recursive: true }); |
| 1113 | fs.writeFileSync(path.join(bundle, 'Contents', 'MacOS', 'accessibility'), ''); |
| 1114 | process.env.CODEWHALE_CU_APP_BUNDLE = bundle; |
| 1115 | const backend = create({ exec: leaseExecutor(async (cmd, args) => { |
| 1116 | if (cmd === 'screencapture') { fs.copyFileSync(fixture, args.at(-1)); return { code: 0, stdout: '', stderr: '' }; } |
| 1117 | const request = JSON.parse(args[0]); |
| 1118 | return { code: 0, stderr: '', stdout: JSON.stringify(request.tool === 'displays' ? displays : { action_sent: true }) }; |
| 1119 | }) }); |
| 1120 | const shot = await backend.screenshot({ display: 1, path: path.join(dir, name) }); |
| 1121 | assert.deepEqual(shot.pixels, { w: 321, h: 123 }, `${name} dimensions`); |
| 1122 | if (old === undefined) delete process.env.CODEWHALE_CU_APP_BUNDLE; else process.env.CODEWHALE_CU_APP_BUNDLE = old; |
| 1123 | fs.rmSync(bundle, { recursive: true, force: true }); |
| 1124 | } |
| 1125 | }); |
| 1126 |