| 1 | import { statSync } from "node:fs"; |
| 2 | import { isAbsolute } from "node:path"; |
| 3 | import type { DocumentBinding, DocumentRegistry, FrameBinding } from "./documents.js"; |
| 4 | import { noGrant, staleReference, takenOver } from "./errors.js"; |
| 5 | import type { GuestFrame, GuestPage } from "./guestView.js"; |
| 6 | import { chordEvents, parseKeySequence } from "./keys.js"; |
| 7 | import { FOCUS_SCRIPT_SOURCE, IDENTITY_SCRIPT_SOURCE, scriptCall, SELECT_SCRIPT_SOURCE, type ResolvedElement, type SelectOutput } from "./pageScripts.js"; |
| 8 | import { frameForRef, locateRef, resolveRef } from "./refResolver.js"; |
| 9 | import { ISOLATED_WORLD, REGISTRY_KEY, runInFrame } from "./snapshot.js"; |
| 10 | import type { BrowserSurfaceManager, BrowserTab } from "./surfaceManager.js"; |
| 11 | import { uploadFiles } from "./upload.js"; |
| 12 | |
| 13 | export const ACT_SETTLE_MS = 150; |
| 14 | |
| 15 | export interface ActRequest { |
| 16 | operationId: string; |
| 17 | tabId: string; |
| 18 | documentToken: string; |
| 19 | action: string; |
| 20 | ref: string; |
| 21 | text: string; |
| 22 | keys: string; |
| 23 | options: string[]; |
| 24 | files: string[]; |
| 25 | submit: boolean; |
| 26 | deltaX: number; |
| 27 | deltaY: number; |
| 28 | } |
| 29 | |
| 30 | export interface ActResult { |
| 31 | executed: boolean; |
| 32 | outcome?: "unknown"; |
| 33 | reason?: string; |
| 34 | documentToken?: string; |
| 35 | } |
| 36 | |
| 37 | export interface ActionDeps { |
| 38 | surfaces: BrowserSurfaceManager; |
| 39 | documents: DocumentRegistry; |
| 40 | settleMs?: number; |
| 41 | sleep?(ms: number): Promise<void>; |
| 42 | fileExists?(path: string): boolean; |
| 43 | } |
| 44 | |
| 45 | interface Point { |
| 46 | x: number; |
| 47 | y: number; |
| 48 | } |
| 49 | |
| 50 | const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)); |
| 51 | const defaultFileExists = (path: string) => { |
| 52 | try { |
| 53 | return statSync(path).isFile(); |
| 54 | } catch { |
| 55 | return false; |
| 56 | } |
| 57 | }; |
| 58 | |
| 59 | export class ActionExecutor { |
| 60 | private readonly sleep: (ms: number) => Promise<void>; |
| 61 | private readonly fileExists: (path: string) => boolean; |
| 62 | |
| 63 | constructor(private readonly deps: ActionDeps) { |
| 64 | this.sleep = deps.sleep ?? defaultSleep; |
| 65 | this.fileExists = deps.fileExists ?? defaultFileExists; |
| 66 | } |
| 67 | |
| 68 | // verify re-checks the grant at every checkpoint, so a revoke or take-over |
| 69 | // that lands while the action is still resolving cancels it before any |
| 70 | // input is dispatched; once dispatched, the receipt is honest about it. |
| 71 | async act(tab: BrowserTab, request: ActRequest, verify: () => void): Promise<ActResult> { |
| 72 | const binding = this.deps.documents.lookup(request.documentToken); |
| 73 | if (!binding || binding.tabId !== tab.id) throw staleReference("documentToken is not current for this tab"); |
| 74 | this.checkpoint(tab, binding, verify); |
| 75 | const page = tab.view.page; |
| 76 | let outcome: ActResult; |
| 77 | let dispatched = false; |
| 78 | const dispatch = () => { dispatched = true; }; |
| 79 | try { |
| 80 | switch (request.action) { |
| 81 | case "click": |
| 82 | outcome = await this.click(tab, binding, request, verify, dispatch); |
| 83 | break; |
| 84 | case "type": |
| 85 | outcome = await this.type(tab, binding, request, verify, dispatch); |
| 86 | break; |
| 87 | case "press": |
| 88 | outcome = await this.press(tab, binding, request, verify, dispatch); |
| 89 | break; |
| 90 | case "scroll": |
| 91 | outcome = await this.scroll(tab, binding, request, verify, dispatch); |
| 92 | break; |
| 93 | case "select": |
| 94 | outcome = await this.select(tab, binding, request, verify, dispatch, () => { dispatched = false; }); |
| 95 | break; |
| 96 | case "upload": |
| 97 | outcome = await this.upload(tab, binding, request, verify, dispatch); |
| 98 | break; |
| 99 | default: |
| 100 | return { executed: false, reason: `unsupported action ${JSON.stringify(request.action)}`, documentToken: request.documentToken }; |
| 101 | } |
| 102 | } catch (error) { |
| 103 | // A checkpoint failure after any dispatch cannot prove zero effects. |
| 104 | // Preserve uncertainty so the durable ledger never permits a replay. |
| 105 | if (dispatched) return { executed: false, outcome: "unknown", reason: String(error) }; |
| 106 | throw error; |
| 107 | } |
| 108 | if (!outcome.executed) return { ...outcome, documentToken: request.documentToken }; |
| 109 | await this.sleep(this.deps.settleMs ?? ACT_SETTLE_MS); |
| 110 | if (tab.epoch !== binding.epoch || page.isDestroyed()) return { executed: true }; |
| 111 | let same = false; |
| 112 | try { |
| 113 | same = (await page.executeJavaScriptInIsolatedWorld(ISOLATED_WORLD, [{ code: scriptCall(IDENTITY_SCRIPT_SOURCE, { key: REGISTRY_KEY, docId: binding.frames[0]?.docId ?? "" }) }])) === true; |
| 114 | } catch { |
| 115 | same = false; |
| 116 | } |
| 117 | if (!same) return { executed: true }; |
| 118 | const token = this.deps.documents.rotate(request.documentToken); |
| 119 | return token ? { executed: true, documentToken: token } : { executed: true }; |
| 120 | } |
| 121 | |
| 122 | private checkpoint(tab: BrowserTab, binding: DocumentBinding, verify: () => void): void { |
| 123 | verify(); |
| 124 | if (this.deps.surfaces.get(tab.id) !== tab) throw noGrant(`browser tab ${tab.id} is closed`); |
| 125 | if (tab.mode !== "agent") throw takenOver(`tab ${tab.id} is in human mode`); |
| 126 | if (tab.epoch !== binding.epoch) throw staleReference(`tab ${tab.id} changed since the snapshot (epoch ${binding.epoch} → ${tab.epoch})`); |
| 127 | if (tab.view.page.isDestroyed()) throw staleReference(`tab ${tab.id} has no page`); |
| 128 | } |
| 129 | |
| 130 | private async target(tab: BrowserTab, binding: DocumentBinding, ref: string, verify: () => void): Promise<{ ok: true; element: ResolvedElement; centre: Point; frame: GuestFrame; binding: FrameBinding } | { ok: false; reason: string }> { |
| 131 | if (ref === "") return { ok: false, reason: "this action needs a ref" }; |
| 132 | const resolved = await resolveRef(tab.view.page, binding, ref, true); |
| 133 | if (!resolved.ok) return resolved; |
| 134 | this.checkpoint(tab, binding, verify); |
| 135 | const zoom = tab.view.inputScale?.() ?? (tab.view.page.getZoomFactor() || 1); |
| 136 | const { element } = resolved.value; |
| 137 | const centre = { x: Math.round((element.x + element.width / 2) * zoom), y: Math.round((element.y + element.height / 2) * zoom) }; |
| 138 | return { ok: true, element, centre, frame: resolved.value.frame, binding: resolved.value.binding }; |
| 139 | } |
| 140 | |
| 141 | private async mouseClick(tab: BrowserTab, at: Point, verify: () => void): Promise<void> { |
| 142 | const page = tab.view.page; |
| 143 | this.deps.surfaces.markAgentInput(tab); |
| 144 | const move = { type: "mouseMove" as const, x: at.x, y: at.y }; |
| 145 | if (tab.view.sendMouseInput) await tab.view.sendMouseInput(move, verify); else page.sendInputEvent(move); |
| 146 | for (const type of ["mouseDown", "mouseUp"] as const) { |
| 147 | verify(); |
| 148 | const event = { type, x: at.x, y: at.y, button: "left" as const, clickCount: 1 }; |
| 149 | if (tab.view.sendMouseInput) await tab.view.sendMouseInput(event, verify); else page.sendInputEvent(event); |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | private sendKeys(tab: BrowserTab, keys: string): string | null { |
| 154 | let chords; |
| 155 | try { |
| 156 | chords = parseKeySequence(keys); |
| 157 | } catch (error) { |
| 158 | return error instanceof Error ? error.message : String(error); |
| 159 | } |
| 160 | if (chords.length === 0) return "no keys given"; |
| 161 | this.deps.surfaces.markAgentInput(tab); |
| 162 | for (const chord of chords) { |
| 163 | for (const event of chordEvents(chord)) tab.view.page.sendInputEvent(event); |
| 164 | } |
| 165 | return null; |
| 166 | } |
| 167 | |
| 168 | private async click(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void): Promise<ActResult> { |
| 169 | const target = await this.target(tab, binding, request.ref, verify); |
| 170 | if (!target.ok) return { executed: false, reason: target.reason }; |
| 171 | if (target.element.disabled) return { executed: false, reason: "element is disabled" }; |
| 172 | if (target.element.tag === "option") return { executed: false, reason: "use the select action for <option> elements" }; |
| 173 | dispatch(); |
| 174 | await this.mouseClick(tab, target.centre, verify); |
| 175 | return { executed: true }; |
| 176 | } |
| 177 | |
| 178 | private async type(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void): Promise<ActResult> { |
| 179 | const target = await this.target(tab, binding, request.ref, verify); |
| 180 | if (!target.ok) return { executed: false, reason: target.reason }; |
| 181 | if (target.element.disabled) return { executed: false, reason: "element is disabled" }; |
| 182 | if (!target.element.editable) return { executed: false, reason: "element is not editable" }; |
| 183 | dispatch(); |
| 184 | await this.mouseClick(tab, target.centre, verify); |
| 185 | await this.sleep(30); |
| 186 | this.checkpoint(tab, binding, verify); |
| 187 | this.deps.surfaces.markAgentInput(tab); |
| 188 | if (request.text !== "") await tab.view.page.insertText(request.text); |
| 189 | if (request.submit) { |
| 190 | this.checkpoint(tab, binding, verify); |
| 191 | const failure = this.sendKeys(tab, "Enter"); |
| 192 | if (failure) return { executed: false, reason: failure }; |
| 193 | } |
| 194 | return { executed: true }; |
| 195 | } |
| 196 | |
| 197 | private async press(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void): Promise<ActResult> { |
| 198 | if (request.keys.trim() === "") return { executed: false, reason: "press needs keys" }; |
| 199 | // Validate before focusing: focusing itself dispatches a physical click. |
| 200 | try { parseKeySequence(request.keys); } catch (error) { return { executed: false, reason: String(error) }; } |
| 201 | if (request.ref !== "") { |
| 202 | const target = await this.target(tab, binding, request.ref, verify); |
| 203 | if (!target.ok) return { executed: false, reason: target.reason }; |
| 204 | if (target.element.editable) { |
| 205 | dispatch(); |
| 206 | await this.mouseClick(tab, target.centre, verify); |
| 207 | await this.sleep(30); |
| 208 | this.checkpoint(tab, binding, verify); |
| 209 | } else { |
| 210 | this.checkpoint(tab, binding, verify); |
| 211 | const focused = await runInFrame(tab.view.page, target.frame, scriptCall(FOCUS_SCRIPT_SOURCE, { |
| 212 | key: REGISTRY_KEY, snapshotId: binding.snapshotId, docId: target.binding.docId, ref: request.ref, |
| 213 | })); |
| 214 | if (focused !== true) return { executed: false, reason: "element could not be focused" }; |
| 215 | this.checkpoint(tab, binding, verify); |
| 216 | } |
| 217 | } |
| 218 | dispatch(); |
| 219 | const failure = this.sendKeys(tab, request.keys); |
| 220 | return failure ? { executed: false, reason: failure } : { executed: true }; |
| 221 | } |
| 222 | |
| 223 | // Blink negates WebMouseWheelEvent deltas when it builds the DOM WheelEvent, |
| 224 | // so the request keeps DOM semantics (positive deltaY scrolls down) and the |
| 225 | // sign flips here. |
| 226 | private async scroll(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void): Promise<ActResult> { |
| 227 | let at: Point; |
| 228 | if (request.ref !== "") { |
| 229 | const target = await this.target(tab, binding, request.ref, verify); |
| 230 | if (!target.ok) return { executed: false, reason: target.reason }; |
| 231 | at = target.centre; |
| 232 | } else { |
| 233 | const size = await this.viewport(tab); |
| 234 | at = { x: Math.round(size.width / 2), y: Math.round(size.height / 2) }; |
| 235 | this.checkpoint(tab, binding, verify); |
| 236 | } |
| 237 | if (!Number.isFinite(request.deltaX) || !Number.isFinite(request.deltaY)) return { executed: false, reason: "scroll deltas must be numbers" }; |
| 238 | if (request.deltaX === 0 && request.deltaY === 0) return { executed: false, reason: "scroll deltas are both zero" }; |
| 239 | this.deps.surfaces.markAgentInput(tab); |
| 240 | dispatch(); |
| 241 | tab.view.page.sendInputEvent({ type: "mouseWheel", x: at.x, y: at.y, deltaX: -request.deltaX, deltaY: -request.deltaY, canScroll: true }); |
| 242 | return { executed: true }; |
| 243 | } |
| 244 | |
| 245 | private async viewport(tab: BrowserTab): Promise<{ width: number; height: number }> { |
| 246 | const zoom = tab.view.inputScale?.() ?? (tab.view.page.getZoomFactor() || 1); |
| 247 | const raw = await tab.view.page.executeJavaScriptInIsolatedWorld(ISOLATED_WORLD, [{ code: "({ width: window.innerWidth, height: window.innerHeight })" }]); |
| 248 | const size = typeof raw === "object" && raw !== null ? (raw as { width?: unknown; height?: unknown }) : {}; |
| 249 | return { width: (typeof size.width === "number" ? size.width : 0) * zoom, height: (typeof size.height === "number" ? size.height : 0) * zoom }; |
| 250 | } |
| 251 | |
| 252 | private async select(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void, noEffects: () => void): Promise<ActResult> { |
| 253 | if (request.ref === "") return { executed: false, reason: "select needs a ref" }; |
| 254 | const page: GuestPage = tab.view.page; |
| 255 | const target = frameForRef(page, binding, request.ref); |
| 256 | const code = scriptCall(SELECT_SCRIPT_SOURCE, { key: REGISTRY_KEY, snapshotId: binding.snapshotId, docId: target.binding.docId, ref: request.ref, options: request.options }); |
| 257 | let raw: unknown; |
| 258 | this.checkpoint(tab, binding, verify); |
| 259 | dispatch(); |
| 260 | try { |
| 261 | raw = await runInFrame(page, target.frame, code); |
| 262 | } catch (error) { |
| 263 | throw staleReference(`frame of ${request.ref} cannot run scripts: ${String(error)}`); |
| 264 | } |
| 265 | const out = raw as SelectOutput | null; |
| 266 | if (!out || typeof out.ok !== "boolean") throw staleReference("select script returned nothing"); |
| 267 | if (!out.ok) { |
| 268 | noEffects(); |
| 269 | if (out.reason === "stale") throw staleReference(`${request.ref} predates the current document`); |
| 270 | return { executed: false, reason: out.reason }; |
| 271 | } |
| 272 | return { executed: true }; |
| 273 | } |
| 274 | |
| 275 | private async upload(tab: BrowserTab, binding: DocumentBinding, request: ActRequest, verify: () => void, dispatch: () => void): Promise<ActResult> { |
| 276 | if (request.files.length === 0) return { executed: false, reason: "upload needs files" }; |
| 277 | for (const file of request.files) { |
| 278 | if (!isAbsolute(file)) return { executed: false, reason: `file path must be absolute: ${file}` }; |
| 279 | if (!this.fileExists(file)) return { executed: false, reason: `file not found: ${file}` }; |
| 280 | } |
| 281 | if (request.ref === "") return { executed: false, reason: "upload needs a ref" }; |
| 282 | const located = await locateRef(tab.view.page, binding, request.ref); |
| 283 | if (!located.ok) return { executed: false, reason: located.reason }; |
| 284 | this.checkpoint(tab, binding, verify); |
| 285 | return uploadFiles(tab.view.page, located.value, request.files, () => this.checkpoint(tab, binding, verify), dispatch); |
| 286 | } |
| 287 | } |
| 288 |