返回 DeepSeek-Reasonix
hostCalls.ts
根目录 / desktop / electron / src / main / browser / hostCalls.ts
1 import type { BrowserNavigateTarget } from "../../shared/ipc.js";
2 import type { HostCallTable } from "../hostCalls.js";
3 import { bool, num, str, strList, type Params } from "../params.js";
4 import type { ActionExecutor, ActResult } from "./actions.js";
5 import type { DocumentRegistry } from "./documents.js";
6 import type { DownloadTracker, HostDownload } from "./downloads.js";
7 import { browserFailure, takenOver } from "./errors.js";
8 import { CaptureQueue, abortable } from "./captureQueue.js";
9 import { queryElements } from "./query.js";
10 import { scriptCall } from "./pageScripts.js";
11 import { ISOLATED_WORLD } from "./snapshot.js";
12 import { randomToken } from "./documents.js";
13 import { resolveRef } from "./refResolver.js";
14 import { withFrameOperationSignal } from "./frameRuntime.js";
15 import { staleReference } from "./errors.js";
16 import type { BrowserRecorder } from "./recorder.js";
17 import { rm } from "node:fs/promises";
18 import { validFileReference } from "./recoveryStore.js";
19 import { diagnosticText } from "./diagnostics.js";
20 import type { BrowserDiagnosticExport } from "./diagnosticExport.js";
21 import { withBrowserDiagnosticRequest } from "./diagnosticContext.js";
22 import type { GrantRegistry } from "./grants.js";
23 import { captureScreenshot, type ScreenshotDeps, type ScreenshotResult } from "./screenshot.js";
24 import { takeSnapshot, type SnapshotResult } from "./snapshot.js";
25 import type { BrowserSurfaceManager, BrowserTab } from "./surfaceManager.js";
26
27 export interface HostBrowserTab {
28 id: string;
29 url: string;
30 title: string;
31 loading: boolean;
32 temporary: boolean;
33 error?: string;
34 }
35
36 export interface BrowserHostDeps {
37 surfaces: BrowserSurfaceManager;
38 grants: GrantRegistry;
39 documents: DocumentRegistry;
40 actions: ActionExecutor;
41 downloads: DownloadTracker;
42 snapshot?(tab: BrowserTab, selector: string): Promise<SnapshotResult>;
43 screenshot?(tab: BrowserTab, request: { ref: string; fullPage: boolean; directory: string }): Promise<ScreenshotResult>;
44 screenshotDeps?: ScreenshotDeps;
45 recorder?: BrowserRecorder;
46 captureQueue?: CaptureQueue;
47 diagnosticExport?: BrowserDiagnosticExport;
48 trace?(event: { requestId: string; phase: string; elapsedMs: number; errorKind?: string }): void;
49 }
50
51 export function hostTab(surfaces: BrowserSurfaceManager, tab: BrowserTab): HostBrowserTab {
52 const view = surfaces.view(tab);
53 return { id: view.id, url: view.url, title: view.title, loading: view.loading, temporary: view.temporary, error: view.error?.description };
54 }
55
56 // Every call after grant re-verifies the grant and that the browser tab
57 // belongs to the grant's task; reads and writes both refuse a tab the user
58 // is operating (-32011).
59 export function buildBrowserHostCalls(deps: BrowserHostDeps): HostCallTable {
60 const { surfaces, grants, documents, downloads } = deps;
61 const captureQueue = deps.captureQueue ?? new CaptureQueue();
62 const pending = new Map<string, { grantId: string; abort: AbortController }>();
63 type Observation = { token: string; epoch: number; revision: number; width: number; height: number; scrollX: number; scrollY: number };
64 const observations = new WeakMap<BrowserTab, Observation>();
65 const geometry = (tab: BrowserTab) => tab.view.page.executeJavaScriptInIsolatedWorld(ISOLATED_WORLD, [{ code: scriptCall("pageReady", {}) }]) as Promise<{ width: number; height: number; scrollX: number; scrollY: number }>;
66 const boundTab = (params: Params): BrowserTab => {
67 const tab = surfaces.get(str(params, "tabId"));
68 grants.verifyTab(str(params, "grantId"), tab?.taskId, tab?.sessionId);
69 return tab as BrowserTab;
70 };
71 const agentTab = (params: Params): BrowserTab => {
72 const tab = boundTab(params);
73 if (tab.mode !== "agent") throw takenOver(`tab ${tab.id} is in human mode`);
74 return tab;
75 };
76 const snapshot = deps.snapshot ?? ((tab: BrowserTab, selector: string, verify: () => void) => takeSnapshot(tab.view.page, tab.id, tab.epoch, selector, documents, verify));
77 const screenshot = deps.screenshot ?? ((tab: BrowserTab, request: { ref: string; fullPage: boolean; directory: string }, verify: () => void) => {
78 const token = documents.currentToken(tab.id);
79 const binding = token ? documents.lookup(token) ?? null : null;
80 return captureScreenshot(tab.view.page, binding, tab.view.inputScale?.() ?? (tab.view.page.getZoomFactor() || 1), request, { ...deps.screenshotDeps, viewport: tab.viewport, pixelRatio: tab.view.capturePixelRatio?.(), verify });
81 });
82
83 const observe = async <T>(params: Params, capture: boolean, work: (tab: BrowserTab, verify: () => void, signal: AbortSignal) => Promise<T>, stableDocument = true, write = false, readPhase = "reading"): Promise<T> => {
84 const tab = agentTab(params);
85 let epoch = tab.epoch;
86 let initializing = tab.view.isPlaceholder?.() ?? false;
87 const viewportRevision = tab.viewportRevision;
88 const surfaceRevision = tab.surfaceRevision;
89 const needsStableSurface = write || readPhase === "capturing";
90 const page = tab.view.page;
91 const zoom = page.getZoomFactor();
92 const grantId = str(params, "grantId");
93 const diagnosticScope = grants.verify(grantId).diagnosticScope ?? "";
94 const requestId = str(params, "requestId") || randomToken(8);
95 const controller = new AbortController();
96 const started = performance.now();
97 const verify = () => {
98 controller.signal.throwIfAborted();
99 grants.verifyTab(grantId, tab.taskId, tab.sessionId);
100 if (tab.mode !== "agent") throw takenOver(`tab ${tab.id} is in human mode`);
101 if (surfaces.get(tab.id) !== tab || (stableDocument && !initializing && tab.epoch !== epoch) || tab.viewportRevision !== viewportRevision || tab.view.page !== page || page.isDestroyed() || page.getZoomFactor() !== zoom) throw browserFailure("stale_document", "page identity or viewport changed during observation");
102 if (needsStableSurface && tab.surfaceRevision !== surfaceRevision) throw browserFailure("stale_document", "capture or input surface changed during operation");
103 };
104 if (requestId && pending.has(requestId)) throw browserFailure("cancelled", "duplicate request identity");
105 if (requestId) pending.set(requestId, { grantId, abort: controller });
106 const deadline = num(params, "deadline", Date.now() + 9000);
107 const timer = setTimeout(() => controller.abort(), Math.max(0, Math.min(deadline - Date.now(), 9000)));
108 const unsubscribe = surfaces.subscribe(() => { try { verify(); } catch (error) { controller.abort(error); } });
109 const offRevoke = grants.onRevoke(grant => { if (grant.grantId === grantId) controller.abort(); });
110 const status = (phase: string, message?: string, errorKind?: string) => {
111 surfaces.setOperation(tab, { id: requestId, phase, message });
112 // Never log page URLs, text, input, credentials, or exception messages.
113 deps.trace?.({ requestId, phase, elapsedMs: Math.round(performance.now() - started), errorKind });
114 deps.diagnosticExport?.trace(diagnosticScope, { requestId, operationId: str(params, "operationId"), tabId: tab.id, method: "observe", phase, elapsedMs: Math.round(performance.now() - started), errorKind });
115 };
116 status("queued");
117 try {
118 const run = async () => {
119 verify();
120 status("preparing");
121 if (initializing) {
122 await abortable(tab.view.ensureLoaded!(), controller.signal);
123 epoch = tab.epoch;
124 initializing = false;
125 verify();
126 }
127 const releaseObservation = tab.view.prepareObservation?.();
128 let release: (() => void) | undefined;
129 try {
130 release = capture ? await tab.view.prepareCapture?.(controller.signal) : undefined;
131 verify();
132 status(write ? "interacting" : readPhase);
133 const result = await abortable(withFrameOperationSignal(controller.signal, () => work(tab, verify, controller.signal)), controller.signal);
134 if (!write) verify();
135 status("completed");
136 return result;
137 } finally { release?.(); releaseObservation?.(); }
138 };
139 return await (capture ? captureQueue.run(controller.signal, run) : run());
140 } catch (error) {
141 const kind = (error as { data?: { kind?: unknown } } | null)?.data?.kind;
142 if (tab.operation?.id === requestId) status("failed", error instanceof Error ? diagnosticText(error.message).slice(0, 300) : "Browser operation failed", typeof kind === "string" ? kind : controller.signal.aborted ? "cancelled" : "host_error");
143 if (write && controller.signal.aborted) throw browserFailure("cancelled", "operation interrupted; outcome may be unknown, do not replay");
144 throw error;
145 } finally {
146 clearTimeout(timer);
147 unsubscribe();
148 offRevoke();
149 if (requestId) pending.delete(requestId);
150 }
151 };
152
153 // Navigation may already have reached the website. Cancellation releases the
154 // host waiter, but never replays or rolls back a possibly completed write.
155 const navigation = async <T>(params: Params, work: (signal: AbortSignal) => Promise<T>): Promise<T> => {
156 const grantId = str(params, "grantId"), requestId = str(params, "requestId") || randomToken(8);
157 grants.verify(grantId);
158 if (pending.has(requestId)) throw browserFailure("cancelled", "duplicate request identity");
159 const controller = new AbortController();
160 pending.set(requestId, { grantId, abort: controller });
161 const off = grants.onRevoke(grant => { if (grant.grantId === grantId) controller.abort(); });
162 const timer = setTimeout(() => controller.abort(), Math.max(0, Math.min(num(params, "deadline", Date.now() + 15000) - Date.now(), 15000)));
163 try { return await work(controller.signal); }
164 catch (error) { if (controller.signal.aborted) throw browserFailure("cancelled", "navigation interrupted; outcome is unknown, do not replay"); throw error; }
165 finally { clearTimeout(timer); off(); pending.delete(requestId); }
166 };
167
168 const calls: HostCallTable = {
169 "host/browser.exportDiagnostics": params => {
170 if (!deps.diagnosticExport) throw browserFailure("capability_unsupported", "browser diagnostic export is unavailable");
171 return deps.diagnosticExport.read(str(params, "scope"));
172 },
173 "host/browser.preview.remember": params => {
174 const tab = boundTab(params);
175 const reference = { source: str(params, "source"), path: str(params, "path"), toolCallId: str(params, "toolCallId") };
176 if (!validFileReference(reference)) throw new Error("invalid file reference");
177 tab.fileReference = reference; tab.fileReferenceURL = tab.view.page.getURL();
178 surfaces.setOperation(tab, { id: randomToken(8), phase: "completed" });
179 return {};
180 },
181 "host/browser.capabilities": (params) => {
182 grants.verify(str(params, "grantId"));
183 return { capabilities: { query: true, wait: true, viewport: true, pointer: true, diagnostics: true, record: Boolean(deps.recorder) } };
184 },
185 "host/browser.record": async params => {
186 if (!deps.recorder) throw browserFailure("capability_unsupported", "recording is unavailable");
187 const action = str(params, "action");
188 const tab = action === "start" ? agentTab(params) : boundTab(params);
189 const request = (signal?: AbortSignal) => deps.recorder!.request(tab, str(params, "grantId"), action, str(params, "recordingId"), str(params, "directory"), num(params, "durationSeconds", 20), signal);
190 const result = action === "start" ? await observe(params, false, (_tab, _verify, signal) => captureQueue.run(signal, () => request(signal)), true, true) : await request();
191 return { recordingId: result.id, tabId: result.tabId, state: result.state, bytes: result.bytes, width: result.width, height: result.height, durationMs: result.durationMs, path: result.path, mime: result.path ? "video/webm" : undefined, error: result.error };
192 },
193 "host/browser.pointer": params => observe(params, true, async (tab, verify) => {
194 const binding = documents.lookup(str(params, "documentToken"));
195 if (!binding || binding.tabId !== tab.id || binding.epoch !== tab.epoch) throw staleReference("pointer requires the current documentToken");
196 const action = str(params, "action");
197 if (!["click", "hover", "drag"].includes(action)) throw new Error("invalid pointer action");
198 const scale = tab.view.inputScale?.() ?? (tab.view.page.getZoomFactor() || 1);
199 const point = async (ref: string) => {
200 const resolved = await resolveRef(tab.view.page, binding, ref, false);
201 if (!resolved.ok) throw staleReference(resolved.reason);
202 const box = resolved.value.element;
203 return { x: Math.round((box.x + box.width / 2) * scale), y: Math.round((box.y + box.height / 2) * scale) };
204 };
205 let from: { x: number; y: number };
206 if (str(params, "ref")) from = await point(str(params, "ref"));
207 else {
208 const observation = observations.get(tab);
209 const current = await geometry(tab);
210 if (!observation || observation.token !== str(params, "observationToken") || observation.epoch !== tab.epoch || observation.revision !== tab.viewportRevision || current.scrollX !== observation.scrollX || current.scrollY !== observation.scrollY || current.width !== observation.width || current.height !== observation.height) throw staleReference("coordinate observation expired; take a new viewport screenshot");
211 const x = num(params, "x", -1), y = num(params, "y", -1);
212 if (x < 0 || y < 0 || x >= current.width || y >= current.height) throw new Error("coordinates fall outside the observed CSS viewport");
213 from = { x: Math.round(x * scale), y: Math.round(y * scale) };
214 }
215 const to = action === "drag" ? await point(str(params, "targetRef")) : from;
216 verify();
217 surfaces.markAgentInput(tab);
218 observations.delete(tab);
219 const mouse = async (event: Electron.MouseInputEvent) => { verify(); if (tab.view.sendMouseInput) await tab.view.sendMouseInput(event, verify); else tab.view.page.sendInputEvent(event); };
220 try {
221 await mouse({ type: "mouseMove", ...from });
222 if (action !== "hover") {
223 await mouse({ type: "mouseDown", button: "left", clickCount: 1, ...from });
224 if (action === "drag") await mouse({ type: "mouseMove", button: "left", movementX: to.x - from.x, movementY: to.y - from.y, ...to });
225 await mouse({ type: "mouseUp", button: "left", clickCount: 1, ...to });
226 }
227 return { executed: true, documentToken: documents.rotate(str(params, "documentToken")) };
228 } catch { return { executed: false, outcome: "unknown" }; }
229 }, false, true),
230 "host/browser.viewport": params => {
231 const tab = agentTab(params);
232 const action = str(params, "action");
233 if (action === "set") surfaces.setViewport(tab.id, { width: num(params, "width"), height: num(params, "height"), scale: "fit" });
234 else if (action === "reset") surfaces.setViewport(tab.id, null);
235 else if (action !== "get") throw new Error("invalid viewport action");
236 return { viewport: tab.viewport, revision: tab.viewportRevision, documentEpoch: tab.epoch };
237 },
238 "host/browser.diagnostics": params => {
239 const tab = agentTab(params);
240 if (!tab.view.diagnostics) throw browserFailure("capability_unsupported", "page diagnostic collection is unavailable");
241 return tab.view.diagnostics.read(num(params, "after"), str(params, "kind"));
242 },
243 "host/browser.query": params => observe(params, true, tab => queryElements(tab, params, documents)),
244 "host/browser.wait": params => observe(params, true, async (tab, verify, signal) => {
245 const timeout = Math.min(8000, Math.max(1, num(params, "timeoutMs", 3000)));
246 const deadline = Date.now() + timeout;
247 const element = Boolean(str(params, "role") || str(params, "text") || str(params, "testId"));
248 while (true) {
249 verify();
250 if (element) {
251 const result = await queryElements(tab, { ...params, state: str(params, "state", "visible") }, documents);
252 if (result.ambiguous) return { ...result, ready: false, reason: "ambiguous query; refine the locator" };
253 if (result.state === true || (str(params, "state") === "hidden" && result.count === 0)) return { ...result, ready: true };
254 } else {
255 const ready = await tab.view.page.executeJavaScriptInIsolatedWorld(ISOLATED_WORLD, [{ code: scriptCall("pageReady", {}) }]) as { ready: boolean; url: string };
256 if (ready.ready && (!str(params, "url") || str(params, "url") === ready.url)) return { ready: true, url: ready.url };
257 }
258 if (Date.now() >= deadline) throw browserFailure("page_not_ready", "wait condition did not become true before its deadline");
259 await abortable(new Promise<void>(resolve => setTimeout(resolve, 50)), signal);
260 }
261 }, Boolean(str(params, "documentToken"))),
262 "host/browser.grant": (params) => {
263 const grant = grants.install({ grantId: str(params, "grantId"), taskId: str(params, "tabId"), sessionId: str(params, "sessionId"), diagnosticScope: str(params, "diagnosticScope") });
264 deps.diagnosticExport?.bind(grant);
265 return {};
266 },
267 "host/browser.revoke": (params) => {
268 for (const request of pending.values()) if (request.grantId === str(params, "grantId")) request.abort.abort();
269 const grant = grants.revoke(str(params, "grantId"));
270 if (grant) for (const tab of surfaces.tabsForSession(grant.taskId, grant.sessionId)) documents.invalidateTab(tab.id);
271 return {};
272 },
273 "host/browser.cancel": (params) => {
274 const request = pending.get(str(params, "requestId"));
275 if (request && request.grantId === str(params, "grantId")) request.abort.abort();
276 return {};
277 },
278 "host/browser.tabs.list": (params) => {
279 const grant = grants.verify(str(params, "grantId"));
280 return { tabs: surfaces.tabsForSession(grant.taskId, grant.sessionId).map((tab) => hostTab(surfaces, tab)) };
281 },
282 "host/browser.tabs.open": params => navigation(params, async signal => {
283 const grant = grants.verify(str(params, "grantId"));
284 const tab = await surfaces.open(str(params, "url"), { taskId: grant.taskId, sessionId: grant.sessionId, temporary: bool(params, "temporary") }, signal);
285 try { grants.verifyTab(str(params, "grantId"), tab.taskId, tab.sessionId); if (surfaces.get(tab.id) !== tab || tab.mode !== "agent") throw new Error("page ownership changed"); }
286 catch {
287 if (surfaces.get(tab.id) === tab && tab.mode === "agent") surfaces.close(tab.id);
288 throw browserFailure("cancelled", "open was dispatched but its task changed; outcome is unknown");
289 }
290 return hostTab(surfaces, tab);
291 }),
292 "host/browser.tabs.navigate": params => navigation(params, async signal => {
293 // allowHuman is emitted only by the renderer's explicit file-refresh
294 // RPC. Agent browser tools never set it, so takeover still blocks them.
295 const tab = bool(params, "allowHuman") ? boundTab(params) : agentTab(params);
296 const action = str(params, "action");
297 const target: BrowserNavigateTarget = action === "back" || action === "forward" || action === "reload" ? { action } : { url: str(params, "url") };
298 await surfaces.navigate(tab.id, target, signal);
299 try { grants.verifyTab(str(params, "grantId"), tab.taskId, tab.sessionId); if (surfaces.get(tab.id) !== tab || !bool(params, "allowHuman") && tab.mode !== "agent") throw new Error("page ownership changed"); }
300 catch { throw browserFailure("cancelled", "navigation was dispatched but its task changed; outcome is unknown"); }
301 return hostTab(surfaces, tab);
302 }),
303 "host/browser.tabs.close": (params) => {
304 const tab = boundTab(params);
305 documents.invalidateTab(tab.id);
306 downloads.forgetTab(tab.id);
307 surfaces.close(tab.id);
308 return {};
309 },
310 "host/browser.snapshot": (params) => observe(params, true, async (tab, verify) => {
311 const result = await snapshot(tab, str(params, "selector"), verify);
312 verify();
313 if (deps.snapshot) return result;
314 const viewport = await geometry(tab);
315 verify();
316 return { ...result, observation: { url: tab.view.page.getURL(), timeMs: Date.now(), documentEpoch: tab.epoch, viewportRevision: tab.viewportRevision, cssWidth: viewport.width, cssHeight: viewport.height } };
317 }),
318 "host/browser.act": (params): Promise<ActResult> => observe(params, true, (tab, verify) => {
319 const directory = str(params, "directory");
320 if (directory !== "") downloads.setTaskDirectory(tab.taskId, directory);
321 return deps.actions.act(
322 tab,
323 {
324 operationId: str(params, "operationId"),
325 tabId: tab.id,
326 documentToken: str(params, "documentToken"),
327 action: str(params, "action"),
328 ref: str(params, "ref"),
329 text: str(params, "text"),
330 keys: str(params, "keys"),
331 options: strList(params, "options"),
332 files: strList(params, "files"),
333 submit: bool(params, "submit"),
334 deltaX: num(params, "deltaX"),
335 deltaY: num(params, "deltaY"),
336 },
337 verify,
338 );
339 }, false, true),
340 "host/browser.screenshot": (params) => observe(params, true, async (tab, verify) => {
341 const directory = str(params, "directory");
342 if (directory !== "") downloads.setTaskDirectory(tab.taskId, directory);
343 if (deps.screenshot) return screenshot(tab, { ref: str(params, "ref"), fullPage: bool(params, "fullPage"), directory }, verify);
344 const before = await geometry(tab);
345 const result = await screenshot(tab, { ref: str(params, "ref"), fullPage: bool(params, "fullPage"), directory }, verify);
346 const observed = { ...result, observation: { url: tab.view.page.getURL(), timeMs: Date.now(), documentEpoch: tab.epoch, viewportRevision: tab.viewportRevision, cssWidth: before.width, cssHeight: before.height } };
347 try {
348 verify();
349 if (!str(params, "ref") && !bool(params, "fullPage")) {
350 const after = await geometry(tab);
351 if (JSON.stringify(before) !== JSON.stringify(after)) throw staleReference("page geometry changed during capture");
352 const token = randomToken();
353 observations.set(tab, { ...after, token, epoch: tab.epoch, revision: tab.viewportRevision });
354 return { ...observed, observationToken: token, cssWidth: after.width, cssHeight: after.height };
355 }
356 return observed;
357 } catch (error) { await rm(result.path, { force: true }); throw error; }
358 }, true, false, "capturing"),
359 "host/browser.downloads": async (params): Promise<{ downloads: HostDownload[] }> => {
360 const tab = boundTab(params);
361 return { downloads: await downloads.wait(tab.id, num(params, "waitForMs")) };
362 },
363 };
364 for (const [method, call] of Object.entries(calls)) {
365 if (method === "host/browser.exportDiagnostics" || method === "host/browser.grant") continue;
366 calls[method] = async params => {
367 let scope = "";
368 try { scope = grants.verify(str(params, "grantId")).diagnosticScope ?? ""; } catch { /* Preserve the handler's original error semantics. */ }
369 const request = { requestId: str(params, "requestId"), operationId: str(params, "operationId"), tabId: str(params, "tabId"), method };
370 const started = performance.now();
371 const emit = (event: import("./diagnosticContext.js").BrowserRequestTrace) => deps.diagnosticExport?.trace(scope, event);
372 return withBrowserDiagnosticRequest(request, emit, async () => {
373 emit({ ...request, phase: "started", elapsedMs: 0 });
374 try {
375 const result = await call(params);
376 emit({ ...request, tabId: request.tabId || (typeof result === "object" && result !== null && "id" in result && typeof result.id === "string" ? result.id : ""), phase: "completed", elapsedMs: Math.round(performance.now() - started) });
377 return result;
378 } catch (error) {
379 const kind = (error as { data?: { kind?: string } } | null)?.data?.kind;
380 emit({ ...request, phase: "failed", elapsedMs: Math.round(performance.now() - started), errorKind: kind ?? "host_error" });
381 throw error;
382 }
383 });
384 };
385 }
386 return calls;
387 }
388
388 lines TYPESCRIPT