| 1 | // Exercise packaged Go -> Electron browser tools -> provider image delivery. |
| 2 | // The loopback provider is deterministic; this does not test model perception. |
| 3 | // Usage: node desktop/packaging/browser-native-smoke.mjs <Reasonix.app> <evidence-dir> |
| 4 | import assert from "node:assert/strict"; |
| 5 | import { createServer } from "node:http"; |
| 6 | import { createRequire } from "node:module"; |
| 7 | import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; |
| 8 | import { tmpdir } from "node:os"; |
| 9 | import { join, resolve } from "node:path"; |
| 10 | import { packagedSmokeEnv } from "./smoke-env.mjs"; |
| 11 | import { waitForSmokeCondition } from "./smoke-poll.mjs"; |
| 12 | import { closeAndVerify } from "./smoke-lifecycle.mjs"; |
| 13 | |
| 14 | const require = createRequire(new URL("../electron/package.json", import.meta.url)); |
| 15 | const { _electron } = require("playwright"); |
| 16 | const bundle = resolve(process.argv[2]), evidence = resolve(process.argv[3]); |
| 17 | const home = mkdtempSync(join(tmpdir(), "reasonix-browser-packaged-")); |
| 18 | const project = join(home, "project"); |
| 19 | mkdirSync(project); mkdirSync(evidence, { recursive: true }); |
| 20 | writeFileSync(join(evidence, "result.json"), JSON.stringify({ completed: false, status: "running" })); |
| 21 | let application, page, step = 0, failure, receivedImage = false, tabId, token, ref, browserPageId; |
| 22 | const steps = []; |
| 23 | function images(value, found = []) { |
| 24 | if (!value || typeof value !== "object") return found; |
| 25 | if (value.type === "image_url") found.push(value.image_url.url); |
| 26 | for (const child of Object.values(value)) if (child && typeof child === "object") images(child, found); |
| 27 | return found; |
| 28 | } |
| 29 | function textOf(message) { |
| 30 | return typeof message?.content === "string" ? message.content : (message?.content ?? []).filter(part => part.type === "text").map(part => part.text).join("\n"); |
| 31 | } |
| 32 | const server = createServer(async (req, res) => { |
| 33 | if (req.method === "GET") { |
| 34 | res.setHeader("Content-Type", "text/html; charset=utf-8"); |
| 35 | if (req.url === "/child") { res.end('<button onclick="this.textContent=\'Frame verified\'">Frame action</button>'); return; } |
| 36 | res.end(`<h1>Browser acceptance</h1><input value="retained"><iframe src="http://localhost:${server.address().port}/child"></iframe>`); |
| 37 | return; |
| 38 | } |
| 39 | try { |
| 40 | let raw = ""; |
| 41 | for await (const chunk of req) raw += chunk; |
| 42 | const request = JSON.parse(raw); |
| 43 | let delta = { content: "Browser fixture" }, finish = "stop"; |
| 44 | if (request.tools?.some(tool => tool.function?.name === "use_capability")) { |
| 45 | const result = textOf([...request.messages].reverse().find(message => message.role === "tool")); |
| 46 | let capability, args; |
| 47 | if (step === 0) { capability = "browser_open"; args = { operationId: "open-fixture", url: `http://127.0.0.1:${server.address().port}/fixture`, temporary: true }; } |
| 48 | if (step === 1) { |
| 49 | tabId = result.match(/opened tab ([^: ]+):/)?.[1]; assert.ok(tabId, result); |
| 50 | capability = "browser_snapshot"; args = { tabId }; |
| 51 | } |
| 52 | if (step === 2) { |
| 53 | token = result.match(/documentToken: (\S+)/)?.[1]; |
| 54 | ref = result.match(/button "Frame action"[^\n]*ref=(f\d+e\d+)/)?.[1]; |
| 55 | assert.ok(token && ref, result); |
| 56 | browserPageId = await application.evaluate(async ({ webContents }, url) => { |
| 57 | const guest = webContents.getAllWebContents().find(page => page.getURL() === url); |
| 58 | const send = guest.debugger.sendCommand.bind(guest.debugger); |
| 59 | guest.fixtureInputs = []; |
| 60 | guest.debugger.sendCommand = async (method, params, session) => { |
| 61 | if (method === "Input.dispatchMouseEvent") guest.fixtureInputs.push(params); |
| 62 | return send(method, params, session); |
| 63 | }; |
| 64 | for (const frame of guest.mainFrame.framesInSubtree) await frame.executeJavaScript(`window.fixtureEvents = []; for (const type of ['mousedown', 'mouseup', 'click']) document.addEventListener(type, e => window.fixtureEvents.push({type, x:e.clientX, y:e.clientY, tag:e.target.tagName}));`); |
| 65 | return guest.id; |
| 66 | }, `http://127.0.0.1:${server.address().port}/fixture`); |
| 67 | capability = "browser_click"; args = { tabId, documentToken: token, operationId: "click-fixture", ref }; |
| 68 | } |
| 69 | if (step === 3) { assert.match(result, /executed|clicked|ok/i); capability = "browser_snapshot"; args = { tabId }; } |
| 70 | if (step === 4) { |
| 71 | const inputEvidence = await application.evaluate(async ({ webContents }, id) => { |
| 72 | const guest = webContents.fromId(id); |
| 73 | return { inputs: guest.fixtureInputs, frames: await Promise.all(guest.mainFrame.framesInSubtree.map(frame => frame.executeJavaScript(`({events:window.fixtureEvents,width:innerWidth,height:innerHeight,rects:[...document.querySelectorAll('iframe,button')].map(e=>({tag:e.tagName,rect:e.getBoundingClientRect().toJSON()}))})`))) }; |
| 74 | }, browserPageId); |
| 75 | writeFileSync(join(evidence, "input-evidence.json"), JSON.stringify(inputEvidence, null, 2)); |
| 76 | assert.match(result, /Frame verified/); |
| 77 | browserPageId = await application.evaluate(({ webContents }, url) => webContents.getAllWebContents().find(page => page.getURL() === url)?.id, |
| 78 | `http://127.0.0.1:${server.address().port}/fixture`); |
| 79 | assert.ok(browserPageId); |
| 80 | await page.locator(".workbench-dock__collapse").click(); |
| 81 | await page.locator(".browser-panel").waitFor({ state: "detached" }); |
| 82 | capability = "browser_screenshot"; args = { tabId }; |
| 83 | } |
| 84 | if (step === 5) { |
| 85 | const data = images(request).find(url => url.startsWith("data:image/png;base64,")); |
| 86 | assert.ok(data, "provider received no structured screenshot image"); |
| 87 | const bytes = Buffer.from(data.split(",")[1], "base64"); |
| 88 | assert.equal(bytes.subarray(0, 8).toString("hex"), "89504e470d0a1a0a"); |
| 89 | assert.ok(bytes.readUInt32BE(16) > 0 && bytes.readUInt32BE(20) > 0); |
| 90 | writeFileSync(join(evidence, "provider-browser.png"), bytes); |
| 91 | receivedImage = true; |
| 92 | delta = { content: "BROWSER_ACCEPTANCE_DONE" }; |
| 93 | } |
| 94 | assert.ok(step <= 5, "unexpected extra provider round trip"); |
| 95 | if (capability) { |
| 96 | steps.push(capability); |
| 97 | delta = { tool_calls: [{ index: 0, id: `browser-fixture-${step}`, type: "function", function: { name: "use_capability", arguments: JSON.stringify({ action: "call", capability_id: `tool:${capability}`, arguments: args }) } }] }; |
| 98 | finish = "tool_calls"; |
| 99 | } |
| 100 | step++; |
| 101 | } |
| 102 | res.writeHead(200, { "Content-Type": "text/event-stream" }); |
| 103 | res.end(`data: ${JSON.stringify({ id: "browser-fixture", choices: [{ index: 0, delta, finish_reason: null }] })}\n\n` |
| 104 | + `data: ${JSON.stringify({ id: "browser-fixture", choices: [{ index: 0, delta: {}, finish_reason: finish }] })}\n\ndata: [DONE]\n\n`); |
| 105 | } catch (error) { failure = error; res.writeHead(500); res.end("fixture assertion failed"); } |
| 106 | }); |
| 107 | await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); |
| 108 | writeFileSync(join(home, "config.toml"), `default_model = "fixture/vision"\n[desktop]\nprovider_access = ["fixture"]\n[[providers]]\nname = "fixture"\nkind = "openai"\nbase_url = "http://127.0.0.1:${server.address().port}/v1"\nmodels = ["vision"]\nvision_models = ["vision"]\ndefault = "vision"\napi_key_env = "BROWSER_FIXTURE_KEY"\n`); |
| 109 | try { |
| 110 | application = await _electron.launch({ executablePath: join(bundle, "Contents/MacOS/Reasonix"), env: { ...packagedSmokeEnv(process.env, home), BROWSER_FIXTURE_KEY: "loopback-only" }, timeout: 60_000 }); |
| 111 | const manifest = JSON.parse(readFileSync(join(bundle, "Contents/Resources/build.json"), "utf8")); |
| 112 | await waitForSmokeCondition(async () => { |
| 113 | for (const candidate of application.windows()) { |
| 114 | try { if (await candidate.evaluate(() => window.reasonixDesktop?.invoke("Version", [])) === manifest.version) { page = candidate; return true; } } catch { /* Splash replacement. */ } |
| 115 | } |
| 116 | return false; |
| 117 | }, { timeout: 60_000 }); |
| 118 | assert.equal(await application.evaluate(({ app }) => app.isPackaged), true); |
| 119 | const invoke = (method, args = []) => page.evaluate(({ method, args }) => window.reasonixDesktop.invoke(method, args), { method, args }); |
| 120 | const tab = await invoke("EnsureBlankSurface", ["project", project]); |
| 121 | await invoke("SetActiveTab", [tab.id]); |
| 122 | await invoke("SetToolApprovalModeForTab", [tab.id, "yolo"]); |
| 123 | await invoke("StartTurnForTab", [tab.id, "BROWSER_FIXTURE: inspect the deterministic local page", "browser-native-fixture"]); |
| 124 | await waitForSmokeCondition(() => { if (failure) throw failure; return receivedImage; }, { timeout: 60_000 }); |
| 125 | await waitForSmokeCondition(async () => (await invoke("ListTabs")).every(tab => !tab.running)); |
| 126 | assert.equal(await page.locator(".browser-panel").count(), 0, "background screenshot must not reopen the panel"); |
| 127 | const retained = await application.evaluate(({ webContents }, id) => webContents.fromId(id)?.executeJavaScript('document.querySelector("input").value'), browserPageId); |
| 128 | assert.equal(retained, "retained", "capture must retain the original page and form"); |
| 129 | const decoded = await application.evaluate(({ nativeImage }, path) => { |
| 130 | const image = nativeImage.createFromPath(path); |
| 131 | return { empty: image.isEmpty(), ...image.getSize(), colors: new Set(image.toBitmap()).size }; |
| 132 | }, join(evidence, "provider-browser.png")); |
| 133 | assert.equal(decoded.empty, false); assert.ok(decoded.colors > 2, "screenshot must contain rendered content"); |
| 134 | const shellPid = await application.evaluate(() => process.pid); |
| 135 | const log = readFileSync(join(home, "desktop-shell/logs/shell.log"), "utf8"); |
| 136 | const servicePid = Number(log.match(/desktop service ready: generation \S+, pid (\d+)/)?.[1]); |
| 137 | assert.ok(servicePid); |
| 138 | await closeAndVerify(application, { shellPid, servicePid }); application = null; |
| 139 | writeFileSync(join(evidence, "result.json"), JSON.stringify({ completed: true, version: manifest.version, steps, providerReceivedPNG: true, backgroundCapturePreservesPage: true, closedPanelStaysClosed: true, decoded, provider: "loopback fixture; no model perception claim", normalExit: true }, null, 2)); |
| 140 | console.log("PASS packaged browser: open, cross-origin snapshot/click/verification, PNG delivered to provider, normal exit"); |
| 141 | } catch (error) { |
| 142 | writeFileSync(join(evidence, "result.json"), JSON.stringify({ completed: false, status: "failed", steps }, null, 2)); |
| 143 | throw error; |
| 144 | } finally { |
| 145 | await application?.close(); |
| 146 | server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); |
| 147 | console.log(`isolated home: ${home}`); |
| 148 | } |
| 149 |