| 1 | // Real-package startup measurements with a fixed active session and project. |
| 2 | // Usage: node desktop/packaging/history-startup-scale.mjs /path/Reasonix.app [--runs=30] [--sizes=100,10000,100000] |
| 3 | // Measures warm catalog startup; this does not certify all formats, platforms, |
| 4 | // body-read counts, memory budgets, or the absence of every background scan. |
| 5 | import assert from "node:assert/strict"; |
| 6 | import { createRequire } from "node:module"; |
| 7 | import { createServer } from "node:http"; |
| 8 | import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises"; |
| 9 | import { join, resolve } from "node:path"; |
| 10 | import { tmpdir, platform, arch, cpus, totalmem } from "node:os"; |
| 11 | import { packagedSmokeEnv } from "./smoke-env.mjs"; |
| 12 | import { parseServiceReady, waitForSmokeCondition } from "./smoke-poll.mjs"; |
| 13 | import { closeAndVerify } from "./smoke-lifecycle.mjs"; |
| 14 | |
| 15 | assert.equal(platform(), "darwin", "This harness measures the macOS production package"); |
| 16 | const bundle = resolve(process.argv[2]); |
| 17 | const args = new Map(process.argv.slice(3).map(value => value.replace(/^--/, "").split("="))); |
| 18 | const runs = Number(args.get("runs") ?? 30); |
| 19 | const sizes = (args.get("sizes") ?? "100,10000,100000").split(",").map(Number); |
| 20 | assert.ok(Number.isSafeInteger(runs) && runs > 0 && runs <= 100); |
| 21 | assert.ok(sizes.length > 0 && sizes.every((n, i) => Number.isSafeInteger(n) && n >= 50 && n <= 100000 && (i === 0 || n > sizes[i - 1]))); |
| 22 | const require = createRequire(new URL("../electron/package.json", import.meta.url)); |
| 23 | const { _electron } = require("playwright"); |
| 24 | const home = await mkdtemp(join(tmpdir(), "reasonix-history-scale-")); |
| 25 | const resultPath = `${home}.results.json`; |
| 26 | const records = []; |
| 27 | const warmups = []; |
| 28 | const build = JSON.parse(await readFile(join(bundle, "Contents/Resources/build.json"), "utf8")); |
| 29 | let application, expectedSessionID, seeded = 0, providerCalls = 0; |
| 30 | const server = createServer(async (req, res) => { |
| 31 | for await (const _ of req) { /* drain the local fixture request */ } |
| 32 | providerCalls++; |
| 33 | res.writeHead(200, { "Content-Type": "text/event-stream" }); |
| 34 | res.end(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "ACTIVE_SCALE_ANSWER" }, finish_reason: null }] })}\n\ndata: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\ndata: [DONE]\n\n`); |
| 35 | }); |
| 36 | await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); |
| 37 | await writeFile(join(home, "config.toml"), `default_model = "fixture/model"\n[desktop]\nprovider_access = ["fixture"]\n[[providers]]\nname = "fixture"\nkind = "openai"\nbase_url = "http://127.0.0.1:${server.address().port}/v1"\nmodels = ["model"]\ndefault = "model"\napi_key_env = "HISTORY_SCALE_FIXTURE_KEY"\n`); |
| 38 | const environment = { ...packagedSmokeEnv(process.env, home), HISTORY_SCALE_FIXTURE_KEY: "local-fixture" }; |
| 39 | const invoke = (page, method, args = []) => page.evaluate(({ method, args }) => window.reasonixDesktop.invoke(method, args), { method, args }); |
| 40 | const now = () => performance.now(); |
| 41 | async function launch() { |
| 42 | const started = now(); |
| 43 | application = await _electron.launch({ executablePath: join(bundle, "Contents/MacOS/Reasonix"), env: environment }); |
| 44 | const page = await application.firstWindow(); |
| 45 | page.setDefaultTimeout(30000); |
| 46 | await page.waitForFunction(() => Boolean(window.reasonixDesktop)); |
| 47 | await invoke(page, "Version"); |
| 48 | return { page, started }; |
| 49 | } |
| 50 | async function close() { |
| 51 | if (!application) return; |
| 52 | // Logs span restarts. Obtain the service of this shell, not the first |
| 53 | // historical ready line in the append-only log. |
| 54 | const log = await readFile(join(home, "desktop-shell/logs/shell.log"), "utf8"); |
| 55 | const latest = log.split("\n").reverse().map(parseServiceReady).find(Boolean); |
| 56 | assert.ok(latest, "service handshake missing"); |
| 57 | await closeAndVerify(application, { shellPid: await application.evaluate(() => process.pid), servicePid: latest.pid }); |
| 58 | application = undefined; |
| 59 | } |
| 60 | async function settleCatalog(page, count) { |
| 61 | const started = now(); |
| 62 | let nextProgress = 0; |
| 63 | await waitForSmokeCondition(async () => { |
| 64 | const snapshot = await invoke(page, "GetProjectTreeSnapshot"); |
| 65 | if (now() >= nextProgress) { |
| 66 | nextProgress = now() + 10000; |
| 67 | console.log(JSON.stringify({ phase: "warm_progress", sessions: count, elapsedMs: now() - started, |
| 68 | indexed: snapshot.catalog.indexed, total: snapshot.catalog.total, state: snapshot.catalog.state })); |
| 69 | } |
| 70 | if (snapshot.catalog.indexed < count) return false; |
| 71 | const first = await invoke(page, "ListProjectTopics", [{ scope: "global", limit: 50 }]); |
| 72 | return first.complete === true && first.items.length === 50; |
| 73 | }, { timeout: 300000, interval: 1000 }); |
| 74 | warmups.push({ sessions: count, elapsedMs: now() - started }); |
| 75 | } |
| 76 | function summary() { |
| 77 | const p95 = values => [...values].sort((a, b) => a - b)[Math.ceil(values.length * .95) - 1]; |
| 78 | const cohorts = sizes.map(size => { |
| 79 | const rows = records.filter(row => row.sessions === size); |
| 80 | return { sessions: size, runs: rows.length, p95: Object.fromEntries(["interactiveMs", "firstPageMs", "trustedWindowMs", "canSendMs"].map(key => [key, rows.length ? p95(rows.map(row => row[key])) : null])) }; |
| 81 | }); |
| 82 | const complete = cohorts.every(cohort => cohort.runs === runs); |
| 83 | const qualifiedSample = complete && runs >= 30 && sizes.join(",") === "100,10000,100000"; |
| 84 | const thresholdPass = qualifiedSample && ["interactiveMs", "firstPageMs"].every(key => cohorts.at(-1).p95[key] <= cohorts[0].p95[key] + Math.max(200, cohorts[0].p95[key] * .2)); |
| 85 | return { build, machine: { platform: platform(), arch: arch(), cpu: cpus()[0]?.model, logicalCPUs: cpus().length, memoryBytes: totalmem() }, |
| 86 | measurement: "elapsed from package launch; existing catalog; fixed active session/project; progressively added inactive legacy files", |
| 87 | qualifiedSample, thresholdPass, warmups, cohorts, records }; |
| 88 | } |
| 89 | try { |
| 90 | const { page } = await launch(); |
| 91 | await invoke(page, "CreateSession", ["global"]); |
| 92 | await page.locator("#composer-input").fill("ACTIVE_SCALE_PROMPT"); |
| 93 | await page.locator(".composer__btn--send").click(); |
| 94 | await page.waitForFunction(() => document.querySelector(".chat-transcript")?.textContent?.includes("ACTIVE_SCALE_ANSWER")); |
| 95 | await waitForSmokeCondition(async () => (await invoke(page, "ListTabs")).every(tab => !tab.running)); |
| 96 | expectedSessionID = (await invoke(page, "ListTabs")).find(tab => tab.active)?.session?.sessionId; |
| 97 | assert.ok(expectedSessionID); |
| 98 | await close(); |
| 99 | await mkdir(join(home, "sessions"), { recursive: true }); |
| 100 | for (const size of sizes) { |
| 101 | for (; seeded < size;) { |
| 102 | const end = Math.min(size, seeded + 64); |
| 103 | await Promise.all(Array.from({ length: end - seeded }, (_, offset) => writeFile(join(home, "sessions", `inactive-${String(seeded + offset).padStart(6, "0")}.jsonl`), '{"role":"user","content":"INACTIVE_SCALE_BODY"}\n'))); |
| 104 | seeded = end; |
| 105 | } |
| 106 | console.log(JSON.stringify({ phase: "warm_catalog", sessions: size })); |
| 107 | const warm = await launch(); |
| 108 | await settleCatalog(warm.page, size); |
| 109 | await close(); |
| 110 | for (let run = 1; run <= runs; run++) { |
| 111 | const { page, started } = await launch(); |
| 112 | const elapsed = () => now() - started; |
| 113 | const interactive = page.locator(".sidebar__quick-action").waitFor({ state: "visible" }).then(elapsed); |
| 114 | const firstPage = waitForSmokeCondition(async () => { |
| 115 | const value = await invoke(page, "ListProjectTopics", [{ scope: "global", limit: 50 }]); |
| 116 | assert.ok(value.items.length <= 50, "first-page limit exceeded"); |
| 117 | return value.items.length === 50; |
| 118 | }).then(elapsed); |
| 119 | const trustedWindow = page.waitForFunction(() => document.querySelector(".chat-transcript")?.textContent?.includes("ACTIVE_SCALE_ANSWER")).then(elapsed); |
| 120 | const canSend = (async () => { |
| 121 | await page.locator("#composer-input").fill("UNSENT_SCALE_DRAFT"); |
| 122 | await waitForSmokeCondition(() => page.locator(".composer__btn--send").isEnabled()); |
| 123 | return elapsed(); |
| 124 | })(); |
| 125 | const [interactiveMs, firstPageMs, trustedWindowMs, canSendMs] = await Promise.all([interactive, firstPage, trustedWindow, canSend]); |
| 126 | assert.equal((await invoke(page, "ListTabs")).find(tab => tab.active)?.session?.sessionId, expectedSessionID); |
| 127 | assert.equal(providerCalls, 1, "startup or draft editing unexpectedly executed a model turn"); |
| 128 | await close(); |
| 129 | const record = { sessions: size, run, interactiveMs, firstPageMs, trustedWindowMs, canSendMs }; |
| 130 | records.push(record); |
| 131 | await writeFile(resultPath, JSON.stringify(summary(), null, 2)); |
| 132 | console.log(JSON.stringify(record)); |
| 133 | } |
| 134 | } |
| 135 | console.log(JSON.stringify({ phase: "complete", resultPath, ...summary() })); |
| 136 | if (summary().qualifiedSample && !summary().thresholdPass) process.exitCode = 1; |
| 137 | await rm(home, { recursive: true, force: true }); |
| 138 | } catch (error) { |
| 139 | await writeFile(resultPath, JSON.stringify({ ...summary(), error: String(error), retainedFixture: home }, null, 2)); |
| 140 | console.error(`Scale fixture retained at ${home}; results at ${resultPath}`); |
| 141 | throw error; |
| 142 | } finally { |
| 143 | await close(); |
| 144 | server.closeAllConnections(); |
| 145 | await new Promise(resolve => server.close(resolve)); |
| 146 | } |
| 147 |