| 1 | // Trajectory recording and replay over the real server: files land in an |
| 2 | // isolated recordings dir; replay re-enters the normal tool pipeline and the |
| 3 | // recorder never records itself. |
| 4 | import { hostKeysLine, attest, attestParams } from "./fixtures/host-decision.mjs"; |
| 5 | import { test, before, after } from "node:test"; |
| 6 | import assert from "node:assert/strict"; |
| 7 | import fs from "node:fs"; |
| 8 | import os from "node:os"; |
| 9 | import path from "node:path"; |
| 10 | import url from "node:url"; |
| 11 | import { spawn } from "node:child_process"; |
| 12 | |
| 13 | const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), ".."); |
| 14 | const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-traj-state-")); |
| 15 | const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-traj-rec-")); |
| 16 | let server; |
| 17 | let buf = ""; |
| 18 | const pending = new Map(); |
| 19 | let nextId = 1; |
| 20 | |
| 21 | function rpc(method, params) { |
| 22 | const id = nextId++; |
| 23 | return new Promise((resolve, reject) => { |
| 24 | const t = setTimeout(() => { pending.delete(id); reject(new Error(`timeout: ${method}`)); }, 20_000); |
| 25 | pending.set(id, (msg) => { clearTimeout(t); resolve(msg); }); |
| 26 | server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: attestParams(method, params) }) + "\n"); |
| 27 | }); |
| 28 | } |
| 29 | async function tool(name, args = {}) { |
| 30 | const res = await rpc("tools/call", { name, arguments: args }); |
| 31 | return JSON.parse(res.result.content[0].text); |
| 32 | } |
| 33 | |
| 34 | before(() => { |
| 35 | server = spawn("node", [path.join(ROOT, "mcp", "server.mjs")], { |
| 36 | env: { ...process.env, CODEWHALE_CU_STATE_DIR: stateDir, CODEWHALE_CU_RECORDINGS_DIR: recDir, CODEWHALE_CU_APP: "off" }, |
| 37 | stdio: ["pipe", "pipe", "pipe"], |
| 38 | }); |
| 39 | server.stdin.write(hostKeysLine()); |
| 40 | server.stdout.on("data", (c) => { |
| 41 | buf += c.toString(); |
| 42 | let i; |
| 43 | while ((i = buf.indexOf("\n")) !== -1) { |
| 44 | const line = buf.slice(0, i).trim(); |
| 45 | buf = buf.slice(i + 1); |
| 46 | if (!line) continue; |
| 47 | const msg = JSON.parse(line); |
| 48 | if (msg.id != null && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); } |
| 49 | } |
| 50 | }); |
| 51 | }); |
| 52 | after(() => { try { server.stdin.end(); } catch {} server?.kill("SIGTERM"); fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(recDir, { recursive: true, force: true }); }); |
| 53 | |
| 54 | test("record → status → stop writes a local JSONL with turns and refusals", async () => { |
| 55 | const started = await tool("trajectory", { action: "start" }); |
| 56 | assert.equal(started.ok, true); |
| 57 | assert.equal(started.recording, true); |
| 58 | assert.ok(fs.existsSync(started.file)); |
| 59 | assert.equal((await tool("trajectory", { action: "status" })).recording, true); |
| 60 | await tool("wait", { seconds: 0.05 }); |
| 61 | await tool("computer", { action: "list" }); |
| 62 | const refused = await tool("click", {}); |
| 63 | assert.equal(refused.error?.code, "bad_args"); |
| 64 | const stopped = await tool("trajectory", { action: "stop" }); |
| 65 | assert.equal(stopped.recording, false); |
| 66 | assert.equal(stopped.turns, 3, `turns=${stopped.turns}`); |
| 67 | const lines = fs.readFileSync(stopped.file, "utf8").trim().split("\n").map(JSON.parse); |
| 68 | assert.equal(lines[0].type, "start"); |
| 69 | assert.equal(lines.at(-1).type, "stop"); |
| 70 | const calls = lines.filter((l) => l.type === "call"); |
| 71 | assert.deepEqual(calls.map((c) => c.tool), ["wait", "computer", "click"]); |
| 72 | assert.equal(calls[2].ok, false, "refusals are part of the record"); |
| 73 | assert.equal(calls[2].code, "bad_args"); |
| 74 | }); |
| 75 | |
| 76 | test("replay dry_run lists the plan without executing", async () => { |
| 77 | const started = await tool("trajectory", { action: "start" }); |
| 78 | assert.equal(started.recording, true); |
| 79 | await tool("wait", { seconds: 0.01 }); |
| 80 | const stopped = await tool("trajectory", { action: "stop" }); |
| 81 | const dry = await tool("trajectory", { action: "replay", id: path.basename(stopped.file), dry_run: true }); |
| 82 | assert.equal(dry.dry_run, true); |
| 83 | assert.equal(dry.replayed, 0); |
| 84 | assert.deepEqual(dry.plan, ["wait"]); |
| 85 | }); |
| 86 | |
| 87 | test("replay re-enters the pipeline, stops at the first refusal, and never records itself", async () => { |
| 88 | const started = await tool("trajectory", { action: "start" }); |
| 89 | assert.equal(started.recording, true); |
| 90 | await tool("wait", { seconds: 0.01 }); |
| 91 | await tool("click", {}); |
| 92 | await tool("wait", { seconds: 0.01 }); |
| 93 | const stopped = await tool("trajectory", { action: "stop" }); |
| 94 | const replay = await tool("trajectory", { action: "replay", id: path.basename(stopped.file) }); |
| 95 | assert.equal(replay.ok, true); |
| 96 | assert.equal(replay.turns_in_file, 3); |
| 97 | assert.equal(replay.replayed, 2, "stops at the refusal instead of continuing"); |
| 98 | assert.deepEqual(replay.results.map((r) => r.tool), ["wait", "click"]); |
| 99 | assert.equal(replay.results[1].ok, false); |
| 100 | const calls = fs.readFileSync(stopped.file, "utf8").trim().split("\n").map(JSON.parse).filter((l) => l.type === "call"); |
| 101 | assert.equal(calls.length, 3, "replayed calls are not re-recorded"); |
| 102 | assert.equal((await tool("trajectory", { action: "status" })).recording, false); |
| 103 | }); |
| 104 | |
| 105 | test("E4: entered text is redacted, the file is 0600 in a 0700 dir, and redacted steps never replay", async () => { |
| 106 | const started = await tool("trajectory", { action: "start" }); |
| 107 | assert.equal(started.recording, true); |
| 108 | // set_value without a target refuses before any backend is touched, but the |
| 109 | // attempt — with its secret — is still part of the record. |
| 110 | const refused = await tool("set_value", { value: "hunter2-secret" }); |
| 111 | assert.equal(refused.error?.code, "bad_args"); |
| 112 | await tool("wait", { seconds: 0.01 }); |
| 113 | const stopped = await tool("trajectory", { action: "stop" }); |
| 114 | const text = fs.readFileSync(stopped.file, "utf8"); |
| 115 | assert.ok(!text.includes("hunter2"), "the secret never reaches disk"); |
| 116 | const call = text.trim().split("\n").map(JSON.parse).find((l) => l.tool === "set_value"); |
| 117 | assert.equal(call.args.value, "[redacted]"); |
| 118 | assert.equal(call.redacted, true); |
| 119 | assert.equal(call.replayable, false); |
| 120 | if (process.platform !== "win32") { |
| 121 | assert.equal(fs.statSync(stopped.file).mode & 0o777, 0o600); |
| 122 | assert.equal(fs.statSync(path.dirname(stopped.file)).mode & 0o777, 0o700); |
| 123 | } |
| 124 | const dry = await tool("trajectory", { action: "replay", id: path.basename(stopped.file), dry_run: true }); |
| 125 | assert.deepEqual(dry.not_replayable, [0]); |
| 126 | const replay = await tool("trajectory", { action: "replay", id: path.basename(stopped.file) }); |
| 127 | assert.equal(replay.replayed, 1); |
| 128 | assert.deepEqual(replay.results, [{ tool: "set_value", ok: false, code: "not_replayable" }]); |
| 129 | }); |
| 130 | |
| 131 | test("trajectory_replay skips app_script", async () => { |
| 132 | const started = await tool("trajectory", { action: "start" }); |
| 133 | assert.equal(started.recording, true); |
| 134 | // The recorded attempt (whatever its outcome) is enough: a script is the |
| 135 | // user's decision each time and never runs from a recording. |
| 136 | await tool("app_script", { script: "return 1" }); |
| 137 | const stopped = await tool("trajectory", { action: "stop" }); |
| 138 | const dry = await tool("trajectory", { action: "replay", id: path.basename(stopped.file), dry_run: true }); |
| 139 | assert.deepEqual(dry.not_replayable, [0]); |
| 140 | const replay = await tool("trajectory", { action: "replay", id: path.basename(stopped.file) }); |
| 141 | assert.deepEqual(replay.results, [{ tool: "app_script", ok: false, code: "not_replayable" }]); |
| 142 | }); |
| 143 | |
| 144 | test("replay refuses escaping ids; the kill switch gates replay but not status", async () => { |
| 145 | const bad = await tool("trajectory", { action: "replay", id: "../escape.jsonl" }); |
| 146 | assert.equal(bad.error?.code, "bad_args"); |
| 147 | const missing = await tool("trajectory", { action: "replay", id: "traj-nope.jsonl" }); |
| 148 | assert.equal(missing.error?.code, "trajectory_not_found"); |
| 149 | await tool("stop_computer_control", { reason: "trajectory test" }); |
| 150 | const afterStop = await tool("trajectory", { action: "replay", dry_run: true }); |
| 151 | assert.equal(afterStop.error?.code, "control_stopped", "replay is an action, not a read"); |
| 152 | const status = await tool("trajectory", { action: "status" }); |
| 153 | assert.equal(status.ok, true, "status stays readable after the stop, like other read-only probes"); |
| 154 | }); |
| 155 |