| 1 | import assert from "node:assert/strict"; |
| 2 | import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 3 | import { tmpdir } from "node:os"; |
| 4 | import { join } from "node:path"; |
| 5 | import { test } from "node:test"; |
| 6 | import { GraphicsFaultRecord, GraphicsRecovery } from "./graphicsRecovery.js"; |
| 7 | |
| 8 | const gpu = { role: "GPU" as const, reason: "crashed", exitCode: 1 }; |
| 9 | const renderer = { ...gpu, role: "renderer" as const }; |
| 10 | const tick = () => new Promise(resolve => setImmediate(resolve)); |
| 11 | function fixture() { |
| 12 | let now = 0, stopping = false, acceleration = true; |
| 13 | const prompts: boolean[] = [], recorded: unknown[] = []; |
| 14 | let acknowledged = 0; |
| 15 | const recovery = new GraphicsRecovery({ |
| 16 | now: () => now, stopping: () => stopping, accelerationEnabled: () => acceleration, |
| 17 | record: f => { recorded.push(f); }, acknowledge: () => { acknowledged++; }, |
| 18 | offer: async gpu => { prompts.push(gpu); }, error: error => { throw error; }, |
| 19 | }); |
| 20 | return { recovery, prompts, recorded, advance: (ms: number) => { now += ms; }, stop: () => { stopping = true; }, disable: () => { acceleration = false; }, acknowledged: () => acknowledged }; |
| 21 | } |
| 22 | |
| 23 | test("GPU crash bursts prompt once, without treating ordinary exits as GPU faults", async () => { |
| 24 | const f = fixture(); |
| 25 | for (const reason of ["clean-exit", "killed", "unknown"]) f.recovery.fault({ ...gpu, reason }); |
| 26 | assert.equal(f.recorded.length, 0); |
| 27 | f.recovery.fault(gpu); |
| 28 | await tick(); |
| 29 | assert.deepEqual(f.prompts, []); |
| 30 | for (let i = 0; i < 20; i++) f.recovery.fault(gpu); |
| 31 | await tick(); |
| 32 | assert.deepEqual(f.prompts, [true]); |
| 33 | f.recovery.fault(gpu); |
| 34 | await tick(); |
| 35 | assert.deepEqual(f.prompts, [true]); |
| 36 | }); |
| 37 | |
| 38 | test("renderer reload budget distinguishes OOM and ordinary stalls from GPU failures", async () => { |
| 39 | const f = fixture(); |
| 40 | assert.equal(f.recovery.fault(renderer), true); |
| 41 | assert.equal(f.recovery.fault(renderer), false); |
| 42 | await tick(); |
| 43 | assert.deepEqual(f.prompts, [false]); |
| 44 | const oom = fixture(); |
| 45 | assert.equal(oom.recovery.fault({ ...renderer, reason: "oom" }), false); |
| 46 | await tick(); |
| 47 | assert.deepEqual(oom.prompts, [false]); |
| 48 | const related = fixture(); |
| 49 | related.recovery.fault(gpu); |
| 50 | related.recovery.fault(renderer); |
| 51 | related.recovery.fault(renderer); |
| 52 | await tick(); |
| 53 | assert.deepEqual(related.prompts, [true]); |
| 54 | const killed = fixture(); |
| 55 | assert.equal(killed.recovery.fault({ ...renderer, reason: "killed" }), true); |
| 56 | assert.equal(killed.recovery.fault({ ...renderer, reason: "killed" }), false); |
| 57 | await tick(); assert.deepEqual(killed.prompts, [false]); |
| 58 | const startup = fixture(); |
| 59 | assert.equal(startup.recovery.fault(renderer, false), false); |
| 60 | await tick(); assert.deepEqual(startup.prompts, [false]); |
| 61 | }); |
| 62 | |
| 63 | test("recovered transients and shutdown never trigger compatibility restart", async () => { |
| 64 | const f = fixture(); |
| 65 | f.recovery.unresponsive(); f.advance(14_000); f.recovery.tick(); |
| 66 | f.recovery.responsive(); f.advance(2000); f.recovery.tick(); |
| 67 | await tick(); assert.deepEqual(f.prompts, []); |
| 68 | f.recovery.unresponsive(); f.advance(15_000); f.recovery.tick(); |
| 69 | await tick(); assert.deepEqual(f.prompts, [false]); |
| 70 | const stopped = fixture(); stopped.stop(); |
| 71 | stopped.recovery.fault(gpu); stopped.recovery.fault(gpu); |
| 72 | await tick(); assert.deepEqual(stopped.recorded, []); |
| 73 | const software = fixture(); software.disable(); |
| 74 | software.recovery.fault(gpu); software.recovery.fault(gpu); |
| 75 | await tick(); assert.deepEqual(software.prompts, [false]); |
| 76 | }); |
| 77 | |
| 78 | test("time window expires and healthy recovery acknowledges prior fault", async () => { |
| 79 | const f = fixture(); f.recovery.fault(gpu); f.advance(60_001); |
| 80 | f.recovery.healthy(); assert.equal(f.acknowledged(), 1); |
| 81 | f.recovery.fault(gpu); await tick(); assert.deepEqual(f.prompts, []); |
| 82 | assert.equal(f.recovery.fault(renderer), true); |
| 83 | f.advance(60_001); assert.equal(f.recovery.fault(renderer), true); |
| 84 | }); |
| 85 | |
| 86 | test("a healthy interval rearms recovery for a later independent failure", async () => { |
| 87 | for (const kind of ["renderer", "GPU", "unresponsive"] as const) { |
| 88 | const f = fixture(); |
| 89 | const fail = () => { |
| 90 | if (kind === "unresponsive") { |
| 91 | f.recovery.unresponsive(); f.advance(15_000); f.recovery.tick(); |
| 92 | } else { |
| 93 | f.recovery.fault(kind === "GPU" ? gpu : renderer); |
| 94 | f.recovery.fault(kind === "GPU" ? gpu : renderer); |
| 95 | } |
| 96 | }; |
| 97 | fail(); await tick(); |
| 98 | assert.equal(f.prompts.length, 1); |
| 99 | f.recovery.responsive(); f.advance(60_001); f.recovery.healthy(); |
| 100 | fail(); await tick(); |
| 101 | assert.equal(f.prompts.length, 2, `${kind}: a dismissed prompt must not suppress all future recovery`); |
| 102 | } |
| 103 | const startup = fixture(); |
| 104 | startup.recovery.offer(true); await tick(); |
| 105 | startup.advance(60_001); startup.recovery.healthy(); |
| 106 | startup.recovery.fault(gpu); startup.recovery.fault(gpu); await tick(); |
| 107 | assert.deepEqual(startup.prompts, [true, true], "a previous-launch warning must also rearm after healthy use"); |
| 108 | }); |
| 109 | |
| 110 | test("acknowledging an old dialog preserves a GPU fault recorded while it was open", t => { |
| 111 | const root = mkdtempSync(join(tmpdir(), "reasonix-gpu-ack-")); |
| 112 | t.after(() => rmSync(root, { recursive: true, force: true })); |
| 113 | const path = join(root, "fault.json"); |
| 114 | const record = new GraphicsFaultRecord(path, "v1", () => 100); |
| 115 | record.record(gpu); |
| 116 | const shown = record.checkpoint(); |
| 117 | record.record(gpu); // Same timestamp still identifies a different failure. |
| 118 | record.acknowledge(shown); |
| 119 | assert.equal(new GraphicsFaultRecord(path, "v1", () => 100).pending(), true); |
| 120 | record.acknowledge(record.checkpoint()); |
| 121 | assert.equal(new GraphicsFaultRecord(path, "v1", () => 100).pending(), false); |
| 122 | }); |
| 123 | |
| 124 | test("GPU failure waits behind a native notice without being lost or opening duplicate dialogs", async () => { |
| 125 | const f = fixture(); let release!: () => void; |
| 126 | assert.equal(f.recovery.notice(() => new Promise<void>(resolve => { release = resolve; })), true); |
| 127 | await tick(); |
| 128 | f.recovery.fault(gpu); f.recovery.fault(gpu); f.recovery.fault(gpu); |
| 129 | assert.deepEqual(f.prompts, []); |
| 130 | release(); await tick(); await tick(); |
| 131 | assert.deepEqual(f.prompts, [true]); |
| 132 | }); |
| 133 | |
| 134 | test("a stall that recovers while another notice is open does not leave a stale recovery prompt", async () => { |
| 135 | const f = fixture(); let release!: () => void; |
| 136 | f.recovery.notice(() => new Promise<void>(resolve => { release = resolve; })); |
| 137 | await tick(); |
| 138 | f.recovery.unresponsive(); f.advance(15_000); f.recovery.tick(); |
| 139 | f.recovery.responsive(); |
| 140 | release(); await tick(); await tick(); |
| 141 | assert.deepEqual(f.prompts, []); |
| 142 | f.recovery.unresponsive(); f.advance(15_000); f.recovery.tick(); |
| 143 | await tick(); assert.deepEqual(f.prompts, [false]); |
| 144 | }); |
| 145 | |
| 146 | test("pending record survives crash, expires, preserves unknown fields and rejects future schemas", t => { |
| 147 | const root = mkdtempSync(join(tmpdir(), "reasonix-gpu-record-")); |
| 148 | t.after(() => rmSync(root, { recursive: true, force: true })); |
| 149 | const path = join(root, "fault.json"); let now = 100; |
| 150 | const open = (build = "v1") => new GraphicsFaultRecord(path, build, () => now); |
| 151 | assert.equal(open().pending(), false); |
| 152 | writeFileSync(path, JSON.stringify({ version: 1, future: { retained: true } })); |
| 153 | open().record(renderer); assert.equal(open().pending(), false); |
| 154 | open().record(gpu); assert.equal(open().pending(), true); |
| 155 | assert.equal(open("v2").pending(), false); |
| 156 | assert.deepEqual(JSON.parse(readFileSync(path, "utf8")).future, { retained: true }); |
| 157 | now += 86_400_001; assert.equal(open().pending(), false); |
| 158 | open().record(gpu); open().acknowledge(); assert.equal(open().pending(), false); |
| 159 | for (const contents of ['{"version":2,"pending":true}', 'broken']) { |
| 160 | writeFileSync(path, contents); open().record(gpu); open().acknowledge(); |
| 161 | assert.equal(open().pending(), false); assert.equal(readFileSync(path, "utf8"), contents); |
| 162 | } |
| 163 | }); |
| 164 |