| 1 | import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; |
| 2 | |
| 3 | export type GraphicsFault = { role: "GPU" | "renderer"; reason: string; exitCode: number }; |
| 4 | const WINDOW_MS = 60_000; |
| 5 | const FAILURE_REASONS = new Set(["crashed", "abnormal-exit", "launch-failed", "oom", "integrity-failure", "memory-eviction"]); |
| 6 | |
| 7 | /** Only explicit GPU failures survive a launch. An unclean exit is not GPU evidence. */ |
| 8 | export class GraphicsFaultRecord { |
| 9 | private writable = true; |
| 10 | private revision = 0; |
| 11 | private value: Record<string, unknown> = { version: 1 }; |
| 12 | constructor(private readonly path: string, private readonly build: string, private readonly now = Date.now) { |
| 13 | try { |
| 14 | if (!existsSync(path)) return; |
| 15 | const value = JSON.parse(readFileSync(path, "utf8")); |
| 16 | if (value?.version !== 1) { this.writable = false; return; } |
| 17 | this.value = value; |
| 18 | } catch { this.writable = false; } |
| 19 | } |
| 20 | pending(): boolean { |
| 21 | const age = this.now() - Number(this.value.at); |
| 22 | return this.value.pending === true && this.value.build === this.build && age >= 0 && age < 86_400_000; |
| 23 | } |
| 24 | record(fault: GraphicsFault): void { |
| 25 | if (fault.role !== "GPU" || !FAILURE_REASONS.has(fault.reason)) return; |
| 26 | this.revision++; |
| 27 | this.save({ ...this.value, build: this.build, at: this.now(), pending: true, fault }); |
| 28 | } |
| 29 | checkpoint(): number { return this.revision; } |
| 30 | acknowledge(revision = this.revision): void { |
| 31 | if (revision === this.revision) this.save({ ...this.value, pending: false }); |
| 32 | } |
| 33 | private save(value: Record<string, unknown>): void { |
| 34 | if (!this.writable) return; |
| 35 | this.value = value; |
| 36 | const temp = `${this.path}.tmp`; |
| 37 | writeFileSync(temp, JSON.stringify(value) + "\n", { mode: 0o600 }); |
| 38 | renameSync(temp, this.path); |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | export interface GraphicsRecoveryDeps { |
| 43 | now?: () => number; |
| 44 | stopping(): boolean; |
| 45 | accelerationEnabled(): boolean; |
| 46 | record(fault: GraphicsFault): void; |
| 47 | acknowledge(): void; |
| 48 | offer(gpu: boolean): Promise<void>; |
| 49 | error(error: unknown): void; |
| 50 | } |
| 51 | |
| 52 | /** One owner for reload budgets and prompts, independent of the renderer's health. */ |
| 53 | export class GraphicsRecovery { |
| 54 | private gpu: number[] = []; |
| 55 | private renderers: number[] = []; |
| 56 | private prompted = new Set<string>(); |
| 57 | private dialogActive = false; |
| 58 | private deferredOffer: boolean | undefined; |
| 59 | private unresponsiveAt: number | undefined; |
| 60 | private lastFault: number | undefined; |
| 61 | private readonly now: () => number; |
| 62 | constructor(private readonly deps: GraphicsRecoveryDeps) { this.now = deps.now ?? Date.now; } |
| 63 | get isPrompting(): boolean { return this.dialogActive; } |
| 64 | |
| 65 | fault(fault: GraphicsFault, canReload = true): boolean { |
| 66 | if (this.deps.stopping()) return false; |
| 67 | // A killed page still needs UI recovery, but killing a GPU process is not |
| 68 | // evidence of a driver failure (task managers and normal shutdown do this). |
| 69 | if (!FAILURE_REASONS.has(fault.reason) && !(fault.role === "renderer" && fault.reason === "killed")) return false; |
| 70 | const now = this.now(); |
| 71 | this.lastFault = now; |
| 72 | try { this.deps.record(fault); } catch (error) { this.deps.error(error); } |
| 73 | if (fault.role === "GPU") { |
| 74 | this.gpu = [...this.gpu.filter(at => now - at < WINDOW_MS), now].slice(-8); |
| 75 | if (this.gpu.length >= 2) this.offer(true); |
| 76 | return false; |
| 77 | } |
| 78 | this.renderers = [...this.renderers.filter(at => now - at < WINDOW_MS), now].slice(-8); |
| 79 | // OOM and integrity errors are not repaired by repeatedly reloading the same page. |
| 80 | if (!canReload || this.renderers.length >= 2 || fault.reason === "oom" || fault.reason === "integrity-failure" || fault.reason === "memory-eviction") { |
| 81 | this.offer(this.gpu.some(at => now - at < WINDOW_MS)); |
| 82 | return false; |
| 83 | } |
| 84 | return true; |
| 85 | } |
| 86 | unresponsive(): void { |
| 87 | this.unresponsiveAt ??= this.now(); |
| 88 | this.lastFault = this.now(); |
| 89 | } |
| 90 | responsive(): void { |
| 91 | if (this.unresponsiveAt !== undefined) this.lastFault = this.now(); |
| 92 | this.unresponsiveAt = undefined; |
| 93 | } |
| 94 | tick(): void { |
| 95 | // Stalls are live state, unlike a recorded process failure. Re-check after |
| 96 | // another dialog closes instead of queueing a possibly recovered stall. |
| 97 | if (!this.dialogActive && this.unresponsiveAt !== undefined && this.now() - this.unresponsiveAt >= 15_000) { |
| 98 | this.offer(this.gpu.some(at => this.now() - at < WINDOW_MS)); |
| 99 | } |
| 100 | } |
| 101 | healthy(): void { |
| 102 | if (this.lastFault !== undefined && this.unresponsiveAt === undefined && !this.dialogActive && this.now() - this.lastFault >= WINDOW_MS) { |
| 103 | this.lastFault = undefined; |
| 104 | this.prompted.clear(); |
| 105 | this.gpu = []; |
| 106 | this.renderers = []; |
| 107 | try { this.deps.acknowledge(); } catch (error) { this.deps.error(error); } |
| 108 | } |
| 109 | } |
| 110 | offer(gpu: boolean): void { |
| 111 | if (this.deps.stopping()) return; |
| 112 | if (this.dialogActive) { |
| 113 | this.deferredOffer = this.deferredOffer === true || gpu; |
| 114 | return; |
| 115 | } |
| 116 | const compatible = gpu && this.deps.accelerationEnabled(); |
| 117 | const key = compatible ? "gpu" : "renderer"; |
| 118 | if (this.prompted.has(key)) return; |
| 119 | // Startup offers come from the previous launch and have no local fault time. |
| 120 | this.lastFault ??= this.now(); |
| 121 | this.prompted.add(key); |
| 122 | this.notice(() => this.deps.offer(compatible)); |
| 123 | } |
| 124 | notice(show: () => Promise<void>): boolean { |
| 125 | if (this.deps.stopping() || this.dialogActive) return false; |
| 126 | this.dialogActive = true; |
| 127 | void Promise.resolve().then(show).catch(this.deps.error).finally(() => { |
| 128 | this.dialogActive = false; |
| 129 | const pending = this.deferredOffer; |
| 130 | this.deferredOffer = undefined; |
| 131 | if (pending !== undefined) this.offer(pending); |
| 132 | }); |
| 133 | return true; |
| 134 | } |
| 135 | } |
| 136 |