| 1 | // HarmonyOS backend — drives a device/emulator through `hdc` (HarmonyOS |
| 2 | // Device Connector) plus the on-device `uitest` and `snapshot_display` tools. |
| 3 | // Observation: `uitest dumpLayout` (the accessibility-tree equivalent). |
| 4 | // Input: `uitest uiInput` (click / swipe / inputText / keyEvent). |
| 5 | // Stills: `snapshot_display`. Video: no CLI screen recorder exists on |
| 6 | // current HarmonyOS shells, so recording is an honest |
| 7 | // snapshot-series mode muxed with ffmpeg on the host. |
| 8 | import fs from "node:fs"; |
| 9 | import os from "node:os"; |
| 10 | import path from "node:path"; |
| 11 | import crypto from "node:crypto"; |
| 12 | import { run, runOk, ExecError, tryJson, have, currentSignal, throwIfAborted } from "../exec.mjs"; |
| 13 | import { recordingsDir, recordingsOutputPath } from "../recordings.mjs"; |
| 14 | |
| 15 | const DEVICE_TMP = "/data/local/tmp/cu"; |
| 16 | |
| 17 | function escDeviceText(s) { |
| 18 | if (/[^\x20-\x7E]/.test(s)) throw new ExecError("harmony uiInput text must be printable ASCII on this backend"); |
| 19 | return `'${String(s).replace(/'/g, `'\\''`)}'`; |
| 20 | } |
| 21 | |
| 22 | function jpegSize(buf) { |
| 23 | // Minimal JPEG SOF parser — enough to learn the panel size of a snapshot. |
| 24 | let i = 2; |
| 25 | while (i + 9 < buf.length) { |
| 26 | if (buf[i] !== 0xff) { i++; continue; } |
| 27 | const marker = buf[i + 1]; |
| 28 | if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { |
| 29 | return { h: buf.readUInt16BE(i + 5), w: buf.readUInt16BE(i + 7) }; |
| 30 | } |
| 31 | i += 2 + buf.readUInt16BE(i + 2); |
| 32 | } |
| 33 | return null; |
| 34 | } |
| 35 | |
| 36 | export function parseBounds(b) { |
| 37 | const m = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(String(b ?? "")); |
| 38 | if (!m) return null; |
| 39 | const [x1, y1, x2, y2] = [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])]; |
| 40 | return { x: x1, y: y1, w: x2 - x1, h: y2 - y1, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) }; |
| 41 | } |
| 42 | |
| 43 | export function flatten(node, pathArr = [], out = []) { |
| 44 | if (!node || out.length >= 600) return out; |
| 45 | const a = node.attributes ?? {}; |
| 46 | out.push({ |
| 47 | index: out.length, |
| 48 | path: pathArr, |
| 49 | role: a.type ?? "node", |
| 50 | label: a.text || a.id || a.description || a.name || null, |
| 51 | value: a.text ?? null, |
| 52 | bounds: parseBounds(a.bounds), |
| 53 | attributes: a, |
| 54 | actions: ["click", "longClick", "inputText"], |
| 55 | }); |
| 56 | for (let i = 0; i < (node.children?.length ?? 0); i++) flatten(node.children[i], [...pathArr, i], out); |
| 57 | return out; |
| 58 | } |
| 59 | |
| 60 | /** Coordinate clicks on this backend are always raw pointer events; strategy="a11y" must fail closed rather than silently degrade. */ |
| 61 | function assertEventStrategy(strategy) { |
| 62 | if (strategy != null && strategy !== "auto" && strategy !== "event") { |
| 63 | throw new ExecError(`strategy "${strategy}" is macOS-only; this backend dispatches coordinate clicks as raw pointer events — use an element target for a semantic action`); |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | function rejectAppSelectors(args) { |
| 68 | if (["app_ref", "window_id", "windowIndex"].some(key => Object.hasOwn(args, key))) throw Object.assign(new ExecError("HarmonyOS cannot select an app or window for observation or element actions; explicit selectors are unsupported"), { code: "unsupported_selector" }); |
| 69 | } |
| 70 | |
| 71 | export function create({ exec }) { |
| 72 | const shell = (args, opts = {}) => exec.shell(args, { timeoutMs: 20_000, ...opts }); |
| 73 | |
| 74 | async function deviceOut(args, opts = {}) { |
| 75 | const r = await shell(args, opts); |
| 76 | if (r.code !== 0) throw new ExecError(`hdc shell ${args[0]} exited ${r.code}: ${(r.stderr || r.stdout).trim().slice(0, 300)}`, r); |
| 77 | return r.stdout; |
| 78 | } |
| 79 | |
| 80 | async function snapshot(localPath) { |
| 81 | const remote = `${DEVICE_TMP}-${crypto.randomBytes(3).toString("hex")}.jpeg`; |
| 82 | await deviceOut(["snapshot_display", "-f", remote], { timeoutMs: 25_000 }); |
| 83 | try { |
| 84 | await exec.pullFile(remote, localPath, { timeoutMs: 30_000 }); |
| 85 | } finally { |
| 86 | await shell(["rm", "-f", remote]).catch(() => {}); |
| 87 | } |
| 88 | return localPath; |
| 89 | } |
| 90 | |
| 91 | async function dumpLayout() { |
| 92 | const remote = `${DEVICE_TMP}-layout.json`; |
| 93 | await deviceOut(["uitest", "dumpLayout", "-p", remote], { timeoutMs: 40_000 }); |
| 94 | let data; |
| 95 | try { |
| 96 | data = await exec.readFile(remote, { timeoutMs: 30_000 }); |
| 97 | } finally { |
| 98 | await shell(["rm", "-f", remote]).catch(() => {}); |
| 99 | } |
| 100 | return JSON.parse(data.toString("utf8")); |
| 101 | } |
| 102 | |
| 103 | async function uiInput(args, opts = {}) { |
| 104 | await deviceOut(["uitest", "uiInput", ...args], opts); |
| 105 | return { action_sent: true, strategy: "event", backend: "uitest" }; |
| 106 | } |
| 107 | |
| 108 | /** |
| 109 | * Re-find an observed element in a fresh dump. uitest indexes are |
| 110 | * positional and shift whenever a sibling appears, so the index alone can |
| 111 | * name a different control; the element is addressed by its observed tree |
| 112 | * path and must still carry the observed role and label, or nothing is sent. |
| 113 | */ |
| 114 | async function centerOf(target) { |
| 115 | rejectAppSelectors(target); |
| 116 | const stale = (why) => Object.assign(new ExecError(`element_stale — ${why}; re-run get_app_state (uitest indexes change with the UI)`), { code: "element_stale" }); |
| 117 | if (!Array.isArray(target?.path)) throw stale("the element target carries no observed tree path"); |
| 118 | const tree = await dumpLayout(); |
| 119 | const key = JSON.stringify(target.path); |
| 120 | const el = flatten(tree).find((candidate) => JSON.stringify(candidate.path) === key); |
| 121 | if (!el || !el.bounds) throw stale("the observed element is gone"); |
| 122 | if (target.role !== undefined && el.role !== target.role) throw stale(`role changed (${target.role} → ${el.role})`); |
| 123 | if (target.label !== undefined && el.label !== target.label) throw stale(`label changed (${target.label} → ${el.label})`); |
| 124 | return el.bounds; |
| 125 | } |
| 126 | |
| 127 | let frameSeq = 0; |
| 128 | let recording = null; // {id, dir, startedAt, intervalMs, timer, display} |
| 129 | let displayPixels = null; |
| 130 | |
| 131 | async function stopFrames() { |
| 132 | const rec = recording; |
| 133 | if (!rec) return null; |
| 134 | rec.stopped = true; |
| 135 | clearInterval(rec.timer); |
| 136 | rec.controller.abort(); |
| 137 | let timer; |
| 138 | try { |
| 139 | await Promise.race([rec.pending, new Promise((_, reject) => { |
| 140 | timer = setTimeout(() => reject(new ExecError("Harmony recording frame did not stop within 2 seconds")), 2_000); |
| 141 | })]); |
| 142 | } finally { clearTimeout(timer); } |
| 143 | recording = null; |
| 144 | return rec; |
| 145 | } |
| 146 | |
| 147 | return { |
| 148 | platform: "harmonyos", |
| 149 | // Session/route cleanup stops capture without a long mux or deleting |
| 150 | // unfinished frames. Failed cleanup retains the recorder for a retry. |
| 151 | closeSession: async () => { |
| 152 | const rec = await stopFrames(); |
| 153 | return rec ? { id: rec.id, frames: rec.seq, framesDir: rec.dir } : null; |
| 154 | }, |
| 155 | probe: async () => { |
| 156 | const r = await exec.run("hdc", [...exec.targetArgs, "list", "targets"], { timeoutMs: 10_000 }); |
| 157 | const targets = r.stdout.trim().split("\n").filter(Boolean); |
| 158 | const connected = r.code === 0 && targets.length > 0 && !targets.includes("[Empty]"); |
| 159 | return { |
| 160 | platform: "harmonyos", |
| 161 | connected, |
| 162 | targets, |
| 163 | capabilities: { screenshot: connected, accessibility_tree: connected, clipboard: false, recording: "snapshot-series" }, |
| 164 | note: "HarmonyOS drives the device over hdc. Clipboard read/write is not exposed by hdc and fails closed. Recording muxes snapshot_display frames with ffmpeg.", |
| 165 | }; |
| 166 | }, |
| 167 | list_displays: async () => { |
| 168 | if (!displayPixels) { |
| 169 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-hm-")); |
| 170 | try { |
| 171 | const shot = path.join(dir, "probe.jpeg"); |
| 172 | await snapshot(shot); |
| 173 | displayPixels = jpegSize(fs.readFileSync(shot)) ?? { w: null, h: null }; |
| 174 | } finally { fs.rmSync(dir, { recursive: true, force: true }).catch(() => {}); } |
| 175 | } |
| 176 | return [{ index: 1, name: "device", pixels: displayPixels, points: displayPixels, scale: 1, main: true }]; |
| 177 | }, |
| 178 | async switch_display({ index }) { |
| 179 | if (index !== 1) throw new ExecError("harmony backend exposes display 1 only"); |
| 180 | return { activeDisplay: 1 }; |
| 181 | }, |
| 182 | list_apps: async () => { |
| 183 | const out = await deviceOut(["bm", "dump", "-a"], { timeoutMs: 25_000 }); |
| 184 | const bundles = out.split("\n").map((s) => s.trim()).filter((s) => /^[a-zA-Z][\w.]*$/.test(s)); |
| 185 | return { apps: bundles.map((b) => ({ name: b, bundle_id: b, kind: "bundle" })) }; |
| 186 | }, |
| 187 | list_windows: async (args = {}) => { |
| 188 | rejectAppSelectors(args); |
| 189 | const out = await deviceOut(["hidumper", "-s", "WindowManagerService", "-a", "-a"], { timeoutMs: 25_000 }).catch(() => ""); |
| 190 | const windows = out.split("\n").filter((l) => /Window Name|bundleName/i.test(l)).slice(0, 40).map((l) => ({ title: l.trim().slice(0, 160) })); |
| 191 | return { windows: windows.length ? windows : [{ title: "(window list unavailable on this HarmonyOS build)" }] }; |
| 192 | }, |
| 193 | open_application: async ({ bundle_id: bid, ability, name } = {}) => { |
| 194 | const bundle = bid ?? name; |
| 195 | const identifier = (value) => typeof value === "string" && value.length <= 256 && /^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$/.test(value); |
| 196 | if (!identifier(bundle)) throw new ExecError("open_application needs a valid Harmony bundle identifier"); |
| 197 | if (ability != null && !identifier(ability)) throw new ExecError("open_application needs a valid Harmony ability identifier"); |
| 198 | const candidates = ability != null ? [ability] : ["EntryAbility", "MainAbility"]; |
| 199 | let last = null; |
| 200 | for (const a of candidates) { |
| 201 | const r = await shell(["aa", "start", "-b", escDeviceText(bundle), "-a", escDeviceText(a)]); |
| 202 | if (r.code === 0 && !/Error|error/.test(r.stdout + r.stderr)) { |
| 203 | return { launched: true, bundle, ability: a }; |
| 204 | } |
| 205 | last = (r.stderr || r.stdout).trim().slice(0, 200); |
| 206 | } |
| 207 | throw new ExecError(`aa start failed: ${last}`); |
| 208 | }, |
| 209 | get_app_state: async (args = {}) => { |
| 210 | rejectAppSelectors(args); |
| 211 | const tree = await dumpLayout(); |
| 212 | const els = flatten(tree); |
| 213 | return { bundle_id: tree.attributes?.bundleName ?? null, elements: els, truncated: els.length >= 600 }; |
| 214 | }, |
| 215 | screenshot: async (args = {}) => { |
| 216 | rejectAppSelectors(args); |
| 217 | const outPath = recordingsOutputPath(args.path); |
| 218 | const dir = recordingsDir(); |
| 219 | fs.mkdirSync(dir, { recursive: true }); |
| 220 | const file = outPath ?? path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.jpeg`); |
| 221 | await snapshot(file); |
| 222 | const buf = fs.readFileSync(file); |
| 223 | displayPixels = jpegSize(buf) ?? displayPixels; |
| 224 | return { file, bytes: buf.length, pixels: jpegSize(buf), scale: 1, points: jpegSize(buf) }; |
| 225 | }, |
| 226 | zoom: async ({ region, path: outPath }) => { |
| 227 | throw new ExecError("zoom is not supported on the harmony backend yet — screenshot + region on the host is the workaround"); |
| 228 | }, |
| 229 | left_click: ({ target, strategy }) => { assertEventStrategy(strategy); return uiInput(["click", String(Math.round(target.x)), String(Math.round(target.y))]); }, |
| 230 | double_click: ({ target }) => uiInput(["doubleClick", String(Math.round(target.x)), String(Math.round(target.y))]), |
| 231 | triple_click: async ({ target }) => { |
| 232 | await uiInput(["doubleClick", String(Math.round(target.x)), String(Math.round(target.y))]); |
| 233 | return uiInput(["click", String(Math.round(target.x)), String(Math.round(target.y))]); |
| 234 | }, |
| 235 | right_click: async () => { throw new ExecError("uitest uiInput has no right-click; use longClick semantics via hold or left_click"); }, |
| 236 | middle_click: async () => { throw new ExecError("middle click is not exposed by uitest uiInput"); }, |
| 237 | mouse_move: async () => ({ action_sent: false, note: "hover without press is not exposed by uitest uiInput" }), |
| 238 | left_click_drag: ({ from_target: from, to }) => |
| 239 | uiInput(["swipe", String(Math.round(from.x)), String(Math.round(from.y)), String(Math.round(to.x)), String(Math.round(to.y)), "200"], { timeoutMs: 30_000 }), |
| 240 | left_mouse_down: async () => { throw new ExecError("low-level press/release is not exposed by uitest uiInput; use left_click_drag"); }, |
| 241 | left_mouse_up: async () => { throw new ExecError("low-level press/release is not exposed by uitest uiInput; use left_click_drag"); }, |
| 242 | scroll: ({ target, direction = "down", amount = 300 }) => { |
| 243 | const dist = Math.max(60, Math.min(1200, amount * 24)); |
| 244 | const dx = direction === "left" ? dist : direction === "right" ? -dist : 0; |
| 245 | const dy = direction === "up" ? dist : direction === "down" ? -dist : 0; |
| 246 | return uiInput(["swipe", String(Math.round(target.x)), String(Math.round(target.y)), String(Math.round(target.x + dx)), String(Math.round(target.y + dy)), "400"]); |
| 247 | }, |
| 248 | type: async ({ text }) => { |
| 249 | if (!text) return { action_sent: false, note: "empty text" }; |
| 250 | await uiInput(["inputText", "300", "300", escDeviceText(text)]).catch(async (e) => { |
| 251 | // Some builds require coordinates of the focused field; retry with a click-first pattern. |
| 252 | throw e; |
| 253 | }); |
| 254 | return { action_sent: true, chars: text.length, note: "inputText at 300,300 — click the field first for focused input" }; |
| 255 | }, |
| 256 | key: ({ text }) => { |
| 257 | const KEYMAP = { enter: "Enter", return: "Enter", escape: "Esc", esc: "Esc", back: "Back", home: "Home", backspace: "Back", delete: "Del", tab: "Tab", left: "DPAD_LEFT", right: "DPAD_RIGHT", up: "DPAD_UP", down: "DPAD_DOWN", power: "Power", menu: "Menu" }; |
| 258 | const k = KEYMAP[String(text).toLowerCase()] ?? String(text); |
| 259 | if (!/^[A-Za-z0-9_]+$/.test(k)) throw new ExecError(`unsupported key "${text}" on harmony backend`); |
| 260 | return uiInput(["keyEvent", k]); |
| 261 | }, |
| 262 | hold_key: async ({ text, duration }) => { |
| 263 | if (String(text).toLowerCase() !== "click") throw new ExecError('hold_key on harmony supports only hold_key({"text":"click"}) = longClick'); |
| 264 | const d = Math.max(1, Math.min(5, Number(duration) || 1)); |
| 265 | return uiInput(["longClick", "300", "300"]); |
| 266 | }, |
| 267 | set_value: async ({ target, value }) => { |
| 268 | const b = await centerOf(target); |
| 269 | await uiInput(["click", String(b.cx), String(b.cy)]); |
| 270 | await new Promise((r) => setTimeout(r, 300)); |
| 271 | await uiInput(["inputText", String(b.cx), String(b.cy), escDeviceText(String(value))]); |
| 272 | return { action_sent: true, strategy: "uitest-element" }; |
| 273 | }, |
| 274 | select_text: async () => { throw new ExecError("select_text is not exposed by uitest dumpLayout/uiInput on the harmony backend"); }, |
| 275 | perform_action: async ({ target, action }) => { |
| 276 | const b = await centerOf(target); |
| 277 | if (action === "longClick") return uiInput(["longClick", String(b.cx), String(b.cy)]); |
| 278 | return uiInput(["click", String(b.cx), String(b.cy)]); |
| 279 | }, |
| 280 | read_clipboard: async () => { throw new ExecError("clipboard read is not exposed by hdc on current HarmonyOS builds"); }, |
| 281 | write_clipboard: async () => { throw new ExecError("clipboard write is not exposed by hdc on current HarmonyOS builds"); }, |
| 282 | cursor_position: async () => { throw new ExecError("cursor position does not exist on touch devices"); }, |
| 283 | recordingStart: async ({ intervalMs = 400 } = {}) => { |
| 284 | if (recording) throw new ExecError(`recording ${recording.id} already running`); |
| 285 | if (!(await have("ffmpeg"))) throw new ExecError("ffmpeg is required on the host to mux harmony snapshot-series recordings"); |
| 286 | const id = crypto.randomBytes(4).toString("hex"); |
| 287 | const dir = fs.mkdtempSync(path.join(os.tmpdir(), `cu-rec-${id}-`)); |
| 288 | const startedAt = new Date().toISOString(); |
| 289 | const rec = recording = { id, dir, startedAt, intervalMs, seq: 0, controller: new AbortController() }; |
| 290 | const tick = () => { |
| 291 | if (rec.stopped || rec.pending) return rec.pending; |
| 292 | rec.pending = (async () => { |
| 293 | const opts = { timeoutMs: 15_000, signal: rec.controller.signal }; |
| 294 | const remote = `${DEVICE_TMP}-rec-${id}-${String(rec.seq).padStart(5, "0")}.jpeg`; |
| 295 | try { |
| 296 | await deviceOut(["snapshot_display", "-f", remote], opts); |
| 297 | await exec.pullFile(remote, path.join(dir, `f${String(rec.seq).padStart(5, "0")}.jpeg`), opts); |
| 298 | rec.seq++; |
| 299 | } finally { await shell(["rm", "-f", remote], opts).catch(() => {}); } |
| 300 | })().catch(() => {}).finally(() => { rec.pending = null; }); |
| 301 | return rec.pending; |
| 302 | }; |
| 303 | const signal = currentSignal(); |
| 304 | const abort = () => rec.controller.abort(); |
| 305 | signal?.addEventListener("abort", abort, { once: true }); |
| 306 | try { |
| 307 | throwIfAborted(signal); |
| 308 | await tick(); |
| 309 | throwIfAborted(signal); |
| 310 | if (rec.stopped) throw new ExecError("Harmony recording closed during startup"); |
| 311 | rec.timer = setInterval(tick, Math.max(150, intervalMs)); |
| 312 | } catch (err) { await stopFrames(); throw err; } |
| 313 | finally { signal?.removeEventListener("abort", abort); } |
| 314 | return { id, mode: "snapshot-series", intervalMs, startedAt, note: "no HarmonyOS CLI screen recorder; frames are muxed into mp4 on stop" }; |
| 315 | }, |
| 316 | recordingStop: async ({ id }) => { |
| 317 | if (!recording || recording.id !== id) throw new ExecError(`unknown recording "${id}"`); |
| 318 | const { dir, seq, startedAt, intervalMs } = await stopFrames(); |
| 319 | const dirOut = recordingsDir(); |
| 320 | fs.mkdirSync(dirOut, { recursive: true }); |
| 321 | const out = path.join(dirOut, `rec-${id}.mp4`); |
| 322 | const fps = Math.max(1, Math.min(15, Math.round(1000 / Math.max(150, intervalMs)))); |
| 323 | const r = await run("ffmpeg", ["-y", "-loglevel", "error", "-framerate", String(fps), "-i", path.join(dir, "f%05d.jpeg"), "-c:v", "libx264", "-pix_fmt", "yuv420p", out], { timeoutMs: 180_000 }); |
| 324 | const bytes = fs.existsSync(out) ? fs.statSync(out).size : 0; |
| 325 | try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} |
| 326 | if (r.code !== 0) throw new ExecError(`ffmpeg mux failed: ${(r.stderr || "").slice(0, 300)}`, r); |
| 327 | return { id, mode: "snapshot-series", frames: seq, fps, file: out, bytes, startedAt, stoppedAt: new Date().toISOString() }; |
| 328 | }, |
| 329 | recordingStatus: ({ id }) => recording && recording.id === id |
| 330 | ? { id, running: true, mode: "snapshot-series", frames: recording.seq, startedAt: recording.startedAt } |
| 331 | : { id, running: false }, |
| 332 | recordingList: async () => { |
| 333 | const dir = recordingsDir(); |
| 334 | const out = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => /\.(mp4|mov|jpeg|png)$/i.test(f)).map((f) => { |
| 335 | const st = fs.statSync(path.join(dir, f)); |
| 336 | return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() }; |
| 337 | }).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50) : []; |
| 338 | return { dir, recordings: out, running: recording ? [recording.id] : [] }; |
| 339 | }, |
| 340 | }; |
| 341 | } |
| 342 | |
| 343 | export default { create }; |
| 344 |