| 1 | import { mkdir, readFile, writeFile } from 'node:fs/promises'; |
| 2 | import { homedir } from 'node:os'; |
| 3 | import path from 'node:path'; |
| 4 | const DEFAULT_CACHE_FILENAME = 'query-ids-cache.json'; |
| 5 | const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000; |
| 6 | const DISCOVERY_PAGES = [ |
| 7 | 'https://x.com/?lang=en', |
| 8 | 'https://x.com/explore', |
| 9 | 'https://x.com/notifications', |
| 10 | 'https://x.com/settings/profile', |
| 11 | ]; |
| 12 | const BUNDLE_URL_REGEX = /https:\/\/abs\.twimg\.com\/responsive-web\/client-web(?:-legacy)?\/[A-Za-z0-9.-]+\.js/g; |
| 13 | const QUERY_ID_REGEX = /^[a-zA-Z0-9_-]+$/; |
| 14 | const OPERATION_PATTERNS = [ |
| 15 | { |
| 16 | regex: /e\.exports=\{queryId\s*:\s*["']([^"']+)["']\s*,\s*operationName\s*:\s*["']([^"']+)["']/gs, |
| 17 | operationGroup: 2, |
| 18 | queryIdGroup: 1, |
| 19 | }, |
| 20 | { |
| 21 | regex: /e\.exports=\{operationName\s*:\s*["']([^"']+)["']\s*,\s*queryId\s*:\s*["']([^"']+)["']/gs, |
| 22 | operationGroup: 1, |
| 23 | queryIdGroup: 2, |
| 24 | }, |
| 25 | { |
| 26 | regex: /operationName\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)queryId\s*[:=]\s*["']([^"']+)["']/gs, |
| 27 | operationGroup: 1, |
| 28 | queryIdGroup: 3, |
| 29 | }, |
| 30 | { |
| 31 | regex: /queryId\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)operationName\s*[:=]\s*["']([^"']+)["']/gs, |
| 32 | operationGroup: 3, |
| 33 | queryIdGroup: 1, |
| 34 | }, |
| 35 | ]; |
| 36 | const HEADERS = { |
| 37 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', |
| 38 | Accept: 'text/html,application/json;q=0.9,*/*;q=0.8', |
| 39 | 'Accept-Language': 'en-US,en;q=0.9', |
| 40 | }; |
| 41 | async function fetchText(fetchImpl, url) { |
| 42 | const response = await fetchImpl(url, { headers: HEADERS }); |
| 43 | if (!response.ok) { |
| 44 | const body = await response.text().catch(() => ''); |
| 45 | throw new Error(`HTTP ${response.status} for ${url}: ${body.slice(0, 120)}`); |
| 46 | } |
| 47 | return response.text(); |
| 48 | } |
| 49 | function resolveDefaultCachePath() { |
| 50 | const override = process.env.BIRD_QUERY_IDS_CACHE; |
| 51 | if (override && override.trim().length > 0) { |
| 52 | return path.resolve(override.trim()); |
| 53 | } |
| 54 | return path.join(homedir(), '.config', 'bird', DEFAULT_CACHE_FILENAME); |
| 55 | } |
| 56 | function parseSnapshot(raw) { |
| 57 | if (!raw || typeof raw !== 'object') { |
| 58 | return null; |
| 59 | } |
| 60 | const record = raw; |
| 61 | const fetchedAt = typeof record.fetchedAt === 'string' ? record.fetchedAt : null; |
| 62 | const ttlMs = typeof record.ttlMs === 'number' && Number.isFinite(record.ttlMs) ? record.ttlMs : null; |
| 63 | const ids = record.ids && typeof record.ids === 'object' ? record.ids : null; |
| 64 | const discovery = record.discovery && typeof record.discovery === 'object' ? record.discovery : null; |
| 65 | if (!fetchedAt || !ttlMs || !ids || !discovery) { |
| 66 | return null; |
| 67 | } |
| 68 | const pages = Array.isArray(discovery.pages) ? discovery.pages : null; |
| 69 | const bundles = Array.isArray(discovery.bundles) ? discovery.bundles : null; |
| 70 | if (!pages || !bundles) { |
| 71 | return null; |
| 72 | } |
| 73 | const normalizedIds = {}; |
| 74 | for (const [key, value] of Object.entries(ids)) { |
| 75 | if (typeof value === 'string' && value.trim().length > 0) { |
| 76 | normalizedIds[key] = value.trim(); |
| 77 | } |
| 78 | } |
| 79 | return { |
| 80 | fetchedAt, |
| 81 | ttlMs, |
| 82 | ids: normalizedIds, |
| 83 | discovery: { |
| 84 | pages: pages.filter((p) => typeof p === 'string'), |
| 85 | bundles: bundles.filter((b) => typeof b === 'string'), |
| 86 | }, |
| 87 | }; |
| 88 | } |
| 89 | async function readSnapshotFromDisk(cachePath) { |
| 90 | try { |
| 91 | const raw = await readFile(cachePath, 'utf8'); |
| 92 | return parseSnapshot(JSON.parse(raw)); |
| 93 | } |
| 94 | catch { |
| 95 | return null; |
| 96 | } |
| 97 | } |
| 98 | async function writeSnapshotToDisk(cachePath, snapshot) { |
| 99 | await mkdir(path.dirname(cachePath), { recursive: true }); |
| 100 | await writeFile(cachePath, `${JSON.stringify(snapshot, null, 2)}\n`, 'utf8'); |
| 101 | } |
| 102 | async function discoverBundles(fetchImpl) { |
| 103 | const bundles = new Set(); |
| 104 | for (const page of DISCOVERY_PAGES) { |
| 105 | try { |
| 106 | const html = await fetchText(fetchImpl, page); |
| 107 | for (const match of html.matchAll(BUNDLE_URL_REGEX)) { |
| 108 | bundles.add(match[0]); |
| 109 | } |
| 110 | } |
| 111 | catch { |
| 112 | // ignore discovery page failures; other pages often work |
| 113 | } |
| 114 | } |
| 115 | const discovered = [...bundles]; |
| 116 | if (discovered.length === 0) { |
| 117 | throw new Error('No client bundles discovered; x.com layout may have changed.'); |
| 118 | } |
| 119 | return discovered; |
| 120 | } |
| 121 | function extractOperations(bundleContents, bundleLabel, targets, discovered) { |
| 122 | for (const pattern of OPERATION_PATTERNS) { |
| 123 | pattern.regex.lastIndex = 0; |
| 124 | while (true) { |
| 125 | const match = pattern.regex.exec(bundleContents); |
| 126 | if (match === null) { |
| 127 | break; |
| 128 | } |
| 129 | const operationName = match[pattern.operationGroup]; |
| 130 | const queryId = match[pattern.queryIdGroup]; |
| 131 | if (!operationName || !queryId) { |
| 132 | continue; |
| 133 | } |
| 134 | if (!targets.has(operationName)) { |
| 135 | continue; |
| 136 | } |
| 137 | if (!QUERY_ID_REGEX.test(queryId)) { |
| 138 | continue; |
| 139 | } |
| 140 | if (discovered.has(operationName)) { |
| 141 | continue; |
| 142 | } |
| 143 | discovered.set(operationName, { queryId, bundle: bundleLabel }); |
| 144 | if (discovered.size === targets.size) { |
| 145 | return; |
| 146 | } |
| 147 | } |
| 148 | } |
| 149 | } |
| 150 | async function fetchAndExtract(fetchImpl, bundleUrls, targets) { |
| 151 | const discovered = new Map(); |
| 152 | const CONCURRENCY = 6; |
| 153 | for (let i = 0; i < bundleUrls.length; i += CONCURRENCY) { |
| 154 | const chunk = bundleUrls.slice(i, i + CONCURRENCY); |
| 155 | await Promise.all(chunk.map(async (url) => { |
| 156 | if (discovered.size === targets.size) { |
| 157 | return; |
| 158 | } |
| 159 | const label = url.split('/').at(-1) ?? url; |
| 160 | try { |
| 161 | const js = await fetchText(fetchImpl, url); |
| 162 | extractOperations(js, label, targets, discovered); |
| 163 | } |
| 164 | catch { |
| 165 | // ignore failed bundles |
| 166 | } |
| 167 | })); |
| 168 | if (discovered.size === targets.size) { |
| 169 | break; |
| 170 | } |
| 171 | } |
| 172 | return discovered; |
| 173 | } |
| 174 | export function createRuntimeQueryIdStore(options = {}) { |
| 175 | const fetchImpl = options.fetchImpl ?? fetch; |
| 176 | const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS; |
| 177 | const cachePath = options.cachePath ? path.resolve(options.cachePath) : resolveDefaultCachePath(); |
| 178 | let memorySnapshot = null; |
| 179 | let loadOnce = null; |
| 180 | let refreshInFlight = null; |
| 181 | const loadSnapshot = async () => { |
| 182 | if (memorySnapshot) { |
| 183 | return memorySnapshot; |
| 184 | } |
| 185 | if (!loadOnce) { |
| 186 | loadOnce = (async () => { |
| 187 | const fromDisk = await readSnapshotFromDisk(cachePath); |
| 188 | memorySnapshot = fromDisk; |
| 189 | return fromDisk; |
| 190 | })(); |
| 191 | } |
| 192 | return loadOnce; |
| 193 | }; |
| 194 | const getSnapshotInfo = async () => { |
| 195 | const snapshot = await loadSnapshot(); |
| 196 | if (!snapshot) { |
| 197 | return null; |
| 198 | } |
| 199 | const fetchedAtMs = new Date(snapshot.fetchedAt).getTime(); |
| 200 | const ageMs = Number.isFinite(fetchedAtMs) ? Math.max(0, Date.now() - fetchedAtMs) : Number.POSITIVE_INFINITY; |
| 201 | const effectiveTtl = Number.isFinite(snapshot.ttlMs) ? snapshot.ttlMs : ttlMs; |
| 202 | const isFresh = ageMs <= effectiveTtl; |
| 203 | return { snapshot, cachePath, ageMs, isFresh }; |
| 204 | }; |
| 205 | const getQueryId = async (operationName) => { |
| 206 | const info = await getSnapshotInfo(); |
| 207 | if (!info) { |
| 208 | return null; |
| 209 | } |
| 210 | return info.snapshot.ids[operationName] ?? null; |
| 211 | }; |
| 212 | const refresh = async (operationNames, opts = {}) => { |
| 213 | if (refreshInFlight) { |
| 214 | return refreshInFlight; |
| 215 | } |
| 216 | refreshInFlight = (async () => { |
| 217 | const current = await getSnapshotInfo(); |
| 218 | if (!opts.force && current?.isFresh) { |
| 219 | return current; |
| 220 | } |
| 221 | const targets = new Set(operationNames); |
| 222 | const bundleUrls = await discoverBundles(fetchImpl); |
| 223 | const discovered = await fetchAndExtract(fetchImpl, bundleUrls, targets); |
| 224 | if (discovered.size === 0) { |
| 225 | return current ?? null; |
| 226 | } |
| 227 | const ids = {}; |
| 228 | for (const name of operationNames) { |
| 229 | const entry = discovered.get(name); |
| 230 | if (entry?.queryId) { |
| 231 | ids[name] = entry.queryId; |
| 232 | } |
| 233 | } |
| 234 | const snapshot = { |
| 235 | fetchedAt: new Date().toISOString(), |
| 236 | ttlMs, |
| 237 | ids, |
| 238 | discovery: { |
| 239 | pages: [...DISCOVERY_PAGES], |
| 240 | bundles: bundleUrls.map((url) => url.split('/').at(-1) ?? url), |
| 241 | }, |
| 242 | }; |
| 243 | await writeSnapshotToDisk(cachePath, snapshot); |
| 244 | memorySnapshot = snapshot; |
| 245 | return getSnapshotInfo(); |
| 246 | })().finally(() => { |
| 247 | refreshInFlight = null; |
| 248 | }); |
| 249 | return refreshInFlight; |
| 250 | }; |
| 251 | return { |
| 252 | cachePath, |
| 253 | ttlMs, |
| 254 | getSnapshotInfo, |
| 255 | getQueryId, |
| 256 | refresh, |
| 257 | clearMemory() { |
| 258 | memorySnapshot = null; |
| 259 | loadOnce = null; |
| 260 | }, |
| 261 | }; |
| 262 | } |
| 263 | export const runtimeQueryIds = createRuntimeQueryIdStore(); |
| 264 | //# sourceMappingURL=runtime-query-ids.js.map |