| 1 | import { TWITTER_API_BASE } from './twitter-client-constants.js'; |
| 2 | import { buildSearchFeatures } from './twitter-client-features.js'; |
| 3 | import { extractCursorFromInstructions, parseTweetsFromInstructions } from './twitter-client-utils.js'; |
| 4 | const RAW_QUERY_MISSING_REGEX = /must be defined/i; |
| 5 | function isQueryIdMismatch(payload) { |
| 6 | try { |
| 7 | const parsed = JSON.parse(payload); |
| 8 | return (parsed.errors?.some((error) => { |
| 9 | if (error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED') { |
| 10 | return true; |
| 11 | } |
| 12 | if (error?.path?.includes('rawQuery') && RAW_QUERY_MISSING_REGEX.test(error.message ?? '')) { |
| 13 | return true; |
| 14 | } |
| 15 | return false; |
| 16 | }) ?? false); |
| 17 | } |
| 18 | catch { |
| 19 | return false; |
| 20 | } |
| 21 | } |
| 22 | export function withSearch(Base) { |
| 23 | class TwitterClientSearch extends Base { |
| 24 | // biome-ignore lint/complexity/noUselessConstructor lint/suspicious/noExplicitAny: TS mixin constructor requirement. |
| 25 | constructor(...args) { |
| 26 | super(...args); |
| 27 | } |
| 28 | /** |
| 29 | * Search for tweets matching a query |
| 30 | */ |
| 31 | async search(query, count = 20, options = {}) { |
| 32 | return this.searchPaged(query, count, options); |
| 33 | } |
| 34 | /** |
| 35 | * Get all search results (paged) |
| 36 | */ |
| 37 | async getAllSearchResults(query, options) { |
| 38 | return this.searchPaged(query, Number.POSITIVE_INFINITY, options); |
| 39 | } |
| 40 | async searchPaged(query, limit, options = {}) { |
| 41 | const features = buildSearchFeatures(); |
| 42 | const pageSize = 20; |
| 43 | const seen = new Set(); |
| 44 | const tweets = []; |
| 45 | let cursor = options.cursor; |
| 46 | let nextCursor; |
| 47 | let pagesFetched = 0; |
| 48 | const { includeRaw = false, maxPages } = options; |
| 49 | const fetchPage = async (pageCount, pageCursor) => { |
| 50 | let lastError; |
| 51 | let had404 = false; |
| 52 | const queryIds = await this.getSearchTimelineQueryIds(); |
| 53 | for (const queryId of queryIds) { |
| 54 | const variables = { |
| 55 | rawQuery: query, |
| 56 | count: pageCount, |
| 57 | querySource: 'typed_query', |
| 58 | product: 'Latest', |
| 59 | ...(pageCursor ? { cursor: pageCursor } : {}), |
| 60 | }; |
| 61 | const params = new URLSearchParams({ |
| 62 | variables: JSON.stringify(variables), |
| 63 | }); |
| 64 | const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params.toString()}`; |
| 65 | try { |
| 66 | const response = await this.fetchWithTimeout(url, { |
| 67 | method: 'POST', |
| 68 | headers: this.getHeaders(), |
| 69 | body: JSON.stringify({ features, queryId }), |
| 70 | }); |
| 71 | if (response.status === 404) { |
| 72 | had404 = true; |
| 73 | lastError = `HTTP ${response.status}`; |
| 74 | continue; |
| 75 | } |
| 76 | if (!response.ok) { |
| 77 | const text = await response.text(); |
| 78 | const shouldRefreshQueryIds = (response.status === 400 || response.status === 422) && isQueryIdMismatch(text); |
| 79 | return { |
| 80 | success: false, |
| 81 | error: `HTTP ${response.status}: ${text.slice(0, 200)}`, |
| 82 | had404: had404 || shouldRefreshQueryIds, |
| 83 | }; |
| 84 | } |
| 85 | const data = (await response.json()); |
| 86 | if (data.errors && data.errors.length > 0) { |
| 87 | const shouldRefreshQueryIds = data.errors.some((error) => error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED'); |
| 88 | return { |
| 89 | success: false, |
| 90 | error: data.errors.map((e) => e.message).join(', '), |
| 91 | had404: had404 || shouldRefreshQueryIds, |
| 92 | }; |
| 93 | } |
| 94 | const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; |
| 95 | const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); |
| 96 | const nextCursor = extractCursorFromInstructions(instructions); |
| 97 | return { success: true, tweets: pageTweets, cursor: nextCursor, had404 }; |
| 98 | } |
| 99 | catch (error) { |
| 100 | lastError = error instanceof Error ? error.message : String(error); |
| 101 | } |
| 102 | } |
| 103 | return { success: false, error: lastError ?? 'Unknown error fetching search results', had404 }; |
| 104 | }; |
| 105 | const fetchWithRefresh = async (pageCount, pageCursor) => { |
| 106 | const firstAttempt = await fetchPage(pageCount, pageCursor); |
| 107 | if (firstAttempt.success) { |
| 108 | return firstAttempt; |
| 109 | } |
| 110 | if (firstAttempt.had404) { |
| 111 | await this.refreshQueryIds(); |
| 112 | const secondAttempt = await fetchPage(pageCount, pageCursor); |
| 113 | if (secondAttempt.success) { |
| 114 | return secondAttempt; |
| 115 | } |
| 116 | return { success: false, error: secondAttempt.error }; |
| 117 | } |
| 118 | return { success: false, error: firstAttempt.error }; |
| 119 | }; |
| 120 | const unlimited = limit === Number.POSITIVE_INFINITY; |
| 121 | while (unlimited || tweets.length < limit) { |
| 122 | const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); |
| 123 | const page = await fetchWithRefresh(pageCount, cursor); |
| 124 | if (!page.success) { |
| 125 | return { success: false, error: page.error }; |
| 126 | } |
| 127 | pagesFetched += 1; |
| 128 | let added = 0; |
| 129 | for (const tweet of page.tweets) { |
| 130 | if (seen.has(tweet.id)) { |
| 131 | continue; |
| 132 | } |
| 133 | seen.add(tweet.id); |
| 134 | tweets.push(tweet); |
| 135 | added += 1; |
| 136 | if (!unlimited && tweets.length >= limit) { |
| 137 | break; |
| 138 | } |
| 139 | } |
| 140 | const pageCursor = page.cursor; |
| 141 | if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { |
| 142 | nextCursor = undefined; |
| 143 | break; |
| 144 | } |
| 145 | if (maxPages && pagesFetched >= maxPages) { |
| 146 | nextCursor = pageCursor; |
| 147 | break; |
| 148 | } |
| 149 | cursor = pageCursor; |
| 150 | nextCursor = pageCursor; |
| 151 | } |
| 152 | return { success: true, tweets, nextCursor }; |
| 153 | } |
| 154 | } |
| 155 | return TwitterClientSearch; |
| 156 | } |
| 157 | //# sourceMappingURL=twitter-client-search.js.map |