| 1 | """Shared query preprocessing utilities: noise-word stripping, core subject |
| 2 | extraction, and compound term detection. Used by all search modules.""" |
| 3 | |
| 4 | import re |
| 5 | from typing import FrozenSet, List, Optional, Set |
| 6 | |
| 7 | # Common multi-word prefixes stripped from all queries (identical across modules) |
| 8 | PREFIXES = [ |
| 9 | 'what are the best', 'what is the best', 'what are the latest', |
| 10 | 'what are people saying about', 'what do people think about', |
| 11 | 'how do i use', 'how to use', 'how to', |
| 12 | 'what are', 'what is', 'tips for', 'best practices for', |
| 13 | # Hebrew question/meta prefixes |
| 14 | 'מה יש חדש ב', 'מה יש חדש על', 'מה אנשים אומרים על', |
| 15 | 'מה חדש ב', 'מה חדש על', 'איך להשתמש ב', |
| 16 | 'מהם המוצרים של', 'מה הם', 'מהם', |
| 17 | ] |
| 18 | |
| 19 | # Multi-word suffixes (used by bird_x) |
| 20 | SUFFIXES = [ |
| 21 | 'best practices', 'use cases', 'prompt techniques', |
| 22 | 'prompting techniques', 'prompting tips', |
| 23 | ] |
| 24 | |
| 25 | # Base noise words shared across most modules |
| 26 | NOISE_WORDS = frozenset({ |
| 27 | # Articles/prepositions/conjunctions |
| 28 | 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or', |
| 29 | 'of', 'in', 'on', 'for', 'with', 'about', 'to', |
| 30 | # Question words |
| 31 | 'how', 'what', 'which', 'who', 'why', 'when', 'where', |
| 32 | 'does', 'should', 'could', 'would', |
| 33 | # Research/meta descriptors |
| 34 | 'best', 'top', 'good', 'great', 'awesome', 'killer', |
| 35 | 'latest', 'new', 'news', 'update', 'updates', |
| 36 | 'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral', |
| 37 | 'practices', 'features', 'guide', 'tutorial', |
| 38 | 'recommendations', 'advice', 'review', 'reviews', |
| 39 | 'usecases', 'examples', 'comparison', 'versus', 'vs', |
| 40 | 'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools', |
| 41 | # Prompting meta words |
| 42 | 'prompt', 'prompts', 'prompting', 'techniques', 'tips', |
| 43 | 'tricks', 'methods', 'strategies', 'approaches', |
| 44 | # Action words |
| 45 | 'using', 'uses', 'use', |
| 46 | # Misc filler |
| 47 | 'people', 'saying', 'think', 'said', 'lately', |
| 48 | # Hebrew function words / prepositions / filler |
| 49 | 'מה', 'מי', 'איך', 'למה', 'איפה', 'מתי', 'כמה', 'האם', |
| 50 | 'של', 'על', 'עם', 'אל', 'את', 'בין', 'כי', 'כן', 'לא', |
| 51 | 'יש', 'אין', 'כבר', 'רק', 'גם', 'אבל', 'כך', 'זה', 'זו', |
| 52 | 'חדש', 'חדשים', 'טוב', 'טובים', 'הכי', 'ביותר', |
| 53 | 'מוצרים', 'מבצע', 'מבצעים', 'חדשות', 'עדכונים', |
| 54 | }) |
| 55 | |
| 56 | |
| 57 | # Shared noise sets for adapter `_extract_core_subject` wrappers. |
| 58 | # |
| 59 | # SOCIAL_NOISE: short-form micro-social platforms (Bluesky, Threads, Truth Social) |
| 60 | # where research/meta words rarely appear in the body of a post. |
| 61 | SOCIAL_NOISE = frozenset({ |
| 62 | 'best', 'top', 'good', 'great', 'awesome', |
| 63 | 'latest', 'new', 'news', 'update', 'updates', |
| 64 | 'trending', 'hottest', 'popular', 'viral', |
| 65 | 'practices', 'features', 'recommendations', 'advice', |
| 66 | 'or', 'and', |
| 67 | }) |
| 68 | |
| 69 | # VIRAL_NOISE: viral / discovery platforms (TikTok, Instagram, Pinterest) and |
| 70 | # the base for YouTube. Adds 'killer', the prompt-meta cluster, and the |
| 71 | # methodology cluster on top of SOCIAL_NOISE. |
| 72 | VIRAL_NOISE = SOCIAL_NOISE | frozenset({ |
| 73 | 'killer', |
| 74 | 'prompt', 'prompts', 'prompting', |
| 75 | 'methods', 'strategies', 'approaches', |
| 76 | }) |
| 77 | |
| 78 | |
| 79 | def extract_core_subject( |
| 80 | topic: str, |
| 81 | *, |
| 82 | noise: Optional[FrozenSet[str]] = None, |
| 83 | max_words: Optional[int] = None, |
| 84 | strip_suffixes: bool = False, |
| 85 | ) -> str: |
| 86 | """Extract core subject from a verbose search query. |
| 87 | |
| 88 | Strips common question/meta prefixes and noise words to produce a |
| 89 | compact search-friendly query. Platforms customize via parameters. |
| 90 | |
| 91 | Args: |
| 92 | topic: Raw user query |
| 93 | noise: Override noise word set (default: NOISE_WORDS) |
| 94 | max_words: Cap result to N words (default: no cap) |
| 95 | strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this) |
| 96 | |
| 97 | Returns: |
| 98 | Cleaned query string |
| 99 | """ |
| 100 | text = topic.lower().strip() |
| 101 | if not text: |
| 102 | return text |
| 103 | |
| 104 | # Phase 1: Strip multi-word prefixes (longest first, stop after first match) |
| 105 | for p in PREFIXES: |
| 106 | if text.startswith(p + ' '): |
| 107 | text = text[len(p):].strip() |
| 108 | break |
| 109 | |
| 110 | # Phase 2: Strip multi-word suffixes (opt-in) |
| 111 | if strip_suffixes: |
| 112 | for s in SUFFIXES: |
| 113 | if text.endswith(' ' + s): |
| 114 | text = text[:-len(s)].strip() |
| 115 | break |
| 116 | |
| 117 | # Phase 3: Filter individual noise words |
| 118 | noise_set = noise if noise is not None else NOISE_WORDS |
| 119 | words = text.split() |
| 120 | filtered = [w for w in words if w not in noise_set] |
| 121 | |
| 122 | # Apply word cap if requested |
| 123 | if max_words is not None and filtered: |
| 124 | filtered = filtered[:max_words] |
| 125 | |
| 126 | result = ' '.join(filtered) if filtered else text |
| 127 | return result.rstrip('?!.') if not max_words else (result or topic.lower().strip()) |
| 128 | |
| 129 | |
| 130 | def infer_query_intent(topic: str) -> str: |
| 131 | """Classify a topic into a coarse intent for adapter query expansion. |
| 132 | |
| 133 | Returns one of: ``comparison``, ``how_to``, ``opinion``, ``product``, |
| 134 | ``prediction``, ``breaking_news`` (default). |
| 135 | |
| 136 | The ``how_to`` regex covers both prefixed forms (``how to install``) |
| 137 | and bare imperatives (``configure``, ``troubleshoot``, ``debug``, |
| 138 | ``fix``). Adapters that previously kept their own copy of this |
| 139 | classifier had drifted to subtly different word lists; this is the |
| 140 | superset. |
| 141 | |
| 142 | Polymarket keeps a custom narrower classifier (prediction-only) and |
| 143 | does NOT delegate here; its expansion only needs that signal. |
| 144 | """ |
| 145 | text = topic.lower().strip() |
| 146 | if re.search(r"\b(vs|versus|compare|difference between)\b", text): |
| 147 | return "comparison" |
| 148 | if re.search( |
| 149 | r"\b(how to|tutorial|guide|setup|step by step|deploy|install|" |
| 150 | r"configuration|configure|troubleshoot|troubleshooting|error|errors|" |
| 151 | r"fix|debug)\b", |
| 152 | text, |
| 153 | ): |
| 154 | return "how_to" |
| 155 | if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text): |
| 156 | return "opinion" |
| 157 | if re.search(r"\b(pricing|feature|features|best .* for)\b", text): |
| 158 | return "product" |
| 159 | if re.search(r"\b(predict|prediction|odds|forecast|chance)\b", text): |
| 160 | return "prediction" |
| 161 | return "breaking_news" |
| 162 | |
| 163 | |
| 164 | def extract_compound_terms(topic: str) -> List[str]: |
| 165 | """Detect multi-word terms that should be quoted in search queries. |
| 166 | |
| 167 | Identifies: |
| 168 | - Hyphenated terms: "multi-agent", "vc-backed" |
| 169 | - Title-cased multi-word names: "Claude Code", "React Native" |
| 170 | |
| 171 | Returns list of terms suitable for quoting (e.g., '"multi-agent"'). |
| 172 | """ |
| 173 | terms: List[str] = [] |
| 174 | |
| 175 | # Hyphenated terms |
| 176 | for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic): |
| 177 | terms.append(match.group()) |
| 178 | |
| 179 | # Title-cased sequences (2+ capitalized words in a row) |
| 180 | for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic): |
| 181 | terms.append(match.group()) |
| 182 | |
| 183 | return terms |
| 184 | |
| 185 | |
| 186 | def leading_mentions(text: Optional[str]) -> List[str]: |
| 187 | """Return the handles a post is directed at: the leading run of @mentions in the text. |
| 188 | |
| 189 | X replies open with the target handle(s) (e.g. "@someone thanks!"), so the |
| 190 | leading run identifies who the post is addressed to. A mention later in the |
| 191 | body is not a reply target and is intentionally ignored. Returns normalized |
| 192 | (``@``-stripped, lowercased) handles, in order. Shared by every X-shaped |
| 193 | source adapter (bird, xquik) so leading-mention parsing has one definition. |
| 194 | """ |
| 195 | out: List[str] = [] |
| 196 | for token in (text or "").split(): |
| 197 | tok = token.strip(",.:;!?") |
| 198 | if tok.startswith("@") and len(tok) > 1: |
| 199 | out.append(tok[1:].lower()) |
| 200 | else: |
| 201 | break |
| 202 | return out |
| 203 |