| 1 | """Shared token-overlap relevance scoring for search result ranking. |
| 2 | |
| 3 | The score is intentionally query-centric: |
| 4 | - exact phrase matches should score very high |
| 5 | - partial matches should pay a meaningful penalty |
| 6 | - matches on generic words alone ("odds", "review") should not pass as relevant |
| 7 | """ |
| 8 | |
| 9 | import re |
| 10 | from typing import List, Optional, Set |
| 11 | |
| 12 | from . import cjk |
| 13 | |
| 14 | # Stopwords for relevance computation (common English words that dilute token overlap) |
| 15 | STOPWORDS = frozenset({ |
| 16 | 'the', 'a', 'an', 'to', 'for', 'how', 'is', 'in', 'of', 'on', |
| 17 | 'and', 'with', 'from', 'by', 'at', 'this', 'that', 'it', 'my', |
| 18 | 'your', 'i', 'me', 'we', 'you', 'what', 'are', 'do', 'can', |
| 19 | 'its', 'be', 'or', 'not', 'no', 'so', 'if', 'but', 'about', |
| 20 | 'all', 'just', 'get', 'has', 'have', 'was', 'will', |
| 21 | # Hebrew function words / prepositions / conjunctions |
| 22 | 'את', 'של', 'על', 'עם', 'אל', 'כי', 'לא', 'הוא', 'היא', 'הם', |
| 23 | 'הן', 'אנו', 'אנחנו', 'זה', 'זו', 'זאת', 'כל', 'יש', 'אין', |
| 24 | 'כבר', 'רק', 'גם', 'כן', 'אם', 'או', 'אבל', 'כך', 'מה', 'מי', |
| 25 | 'איך', 'למה', 'כמה', 'היה', 'הייתה', 'היו', 'יהיה', 'יהיו', |
| 26 | # Hebrew definite article / prefixes appearing as standalone tokens after split |
| 27 | 'ה', 'ב', 'ל', 'מ', 'כ', 'ו', 'ש', |
| 28 | }) | cjk.CHINESE_STOPWORDS |
| 29 | |
| 30 | # Shared relevance-ranking thresholds for the Reddit pipelines (keyed + keyless). |
| 31 | # Single source of truth so both paths apply identical thresholds to the same |
| 32 | # query. RELEVANCE_FLOOR: posts below this are off-topic; the zero-overlap tail is |
| 33 | # dropped when anything relevant remains. MIN_ON_TOPIC: how many posts must clear |
| 34 | # the soft floor before it is applied wholesale. |
| 35 | RELEVANCE_FLOOR = 0.1 |
| 36 | MIN_ON_TOPIC = 5 |
| 37 | |
| 38 | |
| 39 | # Synonym groups for relevance scoring (bidirectional expansion) |
| 40 | # Superset of all platform-specific synonym dicts |
| 41 | SYNONYMS = { |
| 42 | 'hip': {'rap', 'hiphop'}, |
| 43 | 'hop': {'rap', 'hiphop'}, |
| 44 | 'rap': {'hip', 'hop', 'hiphop'}, |
| 45 | 'hiphop': {'rap', 'hip', 'hop'}, |
| 46 | 'js': {'javascript'}, |
| 47 | 'javascript': {'js'}, |
| 48 | 'ts': {'typescript'}, |
| 49 | 'typescript': {'ts'}, |
| 50 | 'ai': {'artificial', 'intelligence'}, |
| 51 | 'ml': {'machine', 'learning'}, |
| 52 | 'react': {'reactjs'}, |
| 53 | 'reactjs': {'react'}, |
| 54 | 'svelte': {'sveltejs'}, |
| 55 | 'sveltejs': {'svelte'}, |
| 56 | 'vue': {'vuejs'}, |
| 57 | 'vuejs': {'vue'}, |
| 58 | } |
| 59 | |
| 60 | # Generic query words that should not carry relevance on their own. |
| 61 | # They still help when paired with stronger entity/topic matches. |
| 62 | LOW_SIGNAL_QUERY_TOKENS = frozenset({ |
| 63 | 'advice', 'animation', 'animations', 'best', 'chance', 'chances', |
| 64 | 'code', 'compare', 'comparison', 'differences', 'explain', 'guide', |
| 65 | 'guides', 'how', 'latest', 'news', 'odds', 'opinion', 'opinions', |
| 66 | 'prediction', 'predictions', 'probability', 'probabilities', 'prompt', |
| 67 | 'prompting', 'prompts', 'rate', 'review', 'reviews', 'thoughts', |
| 68 | 'tip', 'tips', 'tutorial', 'tutorials', 'update', 'updates', 'use', |
| 69 | 'using', 'versus', 'vs', 'worth', |
| 70 | }) |
| 71 | |
| 72 | |
| 73 | def tokenize(text: str) -> Set[str]: |
| 74 | """Lowercase, strip punctuation, remove stopwords, drop single-char tokens. |
| 75 | |
| 76 | Expands tokens with synonyms for better cross-domain matching. |
| 77 | |
| 78 | Chinese text is segmented via cjk.segment (jieba or character bigrams) so |
| 79 | overlap scoring works on Chinese sources; ASCII text keeps the original |
| 80 | whitespace path. |
| 81 | """ |
| 82 | words = cjk.segment(text) |
| 83 | tokens = {w for w in words if w not in STOPWORDS and len(w) > 1} |
| 84 | expanded = set(tokens) |
| 85 | for t in tokens: |
| 86 | if t in SYNONYMS: |
| 87 | expanded.update(SYNONYMS[t]) |
| 88 | return expanded |
| 89 | |
| 90 | |
| 91 | def _normalize_phrase(text: str) -> str: |
| 92 | """Normalize text for phrase containment checks.""" |
| 93 | return ' '.join(re.sub(r'[^\w\s]', ' ', text.lower()).split()) |
| 94 | |
| 95 | |
| 96 | class PreparedQuery: |
| 97 | """Precomputed query shape reused across items in a stream. |
| 98 | |
| 99 | Built once per ranking_query; reused by token_overlap_relevance so the |
| 100 | per-item normalize/score loops don't re-tokenize the same query N times. |
| 101 | """ |
| 102 | |
| 103 | __slots__ = ("raw", "q_tokens", "informative_q_tokens", "normalized_phrase") |
| 104 | |
| 105 | def __init__(self, query: str) -> None: |
| 106 | self.raw = query |
| 107 | self.q_tokens = tokenize(query) |
| 108 | informative = {t for t in self.q_tokens if t not in LOW_SIGNAL_QUERY_TOKENS} |
| 109 | self.informative_q_tokens = informative or self.q_tokens |
| 110 | self.normalized_phrase = _normalize_phrase(query) |
| 111 | |
| 112 | |
| 113 | def _as_prepared(query: "str | PreparedQuery") -> PreparedQuery: |
| 114 | return query if isinstance(query, PreparedQuery) else PreparedQuery(query) |
| 115 | |
| 116 | |
| 117 | def token_overlap_relevance( |
| 118 | query: "str | PreparedQuery", |
| 119 | text: str, |
| 120 | hashtags: Optional[List[str]] = None, |
| 121 | ) -> float: |
| 122 | """Compute a query-centric relevance score between 0.0 and 1.0. |
| 123 | |
| 124 | The score combines: |
| 125 | - query coverage |
| 126 | - informative-token coverage |
| 127 | - a small precision term to penalize extra noise |
| 128 | - an exact phrase bonus |
| 129 | |
| 130 | Generic tokens alone are capped below typical relevance filter thresholds. |
| 131 | |
| 132 | Args: |
| 133 | query: Search query |
| 134 | text: Content text to match against |
| 135 | hashtags: Optional list of hashtags (TikTok/Instagram). Concatenated |
| 136 | hashtags are split to match query tokens (e.g. "claudecode" matches "claude"). |
| 137 | |
| 138 | Returns: |
| 139 | Float between 0.0 and 1.0 (0.5 for empty queries) |
| 140 | """ |
| 141 | prepared = _as_prepared(query) |
| 142 | q_tokens = prepared.q_tokens |
| 143 | |
| 144 | # Combine text and hashtags for matching |
| 145 | combined = text |
| 146 | if hashtags: |
| 147 | combined = f"{text} {' '.join(hashtags)}" |
| 148 | t_tokens = tokenize(combined) |
| 149 | |
| 150 | # Split concatenated hashtags (e.g., "claudecode" -> matches "claude", "code") |
| 151 | if hashtags: |
| 152 | for tag in hashtags: |
| 153 | tag_lower = tag.lower() |
| 154 | for qt in q_tokens: |
| 155 | if qt in tag_lower and qt != tag_lower: |
| 156 | t_tokens.add(qt) |
| 157 | |
| 158 | if not q_tokens: |
| 159 | return 0.5 # Neutral fallback for empty/stopword-only queries |
| 160 | |
| 161 | overlap_tokens = q_tokens & t_tokens |
| 162 | overlap = len(overlap_tokens) |
| 163 | if overlap == 0: |
| 164 | return 0.0 |
| 165 | |
| 166 | informative_q_tokens = prepared.informative_q_tokens |
| 167 | |
| 168 | coverage = overlap / len(q_tokens) |
| 169 | informative_overlap = len(informative_q_tokens & t_tokens) / len(informative_q_tokens) |
| 170 | precision_denominator = min(len(t_tokens), len(q_tokens) + 4) or 1 |
| 171 | precision = overlap / precision_denominator |
| 172 | |
| 173 | phrase_bonus = 0.0 |
| 174 | normalized_query = prepared.normalized_phrase |
| 175 | normalized_text = _normalize_phrase(combined) |
| 176 | if normalized_query: |
| 177 | contained = normalized_query in normalized_text |
| 178 | if not contained and cjk.has_cjk(normalized_query): |
| 179 | # CJK has no inter-word spaces, so a multi-token Chinese query like |
| 180 | # "国产大模型 测评" never appears verbatim in continuous source text |
| 181 | # ("...国产大模型的最新测评"). Retry the containment with spaces |
| 182 | # removed so the phrase bonus isn't permanently dead for Chinese. |
| 183 | # Gated on has_cjk so English ("react hooks") keeps space-sensitive |
| 184 | # matching and doesn't gain spurious bonuses from concatenation. |
| 185 | contained = normalized_query.replace(" ", "") in normalized_text.replace(" ", "") |
| 186 | if contained: |
| 187 | phrase_bonus = 0.12 if len(normalized_query.split()) > 1 else 0.16 |
| 188 | |
| 189 | base = ( |
| 190 | 0.55 * (coverage ** 1.35) + |
| 191 | 0.25 * informative_overlap + |
| 192 | 0.20 * precision |
| 193 | ) |
| 194 | |
| 195 | # If we only matched generic query words, keep the score below the |
| 196 | # normal relevance filter threshold so these do not survive by default. |
| 197 | if informative_q_tokens and not (informative_q_tokens & t_tokens): |
| 198 | return round(min(0.24, base), 2) |
| 199 | |
| 200 | return round(min(1.0, base + phrase_bonus), 2) |
| 201 |