| 1 | """Polymarket prediction market search via Gamma API (free, no auth required). |
| 2 | |
| 3 | Uses gamma-api.polymarket.com for event/market discovery. |
| 4 | No API key needed - public read-only API with generous rate limits (15K req/10s). |
| 5 | """ |
| 6 | |
| 7 | import json |
| 8 | import math |
| 9 | import re |
| 10 | import sys |
| 11 | from concurrent.futures import ThreadPoolExecutor, as_completed |
| 12 | from typing import Any, Dict, List, Optional |
| 13 | from urllib.parse import quote, quote_plus, urlencode |
| 14 | |
| 15 | from . import http, log |
| 16 | from .relevance import LOW_SIGNAL_QUERY_TOKENS, token_overlap_relevance |
| 17 | |
| 18 | GAMMA_SEARCH_URL = "https://gamma-api.polymarket.com/public-search" |
| 19 | GAMMA_EVENTS_URL = "https://gamma-api.polymarket.com/events" |
| 20 | |
| 21 | # Pages to fetch per query (API returns 5 events per page, limit param is a no-op) |
| 22 | DEPTH_CONFIG = { |
| 23 | "quick": 1, |
| 24 | "default": 3, |
| 25 | "deep": 4, |
| 26 | } |
| 27 | |
| 28 | # Max events to return after merge + dedup + re-ranking |
| 29 | RESULT_CAP = { |
| 30 | "quick": 5, |
| 31 | "default": 15, |
| 32 | "deep": 25, |
| 33 | } |
| 34 | |
| 35 | |
| 36 | def _log(msg: str): |
| 37 | log.source_log("PM", msg, tty_only=False) |
| 38 | |
| 39 | |
| 40 | def _extract_core_subject(topic: str) -> str: |
| 41 | """Extract core subject from topic string. |
| 42 | |
| 43 | Strips common prefixes like 'last 7 days', 'what are people saying about', etc. |
| 44 | """ |
| 45 | topic = topic.strip() |
| 46 | # Remove common leading phrases |
| 47 | prefixes = [ |
| 48 | r"^last \d+ days?\s+", |
| 49 | r"^what(?:'s| is| are) (?:people saying about|happening with|going on with)\s+", |
| 50 | r"^how (?:is|are)\s+", |
| 51 | r"^tell me about\s+", |
| 52 | r"^research\s+", |
| 53 | ] |
| 54 | for pattern in prefixes: |
| 55 | topic = re.sub(pattern, "", topic, flags=re.IGNORECASE) |
| 56 | return topic.strip() |
| 57 | |
| 58 | |
| 59 | def _expand_queries(topic: str) -> List[str]: |
| 60 | """Generate search queries to cast a wider net. |
| 61 | |
| 62 | Strategy: |
| 63 | - Always include the core subject |
| 64 | - Add ALL individual words as standalone searches (not just first) |
| 65 | - Include the full topic if different from core |
| 66 | - Cap at 6 queries, dedupe |
| 67 | """ |
| 68 | core = _extract_core_subject(topic) |
| 69 | queries = [core] |
| 70 | |
| 71 | # Add ALL individual words as separate queries |
| 72 | words = core.split() |
| 73 | if len(words) >= 2: |
| 74 | for word in words: |
| 75 | if len(word) > 1 and word.lower() not in LOW_SIGNAL_QUERY_TOKENS and word.lower() not in _NOISE_WORDS: |
| 76 | queries.append(word) |
| 77 | |
| 78 | # Add the full topic if different from core |
| 79 | if topic.lower().strip() != core.lower(): |
| 80 | queries.append(topic.strip()) |
| 81 | |
| 82 | # Dedupe while preserving order, cap at 6 |
| 83 | seen = set() |
| 84 | unique = [] |
| 85 | for q in queries: |
| 86 | q_lower = q.lower().strip() |
| 87 | if q_lower and q_lower not in seen: |
| 88 | seen.add(q_lower) |
| 89 | unique.append(q.strip()) |
| 90 | return unique[:6] |
| 91 | |
| 92 | |
| 93 | _GENERIC_TAGS = frozenset({"sports", "politics", "crypto", "science", "culture", "pop culture"}) |
| 94 | |
| 95 | # Words that are too generic to serve as the sole topic-match signal. |
| 96 | # If ALL core words from the topic are in this set, we skip filtering (can't meaningfully filter). |
| 97 | # But if some words are informative and some are generic, we require at least one informative word. |
| 98 | _NOISE_WORDS = frozenset({ |
| 99 | # Articles, prepositions, conjunctions |
| 100 | "the", "a", "an", "in", "on", "at", "of", "for", "and", "or", "to", "is", "are", |
| 101 | "was", "were", "will", "be", "by", "with", "from", "as", "it", "its", "not", "no", |
| 102 | "but", "if", "so", "do", "has", "had", "have", "this", "that", "what", "who", |
| 103 | # Directional / geographic terms that cause false matches |
| 104 | "west", "east", "north", "south", "central", "southern", "northern", "eastern", "western", |
| 105 | # Common sports / category terms |
| 106 | "champion", "championship", "league", "division", "conference", "cup", "series", |
| 107 | "team", "game", "match", "season", "win", "winner", "finals", |
| 108 | # Common geographic / place nouns that cause false matches |
| 109 | # "club" -> Athletic Club, Racing Club; "island" -> Epstein's Island, Rhode Island |
| 110 | "club", "island", "city", "park", "hill", "lake", "bay", "beach", "valley", |
| 111 | "river", "mountain", "county", "state", "village", "town", "point", "creek", |
| 112 | "springs", "heights", "ridge", "bridge", "harbor", "port", "station", "center", |
| 113 | "square", "field", "forest", "garden", "tower", "school", "church", "camp", |
| 114 | "ranch", "crossing", "shore", "rock", "summit", "falls", "grove", "haven", |
| 115 | # Generic tech terms — see _DOMAIN_WORDS below, which is folded in here |
| 116 | # Generic prediction market terms |
| 117 | "market", "odds", "prediction", "forecast", "chance", "probability", |
| 118 | # Comparison-query conjunctions — should not count as informative filter tokens |
| 119 | # when the topic is "X vs Y vs Z" |
| 120 | "vs", "versus", |
| 121 | }) |
| 122 | |
| 123 | # Generic tech terms that match too broadly to be the sole signal for a NARROW |
| 124 | # topic ("cli" -> any CLI tool market; "ai" -> every AI market), but which ARE |
| 125 | # the subject when the topic is a domain sweep rather than one product. Kept |
| 126 | # separate from the rest of _NOISE_WORDS — the directional/sports/place words |
| 127 | # there exist to PREVENT false matches ("NFC West" vs a "Kanye West" search), |
| 128 | # so they must never be used as a positive signal. |
| 129 | _DOMAIN_WORDS = frozenset({ |
| 130 | "cli", "mcp", "protocol", "tool", "app", "code", "model", "ai", "api", |
| 131 | "software", "plugin", "skill", "agent", "bot", "search", "research", |
| 132 | }) |
| 133 | |
| 134 | # Soft residue left after stripping domain words from a sweep topic |
| 135 | # ("AI frontier developments"). Domain-word fallback may fire when these are |
| 136 | # the only informative leftovers. Distinctive terms like "benchmark" block it. |
| 137 | _SWEEP_RESIDUE = frozenset({ |
| 138 | "frontier", "developments", "development", "news", "trends", "trend", |
| 139 | "latest", "industry", "space", "ecosystem", "landscape", "overview", |
| 140 | "updates", "update", "future", "outlook", "sector", "field", "world", |
| 141 | }) |
| 142 | |
| 143 | _NOISE_WORDS = _NOISE_WORDS | _DOMAIN_WORDS |
| 144 | |
| 145 | |
| 146 | def _domain_stem(word: str) -> str | None: |
| 147 | """Return the canonical domain token if ``word`` is a domain term or plural. |
| 148 | |
| 149 | Exact-set membership alone treats ``models`` as a hard narrowing term even |
| 150 | though ``model`` is a domain word — which blocked soft AI sweeps and broke |
| 151 | ``AI models`` → ``New AI prediction``. |
| 152 | """ |
| 153 | if word in _DOMAIN_WORDS: |
| 154 | return word |
| 155 | if word.endswith("ies") and len(word) > 4: |
| 156 | stem = word[:-3] + "y" |
| 157 | if stem in _DOMAIN_WORDS: |
| 158 | return stem |
| 159 | if len(word) > 3 and word.endswith("es") and word[:-2] in _DOMAIN_WORDS: |
| 160 | return word[:-2] |
| 161 | if len(word) > 2 and word.endswith("s") and word[:-1] in _DOMAIN_WORDS: |
| 162 | return word[:-1] |
| 163 | return None |
| 164 | |
| 165 | |
| 166 | def _informative_words(core_words: list[str]) -> list[str]: |
| 167 | """Topic words that are neither noise nor (possibly plural) domain terms.""" |
| 168 | return [ |
| 169 | w for w in core_words |
| 170 | if w not in _NOISE_WORDS and _domain_stem(w) is None |
| 171 | ] |
| 172 | |
| 173 | |
| 174 | def _domain_word_fallback_allows(core_words: list[str], informative: list[str], |
| 175 | title_lower: str, title_words: set[str]) -> bool: |
| 176 | """Allow domain-word title matches only for pure/soft domain sweeps. |
| 177 | |
| 178 | Blocks mixed topics like \"MCP protocol benchmark\" from accepting a Kyoto |
| 179 | Protocol market via the shared domain token \"protocol\" when the distinctive |
| 180 | informative word (\"benchmark\") missed. |
| 181 | """ |
| 182 | hard_informative = [w for w in informative if w not in _SWEEP_RESIDUE] |
| 183 | if hard_informative: |
| 184 | return False |
| 185 | domain_stems = [] |
| 186 | seen: set[str] = set() |
| 187 | for w in core_words: |
| 188 | stem = _domain_stem(w) |
| 189 | if stem and stem not in seen: |
| 190 | seen.add(stem) |
| 191 | domain_stems.append(stem) |
| 192 | if not domain_stems: |
| 193 | return False |
| 194 | for word in domain_stems: |
| 195 | if word in title_words or f"{word}s" in title_words or f"{word}es" in title_words: |
| 196 | return True |
| 197 | if len(word) >= 4 and word in title_lower: |
| 198 | return True |
| 199 | return False |
| 200 | |
| 201 | |
| 202 | def _acronym_credit(core_words: list[str], title_words: set[str]) -> int: |
| 203 | """Credit matches when the title abbreviates a phrase the topic spells out. |
| 204 | |
| 205 | Prediction-market titles use shorthand ("AGI by 2030?") while topics arrive |
| 206 | spelled out ("artificial general intelligence"), so word overlap scores zero |
| 207 | on a title that is squarely on topic. For each run of 3+ consecutive |
| 208 | informative words, build its initialism and, if the title carries it as a |
| 209 | whole word, credit one match per abbreviated word. Requiring at least three |
| 210 | letters avoids treating ambiguous tokens such as "ML" as expanded phrases. |
| 211 | """ |
| 212 | informative_set = set(_informative_words(core_words)) |
| 213 | credit = 0 |
| 214 | run: list[str] = [] |
| 215 | for word in core_words + [""]: |
| 216 | if word in informative_set: |
| 217 | run.append(word) |
| 218 | continue |
| 219 | if len(run) >= 3: |
| 220 | acronym = "".join(w[0] for w in run) |
| 221 | if len(acronym) >= 3 and acronym in title_words: |
| 222 | credit = max(credit, len(run)) |
| 223 | run = [] |
| 224 | return credit |
| 225 | |
| 226 | |
| 227 | def _passes_topic_filter(topic: str, event_title: str) -> bool: |
| 228 | """Check if event title contains enough informative words from the topic. |
| 229 | |
| 230 | Prevents noise like "Meek Mill" matching "Mill.com food recycler" by requiring |
| 231 | proportional word overlap. For topics with 3+ informative words, at least 2 must |
| 232 | match. For shorter topics, 1 match suffices (existing behavior). |
| 233 | |
| 234 | Returns True if the event should be kept, False if it should be filtered out. |
| 235 | """ |
| 236 | core = _extract_core_subject(topic).lower() |
| 237 | core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1] |
| 238 | |
| 239 | if not core_words: |
| 240 | return True # No words to check against |
| 241 | |
| 242 | # Split into informative vs generic (domain plurals count as domain, not hard) |
| 243 | informative = _informative_words(core_words) |
| 244 | |
| 245 | # If ALL words are generic, we can't meaningfully filter — keep everything |
| 246 | if not informative: |
| 247 | return True |
| 248 | |
| 249 | # Normalize the title for matching |
| 250 | title_lower = " ".join(re.sub(r"[^\w\s]", " ", event_title.lower()).split()) |
| 251 | title_words = set(title_lower.split()) |
| 252 | |
| 253 | # Count how many informative words appear in the title |
| 254 | match_count = 0 |
| 255 | for word in informative: |
| 256 | # Check as whole word in the title word set |
| 257 | if word in title_words: |
| 258 | match_count += 1 |
| 259 | continue |
| 260 | # Also check as substring for compound words (e.g., "kanye" in "kanyewest") |
| 261 | if len(word) >= 4 and word in title_lower: |
| 262 | match_count += 1 |
| 263 | |
| 264 | # A title that abbreviates what the topic spells out ("AGI" for |
| 265 | # "artificial general intelligence") scores zero above; credit it here. |
| 266 | if match_count < 2: |
| 267 | match_count = max(match_count, |
| 268 | _acronym_credit(core_words, title_words)) |
| 269 | |
| 270 | # For topics with 3+ informative words, require at least 2 matches. |
| 271 | # This prevents single-word false positives like "mill" in "Meek Mill" |
| 272 | # when the topic is "Mill.com food recycler" (3 informative words). |
| 273 | min_matches = 2 if len(informative) >= 3 else 1 |
| 274 | |
| 275 | if match_count >= min_matches: |
| 276 | return True |
| 277 | |
| 278 | # Domain-word fallback for soft domain sweeps only (see helper). |
| 279 | return _domain_word_fallback_allows(core_words, informative, title_lower, title_words) |
| 280 | |
| 281 | |
| 282 | def _passes_any_informative_word(topic: str, event_title: str) -> bool: |
| 283 | """Looser variant of _passes_topic_filter that keeps an item if ANY |
| 284 | informative word from the topic appears in the title. |
| 285 | |
| 286 | Designed for post-merge validation of comparison topics (e.g., "OpenClaw vs |
| 287 | Hermes vs Paperclip"), where a market mentioning just one of the entities |
| 288 | is still on-topic. The stricter _passes_topic_filter (min_matches=2 for |
| 289 | 3+ informative words) is correct for single-entity topics like "Mill.com |
| 290 | food recycler" but drops legitimate single-entity comparison results. |
| 291 | """ |
| 292 | core = _extract_core_subject(topic).lower() |
| 293 | core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1] |
| 294 | if not core_words: |
| 295 | return True |
| 296 | informative = _informative_words(core_words) |
| 297 | if not informative: |
| 298 | return True |
| 299 | |
| 300 | title_lower = " ".join(re.sub(r"[^\w\s]", " ", event_title.lower()).split()) |
| 301 | title_words = set(title_lower.split()) |
| 302 | |
| 303 | for word in informative: |
| 304 | if word in title_words: |
| 305 | return True |
| 306 | if len(word) >= 4 and word in title_lower: |
| 307 | return True |
| 308 | |
| 309 | return _domain_word_fallback_allows(core_words, informative, title_lower, title_words) |
| 310 | |
| 311 | |
| 312 | def filter_items_against_topic(topic: str, items: List[Any]) -> List[Any]: |
| 313 | """Drop items whose title shares no informative word with the original topic. |
| 314 | |
| 315 | Called post-merge from pipeline.py so per-entity subquery results for |
| 316 | comparison topics get re-validated against the ORIGINAL full topic before |
| 317 | landing in the footer. Prevents noise like WTI crude oil or Elon tweet |
| 318 | markets from surviving a loose "Hermes" single-entity subquery match. |
| 319 | |
| 320 | Uses the looser _passes_any_informative_word rule (ANY entity name match |
| 321 | is sufficient) so a market mentioning just one of several compared entities |
| 322 | still counts as on-topic. |
| 323 | |
| 324 | Accepts a list of either raw dicts (with 'title') or SourceItem-like objects |
| 325 | (with .title attribute). Returns the filtered list in the same order. |
| 326 | """ |
| 327 | if not topic: |
| 328 | return items |
| 329 | |
| 330 | filtered = [] |
| 331 | for item in items: |
| 332 | title = getattr(item, "title", None) |
| 333 | if title is None and isinstance(item, dict): |
| 334 | title = item.get("title", "") |
| 335 | title = title or "" |
| 336 | |
| 337 | if _passes_any_informative_word(topic, title): |
| 338 | filtered.append(item) |
| 339 | |
| 340 | dropped = len(items) - len(filtered) |
| 341 | if dropped: |
| 342 | _log(f"Post-merge topic filter dropped {dropped} Polymarket items against full topic '{topic}'") |
| 343 | |
| 344 | return filtered |
| 345 | |
| 346 | |
| 347 | def filter_items_against_keywords(items: List[Any], keywords: List[str]) -> List[Any]: |
| 348 | """Keep only items whose title contains at least one keyword (case-insensitive). |
| 349 | |
| 350 | Intended for disambiguating ambiguous single-token topics like 'Warriors' |
| 351 | via --polymarket-keywords (e.g., 'nba,gsw,golden-state') to filter out |
| 352 | Glasgow Warriors rugby, Honor of Kings Rogue Warriors markets that share |
| 353 | the 'Warriors' token but are not the target entity. |
| 354 | """ |
| 355 | if not keywords: |
| 356 | return items |
| 357 | normalized_keywords = [kw.strip().lower() for kw in keywords if kw and kw.strip()] |
| 358 | if not normalized_keywords: |
| 359 | return items |
| 360 | |
| 361 | filtered = [] |
| 362 | for item in items: |
| 363 | title = getattr(item, "title", None) |
| 364 | if title is None and isinstance(item, dict): |
| 365 | title = item.get("title", "") |
| 366 | title = (title or "").lower() |
| 367 | if any(kw in title for kw in normalized_keywords): |
| 368 | filtered.append(item) |
| 369 | |
| 370 | dropped = len(items) - len(filtered) |
| 371 | if dropped: |
| 372 | _log( |
| 373 | f"Keyword filter dropped {dropped} Polymarket items; " |
| 374 | f"kept {len(filtered)} matching {normalized_keywords}" |
| 375 | ) |
| 376 | |
| 377 | return filtered |
| 378 | |
| 379 | |
| 380 | def _extract_domain_queries(topic: str, events: List[Dict]) -> List[str]: |
| 381 | """Extract domain-indicator search terms from first-pass event tags. |
| 382 | |
| 383 | Uses structured tag metadata from Gamma API events to discover broader |
| 384 | domain categories (e.g., 'NCAA CBB' from a Big 12 basketball event). |
| 385 | Falls back to frequent title bigrams if no useful tags exist. |
| 386 | """ |
| 387 | query_words = set(_extract_core_subject(topic).lower().split()) |
| 388 | |
| 389 | # Collect tag labels from all first-pass events, count occurrences |
| 390 | tag_counts: Dict[str, int] = {} |
| 391 | for event in events: |
| 392 | tags = event.get("tags") or [] |
| 393 | for tag in tags: |
| 394 | label = tag.get("label", "") if isinstance(tag, dict) else str(tag) |
| 395 | if not label: |
| 396 | continue |
| 397 | label_lower = label.lower() |
| 398 | # Skip generic category tags and tags matching existing queries |
| 399 | if label_lower in _GENERIC_TAGS: |
| 400 | continue |
| 401 | if label_lower in query_words: |
| 402 | continue |
| 403 | tag_counts[label] = tag_counts.get(label, 0) + 1 |
| 404 | |
| 405 | # Sort by frequency, take top 2 that appear in 2+ events |
| 406 | domain_queries = [ |
| 407 | label for label, count in sorted(tag_counts.items(), key=lambda x: -x[1]) |
| 408 | if count >= 2 |
| 409 | ][:2] |
| 410 | |
| 411 | return domain_queries |
| 412 | |
| 413 | |
| 414 | def _infer_query_intent(topic: str) -> str: |
| 415 | """Narrower local classifier for Polymarket search tuning only. |
| 416 | |
| 417 | Deliberately does NOT delegate to ``query.infer_query_intent``: |
| 418 | Polymarket only needs the prediction/non-prediction split, and the |
| 419 | broader classifier would route queries to ``how_to``, ``opinion``, |
| 420 | ``product``, etc. without any matching expansion branch downstream. |
| 421 | Keep this narrow until polymarket grows additional intents. |
| 422 | """ |
| 423 | text = topic.lower().strip() |
| 424 | if re.search(r"\b(predict|prediction|odds|forecast|chance|probability|will .* win)\b", text): |
| 425 | return "prediction" |
| 426 | return "breaking_news" |
| 427 | |
| 428 | |
| 429 | def _search_single_query(query: str, page: int = 1) -> Dict[str, Any]: |
| 430 | """Run a single search query against Gamma API.""" |
| 431 | params = { |
| 432 | "q": query, |
| 433 | "page": str(page), |
| 434 | "events_status": "active", |
| 435 | "keep_closed_markets": "0", |
| 436 | } |
| 437 | url = f"{GAMMA_SEARCH_URL}?{urlencode(params)}" |
| 438 | |
| 439 | try: |
| 440 | response = http.request("GET", url, timeout=15, retries=2) |
| 441 | return response |
| 442 | except http.HTTPError as e: |
| 443 | _log(f"Search failed for '{query}' page {page}: {e}") |
| 444 | return {"events": [], "error": str(e)} |
| 445 | except Exception as e: |
| 446 | _log(f"Search failed for '{query}' page {page}: {e}") |
| 447 | return {"events": [], "error": str(e)} |
| 448 | |
| 449 | |
| 450 | def _run_queries_parallel( |
| 451 | queries: List[str], pages: int, all_events: Dict, errors: List, start_idx: int = 0, |
| 452 | ) -> None: |
| 453 | """Run (query, page) combinations in parallel, merging into all_events.""" |
| 454 | with ThreadPoolExecutor(max_workers=min(8, len(queries) * pages)) as executor: |
| 455 | futures = {} |
| 456 | for i, q in enumerate(queries, start=start_idx): |
| 457 | for p in range(1, pages + 1): |
| 458 | future = http.submit_with_context(executor, _search_single_query, q, p) |
| 459 | futures[future] = i |
| 460 | |
| 461 | for future in as_completed(futures): |
| 462 | query_idx = futures[future] |
| 463 | try: |
| 464 | response = future.result(timeout=15) |
| 465 | if response.get("error"): |
| 466 | errors.append(response["error"]) |
| 467 | |
| 468 | events = response.get("events", []) |
| 469 | for event in events: |
| 470 | event_id = event.get("id", "") |
| 471 | if not event_id: |
| 472 | continue |
| 473 | if event_id not in all_events: |
| 474 | all_events[event_id] = (event, query_idx) |
| 475 | elif query_idx < all_events[event_id][1]: |
| 476 | all_events[event_id] = (event, query_idx) |
| 477 | except Exception as e: |
| 478 | errors.append(str(e)) |
| 479 | |
| 480 | |
| 481 | def search_polymarket( |
| 482 | topic: str, |
| 483 | from_date: str, |
| 484 | to_date: str, |
| 485 | depth: str = "default", |
| 486 | ) -> Dict[str, Any]: |
| 487 | """Search Polymarket via Gamma API with two-pass query expansion. |
| 488 | |
| 489 | Pass 1: Run expanded queries in parallel, merge and dedupe by event ID. |
| 490 | Pass 2: Extract domain-indicator terms from first-pass titles, search those. |
| 491 | |
| 492 | Args: |
| 493 | topic: Search topic |
| 494 | from_date: Start date (YYYY-MM-DD) - used for activity filtering |
| 495 | to_date: End date (YYYY-MM-DD) |
| 496 | depth: 'quick', 'default', or 'deep' |
| 497 | |
| 498 | Returns: |
| 499 | Dict with 'events' list and optional 'error'. |
| 500 | """ |
| 501 | pages = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"]) |
| 502 | cap = RESULT_CAP.get(depth, RESULT_CAP["default"]) |
| 503 | queries = _expand_queries(topic) |
| 504 | |
| 505 | _log(f"Searching for '{topic}' with queries: {queries} (pages={pages})") |
| 506 | |
| 507 | # Pass 1: run expanded queries in parallel |
| 508 | all_events: Dict[str, tuple] = {} |
| 509 | errors: List[str] = [] |
| 510 | _run_queries_parallel(queries, pages, all_events, errors) |
| 511 | |
| 512 | # Pass 2: extract domain-indicator terms from first-pass titles and search |
| 513 | first_pass_events = [ev for ev, _ in all_events.values()] |
| 514 | domain_queries = _extract_domain_queries(topic, first_pass_events) |
| 515 | # Filter out queries we already ran |
| 516 | seen_queries = {q.lower() for q in queries} |
| 517 | domain_queries = [dq for dq in domain_queries if dq.lower() not in seen_queries] |
| 518 | |
| 519 | if domain_queries: |
| 520 | _log(f"Domain expansion queries: {domain_queries}") |
| 521 | _run_queries_parallel(domain_queries, 1, all_events, errors, start_idx=len(queries)) |
| 522 | |
| 523 | merged_events = [ev for ev, _ in sorted(all_events.values(), key=lambda x: x[1])] |
| 524 | total_queries = len(queries) + len(domain_queries) |
| 525 | _log(f"Found {len(merged_events)} unique events across {total_queries} queries") |
| 526 | |
| 527 | result = {"events": merged_events, "_cap": cap} |
| 528 | if errors and not merged_events: |
| 529 | result["error"] = "; ".join(errors[:2]) |
| 530 | return result |
| 531 | |
| 532 | |
| 533 | def _format_price_movement(market: Dict[str, Any]) -> Optional[str]: |
| 534 | """Pick the most significant price change and format it. |
| 535 | |
| 536 | Returns string like 'down 11.7% this month' or None if no significant change. |
| 537 | """ |
| 538 | changes = [ |
| 539 | (abs(market.get("oneDayPriceChange") or 0), market.get("oneDayPriceChange"), "today"), |
| 540 | (abs(market.get("oneWeekPriceChange") or 0), market.get("oneWeekPriceChange"), "this week"), |
| 541 | (abs(market.get("oneMonthPriceChange") or 0), market.get("oneMonthPriceChange"), "this month"), |
| 542 | ] |
| 543 | |
| 544 | # Pick the largest absolute change |
| 545 | changes.sort(key=lambda x: x[0], reverse=True) |
| 546 | abs_change, raw_change, period = changes[0] |
| 547 | |
| 548 | # Skip if change is less than 1% (noise) |
| 549 | if abs_change < 0.01: |
| 550 | return None |
| 551 | |
| 552 | direction = "up" if raw_change > 0 else "down" |
| 553 | pct = abs_change * 100 |
| 554 | return f"{direction} {pct:.1f}% {period}" |
| 555 | |
| 556 | |
| 557 | def _parse_outcome_prices(market: Dict[str, Any]) -> List[tuple]: |
| 558 | """Parse outcomePrices JSON string into list of (outcome_name, price) tuples.""" |
| 559 | outcomes_raw = market.get("outcomes") or [] |
| 560 | prices_raw = market.get("outcomePrices") |
| 561 | |
| 562 | if not prices_raw: |
| 563 | return [] |
| 564 | |
| 565 | # Both outcomes and outcomePrices can be JSON-encoded strings |
| 566 | try: |
| 567 | if isinstance(outcomes_raw, str): |
| 568 | outcomes = json.loads(outcomes_raw) |
| 569 | else: |
| 570 | outcomes = outcomes_raw |
| 571 | except (json.JSONDecodeError, TypeError): |
| 572 | outcomes = [] |
| 573 | |
| 574 | try: |
| 575 | if isinstance(prices_raw, str): |
| 576 | prices = json.loads(prices_raw) |
| 577 | else: |
| 578 | prices = prices_raw |
| 579 | except (json.JSONDecodeError, TypeError): |
| 580 | return [] |
| 581 | |
| 582 | result = [] |
| 583 | for i, price in enumerate(prices): |
| 584 | try: |
| 585 | p = float(price) |
| 586 | except (ValueError, TypeError): |
| 587 | continue |
| 588 | name = outcomes[i] if i < len(outcomes) else f"Outcome {i+1}" |
| 589 | result.append((name, p)) |
| 590 | |
| 591 | return result |
| 592 | |
| 593 | |
| 594 | def _shorten_question(question: str) -> str: |
| 595 | """Extract a short display name from a market question. |
| 596 | |
| 597 | 'Will Arizona win the 2026 NCAA Tournament?' -> 'Arizona' |
| 598 | 'Will Duke be a number 1 seed in the 2026 NCAA...' -> 'Duke' |
| 599 | """ |
| 600 | q = question.strip().rstrip("?") |
| 601 | # Common patterns: "Will X win/be/...", "X wins/loses..." |
| 602 | m = re.match(r"^Will\s+(.+?)\s+(?:win|be|make|reach|have|lose|qualify|advance|strike|agree|pass|sign|get|become|remain|stay|leave|survive|next)\b", q, re.IGNORECASE) |
| 603 | if m: |
| 604 | return m.group(1).strip() |
| 605 | m = re.match(r"^Will\s+(.+?)\s+", q, re.IGNORECASE) |
| 606 | if m and len(m.group(1).split()) <= 4: |
| 607 | return m.group(1).strip() |
| 608 | # Fallback: truncate, dropping a leading article so the name doesn't read "an"/"the" |
| 609 | text = q[:40] if len(q) > 40 else q |
| 610 | return re.sub(r"^(?:a|an|the)\s+", "", text, flags=re.I) |
| 611 | |
| 612 | |
| 613 | def _compute_text_similarity(topic: str, title: str, outcomes: List[str] = None) -> float: |
| 614 | """Score how well the event title (or outcome names) match the search topic. |
| 615 | |
| 616 | Returns 0.0-1.0. Exact title phrase match gets 1.0. Otherwise we reuse the |
| 617 | shared query-centric relevance scorer and take the best title/outcome match. |
| 618 | """ |
| 619 | core = _extract_core_subject(topic).lower() |
| 620 | title_lower = title.lower() |
| 621 | if not core: |
| 622 | return 0.5 |
| 623 | |
| 624 | # Full substring match in title |
| 625 | if core in title_lower: |
| 626 | return 1.0 |
| 627 | |
| 628 | # Same match, abbreviated: "AGI" standing in for an informative phrase. |
| 629 | # Use the filter's matcher so modifiers and minimum acronym length cannot |
| 630 | # produce different decisions at the filtering and scoring stages. |
| 631 | core_words = [w for w in re.sub(r"[^\w\s]", " ", core).split() if len(w) > 1] |
| 632 | title_words = set(re.sub(r"[^\w\s]", " ", title_lower).split()) |
| 633 | if _acronym_credit(core_words, title_words): |
| 634 | return 1.0 |
| 635 | |
| 636 | query_type = _infer_query_intent(topic) |
| 637 | title_score = token_overlap_relevance(core, title) |
| 638 | best_score = title_score |
| 639 | |
| 640 | if outcomes: |
| 641 | for outcome_name in outcomes: |
| 642 | outcome_lower = outcome_name.lower() |
| 643 | outcome_score = token_overlap_relevance(core, outcome_name) |
| 644 | if _strong_phrase_match(core, outcome_lower): |
| 645 | outcome_score = max(outcome_score, 0.92 if len(outcome_lower.split()) >= 2 else 0.88) |
| 646 | if title_score < 0.3: |
| 647 | outcome_cap = 0.55 if query_type == "prediction" else 0.24 |
| 648 | outcome_score = min(outcome_cap, outcome_score) |
| 649 | else: |
| 650 | outcome_score = max(title_score, 0.75 * title_score + 0.25 * outcome_score) |
| 651 | best_score = max(best_score, outcome_score) |
| 652 | |
| 653 | return round(best_score, 2) |
| 654 | |
| 655 | |
| 656 | def _strong_phrase_match(core: str, candidate: str) -> bool: |
| 657 | """Require real token matches, not accidental short substrings. |
| 658 | |
| 659 | This prevents binary outcomes like "No" from matching "nano" or similar |
| 660 | short-string accidents. |
| 661 | """ |
| 662 | candidate = " ".join(re.sub(r"[^\w\s]", " ", candidate.lower()).split()) |
| 663 | core = " ".join(re.sub(r"[^\w\s]", " ", core.lower()).split()) |
| 664 | if not candidate or not core: |
| 665 | return False |
| 666 | |
| 667 | candidate_tokens = candidate.split() |
| 668 | core_tokens = set(core.split()) |
| 669 | |
| 670 | if len(candidate_tokens) >= 2: |
| 671 | return candidate in core or core in candidate |
| 672 | |
| 673 | token = candidate_tokens[0] |
| 674 | return len(token) > 2 and token in core_tokens |
| 675 | |
| 676 | |
| 677 | def _safe_float(val, default=0.0) -> float: |
| 678 | """Safely convert a value to float.""" |
| 679 | try: |
| 680 | return float(val or default) |
| 681 | except (ValueError, TypeError): |
| 682 | return default |
| 683 | |
| 684 | |
| 685 | def parse_polymarket_response( |
| 686 | response: Dict[str, Any], |
| 687 | topic: str = "", |
| 688 | *, |
| 689 | include_all_outcomes: bool = False, |
| 690 | include_closed: bool = False, |
| 691 | ) -> List[Dict[str, Any]]: |
| 692 | """Parse Gamma API response into normalized item dicts. |
| 693 | |
| 694 | Each event becomes one item showing its title and top markets. |
| 695 | |
| 696 | Args: |
| 697 | response: Raw Gamma API response |
| 698 | topic: Original search topic (for relevance scoring) |
| 699 | |
| 700 | Returns: |
| 701 | List of item dicts ready for normalization. |
| 702 | """ |
| 703 | events = response.get("events", []) |
| 704 | items = [] |
| 705 | |
| 706 | filtered_count = 0 |
| 707 | for i, event in enumerate(events): |
| 708 | event_id = event.get("id", "") |
| 709 | title = event.get("title", "") |
| 710 | slug = event.get("slug", "") |
| 711 | |
| 712 | # Filter: skip closed/resolved events |
| 713 | if not include_closed: |
| 714 | if event.get("closed", False): |
| 715 | continue |
| 716 | if not event.get("active", True): |
| 717 | continue |
| 718 | |
| 719 | # Filter: skip events that don't match the topic's core subject |
| 720 | # This prevents "NFC West" from matching a "Kanye West" search |
| 721 | if topic and not _passes_topic_filter(topic, title): |
| 722 | filtered_count += 1 |
| 723 | continue |
| 724 | |
| 725 | # Get markets for this event |
| 726 | markets = event.get("markets", []) |
| 727 | if not markets: |
| 728 | continue |
| 729 | |
| 730 | # Filter to active, open markets with liquidity (excludes resolved markets) |
| 731 | active_markets = [] |
| 732 | for m in markets: |
| 733 | if not include_closed: |
| 734 | if m.get("closed", False): |
| 735 | continue |
| 736 | if not m.get("active", True): |
| 737 | continue |
| 738 | # Must have liquidity (resolved markets have 0 or None) |
| 739 | try: |
| 740 | liq = float(m.get("liquidity", 0) or 0) |
| 741 | except (ValueError, TypeError): |
| 742 | liq = 0 |
| 743 | if include_closed or liq > 0: |
| 744 | active_markets.append(m) |
| 745 | |
| 746 | if not active_markets: |
| 747 | continue |
| 748 | |
| 749 | # Sort markets by volume (most liquid first) |
| 750 | def market_volume(m): |
| 751 | try: |
| 752 | return float(m.get("volume", 0) or 0) |
| 753 | except (ValueError, TypeError): |
| 754 | return 0 |
| 755 | active_markets.sort(key=market_volume, reverse=True) |
| 756 | |
| 757 | # Take top market for the event |
| 758 | top_market = active_markets[0] |
| 759 | |
| 760 | # Collect outcome names from ALL active markets (not just top) for similarity scoring |
| 761 | # Filter to outcomes with price > 1% to avoid noise |
| 762 | # Also extract subjects from market questions for neg-risk events (outcomes are Yes/No) |
| 763 | all_outcome_names = [] |
| 764 | for m in active_markets: |
| 765 | for name, price in _parse_outcome_prices(m): |
| 766 | if price > 0.01 and name not in all_outcome_names: |
| 767 | all_outcome_names.append(name) |
| 768 | # For neg-risk binary markets (Yes/No outcomes), the team/entity name |
| 769 | # lives in the question, e.g., "Will Arizona win the NCAA Tournament?" |
| 770 | question = m.get("question", "") |
| 771 | if question and question != title: |
| 772 | all_outcome_names.append(question) |
| 773 | |
| 774 | # Parse outcome prices - for multi-market events with Yes/No binary |
| 775 | # sub-markets, synthesize from market questions to show actual |
| 776 | # team/entity probabilities instead of a single market's Yes/No |
| 777 | outcome_prices = _parse_outcome_prices(top_market) |
| 778 | top_outcomes_are_binary = ( |
| 779 | len(outcome_prices) == 2 |
| 780 | and {n.lower() for n, _ in outcome_prices} == {"yes", "no"} |
| 781 | ) |
| 782 | if top_outcomes_are_binary and len(active_markets) > 1: |
| 783 | synth_outcomes = [] |
| 784 | for m in active_markets: |
| 785 | q = m.get("question", "") |
| 786 | if not q: |
| 787 | continue |
| 788 | pairs = _parse_outcome_prices(m) |
| 789 | yes_price = next((p for name, p in pairs if name.lower() == "yes"), None) |
| 790 | if yes_price is not None and yes_price > 0.005: |
| 791 | synth_outcomes.append((q, yes_price)) |
| 792 | if synth_outcomes: |
| 793 | synth_outcomes.sort(key=lambda x: x[1], reverse=True) |
| 794 | outcome_prices = [(_shorten_question(q), p) for q, p in synth_outcomes] |
| 795 | |
| 796 | # Format price movement |
| 797 | price_movement = _format_price_movement(top_market) |
| 798 | |
| 799 | # Volume and liquidity - prefer event-level (more stable), fall back to market-level |
| 800 | event_volume1mo = _safe_float(event.get("volume1mo")) |
| 801 | event_volume1wk = _safe_float(event.get("volume1wk")) |
| 802 | event_liquidity = _safe_float(event.get("liquidity")) |
| 803 | event_competitive = _safe_float(event.get("competitive")) |
| 804 | volume24hr = _safe_float(event.get("volume24hr")) or _safe_float(top_market.get("volume24hr")) |
| 805 | liquidity = event_liquidity or _safe_float(top_market.get("liquidity")) |
| 806 | |
| 807 | # Event URL |
| 808 | url = f"https://polymarket.com/event/{slug}" if slug else f"https://polymarket.com/event/{event_id}" |
| 809 | |
| 810 | # Date: use updatedAt from event |
| 811 | updated_at = event.get("updatedAt", "") |
| 812 | date_str = None |
| 813 | if updated_at: |
| 814 | try: |
| 815 | date_str = updated_at[:10] # YYYY-MM-DD |
| 816 | except (IndexError, TypeError): |
| 817 | pass |
| 818 | |
| 819 | # End date for the market |
| 820 | end_date = top_market.get("endDate") |
| 821 | if end_date: |
| 822 | try: |
| 823 | end_date = end_date[:10] |
| 824 | except (IndexError, TypeError): |
| 825 | end_date = None |
| 826 | |
| 827 | # Semantic relevance should dominate. Market quality should refine |
| 828 | # relevant matches, not rescue unrelated high-liquidity events. |
| 829 | text_score = _compute_text_similarity(topic, title, all_outcome_names) if topic else 0.5 |
| 830 | |
| 831 | # Volume signal: log-scaled monthly volume (most stable signal) |
| 832 | vol_raw = event_volume1mo or event_volume1wk or volume24hr |
| 833 | vol_score = min(1.0, math.log1p(vol_raw) / 16) # ~$9M = 1.0 |
| 834 | |
| 835 | # Liquidity signal |
| 836 | liq_score = min(1.0, math.log1p(liquidity) / 14) # ~$1.2M = 1.0 |
| 837 | |
| 838 | # Price movement: daily weighted more than monthly |
| 839 | day_change = abs(top_market.get("oneDayPriceChange") or 0) * 3 |
| 840 | week_change = abs(top_market.get("oneWeekPriceChange") or 0) * 2 |
| 841 | month_change = abs(top_market.get("oneMonthPriceChange") or 0) |
| 842 | max_change = max(day_change, week_change, month_change) |
| 843 | movement_score = min(1.0, max_change * 5) # 20% change = 1.0 |
| 844 | |
| 845 | # Competitive bonus: markets near 50/50 are more interesting |
| 846 | competitive_score = event_competitive |
| 847 | |
| 848 | market_quality = ( |
| 849 | 0.50 * vol_score + |
| 850 | 0.25 * liq_score + |
| 851 | 0.15 * movement_score + |
| 852 | 0.10 * competitive_score |
| 853 | ) |
| 854 | relevance = min(1.0, text_score * (0.75 + 0.25 * market_quality)) |
| 855 | |
| 856 | # Surface the topic-matching outcome to the front before truncating |
| 857 | if topic and outcome_prices: |
| 858 | core = _extract_core_subject(topic).lower() |
| 859 | core_tokens = set(core.split()) |
| 860 | reordered = [] |
| 861 | rest = [] |
| 862 | for pair in outcome_prices: |
| 863 | name_lower = pair[0].lower() |
| 864 | # Match if full core is substring, or name is substring of core, |
| 865 | # or any core token appears in the name (handles long question strings) |
| 866 | if (core in name_lower or name_lower in core |
| 867 | or any(tok in name_lower for tok in core_tokens if len(tok) > 2)): |
| 868 | reordered.append(pair) |
| 869 | else: |
| 870 | rest.append(pair) |
| 871 | if reordered: |
| 872 | outcome_prices = reordered + rest |
| 873 | |
| 874 | # Normal display payloads stay compact. Verification requests the |
| 875 | # complete snapshot so topic-promoted outcomes remain re-checkable. |
| 876 | top_outcomes = outcome_prices if include_all_outcomes else outcome_prices[:3] |
| 877 | remaining = len(outcome_prices) - 3 |
| 878 | if remaining < 0: |
| 879 | remaining = 0 |
| 880 | |
| 881 | items.append({ |
| 882 | "event_id": event_id, |
| 883 | "title": title, |
| 884 | "question": top_market.get("question", title), |
| 885 | "url": url, |
| 886 | "outcome_prices": top_outcomes, |
| 887 | "outcomes_remaining": remaining, |
| 888 | "price_movement": price_movement, |
| 889 | "volume24hr": volume24hr, |
| 890 | "volume1mo": event_volume1mo, |
| 891 | "liquidity": liquidity, |
| 892 | "date": date_str, |
| 893 | "end_date": end_date, |
| 894 | "relevance": round(relevance, 2), |
| 895 | "why_relevant": f"Prediction market: {title[:60]}", |
| 896 | }) |
| 897 | |
| 898 | if filtered_count: |
| 899 | _log(f"Filtered {filtered_count} noise events (topic: '{topic}')") |
| 900 | |
| 901 | # Sort by relevance (quality-signal ranked) and apply cap |
| 902 | items.sort(key=lambda x: x["relevance"], reverse=True) |
| 903 | |
| 904 | # Drop ALL results if nothing is genuinely on-topic. |
| 905 | # If the best item's relevance is below the threshold, the Gamma API |
| 906 | # returned only tangential matches (e.g., "Anthropic best AI model" |
| 907 | # for a "CLI vs MCP" query). Better to show 0 than noise. |
| 908 | _MIN_RELEVANCE = 0.15 |
| 909 | if items and items[0]["relevance"] < _MIN_RELEVANCE: |
| 910 | _log(f"All {len(items)} Polymarket results below relevance threshold " |
| 911 | f"({items[0]['relevance']:.2f} < {_MIN_RELEVANCE}), dropping all") |
| 912 | return [] |
| 913 | |
| 914 | # Per-item floor: drop individual noise items even if the best item passed |
| 915 | _ITEM_MIN_RELEVANCE = 0.10 |
| 916 | before_count = len(items) |
| 917 | items = [i for i in items if i["relevance"] >= _ITEM_MIN_RELEVANCE] |
| 918 | dropped = before_count - len(items) |
| 919 | if dropped: |
| 920 | _log(f"Dropped {dropped} Polymarket items below per-item relevance floor ({_ITEM_MIN_RELEVANCE})") |
| 921 | |
| 922 | cap = response.get("_cap", len(items)) |
| 923 | return items[:cap] |
| 924 | |
| 925 | |
| 926 | def refetch_datum(item: Any, datum_key: str) -> dict[str, Any]: |
| 927 | """Re-fetch one event datum through the replay-aware HTTP wrapper.""" |
| 928 | event_id = str(getattr(item, "metadata", {}).get("event_id") or "").strip() |
| 929 | slug_match = re.search(r"/event/([^/?#]+)", str(getattr(item, "url", ""))) |
| 930 | cached_item_id = str(getattr(item, "item_id", "") or "").strip() |
| 931 | # On the slug fallback, a slug can be re-used by a re-created event. When |
| 932 | # the cached item still carries the original Gamma event id (numeric; the |
| 933 | # synthetic PM<N> parse fallback carries no identity), the response id |
| 934 | # must match it too, or the verdict would come from another market. |
| 935 | expected_id = ( |
| 936 | cached_item_id |
| 937 | if not event_id and re.fullmatch(r"\d+", cached_item_id) |
| 938 | else "" |
| 939 | ) |
| 940 | if event_id: |
| 941 | payload = http.request( |
| 942 | "GET", f"{GAMMA_EVENTS_URL}/{quote(event_id)}", timeout=10, retries=2, |
| 943 | ) |
| 944 | elif slug_match: |
| 945 | if not expected_id: |
| 946 | # No event id anywhere: slug equality alone cannot verify event |
| 947 | # identity, so fail closed (unsupported) instead of re-deriving a |
| 948 | # verdict from whatever event currently owns the slug. |
| 949 | raise ValueError( |
| 950 | "Polymarket item carries no event id; slug equality alone " |
| 951 | "cannot verify event identity" |
| 952 | ) |
| 953 | requested_slug = slug_match.group(1) |
| 954 | payload = http.request( |
| 955 | "GET", GAMMA_EVENTS_URL, params={"slug": requested_slug}, |
| 956 | timeout=10, retries=2, |
| 957 | ) |
| 958 | else: |
| 959 | raise ValueError("Polymarket item has no event id or slug") |
| 960 | |
| 961 | requested_slug = slug_match.group(1) if slug_match else None |
| 962 | |
| 963 | def _matches_identity(entry: dict) -> bool: |
| 964 | if str(entry.get("slug") or "").strip() != requested_slug: |
| 965 | return False |
| 966 | if expected_id and str(entry.get("id") or "").strip() != expected_id: |
| 967 | return False |
| 968 | return True |
| 969 | |
| 970 | def _pick_event(events: list) -> Any: |
| 971 | candidates = [entry for entry in events if isinstance(entry, dict)] |
| 972 | if requested_slug is None: |
| 973 | return candidates[0] if candidates else None |
| 974 | # Verify identity: Gamma slug queries can return multiple or loosely |
| 975 | # matched events, and verifying a claim against another market's |
| 976 | # prices would fabricate current/stale verdicts. |
| 977 | for entry in candidates: |
| 978 | if _matches_identity(entry): |
| 979 | return entry |
| 980 | return None |
| 981 | |
| 982 | if isinstance(payload, list): |
| 983 | event = _pick_event(payload) |
| 984 | elif isinstance(payload, dict) and isinstance(payload.get("events"), list): |
| 985 | event = _pick_event(payload.get("events") or []) |
| 986 | else: |
| 987 | event = payload |
| 988 | if ( |
| 989 | requested_slug is not None |
| 990 | and isinstance(event, dict) |
| 991 | and ( |
| 992 | str(event.get("slug") or "").strip() not in ("", requested_slug) |
| 993 | or ( |
| 994 | expected_id |
| 995 | and str(event.get("id") or "").strip() not in ("", expected_id) |
| 996 | ) |
| 997 | ) |
| 998 | ): |
| 999 | event = None |
| 1000 | if not isinstance(event, dict): |
| 1001 | raise KeyError("Polymarket event was not found") |
| 1002 | # Mixed events: an active event can carry resolved child markets whose |
| 1003 | # high volume would win the parse and swap the outcome labels. Only fall |
| 1004 | # back to closed markets when nothing is active (fully resolved event - |
| 1005 | # the stale-odds transition verification exists to catch). |
| 1006 | markets = event.get("markets") or [] |
| 1007 | has_active = any( |
| 1008 | isinstance(m, dict) and m.get("active", True) and not m.get("closed", False) |
| 1009 | for m in markets |
| 1010 | ) |
| 1011 | parsed = parse_polymarket_response( |
| 1012 | {"events": [event]}, |
| 1013 | include_all_outcomes=True, |
| 1014 | include_closed=not has_active, |
| 1015 | ) |
| 1016 | if not parsed: |
| 1017 | raise KeyError("Polymarket event is closed, unavailable, or malformed") |
| 1018 | refreshed = parsed[0] |
| 1019 | values: dict[str, Any] = {} |
| 1020 | outcome_pairs = refreshed.get("outcome_prices") or [] |
| 1021 | outcome_totals: dict[str, int] = {} |
| 1022 | for name, _price in outcome_pairs: |
| 1023 | normalized = str(name).casefold() |
| 1024 | outcome_totals[normalized] = outcome_totals.get(normalized, 0) + 1 |
| 1025 | outcome_counts: dict[str, int] = {} |
| 1026 | for name, price in outcome_pairs: |
| 1027 | normalized = str(name).casefold() |
| 1028 | occurrence = outcome_counts.get(normalized, 0) |
| 1029 | outcome_counts[normalized] = occurrence + 1 |
| 1030 | key = f"{name}\x1f{occurrence}" if outcome_totals[normalized] > 1 else str(name) |
| 1031 | values[key] = price |
| 1032 | if refreshed.get("end_date") is not None: |
| 1033 | values["end_date"] = refreshed["end_date"] |
| 1034 | |
| 1035 | if datum_key == "end_date": |
| 1036 | value = values.get("end_date") |
| 1037 | else: |
| 1038 | if "\x1f" in datum_key: |
| 1039 | outcome_name, raw_occurrence = datum_key.rsplit("\x1f", 1) |
| 1040 | occurrence = int(raw_occurrence) |
| 1041 | else: |
| 1042 | outcome_name, occurrence = datum_key, 0 |
| 1043 | matches = [ |
| 1044 | price |
| 1045 | for name, price in refreshed.get("outcome_prices") or [] |
| 1046 | if str(name).casefold() == outcome_name.casefold() |
| 1047 | ] |
| 1048 | value = matches[occurrence] if occurrence < len(matches) else None |
| 1049 | if value is None: |
| 1050 | raise KeyError(f"Polymarket datum {datum_key!r} was not found") |
| 1051 | return { |
| 1052 | "value": value, |
| 1053 | "values": values, |
| 1054 | "url": str(getattr(item, "url", "")), |
| 1055 | "timestamp": event.get("updatedAt"), |
| 1056 | } |
| 1057 |