| 1 | """Auto-resolve subreddits, X handles, and current events context for a topic. |
| 2 | |
| 3 | Uses web search (Brave/Exa/Serper) to discover relevant communities and context |
| 4 | before the planner runs. This is the engine-side equivalent of SKILL.md Steps |
| 5 | 0.55/0.75 which use Claude Code's WebSearch tool. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import re |
| 11 | from concurrent.futures import ThreadPoolExecutor, as_completed |
| 12 | from datetime import datetime, timezone |
| 13 | from typing import Optional |
| 14 | |
| 15 | from . import categories, dates, grounding, log |
| 16 | |
| 17 | MAX_SUBS = 10 |
| 18 | |
| 19 | |
| 20 | def _log(msg: str) -> None: |
| 21 | log.source_log("Resolve", msg, tty_only=False) |
| 22 | |
| 23 | |
| 24 | def _merge_category_peers(topic: str, subreddits: list[str]) -> tuple[list[str], Optional[str]]: |
| 25 | """Extend the WebSearch-extracted subreddit list with category peers. |
| 26 | |
| 27 | Classifies the topic, fetches the category's peer subs, dedupes |
| 28 | case-insensitively against the existing list, and appends missing |
| 29 | peers in priority order. Caps the final list at MAX_SUBS, preserving |
| 30 | every WebSearch-returned sub (they are the freshest signal) and |
| 31 | trimming from the peer-additions end. |
| 32 | |
| 33 | Returns a tuple of (merged_subs, matched_category_id_or_None). |
| 34 | Emits a [Resolve] Matched category log line only when peers were |
| 35 | actually added (not when every peer was already in the WebSearch set). |
| 36 | |
| 37 | Classification failures degrade to "no match" — the unwidened list |
| 38 | is returned and a warning is logged. |
| 39 | """ |
| 40 | try: |
| 41 | category = categories.detect_category(topic) |
| 42 | except Exception as exc: |
| 43 | _log(f"Category classification failed: {exc}") |
| 44 | return list(subreddits)[:MAX_SUBS], None |
| 45 | |
| 46 | if category is None: |
| 47 | return list(subreddits)[:MAX_SUBS], None |
| 48 | |
| 49 | peers = categories.peer_subs_for(category) |
| 50 | if not peers: |
| 51 | return list(subreddits)[:MAX_SUBS], category |
| 52 | |
| 53 | existing_lower = {s.lower() for s in subreddits} |
| 54 | merged = list(subreddits) |
| 55 | added: list[str] = [] |
| 56 | for peer in peers: |
| 57 | if len(merged) >= MAX_SUBS: |
| 58 | break |
| 59 | if peer.lower() in existing_lower: |
| 60 | continue |
| 61 | merged.append(peer) |
| 62 | existing_lower.add(peer.lower()) |
| 63 | added.append(peer) |
| 64 | |
| 65 | if added: |
| 66 | _log(f"Matched category={category}, adding peers: {', '.join(added)}") |
| 67 | |
| 68 | return merged, category |
| 69 | |
| 70 | |
| 71 | def _has_backend(config: dict) -> bool: |
| 72 | """Check if any web search backend is available.""" |
| 73 | return bool( |
| 74 | config.get("BRAVE_API_KEY") |
| 75 | or config.get("EXA_API_KEY") |
| 76 | or config.get("SERPER_API_KEY") |
| 77 | or config.get("PARALLEL_API_KEY") |
| 78 | or config.get("OPENROUTER_API_KEY") |
| 79 | or config.get("PERPLEXITY_API_KEY") |
| 80 | ) |
| 81 | |
| 82 | |
| 83 | def _extract_subreddits(items: list[dict]) -> list[str]: |
| 84 | """Parse subreddit names from search result titles and snippets.""" |
| 85 | pattern = re.compile(r"r/([A-Za-z0-9_]{2,21})") |
| 86 | seen: set[str] = set() |
| 87 | results: list[str] = [] |
| 88 | for item in items: |
| 89 | text = f"{item.get('title', '')} {item.get('snippet', '')} {item.get('url', '')}" |
| 90 | for match in pattern.findall(text): |
| 91 | lower = match.lower() |
| 92 | if lower not in seen: |
| 93 | seen.add(lower) |
| 94 | results.append(match) |
| 95 | return results |
| 96 | |
| 97 | |
| 98 | def _extract_x_handle(items: list[dict]) -> str: |
| 99 | """Extract the most likely X/Twitter handle from search results.""" |
| 100 | pattern = re.compile(r"@([A-Za-z0-9_]{1,15})") |
| 101 | url_pattern = re.compile(r"(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})(?:/|$|\?)") |
| 102 | counts: dict[str, int] = {} |
| 103 | for item in items: |
| 104 | text = f"{item.get('title', '')} {item.get('snippet', '')}" |
| 105 | url = item.get("url", "") |
| 106 | for match in pattern.findall(text): |
| 107 | lower = match.lower() |
| 108 | counts[lower] = counts.get(lower, 0) + 1 |
| 109 | for match in url_pattern.findall(url): |
| 110 | lower = match.lower() |
| 111 | # URL matches are stronger signals |
| 112 | counts[lower] = counts.get(lower, 0) + 3 |
| 113 | # Filter out generic handles |
| 114 | skip = {"twitter", "x", "search", "hashtag", "intent", "share", "i", "home", "explore", "settings"} |
| 115 | counts = {k: v for k, v in counts.items() if k not in skip} |
| 116 | if not counts: |
| 117 | return "" |
| 118 | return max(counts, key=counts.get) |
| 119 | |
| 120 | |
| 121 | def _extract_github_user(items: list[dict]) -> str: |
| 122 | """Extract GitHub username from search results.""" |
| 123 | url_pattern = re.compile(r"github\.com/([A-Za-z0-9_-]{1,39})(?:/|$|\?)") |
| 124 | counts: dict[str, int] = {} |
| 125 | for item in items: |
| 126 | url = item.get("url", "") |
| 127 | text = f"{item.get('title', '')} {item.get('snippet', '')}" |
| 128 | for match in url_pattern.findall(url): |
| 129 | lower = match.lower() |
| 130 | counts[lower] = counts.get(lower, 0) + 3 |
| 131 | for match in url_pattern.findall(text): |
| 132 | lower = match.lower() |
| 133 | counts[lower] = counts.get(lower, 0) + 1 |
| 134 | # Filter out org/repo-like names and generic pages |
| 135 | skip = {"topics", "explore", "settings", "orgs", "search", "features", "about", "pricing", "enterprise"} |
| 136 | counts = {k: v for k, v in counts.items() if k not in skip} |
| 137 | if not counts: |
| 138 | return "" |
| 139 | return max(counts, key=counts.get) |
| 140 | |
| 141 | |
| 142 | # Hosts that can never be a brand's own site; their presence in results is |
| 143 | # platform noise, not an official-domain signal. |
| 144 | _PLATFORM_HOSTS = { |
| 145 | "reddit.com", "x.com", "twitter.com", "github.com", "youtube.com", |
| 146 | "facebook.com", "instagram.com", "tiktok.com", "linkedin.com", |
| 147 | "wikipedia.org", "medium.com", "trustpilot.com", "crunchbase.com", |
| 148 | "bloomberg.com", "apple.com", "play.google.com", "google.com", |
| 149 | "threads.com", "pinterest.com", "glassdoor.com", "indeed.com", |
| 150 | "news.ycombinator.com", "substack.com", "amazon.com", "ebay.com", |
| 151 | } |
| 152 | |
| 153 | |
| 154 | def _extract_official_domain(topic: str, items: list[dict]) -> str: |
| 155 | """Extract the brand's own domain from search-result URLs. |
| 156 | |
| 157 | Conservative on purpose: only a hostname whose registrable label |
| 158 | normalizes to the topic name qualifies ("ThriftBooks" -> thriftbooks.com). |
| 159 | This feeds Trustpilot targeting as a HINT (the engine retries via the |
| 160 | CLI's own search when the hint misses), so a miss here is cheap and a |
| 161 | wrong guess is recoverable. |
| 162 | """ |
| 163 | want = re.sub(r"[^a-z0-9]", "", topic.lower()) |
| 164 | if not want: |
| 165 | return "" |
| 166 | host_pattern = re.compile(r"https?://([A-Za-z0-9.-]+)") |
| 167 | for item in items: |
| 168 | for source in [item.get("url", ""), f"{item.get('title', '')} {item.get('snippet', '')}"]: |
| 169 | for host in host_pattern.findall(source): |
| 170 | host = host.lower().strip(".") |
| 171 | bare = host.removeprefix("www.") |
| 172 | if any(bare == p or bare.endswith("." + p) for p in _PLATFORM_HOSTS): |
| 173 | continue |
| 174 | labels = bare.split(".") |
| 175 | if len(labels) < 2: |
| 176 | continue |
| 177 | registrable = labels[-2] if labels[-2] not in ("co", "com") or len(labels) < 3 else labels[-3] |
| 178 | if re.sub(r"[^a-z0-9]", "", registrable) == want: |
| 179 | return bare |
| 180 | return "" |
| 181 | |
| 182 | |
| 183 | def _extract_github_repos(items: list[dict]) -> list[str]: |
| 184 | """Extract owner/repo strings from search results.""" |
| 185 | repo_pattern = re.compile(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)") |
| 186 | skip_owners = {"topics", "explore", "settings", "orgs", "search", "features", "about", "pricing", "enterprise"} |
| 187 | seen: set[str] = set() |
| 188 | repos: list[str] = [] |
| 189 | for item in items: |
| 190 | url = item.get("url", "") |
| 191 | text = f"{item.get('title', '')} {item.get('snippet', '')}" |
| 192 | for source in [url, text]: |
| 193 | for match in repo_pattern.findall(source): |
| 194 | owner = match.split("/")[0].lower() |
| 195 | if owner in skip_owners: |
| 196 | continue |
| 197 | lower = match.lower() |
| 198 | if lower not in seen: |
| 199 | seen.add(lower) |
| 200 | repos.append(match) |
| 201 | return repos[:5] # cap at 5 repos |
| 202 | |
| 203 | |
| 204 | _INTEGRATION_SUFFIX_KEYWORDS: dict[str, set[str]] = { |
| 205 | "-action": {"action", "actions", "workflow", "workflows"}, |
| 206 | "-sdk": {"sdk", "client", "library"}, |
| 207 | "-plugin": {"plugin", "plugins", "extension", "extensions"}, |
| 208 | "-plugins": {"plugin", "plugins", "extension", "extensions"}, |
| 209 | "-docs": {"docs", "documentation"}, |
| 210 | "-examples": {"example", "examples", "sample", "samples"}, |
| 211 | "-template": {"template", "templates", "starter", "boilerplate"}, |
| 212 | } |
| 213 | |
| 214 | |
| 215 | def _topic_tokens(topic: str) -> set[str]: |
| 216 | return set(re.findall(r"[a-z0-9]+", (topic or "").lower())) |
| 217 | |
| 218 | |
| 219 | def _topic_entity_slugs(topic: str) -> list[str]: |
| 220 | entities = re.split(r"\b(?:vs|versus)\b", (topic or "").lower()) |
| 221 | slugs: list[str] = [] |
| 222 | for entity in entities: |
| 223 | tokens = re.findall(r"[a-z0-9]+", entity) |
| 224 | if tokens: |
| 225 | slugs.append("-".join(tokens)) |
| 226 | return slugs |
| 227 | |
| 228 | |
| 229 | def _repo_slug(repo: str) -> str: |
| 230 | parts = repo.split("/", 1) |
| 231 | if len(parts) != 2: |
| 232 | return "" |
| 233 | return parts[1].lower() |
| 234 | |
| 235 | |
| 236 | def _canonicalize_integration_repo(topic: str, repo: str) -> str: |
| 237 | """Map integration repos back to canonical product repos when intent allows. |
| 238 | |
| 239 | Example: |
| 240 | anthropics/claude-code-action -> anthropics/claude-code |
| 241 | unless topic explicitly asks for "action"/"workflow". |
| 242 | """ |
| 243 | parts = repo.split("/", 1) |
| 244 | if len(parts) != 2: |
| 245 | return repo |
| 246 | owner, name = parts[0], parts[1] |
| 247 | lower_name = name.lower() |
| 248 | topic_words = _topic_tokens(topic) |
| 249 | for suffix, intent_words in _INTEGRATION_SUFFIX_KEYWORDS.items(): |
| 250 | if not lower_name.endswith(suffix): |
| 251 | continue |
| 252 | if topic_words.intersection(intent_words): |
| 253 | return repo |
| 254 | base = name[: -len(suffix)] |
| 255 | if base: |
| 256 | return f"{owner}/{base}" |
| 257 | return repo |
| 258 | |
| 259 | |
| 260 | def canonicalize_github_repos(topic: str, repos: list[str], *, cap: int | None = 5) -> list[str]: |
| 261 | """Normalize/priority-sort GitHub repos for the current topic. |
| 262 | |
| 263 | - Rewrites common integration suffixes to canonical product repos when |
| 264 | topic intent does not mention those integrations. |
| 265 | - Promotes exact topic slug matches (e.g., `claude-code`) over partials. |
| 266 | """ |
| 267 | canonicalized: list[str] = [] |
| 268 | seen: set[str] = set() |
| 269 | for repo in repos: |
| 270 | candidate = _canonicalize_integration_repo(topic, repo.strip()) |
| 271 | if "/" not in candidate: |
| 272 | continue |
| 273 | key = candidate.lower() |
| 274 | if key in seen: |
| 275 | continue |
| 276 | seen.add(key) |
| 277 | canonicalized.append(candidate) |
| 278 | |
| 279 | topic_slugs = set(_topic_entity_slugs(topic)) |
| 280 | if topic_slugs: |
| 281 | exact = [r for r in canonicalized if _repo_slug(r) in topic_slugs] |
| 282 | prefixed = [r for r in canonicalized if any(_repo_slug(r).startswith(f"{slug}-") for slug in topic_slugs) and r not in exact] |
| 283 | rest = [r for r in canonicalized if r not in exact and r not in prefixed] |
| 284 | canonicalized = exact + prefixed + rest |
| 285 | |
| 286 | if cap is not None: |
| 287 | return canonicalized[:cap] |
| 288 | return canonicalized |
| 289 | |
| 290 | |
| 291 | def _build_context_summary(items: list[dict]) -> str: |
| 292 | """Build a 1-2 sentence current events summary from news search results.""" |
| 293 | snippets: list[str] = [] |
| 294 | for item in items[:3]: |
| 295 | snippet = item.get("snippet", "").strip() |
| 296 | if snippet: |
| 297 | snippets.append(snippet) |
| 298 | if not snippets: |
| 299 | return "" |
| 300 | # Take the first two meaningful snippets and truncate to keep it concise |
| 301 | combined = " ".join(snippets[:2]) |
| 302 | if len(combined) > 300: |
| 303 | combined = combined[:297] + "..." |
| 304 | return combined |
| 305 | |
| 306 | |
| 307 | def auto_resolve(topic: str, config: dict) -> dict: |
| 308 | """Discover subreddits, X handles, and current events context for a topic. |
| 309 | |
| 310 | Args: |
| 311 | topic: The research topic. |
| 312 | config: Dict with API keys (BRAVE_API_KEY, EXA_API_KEY, SERPER_API_KEY). |
| 313 | |
| 314 | Returns: |
| 315 | Dict with keys: subreddits, x_handle, github_user, github_repos, |
| 316 | context, category, searches_run. Returns empty result if no web |
| 317 | search backend is available. |
| 318 | """ |
| 319 | empty = { |
| 320 | "subreddits": [], |
| 321 | "x_handle": "", |
| 322 | "github_user": "", |
| 323 | "github_repos": [], |
| 324 | "trustpilot_domain": "", |
| 325 | "context": "", |
| 326 | "category": None, |
| 327 | "searches_run": 0, |
| 328 | } |
| 329 | |
| 330 | if not _has_backend(config): |
| 331 | _log("No web search backend available, skipping resolve") |
| 332 | return empty |
| 333 | |
| 334 | from_date, to_date = dates.get_date_range(30) |
| 335 | date_range = (from_date, to_date) |
| 336 | now = datetime.now(timezone.utc) |
| 337 | current_month = now.strftime("%B") |
| 338 | current_year = now.strftime("%Y") |
| 339 | |
| 340 | queries = { |
| 341 | "subreddit": f"{topic} subreddit reddit", |
| 342 | "news": f"{topic} news {current_month} {current_year}", |
| 343 | "x_handle": f"{topic} X twitter handle", |
| 344 | "github": f"{topic} github profile site:github.com", |
| 345 | } |
| 346 | |
| 347 | results: dict[str, list[dict]] = {} |
| 348 | searches_run = 0 |
| 349 | |
| 350 | def _search(label: str, query: str) -> tuple[str, list[dict]]: |
| 351 | items, _artifact = grounding.web_search(query, date_range, config) |
| 352 | return label, items |
| 353 | |
| 354 | with ThreadPoolExecutor(max_workers=3) as executor: |
| 355 | futures = { |
| 356 | executor.submit(_search, label, q): label |
| 357 | for label, q in queries.items() |
| 358 | } |
| 359 | for future in as_completed(futures): |
| 360 | label = futures[future] |
| 361 | try: |
| 362 | _label, items = future.result() |
| 363 | results[label] = items |
| 364 | searches_run += 1 |
| 365 | except Exception as exc: |
| 366 | _log(f"Search failed for {label}: {exc}") |
| 367 | results[label] = [] |
| 368 | |
| 369 | subreddits = _extract_subreddits(results.get("subreddit", [])) |
| 370 | x_handle = _extract_x_handle(results.get("x_handle", [])) |
| 371 | github_user = _extract_github_user(results.get("github", [])) |
| 372 | github_repos = canonicalize_github_repos(topic, _extract_github_repos(results.get("github", []))) |
| 373 | context = _build_context_summary(results.get("news", [])) |
| 374 | # Official-site domain doubles as the Trustpilot targeting hint (review |
| 375 | # pages are keyed by domain). Scan news first (official sites surface in |
| 376 | # coverage), then the handle query's profile-adjacent results. |
| 377 | trustpilot_domain = _extract_official_domain( |
| 378 | topic, (results.get("news") or []) + (results.get("x_handle") or []) |
| 379 | ) |
| 380 | |
| 381 | subreddits, category = _merge_category_peers(topic, subreddits) |
| 382 | |
| 383 | _log(f"Resolved {len(subreddits)} subreddits, x_handle={x_handle!r}, github_user={github_user!r}, github_repos={github_repos!r}, trustpilot_domain={trustpilot_domain!r}, context_len={len(context)}, category={category!r}") |
| 384 | |
| 385 | return { |
| 386 | "subreddits": subreddits, |
| 387 | "x_handle": x_handle, |
| 388 | "github_user": github_user, |
| 389 | "github_repos": github_repos, |
| 390 | "trustpilot_domain": trustpilot_domain, |
| 391 | "context": context, |
| 392 | "category": category, |
| 393 | "searches_run": searches_run, |
| 394 | } |
| 395 |