| 1 | """Discover peer entities ("competitors") for a topic via web search. |
| 2 | |
| 3 | Mirrors the `resolve.auto_resolve()` pattern: fan out 2-3 web searches via |
| 4 | `grounding.web_search()`, then extract capitalized entity candidates from |
| 5 | titles and snippets with deterministic text mining. No LLM call — the |
| 6 | hosting reasoning model can always override discovery via |
| 7 | `--competitors-list`. |
| 8 | |
| 9 | Returned list is ordered by score (frequency across queries) and capped to |
| 10 | the caller's requested count. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import re |
| 16 | from collections import Counter |
| 17 | from concurrent.futures import ThreadPoolExecutor, as_completed |
| 18 | |
| 19 | from . import dates, grounding, log |
| 20 | from .resolve import _has_backend |
| 21 | |
| 22 | # Peer cap vs total vs-entity cap (main + peers). |
| 23 | COMPETITORS_MIN = 1 |
| 24 | COMPETITORS_MAX = 6 |
| 25 | COMPETITORS_DEFAULT = 2 |
| 26 | COMPARISON_ENTITY_MAX = COMPETITORS_MAX + 1 |
| 27 | # Discovery SERP fan-out is small (3 queries today) but still needs a ceiling |
| 28 | # so a future query expansion cannot open one worker per query unbounded. |
| 29 | MAX_DISCOVERY_WORKERS = 3 |
| 30 | |
| 31 | # A "brand-shaped" token starts with uppercase OR is camelCase with an |
| 32 | # uppercase letter later. Catches "Anthropic", "OpenAI", "xAI", "iPhone", |
| 33 | # "eBay", "Hugging", "Face". |
| 34 | _BRAND_TOKEN = ( |
| 35 | r"(?:[A-Z][A-Za-z0-9&.\-]*" |
| 36 | r"|[a-z][A-Za-z0-9&.\-]*[A-Z][A-Za-z0-9&.\-]*)" |
| 37 | ) |
| 38 | |
| 39 | # A capitalized phrase of 1-4 brand tokens separated by whitespace. |
| 40 | _CAPITALIZED_PHRASE = re.compile( |
| 41 | rf"\b{_BRAND_TOKEN}(?:\s+{_BRAND_TOKEN}){{0,3}}\b" |
| 42 | ) |
| 43 | |
| 44 | # Title-case fillers common in listicle SERPs. Kept flat — extraction |
| 45 | # rejects a candidate whose entire tokens are stopwords, not candidates |
| 46 | # that merely contain one. |
| 47 | _STOPWORD_TOKENS: frozenset[str] = frozenset( |
| 48 | token.lower() |
| 49 | for token in ( |
| 50 | # Listicle fillers |
| 51 | "Top", "Best", "Worst", "Popular", "Leading", "Similar", |
| 52 | "Alternatives", "Alternative", "Competitor", "Competitors", |
| 53 | "vs", "Vs", "Versus", "Review", "Reviews", "Comparison", |
| 54 | "Guide", "List", "Lists", "Full", "Complete", "Free", "Paid", |
| 55 | "Tools", "Tool", "Options", "Rivals", "Rival", "Similar", |
| 56 | "Pick", "Picks", "Ranking", "Ranked", "Recommended", |
| 57 | # Grammar / time |
| 58 | "The", "A", "An", "Of", "In", "For", "To", "With", "On", "At", |
| 59 | "By", "From", "Is", "Are", "And", "Or", "But", "Than", "As", |
| 60 | "This", "That", "These", "Those", "Our", "Your", "Their", |
| 61 | "January", "February", "March", "April", "May", "June", "July", |
| 62 | "August", "September", "October", "November", "December", |
| 63 | # Years likely to appear as standalone tokens |
| 64 | *(str(year) for year in range(2018, 2031)), |
| 65 | # Miscellaneous SERP noise |
| 66 | "AI", "Apps", "App", "Software", "Platform", "Service", "Startups", |
| 67 | "Companies", "Company", "Products", "Product", "Brands", "Brand", |
| 68 | ) |
| 69 | ) |
| 70 | |
| 71 | |
| 72 | def _log(msg: str) -> None: |
| 73 | log.source_log("Competitors", msg, tty_only=False) |
| 74 | |
| 75 | |
| 76 | def _topic_tokens(topic: str) -> set[str]: |
| 77 | """Return lowercase alphanumeric tokens of the topic for filtering.""" |
| 78 | return {tok for tok in re.findall(r"[A-Za-z0-9]+", topic.lower()) if tok} |
| 79 | |
| 80 | |
| 81 | def _candidate_ok(candidate: str, topic_tokens: set[str]) -> bool: |
| 82 | """Filter a candidate phrase against stopwords and topic overlap.""" |
| 83 | tokens = [t for t in re.findall(r"[A-Za-z0-9&.\-]+", candidate) if t] |
| 84 | if not tokens: |
| 85 | return False |
| 86 | # Reject candidates made entirely of stopwords (e.g., "Top Alternatives"). |
| 87 | if all(tok.lower() in _STOPWORD_TOKENS for tok in tokens): |
| 88 | return False |
| 89 | # Reject candidates that overlap with the topic (e.g., topic="OpenAI" |
| 90 | # should not return "OpenAI Alternatives" or "OpenAI"). |
| 91 | lower_tokens = {tok.lower() for tok in tokens} |
| 92 | if lower_tokens & topic_tokens: |
| 93 | return False |
| 94 | # Reject too-short one-letter tokens like "I" or single digits. |
| 95 | if len(tokens) == 1 and len(tokens[0]) < 2: |
| 96 | return False |
| 97 | return True |
| 98 | |
| 99 | |
| 100 | def _normalize_candidate(candidate: str) -> str: |
| 101 | """Collapse whitespace and strip trailing punctuation.""" |
| 102 | return re.sub(r"\s+", " ", candidate).strip(".,;:!?'\"()[] ") |
| 103 | |
| 104 | |
| 105 | def _extract_peer_entities( |
| 106 | items: list[dict], topic: str, limit: int, |
| 107 | ) -> list[str]: |
| 108 | """Score capitalized candidates across SERP items and return top `limit`. |
| 109 | |
| 110 | Scoring is bag-of-phrases frequency across all items in the input. Ties |
| 111 | are broken by first-seen order so the output is deterministic. |
| 112 | """ |
| 113 | topic_tokens = _topic_tokens(topic) |
| 114 | counts: Counter[str] = Counter() |
| 115 | first_seen: dict[str, int] = {} |
| 116 | order = 0 |
| 117 | # Group candidates into a frequency map keyed by lowercased normalized |
| 118 | # form so "xAI" and "xAI" count together regardless of case. |
| 119 | canonical: dict[str, str] = {} |
| 120 | for item in items: |
| 121 | text = f"{item.get('title', '')} {item.get('snippet', '')}" |
| 122 | for raw in _CAPITALIZED_PHRASE.findall(text): |
| 123 | candidate = _normalize_candidate(raw) |
| 124 | if not _candidate_ok(candidate, topic_tokens): |
| 125 | continue |
| 126 | key = candidate.lower() |
| 127 | if key not in canonical: |
| 128 | canonical[key] = candidate |
| 129 | first_seen[key] = order |
| 130 | order += 1 |
| 131 | counts[key] += 1 |
| 132 | |
| 133 | ranked_keys = sorted( |
| 134 | counts.keys(), |
| 135 | key=lambda k: (-counts[k], first_seen[k]), |
| 136 | ) |
| 137 | return [canonical[k] for k in ranked_keys[:limit]] |
| 138 | |
| 139 | |
| 140 | def _queries_for(topic: str) -> dict[str, str]: |
| 141 | return { |
| 142 | "competitors": f"{topic} competitors", |
| 143 | "alternatives": f"{topic} alternatives", |
| 144 | "vs": f"{topic} vs", |
| 145 | } |
| 146 | |
| 147 | |
| 148 | def discover_competitors( |
| 149 | topic: str, |
| 150 | count: int, |
| 151 | config: dict, |
| 152 | *, |
| 153 | lookback_days: int = 30, |
| 154 | ) -> list[str]: |
| 155 | """Discover `count` peer entities for `topic` via web search. |
| 156 | |
| 157 | Args: |
| 158 | topic: The primary research topic. |
| 159 | count: Desired number of competitor entities (1..N). |
| 160 | config: Runtime config dict — expects the same shape as the engine |
| 161 | config (BRAVE_API_KEY / EXA_API_KEY / SERPER_API_KEY / etc.). |
| 162 | lookback_days: Date range for freshness. Defaults to 30. |
| 163 | |
| 164 | Returns: |
| 165 | A list of up to `count` entity names, deduped and ordered by score. |
| 166 | Empty list when no web backend is configured or every search fails |
| 167 | or returns zero usable candidates. |
| 168 | """ |
| 169 | if count < 1: |
| 170 | return [] |
| 171 | if not _has_backend(config): |
| 172 | _log("No web search backend available, skipping competitor discovery") |
| 173 | return [] |
| 174 | |
| 175 | date_range = dates.get_date_range(lookback_days) |
| 176 | queries = _queries_for(topic) |
| 177 | collected: list[dict] = [] |
| 178 | searches_run = 0 |
| 179 | |
| 180 | def _search(label: str, query: str) -> tuple[str, list[dict]]: |
| 181 | items, _artifact = grounding.web_search(query, date_range, config) |
| 182 | return label, items |
| 183 | |
| 184 | with ThreadPoolExecutor(max_workers=min(len(queries), MAX_DISCOVERY_WORKERS)) as executor: |
| 185 | futures = { |
| 186 | executor.submit(_search, label, q): label |
| 187 | for label, q in queries.items() |
| 188 | } |
| 189 | for future in as_completed(futures): |
| 190 | label = futures[future] |
| 191 | try: |
| 192 | _label, items = future.result() |
| 193 | collected.extend(items) |
| 194 | searches_run += 1 |
| 195 | except Exception as exc: |
| 196 | _log(f"Search failed for {label}: {exc}") |
| 197 | |
| 198 | if not collected: |
| 199 | _log(f"No SERP results for {topic!r} across {searches_run}/{len(queries)} queries") |
| 200 | return [] |
| 201 | |
| 202 | entities = _extract_peer_entities(collected, topic, limit=count) |
| 203 | _log( |
| 204 | f"Discovered {len(entities)} competitor(s) for {topic!r} " |
| 205 | f"from {searches_run}/{len(queries)} queries: {entities}" |
| 206 | ) |
| 207 | return entities |
| 208 |