| 1 | """Keyless Reddit listing scrape via shreddit /svc partials — with real scores. |
| 2 | |
| 3 | The subreddit listing partial |
| 4 | ``/svc/shreddit/community-more-posts/{sort}/?name={sub}[&t={range}]`` serves |
| 5 | HTTP 200 with no API key and **server-renders each post's upvote score**, which |
| 6 | neither RSS nor the comments endpoint provides. Each post is a |
| 7 | ``<shreddit-post>`` element whose start-tag attributes carry ``score``, |
| 8 | ``comment-count``, ``post-title``, ``permalink``, ``author``, ``subreddit-name`` |
| 9 | and ``created-timestamp``. |
| 10 | |
| 11 | This is the keyless source of post-level upvotes. It works for normal users on |
| 12 | ordinary connections (verified), so reddit_keyless uses it both as a scored |
| 13 | discovery source and to backfill scores onto RSS-discovered posts. |
| 14 | """ |
| 15 | |
| 16 | import html as _html |
| 17 | import re |
| 18 | import sys |
| 19 | from datetime import datetime, timezone |
| 20 | from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError |
| 21 | from typing import Any, Dict, List, Optional |
| 22 | |
| 23 | from . import http |
| 24 | from .relevance import token_overlap_relevance |
| 25 | |
| 26 | # Listing sorts pulled per subreddit, by depth. |
| 27 | LISTING_SORTS = { |
| 28 | "quick": ["top"], |
| 29 | "default": ["top", "hot"], |
| 30 | "deep": ["top", "hot", "new"], |
| 31 | } |
| 32 | DEPTH_LIMITS = {"quick": 10, "default": 25, "deep": 50} |
| 33 | TIMEFRAME = "month" |
| 34 | MAX_WORKERS = 4 |
| 35 | LISTING_TIMEOUT = 15 |
| 36 | |
| 37 | _POST_CARD = re.compile(r"<shreddit-post(?=[\s>])[^>]*>") |
| 38 | |
| 39 | |
| 40 | def _log(msg: str) -> None: |
| 41 | sys.stderr.write(f"[RedditListing] {msg}\n") |
| 42 | sys.stderr.flush() |
| 43 | |
| 44 | |
| 45 | def _attr(tag: str, name: str) -> Optional[str]: |
| 46 | m = re.search(rf'\b{name}="([^"]*)"', tag) |
| 47 | return _html.unescape(m.group(1)) if m else None |
| 48 | |
| 49 | |
| 50 | def _to_date(value: Optional[str]) -> Optional[str]: |
| 51 | if not value: |
| 52 | return None |
| 53 | try: |
| 54 | return datetime.fromisoformat(value.strip()).date().isoformat() |
| 55 | except (ValueError, TypeError): |
| 56 | return None |
| 57 | |
| 58 | |
| 59 | def _to_epoch(value: Optional[str]) -> Optional[float]: |
| 60 | if not value: |
| 61 | return None |
| 62 | try: |
| 63 | dt = datetime.fromisoformat(value.strip()) |
| 64 | if dt.tzinfo is None: |
| 65 | dt = dt.replace(tzinfo=timezone.utc) |
| 66 | return dt.timestamp() |
| 67 | except (ValueError, TypeError): |
| 68 | return None |
| 69 | |
| 70 | |
| 71 | def _post_id(permalink: str) -> str: |
| 72 | m = re.search(r"/comments/([A-Za-z0-9]+)", permalink or "") |
| 73 | return m.group(1) if m else "" |
| 74 | |
| 75 | |
| 76 | def parse_cards(html_text: str, query: str = "") -> List[Dict[str, Any]]: |
| 77 | """Parse <shreddit-post> cards into normalized post dicts with real scores.""" |
| 78 | posts: List[Dict[str, Any]] = [] |
| 79 | for m in _POST_CARD.finditer(html_text or ""): |
| 80 | tag = m.group(0) |
| 81 | permalink = _attr(tag, "permalink") or "" |
| 82 | if "/comments/" not in permalink: |
| 83 | continue |
| 84 | try: |
| 85 | score = int(_attr(tag, "score") or 0) |
| 86 | except ValueError: |
| 87 | score = 0 |
| 88 | try: |
| 89 | num_comments = int(_attr(tag, "comment-count") or 0) |
| 90 | except ValueError: |
| 91 | num_comments = 0 |
| 92 | title = _attr(tag, "post-title") or "" |
| 93 | author = _attr(tag, "author") or "[deleted]" |
| 94 | subreddit = _attr(tag, "subreddit-name") or "" |
| 95 | created = _attr(tag, "created-timestamp") |
| 96 | url = f"https://www.reddit.com{permalink}" |
| 97 | |
| 98 | posts.append({ |
| 99 | "id": "", |
| 100 | "title": title, |
| 101 | "url": url, |
| 102 | "score": score, |
| 103 | "num_comments": num_comments, |
| 104 | "subreddit": subreddit, |
| 105 | "created_utc": _to_epoch(created), |
| 106 | "author": author if author not in ("[deleted]", "[removed]") else "[deleted]", |
| 107 | "selftext": "", |
| 108 | "date": _to_date(created), |
| 109 | "engagement": { |
| 110 | "score": score, |
| 111 | "num_comments": num_comments, |
| 112 | "upvote_ratio": None, |
| 113 | }, |
| 114 | "relevance": round(token_overlap_relevance(query, title), 3) if query else 0.0, |
| 115 | "why_relevant": "Reddit listing", |
| 116 | "metadata": {"post_id": _post_id(permalink)}, |
| 117 | }) |
| 118 | return posts |
| 119 | |
| 120 | |
| 121 | def _listing_url(subreddit: str, sort: str, timeframe: str = TIMEFRAME) -> str: |
| 122 | sub = subreddit.removeprefix("r/").strip() |
| 123 | if sub.lower() == "all": |
| 124 | url = f"https://www.reddit.com/r/all/{sort}/" |
| 125 | if sort == "top": |
| 126 | url += f"?t={timeframe}" |
| 127 | return url |
| 128 | url = f"https://www.reddit.com/svc/shreddit/community-more-posts/{sort}/?name={sub}" |
| 129 | if sort == "top": |
| 130 | url += f"&t={timeframe}" |
| 131 | return url |
| 132 | |
| 133 | |
| 134 | def _fetch_one( |
| 135 | subreddit: str, |
| 136 | sort: str, |
| 137 | query: str, |
| 138 | timeframe: str = TIMEFRAME, |
| 139 | ) -> List[Dict[str, Any]]: |
| 140 | items, _ = _fetch_one_with_status(subreddit, sort, query, timeframe) |
| 141 | return items |
| 142 | |
| 143 | |
| 144 | def _fetch_one_with_status( |
| 145 | subreddit: str, |
| 146 | sort: str, |
| 147 | query: str, |
| 148 | timeframe: str = TIMEFRAME, |
| 149 | ) -> tuple[List[Dict[str, Any]], Optional[str]]: |
| 150 | try: |
| 151 | # tee_failures, not capture_failures: the latter would replace the |
| 152 | # pipeline's sink and hide this failure from it. get_text launders a |
| 153 | # terminal HTTP failure into None, so the tee is how this lane recovers |
| 154 | # the status code it needs to report (issue #899). |
| 155 | with http.tee_failures() as swallowed: |
| 156 | text = http.reddit_keyless_get_text(_listing_url(subreddit, sort, timeframe), timeout=LISTING_TIMEOUT, |
| 157 | accept="text/html") |
| 158 | if text is None: |
| 159 | # An empty body ("") is a real empty listing; None never is. |
| 160 | return [], (str(swallowed[-1]) if swallowed else "no response") |
| 161 | return parse_cards(text, query), None |
| 162 | except Exception as e: |
| 163 | _log(f"listing fetch failed r/{subreddit} {sort}: {e}") |
| 164 | return [], str(e) |
| 165 | |
| 166 | |
| 167 | def fetch_listings( |
| 168 | subreddits: List[str], |
| 169 | depth: str = "default", |
| 170 | query: str = "", |
| 171 | sorts: Optional[List[str]] = None, |
| 172 | timeframe: str = TIMEFRAME, |
| 173 | ) -> List[Dict[str, Any]]: |
| 174 | """Fetch scored post cards across subreddits × sorts. |
| 175 | |
| 176 | Returns deduped normalized posts (with real scores), unranked/unsliced — |
| 177 | the caller merges these with other sources, ranks, and slices. |
| 178 | |
| 179 | ``sorts`` overrides the depth-derived sort set. Dedicated-subreddit lanes |
| 180 | pass ``["top", "hot", "new"]`` so fresh threads (which the top-of-month |
| 181 | listing misses) are caught with their scores regardless of depth. |
| 182 | """ |
| 183 | if not subreddits: |
| 184 | return [] |
| 185 | sorts = sorts or LISTING_SORTS.get(depth, LISTING_SORTS["default"]) |
| 186 | jobs = [(sub, sort) for sub in subreddits for sort in sorts] |
| 187 | all_posts: List[Dict[str, Any]] = [] |
| 188 | with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor: |
| 189 | # submit_with_context, not executor.submit — see the note in |
| 190 | # fetch_discovery_listings below (issue #899). |
| 191 | futures = {http.submit_with_context(executor, _fetch_one, sub, sort, query, timeframe): (sub, sort) |
| 192 | for sub, sort in jobs} |
| 193 | for future in futures: |
| 194 | try: |
| 195 | all_posts.extend(future.result(timeout=LISTING_TIMEOUT + 5)) |
| 196 | except (Exception, FuturesTimeoutError) as e: |
| 197 | _log(f"listing future failed: {e}") |
| 198 | |
| 199 | seen: set = set() |
| 200 | unique: List[Dict[str, Any]] = [] |
| 201 | for p in all_posts: |
| 202 | if p["url"] not in seen: |
| 203 | seen.add(p["url"]) |
| 204 | unique.append(p) |
| 205 | return unique |
| 206 | |
| 207 | |
| 208 | def fetch_discovery_listings( |
| 209 | subreddits: List[str], |
| 210 | *, |
| 211 | query: str, |
| 212 | depth: str = "default", |
| 213 | ) -> Dict[str, Any]: |
| 214 | """Fetch rising/top-week listings while preserving per-feed failures.""" |
| 215 | if not subreddits: |
| 216 | return {"items": [], "errors": []} |
| 217 | jobs = [(subreddit, sort) for subreddit in subreddits for sort in ("rising", "top")] |
| 218 | items: List[Dict[str, Any]] = [] |
| 219 | errors: List[str] = [] |
| 220 | with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(jobs)) or 1) as executor: |
| 221 | # submit_with_context, not executor.submit: a plain submit starts the |
| 222 | # worker with an empty context, dropping the pipeline's |
| 223 | # capture_failures() sink so a listing's 429/403 is silently discarded |
| 224 | # and the source reports a clean no-results (issue #899). |
| 225 | futures = { |
| 226 | http.submit_with_context( |
| 227 | executor, _fetch_one_with_status, subreddit, sort, query, "week" |
| 228 | ): (subreddit, sort) |
| 229 | for subreddit, sort in jobs |
| 230 | } |
| 231 | for future, (subreddit, sort) in futures.items(): |
| 232 | try: |
| 233 | fetched, error = future.result(timeout=LISTING_TIMEOUT + 5) |
| 234 | except (Exception, FuturesTimeoutError) as exc: |
| 235 | errors.append(f"r/{subreddit} {sort}: {exc}") |
| 236 | continue |
| 237 | items.extend(fetched) |
| 238 | if error: |
| 239 | errors.append(f"r/{subreddit} {sort}: {error}") |
| 240 | |
| 241 | seen: set[str] = set() |
| 242 | unique = [] |
| 243 | for item in items: |
| 244 | if item["url"] in seen: |
| 245 | continue |
| 246 | seen.add(item["url"]) |
| 247 | unique.append(item) |
| 248 | return {"items": unique, "errors": errors} |
| 249 | |
| 250 | |
| 251 | def score_index(subreddits: List[str], depth: str = "default") -> Dict[str, Dict[str, int]]: |
| 252 | """Build a {post_id: {score, num_comments}} map from subreddit listings. |
| 253 | |
| 254 | Used to backfill real scores onto posts discovered via RSS, which carries |
| 255 | no engagement numbers. |
| 256 | """ |
| 257 | index: Dict[str, Dict[str, int]] = {} |
| 258 | for p in fetch_listings(subreddits, depth=depth): |
| 259 | pid = p.get("metadata", {}).get("post_id") or _post_id(p["url"]) |
| 260 | if pid: |
| 261 | index[pid] = {"score": p["score"], "num_comments": p["num_comments"]} |
| 262 | return index |
| 263 |