| 1 | """Keyless Reddit comment enrichment via shreddit /svc endpoints. |
| 2 | |
| 3 | Reddit's ``{thread}.json`` endpoint now returns HTTP 403. The shreddit partial |
| 4 | endpoint ``/svc/shreddit/comments/r/{sub}/t3_{id}`` still serves HTTP 200 HTML |
| 5 | with no API key, embedding each comment as a ``<shreddit-comment>`` custom |
| 6 | element whose start-tag attributes carry ``score`` / ``author`` / ``created`` / |
| 7 | ``permalink``, and whose body lives in a ``<div id="{thingId}-post-rtjson-content">`` |
| 8 | block. This module parses that markup into top comments, matching the |
| 9 | ``top_comments`` / ``comment_insights`` shape produced by ``reddit_enrich`` so |
| 10 | the renderer is unaffected. |
| 11 | |
| 12 | Limitation: the comments endpoint carries the real comment count |
| 13 | (``total-comments``) but not the post's upvote score, so post-level ``score`` |
| 14 | cannot be recovered keylessly here (ScrapeCreators backup still provides it). |
| 15 | """ |
| 16 | |
| 17 | import html as _html |
| 18 | import re |
| 19 | import sys |
| 20 | from datetime import datetime |
| 21 | from typing import Any, Dict, List, Optional |
| 22 | |
| 23 | from . import http |
| 24 | from . import reddit_enrich |
| 25 | |
| 26 | # Up to N posts enriched per run, by depth (mirrors reddit_public.ENRICH_LIMITS). |
| 27 | ENRICH_LIMITS = { |
| 28 | "quick": 3, |
| 29 | "default": 5, |
| 30 | "deep": 8, |
| 31 | } |
| 32 | |
| 33 | # Max comments returned per post (independent of how many posts get enriched). |
| 34 | MAX_COMMENTS = 10 |
| 35 | |
| 36 | SVC_TIMEOUT = 12 |
| 37 | |
| 38 | # Match the exact <shreddit-comment> element start tag, not <shreddit-comment-tree> |
| 39 | # or <shreddit-comment-tree-stats> (lookahead requires whitespace or '>'). |
| 40 | _COMMENT_START = re.compile(r"<shreddit-comment(?=[\s>])[^>]*>") |
| 41 | _TOTAL_COMMENTS = re.compile(r'total-comments="(\d+)"') |
| 42 | _PARA = re.compile(r"<p[^>]*>(.*?)</p>", re.S) |
| 43 | _TAG = re.compile(r"<[^>]+>") |
| 44 | _WS = re.compile(r"\s+") |
| 45 | _NEXT_RTJSON = re.compile(r'id="t1_[A-Za-z0-9]+-(?:comment|post)-rtjson-content"') |
| 46 | |
| 47 | |
| 48 | def _log(msg: str) -> None: |
| 49 | sys.stderr.write(f"[RedditShreddit] {msg}\n") |
| 50 | sys.stderr.flush() |
| 51 | |
| 52 | |
| 53 | def extract_post_ref(url: str) -> Optional[tuple]: |
| 54 | """Return (subreddit, post_id) from a Reddit thread URL, or None.""" |
| 55 | m = re.search(r"/r/([^/]+)/comments/([A-Za-z0-9]+)", url or "") |
| 56 | if not m: |
| 57 | return None |
| 58 | return m.group(1), m.group(2) |
| 59 | |
| 60 | |
| 61 | def _svc_url(subreddit: str, post_id: str) -> str: |
| 62 | # sort=top guarantees Reddit front-loads the highest-scored comments on the |
| 63 | # first page, so the true top comments are captured even on huge threads |
| 64 | # (we still re-sort by score locally as a backstop). |
| 65 | return ( |
| 66 | f"https://www.reddit.com/svc/shreddit/comments/r/{subreddit}/t3_{post_id}" |
| 67 | f"?sort=top" |
| 68 | ) |
| 69 | |
| 70 | |
| 71 | def _attr(tag: str, name: str) -> str: |
| 72 | m = re.search(rf'\b{name}="([^"]*)"', tag) |
| 73 | return _html.unescape(m.group(1)) if m else "" |
| 74 | |
| 75 | |
| 76 | def _iso_to_date(value: str) -> Optional[str]: |
| 77 | if not value: |
| 78 | return None |
| 79 | try: |
| 80 | return datetime.fromisoformat(value.strip()).date().isoformat() |
| 81 | except (ValueError, TypeError): |
| 82 | return None |
| 83 | |
| 84 | |
| 85 | def _body_for(html_text: str, thing_id: str) -> str: |
| 86 | """Extract a comment's text body, anchored on its unique thingId. |
| 87 | |
| 88 | The body div id embeds the comment's thingId, so this assigns body→comment |
| 89 | correctly even for nested replies. The slice is bounded by the next |
| 90 | comment's rtjson anchor to avoid swallowing child-comment text. |
| 91 | """ |
| 92 | if not thing_id: |
| 93 | return "" |
| 94 | anchor = f'id="{thing_id}-post-rtjson-content"' |
| 95 | idx = html_text.find(anchor) |
| 96 | if idx == -1: |
| 97 | return "" |
| 98 | window = html_text[idx + len(anchor): idx + len(anchor) + 8000] |
| 99 | nxt = _NEXT_RTJSON.search(window) |
| 100 | if nxt: |
| 101 | window = window[: nxt.start()] |
| 102 | paras = _PARA.findall(window) |
| 103 | if not paras: |
| 104 | return "" |
| 105 | text = " ".join(_TAG.sub("", p) for p in paras) |
| 106 | return _WS.sub(" ", _html.unescape(text)).strip() |
| 107 | |
| 108 | |
| 109 | def parse_comments(html_text: str, limit: int = MAX_COMMENTS) -> List[Dict[str, Any]]: |
| 110 | """Parse <shreddit-comment> elements into scored comment dicts (sorted desc).""" |
| 111 | comments: List[Dict[str, Any]] = [] |
| 112 | for m in _COMMENT_START.finditer(html_text or ""): |
| 113 | tag = m.group(0) |
| 114 | author = _attr(tag, "author") or "[deleted]" |
| 115 | if author in ("[deleted]", "[removed]"): |
| 116 | continue |
| 117 | thing_id = _attr(tag, "thingId") |
| 118 | body = _body_for(html_text, thing_id) |
| 119 | if not body or body in ("[deleted]", "[removed]"): |
| 120 | continue |
| 121 | try: |
| 122 | score = int(_attr(tag, "score") or 0) |
| 123 | except ValueError: |
| 124 | score = 0 |
| 125 | permalink = _attr(tag, "permalink") |
| 126 | comments.append({ |
| 127 | "score": score, |
| 128 | "author": author, |
| 129 | "body": body[:300], |
| 130 | "excerpt": body[:200], |
| 131 | "permalink": permalink, |
| 132 | "date": _iso_to_date(_attr(tag, "created")), |
| 133 | "url": f"https://reddit.com{permalink}" if permalink else "", |
| 134 | }) |
| 135 | |
| 136 | comments.sort(key=lambda c: c.get("score", 0), reverse=True) |
| 137 | return comments[:limit] |
| 138 | |
| 139 | |
| 140 | def _total_comments(html_text: str) -> Optional[int]: |
| 141 | m = _TOTAL_COMMENTS.search(html_text or "") |
| 142 | return int(m.group(1)) if m else None |
| 143 | |
| 144 | |
| 145 | def fetch_comments( |
| 146 | post_url: str, |
| 147 | timeout: int = SVC_TIMEOUT, |
| 148 | ) -> Dict[str, Any]: |
| 149 | """Fetch and parse top comments for a Reddit post via the shreddit endpoint. |
| 150 | |
| 151 | Args: |
| 152 | post_url: Reddit thread URL (…/r/{sub}/comments/{id}/…) |
| 153 | timeout: HTTP timeout in seconds |
| 154 | |
| 155 | Returns: |
| 156 | Dict with 'top_comments' (list, reddit_enrich shape), 'comment_insights' |
| 157 | (list[str]), and 'num_comments' (int or None). Empty/None on any |
| 158 | failure — never raises, so the caller can fall through to SC backup. |
| 159 | """ |
| 160 | ref = extract_post_ref(post_url) |
| 161 | if not ref: |
| 162 | return {"top_comments": [], "comment_insights": [], "num_comments": None} |
| 163 | sub, post_id = ref |
| 164 | |
| 165 | html_text = http.reddit_keyless_get_text(_svc_url(sub, post_id), timeout=timeout, accept="text/html") |
| 166 | if not html_text: |
| 167 | return {"top_comments": [], "comment_insights": [], "num_comments": None} |
| 168 | |
| 169 | comments = parse_comments(html_text, limit=MAX_COMMENTS) |
| 170 | insights = reddit_enrich.extract_comment_insights(comments) |
| 171 | return { |
| 172 | "top_comments": [ |
| 173 | { |
| 174 | "score": c["score"], |
| 175 | "date": c["date"], |
| 176 | "author": c["author"], |
| 177 | "excerpt": c["excerpt"], |
| 178 | "url": c["url"], |
| 179 | } |
| 180 | for c in comments |
| 181 | ], |
| 182 | "comment_insights": insights, |
| 183 | "num_comments": _total_comments(html_text), |
| 184 | } |
| 185 |