| 1 | """Tests for scripts/lib/reddit_public.py — standalone Reddit public JSON search.""" |
| 2 | |
| 3 | import json |
| 4 | import urllib.error |
| 5 | from unittest import mock |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | # Ensure lib is importable |
| 10 | |
| 11 | from lib import reddit_public |
| 12 | |
| 13 | # --------------------------------------------------------------------------- |
| 14 | # Fixtures / helpers |
| 15 | # --------------------------------------------------------------------------- |
| 16 | |
| 17 | |
| 18 | def _make_reddit_listing(posts): |
| 19 | """Build a Reddit listing JSON structure from a list of post dicts.""" |
| 20 | children = [] |
| 21 | for p in posts: |
| 22 | children.append({ |
| 23 | "kind": "t3", |
| 24 | "data": { |
| 25 | "title": p.get("title", "Test Post"), |
| 26 | "permalink": p.get("permalink", "/r/test/comments/abc123/test_post/"), |
| 27 | "subreddit": p.get("subreddit", "test"), |
| 28 | "score": p.get("score", 42), |
| 29 | "num_comments": p.get("num_comments", 10), |
| 30 | "created_utc": p.get("created_utc", 1711670400), # 2024-03-29 |
| 31 | "author": p.get("author", "testuser"), |
| 32 | "selftext": p.get("selftext", "Some body text"), |
| 33 | "upvote_ratio": p.get("upvote_ratio", 0.95), |
| 34 | }, |
| 35 | }) |
| 36 | return {"data": {"children": children}} |
| 37 | |
| 38 | SAMPLE_LISTING = _make_reddit_listing([ |
| 39 | { |
| 40 | "title": "Claude Code is amazing", |
| 41 | "permalink": "/r/ClaudeAI/comments/abc123/claude_code_is_amazing/", |
| 42 | "subreddit": "ClaudeAI", |
| 43 | "score": 250, |
| 44 | "num_comments": 45, |
| 45 | "created_utc": 1711670400, |
| 46 | "author": "ai_fan", |
| 47 | "selftext": "I've been using Claude Code for a week and it changed my workflow.", |
| 48 | }, |
| 49 | { |
| 50 | "title": "Tips for Claude Code prompting", |
| 51 | "permalink": "/r/ClaudeAI/comments/def456/tips_for_claude_code/", |
| 52 | "subreddit": "ClaudeAI", |
| 53 | "score": 120, |
| 54 | "num_comments": 22, |
| 55 | "created_utc": 1711584000, |
| 56 | "author": "prompt_engineer", |
| 57 | "selftext": "Here are my top tips for getting the most out of Claude Code.", |
| 58 | }, |
| 59 | ]) |
| 60 | |
| 61 | |
| 62 | def _mock_urlopen_ok(listing_data): |
| 63 | """Return a context-manager mock for urllib.request.urlopen that returns listing_data.""" |
| 64 | resp = mock.MagicMock() |
| 65 | resp.read.return_value = json.dumps(listing_data).encode("utf-8") |
| 66 | resp.headers = {"Content-Type": "application/json"} |
| 67 | resp.__enter__ = mock.MagicMock(return_value=resp) |
| 68 | resp.__exit__ = mock.MagicMock(return_value=False) |
| 69 | return resp |
| 70 | |
| 71 | # --------------------------------------------------------------------------- |
| 72 | # Tests |
| 73 | # --------------------------------------------------------------------------- |
| 74 | |
| 75 | |
| 76 | class TestSearchReturnsCorrectFields: |
| 77 | """Search query returns parsed results with correct fields.""" |
| 78 | |
| 79 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 80 | def test_search_returns_parsed_results(self, mock_urlopen): |
| 81 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 82 | |
| 83 | results = reddit_public.search("Claude Code", depth="quick") |
| 84 | |
| 85 | assert len(results) == 2 |
| 86 | first = results[0] |
| 87 | |
| 88 | # Check all required fields exist |
| 89 | assert "id" in first |
| 90 | assert "title" in first |
| 91 | assert "url" in first |
| 92 | assert "score" in first |
| 93 | assert "num_comments" in first |
| 94 | assert "subreddit" in first |
| 95 | assert "created_utc" in first |
| 96 | assert "author" in first |
| 97 | assert "selftext" in first |
| 98 | |
| 99 | # Check values |
| 100 | assert first["title"] == "Claude Code is amazing" |
| 101 | assert first["subreddit"] == "ClaudeAI" |
| 102 | assert first["score"] == 250 |
| 103 | assert first["num_comments"] == 45 |
| 104 | assert first["author"] == "ai_fan" |
| 105 | assert "/comments/" in first["url"] |
| 106 | assert first["id"] == "R1" |
| 107 | |
| 108 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 109 | def test_search_includes_normalized_fields(self, mock_urlopen): |
| 110 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 111 | |
| 112 | results = reddit_public.search("Claude Code") |
| 113 | first = results[0] |
| 114 | |
| 115 | # Normalized fields matching ScrapeCreators format |
| 116 | assert "date" in first |
| 117 | assert "engagement" in first |
| 118 | assert "relevance" in first |
| 119 | assert "why_relevant" in first |
| 120 | |
| 121 | assert isinstance(first["engagement"], dict) |
| 122 | assert "score" in first["engagement"] |
| 123 | assert "num_comments" in first["engagement"] |
| 124 | |
| 125 | |
| 126 | class TestSubredditScopedSearch: |
| 127 | """Subreddit-scoped search works.""" |
| 128 | |
| 129 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 130 | def test_subreddit_search_builds_correct_url(self, mock_urlopen): |
| 131 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 132 | |
| 133 | reddit_public.search("Claude Code", subreddit="ClaudeAI") |
| 134 | |
| 135 | # Check the URL passed to urlopen |
| 136 | call_args = mock_urlopen.call_args |
| 137 | req = call_args[0][0] # First positional arg is the Request object |
| 138 | assert "/r/ClaudeAI/search.json" in req.full_url |
| 139 | assert "restrict_sr=on" in req.full_url |
| 140 | |
| 141 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 142 | def test_subreddit_search_strips_prefix(self, mock_urlopen): |
| 143 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 144 | |
| 145 | reddit_public.search("Claude Code", subreddit="r/ClaudeAI") |
| 146 | |
| 147 | req = mock_urlopen.call_args[0][0] |
| 148 | # Should strip the r/ prefix, not double it |
| 149 | assert "/r/ClaudeAI/search.json" in req.full_url |
| 150 | assert "/r/r/" not in req.full_url |
| 151 | |
| 152 | |
| 153 | class TestRetryOn429: |
| 154 | """429 response triggers retries, eventually returns partial results.""" |
| 155 | |
| 156 | @mock.patch("lib.reddit_public.time.sleep") |
| 157 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 158 | def test_429_retries_then_returns_empty(self, mock_urlopen, mock_sleep): |
| 159 | error = urllib.error.HTTPError( |
| 160 | "https://reddit.com/search.json", 429, "Too Many Requests", |
| 161 | {"Retry-After": "1"}, None, |
| 162 | ) |
| 163 | mock_urlopen.side_effect = error |
| 164 | |
| 165 | results = reddit_public.search("test query") |
| 166 | |
| 167 | assert results == [] |
| 168 | # Should have retried (MAX_RETRIES = 3, sleeps happen between attempts) |
| 169 | assert mock_sleep.call_count == reddit_public.MAX_RETRIES - 1 |
| 170 | |
| 171 | @mock.patch("lib.reddit_public.time.sleep") |
| 172 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 173 | def test_429_then_success(self, mock_urlopen, mock_sleep): |
| 174 | error = urllib.error.HTTPError( |
| 175 | "https://reddit.com/search.json", 429, "Too Many Requests", |
| 176 | {}, None, |
| 177 | ) |
| 178 | success_resp = _mock_urlopen_ok(SAMPLE_LISTING) |
| 179 | |
| 180 | mock_urlopen.side_effect = [error, success_resp] |
| 181 | |
| 182 | results = reddit_public.search("test query") |
| 183 | |
| 184 | assert len(results) == 2 |
| 185 | assert mock_sleep.call_count == 1 |
| 186 | |
| 187 | |
| 188 | class TestHtmlAntiBot: |
| 189 | """HTML response (anti-bot) is detected, returns empty.""" |
| 190 | |
| 191 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 192 | def test_html_response_returns_empty(self, mock_urlopen): |
| 193 | resp = mock.MagicMock() |
| 194 | resp.read.return_value = b"<html><body>Please verify you are human</body></html>" |
| 195 | resp.headers = {"Content-Type": "text/html; charset=utf-8"} |
| 196 | resp.__enter__ = mock.MagicMock(return_value=resp) |
| 197 | resp.__exit__ = mock.MagicMock(return_value=False) |
| 198 | mock_urlopen.return_value = resp |
| 199 | |
| 200 | results = reddit_public.search("test query") |
| 201 | |
| 202 | assert results == [] |
| 203 | |
| 204 | |
| 205 | class TestNetworkTimeout: |
| 206 | """Network timeout returns empty.""" |
| 207 | |
| 208 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 209 | def test_timeout_returns_empty(self, mock_urlopen): |
| 210 | mock_urlopen.side_effect = TimeoutError("Connection timed out") |
| 211 | |
| 212 | results = reddit_public.search("test query") |
| 213 | |
| 214 | assert results == [] |
| 215 | |
| 216 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 217 | def test_url_error_returns_empty(self, mock_urlopen): |
| 218 | mock_urlopen.side_effect = urllib.error.URLError("Connection refused") |
| 219 | |
| 220 | results = reddit_public.search("test query") |
| 221 | |
| 222 | assert results == [] |
| 223 | |
| 224 | |
| 225 | class TestNormalizationMatchesScrapeCreators: |
| 226 | """Results normalize to same schema as ScrapeCreators (field names match).""" |
| 227 | |
| 228 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 229 | def test_field_names_match_scrapecreators(self, mock_urlopen): |
| 230 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 231 | |
| 232 | results = reddit_public.search("Claude Code") |
| 233 | assert len(results) > 0 |
| 234 | item = results[0] |
| 235 | |
| 236 | # These fields must exist to match ScrapeCreators _normalize_post output |
| 237 | sc_required_fields = {"id", "title", "url", "subreddit", "date", |
| 238 | "engagement", "relevance", "why_relevant"} |
| 239 | actual_fields = set(item.keys()) |
| 240 | assert sc_required_fields.issubset(actual_fields), ( |
| 241 | f"Missing fields: {sc_required_fields - actual_fields}" |
| 242 | ) |
| 243 | |
| 244 | # Engagement sub-fields |
| 245 | eng = item["engagement"] |
| 246 | assert "score" in eng |
| 247 | assert "num_comments" in eng |
| 248 | assert "upvote_ratio" in eng |
| 249 | |
| 250 | |
| 251 | class TestDepthLimits: |
| 252 | """Depth-aware limits are respected.""" |
| 253 | |
| 254 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 255 | def test_quick_limit(self, mock_urlopen): |
| 256 | # Create more posts than the quick limit |
| 257 | many_posts = _make_reddit_listing([ |
| 258 | {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"} |
| 259 | for i in range(20) |
| 260 | ]) |
| 261 | mock_urlopen.return_value = _mock_urlopen_ok(many_posts) |
| 262 | |
| 263 | results = reddit_public.search("test", depth="quick") |
| 264 | assert len(results) <= 10 |
| 265 | |
| 266 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 267 | def test_default_limit(self, mock_urlopen): |
| 268 | many_posts = _make_reddit_listing([ |
| 269 | {"title": f"Post {i}", "permalink": f"/r/test/comments/{i:06d}/post_{i}/"} |
| 270 | for i in range(50) |
| 271 | ]) |
| 272 | mock_urlopen.return_value = _mock_urlopen_ok(many_posts) |
| 273 | |
| 274 | results = reddit_public.search("test", depth="default") |
| 275 | assert len(results) <= 25 |
| 276 | |
| 277 | |
| 278 | class TestSearchRedditPublicHighLevel: |
| 279 | """Test the high-level search_reddit_public interface.""" |
| 280 | |
| 281 | def test_date_filtering(self): |
| 282 | # search_reddit_public delegates to the keyless pipeline, which filters |
| 283 | # discovered posts to the requested date window. (search.json is gone, so |
| 284 | # the filter is verified at the keyless discovery seam, not via .json.) |
| 285 | def _p(title, slug, date): |
| 286 | return { |
| 287 | "id": "", "title": title, "score": 0, "num_comments": 0, |
| 288 | "url": f"https://www.reddit.com/r/test/comments/{slug}/x/", |
| 289 | "subreddit": "test", "author": "u", "selftext": "", "date": date, |
| 290 | "engagement": {"score": 0, "num_comments": 0}, "relevance": 0.0, |
| 291 | "metadata": {}, |
| 292 | } |
| 293 | posts = [_p("In range", "aaa", "2024-03-29"), _p("Out of range", "bbb", "2021-01-01")] |
| 294 | with mock.patch("lib.reddit_keyless._discover", return_value=posts), \ |
| 295 | mock.patch("lib.reddit_keyless.reddit_shreddit.fetch_comments", |
| 296 | return_value={"top_comments": [], "comment_insights": [], "num_comments": None}): |
| 297 | results = reddit_public.search_reddit_public("test", "2024-03-01", "2024-03-31") |
| 298 | |
| 299 | titles = [r["title"] for r in results] |
| 300 | assert "In range" in titles |
| 301 | assert "Out of range" not in titles |
| 302 | |
| 303 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 304 | def test_user_agent_header(self, mock_urlopen): |
| 305 | mock_urlopen.return_value = _mock_urlopen_ok(SAMPLE_LISTING) |
| 306 | |
| 307 | reddit_public.search("test") |
| 308 | |
| 309 | req = mock_urlopen.call_args[0][0] |
| 310 | assert "Mozilla/5.0" in req.get_header("User-agent") |
| 311 | |
| 312 | |
| 313 | class TestMissingSubreddit: |
| 314 | """Subreddit doesn't exist returns empty.""" |
| 315 | |
| 316 | @mock.patch("lib.reddit_public.urllib.request.urlopen") |
| 317 | def test_404_subreddit_returns_empty(self, mock_urlopen): |
| 318 | mock_urlopen.side_effect = urllib.error.HTTPError( |
| 319 | "https://reddit.com/r/nonexistent/search.json", |
| 320 | 404, "Not Found", {}, None, |
| 321 | ) |
| 322 | |
| 323 | results = reddit_public.search("test", subreddit="nonexistent") |
| 324 | assert results == [] |
| 325 | |
| 326 | # --------------------------------------------------------------------------- |
| 327 | # search_reddit_public is now a thin shim over the keyless pipeline. |
| 328 | # Full discovery + enrichment behavior is covered in test_reddit_keyless.py. |
| 329 | # --------------------------------------------------------------------------- |
| 330 | |
| 331 | |
| 332 | class TestSearchRedditPublicDelegatesToKeyless: |
| 333 | """search_reddit_public delegates to reddit_keyless.search_and_enrich.""" |
| 334 | |
| 335 | def test_delegates_with_all_args(self): |
| 336 | with mock.patch("lib.reddit_keyless.search_and_enrich") as mock_keyless: |
| 337 | mock_keyless.return_value = [{"id": "R1", "title": "x"}] |
| 338 | results = reddit_public.search_reddit_public( |
| 339 | "test", "2024-03-01", "2024-03-31", |
| 340 | depth="quick", subreddits=["ClaudeAI"], |
| 341 | ) |
| 342 | |
| 343 | assert results == [{"id": "R1", "title": "x"}] |
| 344 | mock_keyless.assert_called_once_with( |
| 345 | "test", "2024-03-01", "2024-03-31", |
| 346 | depth="quick", subreddits=["ClaudeAI"], dedicated_subreddits=None, |
| 347 | ) |
| 348 |