| 1 | """Pipeline Reddit dispatch: free-first, SC thinness-floor backfill (U7), and |
| 2 | that the keyless path never calls search.json (U2).""" |
| 3 | |
| 4 | from unittest import mock |
| 5 | |
| 6 | from pathlib import Path |
| 7 | |
| 8 | from lib import env, pipeline, reddit_keyless, schema |
| 9 | |
| 10 | |
| 11 | def _subquery(): |
| 12 | return schema.SubQuery(label="t", search_query="kanye", ranking_query="kanye", |
| 13 | sources=["reddit"]) |
| 14 | |
| 15 | |
| 16 | def _runtime(): |
| 17 | return schema.ProviderRuntime(reasoning_provider="mock", planner_model="mock", |
| 18 | rerank_model="mock") |
| 19 | |
| 20 | |
| 21 | def _item(rid): |
| 22 | return {"url": f"https://www.reddit.com/r/x/comments/{rid}/t/", "title": rid} |
| 23 | |
| 24 | |
| 25 | def _ids(items): |
| 26 | return [pipeline._reddit_post_key(i) for i in items] |
| 27 | |
| 28 | |
| 29 | class TestThinnessFloor: |
| 30 | KEY = {"SCRAPECREATORS_API_KEY": "k"} |
| 31 | |
| 32 | def _run(self, config, public, sc_parsed): |
| 33 | with mock.patch("lib.reddit_public.search_reddit_public", return_value=public), \ |
| 34 | mock.patch("lib.reddit.search_and_enrich", return_value={"raw": 1}) as sc, \ |
| 35 | mock.patch("lib.reddit.parse_reddit_response", return_value=sc_parsed): |
| 36 | items, _ = pipeline._retrieve_stream( |
| 37 | topic="kanye", subquery=_subquery(), source="reddit", config=config, |
| 38 | depth="quick", date_range=("2026-05-26", "2026-06-25"), |
| 39 | runtime=_runtime(), mock=False, |
| 40 | ) |
| 41 | return items, sc |
| 42 | |
| 43 | def test_default_zero_does_not_call_sc_when_free_has_items(self): |
| 44 | # min_items unset (0): today's empty-only behavior — free wins, no SC. |
| 45 | items, sc = self._run(self.KEY, [_item("a"), _item("b"), _item("c")], [_item("z")]) |
| 46 | assert len(items) == 3 |
| 47 | sc.assert_not_called() |
| 48 | |
| 49 | def test_default_empty_free_falls_to_sc(self): |
| 50 | items, sc = self._run(self.KEY, [], [_item("z")]) |
| 51 | assert _ids(items) == ["z"] |
| 52 | sc.assert_called_once() |
| 53 | |
| 54 | def test_threshold_fires_on_thin_run_and_merges_deduped(self): |
| 55 | cfg = {**self.KEY, "LAST30DAYS_REDDIT_SC_MIN_ITEMS": "5"} |
| 56 | free = [_item("a"), _item("b")] # 2 < 5 -> backfill |
| 57 | sc_parsed = [_item("b"), _item("c")] # overlaps "b" |
| 58 | items, sc = self._run(cfg, free, sc_parsed) |
| 59 | sc.assert_called_once() |
| 60 | assert _ids(items) == ["a", "b", "c"] # free first, dedup b, append c |
| 61 | |
| 62 | def test_threshold_not_fired_when_free_above_floor(self): |
| 63 | cfg = {**self.KEY, "LAST30DAYS_REDDIT_SC_MIN_ITEMS": "2"} |
| 64 | items, sc = self._run(cfg, [_item("a"), _item("b"), _item("c")], [_item("z")]) |
| 65 | sc.assert_not_called() |
| 66 | assert len(items) == 3 |
| 67 | |
| 68 | def test_exactly_floor_is_acceptable_no_backfill(self): |
| 69 | # MIN_ITEMS=N means N results are acceptable; only fewer than N backfills. |
| 70 | cfg = {**self.KEY, "LAST30DAYS_REDDIT_SC_MIN_ITEMS": "3"} |
| 71 | items, sc = self._run(cfg, [_item("a"), _item("b"), _item("c")], [_item("z")]) |
| 72 | sc.assert_not_called() |
| 73 | assert len(items) == 3 |
| 74 | |
| 75 | def test_no_key_never_calls_sc(self): |
| 76 | items, sc = self._run({"LAST30DAYS_REDDIT_SC_MIN_ITEMS": "5"}, [_item("a")], [_item("z")]) |
| 77 | sc.assert_not_called() |
| 78 | assert len(items) == 1 |
| 79 | |
| 80 | def test_bad_threshold_value_defaults_to_empty_only(self): |
| 81 | cfg = {**self.KEY, "LAST30DAYS_REDDIT_SC_MIN_ITEMS": "not-an-int"} |
| 82 | items, sc = self._run(cfg, [_item("a")], [_item("z")]) |
| 83 | sc.assert_not_called() # falls back to 0 -> free (1 item) wins |
| 84 | assert len(items) == 1 |
| 85 | |
| 86 | def test_sc_failure_degrades_to_free(self): |
| 87 | cfg = {**self.KEY, "LAST30DAYS_REDDIT_SC_MIN_ITEMS": "5"} |
| 88 | with mock.patch("lib.reddit_public.search_reddit_public", return_value=[_item("a")]), \ |
| 89 | mock.patch("lib.reddit.search_and_enrich", side_effect=Exception("down")): |
| 90 | items, _ = pipeline._retrieve_stream( |
| 91 | topic="kanye", subquery=_subquery(), source="reddit", config=cfg, |
| 92 | depth="quick", date_range=("2026-05-26", "2026-06-25"), |
| 93 | runtime=_runtime(), mock=False, |
| 94 | ) |
| 95 | assert _ids(items) == ["a"] # backup failed -> keep the free items |
| 96 | |
| 97 | |
| 98 | class TestMergeHelper: |
| 99 | def test_dedup_by_post_id_free_first(self): |
| 100 | out = pipeline._merge_reddit_items([_item("a"), _item("b")], [_item("b"), _item("c")]) |
| 101 | assert _ids(out) == ["a", "b", "c"] |
| 102 | |
| 103 | |
| 104 | class TestNoSearchJson: |
| 105 | def test_keyless_discovery_never_calls_searchjson(self): |
| 106 | # reddit_public.search (the .json caller) must never run in the keyless flow. |
| 107 | with mock.patch("lib.reddit_public.search") as json_search, \ |
| 108 | mock.patch("lib.reddit_keyless.reddit_rss.search_rss", return_value=[]), \ |
| 109 | mock.patch("lib.reddit_keyless.reddit_listing.fetch_listings", return_value=[]): |
| 110 | reddit_keyless._discover("topic", "default", ["test"]) |
| 111 | json_search.assert_not_called() |
| 112 | |
| 113 | |
| 114 | class TestEnvConstantParity: |
| 115 | """F2 regression (restate-as-mirror drift): pipeline's Reddit gating must |
| 116 | key off env's declared constants (env.REDDIT_BACKEND_PIN_VAR / |
| 117 | env.REDDIT_SC_MIN_ITEMS_VAR) — never restated raw strings that can drift |
| 118 | from the single source of truth in lib/env.py.""" |
| 119 | |
| 120 | def _run(self, config, public, sc_parsed): |
| 121 | with mock.patch("lib.reddit_public.search_reddit_public", |
| 122 | return_value=public) as pub, \ |
| 123 | mock.patch("lib.reddit.search_and_enrich", return_value={"raw": 1}) as sc, \ |
| 124 | mock.patch("lib.reddit.parse_reddit_response", return_value=sc_parsed): |
| 125 | items, _ = pipeline._retrieve_stream( |
| 126 | topic="kanye", subquery=_subquery(), source="reddit", config=config, |
| 127 | depth="quick", date_range=("2026-05-26", "2026-06-25"), |
| 128 | runtime=_runtime(), mock=False, |
| 129 | ) |
| 130 | return items, pub, sc |
| 131 | |
| 132 | def test_pipeline_source_has_no_raw_reddit_env_literals(self): |
| 133 | # The declared constants live in env.py; pipeline.py must not restate |
| 134 | # the raw LAST30DAYS_REDDIT_* strings (comments included — they drift too). |
| 135 | source = Path(pipeline.__file__).read_text() |
| 136 | assert "LAST30DAYS_REDDIT_" not in source |
| 137 | |
| 138 | def test_backend_pin_constant_flips_gating_to_sc_primary(self): |
| 139 | # Keyed via the env constant, not a raw string: pin=scrapecreators |
| 140 | # makes SC primary and skips the free path entirely. |
| 141 | cfg = {"SCRAPECREATORS_API_KEY": "k", env.REDDIT_BACKEND_PIN_VAR: "scrapecreators"} |
| 142 | items, pub, sc = self._run(cfg, [_item("a")], [_item("z")]) |
| 143 | sc.assert_called_once() |
| 144 | pub.assert_not_called() |
| 145 | assert _ids(items) == ["z"] |
| 146 | |
| 147 | def test_min_items_constant_drives_thinness_backfill(self): |
| 148 | # Keyed via the env constant: floor of 5 vs 1 free item -> SC backfill. |
| 149 | cfg = {"SCRAPECREATORS_API_KEY": "k", env.REDDIT_SC_MIN_ITEMS_VAR: "5"} |
| 150 | items, pub, sc = self._run(cfg, [_item("a")], [_item("z")]) |
| 151 | sc.assert_called_once() |
| 152 | assert _ids(items) == ["a", "z"] |
| 153 |