| 1 | """Tests for YouTube transcript highlights and yt-dlp safety flags.""" |
| 2 | |
| 3 | import json |
| 4 | import os |
| 5 | import tempfile |
| 6 | import unittest |
| 7 | import urllib.error |
| 8 | from datetime import datetime, timedelta, timezone |
| 9 | from pathlib import Path |
| 10 | from unittest import mock |
| 11 | |
| 12 | from lib import youtube_yt |
| 13 | |
| 14 | |
| 15 | class _DummyProc: |
| 16 | def __init__(self): |
| 17 | self.pid = 12345 |
| 18 | self.returncode = 0 |
| 19 | |
| 20 | def communicate(self, timeout=None): |
| 21 | return "", "" |
| 22 | |
| 23 | def wait(self, timeout=None): |
| 24 | return 0 |
| 25 | |
| 26 | |
| 27 | class TestYouTubeEngagementZero(unittest.TestCase): |
| 28 | """Verify that 0 engagement counts are preserved (not coerced to fallback).""" |
| 29 | |
| 30 | def test_zero_view_count_preserved(self): |
| 31 | """video.get('view_count') == 0 must stay 0, not become the fallback.""" |
| 32 | import json |
| 33 | import tempfile |
| 34 | import os |
| 35 | |
| 36 | video = { |
| 37 | "id": "abc123", |
| 38 | "title": "Test", |
| 39 | "view_count": 0, |
| 40 | "like_count": 0, |
| 41 | "comment_count": 0, |
| 42 | "upload_date": "20260301", |
| 43 | "description": "desc", |
| 44 | } |
| 45 | with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: |
| 46 | f.write(json.dumps(video) + "\n") |
| 47 | f.flush() |
| 48 | with open(f.name) as rf: |
| 49 | lines = rf.readlines() |
| 50 | |
| 51 | # Re-parse as the search function would |
| 52 | parsed = json.loads(lines[0]) |
| 53 | view_count = parsed.get("view_count") if parsed.get("view_count") is not None else 0 |
| 54 | like_count = parsed.get("like_count") if parsed.get("like_count") is not None else 0 |
| 55 | comment_count = parsed.get("comment_count") if parsed.get("comment_count") is not None else 0 |
| 56 | |
| 57 | os.unlink(f.name) |
| 58 | |
| 59 | self.assertEqual(0, view_count) |
| 60 | self.assertEqual(0, like_count) |
| 61 | self.assertEqual(0, comment_count) |
| 62 | |
| 63 | |
| 64 | class TestYtDlpFlags(unittest.TestCase): |
| 65 | def setUp(self): |
| 66 | youtube_yt.reset_search_cache() |
| 67 | |
| 68 | def _fake_result(self, stdout: str = "", returncode: int = 0): |
| 69 | from lib.subproc import SubprocResult |
| 70 | return SubprocResult(returncode=returncode, stdout=stdout, stderr="") |
| 71 | |
| 72 | def test_search_ignores_global_config_and_browser_cookies(self): |
| 73 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 74 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock: |
| 75 | youtube_yt.search_youtube("Claude Code", "2026-02-01", "2026-03-01") |
| 76 | |
| 77 | cmd = run_mock.call_args.args[0] |
| 78 | self.assertIn("--ignore-config", cmd) |
| 79 | self.assertIn("--no-cookies-from-browser", cmd) |
| 80 | |
| 81 | def test_transcript_fetch_ignores_global_config_and_browser_cookies(self): |
| 82 | with tempfile.TemporaryDirectory() as temp_dir, \ |
| 83 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 84 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock: |
| 85 | youtube_yt.fetch_transcript("abc123", temp_dir) |
| 86 | |
| 87 | cmd = run_mock.call_args.args[0] |
| 88 | self.assertIn("--ignore-config", cmd) |
| 89 | self.assertIn("--no-cookies-from-browser", cmd) |
| 90 | |
| 91 | |
| 92 | class TestYtDlpSubLangs(unittest.TestCase): |
| 93 | """Verify LAST30DAYS_YT_SUB_LANGS knob and language-agnostic VTT matching.""" |
| 94 | |
| 95 | def _fake_result(self, stdout: str = "", returncode: int = 0): |
| 96 | from lib.subproc import SubprocResult |
| 97 | return SubprocResult(returncode=returncode, stdout=stdout, stderr="") |
| 98 | |
| 99 | def test_default_sub_langs_when_env_unset(self): |
| 100 | """When LAST30DAYS_YT_SUB_LANGS is not set, the default is en,es,pt.""" |
| 101 | with mock.patch.dict(os.environ, {}, clear=False): |
| 102 | os.environ.pop("LAST30DAYS_YT_SUB_LANGS", None) |
| 103 | self.assertEqual(youtube_yt._ytdlp_sub_langs(), "en,es,pt") |
| 104 | |
| 105 | def test_env_var_overrides_default(self): |
| 106 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": "fr,de"}): |
| 107 | self.assertEqual(youtube_yt._ytdlp_sub_langs(), "fr,de") |
| 108 | |
| 109 | def test_env_var_normalizes_whitespace_and_case(self): |
| 110 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": " EN , Es , PT "}): |
| 111 | self.assertEqual(youtube_yt._ytdlp_sub_langs(), "en,es,pt") |
| 112 | |
| 113 | def test_env_var_handles_empty_segments(self): |
| 114 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": "en,,pt,"}): |
| 115 | self.assertEqual(youtube_yt._ytdlp_sub_langs(), "en,pt") |
| 116 | |
| 117 | def test_env_var_empty_string_falls_back_to_default(self): |
| 118 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": " "}): |
| 119 | self.assertEqual(youtube_yt._ytdlp_sub_langs(), "en,es,pt") |
| 120 | |
| 121 | def test_transcript_cmd_uses_default_sub_langs(self): |
| 122 | """Regression: the --sub-lang arg is en,es,pt by default (issue #469).""" |
| 123 | with tempfile.TemporaryDirectory() as temp_dir, \ |
| 124 | mock.patch.dict(os.environ, {}, clear=False), \ |
| 125 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 126 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock: |
| 127 | os.environ.pop("LAST30DAYS_YT_SUB_LANGS", None) |
| 128 | youtube_yt.fetch_transcript("abc123", temp_dir) |
| 129 | |
| 130 | cmd = run_mock.call_args_list[0].args[0] |
| 131 | idx = cmd.index("--sub-lang") |
| 132 | self.assertEqual(cmd[idx + 1], "en,es,pt") |
| 133 | |
| 134 | def test_transcript_cmd_respects_env_var_override(self): |
| 135 | with tempfile.TemporaryDirectory() as temp_dir, \ |
| 136 | mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": "fr,de,it"}), \ |
| 137 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 138 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()) as run_mock: |
| 139 | youtube_yt.fetch_transcript("abc123", temp_dir) |
| 140 | |
| 141 | cmd = run_mock.call_args_list[0].args[0] |
| 142 | idx = cmd.index("--sub-lang") |
| 143 | self.assertEqual(cmd[idx + 1], "fr,de,it") |
| 144 | |
| 145 | def test_vtt_matching_picks_non_english_track(self): |
| 146 | """When yt-dlp writes a Spanish track (no English available), we read it.""" |
| 147 | with tempfile.TemporaryDirectory() as temp_dir: |
| 148 | # Simulate yt-dlp output: only a Spanish VTT is available |
| 149 | (Path(temp_dir) / "abc123.es.vtt").write_text( |
| 150 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHola mundo esta es una prueba.\n", |
| 151 | encoding="utf-8", |
| 152 | ) |
| 153 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 154 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()): |
| 155 | vtt = youtube_yt._fetch_transcript_ytdlp("abc123", temp_dir) |
| 156 | |
| 157 | self.assertIsNotNone(vtt) |
| 158 | self.assertIn("Hola mundo", vtt) |
| 159 | |
| 160 | def test_partial_success_returns_vtt_despite_nonzero_exit(self): |
| 161 | """A non-zero yt-dlp exit must not discard a VTT already on disk. |
| 162 | |
| 163 | Regression for the 0/N-transcripts bug: with the default |
| 164 | ``--sub-lang en,es,pt``, an English video fetches ``en`` successfully, |
| 165 | then ``es``/``pt`` hit a 429 and yt-dlp exits non-zero. The ``en`` |
| 166 | track is already written and must be returned, not discarded (and not |
| 167 | retried back into the same rate limit). |
| 168 | """ |
| 169 | with tempfile.TemporaryDirectory() as temp_dir: |
| 170 | (Path(temp_dir) / "abc123.en.vtt").write_text( |
| 171 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nThis is the english transcript.\n", |
| 172 | encoding="utf-8", |
| 173 | ) |
| 174 | status: dict = {} |
| 175 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 176 | mock.patch.object( |
| 177 | youtube_yt.subproc, |
| 178 | "run_with_timeout", |
| 179 | return_value=self._fake_result(returncode=1), |
| 180 | ) as run_mock: |
| 181 | vtt = youtube_yt._fetch_transcript_ytdlp("abc123", temp_dir, status) |
| 182 | |
| 183 | self.assertIsNotNone(vtt) |
| 184 | self.assertIn("english transcript", vtt) |
| 185 | self.assertNotIn("ytdlp_error", status) |
| 186 | # Salvage must short-circuit the retry loop: yt-dlp must not be called a |
| 187 | # second time when a partial VTT is already on disk (locks in the |
| 188 | # no-retry guarantee against a future salvage-after-retry regression). |
| 189 | self.assertEqual(run_mock.call_count, 1) |
| 190 | |
| 191 | def test_vtt_matching_respects_non_default_priority(self): |
| 192 | """When multiple tracks exist, the user-requested priority wins |
| 193 | over alphabetical order (regression for the Greptile review on #486).""" |
| 194 | with tempfile.TemporaryDirectory() as temp_dir: |
| 195 | (Path(temp_dir) / "abc123.en.vtt").write_text( |
| 196 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nEnglish first track.\n", |
| 197 | encoding="utf-8", |
| 198 | ) |
| 199 | (Path(temp_dir) / "abc123.es.vtt").write_text( |
| 200 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nSpanish second track.\n", |
| 201 | encoding="utf-8", |
| 202 | ) |
| 203 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": "es,en"}), \ |
| 204 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 205 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()): |
| 206 | vtt = youtube_yt._fetch_transcript_ytdlp("abc123", temp_dir) |
| 207 | |
| 208 | self.assertIsNotNone(vtt) |
| 209 | self.assertIn("Spanish", vtt) |
| 210 | |
| 211 | def test_vtt_matching_unknown_suffix_sorts_last(self): |
| 212 | """A non-lang suffix (e.g. a stray .tmp or .live_chat) must not |
| 213 | win over a real track that just happens to be alphabetically later.""" |
| 214 | with tempfile.TemporaryDirectory() as temp_dir: |
| 215 | (Path(temp_dir) / "abc123.zz.vtt").write_text( |
| 216 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nZZ track content.\n", |
| 217 | encoding="utf-8", |
| 218 | ) |
| 219 | (Path(temp_dir) / "abc123.es.vtt").write_text( |
| 220 | "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nSpanish content.\n", |
| 221 | encoding="utf-8", |
| 222 | ) |
| 223 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_SUB_LANGS": "es,en,pt"}), \ |
| 224 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 225 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=self._fake_result()): |
| 226 | vtt = youtube_yt._fetch_transcript_ytdlp("abc123", temp_dir) |
| 227 | |
| 228 | self.assertIsNotNone(vtt) |
| 229 | self.assertIn("Spanish", vtt) |
| 230 | |
| 231 | |
| 232 | class TestExtractTranscriptHighlights(unittest.TestCase): |
| 233 | def test_extracts_specific_sentences(self): |
| 234 | transcript = ( |
| 235 | "Hey guys welcome back to the channel. " |
| 236 | "In today's video we're looking at something special. " |
| 237 | "The Lego Bugatti Chiron took 13,438 hours to build with over 1 million pieces. " |
| 238 | "Don't forget to subscribe and hit the bell. " |
| 239 | "The tolerance on each brick is 0.002 millimeters which is insane for injection molding. " |
| 240 | "So yeah that's pretty cool. " |
| 241 | "Thanks for watching see you next time." |
| 242 | ) |
| 243 | highlights = youtube_yt.extract_transcript_highlights(transcript, "Lego") |
| 244 | self.assertTrue(len(highlights) > 0) |
| 245 | joined = " ".join(highlights) |
| 246 | self.assertIn("13,438", joined) |
| 247 | self.assertNotIn("subscribe", joined) |
| 248 | self.assertNotIn("welcome back", joined) |
| 249 | |
| 250 | def test_empty_transcript(self): |
| 251 | self.assertEqual(youtube_yt.extract_transcript_highlights("", "test"), []) |
| 252 | |
| 253 | def test_respects_limit(self): |
| 254 | sentences = ". ".join( |
| 255 | f"The model {i} has {i * 100} parameters and runs at {i * 10} tokens per second" |
| 256 | for i in range(20) |
| 257 | ) + "." |
| 258 | highlights = youtube_yt.extract_transcript_highlights(sentences, "model", limit=3) |
| 259 | self.assertEqual(len(highlights), 3) |
| 260 | |
| 261 | def test_punctuation_free_transcript_produces_highlights(self): |
| 262 | # Auto-generated YouTube captions often lack sentence-ending punctuation |
| 263 | words = ( |
| 264 | "the new Tesla Model Y has 350 miles of range and costs about 45000 dollars " |
| 265 | "which makes it one of the most affordable electric vehicles on the market today " |
| 266 | "compared to the BMW iX which starts at 87000 the value proposition is pretty clear " |
| 267 | "and with the 7500 dollar tax credit you can get it for under 40000" |
| 268 | ) |
| 269 | highlights = youtube_yt.extract_transcript_highlights(words, "Tesla Model Y") |
| 270 | self.assertTrue(len(highlights) > 0, "Should produce highlights from punctuation-free text") |
| 271 | |
| 272 | |
| 273 | class TestFetchTranscriptDirect(unittest.TestCase): |
| 274 | """Tests for _fetch_transcript_direct() — direct HTTP transcript fetching.""" |
| 275 | |
| 276 | # Minimal ytInitialPlayerResponse JSON with a caption track |
| 277 | _PLAYER_RESPONSE = json.dumps({ |
| 278 | "captions": { |
| 279 | "playerCaptionsTracklistRenderer": { |
| 280 | "captionTracks": [ |
| 281 | { |
| 282 | "baseUrl": "https://www.youtube.com/api/timedtext?v=abc123&lang=en", |
| 283 | "languageCode": "en", |
| 284 | } |
| 285 | ] |
| 286 | } |
| 287 | } |
| 288 | }) |
| 289 | |
| 290 | _WATCH_HTML = ( |
| 291 | '<html><script>var ytInitialPlayerResponse = ' |
| 292 | + _PLAYER_RESPONSE |
| 293 | + ';</script></html>' |
| 294 | ) |
| 295 | |
| 296 | _SAMPLE_VTT = ( |
| 297 | "WEBVTT\n\n" |
| 298 | "00:00:00.000 --> 00:00:02.000\n" |
| 299 | "Hello world this is a test sentence with enough words to pass.\n\n" |
| 300 | "00:00:02.000 --> 00:00:04.000\n" |
| 301 | "Another line of transcript text here for testing purposes.\n" |
| 302 | ) |
| 303 | |
| 304 | def _mock_urlopen(self, url_or_req, *, timeout=None): |
| 305 | """Return watch HTML or VTT depending on URL.""" |
| 306 | url = url_or_req.full_url if hasattr(url_or_req, 'full_url') else url_or_req |
| 307 | |
| 308 | class _Resp: |
| 309 | def __init__(self, data): |
| 310 | self._data = data.encode("utf-8") |
| 311 | def read(self): |
| 312 | return self._data |
| 313 | def __enter__(self): |
| 314 | return self |
| 315 | def __exit__(self, *a): |
| 316 | pass |
| 317 | |
| 318 | if "watch?" in url: |
| 319 | return _Resp(self._WATCH_HTML) |
| 320 | elif "timedtext" in url: |
| 321 | return _Resp(self._SAMPLE_VTT) |
| 322 | raise urllib.error.URLError("unexpected URL") |
| 323 | |
| 324 | def test_extracts_vtt_from_mock_page(self): |
| 325 | """Happy path: extracts VTT text from a page with captions.""" |
| 326 | with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=self._mock_urlopen): |
| 327 | result = youtube_yt._fetch_transcript_direct("abc123") |
| 328 | self.assertIsNotNone(result) |
| 329 | self.assertIn("WEBVTT", result) |
| 330 | self.assertIn("Hello world", result) |
| 331 | |
| 332 | def test_no_captions_returns_none(self): |
| 333 | """Video with no caption tracks returns None.""" |
| 334 | no_captions_response = json.dumps({"captions": {"playerCaptionsTracklistRenderer": {"captionTracks": []}}}) |
| 335 | html = f'<html><script>var ytInitialPlayerResponse = {no_captions_response};</script></html>' |
| 336 | |
| 337 | class _Resp: |
| 338 | def __init__(self, data): |
| 339 | self._data = data.encode("utf-8") |
| 340 | def read(self): |
| 341 | return self._data |
| 342 | def __enter__(self): |
| 343 | return self |
| 344 | def __exit__(self, *a): |
| 345 | pass |
| 346 | |
| 347 | def mock_open(req, *, timeout=None): |
| 348 | return _Resp(html) |
| 349 | |
| 350 | with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=mock_open): |
| 351 | result = youtube_yt._fetch_transcript_direct("nocaps") |
| 352 | self.assertIsNone(result) |
| 353 | |
| 354 | def test_http_timeout_returns_none(self): |
| 355 | """HTTP timeout on watch page returns None.""" |
| 356 | def timeout_open(req, *, timeout=None): |
| 357 | raise TimeoutError("timed out") |
| 358 | |
| 359 | with mock.patch("lib.youtube_yt.urllib.request.urlopen", side_effect=timeout_open): |
| 360 | result = youtube_yt._fetch_transcript_direct("timeout_vid") |
| 361 | self.assertIsNone(result) |
| 362 | |
| 363 | def test_direct_vtt_feeds_into_clean_vtt(self): |
| 364 | """VTT from direct fetch produces clean plaintext via _clean_vtt().""" |
| 365 | cleaned = youtube_yt._clean_vtt(self._SAMPLE_VTT) |
| 366 | self.assertNotIn("WEBVTT", cleaned) |
| 367 | self.assertNotIn("-->", cleaned) |
| 368 | self.assertIn("Hello world", cleaned) |
| 369 | self.assertIn("Another line", cleaned) |
| 370 | |
| 371 | |
| 372 | class TestFetchTranscriptFallback(unittest.TestCase): |
| 373 | """Tests that fetch_transcript picks yt-dlp or direct path correctly.""" |
| 374 | |
| 375 | def test_uses_ytdlp_when_installed(self): |
| 376 | """When yt-dlp is installed, uses _fetch_transcript_ytdlp.""" |
| 377 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 378 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", return_value="WEBVTT\n\nfake") as yt_mock, \ |
| 379 | mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock: |
| 380 | result = youtube_yt.fetch_transcript("vid1", "/tmp/test") |
| 381 | yt_mock.assert_called_once_with("vid1", "/tmp/test", status=None, fast_fail=False) |
| 382 | direct_mock.assert_not_called() |
| 383 | |
| 384 | def test_uses_direct_when_ytdlp_missing(self): |
| 385 | """When yt-dlp is NOT installed, falls back to _fetch_transcript_direct.""" |
| 386 | sample_vtt = ( |
| 387 | "WEBVTT\n\n" |
| 388 | "00:00:00.000 --> 00:00:02.000\n" |
| 389 | "Direct transcript content with enough words for testing.\n" |
| 390 | ) |
| 391 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \ |
| 392 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp") as yt_mock, \ |
| 393 | mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=sample_vtt) as direct_mock: |
| 394 | result = youtube_yt.fetch_transcript("vid2", "/tmp/test") |
| 395 | yt_mock.assert_not_called() |
| 396 | direct_mock.assert_called_once_with("vid2", status=None) |
| 397 | self.assertIsNotNone(result) |
| 398 | self.assertIn("Direct transcript content", result) |
| 399 | |
| 400 | def test_returns_none_when_both_fail(self): |
| 401 | """Returns None when the chosen path returns None.""" |
| 402 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False), \ |
| 403 | mock.patch.object(youtube_yt, "_fetch_transcript_direct", return_value=None): |
| 404 | result = youtube_yt.fetch_transcript("novid", "/tmp/test") |
| 405 | self.assertIsNone(result) |
| 406 | |
| 407 | |
| 408 | class TestExpandYouTubeQueries(unittest.TestCase): |
| 409 | """Tests for expand_youtube_queries() multi-query generation.""" |
| 410 | |
| 411 | def test_default_depth_returns_two_plus_queries(self): |
| 412 | queries = youtube_yt.expand_youtube_queries("Kanye West", "default") |
| 413 | self.assertGreaterEqual(len(queries), 2) |
| 414 | # First query is the core subject |
| 415 | self.assertEqual(queries[0].lower(), "kanye west") |
| 416 | |
| 417 | def test_how_to_intent_includes_tutorial_variant(self): |
| 418 | # Use deep depth so the intent variant isn't capped out by core + original |
| 419 | queries = youtube_yt.expand_youtube_queries("how to use Docker", "deep") |
| 420 | variant_found = any( |
| 421 | "tutorial" in q.lower() or "guide" in q.lower() or "explained" in q.lower() |
| 422 | for q in queries |
| 423 | ) |
| 424 | self.assertTrue( |
| 425 | variant_found, |
| 426 | f"Expected tutorial/guide/explained in queries: {queries}", |
| 427 | ) |
| 428 | |
| 429 | def test_product_intent_includes_review_variant(self): |
| 430 | # Use deep depth so the intent variant isn't capped out |
| 431 | queries = youtube_yt.expand_youtube_queries("best running shoes", "deep") |
| 432 | variant_found = any("review" in q.lower() for q in queries) |
| 433 | self.assertTrue(variant_found, f"Expected 'review' in queries: {queries}") |
| 434 | |
| 435 | def test_comparison_intent_includes_vs_variant(self): |
| 436 | queries = youtube_yt.expand_youtube_queries("Claude vs Gemini", "default") |
| 437 | variant_found = any("vs" in q.lower() or "compared" in q.lower() for q in queries) |
| 438 | self.assertTrue(variant_found, f"Expected 'vs' or 'compared' in queries: {queries}") |
| 439 | |
| 440 | def test_quick_depth_returns_one_query(self): |
| 441 | queries = youtube_yt.expand_youtube_queries("Kanye West", "quick") |
| 442 | self.assertEqual(len(queries), 1) |
| 443 | |
| 444 | def test_deep_depth_returns_three_queries(self): |
| 445 | queries = youtube_yt.expand_youtube_queries("Kanye West", "deep") |
| 446 | self.assertEqual(len(queries), 3) |
| 447 | |
| 448 | def test_single_word_returns_at_least_one(self): |
| 449 | queries = youtube_yt.expand_youtube_queries("React", "default") |
| 450 | self.assertGreaterEqual(len(queries), 1) |
| 451 | |
| 452 | def test_temporal_words_stripped_from_core(self): |
| 453 | queries = youtube_yt.expand_youtube_queries("kanye west last 30 days", "default") |
| 454 | core = queries[0].lower() |
| 455 | self.assertNotIn("last", core) |
| 456 | self.assertNotIn("days", core) |
| 457 | self.assertIn("kanye", core) |
| 458 | self.assertIn("west", core) |
| 459 | |
| 460 | |
| 461 | class TestTranscriptCandidateSortKey(unittest.TestCase): |
| 462 | """Tests for _transcript_candidate_sort_key recency-boosted ordering.""" |
| 463 | |
| 464 | @staticmethod |
| 465 | def _d(days_ago: int) -> str: |
| 466 | return (datetime.now(timezone.utc) - timedelta(days=days_ago)).strftime("%Y-%m-%d") |
| 467 | |
| 468 | def _make_item(self, video_id, views, date_str): |
| 469 | return { |
| 470 | "video_id": video_id, |
| 471 | "title": f"Video {video_id}", |
| 472 | "url": f"https://www.youtube.com/watch?v={video_id}", |
| 473 | "channel_name": "TestChannel", |
| 474 | "date": date_str, |
| 475 | "engagement": {"views": views, "likes": 10, "comments": 5}, |
| 476 | "relevance": 0.8, |
| 477 | "why_relevant": "test", |
| 478 | "description": "test desc", |
| 479 | "duration": 600, |
| 480 | } |
| 481 | |
| 482 | def test_recency_breaks_views_tie(self): |
| 483 | """When views are equal, the more recent video gets a higher sort key.""" |
| 484 | new = self._make_item("new", 100_000, self._d(1)) |
| 485 | old = self._make_item("old", 100_000, self._d(13)) |
| 486 | self.assertGreater( |
| 487 | youtube_yt._transcript_candidate_sort_key(new), |
| 488 | youtube_yt._transcript_candidate_sort_key(old), |
| 489 | ) |
| 490 | |
| 491 | def test_old_high_view_can_still_qualify_for_transcript(self): |
| 492 | """An old video with very high views still gets a transcript slot; |
| 493 | recency is a tiebreaker, not a gate.""" |
| 494 | old_high = self._make_item("old_high", 10_000_000, self._d(45)) |
| 495 | recent_low = self._make_item("recent_low", 100, self._d(1)) |
| 496 | self.assertGreater( |
| 497 | youtube_yt._transcript_candidate_sort_key(old_high), |
| 498 | youtube_yt._transcript_candidate_sort_key(recent_low), |
| 499 | ) |
| 500 | |
| 501 | def test_no_date_falls_to_back(self): |
| 502 | """An item with no date gets recency 0, sorting behind dated items.""" |
| 503 | no_date = self._make_item("no_date", 50_000, "") |
| 504 | dated = self._make_item("dated", 50_000, self._d(5)) |
| 505 | self.assertGreater( |
| 506 | youtube_yt._transcript_candidate_sort_key(dated), |
| 507 | youtube_yt._transcript_candidate_sort_key(no_date), |
| 508 | ) |
| 509 | |
| 510 | def test_transcript_candidates_pick_recent_over_old_same_views(self): |
| 511 | """search_and_transcribe selects candidates by (views, recency), |
| 512 | so a recent video is tried before an equal-view older video.""" |
| 513 | items = [ |
| 514 | self._make_item("old", 100_000, self._d(13)), |
| 515 | self._make_item("recent", 100_000, self._d(1)), |
| 516 | self._make_item("mid", 50_000, self._d(5)), |
| 517 | ] |
| 518 | |
| 519 | def fake_search(*args, **kwargs): |
| 520 | return {"items": items} |
| 521 | |
| 522 | call_args_list = [] |
| 523 | |
| 524 | def fake_fetch(video_ids, max_workers=5, out_captions_disabled=None, token=None): |
| 525 | call_args_list.extend(video_ids) |
| 526 | return {vid: "transcript" for vid in video_ids} |
| 527 | |
| 528 | with mock.patch.object(youtube_yt, "search_youtube", side_effect=fake_search), \ |
| 529 | mock.patch.object(youtube_yt, "fetch_transcripts_parallel", side_effect=fake_fetch): |
| 530 | youtube_yt.search_and_transcribe("test", self._d(14), self._d(0), depth="default") |
| 531 | |
| 532 | # transcript_limit=2, attempt_count=4 (limited to 3 items) |
| 533 | # Sorted by (views, recency): recent(100k) > old(100k) > mid(50k) |
| 534 | self.assertEqual(call_args_list[:2], ["recent", "old"], |
| 535 | "Recent video should be tried before equal-view older video") |
| 536 | |
| 537 | |
| 538 | class TestSearchAndTranscribe(unittest.TestCase): |
| 539 | """Tests for search_and_transcribe() end-to-end flow.""" |
| 540 | |
| 541 | def _make_item(self, video_id, views): |
| 542 | return { |
| 543 | "video_id": video_id, |
| 544 | "title": f"Video {video_id}", |
| 545 | "url": f"https://www.youtube.com/watch?v={video_id}", |
| 546 | "channel_name": "TestChannel", |
| 547 | "date": "2026-03-15", |
| 548 | "engagement": {"views": views, "likes": 10, "comments": 5}, |
| 549 | "relevance": 0.8, |
| 550 | "why_relevant": "test", |
| 551 | "description": "test desc", |
| 552 | "duration": 600, |
| 553 | } |
| 554 | |
| 555 | def test_transcripts_attached_when_top_videos_lack_captions(self): |
| 556 | """When top-viewed videos have no captions, lower-ranked ones still get transcripts.""" |
| 557 | items = [ |
| 558 | self._make_item("music1", 1_000_000), # no captions (music video) |
| 559 | self._make_item("music2", 500_000), # no captions (music video) |
| 560 | self._make_item("talk1", 50_000), # has captions |
| 561 | self._make_item("talk2", 25_000), # has captions |
| 562 | ] |
| 563 | |
| 564 | # fetch_transcripts_parallel returns None for music videos, text for talks |
| 565 | def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None, token=None): |
| 566 | result = {} |
| 567 | for vid in video_ids: |
| 568 | if vid.startswith("talk"): |
| 569 | result[vid] = "This is a detailed discussion about the topic with 100 data points." |
| 570 | else: |
| 571 | result[vid] = None |
| 572 | return result |
| 573 | |
| 574 | with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \ |
| 575 | mock.patch.object(youtube_yt, "fetch_transcripts_parallel", side_effect=fake_parallel) as ft_mock: |
| 576 | result = youtube_yt.search_and_transcribe("test topic", "2026-03-01", "2026-03-31", depth="default") |
| 577 | |
| 578 | # Should have attempted more than just the top 2 (transcript_limit=2) |
| 579 | called_ids = ft_mock.call_args[0][0] |
| 580 | self.assertGreater(len(called_ids), 2, "Should attempt more than transcript_limit candidates") |
| 581 | self.assertIn("talk1", called_ids) |
| 582 | self.assertIn("talk2", called_ids) |
| 583 | |
| 584 | # talk1 and talk2 should have transcripts |
| 585 | items_by_id = {i["video_id"]: i for i in result["items"]} |
| 586 | self.assertTrue(items_by_id["talk1"]["transcript_snippet"]) |
| 587 | self.assertTrue(items_by_id["talk1"]["transcript_highlights"]) |
| 588 | # music videos should have empty transcripts |
| 589 | self.assertFalse(items_by_id["music1"]["transcript_snippet"]) |
| 590 | |
| 591 | def test_transcript_limit_zero_skips_fetch(self): |
| 592 | """When transcript_limit is 0 (quick depth), no transcripts are fetched.""" |
| 593 | items = [self._make_item("vid1", 1000)] |
| 594 | with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \ |
| 595 | mock.patch.object(youtube_yt, "fetch_transcripts_parallel") as ft_mock: |
| 596 | result = youtube_yt.search_and_transcribe("test", "2026-03-01", "2026-03-31", depth="quick") |
| 597 | |
| 598 | ft_mock.assert_not_called() |
| 599 | self.assertEqual(result["items"][0]["transcript_snippet"], "") |
| 600 | |
| 601 | def test_no_items_returns_early(self): |
| 602 | """When search returns no items, returns without fetching transcripts.""" |
| 603 | with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": []}), \ |
| 604 | mock.patch.object(youtube_yt, "fetch_transcripts_parallel") as ft_mock: |
| 605 | result = youtube_yt.search_and_transcribe("nothing", "2026-03-01", "2026-03-31") |
| 606 | |
| 607 | ft_mock.assert_not_called() |
| 608 | |
| 609 | |
| 610 | class TestTranscriptFetchStats(unittest.TestCase): |
| 611 | """Track yt-dlp fetch outcomes for quality_nudge (#531 false stale-yt-dlp nudge).""" |
| 612 | |
| 613 | FROM_DATE = "2026-03-01" |
| 614 | TO_DATE = "2026-03-31" |
| 615 | |
| 616 | def setUp(self): |
| 617 | youtube_yt.reset_transcript_fetch_stats() |
| 618 | |
| 619 | def _make_item(self, video_id, views, date): |
| 620 | return { |
| 621 | "video_id": video_id, |
| 622 | "title": f"Video {video_id}", |
| 623 | "url": f"https://www.youtube.com/watch?v={video_id}", |
| 624 | "channel_name": "TestChannel", |
| 625 | "date": date, |
| 626 | "engagement": {"views": views, "likes": 10, "comments": 5}, |
| 627 | "relevance": 0.8, |
| 628 | "why_relevant": "test", |
| 629 | "description": "test desc", |
| 630 | "duration": 600, |
| 631 | } |
| 632 | |
| 633 | def _run(self, items, fake_parallel=None): |
| 634 | if fake_parallel is None: |
| 635 | def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None, token=None): |
| 636 | return {vid: "A detailed transcript about the topic." for vid in video_ids} |
| 637 | with mock.patch.object(youtube_yt, "search_youtube", return_value={"items": items}), \ |
| 638 | mock.patch.object(youtube_yt, "fetch_transcripts_parallel", side_effect=fake_parallel): |
| 639 | return youtube_yt.search_and_transcribe( |
| 640 | "test topic", self.FROM_DATE, self.TO_DATE, depth="default", |
| 641 | ) |
| 642 | |
| 643 | def test_fetch_stats_track_attempts_and_failures(self): |
| 644 | items = [ |
| 645 | self._make_item("ok1", 3_000, "2026-03-20"), |
| 646 | self._make_item("fail1", 2_000, "2026-03-15"), |
| 647 | self._make_item("nocap1", 1_000, "2026-03-10"), |
| 648 | ] |
| 649 | |
| 650 | def fake_parallel(video_ids, max_workers=5, out_captions_disabled=None, token=None): |
| 651 | result = {} |
| 652 | for vid in video_ids: |
| 653 | if vid.startswith("nocap"): |
| 654 | result[vid] = None |
| 655 | if out_captions_disabled is not None: |
| 656 | out_captions_disabled.add(vid) |
| 657 | elif vid.startswith("fail"): |
| 658 | result[vid] = None |
| 659 | else: |
| 660 | result[vid] = "A detailed transcript about the topic." |
| 661 | return result |
| 662 | |
| 663 | self._run(items, fake_parallel) |
| 664 | |
| 665 | stats = youtube_yt.get_transcript_fetch_stats() |
| 666 | self.assertEqual(stats["attempts"], 3) |
| 667 | # Captions-disabled videos can never succeed; they are not failures. |
| 668 | self.assertEqual(stats["failures"], 1) |
| 669 | |
| 670 | def test_fetch_stats_zero_failures_when_all_succeed(self): |
| 671 | # The #531 scenario: every fetch succeeds (on videos later pruned by |
| 672 | # freshness scoring). failures must be 0 so quality_nudge does not |
| 673 | # blame a stale yt-dlp binary. |
| 674 | items = [self._make_item(f"v{i}", 1_000 * (i + 1), "2024-01-15") for i in range(4)] |
| 675 | |
| 676 | self._run(items) |
| 677 | |
| 678 | stats = youtube_yt.get_transcript_fetch_stats() |
| 679 | self.assertEqual(stats["attempts"], 4) |
| 680 | self.assertEqual(stats["failures"], 0) |
| 681 | |
| 682 | |
| 683 | class TestYtdlpSSHRouting(unittest.TestCase): |
| 684 | """LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp invocations through SSH for residential IP.""" |
| 685 | |
| 686 | def setUp(self): |
| 687 | # Ensure clean env for each test |
| 688 | self._saved_env = os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) |
| 689 | youtube_yt.reset_search_cache() |
| 690 | |
| 691 | def tearDown(self): |
| 692 | os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) |
| 693 | if self._saved_env is not None: |
| 694 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = self._saved_env |
| 695 | |
| 696 | def test_no_env_var_returns_none(self): |
| 697 | """Without the env var set, _ytdlp_ssh_host returns None.""" |
| 698 | self.assertIsNone(youtube_yt._ytdlp_ssh_host()) |
| 699 | |
| 700 | def test_env_var_returns_host(self): |
| 701 | """With LAST30DAYS_YOUTUBE_SSH_HOST set, _ytdlp_ssh_host returns it.""" |
| 702 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 703 | self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") |
| 704 | |
| 705 | def test_env_var_whitespace_stripped(self): |
| 706 | """Whitespace around the host alias is stripped.""" |
| 707 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = " macmini " |
| 708 | self.assertEqual(youtube_yt._ytdlp_ssh_host(), "macmini") |
| 709 | |
| 710 | def test_empty_env_var_falls_back_to_none(self): |
| 711 | """An empty env var is treated as unset.""" |
| 712 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "" |
| 713 | self.assertIsNone(youtube_yt._ytdlp_ssh_host()) |
| 714 | |
| 715 | def test_wrap_cmd_passthrough_when_unset(self): |
| 716 | """_wrap_ytdlp_cmd returns input unchanged when SSH routing is off.""" |
| 717 | cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] |
| 718 | self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) |
| 719 | |
| 720 | def test_wrap_cmd_prepends_ssh_when_set(self): |
| 721 | """_wrap_ytdlp_cmd prepends ssh <host> when SSH routing is on.""" |
| 722 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 723 | cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"] |
| 724 | wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) |
| 725 | self.assertEqual(wrapped[0], "ssh") |
| 726 | self.assertEqual(wrapped[1], "-o") |
| 727 | self.assertEqual(wrapped[2], "BatchMode=yes") |
| 728 | # `--` terminates SSH option parsing so a host starting with `-` |
| 729 | # (e.g. `-oProxyCommand=...`) cannot be reinterpreted as a flag. |
| 730 | self.assertEqual(wrapped[3], "--") |
| 731 | self.assertEqual(wrapped[4], "macmini") |
| 732 | # Final arg is the shell-quoted command string |
| 733 | self.assertIn("yt-dlp", wrapped[5]) |
| 734 | self.assertIn("ytsearch5:test", wrapped[5]) |
| 735 | |
| 736 | def test_wrap_cmd_quotes_args_with_spaces(self): |
| 737 | """Args containing spaces or special chars are shell-quoted.""" |
| 738 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 739 | cmd = ["yt-dlp", "ytsearch5:hello world", "--dump-json"] |
| 740 | wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) |
| 741 | # shlex.quote wraps the whole arg in single quotes when it contains spaces |
| 742 | self.assertIn("'ytsearch5:hello world'", wrapped[5]) |
| 743 | |
| 744 | def test_wrap_cmd_uses_option_terminator(self): |
| 745 | """`--` is inserted before host as defense-in-depth even for valid hosts.""" |
| 746 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 747 | cmd = ["yt-dlp", "--version"] |
| 748 | wrapped = youtube_yt._wrap_ytdlp_cmd(cmd) |
| 749 | dash_idx = wrapped.index("--") |
| 750 | self.assertEqual(wrapped[dash_idx + 1], "macmini") |
| 751 | |
| 752 | def test_host_alias_with_dash_prefix_is_rejected(self): |
| 753 | """A host value starting with `-` is rejected by the alias validator. |
| 754 | |
| 755 | Without validation, ssh could parse `-oProxyCommand=...` as a flag |
| 756 | instead of a hostname. The `--` terminator in _wrap_ytdlp_cmd is |
| 757 | defense-in-depth; this regex on _ytdlp_ssh_host() rejects the value |
| 758 | before it ever reaches the ssh command line. |
| 759 | """ |
| 760 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-oProxyCommand=evil" |
| 761 | self.assertIsNone(youtube_yt._ytdlp_ssh_host()) |
| 762 | # And the wrap function falls back to the local-execution path. |
| 763 | cmd = ["yt-dlp", "--version"] |
| 764 | self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd) |
| 765 | |
| 766 | def test_host_alias_with_shell_metacharacters_is_rejected(self): |
| 767 | """Host values containing spaces, semicolons, $, etc. are rejected.""" |
| 768 | for bad in ("host;rm -rf /", "host name", "host$IFS", "host`whoami`", "host&cmd"): |
| 769 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = bad |
| 770 | self.assertIsNone( |
| 771 | youtube_yt._ytdlp_ssh_host(), |
| 772 | msg=f"validator should reject {bad!r}", |
| 773 | ) |
| 774 | |
| 775 | def test_host_alias_validator_accepts_realistic_aliases(self): |
| 776 | """Valid SSH config aliases are accepted: bare names, FQDNs, IPs.""" |
| 777 | for good in ("macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"): |
| 778 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = good |
| 779 | self.assertEqual(youtube_yt._ytdlp_ssh_host(), good) |
| 780 | |
| 781 | def test_is_ytdlp_installed_short_circuits_with_ssh(self): |
| 782 | """is_ytdlp_installed returns True without local check when SSH routing is on.""" |
| 783 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 784 | with mock.patch("lib.youtube_yt.shutil.which", return_value=None) as which_mock: |
| 785 | self.assertTrue(youtube_yt.is_ytdlp_installed()) |
| 786 | which_mock.assert_not_called() |
| 787 | |
| 788 | def test_is_ytdlp_installed_falls_through_without_ssh(self): |
| 789 | """is_ytdlp_installed checks PATH normally when SSH routing is off.""" |
| 790 | with mock.patch("lib.youtube_yt.shutil.which", return_value="/usr/bin/yt-dlp"): |
| 791 | self.assertTrue(youtube_yt.is_ytdlp_installed()) |
| 792 | with mock.patch("lib.youtube_yt.shutil.which", return_value=None): |
| 793 | self.assertFalse(youtube_yt.is_ytdlp_installed()) |
| 794 | |
| 795 | def test_search_call_routes_through_ssh(self): |
| 796 | """search_youtube wraps the yt-dlp invocation when SSH routing is on.""" |
| 797 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 798 | from lib.subproc import SubprocResult |
| 799 | fake_result = SubprocResult(returncode=0, stdout="", stderr="") |
| 800 | with mock.patch.object(youtube_yt.subproc, "run_with_timeout", |
| 801 | return_value=fake_result) as run_mock: |
| 802 | youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01") |
| 803 | cmd = run_mock.call_args.args[0] |
| 804 | self.assertEqual(cmd[0], "ssh") |
| 805 | self.assertEqual(cmd[3], "--") |
| 806 | self.assertEqual(cmd[4], "macmini") |
| 807 | # The shell-quoted yt-dlp invocation lives at index 5 |
| 808 | self.assertIn("yt-dlp", cmd[5]) |
| 809 | self.assertIn("--ignore-config", cmd[5]) |
| 810 | self.assertIn("--no-cookies-from-browser", cmd[5]) |
| 811 | |
| 812 | def test_search_surfaces_ssh_failure_as_error(self): |
| 813 | """SSH connection failures surface as an error, not silent '0 results'.""" |
| 814 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 815 | from lib.subproc import SubprocResult |
| 816 | fake_result = SubprocResult( |
| 817 | returncode=255, |
| 818 | stdout="", |
| 819 | stderr="ssh: connect to host macmini port 22: Connection refused\n", |
| 820 | ) |
| 821 | with mock.patch.object(youtube_yt.subproc, "run_with_timeout", |
| 822 | return_value=fake_result): |
| 823 | out = youtube_yt.search_youtube("test", "2026-02-01", "2026-03-01") |
| 824 | self.assertIn("error", out) |
| 825 | self.assertIn("Connection refused", out["error"]) |
| 826 | |
| 827 | |
| 828 | class TestTranscriptSSHRouting(unittest.TestCase): |
| 829 | """LAST30DAYS_YOUTUBE_SSH_HOST routes yt-dlp transcript fetches through SSH.""" |
| 830 | |
| 831 | def setUp(self): |
| 832 | self._saved_env = os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) |
| 833 | |
| 834 | def tearDown(self): |
| 835 | os.environ.pop("LAST30DAYS_YOUTUBE_SSH_HOST", None) |
| 836 | if self._saved_env is not None: |
| 837 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = self._saved_env |
| 838 | |
| 839 | def test_ssh_helper_invokes_remote_mktemp_pipeline(self): |
| 840 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 841 | from lib.subproc import SubprocResult |
| 842 | fake_vtt = "WEBVTT\nKind: captions\nLanguage: en\n\n00:00:00.000 --> 00:00:02.000\nhi\n" |
| 843 | fake_result = SubprocResult(returncode=0, stdout=fake_vtt, stderr="") |
| 844 | with mock.patch.object(youtube_yt.subproc, "run_with_timeout", |
| 845 | return_value=fake_result) as run_mock: |
| 846 | out = youtube_yt._fetch_transcript_ytdlp_via_ssh("vid1", "macmini") |
| 847 | self.assertEqual(out, fake_vtt) |
| 848 | remote_script = run_mock.call_args.args[0][5] |
| 849 | self.assertIn("mktemp -d", remote_script) |
| 850 | self.assertIn("find ", remote_script) |
| 851 | |
| 852 | def test_fetch_transcript_uses_ssh_helper_when_routing_on(self): |
| 853 | os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "macmini" |
| 854 | fake_vtt = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nhello there friends\n" |
| 855 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 856 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp_via_ssh", |
| 857 | return_value=fake_vtt) as ssh_mock, \ |
| 858 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp") as local_mock, \ |
| 859 | mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock: |
| 860 | result = youtube_yt.fetch_transcript("vidX", "/tmp/test") |
| 861 | ssh_mock.assert_called_once_with("vidX", "macmini") |
| 862 | local_mock.assert_not_called() |
| 863 | direct_mock.assert_not_called() |
| 864 | self.assertIn("hello there friends", result) |
| 865 | |
| 866 | |
| 867 | class TestScTranscriptFallback(unittest.TestCase): |
| 868 | """ScrapeCreators fallback wiring in fetch_transcript (U1/U2).""" |
| 869 | |
| 870 | def _ytdlp_hard_fail(self, reason="HTTP Error 429: Too Many Requests"): |
| 871 | def _fake(video_id, temp_dir, status=None, fast_fail=False): |
| 872 | if status is not None: |
| 873 | status["ytdlp_error"] = reason |
| 874 | return None |
| 875 | return _fake |
| 876 | |
| 877 | def test_sc_fallback_fires_on_ytdlp_hard_failure_with_token(self): |
| 878 | """A yt-dlp hard failure (429) with a key falls back to ScrapeCreators.""" |
| 879 | status = {} |
| 880 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 881 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", |
| 882 | side_effect=self._ytdlp_hard_fail()), \ |
| 883 | mock.patch.object(youtube_yt, "_fetch_transcript_direct") as direct_mock, \ |
| 884 | mock.patch.object(youtube_yt, "_sc_fetch_transcript", |
| 885 | return_value="scrapecreators transcript text") as sc_mock: |
| 886 | result = youtube_yt.fetch_transcript("vidA", "/tmp/x", status=status, token="key123") |
| 887 | sc_mock.assert_called_once_with("vidA", "key123") |
| 888 | direct_mock.assert_not_called() # hard error skips the (also-blocked) direct path |
| 889 | self.assertEqual(result, "scrapecreators transcript text") |
| 890 | |
| 891 | def test_sc_rescue_logged_and_flagged_in_status(self): |
| 892 | """A yt-dlp hard failure rescued by ScrapeCreators must be logged and |
| 893 | flagged via status['sc_rescued'] — not just returned silently — so |
| 894 | fetch_transcripts_parallel() can report the rescue instead of letting |
| 895 | the batch summary read as a clean success (#831).""" |
| 896 | status = {} |
| 897 | logs = [] |
| 898 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 899 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", |
| 900 | side_effect=self._ytdlp_hard_fail()), \ |
| 901 | mock.patch.object(youtube_yt, "_sc_fetch_transcript", |
| 902 | return_value="rescued transcript text"), \ |
| 903 | mock.patch.object(youtube_yt, "_log", side_effect=lambda m: logs.append(m)): |
| 904 | result = youtube_yt.fetch_transcript("vidR", "/tmp/x", status=status, token="key123") |
| 905 | self.assertEqual(result, "rescued transcript text") |
| 906 | self.assertTrue(status.get("sc_rescued")) |
| 907 | self.assertTrue(any("ScrapeCreators" in m and "vidR" in m for m in logs)) |
| 908 | |
| 909 | def test_sc_not_called_when_ytdlp_succeeds(self): |
| 910 | """No credit is spent when yt-dlp returns a transcript.""" |
| 911 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 912 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", |
| 913 | return_value="WEBVTT\n\nreal captions here"), \ |
| 914 | mock.patch.object(youtube_yt, "_sc_fetch_transcript") as sc_mock: |
| 915 | result = youtube_yt.fetch_transcript("vidB", "/tmp/x", status={}, token="key123") |
| 916 | sc_mock.assert_not_called() |
| 917 | self.assertIn("real captions", result) |
| 918 | |
| 919 | def test_sc_not_called_without_token(self): |
| 920 | """Keyless behavior unchanged: no token means no ScrapeCreators.""" |
| 921 | status = {} |
| 922 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 923 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", |
| 924 | side_effect=self._ytdlp_hard_fail()), \ |
| 925 | mock.patch.object(youtube_yt, "_sc_fetch_transcript") as sc_mock: |
| 926 | result = youtube_yt.fetch_transcript("vidC", "/tmp/x", status=status, token=None) |
| 927 | sc_mock.assert_not_called() |
| 928 | self.assertIsNone(result) |
| 929 | |
| 930 | def test_sc_skipped_when_proven_captionless(self): |
| 931 | """A video proven to have no caption track must not spend a credit.""" |
| 932 | def _ytdlp_no_captions(video_id, temp_dir, status=None, fast_fail=False): |
| 933 | return None # exit-0 no captions: returns None, no ytdlp_error |
| 934 | |
| 935 | def _direct_no_tracks(video_id, status=None): |
| 936 | if status is not None: |
| 937 | status["no_caption_tracks"] = True |
| 938 | return None |
| 939 | |
| 940 | status = {} |
| 941 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 942 | mock.patch.object(youtube_yt, "_fetch_transcript_ytdlp", side_effect=_ytdlp_no_captions), \ |
| 943 | mock.patch.object(youtube_yt, "_fetch_transcript_direct", side_effect=_direct_no_tracks), \ |
| 944 | mock.patch.object(youtube_yt, "_sc_fetch_transcript") as sc_mock: |
| 945 | result = youtube_yt.fetch_transcript("vidD", "/tmp/x", status=status, token="key123") |
| 946 | sc_mock.assert_not_called() |
| 947 | self.assertIsNone(result) |
| 948 | |
| 949 | def test_should_try_sc_transcript_predicate(self): |
| 950 | self.assertTrue(youtube_yt._should_try_sc_transcript(None)) |
| 951 | self.assertTrue(youtube_yt._should_try_sc_transcript({})) |
| 952 | self.assertTrue(youtube_yt._should_try_sc_transcript({"ytdlp_error": "429"})) |
| 953 | self.assertFalse(youtube_yt._should_try_sc_transcript({"no_caption_tracks": True})) |
| 954 | |
| 955 | def test_token_threads_through_parallel(self): |
| 956 | """fetch_transcripts_parallel passes the token to every fetch_transcript.""" |
| 957 | captured = {} |
| 958 | |
| 959 | def _fake_fetch_transcript(video_id, temp_dir, status=None, token=None): |
| 960 | captured[video_id] = token |
| 961 | return None |
| 962 | |
| 963 | with mock.patch.object(youtube_yt, "fetch_transcript", side_effect=_fake_fetch_transcript): |
| 964 | youtube_yt.fetch_transcripts_parallel(["v1", "v2"], token="tok") |
| 965 | self.assertEqual(captured, {"v1": "tok", "v2": "tok"}) |
| 966 | |
| 967 | def test_summary_reports_sc_rescue_not_bare_success(self): |
| 968 | """Regression for #831: when every video's yt-dlp fetch fails and the |
| 969 | ScrapeCreators fallback rescues all of them, the batch summary must |
| 970 | not read as a bare "0 failed" success — it has to say the videos |
| 971 | were rescued via the fallback, so a fully rate-limited yt-dlp run |
| 972 | doesn't look like nothing went wrong.""" |
| 973 | logs = [] |
| 974 | |
| 975 | def _rescued_fetch_transcript(video_id, temp_dir, status=None, token=None): |
| 976 | if status is not None: |
| 977 | status["sc_rescued"] = True |
| 978 | return "rescued transcript" |
| 979 | |
| 980 | with mock.patch.object(youtube_yt, "fetch_transcript", |
| 981 | side_effect=_rescued_fetch_transcript), \ |
| 982 | mock.patch.object(youtube_yt, "_log", side_effect=lambda m: logs.append(m)): |
| 983 | results = youtube_yt.fetch_transcripts_parallel(["v1", "v2"], token="tok") |
| 984 | |
| 985 | self.assertEqual(results, {"v1": "rescued transcript", "v2": "rescued transcript"}) |
| 986 | summary = next(m for m in logs if m.startswith("Got transcripts for")) |
| 987 | self.assertIn("2/2", summary) |
| 988 | self.assertIn("0 failed", summary) |
| 989 | self.assertIn("2 rescued via ScrapeCreators fallback", summary) |
| 990 | |
| 991 | def test_summary_omits_rescue_note_when_no_fallback_used(self): |
| 992 | """The plain 'N/N (M failed)' format must be unchanged when no video |
| 993 | needed the ScrapeCreators fallback — no rescue tag should appear.""" |
| 994 | logs = [] |
| 995 | |
| 996 | def _plain_fetch_transcript(video_id, temp_dir, status=None, token=None): |
| 997 | return "a normal transcript" |
| 998 | |
| 999 | with mock.patch.object(youtube_yt, "fetch_transcript", |
| 1000 | side_effect=_plain_fetch_transcript), \ |
| 1001 | mock.patch.object(youtube_yt, "_log", side_effect=lambda m: logs.append(m)): |
| 1002 | youtube_yt.fetch_transcripts_parallel(["v1", "v2", "v3"], token="tok") |
| 1003 | |
| 1004 | summary = next(m for m in logs if m.startswith("Got transcripts for")) |
| 1005 | self.assertEqual(summary, "Got transcripts for 3/3 videos (0 failed)") |
| 1006 | |
| 1007 | |
| 1008 | class TestYtdlpFastFail(unittest.TestCase): |
| 1009 | """Fail-fast behavior when a ScrapeCreators key is present (U3).""" |
| 1010 | |
| 1011 | def _transient_fail(self): |
| 1012 | from lib.subproc import SubprocResult |
| 1013 | return SubprocResult( |
| 1014 | returncode=1, stdout="", |
| 1015 | stderr="ERROR: HTTP Error 429: Too Many Requests", |
| 1016 | ) |
| 1017 | |
| 1018 | def test_fast_fail_single_attempt_short_timeout(self): |
| 1019 | """token present -> one attempt, shortened timeout, no retry sleeps.""" |
| 1020 | with tempfile.TemporaryDirectory() as temp_dir: |
| 1021 | status = {} |
| 1022 | with mock.patch.dict(os.environ, {"LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT": ""}), \ |
| 1023 | mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1024 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", |
| 1025 | return_value=self._transient_fail()) as run_mock, \ |
| 1026 | mock.patch.object(youtube_yt.time, "sleep") as sleep_mock: |
| 1027 | vtt = youtube_yt._fetch_transcript_ytdlp("vidF", temp_dir, status, fast_fail=True) |
| 1028 | self.assertIsNone(vtt) |
| 1029 | self.assertEqual(run_mock.call_count, 1) # no retries |
| 1030 | self.assertEqual(run_mock.call_args.kwargs.get("timeout"), 12) |
| 1031 | sleep_mock.assert_not_called() |
| 1032 | self.assertIn("ytdlp_error", status) |
| 1033 | |
| 1034 | def test_no_token_retries_with_full_timeout(self): |
| 1035 | """token absent -> full retry budget and 30s timeout, unchanged.""" |
| 1036 | with tempfile.TemporaryDirectory() as temp_dir: |
| 1037 | status = {} |
| 1038 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1039 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", |
| 1040 | return_value=self._transient_fail()) as run_mock, \ |
| 1041 | mock.patch.object(youtube_yt.time, "sleep"): |
| 1042 | vtt = youtube_yt._fetch_transcript_ytdlp("vidG", temp_dir, status, fast_fail=False) |
| 1043 | self.assertIsNone(vtt) |
| 1044 | self.assertEqual(run_mock.call_count, youtube_yt._TRANSCRIPT_MAX_RETRIES + 1) |
| 1045 | self.assertEqual(run_mock.call_args.kwargs.get("timeout"), 30) |
| 1046 | |
| 1047 | |
| 1048 | class TestScTranscriptParsing(unittest.TestCase): |
| 1049 | """ScrapeCreators transcript parse + credits warning (U4).""" |
| 1050 | |
| 1051 | def test_list_of_dict_segments_parsed_to_text(self): |
| 1052 | payload = { |
| 1053 | "transcript": [ |
| 1054 | {"text": "hello there", "startMs": 0, "endMs": 1000}, |
| 1055 | {"text": "general kenobi", "startMs": 1000, "endMs": 2000}, |
| 1056 | ], |
| 1057 | "credits_remaining": 9999, |
| 1058 | } |
| 1059 | with mock.patch.object(youtube_yt.http, "get", return_value=payload): |
| 1060 | result = youtube_yt._sc_fetch_transcript("vidH", "key") |
| 1061 | self.assertIsNotNone(result) |
| 1062 | self.assertIn("hello there", result) |
| 1063 | self.assertIn("general kenobi", result) |
| 1064 | self.assertNotIn("startMs", result) |
| 1065 | self.assertNotIn("{'text'", result) |
| 1066 | |
| 1067 | def test_null_text_segment_does_not_emit_none(self): |
| 1068 | """A present-but-null text field (silent/music segment) must not become "None".""" |
| 1069 | payload = { |
| 1070 | "transcript": [ |
| 1071 | {"text": "real words here", "startMs": 0}, |
| 1072 | {"text": None, "startMs": 1000}, |
| 1073 | {"text": "more real words", "startMs": 2000}, |
| 1074 | ], |
| 1075 | "credits_remaining": 9999, |
| 1076 | } |
| 1077 | with mock.patch.object(youtube_yt.http, "get", return_value=payload): |
| 1078 | result = youtube_yt._sc_fetch_transcript("vidNull", "key") |
| 1079 | self.assertIsNotNone(result) |
| 1080 | self.assertNotIn("None", result) |
| 1081 | self.assertIn("real words here", result) |
| 1082 | self.assertIn("more real words", result) |
| 1083 | |
| 1084 | def test_plain_string_transcript_preserved(self): |
| 1085 | payload = {"transcript": "just a plain transcript string here", "credits_remaining": 9999} |
| 1086 | with mock.patch.object(youtube_yt.http, "get", return_value=payload): |
| 1087 | result = youtube_yt._sc_fetch_transcript("vidI", "key") |
| 1088 | self.assertIn("just a plain transcript", result) |
| 1089 | |
| 1090 | def test_low_credits_emits_warning(self): |
| 1091 | payload = {"transcript": "some transcript text", "credits_remaining": 5} |
| 1092 | logs = [] |
| 1093 | with mock.patch.object(youtube_yt.http, "get", return_value=payload), \ |
| 1094 | mock.patch.object(youtube_yt, "_log", side_effect=lambda m: logs.append(m)): |
| 1095 | youtube_yt._sc_fetch_transcript("vidJ", "key") |
| 1096 | self.assertTrue(any("credits low" in m.lower() for m in logs)) |
| 1097 | |
| 1098 | def test_healthy_credits_no_warning(self): |
| 1099 | payload = {"transcript": "some transcript text", "credits_remaining": 9999} |
| 1100 | logs = [] |
| 1101 | with mock.patch.object(youtube_yt.http, "get", return_value=payload), \ |
| 1102 | mock.patch.object(youtube_yt, "_log", side_effect=lambda m: logs.append(m)): |
| 1103 | youtube_yt._sc_fetch_transcript("vidK", "key") |
| 1104 | self.assertFalse(any("credits low" in m.lower() for m in logs)) |
| 1105 | |
| 1106 | |
| 1107 | class TestYoutubeCommentsGating(unittest.TestCase): |
| 1108 | """The legacy ScrapeCreators comment path, which applies only when yt-dlp |
| 1109 | is absent. With yt-dlp installed, comments are free and need no opt-in — |
| 1110 | see tests/test_youtube_comments_ytdlp.py.""" |
| 1111 | |
| 1112 | def test_off_with_key_and_no_include_sources(self): |
| 1113 | """SC path: key without INCLUDE_SOURCES does NOT fetch comments.""" |
| 1114 | from lib import env |
| 1115 | with mock.patch.object(env, "is_ytdlp_available", return_value=False): |
| 1116 | self.assertFalse(env.is_youtube_comments_available({"SCRAPECREATORS_API_KEY": "k"})) |
| 1117 | |
| 1118 | def test_on_with_include_sources(self): |
| 1119 | from lib import env |
| 1120 | cfg = {"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "youtube_comments"} |
| 1121 | with mock.patch.object(env, "is_ytdlp_available", return_value=False): |
| 1122 | self.assertTrue(env.is_youtube_comments_available(cfg)) |
| 1123 | |
| 1124 | def test_unavailable_without_key(self): |
| 1125 | """SC path: no key and no yt-dlp means no comments at all.""" |
| 1126 | from lib import env |
| 1127 | with mock.patch.object(env, "is_ytdlp_available", return_value=False): |
| 1128 | self.assertFalse(env.is_youtube_comments_available({"INCLUDE_SOURCES": "youtube_comments"})) |
| 1129 | |
| 1130 | def test_tiktok_comments_still_opt_in(self): |
| 1131 | """Regression: TikTok comments must STILL require INCLUDE_SOURCES.""" |
| 1132 | from lib import env |
| 1133 | self.assertFalse( |
| 1134 | env.is_tiktok_comments_available({"SCRAPECREATORS_API_KEY": "k"}) |
| 1135 | ) |
| 1136 | self.assertTrue(env.is_tiktok_comments_available( |
| 1137 | {"SCRAPECREATORS_API_KEY": "k", "INCLUDE_SOURCES": "tiktok_comments"} |
| 1138 | )) |
| 1139 | |
| 1140 | |
| 1141 | class TestYouTubeSearchTimeoutAndCache(unittest.TestCase): |
| 1142 | """Comparison-mode load: timeouts, status honesty, and in-run dedup.""" |
| 1143 | |
| 1144 | def setUp(self): |
| 1145 | youtube_yt.reset_search_cache() |
| 1146 | |
| 1147 | def _fake_result(self, stdout: str = "", returncode: int = 0): |
| 1148 | from lib.subproc import SubprocResult |
| 1149 | return SubprocResult(returncode=returncode, stdout=stdout, stderr="") |
| 1150 | |
| 1151 | def test_search_timeout_reports_timeout_error_not_empty(self): |
| 1152 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1153 | mock.patch.dict(os.environ, {"LAST30DAYS_YT_SEARCH_TIMEOUT": "1"}), \ |
| 1154 | mock.patch.object( |
| 1155 | youtube_yt.subproc, "run_with_timeout", |
| 1156 | side_effect=youtube_yt.subproc.SubprocTimeout("boom"), |
| 1157 | ): |
| 1158 | out = youtube_yt.search_youtube("Vuori", "2026-06-01", "2026-07-01") |
| 1159 | self.assertEqual(out.get("items"), []) |
| 1160 | self.assertIn("timed out", (out.get("error") or "").lower()) |
| 1161 | self.assertEqual( |
| 1162 | youtube_yt.classify_run_failure(out["error"]), |
| 1163 | youtube_yt.health.TIMEOUT, |
| 1164 | ) |
| 1165 | |
| 1166 | def test_search_timeout_env_is_passed_to_subprocess(self): |
| 1167 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1168 | mock.patch.dict(os.environ, {"LAST30DAYS_YT_SEARCH_TIMEOUT": "7"}), \ |
| 1169 | mock.patch.object( |
| 1170 | youtube_yt.subproc, "run_with_timeout", |
| 1171 | return_value=self._fake_result(), |
| 1172 | ) as run_mock: |
| 1173 | youtube_yt.search_youtube("Alo Yoga", "2026-06-01", "2026-07-01") |
| 1174 | self.assertEqual(run_mock.call_args.kwargs.get("timeout"), 7.0) |
| 1175 | |
| 1176 | def test_identical_searches_are_cached_within_run(self): |
| 1177 | video = { |
| 1178 | "id": "abc123", |
| 1179 | "title": "Vuori review", |
| 1180 | "view_count": 100, |
| 1181 | "like_count": 1, |
| 1182 | "comment_count": 0, |
| 1183 | "upload_date": "20260615", |
| 1184 | "description": "desc", |
| 1185 | "channel": "Tester", |
| 1186 | } |
| 1187 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1188 | mock.patch.object( |
| 1189 | youtube_yt.subproc, "run_with_timeout", |
| 1190 | return_value=self._fake_result(stdout=json.dumps(video) + "\n"), |
| 1191 | ) as run_mock: |
| 1192 | first = youtube_yt.search_youtube("Vuori", "2026-06-01", "2026-07-01") |
| 1193 | second = youtube_yt.search_youtube("Vuori", "2026-06-01", "2026-07-01") |
| 1194 | self.assertEqual(run_mock.call_count, 1) |
| 1195 | self.assertEqual(len(first["items"]), 1) |
| 1196 | self.assertEqual(len(second["items"]), 1) |
| 1197 | self.assertEqual(first["items"][0]["video_id"], second["items"][0]["video_id"]) |
| 1198 | # Callers get independent copies so mutations cannot poison the cache. |
| 1199 | second["items"][0]["title"] = "mutated" |
| 1200 | self.assertNotEqual(first["items"][0]["title"], "mutated") |
| 1201 | |
| 1202 | def test_timeout_errors_are_not_cached(self): |
| 1203 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1204 | mock.patch.object( |
| 1205 | youtube_yt.subproc, "run_with_timeout", |
| 1206 | side_effect=youtube_yt.subproc.SubprocTimeout("boom"), |
| 1207 | ) as run_mock: |
| 1208 | youtube_yt.search_youtube("lululemon", "2026-06-01", "2026-07-01") |
| 1209 | youtube_yt.search_youtube("lululemon", "2026-06-01", "2026-07-01") |
| 1210 | self.assertEqual(run_mock.call_count, 2) |
| 1211 | |
| 1212 | def test_waiter_receives_leader_result_without_synthetic_timeout(self): |
| 1213 | """A coalesced waiter must not invent a timeout while the leader runs.""" |
| 1214 | import threading |
| 1215 | from lib.subproc import SubprocResult |
| 1216 | |
| 1217 | video = { |
| 1218 | "id": "waiter1", |
| 1219 | "title": "Vuori review", |
| 1220 | "view_count": 10, |
| 1221 | "like_count": 1, |
| 1222 | "comment_count": 0, |
| 1223 | "upload_date": "20260615", |
| 1224 | "description": "desc", |
| 1225 | "channel": "Tester", |
| 1226 | } |
| 1227 | started = threading.Event() |
| 1228 | release = threading.Event() |
| 1229 | |
| 1230 | def slow_run(cmd, timeout=None): |
| 1231 | started.set() |
| 1232 | release.wait(timeout=5) |
| 1233 | return SubprocResult( |
| 1234 | returncode=0, stdout=json.dumps(video) + "\n", stderr="", |
| 1235 | ) |
| 1236 | |
| 1237 | results = [] |
| 1238 | |
| 1239 | def leader(): |
| 1240 | results.append( |
| 1241 | youtube_yt.search_youtube("Vuori", "2026-06-01", "2026-07-01") |
| 1242 | ) |
| 1243 | |
| 1244 | def waiter(): |
| 1245 | started.wait(timeout=5) |
| 1246 | results.append( |
| 1247 | youtube_yt.search_youtube("Vuori", "2026-06-01", "2026-07-01") |
| 1248 | ) |
| 1249 | |
| 1250 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 1251 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", side_effect=slow_run): |
| 1252 | t_leader = threading.Thread(target=leader) |
| 1253 | t_waiter = threading.Thread(target=waiter) |
| 1254 | t_leader.start() |
| 1255 | self.assertTrue(started.wait(timeout=5)) |
| 1256 | t_waiter.start() |
| 1257 | release.set() |
| 1258 | t_leader.join(timeout=5) |
| 1259 | t_waiter.join(timeout=5) |
| 1260 | |
| 1261 | self.assertEqual(len(results), 2) |
| 1262 | self.assertTrue(all(r.get("items") for r in results)) |
| 1263 | self.assertTrue(all(not r.get("error") for r in results)) |
| 1264 | |
| 1265 | def test_stale_leader_after_reset_does_not_pop_newer_inflight(self): |
| 1266 | """A leader that outlives reset_search_cache must not drop the new slot.""" |
| 1267 | import threading |
| 1268 | |
| 1269 | key = ("ownership", 8, "2026-06-01") |
| 1270 | old_event = threading.Event() |
| 1271 | old_slot: list = [None] |
| 1272 | new_event = threading.Event() |
| 1273 | new_slot: list = [None] |
| 1274 | |
| 1275 | with youtube_yt._search_cache_lock: |
| 1276 | youtube_yt._search_inflight[key] = (old_event, old_slot) |
| 1277 | |
| 1278 | youtube_yt.reset_search_cache() |
| 1279 | with youtube_yt._search_cache_lock: |
| 1280 | youtube_yt._search_inflight[key] = (new_event, new_slot) |
| 1281 | |
| 1282 | youtube_yt._finish_search_slot( |
| 1283 | key, |
| 1284 | {"items": [{"video_id": "old"}]}, |
| 1285 | event=old_event, |
| 1286 | slot=old_slot, |
| 1287 | ) |
| 1288 | |
| 1289 | with youtube_yt._search_cache_lock: |
| 1290 | current = youtube_yt._search_inflight.get(key) |
| 1291 | cached = youtube_yt._search_cache.get(key) |
| 1292 | |
| 1293 | self.assertIsNotNone(current) |
| 1294 | self.assertIs(current[1], new_slot) |
| 1295 | self.assertIsNone(cached) |
| 1296 | self.assertTrue(old_event.is_set()) |
| 1297 | self.assertEqual(old_slot[0]["items"][0]["video_id"], "old") |
| 1298 | |
| 1299 | def test_multi_query_preserves_timeout_over_later_empty(self): |
| 1300 | responses = [ |
| 1301 | {"items": [], "error": "Search timed out after 1s"}, |
| 1302 | {"items": []}, |
| 1303 | ] |
| 1304 | |
| 1305 | def fake_search(*_args, **_kwargs): |
| 1306 | return responses.pop(0) |
| 1307 | |
| 1308 | with mock.patch.object(youtube_yt, "expand_youtube_queries", return_value=["a", "b"]), \ |
| 1309 | mock.patch.object(youtube_yt, "search_youtube", side_effect=fake_search): |
| 1310 | out = youtube_yt.search_and_transcribe( |
| 1311 | "topic", "2026-06-01", "2026-07-01", depth="default", |
| 1312 | ) |
| 1313 | self.assertEqual(out.get("items"), []) |
| 1314 | self.assertIn("timed out", (out.get("error") or "").lower()) |
| 1315 | self.assertEqual( |
| 1316 | youtube_yt.classify_run_failure(out["error"]), |
| 1317 | youtube_yt.health.TIMEOUT, |
| 1318 | ) |
| 1319 | |
| 1320 | def test_bundle_records_timeout_not_no_results(self): |
| 1321 | from lib import health, schema |
| 1322 | |
| 1323 | bundle = schema.RetrievalBundle() |
| 1324 | bundle.mark_attempted("youtube") |
| 1325 | state = youtube_yt.classify_run_failure("Search timed out after 1s") |
| 1326 | bundle.record_failure("youtube", state, "Search timed out after 1s") |
| 1327 | bundle.add_items("main", "youtube", []) |
| 1328 | self.assertEqual(bundle.source_status["youtube"].state, health.TIMEOUT) |
| 1329 | self.assertNotEqual(bundle.source_status["youtube"].state, schema.NO_RESULTS) |
| 1330 | |
| 1331 | |
| 1332 | if __name__ == "__main__": |
| 1333 | unittest.main() |
| 1334 |