| 1 | """CommentsCollector 测试。""" |
| 2 | |
| 3 | import json |
| 4 | from typing import Any, Dict, List |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from core.comments_collector import CommentsCollector |
| 9 | from storage.metadata_handler import MetadataHandler |
| 10 | |
| 11 | |
| 12 | class _FakeAPIClient: |
| 13 | def __init__(self, pages: List[Dict[str, Any]]): |
| 14 | self._pages = list(pages) |
| 15 | self.call_count = 0 |
| 16 | |
| 17 | async def get_aweme_comments(self, aweme_id, *, cursor, count, include_replies): |
| 18 | self.call_count += 1 |
| 19 | if not self._pages: |
| 20 | return {"items": [], "has_more": False, "max_cursor": cursor} |
| 21 | return self._pages.pop(0) |
| 22 | |
| 23 | |
| 24 | @pytest.mark.asyncio |
| 25 | async def test_collector_paginates_until_no_more(tmp_path): |
| 26 | api = _FakeAPIClient( |
| 27 | [ |
| 28 | { |
| 29 | "items": [{"cid": "1", "text": "a"}, {"cid": "2", "text": "b"}], |
| 30 | "has_more": True, |
| 31 | "max_cursor": 10, |
| 32 | }, |
| 33 | { |
| 34 | "items": [{"cid": "3", "text": "c"}], |
| 35 | "has_more": False, |
| 36 | "max_cursor": 20, |
| 37 | }, |
| 38 | ] |
| 39 | ) |
| 40 | collector = CommentsCollector(api, MetadataHandler()) |
| 41 | out = tmp_path / "out.json" |
| 42 | payload = await collector.collect_and_save("A1", out) |
| 43 | assert payload is not None |
| 44 | assert payload["count"] == 3 |
| 45 | assert out.exists() |
| 46 | data = json.loads(out.read_text(encoding="utf-8")) |
| 47 | assert data["aweme_id"] == "A1" |
| 48 | assert len(data["comments"]) == 3 |
| 49 | |
| 50 | |
| 51 | @pytest.mark.asyncio |
| 52 | async def test_collector_respects_max_comments(tmp_path): |
| 53 | api = _FakeAPIClient( |
| 54 | [ |
| 55 | { |
| 56 | "items": [{"cid": str(i)} for i in range(5)], |
| 57 | "has_more": True, |
| 58 | "max_cursor": 5, |
| 59 | }, |
| 60 | { |
| 61 | "items": [{"cid": str(i)} for i in range(5, 10)], |
| 62 | "has_more": False, |
| 63 | "max_cursor": 10, |
| 64 | }, |
| 65 | ] |
| 66 | ) |
| 67 | collector = CommentsCollector(api, MetadataHandler(), max_comments=3) |
| 68 | out = tmp_path / "out.json" |
| 69 | payload = await collector.collect_and_save("B1", out) |
| 70 | assert payload is not None |
| 71 | assert payload["count"] == 3 |
| 72 | |
| 73 | |
| 74 | @pytest.mark.asyncio |
| 75 | async def test_collector_deduplicates_by_cid(tmp_path): |
| 76 | api = _FakeAPIClient( |
| 77 | [ |
| 78 | { |
| 79 | "items": [{"cid": "1"}, {"cid": "2"}, {"cid": "1"}], |
| 80 | "has_more": False, |
| 81 | "max_cursor": 3, |
| 82 | } |
| 83 | ] |
| 84 | ) |
| 85 | collector = CommentsCollector(api, MetadataHandler()) |
| 86 | out = tmp_path / "out.json" |
| 87 | payload = await collector.collect_and_save("C1", out) |
| 88 | assert payload is not None |
| 89 | cids = [c["cid"] for c in payload["comments"]] |
| 90 | assert cids == ["1", "2"] |
| 91 | |
| 92 | |
| 93 | @pytest.mark.asyncio |
| 94 | async def test_collector_stops_when_cursor_stuck(tmp_path): |
| 95 | # 模拟 cursor 一直未推进、has_more=True 的病态场景，防止死循环。 |
| 96 | same_cursor_page = { |
| 97 | "items": [{"cid": "1"}], |
| 98 | "has_more": True, |
| 99 | "max_cursor": 0, |
| 100 | } |
| 101 | api = _FakeAPIClient([same_cursor_page] * 10) |
| 102 | collector = CommentsCollector(api, MetadataHandler()) |
| 103 | out = tmp_path / "out.json" |
| 104 | payload = await collector.collect_and_save("D1", out) |
| 105 | assert payload is not None |
| 106 | # 第一页后 cursor 未推进，应立即停止 |
| 107 | assert api.call_count == 1 |
| 108 | |
| 109 | |
| 110 | @pytest.mark.asyncio |
| 111 | async def test_collector_returns_none_on_api_error(tmp_path): |
| 112 | class _FlakyAPI: |
| 113 | async def get_aweme_comments(self, *args, **kwargs): |
| 114 | raise RuntimeError("boom") |
| 115 | |
| 116 | collector = CommentsCollector(_FlakyAPI(), MetadataHandler()) |
| 117 | out = tmp_path / "out.json" |
| 118 | payload = await collector.collect_and_save("E1", out) |
| 119 | assert payload is None |
| 120 | assert not out.exists() |
| 121 |