返回 last30days-skill
test_web_search_keyless.py
根目录 / tests / test_web_search_keyless.py
1 """Tests for scripts/lib/web_search_keyless.py — keyless web search floor."""
2
3 from unittest import mock
4
5 from lib import web_search_keyless
6
7 _DDG_HTML = """
8 <div class="result">
9 <a rel="nofollow" class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpost&amp;rut=x">First &amp; Best</a>
10 <a class="result__snippet" href="//x">A snippet about the <b>topic</b>.</a>
11 </div>
12 <div class="result">
13 <a rel="nofollow" class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fnews.example.org%2Fa">Second result</a>
14 <a class="result__snippet" href="//y">Second snippet.</a>
15 </div>
16 """
17
18
19 class TestKeylessSearch:
20 def test_ddg_parsing_happy_path(self):
21 with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):
22 items, artifact = web_search_keyless.keyless_search(
23 "topic", ("2026-02-25", "2026-03-27"), {})
24 assert len(items) == 2
25 assert items[0]["url"] == "https://example.com/post"
26 assert items[0]["title"] == "First & Best"
27 assert items[0]["source_domain"] == "example.com"
28 assert "snippet about the topic" in items[0]["snippet"]
29 assert artifact["keyless_backend"] == "ddg"
30 # Floor relevance is below the paid backends' 0.8.
31 assert items[0]["relevance"] < 0.8
32
33 def test_item_shape_matches_grounding_backends(self):
34 with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):
35 items, _ = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {})
36 required = {"id", "title", "url", "source_domain", "snippet", "date", "relevance", "why_relevant"}
37 assert required.issubset(items[0].keys())
38
39 def test_count_cap(self):
40 with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):
41 items, _ = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {}, count=1)
42 assert len(items) == 1
43
44 def test_ddg_down_no_searxng_returns_degraded(self):
45 with mock.patch.object(web_search_keyless.http, "get_text", return_value=None):
46 items, artifact = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {})
47 assert items == []
48 assert artifact["reason"] == "keyless-search-unavailable"
49
50 def test_searxng_fallback_when_ddg_empty(self):
51 searxng_payload = {"results": [
52 {"url": "https://searxng.example/a", "title": "SX A", "content": "sx snippet"},
53 ]}
54 with mock.patch.object(web_search_keyless.http, "get_text", return_value=None), \
55 mock.patch.object(web_search_keyless.http, "get", return_value=searxng_payload):
56 items, artifact = web_search_keyless.keyless_search(
57 "topic", ("2026-02-25", "2026-03-27"),
58 {"LAST30DAYS_SEARXNG_URL": "https://searxng.example"})
59 assert len(items) == 1
60 assert items[0]["url"] == "https://searxng.example/a"
61 assert artifact["keyless_backend"] == "searxng"
62
63 def test_skips_non_http_results(self):
64 html = '<a class="result__a" href="//duckduckgo.com/l/?uddg=javascript%3Avoid(0)">bad</a>'
65 with mock.patch.object(web_search_keyless.http, "get_text", return_value=html):
66 items, _ = web_search_keyless.keyless_search("topic", ("2026-02-25", "2026-03-27"), {})
67 assert items == []
68
69
70 # Startpage marks each hit with an <a class="result-title result-link …"> whose
71 # href is the target and whose <h2> is the title; the emotion <style> block
72 # (inline CSS) must never leak into the parsed title. Description follows in a
73 # <p class="…description…">.
74 _STARTPAGE_HTML = """
75 <a class="result-title result-link css-1bggj8v" href="https://example.com/post" data-testid="gl-title-link">
76 <style data-emotion="css i3irj7">.css-i3irj7{line-height:18px;color:#2E39B3;}</style>
77 <h2 class="wgl-title css-i3irj7">First &amp; Best Result</h2>
78 </a>
79 <p class="description css-abc">A snippet about the topic.</p>
80 <a class="result-title result-link css-1bggj8v" href="https://news.example.org/a" data-testid="gl-title-link">
81 <h2 class="wgl-title css-i3irj7">Second result</h2>
82 </a>
83 <p class="description css-abc">Second snippet.</p>
84 """
85
86
87 class TestStartpageFallback:
88 def _get_text(self, startpage_html):
89 # DuckDuckGo yields nothing (its datacenter-IP 202 challenge page has no
90 # result anchors); Startpage yields real results.
91 def side_effect(url, *args, **kwargs):
92 return startpage_html if "startpage.com" in url else ""
93 return side_effect
94
95 def test_startpage_used_when_ddg_empty(self):
96 with mock.patch.object(
97 web_search_keyless.http, "get_text",
98 side_effect=self._get_text(_STARTPAGE_HTML),
99 ):
100 items, artifact = web_search_keyless.keyless_search(
101 "topic", ("2026-02-25", "2026-03-27"), {})
102 assert artifact["keyless_backend"] == "startpage"
103 assert artifact.get("reason") is None
104 assert len(items) == 2
105 assert items[0]["url"] == "https://example.com/post"
106 # The inline <style> CSS must not bleed into the title.
107 assert items[0]["title"] == "First & Best Result"
108 assert "css-" not in items[0]["title"]
109 assert items[0]["snippet"] == "A snippet about the topic."
110 assert items[0]["source_domain"] == "example.com"
111
112 def test_ddg_preferred_over_startpage(self):
113 # When DuckDuckGo returns results, Startpage is not consulted.
114 with mock.patch.object(web_search_keyless.http, "get_text", return_value=_DDG_HTML):
115 _, artifact = web_search_keyless.keyless_search(
116 "topic", ("2026-02-25", "2026-03-27"), {})
117 assert artifact["keyless_backend"] == "ddg"
118
119
120 class TestStripHtml:
121 def test_strips_style_block_contents(self):
122 got = web_search_keyless._strip_html(
123 "<style>.a{color:red}</style><h2>Title</h2>")
124 assert got == "Title"
125
125 lines PYTHON