web_search.rs - CodeWhale

web_search.rs

1	//! Web search tool backed by a bounded backend chain: the active route's
2	//! documented provider-native search, configured API search, then DuckDuckGo
3	//! HTML scrape with a one-way Bing fallback. Explicit Bing and private
4	//! DuckDuckGo-compatible routes remain single-provider. API adapters
5	//! include Tavily, Bocha (博查),
6	//! Metaso API (<https://metaso.cn>), SearXNG JSON API, Baidu AI Search,
7	//! Volcengine Ark, and Sofya (<https://sofya.co>).
8	//!
9	//! This is the primary web search surface for agents. For browsing workflows
10	//! (page open, click, screenshot) use a direct URL approach instead.
11	//!
12	//! Set `[search]` in config.toml to switch providers:
13	//! provider = "duckduckgo" # or tavily/bocha/metaso/searxng/baidu/volcengine/sofya
14	//! base_url = `"https://search.example/"` # DDG-compatible URL or SearXNG instance
15	//! api_key = "tvly-..."
16
17	use super::spec::{
18	ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64,
19	};
20	use crate::config::SearchProvider;
21	use crate::network_policy::{Decision, NetworkPolicyDecider};
22	use async_trait::async_trait;
23	use regex::Regex;
24	use serde::Serialize;
25	use serde_json::{Value, json};
26	use std::sync::OnceLock;
27	use std::time::{Duration, Instant};
28
29	use super::web::backend::SearchBackendChain;
30	use super::web::cache;
31	use super::web::contract::{
32	BackendId, BackendSearch, DEFAULT_SEARCH_RESULTS, DEFAULT_SEARCH_TIMEOUT_MS, DegradedReason,
33	HonoredQueryCapabilities, MAX_SEARCH_RESULTS, MAX_SEARCH_TIMEOUT_MS, QueryKnob, Recency,
34	SearchQuery, SearchReceipt, SearchResponse, SearchResult,
35	};
36	use super::web::scrape::{
37	BROWSER_USER_AGENT as USER_AGENT, ScrapedSearchResult, is_duckduckgo_challenge,
38	parse_bing_results as scrape_bing_results,
39	parse_duckduckgo_results as scrape_duckduckgo_results,
40	};
41
42	const DUCKDUCKGO_ENDPOINT: &str = "https://html.duckduckgo.com/html/";
43	const BING_HOST: &str = "www.bing.com";
44	const BING_ENDPOINT: &str = "https://www.bing.com/search";
45	const TAVILY_ENDPOINT: &str = "https://api.tavily.com/search";
46	const BOCHA_ENDPOINT: &str = "https://api.bochaai.com/v1/web-search";
47	const METASO_ENDPOINT: &str = "https://metaso.cn/api/v1";
48	const BAIDU_ENDPOINT: &str = "https://qianfan.baidubce.com/v2/ai_search/web_search";
49	const VOLCENGINE_RESPONSES_ENDPOINT: &str = "https://ark.cn-beijing.volces.com/api/v3/responses";
50	const SOFYA_ENDPOINT: &str = "https://sofya.co/v1/search";
51	const ERROR_BODY_PREVIEW_BYTES: usize = 512;
52	const VOLCENGINE_MIN_TIMEOUT_MS: u64 = 90_000;
53
54	/// Returns `Ok(())` if the policy allows the call, or a `ToolError` otherwise.
55	/// Falls through silently when no policy is attached (back-compat).
56	pub(crate) fn check_policy(
57	decider: Option<&NetworkPolicyDecider>,
58	host: &str,
59	) -> Result<(), ToolError> {
60	let Some(decider) = decider else {
61	return Ok(());
62	};
63	match decider.evaluate(host, "web_search") {
64	Decision::Allow => Ok(()),
65	Decision::Deny => Err(ToolError::permission_denied(format!(
66	"web search to '{host}' blocked by network policy"
67	))),
68	Decision::Prompt => Err(ToolError::permission_denied(format!(
69	"web search to '{host}' requires approval; \
70	re-run after `/network allow {host}` or set network.default = \"allow\" in config"
71	))),
72	}
73	}
74
75	// Cached regex for secret redaction in error bodies
76	static BEARER_TOKEN_RE: OnceLock<Regex> = OnceLock::new();
77
78	fn get_bearer_token_re() -> &'static Regex {
79	BEARER_TOKEN_RE.get_or_init(\|\| {
80	Regex::new(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+")
81	.expect("bearer token regex pattern is valid")
82	})
83	}
84
85	#[derive(Debug, Clone, Serialize)]
86	struct WebSearchEntry {
87	title: String,
88	url: String,
89	snippet: Option<String>,
90	}
91
92	pub struct WebSearchTool;
93
94	#[async_trait]
95	impl ToolSpec for WebSearchTool {
96	fn name(&self) -> &'static str {
97	"web_search"
98	}
99
100	fn model_visible(&self) -> bool {
101	false
102	}
103
104	fn description(&self) -> &'static str {
105	"Search the web and return ranked results with URLs, snippets, session-scoped ref_ids, and an execution receipt. Open a result ref_id with `web.run` when the short summary is not enough; fetch only the few sources needed. When the exact active route reports a documented first-party server-side search tool, it is tried first; otherwise the default backend is DuckDuckGo with Bing fallback. Configured API backends visibly degrade through DuckDuckGo then Bing when unavailable, and every hop is recorded. Configuration and network-policy errors fail closed. Explicit Bing and private DuckDuckGo-compatible routes do not cross providers. Set `[search] provider = \"bing\" \| \"tavily\" \| \"bocha\" \| \"metaso\" \| \"searxng\" \| \"baidu\" \| \"volcengine\" \| \"sofya\"` in config.toml, or `[search] base_url` for a private DuckDuckGo-compatible endpoint or trusted SearXNG instance. For a known canonical URL, prefer `fetch_url` directly."
106	}
107
108	fn input_schema(&self) -> Value {
109	json!({
110	"type": "object",
111	"properties": {
112	"query": {
113	"type": "string",
114	"description": "Search query. Compatibility aliases: q, or search_query[0].q."
115	},
116	"q": {
117	"type": "string",
118	"description": "Search query."
119	},
120	"search_query": {
121	"type": "array",
122	"description": "Array form for advanced queries: [{\"q\":\"...\", \"max_results\": 5}]",
123	"items": {
124	"type": "object",
125	"properties": {
126	"q": { "type": "string" },
127	"query": { "type": "string" },
128	"max_results": { "type": "integer" },
129	"recency": {
130	"oneOf": [
131	{ "type": "string", "enum": ["day", "week", "month", "year"] },
132	{ "type": "integer", "minimum": 1, "maximum": 3650 }
133	]
134	},
135	"domains": { "type": "array", "items": { "type": "string" } },
136	"locale": { "type": "string" }
137	}
138	}
139	},
140	"max_results": {
141	"type": "integer",
142	"description": "Maximum number of results to return (default: 5, max: 10)"
143	},
144	"timeout_ms": {
145	"type": "integer",
146	"description": "Timeout in milliseconds (default: 15000, max: 60000)"
147	},
148	"recency": {
149	"oneOf": [
150	{ "type": "string", "enum": ["day", "week", "month", "year"] },
151	{ "type": "integer", "minimum": 1, "maximum": 3650 }
152	],
153	"description": "Requested freshness window. Unsupported backends report it as degraded instead of silently ignoring it."
154	},
155	"domains": {
156	"type": "array",
157	"items": { "type": "string" },
158	"description": "Restrict returned results to these domains. Backends without native support report post-filtering."
159	},
160	"locale": {
161	"type": "string",
162	"description": "Requested result locale. Unsupported backends report it as degraded."
163	}
164	}
165	})
166	}
167
168	fn capabilities(&self) -> Vec<ToolCapability> {
169	vec![ToolCapability::ReadOnly, ToolCapability::Network]
170	}
171
172	fn approval_requirement(&self) -> ApprovalRequirement {
173	ApprovalRequirement::Auto
174	}
175
176	fn supports_parallel(&self) -> bool {
177	true
178	}
179
180	async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
181	let query = search_query_from_input(&input)?;
182	let timeout_ms = optional_u64(&input, "timeout_ms", DEFAULT_SEARCH_TIMEOUT_MS)?
183	.min(MAX_SEARCH_TIMEOUT_MS);
184	let response = execute_search(query, timeout_ms, context).await?;
185	ToolResult::json(&response).map_err(\|error\| ToolError::execution_failed(error.to_string()))
186	}
187	}
188
189	impl WebSearchTool {
190	/// Search via a configured SearXNG JSON API.
191	///
192	/// SearXNG exposes `/search?q=...&format=json`, but public instances often
193	/// disable JSON output or rate-limit automation. CodeWhale therefore uses
194	/// only the trusted instance configured in `[search] base_url`.
195	async fn run_searxng_search(
196	&self,
197	query: &str,
198	max_results: usize,
199	timeout_ms: u64,
200	context: &ToolContext,
201	) -> Result<(Vec<WebSearchEntry>, String), ToolError> {
202	let (url, host) = searxng_search_url(context.search_base_url.as_deref(), query)?;
203	check_policy(context.network_policy.as_ref(), &host)?;
204
205	let client = crate::tls::reqwest_client_builder()
206	.timeout(Duration::from_millis(timeout_ms))
207	.user_agent(USER_AGENT)
208	.build()
209	.map_err(\|e\| {
210	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
211	})?;
212
213	let resp = client
214	.get(&url)
215	.header("Accept", "application/json")
216	.send()
217	.await
218	.map_err(\|e\| {
219	ToolError::execution_failed(format!("SearXNG search request to {host} failed: {e}"))
220	})?;
221
222	let status = resp.status();
223	let body = resp.text().await.map_err(\|e\| {
224	ToolError::execution_failed(format!("Failed to read SearXNG response from {host}: {e}"))
225	})?;
226
227	if !status.is_success() {
228	let truncated = truncate_error_body(&body);
229	let msg = match status.as_u16() {
230	403 => format!(
231	"SearXNG search failed: HTTP 403 from {host}. Check that JSON output is enabled and this instance permits API access. {truncated}"
232	),
233	429 => format!(
234	"SearXNG search failed: HTTP 429 from {host}. The configured instance is rate-limiting requests; use a trusted/self-hosted instance or retry later. {truncated}"
235	),
236	code => format!("SearXNG search failed: HTTP {code} from {host}. {truncated}"),
237	};
238	return Err(ToolError::execution_failed(msg));
239	}
240
241	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
242	ToolError::execution_failed(format!(
243	"Failed to parse SearXNG JSON response from {host}: {e}. Ensure the instance supports format=json and JSON output is enabled."
244	))
245	})?;
246
247	Ok((parse_searxng_results(&parsed, max_results), host))
248	}
249
250	/// Search via Tavily AI Search API (<https://tavily.com>).
251	async fn run_tavily_search(
252	&self,
253	query: &str,
254	max_results: usize,
255	timeout_ms: u64,
256	context: &ToolContext,
257	) -> Result<Vec<WebSearchEntry>, ToolError> {
258	let api_key = context
259	.search_api_key
260	.as_deref()
261	.ok_or_else(\|\| {
262	ToolError::execution_failed(
263	"Tavily search requires an API key. Set `[search] api_key = \"tvly-...\"` in config.toml.",
264	)
265	})?;
266
267	let client = crate::tls::reqwest_client_builder()
268	.timeout(Duration::from_millis(timeout_ms))
269	.build()
270	.map_err(\|e\| {
271	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
272	})?;
273
274	let payload = json!({
275	"api_key": api_key, // noqa: api-key-in-body
276	"query": query,
277	"search_depth": "basic",
278	"max_results": max_results,
279	});
280
281	let resp = client
282	.post(TAVILY_ENDPOINT)
283	.header("Content-Type", "application/json")
284	.json(&payload)
285	.send()
286	.await
287	.map_err(\|e\| {
288	ToolError::execution_failed(format!("Tavily search request failed: {e}"))
289	})?;
290
291	let status = resp.status();
292	let body = resp.text().await.map_err(\|e\| {
293	ToolError::execution_failed(format!("Failed to read Tavily response: {e}"))
294	})?;
295
296	if !status.is_success() {
297	let truncated = truncate_error_body(&body);
298	return Err(ToolError::execution_failed(format!(
299	"Tavily search failed: HTTP {} — {truncated}",
300	status.as_u16()
301	)));
302	}
303
304	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
305	ToolError::execution_failed(format!("Failed to parse Tavily response: {e}"))
306	})?;
307
308	Ok(parse_tavily_results(&parsed, max_results))
309	}
310
311	/// Search via Sofya web search API (<https://sofya.co>).
312	///
313	/// Sofya returns full extracted page content rather than snippets. The API
314	/// key (`ay_live_...`) comes from `[search] api_key`, falling back to the
315	/// `SOFYA_API_KEY` env var, and is sent as a `Bearer` token.
316	async fn run_sofya_search(
317	&self,
318	query: &str,
319	max_results: usize,
320	timeout_ms: u64,
321	context: &ToolContext,
322	) -> Result<Vec<WebSearchEntry>, ToolError> {
323	let env_key = std::env::var("SOFYA_API_KEY").ok();
324	let api_key = context
325	.search_api_key
326	.as_deref()
327	.or(env_key.as_deref())
328	.ok_or_else(\|\| {
329	ToolError::execution_failed(
330	"Sofya search requires an API key. Set `[search] api_key = \"ay_live_...\"` in config.toml or the SOFYA_API_KEY env var.",
331	)
332	})?;
333
334	let client = crate::tls::reqwest_client_builder()
335	.timeout(Duration::from_millis(timeout_ms))
336	.build()
337	.map_err(\|e\| {
338	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
339	})?;
340
341	let payload = json!({
342	"query": query,
343	"max_results": max_results,
344	});
345
346	let resp = client
347	.post(SOFYA_ENDPOINT)
348	.header("Content-Type", "application/json")
349	.bearer_auth(api_key)
350	.json(&payload)
351	.send()
352	.await
353	.map_err(\|e\| {
354	ToolError::execution_failed(format!("Sofya search request failed: {e}"))
355	})?;
356
357	let status = resp.status();
358	let body = resp.text().await.map_err(\|e\| {
359	ToolError::execution_failed(format!("Failed to read Sofya response: {e}"))
360	})?;
361
362	if !status.is_success() {
363	let truncated = truncate_error_body(&body);
364	return Err(ToolError::execution_failed(format!(
365	"Sofya search failed: HTTP {} — {truncated}",
366	status.as_u16()
367	)));
368	}
369
370	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
371	ToolError::execution_failed(format!("Failed to parse Sofya response: {e}"))
372	})?;
373
374	Ok(parse_sofya_results(&parsed, max_results))
375	}
376
377	/// Search via Bocha AI Search API (<https://bochaai.com>).
378	async fn run_bocha_search(
379	&self,
380	query: &str,
381	max_results: usize,
382	timeout_ms: u64,
383	context: &ToolContext,
384	) -> Result<Vec<WebSearchEntry>, ToolError> {
385	let api_key = context
386	.search_api_key
387	.as_deref()
388	.ok_or_else(\|\| {
389	ToolError::execution_failed(
390	"Bocha search requires an API key. Set `[search] api_key = \"sk-...\"` in config.toml.",
391	)
392	})?;
393
394	let client = crate::tls::reqwest_client_builder()
395	.timeout(Duration::from_millis(timeout_ms))
396	.build()
397	.map_err(\|e\| {
398	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
399	})?;
400
401	let payload = json!({
402	"query": query,
403	"freshness": "noLimit",
404	"count": max_results,
405	});
406
407	let resp = client
408	.post(BOCHA_ENDPOINT)
409	.header("Content-Type", "application/json")
410	.header("Authorization", format!("Bearer {api_key}"))
411	.json(&payload)
412	.send()
413	.await
414	.map_err(\|e\| {
415	ToolError::execution_failed(format!("Bocha search request failed: {e}"))
416	})?;
417
418	let status = resp.status();
419	let body = resp.text().await.map_err(\|e\| {
420	ToolError::execution_failed(format!("Failed to read Bocha response: {e}"))
421	})?;
422
423	if !status.is_success() {
424	let truncated = truncate_error_body(&body);
425	return Err(ToolError::execution_failed(format!(
426	"Bocha search failed: HTTP {} — {truncated}",
427	status.as_u16()
428	)));
429	}
430
431	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
432	ToolError::execution_failed(format!("Failed to parse Bocha response: {e}"))
433	})?;
434
435	if let Some(error) = bocha_error_message(&parsed) {
436	return Err(ToolError::execution_failed(error));
437	}
438
439	Ok(parse_bocha_results(&parsed, max_results))
440	}
441
442	/// Search via Metaso AI Search API (<https://metaso.cn>). Falls back to
443	/// `METASO_API_KEY` when no config key is set.
444	async fn run_metaso_search(
445	&self,
446	query: &str,
447	max_results: usize,
448	timeout_ms: u64,
449	context: &ToolContext,
450	) -> Result<Vec<WebSearchEntry>, ToolError> {
451	let env_key = std::env::var("METASO_API_KEY").ok();
452	let api_key = context
453	.search_api_key
454	.as_deref()
455	.or(env_key.as_deref())
456	.ok_or_else(\|\| {
457	ToolError::execution_failed(
458	"Metaso search requires an API key. Set `METASO_API_KEY` or `[search] api_key` in config.toml.",
459	)
460	})?;
461
462	let client = crate::tls::reqwest_client_builder()
463	.timeout(Duration::from_millis(timeout_ms))
464	.build()
465	.map_err(\|e\| {
466	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
467	})?;
468
469	let size = max_results.clamp(1, 100);
470	let payload = json!({
471	"q": query,
472	"scope": "webpage",
473	"size": size,
474	});
475
476	let resp = client
477	.post(format!("{METASO_ENDPOINT}/search"))
478	.header("Content-Type", "application/json")
479	.header("Authorization", format!("Bearer {api_key}"))
480	.json(&payload)
481	.send()
482	.await
483	.map_err(\|e\| {
484	ToolError::execution_failed(format!("Metaso search request failed: {e}"))
485	})?;
486
487	let status = resp.status();
488	let body = resp.text().await.map_err(\|e\| {
489	ToolError::execution_failed(format!("Failed to read Metaso response: {e}"))
490	})?;
491
492	if !status.is_success() {
493	let msg = match status.as_u16() {
494	401 \| 403 => "Metaso API key rejected — check METASO_API_KEY or set `[search] api_key` in config.toml, or get one at https://metaso.cn/search-api/playground".to_string(),
495	429 => "Metaso rate-limited — wait and retry, or get your own API key at https://metaso.cn/search-api/playground".to_string(),
496	_ => {
497	let truncated = truncate_error_body(&body);
498	format!("Metaso server error (HTTP {status}) — {truncated}")
499	}
500	};
501	return Err(ToolError::execution_failed(msg));
502	}
503
504	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
505	ToolError::execution_failed(format!("Failed to parse Metaso response: {e}"))
506	})?;
507
508	// Check business-logic error codes in the response body.
509	if let Some(code) = parsed.get("code").and_then(\|v\| v.as_i64())
510	&& code != 0
511	{
512	let msg = parsed
513	.get("message")
514	.and_then(\|v\| v.as_str())
515	.unwrap_or("unknown error");
516	return Err(ToolError::execution_failed(match code {
517	3003 => "Metaso: daily search limit reached — set METASO_API_KEY or get one at https://metaso.cn/search-api/playground".to_string(),
518	2005 => "Metaso API key rejected — check METASO_API_KEY or set `[search] api_key` in config.toml".to_string(),
519	_ => format!("Metaso API error (code {code}: {msg})"),
520	}));
521	}
522
523	Ok(parse_metaso_results(&parsed, size))
524	}
525
526	/// Search via Baidu AI Search API (<https://qianfan.baidubce.com>).
527	async fn run_baidu_search(
528	&self,
529	query: &str,
530	max_results: usize,
531	timeout_ms: u64,
532	context: &ToolContext,
533	) -> Result<Vec<WebSearchEntry>, ToolError> {
534	let env_key = std::env::var("BAIDU_SEARCH_API_KEY").ok();
535	let api_key = context
536	.search_api_key
537	.as_deref()
538	.or(env_key.as_deref())
539	.ok_or_else(\|\| {
540	ToolError::execution_failed(
541	"Baidu search requires an API key. Set `BAIDU_SEARCH_API_KEY` or `[search] api_key` in config.toml.",
542	)
543	})?;
544
545	let client = crate::tls::reqwest_client_builder()
546	.timeout(Duration::from_millis(timeout_ms))
547	.build()
548	.map_err(\|e\| {
549	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
550	})?;
551
552	let payload = baidu_search_payload(query, max_results);
553
554	let resp = client
555	.post(BAIDU_ENDPOINT)
556	.header("Authorization", format!("Bearer {api_key}"))
557	.json(&payload)
558	.send()
559	.await
560	.map_err(\|e\| {
561	ToolError::execution_failed(format!("Baidu search request failed: {e}"))
562	})?;
563
564	let status = resp.status();
565	let body = resp.text().await.map_err(\|e\| {
566	ToolError::execution_failed(format!("Failed to read Baidu response: {e}"))
567	})?;
568
569	if !status.is_success() {
570	let msg = match status.as_u16() {
571	401 \| 403 => "Baidu search API key rejected — check BAIDU_SEARCH_API_KEY or `[search] api_key` in config.toml".to_string(),
572	429 => "Baidu search rate-limited — wait and retry, or check your Baidu AI Search quota".to_string(),
573	_ => {
574	let truncated = truncate_error_body(&body);
575	format!("Baidu search failed: HTTP {} — {truncated}", status.as_u16())
576	}
577	};
578	return Err(ToolError::execution_failed(msg));
579	}
580
581	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
582	ToolError::execution_failed(format!("Failed to parse Baidu response: {e}"))
583	})?;
584
585	if let Some(error) = baidu_error_message(&parsed) {
586	return Err(ToolError::execution_failed(error));
587	}
588
589	Ok(parse_baidu_results(&parsed, max_results))
590	}
591
592	/// Search via Volcengine Ark Responses API web_search tool.
593	/// Uses strict JSON prompt constraints to extract structured results
594	/// from the model's search-augmented response.
595	///
596	/// Overrides the user-supplied timeout to a minimum of 90 s because the
597	/// Responses API pipeline (web search → model inference → JSON generation)
598	/// is inherently slower than simple search-API round-trips. A separate
599	/// `connect_timeout` of 15 s lets DNS/TLS failures surface quickly.
600	/// Transient transport errors are retried twice with exponential backoff.
601	async fn run_volcengine_search(
602	&self,
603	query: &str,
604	max_results: usize,
605	timeout_ms: u64,
606	context: &ToolContext,
607	) -> Result<Vec<WebSearchEntry>, ToolError> {
608	let volc_key = std::env::var("VOLCENGINE_API_KEY").ok();
609	let volc_ark_key = std::env::var("VOLCENGINE_ARK_API_KEY").ok();
610	let ark_key = std::env::var("ARK_API_KEY").ok();
611	let api_key = context
612	.search_api_key
613	.as_deref()
614	.or(volc_key.as_deref())
615	.or(volc_ark_key.as_deref())
616	.or(ark_key.as_deref())
617	.ok_or_else(\|\| {
618	ToolError::execution_failed(
619	"Volcengine search requires an API key. Set `[search] api_key`, \
620	or VOLCENGINE_API_KEY / VOLCENGINE_ARK_API_KEY / ARK_API_KEY env var.",
621	)
622	})?;
623
624	// Volcengine Responses API pipeline (search + model inference) is
625	// slow, so enforce a floor of 90 s. The caller's value is used only
626	// when it exceeds 90_000 ms.
627	let effective_timeout = timeout_ms.max(90_000);
628
629	let client = crate::tls::reqwest_client_builder()
630	.connect_timeout(Duration::from_secs(15))
631	.timeout(Duration::from_millis(effective_timeout))
632	.tcp_keepalive(Some(Duration::from_secs(30)))
633	.http2_keep_alive_interval(Some(Duration::from_secs(15)))
634	.http2_keep_alive_timeout(Duration::from_secs(20))
635	.user_agent(USER_AGENT)
636	.build()
637	.map_err(\|e\| {
638	ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
639	})?;
640
641	let payload = volcengine_search_payload(query, max_results);
642
643	// Retry transient transport errors (DNS, connection reset, timeout)
644	// up to 2 times with exponential backoff: 1 s, 2 s.
645	let mut last_err: Option<ToolError> = None;
646	for attempt in 0..3 {
647	if attempt > 0 {
648	tokio::time::sleep(Duration::from_millis(1000 * (1 << (attempt - 1)))).await;
649	}
650
651	match client
652	.post(VOLCENGINE_RESPONSES_ENDPOINT)
653	.header("Authorization", format!("Bearer {api_key}"))
654	.json(&payload)
655	.send()
656	.await
657	{
658	Ok(resp) => {
659	let status = resp.status();
660	let body = resp.text().await.map_err(\|e\| {
661	ToolError::execution_failed(format!(
662	"Failed to read Volcengine response: {e}"
663	))
664	})?;
665
666	if !status.is_success() {
667	let msg = match status.as_u16() {
668	401 \| 403 => "Volcengine API key rejected — check `[search] api_key` in config.toml or VOLCENGINE_API_KEY / VOLCENGINE_ARK_API_KEY / ARK_API_KEY".to_string(),
669	429 => "Volcengine API rate-limited — wait and retry, or check your quota".to_string(),
670	_ => {
671	let truncated = truncate_error_body(&body);
672	format!("Volcengine search failed: HTTP {} — {truncated}", status.as_u16())
673	}
674	};
675	return Err(ToolError::execution_failed(msg));
676	}
677
678	let parsed: serde_json::Value = serde_json::from_str(&body).map_err(\|e\| {
679	ToolError::execution_failed(format!(
680	"Failed to parse Volcengine response: {e}"
681	))
682	})?;
683
684	if let Some(error) = volcengine_error_message(&parsed) {
685	return Err(ToolError::execution_failed(error));
686	}
687
688	let response_text = volcengine_extract_text(&parsed).ok_or_else(\|\| {
689	ToolError::execution_failed("Volcengine response contains no output text")
690	})?;
691
692	return Ok(parse_volcengine_results(&response_text, max_results));
693	}
694	Err(e) => {
695	let is_transient = e.is_timeout() \|\| e.is_connect();
696	if !is_transient \|\| attempt == 2 {
697	return Err(ToolError::execution_failed(format!(
698	"Volcengine search request failed: {e}"
699	)));
700	}
701	last_err = Some(ToolError::execution_failed(format!(
702	"Volcengine search request failed (attempt {}/3): {e}",
703	attempt + 1
704	)));
705	}
706	}
707	}
708
709	// Unreachable — the final iteration always returns above.
710	Err(last_err.unwrap_or_else(\|\| {
711	ToolError::execution_failed("Volcengine search: unexpected retry exit")
712	}))
713	}
714	}
715
716	pub(crate) async fn execute_search(
717	query: SearchQuery,
718	timeout_ms: u64,
719	context: &ToolContext,
720	) -> Result<SearchResponse, ToolError> {
721	if configured_search_base_url(context.search_base_url.as_deref()).is_some()
722	&& !matches!(
723	context.search_provider,
724	SearchProvider::DuckDuckGo \| SearchProvider::Searxng
725	)
726	{
727	return Err(ToolError::invalid_input(format!(
728	"[search].base_url is only supported with provider = \"duckduckgo\" or \"searxng\"; current provider is \"{}\"",
729	context.search_provider.as_str()
730	)));
731	}
732
733	let chain = SearchBackendChain::from_context(context);
734	let initial_backend = chain.initial_backend();
735	if initial_backend != BackendId::ProviderNative {
736	debug_assert_eq!(initial_backend.as_str(), context.search_provider.as_str());
737	preflight_search_provider(context)?;
738	}
739	let cache_scope = if initial_backend == BackendId::ProviderNative {
740	context
741	.provider_native_search
742	.as_ref()
743	.map(crate::client::ProviderNativeSearchClient::cache_identity)
744	} else {
745	normalized_search_base_url(context.search_base_url.as_deref())
746	};
747
748	if let Some(mut cached) = cache::get_search(
749	&context.state_namespace,
750	initial_backend,
751	cache_scope.as_deref(),
752	&query,
753	) {
754	validate_cached_search_policy(&cached, context)?;
755	register_search_citations(&mut cached, context);
756	cached.receipt.cache_hit = true;
757	cached.receipt.latency_ms = 0;
758	return Ok(cached);
759	}
760
761	let started = Instant::now();
762	let requested_timeout = Duration::from_millis(timeout_ms.max(1));
763	let (total_timeout, first_attempt_budget) = if initial_backend == BackendId::Volcengine {
764	let provider_budget = Duration::from_millis(VOLCENGINE_MIN_TIMEOUT_MS);
765	(provider_budget + requested_timeout, Some(provider_budget))
766	} else {
767	(requested_timeout, None)
768	};
769	let deadline = started + total_timeout;
770	let chained = chain.search(&query, deadline, first_attempt_budget).await?;
771	let mut response =
772	finalize_search_response(query.clone(), chained.capabilities, chained.raw, started);
773	register_search_citations(&mut response, context);
774	cache::insert_search(
775	&context.state_namespace,
776	initial_backend,
777	cache_scope.as_deref(),
778	&query,
779	response.clone(),
780	);
781	Ok(response)
782	}
783
784	fn register_search_citations(response: &mut SearchResponse, context: &ToolContext) {
785	let mut seen = std::collections::HashSet::new();
786	response.results.retain_mut(\|result\| {
787	let Some(citation) = super::web::citations::register(
788	&context.state_namespace,
789	&result.url,
790	Some(&result.title),
791	) else {
792	return false;
793	};
794	result.ref_id = citation.ref_id;
795	result.url = citation.url;
796	seen.insert(result.ref_id.clone())
797	});
798	if response.count != response.results.len() {
799	rerank(&mut response.results);
800	response.count = response.results.len();
801	response.message = if response.count == 0 {
802	"No usable web citations found".to_string()
803	} else {
804	format!("Found {} result(s)", response.count)
805	};
806	}
807	}
808
809	/// Reject a misconfigured provider before any cache lookup or network attempt.
810	///
811	/// Configuration gaps are reported as `InvalidInput` ("not configured", with
812	/// the exact config fix) so they stay distinguishable from transport failures
813	/// (`ExecutionFailed`) and from successful-but-empty results.
814	fn preflight_search_provider(context: &ToolContext) -> Result<(), ToolError> {
815	let configured_key = context
816	.search_api_key
817	.as_deref()
818	.is_some_and(\|key\| !key.trim().is_empty());
819	let env_key = \|name: &str\| std::env::var_os(name).is_some_and(\|value\| !value.is_empty());
820	let not_configured = \|message: &str\| Err(ToolError::invalid_input(message));
821
822	match context.search_provider {
823	SearchProvider::Tavily if !configured_key => not_configured(
824	"Tavily search is not configured: it requires an API key. Set `[search] api_key = \"tvly-...\"` in config.toml.",
825	),
826	SearchProvider::Bocha if !configured_key => not_configured(
827	"Bocha search is not configured: it requires an API key. Set `[search] api_key = \"sk-...\"` in config.toml.",
828	),
829	SearchProvider::Metaso if !configured_key && !env_key("METASO_API_KEY") => not_configured(
830	"Metaso search is not configured: it requires an API key. Set `METASO_API_KEY` or `[search] api_key` in config.toml.",
831	),
832	SearchProvider::Baidu if !configured_key && !env_key("BAIDU_SEARCH_API_KEY") => {
833	not_configured(
834	"Baidu search is not configured: it requires an API key. Set `BAIDU_SEARCH_API_KEY` or `[search] api_key` in config.toml.",
835	)
836	}
837	SearchProvider::Volcengine
838	if !configured_key
839	&& !env_key("VOLCENGINE_API_KEY")
840	&& !env_key("VOLCENGINE_ARK_API_KEY")
841	&& !env_key("ARK_API_KEY") =>
842	{
843	not_configured(
844	"Volcengine search is not configured: it requires an API key. Set `[search] api_key`, or VOLCENGINE_API_KEY / VOLCENGINE_ARK_API_KEY / ARK_API_KEY env var.",
845	)
846	}
847	SearchProvider::Sofya if !configured_key && !env_key("SOFYA_API_KEY") => not_configured(
848	"Sofya search is not configured: it requires an API key. Set `[search] api_key = \"ay_live_...\"` in config.toml or the SOFYA_API_KEY env var.",
849	),
850	SearchProvider::Searxng
851	if configured_search_base_url(context.search_base_url.as_deref()).is_none() =>
852	{
853	not_configured(
854	"SearXNG search requires [search] base_url = \"https://your-searxng.example\"; no public instance is used by default.",
855	)
856	}
857	_ => Ok(()),
858	}
859	}
860
861	fn normalized_search_base_url(base_url: Option<&str>) -> Option<String> {
862	let raw = configured_search_base_url(base_url)?;
863	let Ok(mut url) = reqwest::Url::parse(raw) else {
864	return Some(raw.to_string());
865	};
866	url.set_fragment(None);
867	Some(url.to_string())
868	}
869
870	fn validate_cached_search_policy(
871	response: &SearchResponse,
872	context: &ToolContext,
873	) -> Result<(), ToolError> {
874	let host = response
875	.receipt
876	.backend_detail
877	.as_deref()
878	.or_else(\|\| default_backend_host(response.receipt.backend))
879	.ok_or_else(\|\| {
880	ToolError::execution_failed("cached search receipt did not identify its backend host")
881	})?;
882	check_policy(context.network_policy.as_ref(), host)
883	}
884
885	const fn default_backend_host(backend: BackendId) -> Option<&'static str> {
886	match backend {
887	BackendId::ProviderNative => None,
888	BackendId::Bing => Some(BING_HOST),
889	BackendId::DuckDuckGo => Some("html.duckduckgo.com"),
890	BackendId::Tavily => Some("api.tavily.com"),
891	BackendId::Bocha => Some("api.bochaai.com"),
892	BackendId::Metaso => Some("metaso.cn"),
893	BackendId::Searxng => None,
894	BackendId::Baidu => Some("qianfan.baidubce.com"),
895	BackendId::Volcengine => Some("ark.cn-beijing.volces.com"),
896	BackendId::Sofya => Some("sofya.co"),
897	}
898	}
899
900	fn finalize_search_response(
901	query: SearchQuery,
902	capabilities: super::web::contract::QueryCapabilities,
903	mut raw: BackendSearch,
904	started: Instant,
905	) -> SearchResponse {
906	let mut honored = HonoredQueryCapabilities {
907	max_results: matches!(
908	capabilities.max_results,
909	super::web::contract::CapabilityState::Supported
910	),
911	..HonoredQueryCapabilities::default()
912	};
913
914	if query.recency.is_some() {
915	if matches!(
916	capabilities.recency,
917	super::web::contract::CapabilityState::Supported
918	) {
919	honored.recency = true;
920	} else {
921	raw.degraded.push(DegradedReason::KnobIgnored {
922	knob: QueryKnob::Recency,
923	});
924	}
925	}
926	if !query.domains.is_empty() {
927	let before = raw.results.len();
928	raw.results
929	.retain(\|result\| domain_matches(&result.url, &query.domains));
930	rerank(&mut raw.results);
931	honored.domains = true;
932	let provider_honored = matches!(
933	capabilities.domains,
934	super::web::contract::CapabilityState::Supported
935	);
936	if !provider_honored \|\| raw.results.len() != before {
937	raw.degraded.push(DegradedReason::PostFiltered {
938	knob: QueryKnob::Domains,
939	});
940	}
941	}
942	if query.locale.is_some() {
943	if matches!(
944	capabilities.locale,
945	super::web::contract::CapabilityState::Supported
946	) {
947	honored.locale = true;
948	} else {
949	raw.degraded.push(DegradedReason::KnobIgnored {
950	knob: QueryKnob::Locale,
951	});
952	}
953	}
954
955	raw.results.truncate(usize::from(query.max_results));
956	rerank(&mut raw.results);
957	let latency_ms = u32::try_from(started.elapsed().as_millis()).unwrap_or(u32::MAX);
958	let receipt = SearchReceipt {
959	backend: raw.backend,
960	backend_detail: raw.backend_detail,
961	requested: query.clone(),
962	capabilities,
963	honored,
964	degraded: raw.degraded,
965	latency_ms,
966	cache_hit: false,
967	};
968	let count = raw.results.len();
969	let message = match (count, raw.note.as_deref()) {
970	(0, Some(note)) => format!("No results found. {note}"),
971	(0, None) => "No results found".to_string(),
972	(_, Some(note)) => format!("Found {count} result(s). {note}"),
973	(_, None) => format!("Found {count} result(s)"),
974	};
975
976	SearchResponse {
977	query: query.query,
978	source: raw.source,
979	count,
980	message,
981	results: raw.results,
982	receipt,
983	}
984	}
985
986	pub(crate) async fn run_backend_search(
987	provider: SearchProvider,
988	query: &SearchQuery,
989	deadline: Instant,
990	context: &ToolContext,
991	) -> Result<BackendSearch, ToolError> {
992	let timeout_ms = u64::try_from(
993	deadline
994	.saturating_duration_since(Instant::now())
995	.as_millis()
996	.max(1),
997	)
998	.unwrap_or(u64::MAX);
999	let max_results = usize::from(query.max_results);
1000	let tool = WebSearchTool;
1001	let simple = \|backend, entries: Vec<WebSearchEntry>\| BackendSearch {
1002	backend,
1003	source: backend.as_str().to_string(),
1004	backend_detail: None,
1005	results: normalize_entries(entries),
1006	degraded: Vec::new(),
1007	note: None,
1008	};
1009
1010	match provider {
1011	SearchProvider::Tavily => {
1012	check_policy(context.network_policy.as_ref(), "api.tavily.com")?;
1013	Ok(simple(
1014	BackendId::Tavily,
1015	tool.run_tavily_search(&query.query, max_results, timeout_ms, context)
1016	.await?,
1017	))
1018	}
1019	SearchProvider::Bocha => {
1020	check_policy(context.network_policy.as_ref(), "api.bochaai.com")?;
1021	Ok(simple(
1022	BackendId::Bocha,
1023	tool.run_bocha_search(&query.query, max_results, timeout_ms, context)
1024	.await?,
1025	))
1026	}
1027	SearchProvider::Metaso => {
1028	check_policy(context.network_policy.as_ref(), "metaso.cn")?;
1029	Ok(simple(
1030	BackendId::Metaso,
1031	tool.run_metaso_search(&query.query, max_results, timeout_ms, context)
1032	.await?,
1033	))
1034	}
1035	SearchProvider::Searxng => {
1036	let (entries, host) = tool
1037	.run_searxng_search(&query.query, max_results, timeout_ms, context)
1038	.await?;
1039	let note = format!("Backend: searxng at {host}");
1040	Ok(BackendSearch {
1041	backend: BackendId::Searxng,
1042	source: "searxng".to_string(),
1043	backend_detail: Some(host),
1044	results: normalize_entries(entries),
1045	degraded: Vec::new(),
1046	note: Some(note),
1047	})
1048	}
1049	SearchProvider::Baidu => {
1050	check_policy(context.network_policy.as_ref(), "qianfan.baidubce.com")?;
1051	Ok(simple(
1052	BackendId::Baidu,
1053	tool.run_baidu_search(&query.query, max_results, timeout_ms, context)
1054	.await?,
1055	))
1056	}
1057	SearchProvider::Volcengine => {
1058	check_policy(context.network_policy.as_ref(), "ark.cn-beijing.volces.com")?;
1059	let mut response = simple(
1060	BackendId::Volcengine,
1061	tool.run_volcengine_search(&query.query, max_results, timeout_ms, context)
1062	.await?,
1063	);
1064	response.degraded.push(DegradedReason::SynthesizedResults);
1065	Ok(response)
1066	}
1067	SearchProvider::Sofya => {
1068	check_policy(context.network_policy.as_ref(), "sofya.co")?;
1069	Ok(simple(
1070	BackendId::Sofya,
1071	tool.run_sofya_search(&query.query, max_results, timeout_ms, context)
1072	.await?,
1073	))
1074	}
1075	SearchProvider::Bing \| SearchProvider::DuckDuckGo => {
1076	run_scrape_search(provider, query, timeout_ms, context).await
1077	}
1078	}
1079	}
1080
1081	#[derive(Clone, Copy)]
1082	struct ScrapeEndpoints<'a> {
1083	bing: &'a str,
1084	allow_bing_fallback: Option<bool>,
1085	}
1086
1087	impl Default for ScrapeEndpoints<'static> {
1088	fn default() -> Self {
1089	Self {
1090	bing: BING_ENDPOINT,
1091	allow_bing_fallback: None,
1092	}
1093	}
1094	}
1095
1096	async fn run_scrape_search(
1097	provider: SearchProvider,
1098	query: &SearchQuery,
1099	timeout_ms: u64,
1100	context: &ToolContext,
1101	) -> Result<BackendSearch, ToolError> {
1102	// A configured API backend may carry a provider-specific base URL (most
1103	// notably SearXNG). The public scrape fallback must never reinterpret
1104	// that endpoint as DuckDuckGo-compatible HTML.
1105	let fallback_context = (provider == SearchProvider::DuckDuckGo
1106	&& context.search_provider != SearchProvider::DuckDuckGo)
1107	.then(\|\| {
1108	let mut cloned = context.clone();
1109	cloned.search_base_url = None;
1110	cloned
1111	});
1112	let context = fallback_context.as_ref().unwrap_or(context);
1113	run_scrape_search_with_endpoints(
1114	provider,
1115	query,
1116	timeout_ms,
1117	context,
1118	ScrapeEndpoints::default(),
1119	)
1120	.await
1121	}
1122
1123	async fn run_scrape_search_with_endpoints(
1124	provider: SearchProvider,
1125	query: &SearchQuery,
1126	timeout_ms: u64,
1127	context: &ToolContext,
1128	endpoints: ScrapeEndpoints<'_>,
1129	) -> Result<BackendSearch, ToolError> {
1130	let decider = context.network_policy.as_ref();
1131	let client = crate::tls::reqwest_client_builder()
1132	.timeout(Duration::from_millis(timeout_ms))
1133	.user_agent(USER_AGENT)
1134	.build()
1135	.map_err(\|error\| {
1136	ToolError::execution_failed(format!("Failed to build HTTP client: {error}"))
1137	})?;
1138	let max_results = usize::from(query.max_results);
1139	let mut degraded = Vec::new();
1140
1141	if provider == SearchProvider::Bing {
1142	check_policy(decider, BING_HOST)?;
1143	let results = run_bing_search(&client, &query.query, max_results, endpoints.bing).await?;
1144	return Ok(BackendSearch {
1145	backend: BackendId::Bing,
1146	source: "bing".to_string(),
1147	backend_detail: None,
1148	results: normalize_entries(results),
1149	degraded,
1150	note: None,
1151	});
1152	}
1153
1154	let (url, duckduckgo_host) =
1155	duckduckgo_search_url(context.search_base_url.as_deref(), &query.query)?;
1156	let allow_bing_fallback = endpoints
1157	.allow_bing_fallback
1158	.unwrap_or_else(\|\| duckduckgo_allows_bing_fallback(context.search_base_url.as_deref()));
1159	check_policy(decider, &duckduckgo_host)?;
1160	let resp = client
1161	.get(&url)
1162	.header(
1163	"Accept",
1164	"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8",
1165	)
1166	.header("Accept-Language", "en-US,en;q=0.5")
1167	.send()
1168	.await
1169	.map_err(\|error\| {
1170	ToolError::execution_failed(format!("Web search request failed: {error}"))
1171	})?;
1172	let status = resp.status();
1173	let body = resp.text().await.map_err(\|error\| {
1174	ToolError::execution_failed(format!("Failed to read response: {error}"))
1175	})?;
1176	if !status.is_success() {
1177	return Err(ToolError::execution_failed(format!(
1178	"Web search failed: HTTP {}",
1179	status.as_u16()
1180	)));
1181	}
1182
1183	let results = parse_duckduckgo_results(&body, max_results);
1184	let blocked = is_duckduckgo_challenge(&body);
1185	if !results.is_empty() {
1186	return Ok(BackendSearch {
1187	backend: BackendId::DuckDuckGo,
1188	source: if allow_bing_fallback {
1189	"duckduckgo".to_string()
1190	} else {
1191	duckduckgo_host.clone()
1192	},
1193	backend_detail: (!allow_bing_fallback).then_some(duckduckgo_host),
1194	results: normalize_entries(results),
1195	degraded,
1196	note: None,
1197	});
1198	}
1199	if blocked {
1200	degraded.push(DegradedReason::ChallengeDetected {
1201	backend: BackendId::DuckDuckGo,
1202	});
1203	}
1204	if !allow_bing_fallback {
1205	if blocked {
1206	return Err(ToolError::execution_failed(format!(
1207	"DuckDuckGo-compatible search endpoint at {duckduckgo_host} returned a bot challenge; check the private search service, credentials, or network policy"
1208	)));
1209	}
1210	return Ok(BackendSearch {
1211	backend: BackendId::DuckDuckGo,
1212	source: duckduckgo_host.clone(),
1213	backend_detail: Some(duckduckgo_host),
1214	results: Vec::new(),
1215	degraded,
1216	note: None,
1217	});
1218	}
1219
1220	check_policy(decider, BING_HOST)?;
1221	match run_bing_search(&client, &query.query, max_results, endpoints.bing).await {
1222	Ok(results) if !results.is_empty() => {
1223	degraded.push(DegradedReason::ScrapeFallback {
1224	from: BackendId::DuckDuckGo,
1225	to: BackendId::Bing,
1226	});
1227	Ok(BackendSearch {
1228	backend: BackendId::Bing,
1229	source: "bing".to_string(),
1230	backend_detail: None,
1231	results: normalize_entries(results),
1232	degraded,
1233	note: Some(if blocked {
1234	"DuckDuckGo returned a bot challenge; used Bing fallback".to_string()
1235	} else {
1236	"DuckDuckGo returned no parseable results; used Bing fallback".to_string()
1237	}),
1238	})
1239	}
1240	Ok(_) if blocked => Err(ToolError::execution_failed(
1241	"DuckDuckGo returned a bot challenge and Bing fallback returned no results",
1242	)),
1243	Err(error) if blocked => Err(ToolError::execution_failed(format!(
1244	"DuckDuckGo returned a bot challenge and Bing fallback failed: {error}"
1245	))),
1246	Ok(_) \| Err(_) => Ok(BackendSearch {
1247	backend: BackendId::DuckDuckGo,
1248	source: "duckduckgo".to_string(),
1249	backend_detail: None,
1250	results: Vec::new(),
1251	degraded,
1252	note: None,
1253	}),
1254	}
1255	}
1256
1257	fn normalize_entries(entries: Vec<WebSearchEntry>) -> Vec<SearchResult> {
1258	entries
1259	.into_iter()
1260	.enumerate()
1261	.map(\|(index, entry)\| {
1262	SearchResult::new(index + 1, entry.title, entry.url, entry.snippet, None)
1263	})
1264	.collect()
1265	}
1266
1267	fn rerank(results: &mut [SearchResult]) {
1268	for (index, result) in results.iter_mut().enumerate() {
1269	result.rank = u8::try_from(index + 1).unwrap_or(u8::MAX);
1270	}
1271	}
1272
1273	pub(crate) fn domain_matches(url: &str, domains: &[String]) -> bool {
1274	if domains.is_empty() {
1275	return true;
1276	}
1277	let Ok(parsed) = reqwest::Url::parse(url) else {
1278	return false;
1279	};
1280	let Some(host) = parsed.host_str() else {
1281	return false;
1282	};
1283	let host = host.trim_start_matches("www.").to_ascii_lowercase();
1284	domains.iter().any(\|domain\| {
1285	let domain = domain.trim_start_matches("www.").to_ascii_lowercase();
1286	host == domain \|\| host.ends_with(&format!(".{domain}"))
1287	})
1288	}
1289
1290	fn truncate_error_body(body: &str) -> String {
1291	let stripped = sanitize_error_body(body);
1292	if stripped.len() <= ERROR_BODY_PREVIEW_BYTES {
1293	stripped
1294	} else {
1295	let mut end = ERROR_BODY_PREVIEW_BYTES;
1296	while !stripped.is_char_boundary(end) {
1297	end -= 1;
1298	}
1299	format!("{}...", &stripped[..end])
1300	}
1301	}
1302
1303	static TAG_RE: OnceLock<Regex> = OnceLock::new();
1304
1305	fn get_tag_re() -> &'static Regex {
1306	TAG_RE.get_or_init(\|\| Regex::new(r"<[^>]+>").expect("tag regex pattern is valid"))
1307	}
1308
1309	fn strip_html_tags(text: &str) -> String {
1310	get_tag_re().replace_all(text, "").to_string()
1311	}
1312
1313	fn sanitize_error_body(body: &str) -> String {
1314	let stripped = strip_html_tags(body);
1315	let visible: String = stripped
1316	.chars()
1317	.filter(\|c\| !c.is_control() \|\| c.is_ascii_whitespace())
1318	.collect();
1319	get_bearer_token_re()
1320	.replace_all(&visible, "Bearer [REDACTED]")
1321	.to_string()
1322	}
1323
1324	fn parse_tavily_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1325	parsed
1326	.get("results")
1327	.and_then(Value::as_array)
1328	.into_iter()
1329	.flatten()
1330	.filter_map(\|item\| {
1331	let title = item.get("title")?.as_str()?.trim();
1332	let url = item.get("url")?.as_str()?.trim();
1333	if title.is_empty() \|\| url.is_empty() {
1334	return None;
1335	}
1336	Some(WebSearchEntry {
1337	title: title.to_string(),
1338	url: url.to_string(),
1339	snippet: first_non_empty_string(item, &["content", "snippet"]),
1340	})
1341	})
1342	.take(max_results)
1343	.collect()
1344	}
1345
1346	fn parse_metaso_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1347	parsed
1348	.get("webpages")
1349	.and_then(Value::as_array)
1350	.into_iter()
1351	.flatten()
1352	.filter_map(\|item\| {
1353	let title = item.get("title")?.as_str()?.trim();
1354	let url = item.get("link")?.as_str()?.trim();
1355	if title.is_empty() \|\| url.is_empty() {
1356	return None;
1357	}
1358	Some(WebSearchEntry {
1359	title: title.to_string(),
1360	url: url.to_string(),
1361	snippet: first_non_empty_string(item, &["snippet", "summary"]),
1362	})
1363	})
1364	.take(max_results)
1365	.collect()
1366	}
1367
1368	fn parse_bocha_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1369	parsed
1370	.get("data")
1371	.and_then(\|d\| {
1372	d.get("webPages")
1373	.and_then(\|w\| w.get("value"))
1374	.or_else(\|\| d.get("pages"))
1375	})
1376	.or_else(\|\| parsed.get("pages"))
1377	.and_then(\|v\| v.as_array())
1378	.into_iter()
1379	.flat_map(\|arr\| arr.iter())
1380	.filter_map(\|item\| {
1381	let title = item
1382	.get("name")
1383	.or_else(\|\| item.get("title"))
1384	.and_then(\|s\| s.as_str())?
1385	.trim();
1386	let url = item
1387	.get("url")
1388	.or_else(\|\| item.get("link"))
1389	.and_then(\|s\| s.as_str())?
1390	.trim();
1391	if title.is_empty() \|\| url.is_empty() {
1392	return None;
1393	}
1394	let snippet = item
1395	.get("summary")
1396	.or_else(\|\| item.get("snippet"))
1397	.or_else(\|\| item.get("description"))
1398	.and_then(\|s\| s.as_str())
1399	.map(str::trim)
1400	.filter(\|s\| !s.is_empty())
1401	.map(ToString::to_string);
1402	Some(WebSearchEntry {
1403	title: title.to_string(),
1404	url: url.to_string(),
1405	snippet,
1406	})
1407	})
1408	.take(max_results)
1409	.collect()
1410	}
1411
1412	fn bocha_error_message(parsed: &Value) -> Option<String> {
1413	let code = parsed.get("code").and_then(\|v\| v.as_i64())?;
1414	if code == 0 \|\| code == 200 {
1415	return None;
1416	}
1417	let message = parsed
1418	.get("msg")
1419	.or_else(\|\| parsed.get("message"))
1420	.and_then(\|v\| v.as_str())
1421	.unwrap_or("unknown error");
1422	Some(format!("Bocha search API error (code {code}: {message})"))
1423	}
1424
1425	fn parse_baidu_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1426	parsed
1427	.get("references")
1428	.and_then(\|v\| v.as_array())
1429	.into_iter()
1430	.flat_map(\|arr\| arr.iter())
1431	.filter_map(\|item\| {
1432	let title = item
1433	.get("title")
1434	.or_else(\|\| item.get("name"))
1435	.and_then(\|s\| s.as_str())?
1436	.trim();
1437	let url = item
1438	.get("url")
1439	.or_else(\|\| item.get("link"))
1440	.and_then(\|s\| s.as_str())?
1441	.trim();
1442	if title.is_empty() \|\| url.is_empty() {
1443	return None;
1444	}
1445	let snippet = item
1446	.get("content")
1447	.or_else(\|\| item.get("snippet"))
1448	.or_else(\|\| item.get("summary"))
1449	.and_then(\|s\| s.as_str())
1450	.map(str::trim)
1451	.filter(\|s\| !s.is_empty())
1452	.map(ToString::to_string);
1453	Some(WebSearchEntry {
1454	title: title.to_string(),
1455	url: url.to_string(),
1456	snippet,
1457	})
1458	})
1459	.take(max_results)
1460	.collect()
1461	}
1462
1463	fn parse_searxng_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1464	parsed
1465	.get("results")
1466	.and_then(\|v\| v.as_array())
1467	.into_iter()
1468	.flat_map(\|arr\| arr.iter())
1469	.filter_map(\|item\| {
1470	let title = item.get("title").and_then(Value::as_str)?.trim();
1471	let url = item.get("url").and_then(Value::as_str)?.trim();
1472	if title.is_empty() \|\| url.is_empty() {
1473	return None;
1474	}
1475	let snippet = first_non_empty_string(item, &["content", "snippet"]);
1476	Some(WebSearchEntry {
1477	title: title.to_string(),
1478	url: url.to_string(),
1479	snippet,
1480	})
1481	})
1482	.take(max_results)
1483	.collect()
1484	}
1485
1486	fn baidu_error_message(parsed: &Value) -> Option<String> {
1487	let code = parsed
1488	.get("error_code")
1489	.or_else(\|\| parsed.get("code"))
1490	.and_then(\|v\| v.as_i64())?;
1491	if code == 0 {
1492	return None;
1493	}
1494	let message = parsed
1495	.get("error_msg")
1496	.or_else(\|\| parsed.get("message"))
1497	.and_then(\|v\| v.as_str())
1498	.unwrap_or("unknown error");
1499	Some(format!("Baidu search API error (code {code}: {message})"))
1500	}
1501
1502	fn parse_sofya_results(parsed: &Value, max_results: usize) -> Vec<WebSearchEntry> {
1503	parsed
1504	.get("results")
1505	.and_then(\|v\| v.as_array())
1506	.into_iter()
1507	.flat_map(\|arr\| arr.iter())
1508	.filter_map(\|item\| {
1509	let title = item.get("title")?.as_str()?.to_string();
1510	let url = item.get("url")?.as_str()?.to_string();
1511	let snippet = first_non_empty_string(item, &["content", "description"]);
1512	Some(WebSearchEntry {
1513	title,
1514	url,
1515	snippet,
1516	})
1517	})
1518	.take(max_results)
1519	.collect()
1520	}
1521
1522	fn first_non_empty_string(item: &Value, keys: &[&str]) -> Option<String> {
1523	keys.iter().find_map(\|key\| {
1524	item.get(*key)
1525	.and_then(Value::as_str)
1526	.map(str::trim)
1527	.filter(\|value\| !value.is_empty())
1528	.map(str::to_string)
1529	})
1530	}
1531
1532	fn baidu_search_payload(query: &str, max_results: usize) -> Value {
1533	json!({
1534	"messages": [
1535	{
1536	"role": "user",
1537	"content": query,
1538	}
1539	],
1540	"search_source": "baidu_search_v2",
1541	"resource_type_filter": [
1542	{
1543	"type": "web",
1544	"top_k": max_results,
1545	}
1546	],
1547	})
1548	}
1549
1550	fn volcengine_search_payload(query: &str, max_results: usize) -> Value {
1551	json!({
1552	"model": "doubao-seed-2-0-lite-260428",
1553	"stream": false,
1554	"tools": [{"type": "web_search"}],
1555	"input": [{
1556	"role": "user",
1557	"content": [{
1558	"type": "input_text",
1559	"text": format!(
1560	"Search the web for: {query}\n\n\
1561	CRITICAL: Respond ONLY with a valid JSON object. No markdown, no explanation.\n\
1562	Schema: {{\"results\":[{{\"title\":\"...\",\"url\":\"https://...\",\"snippet\":\"...\"}}]}}\n\
1563	- results: 1-{max_results} most relevant pages\n\
1564	- title: page title (required)\n\
1565	- url: full URL starting with https:// (required)\n\
1566	- snippet: 1-2 sentence factual summary (required)\n\
1567	- If zero results: {{\"results\":[]}}\n\
1568	- Your entire response must be valid, parseable JSON."
1569	)
1570	}]
1571	}]
1572	})
1573	}
1574
1575	/// Extracts the model's text response from a Volcengine Responses API output.
1576	fn volcengine_extract_text(parsed: &Value) -> Option<String> {
1577	parsed
1578	.get("output")
1579	.and_then(\|v\| v.as_array())
1580	.into_iter()
1581	.flat_map(\|arr\| arr.iter().rev())
1582	.find(\|item\| item.get("type").and_then(\|t\| t.as_str()) == Some("message"))
1583	.and_then(\|msg\| msg.get("content").and_then(\|c\| c.as_array()))
1584	.and_then(\|content\| {
1585	content
1586	.iter()
1587	.find(\|c\| c.get("text").and_then(\|t\| t.as_str()).is_some())
1588	})
1589	.and_then(\|c\| c.get("text").and_then(\|t\| t.as_str()))
1590	.map(\|s\| s.to_string())
1591	}
1592
1593	/// Checks for business-logic errors in a Volcengine Responses API response.
1594	fn volcengine_error_message(parsed: &Value) -> Option<String> {
1595	let error = parsed.get("error")?;
1596	let code = error
1597	.get("code")
1598	.and_then(\|v\| v.as_str())
1599	.unwrap_or("unknown");
1600	let message = error
1601	.get("message")
1602	.and_then(\|v\| v.as_str())
1603	.unwrap_or("no details");
1604	Some(format!("Volcengine API error (code {code}: {message})"))
1605	}
1606
1607	/// Parses Volcengine model-generated JSON results into `WebSearchEntry` items.
1608	fn parse_volcengine_results(response_text: &str, max_results: usize) -> Vec<WebSearchEntry> {
1609	let json_text = extract_json_block(response_text).unwrap_or(response_text);
1610
1611	let parsed: Value = match serde_json::from_str(json_text) {
1612	Ok(v) => v,
1613	Err(_) => return Vec::new(),
1614	};
1615
1616	parsed
1617	.get("results")
1618	.and_then(\|v\| v.as_array())
1619	.into_iter()
1620	.flat_map(\|arr\| arr.iter())
1621	.filter_map(\|item\| {
1622	let title = item.get("title").and_then(\|s\| s.as_str())?.trim();
1623	let url = item.get("url").and_then(\|s\| s.as_str())?.trim();
1624	if title.is_empty() \|\| url.is_empty() {
1625	return None;
1626	}
1627	let snippet = item
1628	.get("snippet")
1629	.and_then(\|s\| s.as_str())
1630	.map(str::trim)
1631	.filter(\|s\| !s.is_empty())
1632	.map(ToString::to_string);
1633	Some(WebSearchEntry {
1634	title: title.to_string(),
1635	url: url.to_string(),
1636	snippet,
1637	})
1638	})
1639	.take(max_results)
1640	.collect()
1641	}
1642
1643	/// Attempts to extract a JSON block from text that may be wrapped in
1644	/// markdown fences (```json ... ```) or contain surrounding commentary.
1645	fn extract_json_block(text: &str) -> Option<&str> {
1646	if let Some(start) = text.find("```json") {
1647	let inner = &text[start + 7..];
1648	if let Some(end) = inner.find("```") {
1649	return Some(inner[..end].trim());
1650	}
1651	}
1652	if let Some(start) = text.find('{')
1653	&& let Some(end) = text.rfind('}')
1654	{
1655	return Some(&text[start..=end]);
1656	}
1657	None
1658	}
1659
1660	fn extract_search_query(input: &Value) -> Result<String, ToolError> {
1661	for key in ["query", "q"] {
1662	if let Some(value) = input.get(key) {
1663	let Some(query) = value.as_str() else {
1664	return Err(ToolError::invalid_input(format!(
1665	"Field '{key}' must be a string"
1666	)));
1667	};
1668	let query = query.trim();
1669	if !query.is_empty() {
1670	return Ok(query.to_string());
1671	}
1672	}
1673	}
1674
1675	for item in search_query_items(input) {
1676	for key in ["q", "query"] {
1677	if let Some(value) = item.get(key) {
1678	let Some(query) = value.as_str() else {
1679	return Err(ToolError::invalid_input(format!(
1680	"Field 'search_query[].{key}' must be a string"
1681	)));
1682	};
1683	let query = query.trim();
1684	if !query.is_empty() {
1685	return Ok(query.to_string());
1686	}
1687	}
1688	}
1689	}
1690
1691	Err(ToolError::missing_field("query"))
1692	}
1693
1694	fn optional_search_max_results(input: &Value) -> u64 {
1695	if let Some(value) = input.get("max_results").and_then(Value::as_u64) {
1696	return value;
1697	}
1698	search_query_items(input)
1699	.filter_map(\|item\| item.get("max_results").and_then(Value::as_u64))
1700	.next()
1701	.unwrap_or(DEFAULT_SEARCH_RESULTS as u64)
1702	}
1703
1704	fn search_query_from_input(input: &Value) -> Result<SearchQuery, ToolError> {
1705	let query = extract_search_query(input)?;
1706	if query.is_empty() {
1707	return Err(ToolError::invalid_input("Query cannot be empty"));
1708	}
1709	let max_results = usize::try_from(optional_search_max_results(input))
1710	.unwrap_or(DEFAULT_SEARCH_RESULTS)
1711	.clamp(1, usize::from(MAX_SEARCH_RESULTS));
1712	let recency = search_option(input, "recency")
1713	.map(parse_recency)
1714	.transpose()?;
1715	let domains = match search_option(input, "domains") {
1716	Some(value) => value
1717	.as_array()
1718	.ok_or_else(\|\| ToolError::invalid_input("Field 'domains' must be an array"))?
1719	.iter()
1720	.map(\|value\| {
1721	value.as_str().map(str::to_string).ok_or_else(\|\| {
1722	ToolError::invalid_input("Every 'domains' entry must be a string")
1723	})
1724	})
1725	.collect::<Result<Vec<_>, _>>()?,
1726	None => Vec::new(),
1727	};
1728	let locale = search_option(input, "locale")
1729	.map(\|value\| {
1730	value
1731	.as_str()
1732	.map(str::to_string)
1733	.ok_or_else(\|\| ToolError::invalid_input("Field 'locale' must be a string"))
1734	})
1735	.transpose()?;
1736
1737	Ok(SearchQuery::new(
1738	query,
1739	max_results,
1740	recency,
1741	domains,
1742	locale,
1743	))
1744	}
1745
1746	fn search_option<'a>(input: &'a Value, key: &str) -> Option<&'a Value> {
1747	input
1748	.get(key)
1749	.or_else(\|\| search_query_items(input).find_map(\|item\| item.get(key)))
1750	}
1751
1752	fn parse_recency(value: &Value) -> Result<Recency, ToolError> {
1753	if let Some(days) = value.as_u64() {
1754	let days = u16::try_from(days)
1755	.ok()
1756	.filter(\|days\| (1..=3650).contains(days))
1757	.ok_or_else(\|\| {
1758	ToolError::invalid_input("Field 'recency' must be between 1 and 3650 days")
1759	})?;
1760	return Ok(Recency::Days(days));
1761	}
1762	match value.as_str() {
1763	Some("day") => Ok(Recency::Day),
1764	Some("week") => Ok(Recency::Week),
1765	Some("month") => Ok(Recency::Month),
1766	Some("year") => Ok(Recency::Year),
1767	_ => Err(ToolError::invalid_input(
1768	"Field 'recency' must be day, week, month, year, or an integer day count",
1769	)),
1770	}
1771	}
1772
1773	fn search_query_items(input: &Value) -> impl Iterator<Item = &Value> {
1774	input
1775	.get("search_query")
1776	.and_then(Value::as_array)
1777	.into_iter()
1778	.flat_map(\|items\| items.iter())
1779	}
1780
1781	async fn run_bing_search(
1782	client: &reqwest::Client,
1783	query: &str,
1784	max_results: usize,
1785	endpoint: &str,
1786	) -> Result<Vec<WebSearchEntry>, ToolError> {
1787	let mut url = reqwest::Url::parse(endpoint)
1788	.map_err(\|error\| ToolError::invalid_input(format!("Invalid Bing endpoint: {error}")))?;
1789	url.query_pairs_mut().append_pair("q", query);
1790	let resp = client
1791	.get(url)
1792	.header(
1793	"Accept",
1794	"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8",
1795	)
1796	.header("Accept-Language", "en-US,en;q=0.9")
1797	.send()
1798	.await
1799	.map_err(\|e\| ToolError::execution_failed(format!("Bing search request failed: {e}")))?;
1800
1801	let status = resp.status();
1802	let body = resp.text().await.map_err(\|e\| {
1803	ToolError::execution_failed(format!("Failed to read Bing search response: {e}"))
1804	})?;
1805
1806	if !status.is_success() {
1807	return Err(ToolError::execution_failed(format!(
1808	"Bing search failed: HTTP {}",
1809	status.as_u16()
1810	)));
1811	}
1812
1813	Ok(parse_bing_results(&body, max_results))
1814	}
1815
1816	fn parse_duckduckgo_results(html: &str, max_results: usize) -> Vec<WebSearchEntry> {
1817	scrape_duckduckgo_results(html, max_results)
1818	.into_iter()
1819	.map(web_search_entry_from_scraped)
1820	.collect()
1821	}
1822
1823	fn parse_bing_results(html: &str, max_results: usize) -> Vec<WebSearchEntry> {
1824	scrape_bing_results(html, max_results)
1825	.into_iter()
1826	.map(web_search_entry_from_scraped)
1827	.collect()
1828	}
1829
1830	fn web_search_entry_from_scraped(entry: ScrapedSearchResult) -> WebSearchEntry {
1831	WebSearchEntry {
1832	title: entry.title,
1833	url: entry.url,
1834	snippet: entry.snippet,
1835	}
1836	}
1837
1838	fn duckduckgo_search_url(
1839	base_url: Option<&str>,
1840	query: &str,
1841	) -> Result<(String, String), ToolError> {
1842	let raw = configured_search_base_url(base_url).unwrap_or(DUCKDUCKGO_ENDPOINT);
1843	let mut url = reqwest::Url::parse(raw).map_err(\|err\| {
1844	ToolError::invalid_input(format!(
1845	"Invalid DuckDuckGo-compatible search base_url: {err}"
1846	))
1847	})?;
1848	url.query_pairs_mut().append_pair("q", query);
1849	let host = url.host_str().ok_or_else(\|\| {
1850	ToolError::invalid_input("DuckDuckGo-compatible search base_url must include a host")
1851	})?;
1852	Ok((url.to_string(), host.to_string()))
1853	}
1854
1855	fn searxng_search_url(base_url: Option<&str>, query: &str) -> Result<(String, String), ToolError> {
1856	let raw = configured_search_base_url(base_url).ok_or_else(\|\| {
1857	ToolError::invalid_input(
1858	"SearXNG search requires [search] base_url = \"https://your-searxng.example\"; no public instance is used by default.",
1859	)
1860	})?;
1861	let mut url = reqwest::Url::parse(raw).map_err(\|err\| {
1862	ToolError::invalid_input(format!("Invalid SearXNG search base_url: {err}"))
1863	})?;
1864	let host = url
1865	.host_str()
1866	.ok_or_else(\|\| ToolError::invalid_input("SearXNG search base_url must include a host"))?
1867	.to_string();
1868
1869	let path = url.path().trim_end_matches('/');
1870	if path.is_empty() {
1871	url.set_path("search");
1872	} else if path != "/search" && !path.ends_with("/search") {
1873	url.set_path(&format!("{path}/search"));
1874	}
1875	url.query_pairs_mut()
1876	.append_pair("q", query)
1877	.append_pair("format", "json");
1878
1879	Ok((url.to_string(), host))
1880	}
1881
1882	fn configured_search_base_url(base_url: Option<&str>) -> Option<&str> {
1883	base_url.map(str::trim).filter(\|value\| !value.is_empty())
1884	}
1885
1886	fn duckduckgo_allows_bing_fallback(base_url: Option<&str>) -> bool {
1887	configured_search_base_url(base_url).is_none()
1888	}
1889
1890	#[cfg(test)]
1891	mod tests {
1892	use super::{
1893	ERROR_BODY_PREVIEW_BYTES, ScrapeEndpoints, WebSearchTool, baidu_search_payload,
1894	bocha_error_message, domain_matches, duckduckgo_search_url, extract_search_query,
1895	finalize_search_response, optional_search_max_results, parse_baidu_results,
1896	parse_bocha_results, parse_metaso_results, parse_searxng_results, parse_sofya_results,
1897	parse_tavily_results, parse_volcengine_results, register_search_citations, rerank,
1898	run_scrape_search_with_endpoints, sanitize_error_body, searxng_search_url,
1899	truncate_error_body, volcengine_extract_text,
1900	};
1901	use crate::tools::web::contract::{
1902	BackendId, BackendSearch, CapabilityState, DegradedReason, QueryCapabilities, QueryKnob,
1903	Recency, SearchQuery, SearchResult,
1904	};
1905	use crate::tools::web::scrape::{decode_html_entities, normalize_bing_url};
1906	use serde_json::json;
1907	use std::time::Instant;
1908
1909	// Regression guard: Bing /ck/a redirect hrefs are HTML-entity-encoded
1910	// (`&`). normalize_bing_url must decode entities before extracting the
1911	// `u=` base64 payload, otherwise the real URL is never recovered and the
1912	// result remains a Bing tracking URL instead of the cited source.
1913	#[test]
1914	fn bing_ckurl_with_html_entities_decodes_real_url() {
1915	let href = "https://www.bing.com/ck/a?!&&p=abc&u=a1aHR0cHM6Ly9ydXN0LWxhbmcub3JnLw&ntb=1";
1916	assert_eq!(normalize_bing_url(href), "https://rust-lang.org/");
1917	}
1918
1919	#[test]
1920	fn decode_html_entities_handles_named_entities() {
1921	assert_eq!(decode_html_entities("&"), "&");
1922	assert_eq!(decode_html_entities("<"), "<");
1923	assert_eq!(decode_html_entities(">"), ">");
1924	assert_eq!(decode_html_entities("""), "\"");
1925	assert_eq!(decode_html_entities("'"), "'");
1926	assert_eq!(decode_html_entities(" "), " ");
1927	assert_eq!(decode_html_entities("©"), "\u{00A9}");
1928	assert_eq!(decode_html_entities("—"), "\u{2014}");
1929	}
1930
1931	#[test]
1932	fn decode_html_entities_handles_decimal_numeric_references() {
1933	assert_eq!(decode_html_entities("A"), "A");
1934	assert_eq!(decode_html_entities("<"), "<");
1935	assert_eq!(decode_html_entities("–"), "\u{2013}");
1936	}
1937
1938	#[test]
1939	fn decode_html_entities_handles_hex_numeric_references() {
1940	assert_eq!(decode_html_entities("A"), "A");
1941	assert_eq!(decode_html_entities("<"), "<");
1942	assert_eq!(decode_html_entities("—"), "\u{2014}");
1943	}
1944
1945	#[test]
1946	fn decode_html_entities_passthrough_unknown() {
1947	assert_eq!(decode_html_entities("&unknown;"), "&unknown;");
1948	}
1949
1950	#[test]
1951	fn decode_html_entities_mixed_content() {
1952	let input = "Hello & welcome to "Rust's world" — enjoy!";
1953	let expected = "Hello & welcome to \"Rust's world\" \u{2014} enjoy!";
1954	assert_eq!(decode_html_entities(input), expected);
1955	}
1956
1957	#[test]
1958	fn extract_search_query_accepts_legacy_query() {
1959	let query =
1960	extract_search_query(&json!({"query": " deepseek v4 "})).expect("query should parse");
1961	assert_eq!(query, "deepseek v4");
1962	}
1963
1964	#[test]
1965	fn extract_search_query_accepts_q_alias() {
1966	let query =
1967	extract_search_query(&json!({"q": "deepseek v4 pro"})).expect("q alias should parse");
1968	assert_eq!(query, "deepseek v4 pro");
1969	}
1970
1971	#[test]
1972	fn extract_search_query_accepts_array_form() {
1973	let input = json!({"search_query": [{"q": "deepseek api", "max_results": 3}]});
1974	let query = extract_search_query(&input).expect("array form should parse");
1975	assert_eq!(query, "deepseek api");
1976	assert_eq!(optional_search_max_results(&input), 3);
1977	}
1978
1979	#[test]
1980	fn extract_search_query_rejects_missing_query() {
1981	let err = extract_search_query(&json!({"max_results": 2}))
1982	.expect_err("missing query should fail");
1983	assert!(format!("{err}").contains("missing required field 'query'"));
1984	}
1985
1986	#[test]
1987	fn optional_max_results_prefers_top_level_value() {
1988	// Top-level `max_results` wins over the array-form sibling
1989	// because callers using the array form usually copy-paste it
1990	// wholesale and then tweak the outer max_results afterwards.
1991	assert_eq!(
1992	optional_search_max_results(
1993	&json!({"query": "x", "max_results": 8, "search_query": [{"q": "y", "max_results": 2}]})
1994	),
1995	8,
1996	);
1997	}
1998
1999	#[test]
2000	fn optional_max_results_falls_back_to_array_form() {
2001	// When only the array form sets max_results, that value is the
2002	// one that should reach the caller. This is the path V4 uses
2003	// when it emits the structured `search_query: [{…}]` shape.
2004	assert_eq!(
2005	optional_search_max_results(&json!({"search_query": [{"q": "y", "max_results": 3}]})),
2006	3,
2007	);
2008	}
2009
2010	#[test]
2011	fn optional_max_results_uses_default_when_neither_set() {
2012	// No explicit bound anywhere → the DEFAULT (currently 5)
2013	// applies, so the model can't accidentally pull the maximum
2014	// worth of bandwidth just by omitting the field.
2015	assert_eq!(optional_search_max_results(&json!({"query": "x"})), 5);
2016	assert_eq!(
2017	optional_search_max_results(&json!({"search_query": [{"q": "y"}]})),
2018	5,
2019	);
2020	}
2021
2022	#[test]
2023	fn optional_max_results_only_reads_first_array_entry() {
2024	// Sub-search support is a future feature; for now the array
2025	// entries beyond the first are ignored. Pin so a future
2026	// multi-query implementation has to update this test
2027	// intentionally rather than silently start fanning out.
2028	assert_eq!(
2029	optional_search_max_results(
2030	&json!({"search_query": [{"q": "first", "max_results": 1}, {"q": "second", "max_results": 9}]})
2031	),
2032	1,
2033	);
2034	}
2035
2036	#[test]
2037	fn extract_search_query_trims_whitespace_from_array_form_q_alias() {
2038	// The "trimmed" contract is part of the helper's invariant —
2039	// a model sometimes pads `q` with newlines from a heredoc.
2040	let q = extract_search_query(&json!({"search_query": [{"q": " deepseek tui "}]}))
2041	.expect("array form should parse with trim");
2042	assert_eq!(q, "deepseek tui");
2043	}
2044
2045	#[test]
2046	fn extract_search_query_rejects_empty_query() {
2047	// A "" query lands in extract_search_query → propagates as
2048	// missing_field rather than a confusing engine error a few
2049	// layers down. Lock the failure mode.
2050	for body in [json!({"query": ""}), json!({"q": " "}), json!({})] {
2051	let err = extract_search_query(&body).expect_err("empty query must reject");
2052	let msg = format!("{err}");
2053	assert!(
2054	msg.contains("missing required field 'query'") \|\| msg.contains("Query"),
2055	"expected query-missing error, got `{msg}`"
2056	);
2057	}
2058	}
2059
2060	#[test]
2061	fn truncate_error_body_truncates_long_body() {
2062	let body = "a".repeat(ERROR_BODY_PREVIEW_BYTES + 100);
2063	let truncated = truncate_error_body(&body);
2064	assert!(truncated.len() <= ERROR_BODY_PREVIEW_BYTES + 3);
2065	assert!(truncated.ends_with("..."));
2066	}
2067
2068	#[test]
2069	fn truncate_error_body_keeps_short_body_intact() {
2070	let body = "short error";
2071	assert_eq!(truncate_error_body(body), body);
2072	}
2073
2074	#[test]
2075	fn sanitize_error_body_strips_html_and_control_chars() {
2076	let body = "<p>error</p>\x00\x01\x02";
2077	let sanitized = sanitize_error_body(body);
2078	assert_eq!(sanitized, "error");
2079	}
2080
2081	#[test]
2082	fn sanitize_error_body_redacts_bearer_tokens() {
2083	let body = r#"{"error":"bad token","authorization":"Bearer test-token/with+chars="}"#;
2084
2085	let sanitized = sanitize_error_body(body);
2086
2087	assert!(!sanitized.contains("test-token/with+chars="));
2088	assert!(sanitized.contains("Bearer [REDACTED]"));
2089	}
2090
2091	#[test]
2092	fn parse_bocha_web_pages_value_extracts_ranked_results() {
2093	let body = json!({
2094	"code": 200,
2095	"msg": null,
2096	"data": {
2097	"webPages": {
2098	"value": [
2099	{
2100	"name": "广州天气",
2101	"url": "https://bocha.cn/share/weather",
2102	"snippet": "广州今日雷阵雨转晴。"
2103	},
2104	{
2105	"name": "中央气象台",
2106	"url": "https://www.weather.com.cn/",
2107	"summary": "天气实况。"
2108	}
2109	]
2110	}
2111	}
2112	});
2113
2114	let results = parse_bocha_results(&body, 10);
2115
2116	assert_eq!(results.len(), 2);
2117	assert_eq!(results[0].title, "广州天气");
2118	assert_eq!(results[0].url, "https://bocha.cn/share/weather");
2119	assert_eq!(results[0].snippet.as_deref(), Some("广州今日雷阵雨转晴。"));
2120	assert_eq!(results[1].title, "中央气象台");
2121	}
2122
2123	#[test]
2124	fn parse_bocha_keeps_legacy_pages_shape() {
2125	let body = json!({
2126	"code": 200,
2127	"data": {
2128	"pages": [
2129	{
2130	"title": "Legacy title",
2131	"link": "https://example.com/legacy",
2132	"description": "Legacy description"
2133	}
2134	]
2135	}
2136	});
2137
2138	let results = parse_bocha_results(&body, 5);
2139
2140	assert_eq!(results.len(), 1);
2141	assert_eq!(results[0].title, "Legacy title");
2142	assert_eq!(results[0].url, "https://example.com/legacy");
2143	assert_eq!(results[0].snippet.as_deref(), Some("Legacy description"));
2144	}
2145
2146	#[test]
2147	fn bocha_error_message_flags_non_success_business_code() {
2148	let body = json!({"code": 401, "msg": "invalid api key"});
2149
2150	let error = bocha_error_message(&body).expect("non-success code should error");
2151
2152	assert!(error.contains("Bocha"));
2153	assert!(error.contains("401"));
2154	assert!(error.contains("invalid api key"));
2155	}
2156
2157	#[test]
2158	fn parse_baidu_references_extracts_ranked_results() {
2159	let body = json!({
2160	"references": [
2161	{
2162	"title": "Rust 官方文档",
2163	"url": "https://www.rust-lang.org/",
2164	"content": "Rust 是一门注重性能和可靠性的语言。"
2165	},
2166	{
2167	"title": "Cargo Book",
2168	"url": "https://doc.rust-lang.org/cargo/",
2169	"snippet": "Cargo is Rust's package manager."
2170	}
2171	]
2172	});
2173
2174	let results = parse_baidu_results(&body, 10);
2175
2176	assert_eq!(results.len(), 2);
2177	assert_eq!(results[0].title, "Rust 官方文档");
2178	assert_eq!(results[0].url, "https://www.rust-lang.org/");
2179	assert_eq!(
2180	results[0].snippet.as_deref(),
2181	Some("Rust 是一门注重性能和可靠性的语言。")
2182	);
2183	assert_eq!(results[1].title, "Cargo Book");
2184	assert_eq!(results[1].url, "https://doc.rust-lang.org/cargo/");
2185	assert_eq!(
2186	results[1].snippet.as_deref(),
2187	Some("Cargo is Rust's package manager.")
2188	);
2189	}
2190
2191	#[test]
2192	fn parse_baidu_references_skips_incomplete_entries() {
2193	let body = json!({
2194	"references": [
2195	{"title": "No URL", "content": "missing url"},
2196	{"url": "https://example.com/no-title", "content": "missing title"},
2197	{"title": "Valid", "url": "https://example.com/valid"}
2198	]
2199	});
2200
2201	let results = parse_baidu_results(&body, 10);
2202
2203	assert_eq!(results.len(), 1);
2204	assert_eq!(results[0].title, "Valid");
2205	assert_eq!(results[0].url, "https://example.com/valid");
2206	assert_eq!(results[0].snippet, None);
2207	}
2208
2209	#[test]
2210	fn baidu_search_payload_uses_official_search_source() {
2211	let payload = baidu_search_payload("Rust cargo workspace", 3);
2212
2213	assert_eq!(
2214	payload.get("search_source").and_then(\|v\| v.as_str()),
2215	Some("baidu_search_v2")
2216	);
2217	assert_eq!(
2218	payload
2219	.get("messages")
2220	.and_then(\|v\| v.as_array())
2221	.and_then(\|messages\| messages.first())
2222	.and_then(\|message\| message.get("content"))
2223	.and_then(\|v\| v.as_str()),
2224	Some("Rust cargo workspace")
2225	);
2226	assert_eq!(
2227	payload
2228	.get("resource_type_filter")
2229	.and_then(\|v\| v.as_array())
2230	.and_then(\|filters\| filters.first())
2231	.and_then(\|filter\| filter.get("top_k"))
2232	.and_then(\|v\| v.as_u64()),
2233	Some(3)
2234	);
2235	}
2236
2237	#[test]
2238	fn parse_sofya_results_falls_back_to_description_for_empty_content() {
2239	let body = json!({
2240	"results": [
2241	{
2242	"title": "Full content",
2243	"url": "https://example.com/full",
2244	"content": "full extracted page content",
2245	"description": "unused description"
2246	},
2247	{
2248	"title": "Null content",
2249	"url": "https://example.com/null",
2250	"content": null,
2251	"description": "description for null content"
2252	},
2253	{
2254	"title": "Empty content",
2255	"url": "https://example.com/empty",
2256	"content": "",
2257	"description": "description for empty content"
2258	},
2259	{
2260	"title": "Whitespace content",
2261	"url": "https://example.com/blank",
2262	"content": " ",
2263	"description": "description for blank content"
2264	},
2265	{
2266	"title": "No snippet",
2267	"url": "https://example.com/no-snippet"
2268	}
2269	]
2270	});
2271
2272	let results = parse_sofya_results(&body, 10);
2273
2274	assert_eq!(results.len(), 5);
2275	assert_eq!(
2276	results[0].snippet.as_deref(),
2277	Some("full extracted page content")
2278	);
2279	assert_eq!(
2280	results[1].snippet.as_deref(),
2281	Some("description for null content")
2282	);
2283	assert_eq!(
2284	results[2].snippet.as_deref(),
2285	Some("description for empty content")
2286	);
2287	assert_eq!(
2288	results[3].snippet.as_deref(),
2289	Some("description for blank content")
2290	);
2291	assert_eq!(results[4].snippet, None);
2292	}
2293
2294	#[test]
2295	fn tavily_metaso_and_volcengine_payloads_use_normalized_entry_shape() {
2296	let tavily = parse_tavily_results(
2297	&json!({"results": [{
2298	"title": " Tavily result ",
2299	"url": "https://tavily.example/result",
2300	"content": " content "
2301	}]}),
2302	5,
2303	);
2304	let metaso = parse_metaso_results(
2305	&json!({"webpages": [{
2306	"title": " Metaso result ",
2307	"link": "https://metaso.example/result",
2308	"summary": " summary "
2309	}]}),
2310	5,
2311	);
2312	let volcengine = parse_volcengine_results(
2313	r#"{"results":[{"title":"Volcengine result","url":"https://volc.example/result","snippet":"summary"}]}"#,
2314	5,
2315	);
2316
2317	for (entries, title, snippet) in [
2318	(tavily, "Tavily result", "content"),
2319	(metaso, "Metaso result", "summary"),
2320	(volcengine, "Volcengine result", "summary"),
2321	] {
2322	assert_eq!(entries.len(), 1);
2323	assert_eq!(entries[0].title, title);
2324	assert_eq!(entries[0].snippet.as_deref(), Some(snippet));
2325	}
2326	}
2327
2328	#[test]
2329	fn volcengine_extract_text_skips_non_text_content_blocks() {
2330	let body = json!({
2331	"output": [
2332	{
2333	"type": "message",
2334	"content": [
2335	{"type": "reasoning", "summary": "thinking first"},
2336	{"type": "output_text", "text": "{\"results\":[]}"}
2337	]
2338	}
2339	]
2340	});
2341
2342	assert_eq!(
2343	volcengine_extract_text(&body).as_deref(),
2344	Some("{\"results\":[]}")
2345	);
2346	}
2347
2348	#[tokio::test]
2349	async fn tavily_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
2350	// Trust-boundary pin: if a user has opted into Tavily but
2351	// forgot the api_key, the tool must NOT silently fall through
2352	// to DuckDuckGo (which would expose the query to a different
2353	// provider than the user authorised). Instead it returns a
2354	// ToolError that names the missing key explicitly.
2355	use crate::config::SearchProvider;
2356	use crate::tools::spec::{ToolContext, ToolSpec};
2357
2358	let tmp = tempfile::tempdir().expect("tempdir");
2359	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2360	ctx.search_provider = SearchProvider::Tavily;
2361	ctx.search_api_key = None;
2362	let err = WebSearchTool
2363	.execute(json!({"query": "anything"}), &ctx)
2364	.await
2365	.expect_err("missing api_key must surface as ToolError");
2366	let msg = err.to_string();
2367	assert!(
2368	msg.contains("Tavily") && msg.contains("API key"),
2369	"error must name the provider and missing key; got `{msg}`"
2370	);
2371	}
2372
2373	#[tokio::test]
2374	async fn bocha_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
2375	// Same trust-boundary pin for Bocha.
2376	use crate::config::SearchProvider;
2377	use crate::tools::spec::{ToolContext, ToolSpec};
2378
2379	let tmp = tempfile::tempdir().expect("tempdir");
2380	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2381	ctx.search_provider = SearchProvider::Bocha;
2382	ctx.search_api_key = None;
2383	let err = WebSearchTool
2384	.execute(json!({"query": "anything"}), &ctx)
2385	.await
2386	.expect_err("missing api_key must surface as ToolError");
2387	let msg = err.to_string();
2388	assert!(
2389	msg.contains("Bocha") && msg.contains("API key"),
2390	"error must name the provider and missing key; got `{msg}`"
2391	);
2392	}
2393
2394	#[tokio::test]
2395	async fn baidu_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
2396	use crate::config::SearchProvider;
2397	use crate::tools::spec::{ToolContext, ToolSpec};
2398
2399	let prev = std::env::var_os("BAIDU_SEARCH_API_KEY");
2400	unsafe { std::env::remove_var("BAIDU_SEARCH_API_KEY") };
2401
2402	let tmp = tempfile::tempdir().expect("tempdir");
2403	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2404	ctx.search_provider = SearchProvider::Baidu;
2405	ctx.search_api_key = None;
2406	let err = WebSearchTool
2407	.execute(json!({"query": "anything"}), &ctx)
2408	.await
2409	.expect_err("missing api_key must surface as ToolError");
2410
2411	match prev {
2412	Some(value) => unsafe { std::env::set_var("BAIDU_SEARCH_API_KEY", value) },
2413	None => unsafe { std::env::remove_var("BAIDU_SEARCH_API_KEY") },
2414	}
2415
2416	let msg = err.to_string();
2417	assert!(
2418	msg.contains("Baidu") && msg.contains("API key"),
2419	"error must name the provider and missing key; got `{msg}`"
2420	);
2421	}
2422
2423	#[tokio::test]
2424	#[allow(clippy::await_holding_lock)]
2425	async fn sofya_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
2426	// Same trust-boundary pin as Tavily/Bocha: opting into Sofya without a
2427	// key must surface a ToolError naming the provider, not silently fall
2428	// through to DuckDuckGo.
2429	use crate::config::SearchProvider;
2430	use crate::tools::spec::{ToolContext, ToolSpec};
2431
2432	// This test holds the process-env lock through the awaited tool
2433	// execution because the tool reads SOFYA_API_KEY during that call.
2434	let _guard = crate::test_support::lock_test_env();
2435	let prev = std::env::var_os("SOFYA_API_KEY");
2436	unsafe { std::env::remove_var("SOFYA_API_KEY") };
2437
2438	let tmp = tempfile::tempdir().expect("tempdir");
2439	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2440	ctx.search_provider = SearchProvider::Sofya;
2441	ctx.search_api_key = None;
2442	let err = WebSearchTool
2443	.execute(json!({"query": "anything"}), &ctx)
2444	.await
2445	.expect_err("missing api_key must surface as ToolError");
2446
2447	match prev {
2448	Some(value) => unsafe { std::env::set_var("SOFYA_API_KEY", value) },
2449	None => unsafe { std::env::remove_var("SOFYA_API_KEY") },
2450	}
2451
2452	let msg = err.to_string();
2453	assert!(
2454	msg.contains("Sofya") && msg.contains("API key"),
2455	"error must name the provider and missing key; got `{msg}`"
2456	);
2457	}
2458
2459	#[tokio::test]
2460	#[allow(clippy::await_holding_lock)]
2461	async fn volcengine_provider_without_api_key_lists_supported_env_fallbacks() {
2462	use crate::config::SearchProvider;
2463	use crate::tools::spec::{ToolContext, ToolSpec};
2464
2465	// This test intentionally keeps the process-env lock through the
2466	// awaited tool execution because the tool reads env fallbacks during
2467	// that call. Dropping the lock before await would reintroduce races
2468	// with other env-mutating tests.
2469	let _guard = crate::test_support::lock_test_env();
2470	let prev_volc = std::env::var_os("VOLCENGINE_API_KEY");
2471	let prev_volc_ark = std::env::var_os("VOLCENGINE_ARK_API_KEY");
2472	let prev_ark = std::env::var_os("ARK_API_KEY");
2473	unsafe {
2474	std::env::remove_var("VOLCENGINE_API_KEY");
2475	std::env::remove_var("VOLCENGINE_ARK_API_KEY");
2476	std::env::remove_var("ARK_API_KEY");
2477	}
2478
2479	let tmp = tempfile::tempdir().expect("tempdir");
2480	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2481	ctx.search_provider = SearchProvider::Volcengine;
2482	ctx.search_api_key = None;
2483	let err = WebSearchTool
2484	.execute(json!({"query": "anything"}), &ctx)
2485	.await
2486	.expect_err("missing api_key must surface as ToolError");
2487
2488	match prev_volc {
2489	Some(value) => unsafe { std::env::set_var("VOLCENGINE_API_KEY", value) },
2490	None => unsafe { std::env::remove_var("VOLCENGINE_API_KEY") },
2491	}
2492	match prev_volc_ark {
2493	Some(value) => unsafe { std::env::set_var("VOLCENGINE_ARK_API_KEY", value) },
2494	None => unsafe { std::env::remove_var("VOLCENGINE_ARK_API_KEY") },
2495	}
2496	match prev_ark {
2497	Some(value) => unsafe { std::env::set_var("ARK_API_KEY", value) },
2498	None => unsafe { std::env::remove_var("ARK_API_KEY") },
2499	}
2500
2501	let msg = err.to_string();
2502	assert!(msg.contains("Volcengine") && msg.contains("API key"));
2503	assert!(msg.contains("VOLCENGINE_API_KEY"));
2504	assert!(msg.contains("VOLCENGINE_ARK_API_KEY"));
2505	assert!(msg.contains("ARK_API_KEY"));
2506	assert!(!msg.contains("DEEPSEEK_SEARCH_API_KEY"));
2507	}
2508
2509	#[tokio::test]
2510	#[allow(clippy::await_holding_lock)]
2511	async fn metaso_provider_without_api_key_fails_closed_before_fallback() {
2512	use crate::config::SearchProvider;
2513	use crate::tools::spec::{ToolContext, ToolSpec};
2514
2515	let _guard = crate::test_support::lock_test_env();
2516	let previous = std::env::var_os("METASO_API_KEY");
2517	unsafe { std::env::remove_var("METASO_API_KEY") };
2518
2519	let tmp = tempfile::tempdir().expect("tempdir");
2520	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2521	ctx.search_provider = SearchProvider::Metaso;
2522	ctx.search_api_key = None;
2523	let error = WebSearchTool
2524	.execute(json!({"query": "anything"}), &ctx)
2525	.await
2526	.expect_err("missing Metaso key must fail before the fallback chain");
2527
2528	match previous {
2529	Some(value) => unsafe { std::env::set_var("METASO_API_KEY", value) },
2530	None => unsafe { std::env::remove_var("METASO_API_KEY") },
2531	}
2532
2533	let message = error.to_string();
2534	assert!(
2535	message.contains("Metaso")
2536	&& message.contains("API key")
2537	&& message.contains("METASO_API_KEY"),
2538	"got `{message}`"
2539	);
2540	assert!(
2541	!message.contains("duckduckgo"),
2542	"missing configuration must not cross providers: `{message}`"
2543	);
2544	}
2545
2546	#[test]
2547	fn duckduckgo_compatible_url_uses_custom_base_url_and_preserves_query() {
2548	let (url, host) = duckduckgo_search_url(
2549	Some("https://search.internal.example/html/?region=us"),
2550	"rust async",
2551	)
2552	.expect("custom duckduckgo-compatible url");
2553
2554	assert_eq!(host, "search.internal.example");
2555	assert_eq!(
2556	url,
2557	"https://search.internal.example/html/?region=us&q=rust+async"
2558	);
2559	}
2560
2561	#[test]
2562	fn custom_duckduckgo_endpoint_disables_public_bing_fallback() {
2563	assert!(super::duckduckgo_allows_bing_fallback(None));
2564	assert!(super::duckduckgo_allows_bing_fallback(Some(" ")));
2565	assert!(!super::duckduckgo_allows_bing_fallback(Some(
2566	"https://search.internal.example/html/"
2567	)));
2568	}
2569
2570	#[test]
2571	fn searxng_url_uses_search_path_and_json_format() {
2572	let (url, host) =
2573	searxng_search_url(Some("https://search.example/"), "rust async").expect("searxng url");
2574	let parsed = reqwest::Url::parse(&url).expect("valid url");
2575	assert_eq!(host, "search.example");
2576	assert_eq!(parsed.path(), "/search");
2577	assert_eq!(
2578	parsed.query_pairs().find(\|(key, _)\| key == "q").unwrap().1,
2579	"rust async"
2580	);
2581	assert_eq!(
2582	parsed
2583	.query_pairs()
2584	.find(\|(key, _)\| key == "format")
2585	.unwrap()
2586	.1,
2587	"json"
2588	);
2589
2590	let (subpath_url, _) = searxng_search_url(
2591	Some("https://search.example/searxng?language=en"),
2592	"codewhale",
2593	)
2594	.expect("searxng subpath url");
2595	let parsed = reqwest::Url::parse(&subpath_url).expect("valid subpath url");
2596	assert_eq!(parsed.path(), "/searxng/search");
2597	assert_eq!(
2598	parsed
2599	.query_pairs()
2600	.find(\|(key, _)\| key == "language")
2601	.unwrap()
2602	.1,
2603	"en"
2604	);
2605
2606	let (search_url, _) =
2607	searxng_search_url(Some("https://search.example/searxng/search"), "codewhale")
2608	.expect("searxng search endpoint");
2609	assert_eq!(
2610	reqwest::Url::parse(&search_url)
2611	.expect("valid search url")
2612	.path(),
2613	"/searxng/search"
2614	);
2615	}
2616
2617	#[test]
2618	fn searxng_parser_normalizes_results() {
2619	let parsed = json!({
2620	"results": [
2621	{
2622	"title": " Rust async ",
2623	"url": " https://example.com/rust ",
2624	"content": " Result content "
2625	},
2626	{
2627	"title": "Empty snippet",
2628	"url": "https://example.com/empty",
2629	"content": " ",
2630	"snippet": " Fallback snippet "
2631	},
2632	{
2633	"title": "",
2634	"url": "https://example.com/missing-title",
2635	"content": "ignored"
2636	},
2637	{
2638	"title": "Missing URL",
2639	"content": "ignored"
2640	}
2641	]
2642	});
2643
2644	let results = parse_searxng_results(&parsed, 10);
2645	assert_eq!(results.len(), 2);
2646	assert_eq!(results[0].title, "Rust async");
2647	assert_eq!(results[0].url, "https://example.com/rust");
2648	assert_eq!(results[0].snippet.as_deref(), Some("Result content"));
2649	assert_eq!(results[1].snippet.as_deref(), Some("Fallback snippet"));
2650	}
2651
2652	#[tokio::test]
2653	async fn searxng_provider_requires_base_url() {
2654	use crate::config::SearchProvider;
2655	use crate::tools::spec::{ToolContext, ToolSpec};
2656
2657	let tmp = tempfile::tempdir().expect("tempdir");
2658	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2659	ctx.search_provider = SearchProvider::Searxng;
2660	ctx.search_base_url = None;
2661
2662	let err = WebSearchTool
2663	.execute(json!({"query": "rust async"}), &ctx)
2664	.await
2665	.expect_err("searxng requires explicit base_url");
2666	let msg = err.to_string();
2667	assert!(
2668	matches!(err, crate::tools::spec::ToolError::InvalidInput { .. }),
2669	"missing base_url is a configuration gap, not a transport failure: {err:?}"
2670	);
2671	assert!(
2672	msg.contains("SearXNG")
2673	&& msg.contains("base_url")
2674	&& msg.contains("no public instance"),
2675	"got `{msg}`"
2676	);
2677	}
2678
2679	#[tokio::test]
2680	async fn missing_provider_key_fails_closed_as_not_configured() {
2681	use crate::config::SearchProvider;
2682	use crate::tools::spec::{ToolContext, ToolError, ToolSpec};
2683
2684	for provider in [SearchProvider::Tavily, SearchProvider::Bocha] {
2685	let tmp = tempfile::tempdir().expect("tempdir");
2686	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2687	ctx.search_provider = provider;
2688	ctx.search_api_key = None;
2689
2690	let error = WebSearchTool
2691	.execute(json!({"query": "needs configuration"}), &ctx)
2692	.await
2693	.expect_err("a keyed provider without an API key must fail closed");
2694	assert!(
2695	matches!(error, ToolError::InvalidInput { .. }),
2696	"config gaps must stay distinguishable from transport failures: {error:?}"
2697	);
2698	let message = error.to_string();
2699	assert!(message.contains("is not configured"), "got `{message}`");
2700	assert!(message.contains("api_key"), "got `{message}`");
2701	}
2702	}
2703
2704	#[tokio::test]
2705	async fn searxng_search_returns_json_results() {
2706	use crate::config::SearchProvider;
2707	use crate::tools::spec::{ToolContext, ToolSpec};
2708	use wiremock::matchers::{method, path, query_param};
2709	use wiremock::{Mock, MockServer, ResponseTemplate};
2710
2711	let server = MockServer::start().await;
2712	Mock::given(method("GET"))
2713	.and(path("/search"))
2714	.and(query_param("q", "rust async"))
2715	.and(query_param("format", "json"))
2716	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2717	"results": [
2718	{
2719	"title": "Rust async",
2720	"url": "https://example.com/rust",
2721	"content": "Async Rust result"
2722	}
2723	]
2724	})))
2725	.mount(&server)
2726	.await;
2727
2728	let tmp = tempfile::tempdir().expect("tempdir");
2729	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2730	ctx.search_provider = SearchProvider::Searxng;
2731	ctx.search_base_url = Some(server.uri());
2732
2733	let result = WebSearchTool
2734	.execute(json!({"query": "rust async"}), &ctx)
2735	.await
2736	.expect("searxng endpoint should return results");
2737	let value: serde_json::Value =
2738	serde_json::from_str(&result.content).expect("web search json response");
2739
2740	assert_eq!(value["source"].as_str(), Some("searxng"));
2741	assert_eq!(value["count"].as_u64(), Some(1));
2742	assert_eq!(value["results"][0]["rank"].as_u64(), Some(1));
2743	assert_eq!(value["results"][0]["domain"], "example.com");
2744	assert_eq!(value["receipt"]["backend"], "searxng");
2745	assert_eq!(
2746	value["receipt"]["backend_detail"].as_str(),
2747	Some("127.0.0.1")
2748	);
2749	assert!(
2750	value["message"]
2751	.as_str()
2752	.expect("message")
2753	.contains("Backend: searxng at")
2754	);
2755	}
2756
2757	#[tokio::test]
2758	async fn unsupported_knobs_are_visible_and_domains_are_post_filtered() {
2759	use crate::config::SearchProvider;
2760	use crate::tools::spec::{ToolContext, ToolSpec};
2761	use wiremock::matchers::{method, path, query_param};
2762	use wiremock::{Mock, MockServer, ResponseTemplate};
2763
2764	let server = MockServer::start().await;
2765	Mock::given(method("GET"))
2766	.and(path("/search"))
2767	.and(query_param("q", "fresh rust"))
2768	.and(query_param("format", "json"))
2769	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2770	"results": [
2771	{"title": "Keep", "url": "https://docs.example.com/rust", "content": "kept"},
2772	{"title": "Drop", "url": "https://other.test/rust", "content": "dropped"}
2773	]
2774	})))
2775	.mount(&server)
2776	.await;
2777
2778	let tmp = tempfile::tempdir().expect("tempdir");
2779	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2780	ctx.search_provider = SearchProvider::Searxng;
2781	ctx.search_base_url = Some(server.uri());
2782
2783	let result = WebSearchTool
2784	.execute(
2785	json!({
2786	"query": "fresh rust",
2787	"recency": "week",
2788	"domains": ["example.com"],
2789	"locale": "en-US"
2790	}),
2791	&ctx,
2792	)
2793	.await
2794	.expect("structured query should execute");
2795	let value: serde_json::Value =
2796	serde_json::from_str(&result.content).expect("web search json response");
2797
2798	assert_eq!(value["count"], 1);
2799	assert_eq!(value["results"][0]["domain"], "docs.example.com");
2800	assert_eq!(value["receipt"]["honored"]["domains"], true);
2801	assert_eq!(value["receipt"]["honored"]["recency"], false);
2802	assert_eq!(value["receipt"]["honored"]["locale"], false);
2803	let degraded = value["receipt"]["degraded"]
2804	.as_array()
2805	.expect("degraded receipt array");
2806	assert!(
2807	degraded
2808	.iter()
2809	.any(\|item\| { item["kind"] == "post_filtered" && item["knob"] == "domains" })
2810	);
2811	assert!(
2812	degraded
2813	.iter()
2814	.any(\|item\| { item["kind"] == "knob_ignored" && item["knob"] == "recency" })
2815	);
2816	assert!(
2817	degraded
2818	.iter()
2819	.any(\|item\| { item["kind"] == "knob_ignored" && item["knob"] == "locale" })
2820	);
2821	}
2822
2823	#[test]
2824	fn provider_native_domain_filter_is_reported_as_provider_honored() {
2825	let query = SearchQuery::new(
2826	"current release".to_string(),
2827	3,
2828	Some(Recency::Week),
2829	vec!["example.com".to_string()],
2830	None,
2831	);
2832	let raw = BackendSearch {
2833	backend: BackendId::ProviderNative,
2834	source: "provider-native/xai/grok-4.5".to_string(),
2835	backend_detail: Some("api.x.ai".to_string()),
2836	results: vec![SearchResult::new(
2837	1,
2838	"Exact source".to_string(),
2839	"https://docs.example.com/release".to_string(),
2840	None,
2841	None,
2842	)],
2843	degraded: Vec::new(),
2844	note: Some("Grounded answer.".to_string()),
2845	};
2846	let response = finalize_search_response(
2847	query,
2848	QueryCapabilities {
2849	max_results: CapabilityState::Supported,
2850	recency: CapabilityState::Unsupported,
2851	domains: CapabilityState::Supported,
2852	locale: CapabilityState::Unsupported,
2853	published_date: CapabilityState::Unknown,
2854	},
2855	raw,
2856	Instant::now(),
2857	);
2858
2859	assert!(response.receipt.honored.max_results);
2860	assert!(response.receipt.honored.domains);
2861	assert!(!response.receipt.honored.recency);
2862	assert!(response.receipt.degraded.iter().any(\|reason\| matches!(
2863	reason,
2864	DegradedReason::KnobIgnored {
2865	knob: QueryKnob::Recency
2866	}
2867	)));
2868	assert!(!response.receipt.degraded.iter().any(\|reason\| matches!(
2869	reason,
2870	DegradedReason::PostFiltered {
2871	knob: QueryKnob::Domains
2872	}
2873	)));
2874	}
2875
2876	#[test]
2877	fn search_results_receive_session_scoped_refs_and_sanitize_credential_urls() {
2878	let query = SearchQuery::new("sources".to_string(), 5, None, Vec::new(), None);
2879	let raw = BackendSearch {
2880	backend: BackendId::DuckDuckGo,
2881	source: "duckduckgo".to_string(),
2882	backend_detail: None,
2883	results: vec![
2884	SearchResult::new(
2885	1,
2886	"Valid".to_string(),
2887	"https://example.com/source#section".to_string(),
2888	None,
2889	None,
2890	),
2891	SearchResult::new(
2892	2,
2893	"Protected".to_string(),
2894	"https://example.com/protected?access_token=sensitive&view=full".to_string(),
2895	None,
2896	None,
2897	),
2898	],
2899	degraded: Vec::new(),
2900	note: None,
2901	};
2902	let mut response =
2903	finalize_search_response(query, QueryCapabilities::count_only(), raw, Instant::now());
2904	let context = crate::tools::spec::ToolContext::new(std::path::PathBuf::from("."))
2905	.with_state_namespace("search-citation-session");
2906
2907	register_search_citations(&mut response, &context);
2908
2909	assert_eq!(response.count, 2);
2910	assert_eq!(response.results[0].url, "https://example.com/source");
2911	assert_eq!(
2912	response.results[1].url,
2913	"https://example.com/protected?view=full"
2914	);
2915	assert!(!response.results[1].url.contains("sensitive"));
2916	assert!(response.results[0].ref_id.starts_with("web_"));
2917	assert!(
2918	crate::tools::web::citations::resolve(
2919	"search-citation-session",
2920	&response.results[0].ref_id
2921	)
2922	.is_some()
2923	);
2924	assert!(
2925	crate::tools::web::citations::resolve(
2926	"foreign-search-citation-session",
2927	&response.results[0].ref_id
2928	)
2929	.is_none()
2930	);
2931	}
2932
2933	#[tokio::test]
2934	async fn searxng_empty_results_report_backend() {
2935	use crate::config::SearchProvider;
2936	use crate::tools::spec::ToolContext;
2937	use wiremock::matchers::{method, path, query_param};
2938	use wiremock::{Mock, MockServer, ResponseTemplate};
2939
2940	let server = MockServer::start().await;
2941	Mock::given(method("GET"))
2942	.and(path("/search"))
2943	.and(query_param("q", "empty"))
2944	.and(query_param("format", "json"))
2945	.respond_with(ResponseTemplate::new(200).set_body_json(json!({"results": []})))
2946	.mount(&server)
2947	.await;
2948
2949	let tmp = tempfile::tempdir().expect("tempdir");
2950	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2951	ctx.search_provider = SearchProvider::Searxng;
2952	ctx.search_base_url = Some(server.uri());
2953
2954	let (results, host) = WebSearchTool
2955	.run_searxng_search("empty", 5, 5_000, &ctx)
2956	.await
2957	.expect("empty SearXNG adapter response should be successful");
2958	let expected_host = reqwest::Url::parse(&server.uri())
2959	.expect("mock URL")
2960	.host_str()
2961	.expect("mock host")
2962	.to_string();
2963
2964	assert!(results.is_empty());
2965	assert_eq!(host, expected_host);
2966	}
2967
2968	#[tokio::test]
2969	async fn searxng_http_errors_are_actionable() {
2970	use crate::config::SearchProvider;
2971	use crate::tools::spec::ToolContext;
2972	use wiremock::matchers::{method, path, query_param};
2973	use wiremock::{Mock, MockServer, ResponseTemplate};
2974
2975	let server = MockServer::start().await;
2976	Mock::given(method("GET"))
2977	.and(path("/search"))
2978	.and(query_param("q", "blocked"))
2979	.and(query_param("format", "json"))
2980	.respond_with(ResponseTemplate::new(403).set_body_string("json disabled"))
2981	.mount(&server)
2982	.await;
2983
2984	let tmp = tempfile::tempdir().expect("tempdir");
2985	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
2986	ctx.search_provider = SearchProvider::Searxng;
2987	ctx.search_base_url = Some(server.uri());
2988
2989	let err = WebSearchTool
2990	.run_searxng_search("blocked", 5, 5_000, &ctx)
2991	.await
2992	.expect_err("403 should be actionable");
2993	let msg = err.to_string();
2994	assert!(
2995	msg.contains("HTTP 403")
2996	&& msg.contains("JSON output")
2997	&& msg.contains("permits API access"),
2998	"got `{msg}`"
2999	);
3000	}
3001
3002	#[tokio::test]
3003	async fn searxng_rate_limit_error_mentions_configured_instance() {
3004	use crate::config::SearchProvider;
3005	use crate::tools::spec::ToolContext;
3006	use wiremock::matchers::{method, path, query_param};
3007	use wiremock::{Mock, MockServer, ResponseTemplate};
3008
3009	let server = MockServer::start().await;
3010	Mock::given(method("GET"))
3011	.and(path("/search"))
3012	.and(query_param("q", "later"))
3013	.and(query_param("format", "json"))
3014	.respond_with(ResponseTemplate::new(429).set_body_string("too many requests"))
3015	.mount(&server)
3016	.await;
3017
3018	let tmp = tempfile::tempdir().expect("tempdir");
3019	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
3020	ctx.search_provider = SearchProvider::Searxng;
3021	ctx.search_base_url = Some(server.uri());
3022
3023	let err = WebSearchTool
3024	.run_searxng_search("later", 5, 5_000, &ctx)
3025	.await
3026	.expect_err("429 should be actionable");
3027	let msg = err.to_string();
3028	assert!(
3029	msg.contains("HTTP 429")
3030	&& msg.contains("rate-limiting")
3031	&& msg.contains("trusted/self-hosted instance"),
3032	"got `{msg}`"
3033	);
3034	}
3035
3036	#[tokio::test]
3037	async fn searxng_invalid_json_is_actionable() {
3038	use crate::config::SearchProvider;
3039	use crate::tools::spec::ToolContext;
3040	use wiremock::matchers::{method, path, query_param};
3041	use wiremock::{Mock, MockServer, ResponseTemplate};
3042
3043	let server = MockServer::start().await;
3044	Mock::given(method("GET"))
3045	.and(path("/search"))
3046	.and(query_param("q", "html"))
3047	.and(query_param("format", "json"))
3048	.respond_with(ResponseTemplate::new(200).set_body_string("<html>not json</html>"))
3049	.mount(&server)
3050	.await;
3051
3052	let tmp = tempfile::tempdir().expect("tempdir");
3053	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
3054	ctx.search_provider = SearchProvider::Searxng;
3055	ctx.search_base_url = Some(server.uri());
3056
3057	let err = WebSearchTool
3058	.run_searxng_search("html", 5, 5_000, &ctx)
3059	.await
3060	.expect_err("invalid JSON should be actionable");
3061	let msg = err.to_string();
3062	assert!(
3063	msg.contains("Failed to parse SearXNG JSON response")
3064	&& msg.contains("format=json")
3065	&& msg.contains("JSON output"),
3066	"got `{msg}`"
3067	);
3068	}
3069
3070	#[tokio::test]
3071	async fn custom_duckduckgo_results_report_custom_host_source() {
3072	use crate::config::SearchProvider;
3073	use crate::tools::spec::{ToolContext, ToolSpec};
3074	use wiremock::matchers::{method, path, query_param};
3075	use wiremock::{Mock, MockServer, ResponseTemplate};
3076
3077	let server = MockServer::start().await;
3078	Mock::given(method("GET"))
3079	.and(path("/html/"))
3080	.and(query_param("q", "rust async"))
3081	.respond_with(ResponseTemplate::new(200).set_body_string(
3082	r#"
3083	<html><body>
3084	<a class="result__a" href="https://example.com/rust">Rust async</a>
3085	<div class="result__snippet">Async Rust result</div>
3086	</body></html>
3087	"#,
3088	))
3089	.mount(&server)
3090	.await;
3091
3092	let tmp = tempfile::tempdir().expect("tempdir");
3093	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
3094	ctx.search_provider = SearchProvider::DuckDuckGo;
3095	let base_url = format!("{}/html/", server.uri());
3096	let expected_host = reqwest::Url::parse(&base_url)
3097	.expect("mock server url")
3098	.host_str()
3099	.expect("mock server host")
3100	.to_string();
3101	ctx.search_base_url = Some(base_url);
3102
3103	let result = WebSearchTool
3104	.execute(json!({"query": "rust async"}), &ctx)
3105	.await
3106	.expect("custom endpoint should return results");
3107	let value: serde_json::Value =
3108	serde_json::from_str(&result.content).expect("web search json response");
3109
3110	assert_eq!(value["source"].as_str(), Some(expected_host.as_str()));
3111	assert_eq!(value["count"].as_u64(), Some(1));
3112	}
3113
3114	#[tokio::test]
3115	async fn repeated_search_uses_session_cache_and_marks_receipt() {
3116	use crate::config::SearchProvider;
3117	use crate::tools::spec::{ToolContext, ToolSpec};
3118	use crate::tools::web::cache;
3119	use wiremock::matchers::{method, path, query_param};
3120	use wiremock::{Mock, MockServer, ResponseTemplate};
3121
3122	cache::reset_search();
3123	let server = MockServer::start().await;
3124	Mock::given(method("GET"))
3125	.and(path("/html/"))
3126	.and(query_param("q", "session cache receipt"))
3127	.respond_with(ResponseTemplate::new(200).set_body_string(
3128	r#"
3129	<html><body>
3130	<a class="result__a" href="https://example.com/cached">Cached result</a>
3131	<div class="result__snippet">Fetched once.</div>
3132	</body></html>
3133	"#,
3134	))
3135	.mount(&server)
3136	.await;
3137
3138	let tmp = tempfile::tempdir().expect("tempdir");
3139	let mut context = ToolContext::new(tmp.path().to_path_buf())
3140	.with_state_namespace("web-search-query-cache");
3141	context.search_provider = SearchProvider::DuckDuckGo;
3142	context.search_base_url = Some(format!("{}/html/", server.uri()));
3143
3144	let first = WebSearchTool
3145	.execute(json!({"query": "session cache receipt"}), &context)
3146	.await
3147	.expect("first search should succeed");
3148	let second = WebSearchTool
3149	.execute(json!({"query": "session cache receipt"}), &context)
3150	.await
3151	.expect("second search should hit cache");
3152	let first: serde_json::Value =
3153	serde_json::from_str(&first.content).expect("first response json");
3154	let second: serde_json::Value =
3155	serde_json::from_str(&second.content).expect("second response json");
3156	let requests = server.received_requests().await.expect("recorded requests");
3157
3158	assert_eq!(requests.len(), 1);
3159	assert_eq!(first["receipt"]["cache_hit"], false);
3160	assert_eq!(second["receipt"]["cache_hit"], true);
3161	assert_eq!(second["receipt"]["latency_ms"], 0);
3162	assert_eq!(second["results"], first["results"]);
3163
3164	use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider};
3165	let denied_host = reqwest::Url::parse(&server.uri())
3166	.expect("mock server URL")
3167	.host_str()
3168	.expect("mock server host")
3169	.to_string();
3170	let policy = NetworkPolicy {
3171	default: Decision::Allow.into(),
3172	allow: Vec::new(),
3173	deny: vec![denied_host],
3174	proxy: Vec::new(),
3175	proxy_fake_ip_cidrs: Vec::new(),
3176	audit: false,
3177	};
3178	let blocked = context
3179	.clone()
3180	.with_network_policy(NetworkPolicyDecider::new(policy, None));
3181	let error = WebSearchTool
3182	.execute(json!({"query": "session cache receipt"}), &blocked)
3183	.await
3184	.expect_err("tightened policy must win over the query cache");
3185	assert!(error.to_string().contains("blocked by network policy"));
3186	assert_eq!(
3187	server
3188	.received_requests()
3189	.await
3190	.expect("recorded requests")
3191	.len(),
3192	1
3193	);
3194	}
3195
3196	#[tokio::test]
3197	async fn explicit_bing_does_not_fall_back_to_duckduckgo() {
3198	use crate::config::SearchProvider;
3199	use crate::tools::spec::ToolContext;
3200	use wiremock::matchers::{method, path, query_param};
3201	use wiremock::{Mock, MockServer, ResponseTemplate};
3202
3203	let server = MockServer::start().await;
3204	Mock::given(method("GET"))
3205	.and(path("/bing"))
3206	.and(query_param("q", "one way fallback"))
3207	.respond_with(ResponseTemplate::new(200).set_body_string("<html></html>"))
3208	.mount(&server)
3209	.await;
3210
3211	let tmp = tempfile::tempdir().expect("tempdir");
3212	let mut context = ToolContext::new(tmp.path().to_path_buf());
3213	context.search_provider = SearchProvider::Bing;
3214	context.search_base_url = Some(format!("{}/must-not-be-used", server.uri()));
3215	let query = SearchQuery::new("one way fallback".to_string(), 5, None, Vec::new(), None);
3216	let raw = run_scrape_search_with_endpoints(
3217	SearchProvider::Bing,
3218	&query,
3219	5_000,
3220	&context,
3221	ScrapeEndpoints {
3222	bing: &format!("{}/bing", server.uri()),
3223	allow_bing_fallback: Some(true),
3224	},
3225	)
3226	.await
3227	.expect("empty Bing response is a successful empty search");
3228	let requests = server.received_requests().await.expect("recorded requests");
3229
3230	assert_eq!(raw.backend, BackendId::Bing);
3231	assert!(raw.results.is_empty());
3232	assert!(raw.degraded.is_empty());
3233	assert_eq!(requests.len(), 1);
3234	assert_eq!(requests[0].url.path(), "/bing");
3235	}
3236
3237	#[tokio::test]
3238	async fn custom_duckduckgo_challenge_returns_actionable_error() {
3239	use crate::config::SearchProvider;
3240	use crate::tools::spec::{ToolContext, ToolSpec};
3241	use wiremock::matchers::{method, path, query_param};
3242	use wiremock::{Mock, MockServer, ResponseTemplate};
3243
3244	let server = MockServer::start().await;
3245	Mock::given(method("GET"))
3246	.and(path("/html/"))
3247	.and(query_param("q", "rust async"))
3248	.respond_with(ResponseTemplate::new(200).set_body_string(
3249	r#"<html><body><div class="anomaly-modal">Unfortunately, bots use DuckDuckGo too</div></body></html>"#,
3250	))
3251	.mount(&server)
3252	.await;
3253
3254	let tmp = tempfile::tempdir().expect("tempdir");
3255	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
3256	ctx.search_provider = SearchProvider::DuckDuckGo;
3257	ctx.search_base_url = Some(format!("{}/html/", server.uri()));
3258
3259	let err = WebSearchTool
3260	.execute(json!({"query": "rust async"}), &ctx)
3261	.await
3262	.expect_err("custom endpoint challenge should error");
3263	let msg = err.to_string();
3264	assert!(
3265	msg.contains("DuckDuckGo-compatible search endpoint")
3266	&& msg.contains("bot challenge")
3267	&& msg.contains("private search service"),
3268	"got `{msg}`"
3269	);
3270	}
3271
3272	#[tokio::test]
3273	async fn duckduckgo_challenge_to_bing_success_populates_fallback_receipt() {
3274	use crate::config::SearchProvider;
3275	use crate::tools::spec::ToolContext;
3276	use std::time::Instant;
3277	use wiremock::matchers::{method, path, query_param};
3278	use wiremock::{Mock, MockServer, ResponseTemplate};
3279
3280	let server = MockServer::start().await;
3281	Mock::given(method("GET"))
3282	.and(path("/html/"))
3283	.and(query_param("q", "fallback receipt"))
3284	.respond_with(ResponseTemplate::new(200).set_body_string(
3285	r#"<html><body><div class="anomaly-modal">Unfortunately, bots use DuckDuckGo too</div></body></html>"#,
3286	))
3287	.mount(&server)
3288	.await;
3289	Mock::given(method("GET"))
3290	.and(path("/bing"))
3291	.and(query_param("q", "fallback receipt"))
3292	.respond_with(ResponseTemplate::new(200).set_body_string(
3293	r#"
3294	<ol><li class="b_algo">
3295	<h2><a href="https://example.com/fallback">Fallback result</a></h2>
3296	<div class="b_caption"><p>Bing result after challenge.</p></div>
3297	</li></ol>
3298	"#,
3299	))
3300	.mount(&server)
3301	.await;
3302
3303	let tmp = tempfile::tempdir().expect("tempdir");
3304	let mut context = ToolContext::new(tmp.path().to_path_buf());
3305	context.search_provider = SearchProvider::DuckDuckGo;
3306	context.search_base_url = Some(format!("{}/html/", server.uri()));
3307	let query = SearchQuery::new("fallback receipt".to_string(), 5, None, Vec::new(), None);
3308	let started = Instant::now();
3309	let raw = run_scrape_search_with_endpoints(
3310	SearchProvider::DuckDuckGo,
3311	&query,
3312	5_000,
3313	&context,
3314	ScrapeEndpoints {
3315	bing: &format!("{}/bing", server.uri()),
3316	allow_bing_fallback: Some(true),
3317	},
3318	)
3319	.await
3320	.expect("Bing fallback should succeed");
3321	let response =
3322	finalize_search_response(query, QueryCapabilities::count_only(), raw, started);
3323	let value = serde_json::to_value(&response).expect("response serializes");
3324
3325	assert_eq!(value["source"], "bing");
3326	assert_eq!(value["count"], 1);
3327	assert_eq!(value["receipt"]["backend"], "bing");
3328	assert_eq!(
3329	value["receipt"]["degraded"][0],
3330	json!({"kind": "challenge_detected", "backend": "duckduckgo"})
3331	);
3332	assert_eq!(
3333	value["receipt"]["degraded"][1],
3334	json!({"kind": "scrape_fallback", "from": "duckduckgo", "to": "bing"})
3335	);
3336	assert!(
3337	response
3338	.receipt
3339	.warning()
3340	.expect("warning")
3341	.contains("used bing fallback")
3342	);
3343	}
3344
3345	#[tokio::test]
3346	async fn search_base_url_with_non_duckduckgo_provider_is_explicit_error() {
3347	use crate::config::SearchProvider;
3348	use crate::tools::spec::{ToolContext, ToolSpec};
3349
3350	let tmp = tempfile::tempdir().expect("tempdir");
3351	let mut ctx = ToolContext::new(tmp.path().to_path_buf());
3352	ctx.search_provider = SearchProvider::Tavily;
3353	ctx.search_base_url = Some("https://search.internal.example/html/".to_string());
3354
3355	let err = WebSearchTool
3356	.execute(json!({"query": "rust async"}), &ctx)
3357	.await
3358	.expect_err("non-duckduckgo provider with base_url should error");
3359	let msg = err.to_string();
3360	assert!(
3361	msg.contains("[search].base_url")
3362	&& msg.contains("provider = \"duckduckgo\" or \"searxng\"")
3363	&& msg.contains("tavily"),
3364	"got `{msg}`"
3365	);
3366	}
3367
3368	// ── Offline deterministic corpus ─────────────────────────────────────────
3369	// These tests exercise ranking, deduplication, domain filtering, truncation,
3370	// and citation metadata without any network calls.
3371
3372	#[test]
3373	fn rerank_assigns_sequential_ranks_starting_at_one() {
3374	// Simulates the post-dedup path: ranks may be non-contiguous after a
3375	// result is dropped; rerank must restore a clean 1..N sequence.
3376	let mut results = vec![
3377	SearchResult::new(
3378	5,
3379	"C".to_string(),
3380	"https://c.example.com/".to_string(),
3381	None,
3382	None,
3383	),
3384	SearchResult::new(
3385	3,
3386	"A".to_string(),
3387	"https://a.example.com/".to_string(),
3388	None,
3389	None,
3390	),
3391	SearchResult::new(
3392	1,
3393	"B".to_string(),
3394	"https://b.example.com/".to_string(),
3395	None,
3396	None,
3397	),
3398	];
3399	rerank(&mut results);
3400	assert_eq!(results[0].rank, 1);
3401	assert_eq!(results[1].rank, 2);
3402	assert_eq!(results[2].rank, 3);
3403	}
3404
3405	#[test]
3406	fn rerank_on_empty_slice_is_a_no_op() {
3407	let mut results: Vec<SearchResult> = Vec::new();
3408	rerank(&mut results); // must not panic
3409	}
3410
3411	#[test]
3412	fn register_search_citations_deduplicates_results_with_same_canonical_url() {
3413	// Two result URLs that differ only by fragment normalize to the same
3414	// canonical URL and receive the same ref_id. The second occurrence must
3415	// be removed and ranks re-assigned contiguously.
3416	let namespace = "dedup-test-session-fragments";
3417	let query = SearchQuery::new("deduplicate".to_string(), 5, None, Vec::new(), None);
3418	let raw = BackendSearch {
3419	backend: BackendId::DuckDuckGo,
3420	source: "duckduckgo".to_string(),
3421	backend_detail: None,
3422	results: vec![
3423	SearchResult::new(
3424	1,
3425	"First occurrence".to_string(),
3426	"https://dedup.example.com/page#section-a".to_string(),
3427	Some("first snippet".to_string()),
3428	None,
3429	),
3430	SearchResult::new(
3431	2,
3432	"Unique result".to_string(),
3433	"https://other.dedup.example.com/different".to_string(),
3434	None,
3435	None,
3436	),
3437	SearchResult::new(
3438	3,
3439	"Duplicate of first".to_string(),
3440	"https://dedup.example.com/page#section-b".to_string(),
3441	Some("duplicate snippet".to_string()),
3442	None,
3443	),
3444	],
3445	degraded: Vec::new(),
3446	note: None,
3447	};
3448	let mut response =
3449	finalize_search_response(query, QueryCapabilities::count_only(), raw, Instant::now());
3450	let context = crate::tools::spec::ToolContext::new(std::path::PathBuf::from("."))
3451	.with_state_namespace(namespace);
3452
3453	register_search_citations(&mut response, &context);
3454
3455	assert_eq!(
3456	response.count, 2,
3457	"duplicate canonical URL must reduce the result count"
3458	);
3459	assert_eq!(response.results.len(), 2);
3460	// After deduplication, ranks must be re-assigned contiguously.
3461	assert_eq!(response.results[0].rank, 1);
3462	assert_eq!(response.results[1].rank, 2);
3463	// First occurrence survives with its canonical (fragment-stripped) URL.
3464	assert_eq!(response.results[0].url, "https://dedup.example.com/page");
3465	// The unique URL survives untouched.
3466	assert_eq!(
3467	response.results[1].url,
3468	"https://other.dedup.example.com/different"
3469	);
3470	// The two surviving results must carry distinct ref_ids.
3471	assert_ne!(
3472	response.results[0].ref_id, response.results[1].ref_id,
3473	"surviving results must have distinct ref_ids"
3474	);
3475	// Updated count and message must reflect the deduplicated set.
3476	assert!(response.message.contains('2'), "{}", response.message);
3477	}
3478
3479	#[test]
3480	fn register_search_citations_preserves_title_url_and_ref_id_metadata() {
3481	// Verifies that after citation registration, every result has a
3482	// non-empty web_-prefixed ref_id, the normalized URL, and the
3483	// original title; and that the ref_id is resolvable in the session.
3484	let namespace = "citation-metadata-test-session";
3485	let query = SearchQuery::new("docs".to_string(), 5, None, Vec::new(), None);
3486	let raw = BackendSearch {
3487	backend: BackendId::DuckDuckGo,
3488	source: "duckduckgo".to_string(),
3489	backend_detail: None,
3490	results: vec![SearchResult::new(
3491	1,
3492	"Official Docs".to_string(),
3493	"https://docs.citation-meta.example.com/reference".to_string(),
3494	Some("Comprehensive reference documentation.".to_string()),
3495	None,
3496	)],
3497	degraded: Vec::new(),
3498	note: None,
3499	};
3500	let mut response =
3501	finalize_search_response(query, QueryCapabilities::count_only(), raw, Instant::now());
3502	let context = crate::tools::spec::ToolContext::new(std::path::PathBuf::from("."))
3503	.with_state_namespace(namespace);
3504
3505	register_search_citations(&mut response, &context);
3506
3507	assert_eq!(response.count, 1);
3508	let result = &response.results[0];
3509	// ref_id must be populated with the canonical prefix.
3510	assert!(
3511	result.ref_id.starts_with("web_"),
3512	"ref_id must use web_ prefix; got `{}`",
3513	result.ref_id
3514	);
3515	assert_eq!(result.title, "Official Docs");
3516	assert_eq!(
3517	result.url,
3518	"https://docs.citation-meta.example.com/reference"
3519	);
3520	assert_eq!(result.rank, 1);
3521	// The citation must be resolvable in the session that minted it.
3522	let citation = crate::tools::web::citations::resolve(namespace, &result.ref_id)
3523	.expect("citation must be registered and resolvable in its session");
3524	assert_eq!(citation.ref_id, result.ref_id);
3525	assert_eq!(citation.url, result.url);
3526	assert_eq!(citation.title.as_deref(), Some("Official Docs"));
3527	assert!(
3528	!citation.retrieved_at.is_empty(),
3529	"retrieved_at must be set to the retrieval timestamp"
3530	);
3531	// Must not be resolvable in a foreign session.
3532	assert!(
3533	crate::tools::web::citations::resolve("other-session", &result.ref_id).is_none(),
3534	"citation must not leak to foreign sessions"
3535	);
3536	}
3537
3538	#[test]
3539	fn finalize_search_response_truncates_to_max_results_and_reranks() {
3540	// Backends may return more than max_results; finalize must clip and
3541	// reassign contiguous ranks so no gap appears in the model-visible output.
3542	let query = SearchQuery::new("truncate me".to_string(), 2, None, Vec::new(), None);
3543	let raw = BackendSearch {
3544	backend: BackendId::DuckDuckGo,
3545	source: "duckduckgo".to_string(),
3546	backend_detail: None,
3547	results: vec![
3548	SearchResult::new(
3549	1,
3550	"A".to_string(),
3551	"https://a.trunc.example.com/".to_string(),
3552	None,
3553	None,
3554	),
3555	SearchResult::new(
3556	2,
3557	"B".to_string(),
3558	"https://b.trunc.example.com/".to_string(),
3559	None,
3560	None,
3561	),
3562	SearchResult::new(
3563	3,
3564	"C".to_string(),
3565	"https://c.trunc.example.com/".to_string(),
3566	None,
3567	None,
3568	),
3569	],
3570	degraded: Vec::new(),
3571	note: None,
3572	};
3573
3574	let response =
3575	finalize_search_response(query, QueryCapabilities::count_only(), raw, Instant::now());
3576
3577	assert_eq!(response.count, 2, "must be truncated to max_results");
3578	assert_eq!(response.results.len(), 2);
3579	assert_eq!(response.results[0].rank, 1);
3580	assert_eq!(response.results[1].rank, 2);
3581	assert_eq!(response.results[0].title, "A");
3582	assert_eq!(response.results[1].title, "B");
3583	assert!(response.message.contains('2'), "{}", response.message);
3584	}
3585
3586	#[test]
3587	fn domain_matches_handles_subdomains_www_prefix_and_empty_list() {
3588	// Empty domain list: every URL passes (no filtering requested).
3589	assert!(
3590	domain_matches("https://any.example.com/page", &[]),
3591	"empty domain list must accept all URLs"
3592	);
3593	// Exact host match.
3594	assert!(domain_matches(
3595	"https://example.com/page",
3596	&["example.com".to_string()]
3597	));
3598	// Subdomain match: docs.example.com is a subdomain of example.com.
3599	assert!(domain_matches(
3600	"https://docs.example.com/page",
3601	&["example.com".to_string()]
3602	));
3603	// www-prefixed URL matches bare domain.
3604	assert!(domain_matches(
3605	"https://www.example.com/page",
3606	&["example.com".to_string()]
3607	));
3608	// www-prefixed domain filter matches bare URL.
3609	assert!(domain_matches(
3610	"https://example.com/page",
3611	&["www.example.com".to_string()]
3612	));
3613	// Completely different domain must not match.
3614	assert!(!domain_matches(
3615	"https://other.com/page",
3616	&["example.com".to_string()]
3617	));
3618	// A domain that shares a suffix but is not a subdomain must not match
3619	// (prevents "notexample.com" matching filter "example.com").
3620	assert!(!domain_matches(
3621	"https://notexample.com/page",
3622	&["example.com".to_string()]
3623	));
3624	}
3625
3626	#[test]
3627	fn finalize_search_response_domain_post_filter_reranks_survivors() {
3628	// When domain filtering is applied post-search, results that do not
3629	// match must be removed and the survivors re-ranked from 1.
3630	let query = SearchQuery::new(
3631	"domain filter".to_string(),
3632	5,
3633	None,
3634	vec!["keep.example.com".to_string()],
3635	None,
3636	);
3637	let raw = BackendSearch {
3638	backend: BackendId::DuckDuckGo,
3639	source: "duckduckgo".to_string(),
3640	backend_detail: None,
3641	results: vec![
3642	SearchResult::new(
3643	1,
3644	"Drop this".to_string(),
3645	"https://other.example.com/page".to_string(),
3646	None,
3647	None,
3648	),
3649	SearchResult::new(
3650	2,
3651	"Keep this".to_string(),
3652	"https://keep.example.com/page".to_string(),
3653	None,
3654	None,
3655	),
3656	SearchResult::new(
3657	3,
3658	"Also drop".to_string(),
3659	"https://unrelated.example.com/page".to_string(),
3660	None,
3661	None,
3662	),
3663	],
3664	degraded: Vec::new(),
3665	note: None,
3666	};
3667
3668	let response =
3669	finalize_search_response(query, QueryCapabilities::count_only(), raw, Instant::now());
3670
3671	assert_eq!(response.count, 1, "only the matching domain must survive");
3672	assert_eq!(
3673	response.results[0].rank, 1,
3674	"survivor must be re-ranked to 1"
3675	);
3676	assert_eq!(response.results[0].title, "Keep this");
3677	// The receipt must record post-filtering as a degraded reason.
3678	assert!(
3679	response.receipt.degraded.iter().any(\|reason\| matches!(
3680	reason,
3681	DegradedReason::PostFiltered {
3682	knob: QueryKnob::Domains
3683	}
3684	)),
3685	"post-filtered degraded reason must be present"
3686	);
3687	}
3688
3689	#[test]
3690	fn fallback_receipt_carries_full_backend_chain_history() {
3691	// Verifies that the machine-readable degraded vec records every hop in
3692	// the fallback chain so callers can audit exactly what happened.
3693	let receipt = crate::tools::web::contract::SearchReceipt {
3694	backend: BackendId::Bing,
3695	backend_detail: None,
3696	requested: SearchQuery::new("fallback chain".to_string(), 5, None, Vec::new(), None),
3697	capabilities: QueryCapabilities::count_only(),
3698	honored: crate::tools::web::contract::HonoredQueryCapabilities {
3699	max_results: true,
3700	..Default::default()
3701	},
3702	degraded: vec![
3703	DegradedReason::ChallengeDetected {
3704	backend: BackendId::DuckDuckGo,
3705	},
3706	DegradedReason::ScrapeFallback {
3707	from: BackendId::DuckDuckGo,
3708	to: BackendId::Bing,
3709	},
3710	],
3711	latency_ms: 42,
3712	cache_hit: false,
3713	};
3714
3715	let value = serde_json::to_value(&receipt).expect("receipt must serialize");
3716	assert_eq!(value["backend"], "bing");
3717	assert_eq!(value["degraded"].as_array().unwrap().len(), 2);
3718	assert_eq!(value["degraded"][0]["kind"], "challenge_detected");
3719	assert_eq!(value["degraded"][0]["backend"], "duckduckgo");
3720	assert_eq!(value["degraded"][1]["kind"], "scrape_fallback");
3721	assert_eq!(value["degraded"][1]["from"], "duckduckgo");
3722	assert_eq!(value["degraded"][1]["to"], "bing");
3723
3724	let warning = receipt
3725	.warning()
3726	.expect("degraded receipt must produce a warning");
3727	assert!(warning.contains("bot challenge"), "{warning}");
3728	assert!(warning.contains("used bing fallback"), "{warning}");
3729	}
3730	}
3731

3731 lines RUST