| 1 | //! Web search tool backed by DuckDuckGo HTML results (with Bing fallback). |
| 2 | //! |
| 3 | //! This is the primary web search surface for agents. For browsing workflows |
| 4 | //! (page open, click, screenshot) use a direct URL approach instead. |
| 5 | |
| 6 | use super::spec::{ |
| 7 | ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64, |
| 8 | }; |
| 9 | use crate::network_policy::{Decision, NetworkPolicyDecider}; |
| 10 | use async_trait::async_trait; |
| 11 | use base64::{Engine as _, engine::general_purpose}; |
| 12 | use regex::Regex; |
| 13 | use serde::Serialize; |
| 14 | use serde_json::{Value, json}; |
| 15 | use std::sync::OnceLock; |
| 16 | use std::time::Duration; |
| 17 | |
| 18 | const DUCKDUCKGO_HOST: &str = "html.duckduckgo.com"; |
| 19 | const BING_HOST: &str = "www.bing.com"; |
| 20 | |
| 21 | /// Returns `Ok(())` if the policy allows the call, or a `ToolError` otherwise. |
| 22 | /// Falls through silently when no policy is attached (back-compat). |
| 23 | fn check_policy(decider: Option<&NetworkPolicyDecider>, host: &str) -> Result<(), ToolError> { |
| 24 | let Some(decider) = decider else { |
| 25 | return Ok(()); |
| 26 | }; |
| 27 | match decider.evaluate(host, "web_search") { |
| 28 | Decision::Allow => Ok(()), |
| 29 | Decision::Deny => Err(ToolError::permission_denied(format!( |
| 30 | "web search to '{host}' blocked by network policy" |
| 31 | ))), |
| 32 | Decision::Prompt => Err(ToolError::permission_denied(format!( |
| 33 | "web search to '{host}' requires approval; \ |
| 34 | re-run after `/network allow {host}` or set network.default = \"allow\" in config" |
| 35 | ))), |
| 36 | } |
| 37 | } |
| 38 | |
| 39 | // Cached regex patterns for HTML parsing |
| 40 | static TITLE_RE: OnceLock<Regex> = OnceLock::new(); |
| 41 | static SNIPPET_RE: OnceLock<Regex> = OnceLock::new(); |
| 42 | static TAG_RE: OnceLock<Regex> = OnceLock::new(); |
| 43 | static BING_RESULT_RE: OnceLock<Regex> = OnceLock::new(); |
| 44 | static BING_TITLE_RE: OnceLock<Regex> = OnceLock::new(); |
| 45 | static BING_SNIPPET_RE: OnceLock<Regex> = OnceLock::new(); |
| 46 | |
| 47 | fn get_title_re() -> &'static Regex { |
| 48 | TITLE_RE.get_or_init(|| { |
| 49 | Regex::new(r#"<a[^>]*class=\"result__a\"[^>]*href=\"([^\"]+)\"[^>]*>(.*?)</a>"#) |
| 50 | .expect("title regex pattern is valid") |
| 51 | }) |
| 52 | } |
| 53 | |
| 54 | fn get_snippet_re() -> &'static Regex { |
| 55 | SNIPPET_RE.get_or_init(|| { |
| 56 | Regex::new( |
| 57 | r#"<a[^>]*class=\"result__snippet\"[^>]*>(.*?)</a>|<div[^>]*class=\"result__snippet\"[^>]*>(.*?)</div>"#, |
| 58 | ) |
| 59 | .expect("snippet regex pattern is valid") |
| 60 | }) |
| 61 | } |
| 62 | |
| 63 | fn get_tag_re() -> &'static Regex { |
| 64 | TAG_RE.get_or_init(|| Regex::new(r"<[^>]+>").expect("tag regex pattern is valid")) |
| 65 | } |
| 66 | |
| 67 | fn get_bing_result_re() -> &'static Regex { |
| 68 | BING_RESULT_RE.get_or_init(|| { |
| 69 | Regex::new(r#"(?is)<li[^>]*class=\"[^\"]*\bb_algo\b[^\"]*\"[^>]*>(.*?)</li>"#) |
| 70 | .expect("bing result regex pattern is valid") |
| 71 | }) |
| 72 | } |
| 73 | |
| 74 | fn get_bing_title_re() -> &'static Regex { |
| 75 | BING_TITLE_RE.get_or_init(|| { |
| 76 | Regex::new(r#"(?is)<h2[^>]*>.*?<a[^>]*href=\"([^\"]+)\"[^>]*>(.*?)</a>"#) |
| 77 | .expect("bing title regex pattern is valid") |
| 78 | }) |
| 79 | } |
| 80 | |
| 81 | fn get_bing_snippet_re() -> &'static Regex { |
| 82 | BING_SNIPPET_RE.get_or_init(|| { |
| 83 | Regex::new(r#"(?is)<div[^>]*class=\"[^\"]*\bb_caption\b[^\"]*\"[^>]*>.*?<p[^>]*>(.*?)</p>"#) |
| 84 | .expect("bing snippet regex pattern is valid") |
| 85 | }) |
| 86 | } |
| 87 | |
| 88 | const DEFAULT_MAX_RESULTS: usize = 5; |
| 89 | const MAX_RESULTS: usize = 10; |
| 90 | const DEFAULT_TIMEOUT_MS: u64 = 15_000; |
| 91 | const USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"; |
| 92 | |
| 93 | #[derive(Debug, Clone, Serialize)] |
| 94 | struct WebSearchEntry { |
| 95 | title: String, |
| 96 | url: String, |
| 97 | snippet: Option<String>, |
| 98 | } |
| 99 | |
| 100 | #[derive(Debug, Clone, Serialize)] |
| 101 | struct WebSearchResponse { |
| 102 | query: String, |
| 103 | source: String, |
| 104 | count: usize, |
| 105 | message: String, |
| 106 | results: Vec<WebSearchEntry>, |
| 107 | } |
| 108 | |
| 109 | pub struct WebSearchTool; |
| 110 | |
| 111 | #[async_trait] |
| 112 | impl ToolSpec for WebSearchTool { |
| 113 | fn name(&self) -> &'static str { |
| 114 | "web_search" |
| 115 | } |
| 116 | |
| 117 | fn description(&self) -> &'static str { |
| 118 | "Search the web using DuckDuckGo or Bing and return structured results with URLs and snippets." |
| 119 | } |
| 120 | |
| 121 | fn input_schema(&self) -> Value { |
| 122 | json!({ |
| 123 | "type": "object", |
| 124 | "properties": { |
| 125 | "query": { |
| 126 | "type": "string", |
| 127 | "description": "Search query. Compatibility aliases: q, or search_query[0].q." |
| 128 | }, |
| 129 | "q": { |
| 130 | "type": "string", |
| 131 | "description": "Search query." |
| 132 | }, |
| 133 | "search_query": { |
| 134 | "type": "array", |
| 135 | "description": "Array form for advanced queries: [{\"q\":\"...\", \"max_results\": 5}]", |
| 136 | "items": { |
| 137 | "type": "object", |
| 138 | "properties": { |
| 139 | "q": { "type": "string" }, |
| 140 | "query": { "type": "string" }, |
| 141 | "max_results": { "type": "integer" } |
| 142 | } |
| 143 | } |
| 144 | }, |
| 145 | "max_results": { |
| 146 | "type": "integer", |
| 147 | "description": "Maximum number of results to return (default: 5, max: 10)" |
| 148 | }, |
| 149 | "timeout_ms": { |
| 150 | "type": "integer", |
| 151 | "description": "Timeout in milliseconds (default: 15000, max: 60000)" |
| 152 | } |
| 153 | } |
| 154 | }) |
| 155 | } |
| 156 | |
| 157 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 158 | vec![ToolCapability::ReadOnly, ToolCapability::Network] |
| 159 | } |
| 160 | |
| 161 | fn approval_requirement(&self) -> ApprovalRequirement { |
| 162 | ApprovalRequirement::Auto |
| 163 | } |
| 164 | |
| 165 | async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> { |
| 166 | let query = extract_search_query(&input)?; |
| 167 | if query.is_empty() { |
| 168 | return Err(ToolError::invalid_input("Query cannot be empty")); |
| 169 | } |
| 170 | let max_results = |
| 171 | usize::try_from(optional_search_max_results(&input)).unwrap_or(DEFAULT_MAX_RESULTS); |
| 172 | let max_results = max_results.clamp(1, MAX_RESULTS); |
| 173 | let timeout_ms = optional_u64(&input, "timeout_ms", DEFAULT_TIMEOUT_MS).min(60_000); |
| 174 | |
| 175 | // Per-domain network policy gate (#135). The "host" for web search is |
| 176 | // the upstream search engine domain — DuckDuckGo first, Bing on |
| 177 | // fallback. We gate DuckDuckGo here; Bing is gated separately inside |
| 178 | // `run_bing_search` so a deny on one engine doesn't block the other. |
| 179 | let decider = context.network_policy.as_ref(); |
| 180 | check_policy(decider, DUCKDUCKGO_HOST)?; |
| 181 | |
| 182 | let client = reqwest::Client::builder() |
| 183 | .timeout(Duration::from_millis(timeout_ms)) |
| 184 | .user_agent(USER_AGENT) |
| 185 | .build() |
| 186 | .map_err(|e| { |
| 187 | ToolError::execution_failed(format!("Failed to build HTTP client: {e}")) |
| 188 | })?; |
| 189 | |
| 190 | let encoded = url_encode(&query); |
| 191 | let url = format!("https://html.duckduckgo.com/html/?q={encoded}"); |
| 192 | let resp = client |
| 193 | .get(&url) |
| 194 | .header( |
| 195 | "Accept", |
| 196 | "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", |
| 197 | ) |
| 198 | .header("Accept-Language", "en-US,en;q=0.5") |
| 199 | .send() |
| 200 | .await |
| 201 | .map_err(|e| ToolError::execution_failed(format!("Web search request failed: {e}")))?; |
| 202 | |
| 203 | let status = resp.status(); |
| 204 | let body = resp |
| 205 | .text() |
| 206 | .await |
| 207 | .map_err(|e| ToolError::execution_failed(format!("Failed to read response: {e}")))?; |
| 208 | |
| 209 | if !status.is_success() { |
| 210 | return Err(ToolError::execution_failed(format!( |
| 211 | "Web search failed: HTTP {}", |
| 212 | status.as_u16() |
| 213 | ))); |
| 214 | } |
| 215 | |
| 216 | let mut results = parse_duckduckgo_results(&body, max_results); |
| 217 | let mut source = "duckduckgo".to_string(); |
| 218 | let mut message_suffix = None; |
| 219 | if results.is_empty() { |
| 220 | let duckduckgo_blocked = is_duckduckgo_challenge(&body); |
| 221 | // Bing is a separate host — gate it independently so a deny on |
| 222 | // DuckDuckGo doesn't silently let Bing through (and vice versa). |
| 223 | check_policy(decider, BING_HOST)?; |
| 224 | match run_bing_search(&client, &query, max_results).await { |
| 225 | Ok(fallback_results) if !fallback_results.is_empty() => { |
| 226 | results = fallback_results; |
| 227 | source = "bing".to_string(); |
| 228 | message_suffix = Some(if duckduckgo_blocked { |
| 229 | "DuckDuckGo returned a bot challenge; used Bing fallback" |
| 230 | } else { |
| 231 | "DuckDuckGo returned no parseable results; used Bing fallback" |
| 232 | }); |
| 233 | } |
| 234 | Ok(_) if duckduckgo_blocked => { |
| 235 | return Err(ToolError::execution_failed( |
| 236 | "DuckDuckGo returned a bot challenge and Bing fallback returned no results", |
| 237 | )); |
| 238 | } |
| 239 | Err(err) if duckduckgo_blocked => { |
| 240 | return Err(ToolError::execution_failed(format!( |
| 241 | "DuckDuckGo returned a bot challenge and Bing fallback failed: {err}" |
| 242 | ))); |
| 243 | } |
| 244 | Ok(_) | Err(_) => {} |
| 245 | } |
| 246 | } |
| 247 | let message = if results.is_empty() { |
| 248 | "No results found".to_string() |
| 249 | } else if let Some(suffix) = message_suffix { |
| 250 | format!("Found {} result(s). {suffix}", results.len()) |
| 251 | } else { |
| 252 | format!("Found {} result(s)", results.len()) |
| 253 | }; |
| 254 | |
| 255 | let response = WebSearchResponse { |
| 256 | query, |
| 257 | source, |
| 258 | count: results.len(), |
| 259 | message, |
| 260 | results, |
| 261 | }; |
| 262 | |
| 263 | ToolResult::json(&response).map_err(|e| ToolError::execution_failed(e.to_string())) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | fn extract_search_query(input: &Value) -> Result<String, ToolError> { |
| 268 | for key in ["query", "q"] { |
| 269 | if let Some(value) = input.get(key) { |
| 270 | let Some(query) = value.as_str() else { |
| 271 | return Err(ToolError::invalid_input(format!( |
| 272 | "Field '{key}' must be a string" |
| 273 | ))); |
| 274 | }; |
| 275 | let query = query.trim(); |
| 276 | if !query.is_empty() { |
| 277 | return Ok(query.to_string()); |
| 278 | } |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | for item in search_query_items(input) { |
| 283 | for key in ["q", "query"] { |
| 284 | if let Some(value) = item.get(key) { |
| 285 | let Some(query) = value.as_str() else { |
| 286 | return Err(ToolError::invalid_input(format!( |
| 287 | "Field 'search_query[].{key}' must be a string" |
| 288 | ))); |
| 289 | }; |
| 290 | let query = query.trim(); |
| 291 | if !query.is_empty() { |
| 292 | return Ok(query.to_string()); |
| 293 | } |
| 294 | } |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | Err(ToolError::missing_field("query")) |
| 299 | } |
| 300 | |
| 301 | fn optional_search_max_results(input: &Value) -> u64 { |
| 302 | if let Some(value) = input.get("max_results").and_then(Value::as_u64) { |
| 303 | return value; |
| 304 | } |
| 305 | search_query_items(input) |
| 306 | .filter_map(|item| item.get("max_results").and_then(Value::as_u64)) |
| 307 | .next() |
| 308 | .unwrap_or(DEFAULT_MAX_RESULTS as u64) |
| 309 | } |
| 310 | |
| 311 | fn search_query_items(input: &Value) -> impl Iterator<Item = &Value> { |
| 312 | input |
| 313 | .get("search_query") |
| 314 | .and_then(Value::as_array) |
| 315 | .into_iter() |
| 316 | .flat_map(|items| items.iter()) |
| 317 | } |
| 318 | |
| 319 | async fn run_bing_search( |
| 320 | client: &reqwest::Client, |
| 321 | query: &str, |
| 322 | max_results: usize, |
| 323 | ) -> Result<Vec<WebSearchEntry>, ToolError> { |
| 324 | let encoded = url_encode(query); |
| 325 | let url = format!("https://www.bing.com/search?q={encoded}"); |
| 326 | let resp = client |
| 327 | .get(&url) |
| 328 | .header( |
| 329 | "Accept", |
| 330 | "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", |
| 331 | ) |
| 332 | .header("Accept-Language", "en-US,en;q=0.9") |
| 333 | .send() |
| 334 | .await |
| 335 | .map_err(|e| ToolError::execution_failed(format!("Bing fallback request failed: {e}")))?; |
| 336 | |
| 337 | let status = resp.status(); |
| 338 | let body = resp.text().await.map_err(|e| { |
| 339 | ToolError::execution_failed(format!("Failed to read Bing fallback response: {e}")) |
| 340 | })?; |
| 341 | |
| 342 | if !status.is_success() { |
| 343 | return Err(ToolError::execution_failed(format!( |
| 344 | "Bing fallback failed: HTTP {}", |
| 345 | status.as_u16() |
| 346 | ))); |
| 347 | } |
| 348 | |
| 349 | Ok(parse_bing_results(&body, max_results)) |
| 350 | } |
| 351 | |
| 352 | fn parse_duckduckgo_results(html: &str, max_results: usize) -> Vec<WebSearchEntry> { |
| 353 | let title_re = get_title_re(); |
| 354 | let snippet_re = get_snippet_re(); |
| 355 | let snippets: Vec<String> = snippet_re |
| 356 | .captures_iter(html) |
| 357 | .filter_map(|cap| cap.get(1).or_else(|| cap.get(2))) |
| 358 | .map(|m| normalize_text(m.as_str())) |
| 359 | .collect(); |
| 360 | |
| 361 | let mut results = Vec::new(); |
| 362 | for (idx, cap) in title_re.captures_iter(html).enumerate() { |
| 363 | if results.len() >= max_results { |
| 364 | break; |
| 365 | } |
| 366 | let href = cap.get(1).map(|m| m.as_str()).unwrap_or(""); |
| 367 | let title_raw = cap.get(2).map(|m| m.as_str()).unwrap_or(""); |
| 368 | let title = normalize_text(title_raw); |
| 369 | if title.is_empty() { |
| 370 | continue; |
| 371 | } |
| 372 | let url = normalize_url(href); |
| 373 | let snippet = snippets |
| 374 | .get(idx) |
| 375 | .map(|s| s.to_string()) |
| 376 | .filter(|s| !s.is_empty()); |
| 377 | |
| 378 | results.push(WebSearchEntry { |
| 379 | title, |
| 380 | url, |
| 381 | snippet, |
| 382 | }); |
| 383 | } |
| 384 | |
| 385 | results |
| 386 | } |
| 387 | |
| 388 | fn is_duckduckgo_challenge(html: &str) -> bool { |
| 389 | html.contains("anomaly-modal") || html.contains("Unfortunately, bots use DuckDuckGo too") |
| 390 | } |
| 391 | |
| 392 | fn parse_bing_results(html: &str, max_results: usize) -> Vec<WebSearchEntry> { |
| 393 | let mut results = Vec::new(); |
| 394 | for cap in get_bing_result_re().captures_iter(html) { |
| 395 | if results.len() >= max_results { |
| 396 | break; |
| 397 | } |
| 398 | let Some(block) = cap.get(1).map(|m| m.as_str()) else { |
| 399 | continue; |
| 400 | }; |
| 401 | let Some(title_cap) = get_bing_title_re().captures(block) else { |
| 402 | continue; |
| 403 | }; |
| 404 | let href = title_cap.get(1).map(|m| m.as_str()).unwrap_or(""); |
| 405 | let title_raw = title_cap.get(2).map(|m| m.as_str()).unwrap_or(""); |
| 406 | let title = normalize_text(title_raw); |
| 407 | if title.is_empty() { |
| 408 | continue; |
| 409 | } |
| 410 | let snippet = get_bing_snippet_re() |
| 411 | .captures(block) |
| 412 | .and_then(|snippet_cap| snippet_cap.get(1)) |
| 413 | .map(|m| normalize_text(m.as_str())) |
| 414 | .filter(|s| !s.is_empty()); |
| 415 | |
| 416 | results.push(WebSearchEntry { |
| 417 | title, |
| 418 | url: normalize_bing_url(href), |
| 419 | snippet, |
| 420 | }); |
| 421 | } |
| 422 | |
| 423 | results |
| 424 | } |
| 425 | |
| 426 | fn normalize_url(href: &str) -> String { |
| 427 | if let Some(uddg) = extract_query_param(href, "uddg") { |
| 428 | let decoded = percent_decode(&uddg); |
| 429 | if !decoded.is_empty() { |
| 430 | return decoded; |
| 431 | } |
| 432 | } |
| 433 | if href.starts_with("//") { |
| 434 | return format!("https:{href}"); |
| 435 | } |
| 436 | if href.starts_with('/') { |
| 437 | return format!("https://duckduckgo.com{href}"); |
| 438 | } |
| 439 | href.to_string() |
| 440 | } |
| 441 | |
| 442 | fn normalize_bing_url(href: &str) -> String { |
| 443 | if let Some(encoded) = extract_query_param(href, "u") { |
| 444 | let decoded = percent_decode(&encoded); |
| 445 | let token = decoded.strip_prefix("a1").unwrap_or(&decoded); |
| 446 | let mut padded = token.replace('-', "+").replace('_', "/"); |
| 447 | while !padded.len().is_multiple_of(4) { |
| 448 | padded.push('='); |
| 449 | } |
| 450 | if let Ok(bytes) = general_purpose::STANDARD.decode(padded) |
| 451 | && let Ok(url) = String::from_utf8(bytes) |
| 452 | && (url.starts_with("http://") || url.starts_with("https://")) |
| 453 | { |
| 454 | return url; |
| 455 | } |
| 456 | } |
| 457 | if href.starts_with("//") { |
| 458 | return format!("https:{href}"); |
| 459 | } |
| 460 | if href.starts_with('/') { |
| 461 | return format!("https://www.bing.com{href}"); |
| 462 | } |
| 463 | href.to_string() |
| 464 | } |
| 465 | |
| 466 | fn normalize_text(text: &str) -> String { |
| 467 | let stripped = strip_html_tags(text); |
| 468 | let decoded = decode_html_entities(&stripped); |
| 469 | decoded.split_whitespace().collect::<Vec<_>>().join(" ") |
| 470 | } |
| 471 | |
| 472 | fn strip_html_tags(text: &str) -> String { |
| 473 | get_tag_re().replace_all(text, "").to_string() |
| 474 | } |
| 475 | |
| 476 | fn decode_html_entities(text: &str) -> String { |
| 477 | use regex::Regex; |
| 478 | use std::sync::OnceLock; |
| 479 | |
| 480 | static ENTITY_RE: OnceLock<Regex> = OnceLock::new(); |
| 481 | let re = ENTITY_RE.get_or_init(|| { |
| 482 | Regex::new(r"&(?:#(\d+)|#x([0-9A-Fa-f]+)|([a-zA-Z]+));").expect("HTML entity regex") |
| 483 | }); |
| 484 | |
| 485 | re.replace_all(text, |caps: ®ex::Captures| { |
| 486 | if let Some(dec) = caps.get(1) { |
| 487 | return dec |
| 488 | .as_str() |
| 489 | .parse::<u32>() |
| 490 | .ok() |
| 491 | .and_then(std::char::from_u32) |
| 492 | .unwrap_or('\u{FFFD}') |
| 493 | .to_string(); |
| 494 | } |
| 495 | if let Some(hex) = caps.get(2) { |
| 496 | return u32::from_str_radix(hex.as_str(), 16) |
| 497 | .ok() |
| 498 | .and_then(std::char::from_u32) |
| 499 | .unwrap_or('\u{FFFD}') |
| 500 | .to_string(); |
| 501 | } |
| 502 | let named = caps.get(3).map(|m| m.as_str()); |
| 503 | match named { |
| 504 | Some("amp") => "&", |
| 505 | Some("lt") => "<", |
| 506 | Some("gt") => ">", |
| 507 | Some("quot") => "\"", |
| 508 | Some("apos") => "'", |
| 509 | Some("nbsp") => " ", |
| 510 | Some("copy") => "\u{00A9}", |
| 511 | Some("reg") => "\u{00AE}", |
| 512 | Some("mdash") => "\u{2014}", |
| 513 | Some("ndash") => "\u{2013}", |
| 514 | Some("lsquo") => "\u{2018}", |
| 515 | Some("rsquo") => "\u{2019}", |
| 516 | Some("ldquo") => "\u{201C}", |
| 517 | Some("rdquo") => "\u{201D}", |
| 518 | Some("hellip") => "\u{2026}", |
| 519 | _ => return caps.get(0).map(|m| m.as_str()).unwrap_or("").to_string(), |
| 520 | } |
| 521 | .to_string() |
| 522 | }) |
| 523 | .to_string() |
| 524 | } |
| 525 | |
| 526 | fn url_encode(input: &str) -> String { |
| 527 | crate::utils::url_encode(input) |
| 528 | } |
| 529 | |
| 530 | fn percent_decode(input: &str) -> String { |
| 531 | let bytes = input.as_bytes(); |
| 532 | let mut out = Vec::new(); |
| 533 | let mut i = 0; |
| 534 | while i < bytes.len() { |
| 535 | match bytes[i] { |
| 536 | b'%' if i + 2 < bytes.len() => { |
| 537 | let hex = &input[i + 1..i + 3]; |
| 538 | if let Ok(val) = u8::from_str_radix(hex, 16) { |
| 539 | out.push(val); |
| 540 | i += 3; |
| 541 | continue; |
| 542 | } |
| 543 | out.push(bytes[i]); |
| 544 | } |
| 545 | b'+' => out.push(b' '), |
| 546 | _ => out.push(bytes[i]), |
| 547 | } |
| 548 | i += 1; |
| 549 | } |
| 550 | String::from_utf8_lossy(&out).to_string() |
| 551 | } |
| 552 | |
| 553 | fn extract_query_param(url: &str, key: &str) -> Option<String> { |
| 554 | let query = url.split_once('?')?.1; |
| 555 | for part in query.split('&') { |
| 556 | let mut iter = part.splitn(2, '='); |
| 557 | let name = iter.next().unwrap_or(""); |
| 558 | if name == key { |
| 559 | return iter.next().map(str::to_string); |
| 560 | } |
| 561 | } |
| 562 | None |
| 563 | } |
| 564 | |
| 565 | #[cfg(test)] |
| 566 | mod tests { |
| 567 | use super::{decode_html_entities, extract_search_query, optional_search_max_results}; |
| 568 | use serde_json::json; |
| 569 | |
| 570 | #[test] |
| 571 | fn decode_html_entities_handles_named_entities() { |
| 572 | assert_eq!(decode_html_entities("&"), "&"); |
| 573 | assert_eq!(decode_html_entities("<"), "<"); |
| 574 | assert_eq!(decode_html_entities(">"), ">"); |
| 575 | assert_eq!(decode_html_entities("""), "\""); |
| 576 | assert_eq!(decode_html_entities("'"), "'"); |
| 577 | assert_eq!(decode_html_entities(" "), " "); |
| 578 | assert_eq!(decode_html_entities("©"), "\u{00A9}"); |
| 579 | assert_eq!(decode_html_entities("—"), "\u{2014}"); |
| 580 | } |
| 581 | |
| 582 | #[test] |
| 583 | fn decode_html_entities_handles_decimal_numeric_references() { |
| 584 | assert_eq!(decode_html_entities("A"), "A"); |
| 585 | assert_eq!(decode_html_entities("<"), "<"); |
| 586 | assert_eq!(decode_html_entities("–"), "\u{2013}"); |
| 587 | } |
| 588 | |
| 589 | #[test] |
| 590 | fn decode_html_entities_handles_hex_numeric_references() { |
| 591 | assert_eq!(decode_html_entities("A"), "A"); |
| 592 | assert_eq!(decode_html_entities("<"), "<"); |
| 593 | assert_eq!(decode_html_entities("—"), "\u{2014}"); |
| 594 | } |
| 595 | |
| 596 | #[test] |
| 597 | fn decode_html_entities_passthrough_unknown() { |
| 598 | assert_eq!(decode_html_entities("&unknown;"), "&unknown;"); |
| 599 | } |
| 600 | |
| 601 | #[test] |
| 602 | fn decode_html_entities_mixed_content() { |
| 603 | let input = "Hello & welcome to "Rust's world" — enjoy!"; |
| 604 | let expected = "Hello & welcome to \"Rust's world\" \u{2014} enjoy!"; |
| 605 | assert_eq!(decode_html_entities(input), expected); |
| 606 | } |
| 607 | |
| 608 | #[test] |
| 609 | fn extract_search_query_accepts_legacy_query() { |
| 610 | let query = |
| 611 | extract_search_query(&json!({"query": " deepseek v4 "})).expect("query should parse"); |
| 612 | assert_eq!(query, "deepseek v4"); |
| 613 | } |
| 614 | |
| 615 | #[test] |
| 616 | fn extract_search_query_accepts_q_alias() { |
| 617 | let query = |
| 618 | extract_search_query(&json!({"q": "deepseek v4 pro"})).expect("q alias should parse"); |
| 619 | assert_eq!(query, "deepseek v4 pro"); |
| 620 | } |
| 621 | |
| 622 | #[test] |
| 623 | fn extract_search_query_accepts_array_form() { |
| 624 | let input = json!({"search_query": [{"q": "deepseek api", "max_results": 3}]}); |
| 625 | let query = extract_search_query(&input).expect("array form should parse"); |
| 626 | assert_eq!(query, "deepseek api"); |
| 627 | assert_eq!(optional_search_max_results(&input), 3); |
| 628 | } |
| 629 | |
| 630 | #[test] |
| 631 | fn extract_search_query_rejects_missing_query() { |
| 632 | let err = extract_search_query(&json!({"max_results": 2})) |
| 633 | .expect_err("missing query should fail"); |
| 634 | assert!(format!("{err}").contains("missing required field 'query'")); |
| 635 | } |
| 636 | } |
| 637 |