web_run.rs - CodeWhale

web_run.rs

1	//! Web browsing tool with multi-command support (search/open/click/find/screenshot).
2	//!
3	//! This mirrors the Codex harness `web.run` interface so models can use a single
4	//! tool call to perform multiple web actions and cite sources with ref_ids.
5
6	use super::spec::{
7	ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
8	optional_u64, required_str,
9	};
10	use super::web::extract::{DocumentKind, ExtractedDocument, extract_document};
11	#[cfg(test)]
12	use super::web::fetch::fetch_readable_with_initial_pin;
13	use super::web::fetch::{FetchOptions, HARD_MAX_BYTES, fetch_readable};
14	#[cfg(test)]
15	use super::web::guard::DnsPin;
16	use super::web::overflow::bound_text as bound_web_text;
17	#[cfg(test)]
18	use super::web::overflow::inline_char_budget;
19	use async_trait::async_trait;
20	use regex::Regex;
21	use serde::{Deserialize, Serialize};
22	use serde_json::{Value, json};
23	use std::collections::{HashMap, HashSet, VecDeque};
24	use std::hash::{Hash, Hasher};
25	use std::sync::{Arc, OnceLock};
26	use std::time::{Duration, Instant};
27	use unicode_width::UnicodeWidthStr;
28
29	use parking_lot::{RwLock, RwLockWriteGuard};
30
31	use super::web::contract::{
32	DEFAULT_SEARCH_RESULTS, DEFAULT_SEARCH_TIMEOUT_MS, MAX_SEARCH_RESULTS, MAX_SEARCH_TIMEOUT_MS,
33	Recency, SearchQuery, SearchReceipt, SearchResult as NormalizedSearchResult,
34	};
35	use super::web::scrape::BROWSER_USER_AGENT as USER_AGENT;
36	use super::web_search::{domain_matches, execute_search};
37
38	// Search and open share the retrieval-path defaults from `web::contract` and
39	// `web::fetch` so `web.run` cannot drift from `web_search` / `fetch_url`.
40	const DEFAULT_OPEN_TIMEOUT_MS: u64 = super::web::fetch::DEFAULT_TIMEOUT.as_millis() as u64;
41	const MAX_WEB_RUN_SESSIONS: usize = 64;
42	const MAX_PAGES_PER_SESSION: usize = 256;
43	const WEB_RUN_SESSION_TTL: Duration = Duration::from_secs(30 * 60);
44
45	static WEB_RUN_STATE: OnceLock<WebRunCache> = OnceLock::new();
46
47	#[derive(Default)]
48	struct WebRunCache {
49	sessions: RwLock<HashMap<String, WebRunSessionState>>,
50	pages: RwLock<HashMap<String, StoredWebPage>>,
51	}
52
53	#[derive(Default)]
54	struct WebRunState {
55	sessions: HashMap<String, WebRunSessionState>,
56	pages: HashMap<String, StoredWebPage>,
57	}
58
59	struct WebRunSessionState {
60	next_turn: u64,
61	refs: VecDeque<String>,
62	last_access: Instant,
63	/// Hosts that refused to serve a page this session (HTTP 401/403 even
64	/// after the browser-agent fallback). Later search results from them are
65	/// ranked after sources that are likely to load.
66	refusing_hosts: HashSet<String>,
67	}
68
69	impl Default for WebRunSessionState {
70	fn default() -> Self {
71	Self {
72	next_turn: 0,
73	refs: VecDeque::new(),
74	last_access: Instant::now(),
75	refusing_hosts: HashSet::new(),
76	}
77	}
78	}
79
80	#[derive(Debug, Clone)]
81	struct StoredWebPage {
82	namespace: String,
83	page: Arc<WebPage>,
84	}
85
86	impl WebRunState {
87	fn cleanup(&mut self) {
88	let now = Instant::now();
89	let expired = self
90	.sessions
91	.iter()
92	.filter_map(\|(namespace, session)\| {
93	if now.duration_since(session.last_access) > WEB_RUN_SESSION_TTL {
94	Some(namespace.clone())
95	} else {
96	None
97	}
98	})
99	.collect::<Vec<_>>();
100	for namespace in expired {
101	self.remove_session(&namespace);
102	}
103
104	while self.sessions.len() > MAX_WEB_RUN_SESSIONS {
105	let Some(oldest_namespace) = self
106	.sessions
107	.iter()
108	.min_by_key(\|(_, session)\| session.last_access)
109	.map(\|(namespace, _)\| namespace.clone())
110	else {
111	break;
112	};
113	self.remove_session(&oldest_namespace);
114	}
115	}
116
117	fn remove_session(&mut self, namespace: &str) {
118	if let Some(session) = self.sessions.remove(namespace) {
119	for ref_id in session.refs {
120	self.pages.remove(&ref_id);
121	}
122	}
123	}
124
125	fn touch_session(&mut self, namespace: &str) {
126	self.cleanup();
127	if !self.sessions.contains_key(namespace)
128	&& self.sessions.len() >= MAX_WEB_RUN_SESSIONS
129	&& let Some(oldest_namespace) = self
130	.sessions
131	.iter()
132	.min_by_key(\|(_, session)\| session.last_access)
133	.map(\|(existing_namespace, _)\| existing_namespace.clone())
134	{
135	self.remove_session(&oldest_namespace);
136	}
137
138	let session = self.sessions.entry(namespace.to_string()).or_default();
139	session.last_access = Instant::now();
140	}
141
142	fn next_turn(&mut self, namespace: &str) -> u64 {
143	self.touch_session(namespace);
144	let session = self
145	.sessions
146	.get_mut(namespace)
147	.expect("session should exist after touch");
148	let current = session.next_turn;
149	session.next_turn = session.next_turn.saturating_add(1);
150	current
151	}
152
153	fn note_refusing_host(&mut self, namespace: &str, host: &str) {
154	self.touch_session(namespace);
155	if let Some(session) = self.sessions.get_mut(namespace) {
156	session.refusing_hosts.insert(host.to_string());
157	}
158	}
159
160	fn refusing_hosts(&self, namespace: &str) -> HashSet<String> {
161	self.sessions
162	.get(namespace)
163	.map(\|session\| session.refusing_hosts.clone())
164	.unwrap_or_default()
165	}
166
167	fn store_page(&mut self, namespace: &str, ref_id: &str, page: WebPage) {
168	self.touch_session(namespace);
169	let mut evicted_refs = Vec::new();
170	{
171	let session = self
172	.sessions
173	.get_mut(namespace)
174	.expect("session should exist after touch");
175	if let Some(existing_idx) = session.refs.iter().position(\|existing\| existing == ref_id)
176	{
177	session.refs.remove(existing_idx);
178	}
179	session.refs.push_back(ref_id.to_string());
180
181	while session.refs.len() > MAX_PAGES_PER_SESSION {
182	if let Some(evicted_ref) = session.refs.pop_front() {
183	evicted_refs.push(evicted_ref);
184	}
185	}
186	}
187
188	self.pages.insert(
189	ref_id.to_string(),
190	StoredWebPage {
191	namespace: namespace.to_string(),
192	page: Arc::new(page),
193	},
194	);
195	for evicted_ref in evicted_refs {
196	self.pages.remove(&evicted_ref);
197	}
198	}
199	}
200
201	#[derive(Debug, Clone, Serialize)]
202	struct WebLink {
203	id: usize,
204	url: String,
205	text: String,
206	}
207
208	#[derive(Debug, Clone)]
209	struct WebPage {
210	url: String,
211	title: Option<String>,
212	content_type: Option<String>,
213	lines: Vec<String>,
214	links: Vec<WebLink>,
215	pdf_pages: Option<Vec<Vec<String>>>,
216	truncated: bool,
217	}
218
219	#[derive(Debug, Clone, Copy)]
220	enum ResponseLength {
221	Short,
222	Medium,
223	Long,
224	}
225
226	impl ResponseLength {
227	fn from_input(input: Option<&Value>) -> Self {
228	let raw = input.and_then(\|v\| v.as_str()).unwrap_or("medium");
229	match raw.to_lowercase().as_str() {
230	"short" => Self::Short,
231	"long" => Self::Long,
232	_ => Self::Medium,
233	}
234	}
235
236	fn view_lines(self) -> usize {
237	match self {
238	Self::Short => 40,
239	Self::Medium => 80,
240	Self::Long => 160,
241	}
242	}
243
244	fn wrap_width(self) -> usize {
245	match self {
246	Self::Short => 88,
247	Self::Medium => 110,
248	Self::Long => 140,
249	}
250	}
251
252	fn max_results(self) -> usize {
253	match self {
254	Self::Short => DEFAULT_SEARCH_RESULTS,
255	Self::Medium => 8,
256	Self::Long => usize::from(MAX_SEARCH_RESULTS),
257	}
258	}
259
260	fn max_find_matches(self) -> usize {
261	match self {
262	Self::Short => 8,
263	Self::Medium => 15,
264	Self::Long => 30,
265	}
266	}
267	}
268
269	#[derive(Debug, Clone, Serialize)]
270	struct WebRunSearchResult {
271	ref_id: String,
272	query: String,
273	source: String,
274	count: usize,
275	results: Vec<NormalizedSearchResult>,
276	#[serde(skip_serializing_if = "Option::is_none")]
277	warning: Option<String>,
278	receipt: SearchReceipt,
279	}
280
281	#[derive(Debug, Clone, Serialize)]
282	struct PageViewResult {
283	ref_id: String,
284	url: String,
285	#[serde(skip_serializing_if = "Option::is_none")]
286	title: Option<String>,
287	#[serde(skip_serializing_if = "Option::is_none")]
288	content_type: Option<String>,
289	line_start: usize,
290	line_end: usize,
291	total_lines: usize,
292	#[serde(default, skip_serializing_if = "is_false")]
293	truncated: bool,
294	content: String,
295	links: Vec<WebLink>,
296	}
297
298	#[derive(Debug, Clone, Serialize)]
299	struct FindMatch {
300	line: usize,
301	text: String,
302	}
303
304	fn is_false(value: &bool) -> bool {
305	!*value
306	}
307
308	#[derive(Debug, Clone, Serialize)]
309	struct FindResult {
310	ref_id: String,
311	pattern: String,
312	count: usize,
313	matches: Vec<FindMatch>,
314	}
315
316	#[derive(Debug, Clone, Serialize)]
317	struct ScreenshotResult {
318	ref_id: String,
319	pageno: usize,
320	total_pages: usize,
321	content: String,
322	}
323
324	#[derive(Debug, Clone, Serialize)]
325	struct ImageResultEntry {
326	ref_id: String,
327	image: String,
328	#[serde(skip_serializing_if = "Option::is_none")]
329	thumbnail: Option<String>,
330	#[serde(skip_serializing_if = "Option::is_none")]
331	title: Option<String>,
332	#[serde(skip_serializing_if = "Option::is_none")]
333	url: Option<String>,
334	#[serde(skip_serializing_if = "Option::is_none")]
335	source: Option<String>,
336	#[serde(skip_serializing_if = "Option::is_none")]
337	width: Option<u32>,
338	#[serde(skip_serializing_if = "Option::is_none")]
339	height: Option<u32>,
340	}
341
342	#[derive(Debug, Clone, Serialize)]
343	struct ImageQueryResult {
344	query: String,
345	source: String,
346	count: usize,
347	results: Vec<ImageResultEntry>,
348	#[serde(skip_serializing_if = "Option::is_none")]
349	warning: Option<String>,
350	}
351
352	#[derive(Debug, Clone, Serialize, Default)]
353	struct WebRunOutput {
354	#[serde(skip_serializing_if = "Option::is_none")]
355	search_query: Option<Vec<WebRunSearchResult>>,
356	#[serde(skip_serializing_if = "Option::is_none")]
357	image_query: Option<Vec<ImageQueryResult>>,
358	#[serde(skip_serializing_if = "Option::is_none")]
359	open: Option<Vec<PageViewResult>>,
360	#[serde(skip_serializing_if = "Option::is_none")]
361	click: Option<Vec<PageViewResult>>,
362	#[serde(skip_serializing_if = "Option::is_none")]
363	find: Option<Vec<FindResult>>,
364	#[serde(skip_serializing_if = "Option::is_none")]
365	screenshot: Option<Vec<ScreenshotResult>>,
366	#[serde(skip_serializing_if = "Vec::is_empty", default)]
367	warnings: Vec<String>,
368	/// Every page this call tried to open or click, loaded ones first, so the
369	/// model cites what actually loaded and moves past what did not.
370	#[serde(skip_serializing_if = "Vec::is_empty", default)]
371	sources: Vec<SourceEntry>,
372	}
373
374	#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
375	#[serde(rename_all = "snake_case")]
376	enum SourceStatus {
377	Loaded,
378	/// The site answered but would not serve a readable page (403, 404,
379	/// script-only body). Not retryable as-is; use another source.
380	Unavailable,
381	/// Network or server trouble (timeout, 5xx, 429). May load on a later try.
382	Transient,
383	}
384
385	#[derive(Debug, Clone, Serialize)]
386	struct SourceEntry {
387	#[serde(skip_serializing_if = "Option::is_none")]
388	ref_id: Option<String>,
389	url: String,
390	#[serde(skip_serializing_if = "Option::is_none")]
391	title: Option<String>,
392	status: SourceStatus,
393	#[serde(skip_serializing_if = "Option::is_none")]
394	reason: Option<String>,
395	}
396
397	impl SourceEntry {
398	fn loaded(ref_id: &str, page: &WebPage) -> Self {
399	Self {
400	ref_id: Some(ref_id.to_string()),
401	url: page.url.clone(),
402	title: page.title.clone(),
403	status: SourceStatus::Loaded,
404	reason: None,
405	}
406	}
407	}
408
409	pub struct WebRunTool;
410
411	#[async_trait]
412	impl ToolSpec for WebRunTool {
413	fn name(&self) -> &'static str {
414	"web.run"
415	}
416
417	fn description(&self) -> &'static str {
418	"Browse the web (search/open/click/find/screenshot/image_query) and return structured results with ref_ids for citations."
419	}
420
421	fn input_schema(&self) -> Value {
422	json!({
423	"type": "object",
424	"properties": {
425	"search_query": {
426	"type": "array",
427	"items": {
428	"type": "object",
429	"properties": {
430	"q": { "type": "string" },
431	"recency": { "type": "integer", "minimum": 1, "maximum": 3650 },
432	"max_results": { "type": "integer" },
433	"timeout_ms": { "type": "integer" },
434	"domains": { "type": "array", "items": { "type": "string" } }
435	},
436	"required": ["q"]
437	}
438	},
439	"image_query": {
440	"type": "array",
441	"items": {
442	"type": "object",
443	"properties": {
444	"q": { "type": "string" },
445	"recency": { "type": "integer" },
446	"max_results": { "type": "integer" },
447	"timeout_ms": { "type": "integer" },
448	"domains": { "type": "array", "items": { "type": "string" } }
449	},
450	"required": ["q"]
451	}
452	},
453	"open": {
454	"type": "array",
455	"items": {
456	"type": "object",
457	"properties": {
458	"ref_id": { "type": "string" },
459	"lineno": { "type": "integer" }
460	},
461	"required": ["ref_id"]
462	}
463	},
464	"click": {
465	"type": "array",
466	"items": {
467	"type": "object",
468	"properties": {
469	"ref_id": { "type": "string" },
470	"id": { "type": "integer" }
471	},
472	"required": ["ref_id", "id"]
473	}
474	},
475	"find": {
476	"type": "array",
477	"items": {
478	"type": "object",
479	"properties": {
480	"ref_id": { "type": "string" },
481	"pattern": { "type": "string" }
482	},
483	"required": ["ref_id", "pattern"]
484	}
485	},
486	"screenshot": {
487	"type": "array",
488	"items": {
489	"type": "object",
490	"properties": {
491	"ref_id": { "type": "string" },
492	"pageno": { "type": "integer" }
493	},
494	"required": ["ref_id", "pageno"]
495	}
496	},
497	"response_length": {
498	"type": "string",
499	"enum": ["short", "medium", "long"],
500	"description": "Controls result verbosity"
501	}
502	}
503	})
504	}
505
506	fn capabilities(&self) -> Vec<ToolCapability> {
507	vec![ToolCapability::ReadOnly, ToolCapability::Network]
508	}
509
510	fn approval_requirement(&self) -> ApprovalRequirement {
511	// Read-only HTTP can still disclose local data through a URL or query.
512	// Host allowlisting controls reachability, not approval of this payload.
513	ApprovalRequirement::Required
514	}
515
516	async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
517	let response_length = ResponseLength::from_input(input.get("response_length"));
518	let mut output = WebRunOutput::default();
519	let scope = scoped_ref_prefix(&context.state_namespace);
520	let turn = with_state(\|state\| state.next_turn(&context.state_namespace));
521
522	let mut search_counter = 0usize;
523	let mut view_counter = 0usize;
524	let mut click_counter = 0usize;
525
526	if let Some(searches) = input.get("search_query").and_then(\|v\| v.as_array()) {
527	let mut results = Vec::new();
528	for search in searches {
529	let query = required_str(search, "q")?.trim().to_string();
530	if query.is_empty() {
531	continue;
532	}
533	let recency = optional_u64(search, "recency", 0)?;
534	let max_results = usize::try_from(optional_u64(
535	search,
536	"max_results",
537	response_length.max_results() as u64,
538	)?)
539	.unwrap_or(response_length.max_results())
540	.clamp(1, usize::from(MAX_SEARCH_RESULTS));
541	let timeout_ms = optional_u64(search, "timeout_ms", DEFAULT_SEARCH_TIMEOUT_MS)?
542	.min(MAX_SEARCH_TIMEOUT_MS);
543
544	let domains = search
545	.get("domains")
546	.and_then(\|v\| v.as_array())
547	.map(\|arr\| {
548	arr.iter()
549	.filter_map(\|v\| v.as_str().map(\|s\| s.to_string()))
550	.collect::<Vec<_>>()
551	})
552	.unwrap_or_default();
553
554	let requested_recency = if recency == 0 {
555	None
556	} else {
557	let days = u16::try_from(recency)
558	.ok()
559	.filter(\|days\| *days <= 3650)
560	.ok_or_else(\|\| {
561	ToolError::invalid_input(
562	"Field 'search_query[].recency' must be between 1 and 3650 days",
563	)
564	})?;
565	Some(Recency::Days(days))
566	};
567	let mut response = execute_search(
568	SearchQuery::new(query, max_results, requested_recency, domains, None),
569	timeout_ms,
570	context,
571	)
572	.await?;
573	let refusing_hosts =
574	with_state(\|state\| state.refusing_hosts(&context.state_namespace));
575	prefer_loading_sources(&mut response.results, &refusing_hosts);
576	let warning = response.receipt.warning();
577	search_counter += 1;
578	let ref_id = format!("{scope}turn{turn}search{search_counter}");
579
580	let page = page_from_search(&response.query, &response.results);
581	store_page(&context.state_namespace, &ref_id, page);
582
583	results.push(WebRunSearchResult {
584	ref_id,
585	query: response.query,
586	source: response.source,
587	count: response.count,
588	results: response.results,
589	warning,
590	receipt: response.receipt,
591	});
592	}
593	if !results.is_empty() {
594	output.search_query = Some(results);
595	}
596	}
597
598	if let Some(images) = input.get("image_query").and_then(\|v\| v.as_array()) {
599	let mut results = Vec::new();
600	for image in images {
601	let query = required_str(image, "q")?.trim().to_string();
602	if query.is_empty() {
603	continue;
604	}
605	let recency = optional_u64(image, "recency", 0)?;
606	let max_results = usize::try_from(optional_u64(
607	image,
608	"max_results",
609	response_length.max_results() as u64,
610	)?)
611	.unwrap_or(response_length.max_results())
612	.clamp(1, usize::from(MAX_SEARCH_RESULTS));
613	let timeout_ms = optional_u64(image, "timeout_ms", DEFAULT_SEARCH_TIMEOUT_MS)?
614	.min(MAX_SEARCH_TIMEOUT_MS);
615
616	let domains = image
617	.get("domains")
618	.and_then(\|v\| v.as_array())
619	.map(\|arr\| {
620	arr.iter()
621	.filter_map(\|v\| v.as_str().map(\|s\| s.to_string()))
622	.collect::<Vec<_>>()
623	})
624	.unwrap_or_default();
625
626	let (mut entries, warning) =
627	run_image_search(&query, max_results, timeout_ms, &domains, context).await?;
628	entries.retain_mut(\|entry\| {
629	let canonical_url = entry.url.as_deref().unwrap_or(&entry.image);
630	let Some(citation) = super::web::citations::register(
631	&context.state_namespace,
632	canonical_url,
633	entry.title.as_deref(),
634	) else {
635	return false;
636	};
637	entry.ref_id = citation.ref_id;
638	true
639	});
640
641	let mut warnings = Vec::new();
642	if recency > 0 {
643	warnings.push(format!(
644	"Recency filter not enforced (requested last {recency} days)"
645	));
646	}
647	if let Some(w) = warning {
648	warnings.push(w);
649	}
650
651	results.push(ImageQueryResult {
652	query,
653	source: "duckduckgo_images".to_string(),
654	count: entries.len(),
655	results: entries,
656	warning: if warnings.is_empty() {
657	None
658	} else {
659	Some(warnings.join("; "))
660	},
661	});
662	}
663	if !results.is_empty() {
664	output.image_query = Some(results);
665	}
666	}
667
668	if let Some(opens) = input.get("open").and_then(\|v\| v.as_array()) {
669	let mut views = Vec::new();
670	for open in opens {
671	let ref_id = required_str(open, "ref_id")?.to_string();
672	let lineno = optional_u64(open, "lineno", 1)?.max(1) as usize;
673
674	let page =
675	match resolve_or_fetch_page(&ref_id, DEFAULT_OPEN_TIMEOUT_MS, context).await {
676	Ok(page) => page,
677	Err(error) => {
678	let target = open_target_url(&context.state_namespace, &ref_id);
679	output.sources.push(source_failure(
680	&context.state_namespace,
681	target.as_deref().unwrap_or(&ref_id),
682	error,
683	)?);
684	continue;
685	}
686	};
687	view_counter += 1;
688	let view_ref = format!("{scope}turn{turn}view{view_counter}");
689	store_page(&context.state_namespace, &view_ref, (*page).clone());
690	output.sources.push(SourceEntry::loaded(&view_ref, &page));
691
692	let view = render_view(&view_ref, &page, lineno, response_length);
693	views.push(view);
694	}
695	if !views.is_empty() {
696	output.open = Some(views);
697	}
698	}
699
700	if let Some(clicks) = input.get("click").and_then(\|v\| v.as_array()) {
701	let mut views = Vec::new();
702	for click in clicks {
703	let ref_id = required_str(click, "ref_id")?.to_string();
704	let link_id = optional_u64(click, "id", 0)? as usize;
705	if link_id == 0 {
706	return Err(ToolError::invalid_input("click.id must be >= 1"));
707	}
708	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
709	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
710	})?;
711	let link = page.links.iter().find(\|l\| l.id == link_id).ok_or_else(\|\| {
712	ToolError::invalid_input(format!(
713	"Link id {link_id} not found for ref_id '{ref_id}'"
714	))
715	})?;
716	let target = link.url.clone();
717	let fetched =
718	match resolve_or_fetch_page(&target, DEFAULT_OPEN_TIMEOUT_MS, context).await {
719	Ok(page) => page,
720	Err(error) => {
721	output.sources.push(source_failure(
722	&context.state_namespace,
723	&target,
724	error,
725	)?);
726	continue;
727	}
728	};
729	click_counter += 1;
730	let click_ref = format!("{scope}turn{turn}click{click_counter}");
731	store_page(&context.state_namespace, &click_ref, (*fetched).clone());
732	output
733	.sources
734	.push(SourceEntry::loaded(&click_ref, &fetched));
735	let view = render_view(&click_ref, &fetched, 1, response_length);
736	views.push(view);
737	}
738	if !views.is_empty() {
739	output.click = Some(views);
740	}
741	}
742
743	if let Some(find_requests) = input.get("find").and_then(\|v\| v.as_array()) {
744	let mut finds = Vec::new();
745	for find_req in find_requests {
746	let ref_id = required_str(find_req, "ref_id")?.to_string();
747	let pattern = required_str(find_req, "pattern")?.to_string();
748	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
749	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
750	})?;
751	let find_result = find_in_page(&ref_id, &pattern, &page, response_length);
752	finds.push(find_result);
753	}
754	if !finds.is_empty() {
755	output.find = Some(finds);
756	}
757	}
758
759	if let Some(shots) = input.get("screenshot").and_then(\|v\| v.as_array()) {
760	let mut screenshots = Vec::new();
761	for shot in shots {
762	let ref_id = required_str(shot, "ref_id")?.to_string();
763	let pageno = optional_u64(shot, "pageno", 0)? as usize;
764	let page = get_page(&context.state_namespace, &ref_id).ok_or_else(\|\| {
765	ToolError::invalid_input(format!("Unknown ref_id '{ref_id}'"))
766	})?;
767	let screenshot = screenshot_page(&ref_id, pageno, &page)?;
768	screenshots.push(screenshot);
769	}
770	if !screenshots.is_empty() {
771	output.screenshot = Some(screenshots);
772	}
773	}
774
775	let failed = output
776	.sources
777	.iter()
778	.filter(\|source\| source.status != SourceStatus::Loaded)
779	.count();
780	if failed > 0 {
781	output
782	.sources
783	.sort_by_key(\|source\| source.status != SourceStatus::Loaded);
784	output.warnings.push(format!(
785	"{failed} source(s) did not load. Cite only sources marked loaded; open another \
786	search result instead of retrying an unavailable URL."
787	));
788	}
789
790	if output.performed_no_op() && output.sources.is_empty() {
791	// #5123-class: an empty success here reads as "nothing found"
792	// rather than "you called the tool wrong" (e.g. the natural
793	// {"query": …} shape, which matches no op key).
794	let received: Vec<&str> = input
795	.as_object()
796	.map(\|object\| object.keys().map(String::as_str).collect())
797	.unwrap_or_default();
798	return Err(ToolError::invalid_input(format!(
799	"web.run performed no operation. Pass at least one non-empty op array \
800	({}). Received keys: [{}].",
801	WEB_RUN_OP_KEYS.join(", "),
802	received.join(", ")
803	)));
804	}
805
806	let mut result = bounded_web_run_result(&output, context)?;
807	if failed > 0 {
808	// A site refusing a page is an outcome of the browse, not a tool
809	// failure: the call still succeeds, and clients render these
810	// neutrally instead of as errors.
811	let metadata = result.metadata.get_or_insert_with(\|\| json!({}));
812	metadata["source_failures"] = json!(failed);
813	metadata["source_failure_severity"] = json!("neutral");
814	}
815	Ok(result)
816	}
817	}
818
819	/// Where an `open` ref would be fetched from, for the failure receipt.
820	fn open_target_url(namespace: &str, ref_id: &str) -> Option<String> {
821	if let Some(citation) = super::web::citations::resolve(namespace, ref_id) {
822	return Some(citation.url);
823	}
824	looks_like_url(ref_id).then(\|\| ref_id.to_string())
825	}
826
827	/// Turn a failed page fetch into a source receipt, or pass the error through
828	/// when it is the caller's mistake or a gate (bad ref, denied host, cancel).
829	fn source_failure(namespace: &str, url: &str, error: ToolError) -> Result<SourceEntry, ToolError> {
830	let (status, reason) = match &error {
831	ToolError::Timeout { .. } => (SourceStatus::Transient, error.to_string()),
832	ToolError::ExecutionFailed { message, .. } => {
833	let status = match http_status_of(message) {
834	Some(code) if code == 429 \|\| (500..600).contains(&code) => SourceStatus::Transient,
835	Some(_) => SourceStatus::Unavailable,
836	None if message.contains("timed out") \|\| message.contains("after one retry") => {
837	SourceStatus::Transient
838	}
839	None => SourceStatus::Unavailable,
840	};
841	(status, message.clone())
842	}
843	_ => return Err(error),
844	};
845	if let Some(code) = http_status_of(&reason)
846	&& matches!(code, 401 \| 403)
847	&& let Some(host) = reqwest::Url::parse(url)
848	.ok()
849	.and_then(\|parsed\| parsed.host_str().map(str::to_ascii_lowercase))
850	{
851	with_state(\|state\| state.note_refusing_host(namespace, &host));
852	}
853	Ok(SourceEntry {
854	ref_id: None,
855	url: url.to_string(),
856	title: None,
857	status,
858	reason: Some(reason),
859	})
860	}
861
862	/// The HTTP status carried by a `document_from_fetched` rejection.
863	fn http_status_of(message: &str) -> Option<u16> {
864	let (_, tail) = message.rsplit_once(" failed: HTTP ")?;
865	tail.get(..3)?.parse().ok()
866	}
867
868	/// Rank results from hosts that already refused this session after the ones
869	/// likely to load, keeping each group's order and renumbering `rank`.
870	fn prefer_loading_sources(
871	results: &mut [NormalizedSearchResult],
872	refusing_hosts: &HashSet<String>,
873	) {
874	if refusing_hosts.is_empty() {
875	return;
876	}
877	results.sort_by_key(\|result\| refusing_hosts.contains(&result.domain));
878	for (index, result) in results.iter_mut().enumerate() {
879	result.rank = u8::try_from(index + 1).unwrap_or(u8::MAX);
880	}
881	}
882
883	const WEB_RUN_OP_KEYS: [&str; 6] = [
884	"search_query",
885	"image_query",
886	"open",
887	"click",
888	"find",
889	"screenshot",
890	];
891
892	impl WebRunOutput {
893	fn performed_no_op(&self) -> bool {
894	self.search_query.is_none()
895	&& self.image_query.is_none()
896	&& self.open.is_none()
897	&& self.click.is_none()
898	&& self.find.is_none()
899	&& self.screenshot.is_none()
900	}
901	}
902
903	fn with_state<T>(f: impl FnOnce(&mut WebRunState) -> T) -> T {
904	let cache = WEB_RUN_STATE.get_or_init(WebRunCache::default);
905	let sessions = cache.sessions.write();
906	let pages = cache.pages.write();
907	let mut guard = WebRunStateWriteBack::new(sessions, pages);
908	guard.state_mut().cleanup();
909	let result = f(guard.state_mut());
910	guard.write_back();
911	result
912	}
913
914	struct WebRunStateWriteBack<'a> {
915	sessions: RwLockWriteGuard<'a, HashMap<String, WebRunSessionState>>,
916	pages: RwLockWriteGuard<'a, HashMap<String, StoredWebPage>>,
917	state: Option<WebRunState>,
918	}
919
920	impl<'a> WebRunStateWriteBack<'a> {
921	fn new(
922	mut sessions: RwLockWriteGuard<'a, HashMap<String, WebRunSessionState>>,
923	mut pages: RwLockWriteGuard<'a, HashMap<String, StoredWebPage>>,
924	) -> Self {
925	let state = WebRunState {
926	sessions: std::mem::take(&mut *sessions),
927	pages: std::mem::take(&mut *pages),
928	};
929	Self {
930	sessions,
931	pages,
932	state: Some(state),
933	}
934	}
935
936	fn state_mut(&mut self) -> &mut WebRunState {
937	self.state
938	.as_mut()
939	.expect("web run state should be present until write-back")
940	}
941
942	fn write_back(mut self) {
943	self.restore();
944	}
945
946	fn restore(&mut self) {
947	if let Some(state) = self.state.take() {
948	*self.sessions = state.sessions;
949	*self.pages = state.pages;
950	}
951	}
952	}
953
954	impl Drop for WebRunStateWriteBack<'_> {
955	fn drop(&mut self) {
956	self.restore();
957	}
958	}
959
960	fn scoped_ref_prefix(namespace: &str) -> String {
961	let mut hasher = std::collections::hash_map::DefaultHasher::new();
962	namespace.hash(&mut hasher);
963	format!("s{:016x}_", hasher.finish())
964	}
965
966	fn store_page(namespace: &str, ref_id: &str, page: WebPage) {
967	let _ = super::web::citations::register_with_ref(
968	namespace,
969	ref_id,
970	&page.url,
971	page.title.as_deref(),
972	);
973	with_state(\|state\| {
974	state.store_page(namespace, ref_id, page);
975	});
976	}
977
978	fn get_page(namespace: &str, ref_id: &str) -> Option<Arc<WebPage>> {
979	let cache = WEB_RUN_STATE.get_or_init(WebRunCache::default);
980	let stored = {
981	let pages = cache.pages.read();
982	pages.get(ref_id).cloned()
983	}?;
984	if stored.namespace != namespace {
985	return None;
986	}
987	{
988	let mut sessions = cache.sessions.write();
989	if let Some(session) = sessions.get_mut(namespace) {
990	session.last_access = Instant::now();
991	}
992	}
993	Some(stored.page)
994	}
995
996	#[cfg(test)]
997	fn reset_web_run_state() {
998	with_state(\|state\| {
999	*state = WebRunState::default();
1000	});
1001	}
1002
1003	#[cfg(test)]
1004	fn next_turn_for_namespace(namespace: &str) -> u64 {
1005	with_state(\|state\| state.next_turn(namespace))
1006	}
1007
1008	async fn resolve_or_fetch_page(
1009	ref_id: &str,
1010	timeout_ms: u64,
1011	context: &ToolContext,
1012	) -> Result<Arc<WebPage>, ToolError> {
1013	if let Some(page) = get_page(&context.state_namespace, ref_id) {
1014	return Ok(page);
1015	}
1016	if let Some(citation) = super::web::citations::resolve(&context.state_namespace, ref_id) {
1017	return fetch_page(&citation.url, timeout_ms, context)
1018	.await
1019	.map(Arc::new);
1020	}
1021	if looks_like_url(ref_id) {
1022	return fetch_page(ref_id, timeout_ms, context).await.map(Arc::new);
1023	}
1024	Err(ToolError::invalid_input(format!(
1025	"Unknown ref_id '{ref_id}'"
1026	)))
1027	}
1028
1029	fn looks_like_url(value: &str) -> bool {
1030	value.starts_with("http://") \|\| value.starts_with("https://")
1031	}
1032
1033	#[derive(Debug, Clone, Serialize, Deserialize)]
1034	struct DuckDuckGoImageResponse {
1035	#[serde(default)]
1036	results: Vec<DuckDuckGoImageResult>,
1037	}
1038
1039	#[derive(Debug, Clone, Serialize, Deserialize)]
1040	struct DuckDuckGoImageResult {
1041	image: String,
1042	#[serde(default)]
1043	thumbnail: Option<String>,
1044	#[serde(default)]
1045	title: Option<String>,
1046	#[serde(default)]
1047	url: Option<String>,
1048	#[serde(default)]
1049	source: Option<String>,
1050	#[serde(default)]
1051	width: Option<u32>,
1052	#[serde(default)]
1053	height: Option<u32>,
1054	}
1055
1056	fn extract_duckduckgo_vqd(html: &str) -> Option<String> {
1057	let html = html.trim();
1058	if html.is_empty() {
1059	return None;
1060	}
1061
1062	for (prefix, suffix) in [("vqd='", "'"), ("vqd=\"", "\"")] {
1063	if let Some(start) = html.find(prefix) {
1064	let rest = &html[start + prefix.len()..];
1065	if let Some(end) = rest.find(suffix) {
1066	let token = rest[..end].trim();
1067	if !token.is_empty() {
1068	return Some(token.to_string());
1069	}
1070	}
1071	}
1072	}
1073
1074	// Fallback: look for `vqd=` and accept a conservative token charset.
1075	if let Some(start) = html.find("vqd=") {
1076	let rest = &html[start + 4..];
1077	let mut token = String::new();
1078	for ch in rest.chars() {
1079	if ch.is_ascii_alphanumeric() \|\| ch == '-' \|\| ch == '_' {
1080	token.push(ch);
1081	} else {
1082	break;
1083	}
1084	}
1085	if !token.is_empty() {
1086	return Some(token);
1087	}
1088	}
1089
1090	None
1091	}
1092
1093	async fn run_image_search(
1094	query: &str,
1095	max_results: usize,
1096	timeout_ms: u64,
1097	domains: &[String],
1098	context: &ToolContext,
1099	) -> Result<(Vec<ImageResultEntry>, Option<String>), ToolError> {
1100	super::web_search::check_policy(context.network_policy.as_ref(), "duckduckgo.com")?;
1101	let client = crate::tls::reqwest_client_builder()
1102	.timeout(Duration::from_millis(timeout_ms))
1103	.user_agent(USER_AGENT)
1104	.build()
1105	.map_err(\|e\| ToolError::execution_failed(format!("Failed to build HTTP client: {e}")))?;
1106
1107	// Step 1: fetch the HTML page to obtain the `vqd` token used by the images API.
1108	let encoded = url_encode(query);
1109	let seed_url = format!("https://duckduckgo.com/?q={encoded}&iax=images&ia=images");
1110	let seed_resp = client
1111	.get(&seed_url)
1112	.header(
1113	"Accept",
1114	"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8",
1115	)
1116	.header("Accept-Language", "en-US,en;q=0.5")
1117	.send()
1118	.await
1119	.map_err(\|e\| {
1120	ToolError::execution_failed(format!("Image search seed request failed: {e}"))
1121	})?;
1122
1123	let seed_status = seed_resp.status();
1124	let seed_body = if super::web::adapter::search_selected(context) {
1125	super::web::adapter::read_response_with_limit(seed_resp, context, HARD_MAX_BYTES)
1126	.await
1127	.map_err(ToolError::from)?
1128	} else {
1129	seed_resp.text().await.map_err(\|e\| {
1130	ToolError::execution_failed(format!("Failed to read image seed response: {e}"))
1131	})?
1132	};
1133
1134	if !seed_status.is_success() {
1135	return Err(ToolError::execution_failed(format!(
1136	"Image search seed request failed: HTTP {}",
1137	seed_status.as_u16()
1138	)));
1139	}
1140
1141	let vqd = extract_duckduckgo_vqd(&seed_body).ok_or_else(\|\| {
1142	ToolError::execution_failed("Failed to extract DuckDuckGo image token (vqd)")
1143	})?;
1144
1145	// Step 2: query the DuckDuckGo images JSON endpoint.
1146	let api_url = format!("https://duckduckgo.com/i.js?l=us-en&o=json&q={encoded}&vqd={vqd}&p=1");
1147	super::web_search::check_policy(context.network_policy.as_ref(), "duckduckgo.com")?;
1148	let api_resp = client
1149	.get(&api_url)
1150	.header("Accept", "application/json")
1151	.header("Referer", "https://duckduckgo.com/")
1152	.send()
1153	.await
1154	.map_err(\|e\| ToolError::execution_failed(format!("Image search request failed: {e}")))?;
1155
1156	let api_status = api_resp.status();
1157	let api_body = if super::web::adapter::search_selected(context) {
1158	super::web::adapter::read_response(api_resp, context)
1159	.await
1160	.map_err(ToolError::from)?
1161	} else {
1162	api_resp.text().await.map_err(\|e\| {
1163	ToolError::execution_failed(format!("Failed to read image response: {e}"))
1164	})?
1165	};
1166
1167	if !api_status.is_success() {
1168	return Err(ToolError::execution_failed(format!(
1169	"Image search failed: HTTP {}",
1170	api_status.as_u16()
1171	)));
1172	}
1173
1174	let mut parsed: DuckDuckGoImageResponse = serde_json::from_str(&api_body).map_err(\|e\| {
1175	ToolError::execution_failed(format!("Failed to parse image search JSON: {e}"))
1176	})?;
1177
1178	if super::web::adapter::search_selected(context) {
1179	#[derive(Deserialize)]
1180	#[serde(deny_unknown_fields)]
1181	struct Proposal {
1182	kind: String,
1183	entries: Vec<DuckDuckGoImageResult>,
1184	max_results: usize,
1185	}
1186	let mut urls = super::web_search::OpaqueUrls::default();
1187	let mut captured = parsed.clone();
1188	for entry in &mut captured.results {
1189	entry.image = urls.capture(&entry.image);
1190	if let Some(value) = entry.thumbnail.as_mut() {
1191	*value = urls.capture(value);
1192	}
1193	if let Some(value) = entry.url.as_mut() {
1194	*value = urls.capture(value);
1195	}
1196	}
1197	let proposal: Proposal = super::web::adapter::transform(
1198	crate::extension_host::StockOperation::WebImages,
1199	json!({"max_results":max_results,"parsed":captured}),
1200	context,
1201	Duration::from_millis(timeout_ms),
1202	)
1203	.await
1204	.map_err(ToolError::from)?;
1205	if proposal.kind != "web_images"
1206	\|\| proposal.max_results != max_results
1207	\|\| proposal.entries.len() > parsed.results.len()
1208	{
1209	return Err(ToolError::execution_failed(
1210	"Web Host returned inconsistent image candidates; no fallback was attempted",
1211	));
1212	}
1213	parsed.results = proposal.entries;
1214	for entry in &mut parsed.results {
1215	entry.image = urls.restore(&entry.image).map_err(ToolError::from)?;
1216	if let Some(value) = entry.thumbnail.as_mut() {
1217	*value = urls.restore(value).map_err(ToolError::from)?;
1218	}
1219	if let Some(value) = entry.url.as_mut() {
1220	*value = urls.restore(value).map_err(ToolError::from)?;
1221	}
1222	}
1223	}
1224
1225	let mut results = parsed
1226	.results
1227	.into_iter()
1228	.filter(\|item\| !item.image.trim().is_empty())
1229	.map(\|item\| ImageResultEntry {
1230	ref_id: String::new(),
1231	image: item.image,
1232	thumbnail: item.thumbnail,
1233	title: item.title,
1234	url: item.url,
1235	source: item.source,
1236	width: item.width,
1237	height: item.height,
1238	})
1239	.collect::<Vec<_>>();
1240
1241	// Domain filter is applied to the source page URL when available.
1242	let warning = if !domains.is_empty() {
1243	let before = results.len();
1244	results.retain(\|entry\| match entry.url.as_deref() {
1245	Some(url) => domain_matches(url, domains),
1246	None => true,
1247	});
1248	if before != results.len() {
1249	Some("Filtered image results by domain list".to_string())
1250	} else {
1251	None
1252	}
1253	} else {
1254	None
1255	};
1256
1257	results.truncate(max_results);
1258	Ok((results, warning))
1259	}
1260
1261	fn page_from_search(query: &str, results: &[NormalizedSearchResult]) -> WebPage {
1262	let mut lines = Vec::new();
1263	let mut links = Vec::new();
1264
1265	lines.push(format!("Search results for: {query}"));
1266	for (idx, entry) in results.iter().enumerate() {
1267	let id = idx + 1;
1268	links.push(WebLink {
1269	id,
1270	url: entry.url.clone(),
1271	text: entry.title.clone(),
1272	});
1273	lines.push(format!("{}. [{}] {}", id, id, entry.title));
1274	if let Some(snippet) = entry.snippet.as_ref()
1275	&& !snippet.trim().is_empty()
1276	{
1277	lines.push(format!(" {snippet}"));
1278	}
1279	lines.push(format!(" {url}", url = entry.url));
1280	}
1281
1282	WebPage {
1283	url: "https://html.duckduckgo.com/html/".to_string(),
1284	title: Some("Search Results".to_string()),
1285	content_type: Some("text/html".to_string()),
1286	lines,
1287	links,
1288	pdf_pages: None,
1289	truncated: false,
1290	}
1291	}
1292
1293	fn open_fetch_options(timeout_ms: u64) -> FetchOptions {
1294	FetchOptions::new(
1295	Duration::from_millis(timeout_ms),
1296	HARD_MAX_BYTES,
1297	"text/html,text/markdown,text/plain,application/xhtml+xml,application/pdf,image/,audio/,video/,/*;q=0.5",
1298	)
1299	}
1300
1301	/// A 401/403 earns one browser-agent retry inside [`fetch_readable`].
1302	async fn fetch_page(
1303	url: &str,
1304	timeout_ms: u64,
1305	context: &ToolContext,
1306	) -> Result<WebPage, ToolError> {
1307	fetch_page_with(url, &open_fetch_options(timeout_ms), context).await
1308	}
1309
1310	async fn fetch_page_with(
1311	url: &str,
1312	options: &FetchOptions,
1313	context: &ToolContext,
1314	) -> Result<WebPage, ToolError> {
1315	let readable = fetch_readable(
1316	url,
1317	options,
1318	context,
1319	"web_run",
1320	\|payload: super::web::fetch::FetchedPayload\| {
1321	Box::pin(async move { document_from_fetched(&payload, context).await })
1322	},
1323	)
1324	.await?;
1325	page_from_document(readable.payload, readable.document, context)
1326	}
1327
1328	#[cfg(test)]
1329	async fn fetch_page_with_initial_pin(
1330	url: &str,
1331	timeout_ms: u64,
1332	context: &ToolContext,
1333	initial_pin: Option<DnsPin>,
1334	) -> Result<WebPage, ToolError> {
1335	let readable = fetch_readable_with_initial_pin(
1336	url,
1337	&open_fetch_options(timeout_ms),
1338	context,
1339	"web_run",
1340	initial_pin.flatten(),
1341	\|payload: super::web::fetch::FetchedPayload\| {
1342	Box::pin(async move { document_from_fetched(&payload, context).await })
1343	},
1344	)
1345	.await?;
1346	page_from_document(readable.payload, readable.document, context)
1347	}
1348
1349	/// Reject non-2xx responses, then extract one readable document.
1350	///
1351	/// Kept separate from rendering so `fetch_readable` can re-run exactly this
1352	/// step against a cache-busted second response (#5904).
1353	async fn document_from_fetched(
1354	payload: &super::web::fetch::FetchedPayload,
1355	context: &ToolContext,
1356	) -> super::web::adapter::AdapterResult<ExtractedDocument> {
1357	if !(200..300).contains(&payload.status) {
1358	return Err((ToolError::execution_failed(format!(
1359	"Web request to {} failed: HTTP {}",
1360	payload.url, payload.status
1361	)))
1362	.into());
1363	}
1364	extract_document(
1365	&payload.url,
1366	Some(&payload.content_type),
1367	&payload.bytes,
1368	Some(context),
1369	)
1370	.await
1371	}
1372
1373	fn page_from_document(
1374	payload: super::web::fetch::FetchedPayload,
1375	document: ExtractedDocument,
1376	context: &ToolContext,
1377	) -> Result<WebPage, ToolError> {
1378	let content_type = Some(payload.content_type);
1379	match document.kind {
1380	DocumentKind::Html => {
1381	let html = document.cleaned_html.ok_or_else(\|\| {
1382	ToolError::execution_failed("Readable HTML extraction returned no document")
1383	})?;
1384	let (lines, links, parsed_title) = parse_html(&html, &payload.url);
1385	Ok(WebPage {
1386	url: payload.url,
1387	title: document.title.or(parsed_title),
1388	content_type,
1389	lines,
1390	links,
1391	pdf_pages: None,
1392	truncated: payload.truncated,
1393	})
1394	}
1395	DocumentKind::Markdown => {
1396	let (lines, links) = parse_markdown(&document.markdown, &payload.url);
1397	Ok(WebPage {
1398	url: payload.url,
1399	title: document.title,
1400	content_type,
1401	lines,
1402	links,
1403	pdf_pages: None,
1404	truncated: payload.truncated,
1405	})
1406	}
1407	DocumentKind::Text => Ok(WebPage {
1408	url: payload.url,
1409	title: document.title,
1410	content_type,
1411	lines: readable_lines(&document.text),
1412	links: Vec::new(),
1413	pdf_pages: None,
1414	truncated: payload.truncated,
1415	}),
1416	DocumentKind::Pdf => {
1417	let pages = document.pdf_pages.unwrap_or_default();
1418	Ok(WebPage {
1419	url: payload.url,
1420	title: document.title,
1421	content_type,
1422	lines: pages.first().cloned().unwrap_or_default(),
1423	links: Vec::new(),
1424	pdf_pages: Some(pages),
1425	truncated: payload.truncated,
1426	})
1427	}
1428	DocumentKind::Media => {
1429	let extension = document.media_extension.unwrap_or("bin");
1430	let digest = crate::hashing::sha256_hex(&*payload.bytes);
1431	let artifact_id = format!("web_media_{}", &digest[..16]);
1432	let (_absolute, relative) = crate::artifacts::write_session_artifact_bytes(
1433	&context.state_namespace,
1434	&artifact_id,
1435	extension,
1436	&payload.bytes,
1437	)
1438	.map_err(\|error\| {
1439	ToolError::execution_failed(format!(
1440	"failed to preserve fetched media artifact: {error}"
1441	))
1442	})?;
1443	let path = crate::artifacts::format_artifact_relative_path(&relative);
1444	Ok(WebPage {
1445	url: payload.url,
1446	title: Some("Media artifact".to_string()),
1447	content_type,
1448	lines: vec![format!("Fetched media saved to {path}")],
1449	links: Vec::new(),
1450	pdf_pages: None,
1451	truncated: payload.truncated,
1452	})
1453	}
1454	}
1455	}
1456
1457	fn bounded_web_run_result(
1458	output: &WebRunOutput,
1459	context: &ToolContext,
1460	) -> Result<ToolResult, ToolError> {
1461	let full = serde_json::to_string_pretty(output)
1462	.map_err(\|error\| ToolError::execution_failed(error.to_string()))?;
1463	let bounded = bound_web_text(
1464	full,
1465	context,
1466	\|body\| {
1467	let digest = crate::hashing::sha256_hex(body.as_bytes());
1468	format!("web_run_{}", &digest[..16])
1469	},
1470	"web.run result",
1471	)?;
1472	let metadata = bounded.artifact.map(\|artifact\| {
1473	json!({
1474	"spillover_path": artifact.absolute_path.display().to_string(),
1475	"artifact_session_id": artifact.session_id,
1476	"artifact_relative_path": crate::artifacts::format_artifact_relative_path(&artifact.relative_path),
1477	"artifact_byte_size": artifact.byte_size,
1478	"artifact_preview": artifact.preview,
1479	// The overflow footer points at `retrieve_tool_result`; flag the
1480	// evidence so the engine auto-activates that tool next turn (same
1481	// contract as the shell-truncation spillover).
1482	"evidence_available": true,
1483	})
1484	});
1485
1486	Ok(ToolResult {
1487	content: bounded.content,
1488	success: true,
1489	metadata,
1490	})
1491	}
1492
1493	fn render_view(
1494	ref_id: &str,
1495	page: &WebPage,
1496	lineno: usize,
1497	response: ResponseLength,
1498	) -> PageViewResult {
1499	let total = page.lines.len();
1500	let view_lines = response.view_lines();
1501	let start = if total == 0 {
1502	1
1503	} else if lineno > total {
1504	total.saturating_sub(view_lines.saturating_sub(1)).max(1)
1505	} else {
1506	lineno
1507	};
1508	let end = if total == 0 {
1509	0
1510	} else {
1511	(start + view_lines - 1).min(total)
1512	};
1513
1514	let content = if total == 0 {
1515	"(no content)".to_string()
1516	} else {
1517	render_lines(&page.lines, start, end)
1518	};
1519
1520	PageViewResult {
1521	ref_id: ref_id.to_string(),
1522	url: page.url.clone(),
1523	title: page.title.clone(),
1524	content_type: page.content_type.clone(),
1525	line_start: start,
1526	line_end: end,
1527	total_lines: total,
1528	truncated: page.truncated,
1529	content,
1530	links: page.links.clone(),
1531	}
1532	}
1533
1534	fn render_lines(lines: &[String], start: usize, end: usize) -> String {
1535	lines
1536	.iter()
1537	.enumerate()
1538	.filter_map(\|(idx, line)\| {
1539	let line_no = idx + 1;
1540	if line_no < start \|\| line_no > end {
1541	return None;
1542	}
1543	Some(format!("{line_no:>4} {line}"))
1544	})
1545	.collect::<Vec<_>>()
1546	.join("\n")
1547	}
1548
1549	fn find_in_page(
1550	ref_id: &str,
1551	pattern: &str,
1552	page: &WebPage,
1553	response: ResponseLength,
1554	) -> FindResult {
1555	let needle = pattern.to_lowercase();
1556	let mut matches = Vec::new();
1557	for (idx, line) in page.lines.iter().enumerate() {
1558	if line.to_lowercase().contains(&needle) {
1559	matches.push(FindMatch {
1560	line: idx + 1,
1561	text: line.clone(),
1562	});
1563	}
1564	if matches.len() >= response.max_find_matches() {
1565	break;
1566	}
1567	}
1568
1569	FindResult {
1570	ref_id: ref_id.to_string(),
1571	pattern: pattern.to_string(),
1572	count: matches.len(),
1573	matches,
1574	}
1575	}
1576
1577	fn screenshot_page(
1578	ref_id: &str,
1579	pageno: usize,
1580	page: &WebPage,
1581	) -> Result<ScreenshotResult, ToolError> {
1582	let pages = page
1583	.pdf_pages
1584	.as_ref()
1585	.ok_or_else(\|\| ToolError::invalid_input("screenshot is only supported for PDF pages"))?;
1586	if pages.is_empty() {
1587	return Err(ToolError::execution_failed("PDF has no pages"));
1588	}
1589	if pageno >= pages.len() {
1590	return Err(ToolError::invalid_input(format!(
1591	"pageno {pageno} out of range (0..{max})",
1592	max = pages.len().saturating_sub(1)
1593	)));
1594	}
1595	let content = pages[pageno].join("\n");
1596	Ok(ScreenshotResult {
1597	ref_id: ref_id.to_string(),
1598	pageno,
1599	total_pages: pages.len(),
1600	content,
1601	})
1602	}
1603
1604	// === HTML Parsing ===
1605
1606	static ANCHOR_RE: OnceLock<Regex> = OnceLock::new();
1607	static TAG_RE: OnceLock<Regex> = OnceLock::new();
1608	static BLOCK_RE: OnceLock<Regex> = OnceLock::new();
1609	static SCRIPT_RE: OnceLock<Regex> = OnceLock::new();
1610	static STYLE_RE: OnceLock<Regex> = OnceLock::new();
1611	static TITLE_RE: OnceLock<Regex> = OnceLock::new();
1612	static MARKDOWN_LINK_RE: OnceLock<Regex> = OnceLock::new();
1613
1614	fn get_anchor_re() -> &'static Regex {
1615	ANCHOR_RE.get_or_init(\|\| {
1616	Regex::new(r#"(?is)<a\s+[^>]href\s=\s['\"]([^'\"]+)['\"][^>]>(.*?)</a>"#)
1617	.expect("anchor regex")
1618	})
1619	}
1620
1621	fn get_tag_re() -> &'static Regex {
1622	TAG_RE.get_or_init(\|\| Regex::new(r"<[^>]+>").expect("tag regex"))
1623	}
1624
1625	fn get_block_re() -> &'static Regex {
1626	BLOCK_RE.get_or_init(\|\| {
1627	Regex::new(r"(?is)</?(p\|div\|li\|ul\|ol\|br\|h[1-6]\|tr\|td\|th\|table\|section\|article)[^>]*>")
1628	.expect("block regex")
1629	})
1630	}
1631
1632	fn get_script_re() -> &'static Regex {
1633	SCRIPT_RE.get_or_init(\|\| Regex::new(r"(?is)<script[^>]>.?</script>").unwrap())
1634	}
1635
1636	fn get_style_re() -> &'static Regex {
1637	STYLE_RE.get_or_init(\|\| Regex::new(r"(?is)<style[^>]>.?</style>").unwrap())
1638	}
1639
1640	fn get_title_re() -> &'static Regex {
1641	TITLE_RE.get_or_init(\|\| Regex::new(r"(?is)<title[^>]>(.?)</title>").unwrap())
1642	}
1643
1644	fn parse_html(html: &str, base_url: &str) -> (Vec<String>, Vec<WebLink>, Option<String>) {
1645	let title = extract_title(html);
1646	let without_scripts = get_script_re().replace_all(html, "").to_string();
1647	let without_styles = get_style_re().replace_all(&without_scripts, "").to_string();
1648
1649	let (with_links, links) = replace_links(&without_styles, base_url);
1650	let with_breaks = get_block_re().replace_all(&with_links, "\n").to_string();
1651	let without_tags = get_tag_re().replace_all(&with_breaks, "").to_string();
1652	let decoded = decode_html_entities(&without_tags);
1653
1654	let mut lines = Vec::new();
1655	for line in decoded.lines() {
1656	let trimmed = normalize_whitespace(line);
1657	if trimmed.is_empty() {
1658	continue;
1659	}
1660	for wrapped in wrap_line(&trimmed, ResponseLength::Medium.wrap_width()) {
1661	lines.push(wrapped);
1662	}
1663	}
1664
1665	(lines, links, title)
1666	}
1667
1668	fn parse_markdown(markdown: &str, base_url: &str) -> (Vec<String>, Vec<WebLink>) {
1669	let re = MARKDOWN_LINK_RE.get_or_init(\|\| {
1670	Regex::new(r#"\[([^\]]+)\]\(([^\s)]+)(?:\s+"[^"]*")?\)"#).expect("markdown link regex")
1671	});
1672	let mut links = Vec::new();
1673	let mut replaced = String::with_capacity(markdown.len());
1674	let mut last = 0;
1675	for capture in re.captures_iter(markdown) {
1676	let Some(full) = capture.get(0) else { continue };
1677	let Some(text) = capture.get(1) else { continue };
1678	let Some(target) = capture.get(2) else {
1679	continue;
1680	};
1681	replaced.push_str(&markdown[last..full.start()]);
1682	let id = links.len() + 1;
1683	let text = normalize_whitespace(text.as_str());
1684	let url = resolve_url(base_url, target.as_str());
1685	links.push(WebLink {
1686	id,
1687	url,
1688	text: text.clone(),
1689	});
1690	replaced.push_str(&format!("[{id}] {text}"));
1691	last = full.end();
1692	}
1693	replaced.push_str(&markdown[last..]);
1694	(readable_lines(&replaced), links)
1695	}
1696
1697	fn readable_lines(text: &str) -> Vec<String> {
1698	text.lines()
1699	.flat_map(\|line\| {
1700	let line = normalize_whitespace(line);
1701	wrap_line(&line, ResponseLength::Medium.wrap_width())
1702	})
1703	.filter(\|line\| !line.is_empty())
1704	.collect()
1705	}
1706
1707	fn extract_title(html: &str) -> Option<String> {
1708	let re = get_title_re();
1709	let cap = re.captures(html)?;
1710	let raw = cap.get(1)?.as_str();
1711	let cleaned = normalize_whitespace(&decode_html_entities(raw));
1712	if cleaned.is_empty() {
1713	None
1714	} else {
1715	Some(cleaned)
1716	}
1717	}
1718
1719	fn replace_links(html: &str, base_url: &str) -> (String, Vec<WebLink>) {
1720	let re = get_anchor_re();
1721	let mut links = Vec::new();
1722	let mut output = String::with_capacity(html.len());
1723	let mut last = 0;
1724
1725	for cap in re.captures_iter(html) {
1726	let Some(full) = cap.get(0) else { continue };
1727	let Some(href) = cap.get(1) else { continue };
1728	let Some(text_match) = cap.get(2) else {
1729	continue;
1730	};
1731
1732	output.push_str(&html[last..full.start()]);
1733	let text = normalize_whitespace(&strip_tags(text_match.as_str()));
1734	let resolved = resolve_url(base_url, href.as_str());
1735	if !text.is_empty() {
1736	let id = links.len() + 1;
1737	links.push(WebLink {
1738	id,
1739	url: resolved.clone(),
1740	text: text.clone(),
1741	});
1742	output.push_str(&format!("[{id}] {text}"));
1743	} else {
1744	output.push_str(&resolved);
1745	}
1746	last = full.end();
1747	}
1748
1749	output.push_str(&html[last..]);
1750	(output, links)
1751	}
1752
1753	fn resolve_url(base: &str, href: &str) -> String {
1754	if href.starts_with("http://") \|\| href.starts_with("https://") {
1755	return href.to_string();
1756	}
1757	if href.starts_with("//") {
1758	return format!("https:{href}");
1759	}
1760	if let Ok(base_url) = reqwest::Url::parse(base)
1761	&& let Ok(joined) = base_url.join(href)
1762	{
1763	return joined.to_string();
1764	}
1765	href.to_string()
1766	}
1767
1768	fn strip_tags(text: &str) -> String {
1769	get_tag_re().replace_all(text, "").to_string()
1770	}
1771
1772	fn normalize_whitespace(text: &str) -> String {
1773	text.split_whitespace().collect::<Vec<_>>().join(" ")
1774	}
1775
1776	/// Reflow one line to `width` terminal columns.
1777	///
1778	/// `width` is a column budget, so every measurement here is a display width.
1779	/// Measuring `str::len()` instead made the budget script-dependent: Cyrillic
1780	/// and Greek are two bytes per single-column character and CJK three bytes per
1781	/// double-column character, so a Russian page wrapped at half the requested
1782	/// width and a Japanese one at two thirds. That is not only ragged output —
1783	/// `render_view` pages these lines by count, so the extra lines pushed real
1784	/// content past `ResponseLength::view_lines()` and the model saw a fraction of
1785	/// the page an English URL would have returned. Widths equal byte lengths for
1786	/// ASCII, so Latin-script wrapping is unchanged.
1787	fn wrap_line(text: &str, width: usize) -> Vec<String> {
1788	if UnicodeWidthStr::width(text) <= width {
1789	return vec![text.to_string()];
1790	}
1791	let mut lines = Vec::new();
1792	let mut current = String::new();
1793	let mut current_width = 0usize;
1794	for word in text.split_whitespace() {
1795	let word_width = UnicodeWidthStr::width(word);
1796	if current.is_empty() {
1797	current.push_str(word);
1798	current_width = word_width;
1799	} else if current_width + word_width < width {
1800	current.push(' ');
1801	current.push_str(word);
1802	current_width += 1 + word_width;
1803	} else {
1804	lines.push(std::mem::take(&mut current));
1805	current.push_str(word);
1806	current_width = word_width;
1807	}
1808	}
1809	if !current.is_empty() {
1810	lines.push(current);
1811	}
1812	lines
1813	}
1814
1815	fn decode_html_entities(text: &str) -> String {
1816	text.replace("&", "&")
1817	.replace(""", "\"")
1818	.replace("'", "'")
1819	.replace("'", "'")
1820	.replace("<", "<")
1821	.replace(">", ">")
1822	.replace(" ", " ")
1823	}
1824
1825	fn url_encode(input: &str) -> String {
1826	crate::utils::url_encode(input)
1827	}
1828
1829	// === Tests ===
1830
1831	#[cfg(test)]
1832	mod tests {
1833	use super::*;
1834	use crate::tools::web::scrape::{parse_bing_results, parse_duckduckgo_results};
1835	use std::path::PathBuf;
1836	use tokio::sync::{Mutex, MutexGuard};
1837
1838	static WEB_RUN_TEST_LOCK: Mutex<()> = Mutex::const_new(());
1839
1840	struct ArtifactRootRestore(Option<PathBuf>);
1841
1842	impl Drop for ArtifactRootRestore {
1843	fn drop(&mut self) {
1844	crate::artifacts::set_test_artifact_sessions_root(self.0.take());
1845	}
1846	}
1847
1848	fn lock_web_run_test_state() -> MutexGuard<'static, ()> {
1849	WEB_RUN_TEST_LOCK.blocking_lock()
1850	}
1851
1852	fn sample_page(url: &str) -> WebPage {
1853	WebPage {
1854	url: url.to_string(),
1855	title: Some("Example".to_string()),
1856	content_type: Some("text/html".to_string()),
1857	lines: vec!["example line".to_string()],
1858	links: Vec::new(),
1859	pdf_pages: None,
1860	truncated: false,
1861	}
1862	}
1863
1864	fn sample_page_with_link(url: &str, target: &str) -> WebPage {
1865	let mut page = sample_page(url);
1866	page.links.push(WebLink {
1867	id: 1,
1868	url: target.to_string(),
1869	text: "target".to_string(),
1870	});
1871	page
1872	}
1873
1874	#[test]
1875	fn html_link_parsing_extracts_links() {
1876	let html = r#"
1877	<html><body>
1878	<p>Hello <a href="https://example.com">Example</a> world.</p>
1879	</body></html>
1880	"#;
1881	let (lines, links, title) = parse_html(html, "https://example.com");
1882	assert!(title.is_none());
1883	assert_eq!(links.len(), 1);
1884	assert_eq!(links[0].url, "https://example.com");
1885	assert!(lines.iter().any(\|line\| line.contains("Example")));
1886	}
1887
1888	#[test]
1889	fn markdown_link_parsing_preserves_click_targets() {
1890	let (lines, links) = parse_markdown(
1891	"## Guide\n\nRead [the proof](/proof) before shipping.",
1892	"https://example.com/docs/start",
1893	);
1894
1895	assert_eq!(links.len(), 1);
1896	assert_eq!(links[0].url, "https://example.com/proof");
1897	assert!(lines.iter().any(\|line\| line.contains("[1] the proof")));
1898	}
1899
1900	#[test]
1901	fn oversized_web_run_output_round_trips_through_session_artifact() {
1902	let _lock = crate::artifacts::TEST_ARTIFACT_SESSIONS_GUARD
1903	.lock()
1904	.unwrap_or_else(\|error\| error.into_inner());
1905	let tmp = tempfile::tempdir().unwrap();
1906	let prior =
1907	crate::artifacts::set_test_artifact_sessions_root(Some(tmp.path().join("sessions")));
1908	let _restore = ArtifactRootRestore(prior);
1909	let context = ToolContext::new(".")
1910	.with_state_namespace("web-run-overflow")
1911	.with_route_context_window(10_000);
1912	let output = WebRunOutput {
1913	warnings: vec!["large receipt ".repeat(200)],
1914	..WebRunOutput::default()
1915	};
1916
1917	let result = bounded_web_run_result(&output, &context).unwrap();
1918	let metadata = result.metadata.expect("artifact metadata");
1919	let path = metadata["spillover_path"].as_str().unwrap();
1920	let full = std::fs::read_to_string(path).unwrap();
1921
1922	assert!(result.content.contains("retrieve_tool_result"));
1923	assert!(result.content.chars().count() <= inline_char_budget(&context));
1924	assert_eq!(
1925	metadata["evidence_available"],
1926	json!(true),
1927	"overflow footer points at retrieve_tool_result; the metadata must \
1928	flag the evidence so the engine auto-activates that tool:\n{metadata}"
1929	);
1930	assert_eq!(
1931	serde_json::from_str::<Value>(&full).unwrap()["warnings"][0],
1932	output.warnings[0]
1933	);
1934	}
1935
1936	#[test]
1937	fn wrap_line_measures_display_width_not_bytes() {
1938	// Same shape, two scripts: twelve four-character words. Cyrillic is two
1939	// bytes per single-column character, so measuring `len()` wrapped the
1940	// Russian text at half the requested column budget and handed
1941	// `render_view` twice as many lines to page through.
1942	let latin = ["abcd"; 12].join(" ");
1943	let cyrillic = ["абвг"; 12].join(" ");
1944	assert_eq!(
1945	wrap_line(&cyrillic, 20).len(),
1946	wrap_line(&latin, 20).len(),
1947	"cyrillic: {:?}\nlatin: {:?}",
1948	wrap_line(&cyrillic, 20),
1949	wrap_line(&latin, 20)
1950	);
1951	for line in wrap_line(&cyrillic, 20) {
1952	assert!(
1953	UnicodeWidthStr::width(line.as_str()) <= 20,
1954	"wrapped past the column budget: {line:?}"
1955	);
1956	}
1957	}
1958
1959	#[test]
1960	fn wrap_line_splits_long_lines() {
1961	let line = "This is a long line that should wrap cleanly at word boundaries";
1962	let wrapped = wrap_line(line, 20);
1963	assert!(wrapped.len() > 1);
1964	assert!(wrapped.iter().all(\|l\| l.len() <= 20));
1965	}
1966
1967	#[test]
1968	fn extracts_duckduckgo_vqd_token() {
1969	let html_single = "<script>var x = {vqd='3-1234567890'};</script>";
1970	assert_eq!(
1971	extract_duckduckgo_vqd(html_single),
1972	Some("3-1234567890".to_string())
1973	);
1974
1975	let html_double = "<script>var x = {vqd=\"3-abcdef\"};</script>";
1976	assert_eq!(
1977	extract_duckduckgo_vqd(html_double),
1978	Some("3-abcdef".to_string())
1979	);
1980
1981	let html_plain = "https://duckduckgo.com/?q=test&vqd=3-xyz_123&ia=images";
1982	assert_eq!(
1983	extract_duckduckgo_vqd(html_plain),
1984	Some("3-xyz_123".to_string())
1985	);
1986	}
1987
1988	#[tokio::test]
1989	async fn text_search_uses_configured_shared_backend_and_exposes_receipt() {
1990	use crate::config::SearchProvider;
1991	use crate::tools::spec::ToolSpec;
1992	use wiremock::matchers::{method, path, query_param};
1993	use wiremock::{Mock, MockServer, ResponseTemplate};
1994
1995	let server = MockServer::start().await;
1996	Mock::given(method("GET"))
1997	.and(path("/search"))
1998	.and(query_param("q", "shared seam"))
1999	.and(query_param("format", "json"))
2000	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2001	"results": [{
2002	"title": "Shared result",
2003	"url": "https://docs.example.com/shared",
2004	"content": "one adapter path"
2005	}]
2006	})))
2007	.mount(&server)
2008	.await;
2009
2010	let tmp = tempfile::tempdir().expect("tempdir");
2011	let mut context = ToolContext::new(tmp.path().to_path_buf());
2012	context.search_provider = SearchProvider::Searxng;
2013	context.search_base_url = Some(server.uri());
2014	context.state_namespace = "shared-backend-test".to_string();
2015
2016	let result = WebRunTool
2017	.execute(
2018	json!({
2019	"search_query": [{
2020	"q": "shared seam",
2021	"recency": 7,
2022	"domains": ["example.com"]
2023	}]
2024	}),
2025	&context,
2026	)
2027	.await
2028	.expect("web.run should use configured SearXNG backend");
2029	let value: Value = serde_json::from_str(&result.content).expect("web.run json");
2030	let search = &value["search_query"][0];
2031
2032	assert_eq!(search["source"], "searxng");
2033	assert_eq!(search["count"], 1);
2034	assert_eq!(search["results"][0]["rank"], 1);
2035	assert_eq!(search["results"][0]["domain"], "docs.example.com");
2036	assert_eq!(search["receipt"]["backend"], "searxng");
2037	assert_eq!(search["receipt"]["honored"]["domains"], true);
2038	assert_eq!(
2039	search["receipt"]["honored"]["recency"], true,
2040	"SearXNG forwards recency as time_range"
2041	);
2042	}
2043
2044	#[tokio::test]
2045	async fn search_ref_ids_resolve_to_their_source_urls_for_open() {
2046	use crate::config::SearchProvider;
2047	use crate::tools::spec::ToolSpec;
2048	use wiremock::matchers::{method, path, query_param};
2049	use wiremock::{Mock, MockServer, ResponseTemplate};
2050
2051	let server = MockServer::start().await;
2052	Mock::given(method("GET"))
2053	.and(path("/search"))
2054	.and(query_param("q", "citation handoff"))
2055	.and(query_param("format", "json"))
2056	.respond_with(ResponseTemplate::new(200).set_body_json(json!({
2057	"results": [{
2058	"title": "Handoff target",
2059	"url": "https://docs.example.com/handoff",
2060	"content": "the page a later open command fetches"
2061	}]
2062	})))
2063	.mount(&server)
2064	.await;
2065
2066	let tmp = tempfile::tempdir().expect("tempdir");
2067	let mut context = ToolContext::new(tmp.path().to_path_buf());
2068	context.search_provider = SearchProvider::Searxng;
2069	context.search_base_url = Some(server.uri());
2070	context.state_namespace = "handoff-citation-test".to_string();
2071
2072	let result = WebRunTool
2073	.execute(
2074	json!({"search_query": [{"q": "citation handoff"}]}),
2075	&context,
2076	)
2077	.await
2078	.expect("web.run search should succeed");
2079	let value: Value = serde_json::from_str(&result.content).expect("web.run json");
2080	let ref_id = value["search_query"][0]["results"][0]["ref_id"]
2081	.as_str()
2082	.expect("every search result carries a minted ref_id")
2083	.to_string();
2084
2085	// `open` resolves search-result refs through the shared citation
2086	// registry; the handoff must preserve the exact source URL.
2087	let citation = crate::tools::web::citations::resolve(&context.state_namespace, &ref_id)
2088	.expect("search result ref must resolve for a later open");
2089	assert_eq!(citation.url, "https://docs.example.com/handoff");
2090	assert_eq!(citation.title.as_deref(), Some("Handoff target"));
2091	assert!(
2092	crate::tools::web::citations::resolve("foreign-session", &ref_id).is_none(),
2093	"citation handles must stay session-scoped"
2094	);
2095	}
2096
2097	#[tokio::test]
2098	async fn open_failure_reports_url_and_status() {
2099	let payload = crate::tools::web::fetch::FetchedPayload {
2100	url: "https://example.com/missing".to_string(),
2101	status: 404,
2102	headers: std::collections::BTreeMap::new(),
2103	content_type: "text/html".to_string(),
2104	bytes: Arc::new(Vec::new()),
2105	truncated: false,
2106	cache_hit: false,
2107	retries: 0,
2108	redirects: 0,
2109	};
2110	let context = ToolContext::new(PathBuf::from("."));
2111
2112	let error = document_from_fetched(&payload, &context)
2113	.await
2114	.expect_err("non-2xx pages must not be rendered");
2115	let message = error.to_string();
2116	assert!(
2117	message.contains("https://example.com/missing"),
2118	"transport failures must name the URL: {message}"
2119	);
2120	assert!(message.contains("404"), "got `{message}`");
2121	}
2122
2123	#[test]
2124	fn response_length_result_counts_are_anchored_to_the_shared_contract() {
2125	assert_eq!(ResponseLength::Short.max_results(), DEFAULT_SEARCH_RESULTS);
2126	assert_eq!(
2127	ResponseLength::Long.max_results(),
2128	usize::from(MAX_SEARCH_RESULTS)
2129	);
2130	assert!(ResponseLength::Medium.max_results() <= usize::from(MAX_SEARCH_RESULTS));
2131	}
2132
2133	#[test]
2134	fn parses_bing_results_and_decodes_redirect_url() {
2135	let html = r#"
2136	<ol>
2137	<li class="b_algo">
2138	<h2><a href="https://www.bing.com/ck/a?u=a1aHR0cHM6Ly9leGFtcGxlLmNvbS9wYXRoP3E9MQ">Example & Result</a></h2>
2139	<div class="b_caption"><p>A <strong>useful</strong> snippet.</p></div>
2140	</li>
2141	</ol>
2142	"#;
2143
2144	let results = parse_bing_results(html, 5);
2145
2146	assert_eq!(results.len(), 1);
2147	assert_eq!(results[0].title, "Example & Result");
2148	assert_eq!(results[0].url, "https://example.com/path?q=1");
2149	assert_eq!(results[0].snippet.as_deref(), Some("A useful snippet."));
2150	}
2151
2152	#[test]
2153	fn web_run_search_path_filters_known_spam_domain() {
2154	// The shared scraper used by web_run filters the known #964 spam family.
2155	let html = r#"
2156	<a class="result__a" href="https://astralia.forumgratuit.org/a">A</a>
2157	<a class="result__snippet">spam</a>
2158	<a class="result__a" href="https://russia.forumgratuit.org/b">B</a>
2159	<a class="result__snippet">spam</a>
2160	<a class="result__a" href="https://other.forumgratuit.org/c">C</a>
2161	<a class="result__snippet">spam</a>
2162	<a class="result__a" href="https://hello.forumgratuit.org/d">D</a>
2163	<a class="result__snippet">spam</a>
2164	<a class="result__a" href="https://world.forumgratuit.org/e">E</a>
2165	<a class="result__snippet">spam</a>
2166	"#;
2167	let results = parse_duckduckgo_results(html, 10);
2168	assert!(
2169	results.is_empty(),
2170	"web_run path must drop the known spam family via the shared scraper"
2171	);
2172	}
2173
2174	#[test]
2175	fn domain_scoped_fixture_preserves_legitimate_same_site_results() {
2176	let html = r#"
2177	<a class="result__a" href="https://docs.example.co.uk/a">A</a>
2178	<a class="result__snippet">s</a>
2179	<a class="result__a" href="https://docs.example.co.uk/b">B</a>
2180	<a class="result__snippet">s</a>
2181	<a class="result__a" href="https://docs.example.co.uk/c">C</a>
2182	<a class="result__snippet">s</a>
2183	<a class="result__a" href="https://other.example/d">D</a>
2184	<a class="result__snippet">s</a>
2185	"#;
2186	let domains = vec!["docs.example.co.uk".to_string()];
2187	let mut results = parse_duckduckgo_results(html, 10);
2188	results.retain(\|entry\| domain_matches(&entry.url, &domains));
2189
2190	assert_eq!(results.len(), 3);
2191	assert!(
2192	results
2193	.iter()
2194	.all(\|entry\| entry.url.contains("docs.example.co.uk"))
2195	);
2196	}
2197
2198	#[test]
2199	fn scoped_ref_prefix_is_session_specific() {
2200	let _lock = lock_web_run_test_state();
2201	reset_web_run_state();
2202	let alpha = scoped_ref_prefix("session-alpha");
2203	let beta = scoped_ref_prefix("session-beta");
2204
2205	assert_ne!(alpha, beta);
2206	assert!(alpha.starts_with('s'));
2207	assert!(alpha.ends_with('_'));
2208	assert_eq!(alpha.len(), 18);
2209	}
2210
2211	#[test]
2212	fn stored_pages_do_not_cross_scoped_sessions() {
2213	let _lock = lock_web_run_test_state();
2214	reset_web_run_state();
2215	let shared_suffix = "turn1search1";
2216	let ref_alpha = format!("{}{}", scoped_ref_prefix("session-alpha"), shared_suffix);
2217	let ref_beta = format!("{}{}", scoped_ref_prefix("session-beta"), shared_suffix);
2218
2219	store_page(
2220	"session-alpha",
2221	&ref_alpha,
2222	sample_page("https://example.com/alpha"),
2223	);
2224
2225	assert!(get_page("session-alpha", &ref_alpha).is_some());
2226	assert!(get_page("session-beta", &ref_alpha).is_none());
2227	assert!(get_page("session-beta", &ref_beta).is_none());
2228	}
2229
2230	#[tokio::test(flavor = "current_thread")]
2231	async fn execute_open_rejects_exact_foreign_session_ref() {
2232	let _lock = WEB_RUN_TEST_LOCK.lock().await;
2233	reset_web_run_state();
2234	let ref_id = format!("{}turn0search1", scoped_ref_prefix("foreign-open-owner"));
2235	store_page(
2236	"foreign-open-owner",
2237	&ref_id,
2238	sample_page("https://example.com/private-session-page"),
2239	);
2240	let context =
2241	ToolContext::new(PathBuf::from(".")).with_state_namespace("foreign-open-caller");
2242
2243	let err = WebRunTool
2244	.execute(json!({"open": [{"ref_id": ref_id}]}), &context)
2245	.await
2246	.expect_err("foreign exact ref must not open");
2247
2248	assert!(format!("{err}").contains("Unknown ref_id"));
2249	}
2250
2251	#[tokio::test(flavor = "current_thread")]
2252	async fn execute_click_rejects_exact_foreign_session_ref() {
2253	let _lock = WEB_RUN_TEST_LOCK.lock().await;
2254	reset_web_run_state();
2255	let ref_id = format!("{}turn0search1", scoped_ref_prefix("foreign-click-owner"));
2256	store_page(
2257	"foreign-click-owner",
2258	&ref_id,
2259	sample_page_with_link(
2260	"https://example.com/private-session-page",
2261	"https://example.com/target",
2262	),
2263	);
2264	let context =
2265	ToolContext::new(PathBuf::from(".")).with_state_namespace("foreign-click-caller");
2266
2267	let err = WebRunTool
2268	.execute(json!({"click": [{"ref_id": ref_id, "id": 1}]}), &context)
2269	.await
2270	.expect_err("foreign exact ref must not be clickable");
2271
2272	assert!(format!("{err}").contains("Unknown ref_id"));
2273	}
2274
2275	#[tokio::test(flavor = "current_thread")]
2276	async fn execute_click_routes_target_through_shared_ssrf_guard() {
2277	let _lock = WEB_RUN_TEST_LOCK.lock().await;
2278	reset_web_run_state();
2279	let namespace = "guarded-click-session";
2280	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2281	store_page(
2282	namespace,
2283	&ref_id,
2284	sample_page_with_link("https://example.com/source", "http://127.0.0.1/admin"),
2285	);
2286	let context = ToolContext::new(PathBuf::from(".")).with_state_namespace(namespace);
2287
2288	let err = WebRunTool
2289	.execute(json!({"click": [{"ref_id": ref_id, "id": 1}]}), &context)
2290	.await
2291	.expect_err("click target must be SSRF-guarded");
2292
2293	assert!(format!("{err}").contains("restricted address"));
2294	}
2295
2296	#[test]
2297	fn cached_page_reads_share_page_arc() {
2298	let _lock = lock_web_run_test_state();
2299	reset_web_run_state();
2300	let namespace = "session-alpha";
2301	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2302	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
2303
2304	let first = get_page(namespace, &ref_id).expect("first page read");
2305	let second = get_page(namespace, &ref_id).expect("second page read");
2306
2307	assert!(Arc::ptr_eq(&first, &second));
2308	}
2309
2310	#[test]
2311	fn turn_counters_are_scoped_per_session() {
2312	let _lock = lock_web_run_test_state();
2313	reset_web_run_state();
2314
2315	assert_eq!(next_turn_for_namespace("session-alpha"), 0);
2316	assert_eq!(next_turn_for_namespace("session-alpha"), 1);
2317	assert_eq!(next_turn_for_namespace("session-beta"), 0);
2318	}
2319
2320	#[test]
2321	fn with_state_restores_cache_after_panic() {
2322	let _lock = lock_web_run_test_state();
2323	reset_web_run_state();
2324	let namespace = "session-alpha";
2325	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2326	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
2327
2328	let panic_result = std::panic::catch_unwind(\|\| {
2329	with_state(\|state\| {
2330	let session = state
2331	.sessions
2332	.get_mut(namespace)
2333	.expect("session should exist");
2334	session.next_turn = 42;
2335	panic!("exercise web_run write-back guard");
2336	});
2337	});
2338
2339	assert!(panic_result.is_err());
2340	assert!(get_page(namespace, &ref_id).is_some());
2341	assert_eq!(next_turn_for_namespace(namespace), 42);
2342	}
2343
2344	#[test]
2345	fn stale_session_pages_are_evicted() {
2346	let _lock = lock_web_run_test_state();
2347	reset_web_run_state();
2348	let namespace = "session-alpha";
2349	let ref_id = format!("{}turn0search1", scoped_ref_prefix(namespace));
2350	store_page(namespace, &ref_id, sample_page("https://example.com/alpha"));
2351
2352	// On Windows, Instant's epoch is system boot. If the CI runner has
2353	// been up for less than WEB_RUN_SESSION_TTL the subtraction would
2354	// underflow, so we skip the test in that case.
2355	let stale = WEB_RUN_SESSION_TTL + Duration::from_secs(1);
2356	let can_test = with_state(\|state\| {
2357	let session = state
2358	.sessions
2359	.get_mut(namespace)
2360	.expect("session should exist");
2361	match Instant::now().checked_sub(stale) {
2362	Some(past) => {
2363	session.last_access = past;
2364	true
2365	}
2366	None => false,
2367	}
2368	});
2369	if !can_test {
2370	// System uptime shorter than session TTL; can't test eviction.
2371	return;
2372	}
2373
2374	let _ = next_turn_for_namespace("session-beta");
2375
2376	assert!(get_page(namespace, &ref_id).is_none());
2377	}
2378
2379	#[test]
2380	fn direct_urls_remain_compatible_open_refs() {
2381	assert!(looks_like_url("https://example.com"));
2382	assert!(looks_like_url("http://example.com"));
2383	assert!(!looks_like_url("turn0search0"));
2384	}
2385
2386	#[tokio::test]
2387	async fn network_policy_denies_direct_open_url() {
2388	use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider};
2389
2390	let policy = NetworkPolicy {
2391	default: Decision::Deny.into(),
2392	allow: vec!["api.deepseek.com".to_string()],
2393	deny: vec![],
2394	proxy: Vec::new(),
2395	proxy_fake_ip_cidrs: Vec::new(),
2396	audit: false,
2397	};
2398	let decider = NetworkPolicyDecider::new(policy, None);
2399	let ctx = ToolContext::new(PathBuf::from(".")).with_network_policy(decider);
2400
2401	let err = fetch_page("https://example.com/private", 5_000, &ctx)
2402	.await
2403	.expect_err("blocked host should fail");
2404	assert!(format!("{err}").contains("blocked by network policy"));
2405	}
2406
2407	fn ssrf_ctx() -> ToolContext {
2408	ToolContext::new(PathBuf::from("."))
2409	}
2410
2411	#[tokio::test]
2412	async fn open_refuses_loopback_ip_url() {
2413	let err = resolve_or_fetch_page("http://127.0.0.1/", 5_000, &ssrf_ctx())
2414	.await
2415	.expect_err("loopback open must be refused");
2416	assert!(
2417	format!("{err}").contains("restricted address"),
2418	"expected restricted-address error; got {err}"
2419	);
2420	}
2421
2422	#[tokio::test]
2423	async fn open_resolves_shared_citation_refs_only_in_their_session() {
2424	let namespace = "shared-citation-open-session";
2425	let citation = crate::tools::web::citations::register(
2426	namespace,
2427	"http://127.0.0.1/private",
2428	Some("Private target"),
2429	)
2430	.expect("valid HTTP citation metadata");
2431	let owner_context = ToolContext::new(PathBuf::from(".")).with_state_namespace(namespace);
2432	let owner_error = resolve_or_fetch_page(&citation.ref_id, 5_000, &owner_context)
2433	.await
2434	.expect_err("resolved citation must still use the SSRF guard");
2435	assert!(format!("{owner_error}").contains("restricted address"));
2436
2437	let foreign_context = ToolContext::new(PathBuf::from("."))
2438	.with_state_namespace("shared-citation-foreign-session");
2439	let foreign_error = resolve_or_fetch_page(&citation.ref_id, 5_000, &foreign_context)
2440	.await
2441	.expect_err("foreign session must not resolve citation ref");
2442	assert!(format!("{foreign_error}").contains("Unknown ref_id"));
2443	}
2444
2445	#[tokio::test]
2446	async fn open_refuses_private_range_ip_url() {
2447	let err = resolve_or_fetch_page("http://192.168.1.50/admin", 5_000, &ssrf_ctx())
2448	.await
2449	.expect_err("private-range open must be refused");
2450	assert!(
2451	format!("{err}").contains("restricted address"),
2452	"expected restricted-address error; got {err}"
2453	);
2454	}
2455
2456	#[tokio::test]
2457	async fn open_refuses_metadata_endpoint_ip_url() {
2458	let err = resolve_or_fetch_page(
2459	"http://169.254.169.254/latest/meta-data",
2460	5_000,
2461	&ssrf_ctx(),
2462	)
2463	.await
2464	.expect_err("cloud metadata open must be refused");
2465	assert!(
2466	format!("{err}").contains("restricted address"),
2467	"expected restricted-address error; got {err}"
2468	);
2469	}
2470
2471	#[tokio::test]
2472	async fn open_refuses_redirect_from_public_host_to_private_ip() {
2473	use wiremock::matchers::method;
2474	use wiremock::{Mock, MockServer, ResponseTemplate};
2475
2476	// Use a public-looking hostname pinned to the local fixture for the
2477	// already-validated first hop. The redirect itself still goes through
2478	// the real shared guard inside fetch_page's redirect loop.
2479	let server = MockServer::start().await;
2480	let private_location = "http://10.0.0.5/internal";
2481	Mock::given(method("GET"))
2482	.respond_with(ResponseTemplate::new(302).insert_header("Location", private_location))
2483	.mount(&server)
2484	.await;
2485	let host = "public-redirect.example.test";
2486	let initial_url = format!("http://{host}:{}/", server.address().port());
2487	let pin = Some((host.to_string(), "127.0.0.1".parse().unwrap()));
2488
2489	let err = fetch_page_with_initial_pin(&initial_url, 5_000, &ssrf_ctx(), Some(pin))
2490	.await
2491	.expect_err("guarded redirect to private IP must be refused");
2492	assert!(
2493	format!("{err}").contains("restricted address"),
2494	"redirect loop must surface the shared guard rejection; got {err}"
2495	);
2496	}
2497
2498	#[tokio::test(flavor = "current_thread")]
2499	async fn execute_fails_fast_when_no_op_key_matches() {
2500	// #5123-class: the natural {"query": …} shape matched no op key and
2501	// previously returned an empty SUCCESS ({"warnings":[]}).
2502	let _lock = WEB_RUN_TEST_LOCK.lock().await;
2503	reset_web_run_state();
2504	let tmp = tempfile::tempdir().expect("tempdir");
2505	let context = ToolContext::new(tmp.path());
2506
2507	let err = WebRunTool
2508	.execute(json!({"query": "rust async runtime"}), &context)
2509	.await
2510	.expect_err("unmatched op keys must fail, not return empty success");
2511	let message = format!("{err}");
2512	assert!(message.contains("performed no operation"), "{message}");
2513	assert!(message.contains("search_query"), "{message}");
2514	assert!(message.contains("query"), "{message}");
2515
2516	let err = WebRunTool
2517	.execute(json!({}), &context)
2518	.await
2519	.expect_err("empty input must fail fast");
2520	assert!(format!("{err}").contains("performed no operation"), "{err}");
2521	}
2522
2523	#[tokio::test]
2524	async fn open_retries_as_browser_once_after_a_refusal() {
2525	use std::sync::atomic::{AtomicUsize, Ordering};
2526	use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate, matchers::method};
2527
2528	#[derive(Clone)]
2529	struct RefuseBots(Arc<AtomicUsize>);
2530	impl Respond for RefuseBots {
2531	fn respond(&self, request: &Request) -> ResponseTemplate {
2532	self.0.fetch_add(1, Ordering::SeqCst);
2533	let agent = request
2534	.headers
2535	.get("user-agent")
2536	.and_then(\|value\| value.to_str().ok())
2537	.unwrap_or_default();
2538	if agent.contains("codewhale") {
2539	ResponseTemplate::new(403)
2540	} else {
2541	ResponseTemplate::new(200)
2542	.insert_header("content-type", "text/plain")
2543	.set_body_string("review body")
2544	}
2545	}
2546	}
2547
2548	let server = MockServer::start().await;
2549	let calls = Arc::new(AtomicUsize::new(0));
2550	Mock::given(method("GET"))
2551	.respond_with(RefuseBots(Arc::clone(&calls)))
2552	.mount(&server)
2553	.await;
2554	let host = "refuses-bots.example.test";
2555	let url = format!("http://{host}:{}/review", server.address().port());
2556	let pin = Some((host.to_string(), "127.0.0.1".parse().unwrap()));
2557	let context = ToolContext::new(PathBuf::from(".")).with_state_namespace("refuse-bots");
2558
2559	let page = fetch_page_with_initial_pin(&url, 5_000, &context, Some(pin))
2560	.await
2561	.expect("browser-agent fallback loads the page");
2562	assert!(page.lines.iter().any(\|line\| line.contains("review body")));
2563	assert_eq!(
2564	calls.load(Ordering::SeqCst),
2565	2,
2566	"exactly one fallback request"
2567	);
2568	}
2569
2570	#[tokio::test]
2571	async fn open_does_not_retry_a_missing_page_as_browser() {
2572	use std::sync::atomic::{AtomicUsize, Ordering};
2573	use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate, matchers::method};
2574
2575	#[derive(Clone)]
2576	struct Missing(Arc<AtomicUsize>);
2577	impl Respond for Missing {
2578	fn respond(&self, _request: &Request) -> ResponseTemplate {
2579	self.0.fetch_add(1, Ordering::SeqCst);
2580	ResponseTemplate::new(404)
2581	}
2582	}
2583
2584	let server = MockServer::start().await;
2585	let calls = Arc::new(AtomicUsize::new(0));
2586	Mock::given(method("GET"))
2587	.respond_with(Missing(Arc::clone(&calls)))
2588	.mount(&server)
2589	.await;
2590	let host = "missing.example.test";
2591	let url = format!("http://{host}:{}/gone", server.address().port());
2592	let pin = Some((host.to_string(), "127.0.0.1".parse().unwrap()));
2593	let context = ToolContext::new(PathBuf::from(".")).with_state_namespace("missing-page");
2594
2595	let err = fetch_page_with_initial_pin(&url, 5_000, &context, Some(pin))
2596	.await
2597	.expect_err("404 stays a failure");
2598	assert_eq!(http_status_of(&err.to_string()), Some(404));
2599	assert_eq!(calls.load(Ordering::SeqCst), 1);
2600	}
2601
2602	#[test]
2603	fn source_failures_are_classified_and_gates_pass_through() {
2604	let _lock = lock_web_run_test_state();
2605	reset_web_run_state();
2606	let namespace = "source-failure-session";
2607
2608	let refused = source_failure(
2609	namespace,
2610	"https://reviews.example.com/espresso",
2611	ToolError::execution_failed(
2612	"Web request to https://reviews.example.com/espresso failed: HTTP 403",
2613	),
2614	)
2615	.expect("a refusal is a source outcome");
2616	assert_eq!(refused.status, SourceStatus::Unavailable);
2617	assert!(
2618	with_state(\|state\| state.refusing_hosts(namespace)).contains("reviews.example.com"),
2619	"a refusing host is remembered for this session"
2620	);
2621
2622	let flaky = source_failure(
2623	namespace,
2624	"https://slow.example.com/",
2625	ToolError::execution_failed(
2626	"Web request to https://slow.example.com/ failed: HTTP 503",
2627	),
2628	)
2629	.expect("a 5xx is a source outcome");
2630	assert_eq!(flaky.status, SourceStatus::Transient);
2631	let timeout = source_failure(
2632	namespace,
2633	"https://slow.example.com/",
2634	ToolError::execution_failed("request timed out before retry completed"),
2635	)
2636	.expect("a timeout is a source outcome");
2637	assert_eq!(timeout.status, SourceStatus::Transient);
2638
2639	let gate = source_failure(
2640	namespace,
2641	"http://10.0.0.5/",
2642	ToolError::permission_denied("IP 10.0.0.5 is a restricted address"),
2643	);
2644	assert!(
2645	gate.is_err(),
2646	"gates must never be softened into source outcomes"
2647	);
2648	let bad_ref = source_failure(
2649	namespace,
2650	"turn9search9",
2651	ToolError::invalid_input("Unknown ref_id 'turn9search9'"),
2652	);
2653	assert!(bad_ref.is_err(), "caller mistakes stay errors");
2654	}
2655
2656	#[test]
2657	fn search_results_prefer_hosts_that_load() {
2658	let mut results = vec![
2659	NormalizedSearchResult::new(
2660	1,
2661	"a".into(),
2662	"https://blocked.example/a".into(),
2663	None,
2664	None,
2665	),
2666	NormalizedSearchResult::new(2, "b".into(), "https://open.example/b".into(), None, None),
2667	NormalizedSearchResult::new(3, "c".into(), "https://open.example/c".into(), None, None),
2668	];
2669	let refusing = HashSet::from(["blocked.example".to_string()]);
2670	prefer_loading_sources(&mut results, &refusing);
2671	let order: Vec<_> = results.iter().map(\|r\| (r.rank, r.url.as_str())).collect();
2672	assert_eq!(
2673	order,
2674	vec![
2675	(1, "https://open.example/b"),
2676	(2, "https://open.example/c"),
2677	(3, "https://blocked.example/a"),
2678	]
2679	);
2680	}
2681	}
2682

2682 lines RUST