| 1 | //! Direct-fetch HTTP tool. Complements `web_search` for cases where the user |
| 2 | //! already knows the URL — a known repo, a blog post, a spec page — and |
| 3 | //! search is overkill or actively unhelpful. |
| 4 | //! |
| 5 | //! Returns a structured `{url, status, content_type, content, truncated}` |
| 6 | //! payload. HTML responses are stripped to readable text by default |
| 7 | //! (`format = "markdown"`); pass `format = "raw"` to keep the bytes intact |
| 8 | //! when the model wants to do its own parsing. |
| 9 | |
| 10 | use super::handle::query_jsonpath; |
| 11 | use super::pdf::PdfTextCommand; |
| 12 | use super::spec::{ |
| 13 | ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64, |
| 14 | }; |
| 15 | use super::web::extract::{DocumentKind, ExtractedDocument, decode_response_body}; |
| 16 | use super::web::fetch::{ |
| 17 | DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT, FetchOptions, HARD_MAX_BYTES, HARD_MAX_TIMEOUT, fetch, |
| 18 | }; |
| 19 | use super::web::overflow::bound_text as bound_web_text; |
| 20 | #[cfg(test)] |
| 21 | use super::web::overflow::inline_char_budget; |
| 22 | use async_trait::async_trait; |
| 23 | use serde::Serialize; |
| 24 | use serde_json::{Value, json}; |
| 25 | use std::collections::BTreeMap; |
| 26 | use std::time::Duration; |
| 27 | |
| 28 | const FETCH_ACCEPT: &str = "text/html,text/markdown,text/plain,application/json,application/pdf,image/*,audio/*,video/*,*/*;q=0.5"; |
| 29 | |
| 30 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 31 | enum Format { |
| 32 | Text, |
| 33 | Markdown, |
| 34 | Raw, |
| 35 | } |
| 36 | |
| 37 | impl Format { |
| 38 | fn parse(value: Option<&str>) -> Result<Self, ToolError> { |
| 39 | match value |
| 40 | .unwrap_or("markdown") |
| 41 | .trim() |
| 42 | .to_ascii_lowercase() |
| 43 | .as_str() |
| 44 | { |
| 45 | "text" | "txt" | "plain" => Ok(Self::Text), |
| 46 | "markdown" | "md" => Ok(Self::Markdown), |
| 47 | "raw" | "html" | "bytes" => Ok(Self::Raw), |
| 48 | other => Err(ToolError::invalid_input(format!( |
| 49 | "unknown format `{other}` (allowed: text, markdown, raw)" |
| 50 | ))), |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | #[derive(Debug, Serialize)] |
| 56 | struct FetchResponse { |
| 57 | ref_id: String, |
| 58 | url: String, |
| 59 | status: u16, |
| 60 | headers: BTreeMap<String, String>, |
| 61 | content_type: String, |
| 62 | content: String, |
| 63 | truncated: bool, |
| 64 | receipt: FetchReceipt, |
| 65 | #[serde(skip_serializing_if = "Option::is_none")] |
| 66 | artifact: Option<String>, |
| 67 | #[serde(skip_serializing_if = "Option::is_none")] |
| 68 | fields: Option<BTreeMap<String, Vec<Value>>>, |
| 69 | } |
| 70 | |
| 71 | #[derive(Debug, Serialize)] |
| 72 | struct FetchReceipt { |
| 73 | cache_hit: bool, |
| 74 | retries: usize, |
| 75 | redirects: usize, |
| 76 | } |
| 77 | |
| 78 | #[derive(Debug)] |
| 79 | struct ArtifactWrite { |
| 80 | session_id: String, |
| 81 | absolute_path: std::path::PathBuf, |
| 82 | relative_path: std::path::PathBuf, |
| 83 | byte_size: u64, |
| 84 | preview: String, |
| 85 | } |
| 86 | |
| 87 | pub struct FetchUrlTool; |
| 88 | |
| 89 | #[async_trait] |
| 90 | impl ToolSpec for FetchUrlTool { |
| 91 | fn name(&self) -> &'static str { |
| 92 | "fetch_url" |
| 93 | } |
| 94 | |
| 95 | fn model_visible(&self) -> bool { |
| 96 | false |
| 97 | } |
| 98 | |
| 99 | fn description(&self) -> &'static str { |
| 100 | "Fetch a known URL directly (HTTP GET) and return its content with a session-scoped citation ref_id. Use this instead of `curl` in `exec_shell` — sandboxed, network-policy aware, and properly decoded. Plain-text endpoints (`.md`, `.txt`, `.json`, `.yaml`, `raw.githubusercontent.com`, public APIs) prefer this over the browser/automation stack. For unknown queries, use `web_search` first. If a login or authorization wall is returned, treat the wall as the result; do not claim the protected page was read." |
| 101 | } |
| 102 | |
| 103 | fn input_schema(&self) -> Value { |
| 104 | json!({ |
| 105 | "type": "object", |
| 106 | "properties": { |
| 107 | "url": { |
| 108 | "type": "string", |
| 109 | "description": "Absolute HTTP/HTTPS URL to fetch." |
| 110 | }, |
| 111 | "format": { |
| 112 | "type": "string", |
| 113 | "enum": ["text", "markdown", "raw"], |
| 114 | "description": "Post-processing for the response body. `markdown` (default) uses readability extraction and real HTML-to-Markdown conversion; `text` returns readable plain text; `raw` preserves textual response bytes. Binary media is saved as a session artifact." |
| 115 | }, |
| 116 | "max_bytes": { |
| 117 | "type": "integer", |
| 118 | "description": "Truncate response body after this many bytes (default 1,000,000; hard max 10,485,760)." |
| 119 | }, |
| 120 | "timeout_ms": { |
| 121 | "type": "integer", |
| 122 | "description": "Request timeout in milliseconds (default 15,000; max 60,000)." |
| 123 | }, |
| 124 | "fields": { |
| 125 | "type": "array", |
| 126 | "items": { "type": "string" }, |
| 127 | "description": "Optional JSONPath projections for JSON responses. Supports $, .field, [index], [*], and ['field']; returns matches under `fields`." |
| 128 | } |
| 129 | }, |
| 130 | "required": ["url"] |
| 131 | }) |
| 132 | } |
| 133 | |
| 134 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 135 | vec![ToolCapability::ReadOnly, ToolCapability::Network] |
| 136 | } |
| 137 | |
| 138 | fn approval_requirement(&self) -> ApprovalRequirement { |
| 139 | ApprovalRequirement::Auto |
| 140 | } |
| 141 | |
| 142 | async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> { |
| 143 | let url = input |
| 144 | .get("url") |
| 145 | .and_then(Value::as_str) |
| 146 | .ok_or_else(|| ToolError::invalid_input("`url` is required"))? |
| 147 | .trim() |
| 148 | .to_string(); |
| 149 | |
| 150 | if url.is_empty() { |
| 151 | return Err(ToolError::invalid_input("`url` cannot be empty")); |
| 152 | } |
| 153 | let scheme_ok = url.starts_with("http://") || url.starts_with("https://"); |
| 154 | if !scheme_ok { |
| 155 | return Err(ToolError::invalid_input( |
| 156 | "only http:// and https:// URLs are supported", |
| 157 | )); |
| 158 | } |
| 159 | |
| 160 | let format = Format::parse(input.get("format").and_then(Value::as_str))?; |
| 161 | let max_bytes = |
| 162 | usize::try_from(optional_u64(&input, "max_bytes", DEFAULT_MAX_BYTES as u64)?) |
| 163 | .unwrap_or(HARD_MAX_BYTES) |
| 164 | .clamp(1, HARD_MAX_BYTES); |
| 165 | let timeout_ms = optional_u64(&input, "timeout_ms", DEFAULT_TIMEOUT.as_millis() as u64)? |
| 166 | .clamp(1, HARD_MAX_TIMEOUT.as_millis() as u64); |
| 167 | let requested_fields = parse_fields(&input)?; |
| 168 | let fetched = fetch( |
| 169 | &url, |
| 170 | &FetchOptions::new(Duration::from_millis(timeout_ms), max_bytes, FETCH_ACCEPT), |
| 171 | context, |
| 172 | "fetch_url", |
| 173 | ) |
| 174 | .await?; |
| 175 | let is_success = (200..300).contains(&fetched.status); |
| 176 | let body_text = if requested_fields.is_empty() { |
| 177 | None |
| 178 | } else { |
| 179 | // JSON is never allowed to discover an encoding from body markup. |
| 180 | Some(decode_response_body( |
| 181 | &fetched.bytes, |
| 182 | Some(&fetched.content_type), |
| 183 | false, |
| 184 | )?) |
| 185 | }; |
| 186 | let fields = match body_text.as_deref() { |
| 187 | Some(body) => project_json_fields(body, &fetched.content_type, &requested_fields)?, |
| 188 | None => None, |
| 189 | }; |
| 190 | let extracted = extract_fetched_document( |
| 191 | format, |
| 192 | &fetched.url, |
| 193 | &fetched.content_type, |
| 194 | &fetched.bytes, |
| 195 | is_success, |
| 196 | body_text.as_deref(), |
| 197 | PdfTextCommand::system(context.cancel_token.as_ref()), |
| 198 | ) |
| 199 | .await?; |
| 200 | |
| 201 | let citation_title = extracted.title.clone(); |
| 202 | let (processed, artifact_write) = render_extracted( |
| 203 | &fetched.url, |
| 204 | &fetched.content_type, |
| 205 | format, |
| 206 | extracted, |
| 207 | &fetched.bytes, |
| 208 | context, |
| 209 | )?; |
| 210 | let artifact = artifact_write |
| 211 | .as_ref() |
| 212 | .map(|write| crate::artifacts::format_artifact_relative_path(&write.relative_path)); |
| 213 | |
| 214 | let citation = super::web::citations::register( |
| 215 | &context.state_namespace, |
| 216 | &fetched.url, |
| 217 | citation_title.as_deref(), |
| 218 | ) |
| 219 | .ok_or_else(|| ToolError::execution_failed("fetched URL could not be registered"))?; |
| 220 | let response = FetchResponse { |
| 221 | ref_id: citation.ref_id, |
| 222 | url: citation.url, |
| 223 | status: fetched.status, |
| 224 | headers: fetched.headers, |
| 225 | content_type: fetched.content_type, |
| 226 | content: processed, |
| 227 | truncated: fetched.truncated, |
| 228 | receipt: FetchReceipt { |
| 229 | cache_hit: fetched.cache_hit, |
| 230 | retries: fetched.retries, |
| 231 | redirects: fetched.redirects, |
| 232 | }, |
| 233 | artifact, |
| 234 | fields, |
| 235 | }; |
| 236 | |
| 237 | let content = serde_json::to_string_pretty(&response).map_err(|error| { |
| 238 | ToolError::execution_failed(format!("failed to serialize response: {error}")) |
| 239 | })?; |
| 240 | let metadata = artifact_write.map(artifact_metadata); |
| 241 | |
| 242 | if !is_success { |
| 243 | // Don't `Err` on 4xx/5xx — the caller often wants to see the body |
| 244 | // (e.g. a JSON error envelope). Mark the result as a failure so the |
| 245 | // engine renders it as such. |
| 246 | return Ok(ToolResult { |
| 247 | content, |
| 248 | success: false, |
| 249 | metadata, |
| 250 | }); |
| 251 | } |
| 252 | |
| 253 | Ok(ToolResult { |
| 254 | content, |
| 255 | success: true, |
| 256 | metadata, |
| 257 | }) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | async fn extract_fetched_document( |
| 262 | format: Format, |
| 263 | url: &str, |
| 264 | content_type: &str, |
| 265 | bytes: &[u8], |
| 266 | is_success: bool, |
| 267 | decoded_body: Option<&str>, |
| 268 | pdf_command: PdfTextCommand<'_>, |
| 269 | ) -> Result<ExtractedDocument, ToolError> { |
| 270 | let extraction = if format == Format::Raw |
| 271 | && super::web::extract::validate_pdf_response(url, Some(content_type), bytes)? |
| 272 | { |
| 273 | Ok(ExtractedDocument { |
| 274 | kind: DocumentKind::Pdf, |
| 275 | title: Some("PDF Document".to_string()), |
| 276 | text: String::new(), |
| 277 | markdown: String::new(), |
| 278 | cleaned_html: None, |
| 279 | pdf_pages: None, |
| 280 | media_extension: None, |
| 281 | }) |
| 282 | } else { |
| 283 | super::web::extract::extract_document_with_pdf_command( |
| 284 | url, |
| 285 | Some(content_type), |
| 286 | bytes, |
| 287 | pdf_command, |
| 288 | ) |
| 289 | .await |
| 290 | }; |
| 291 | match extraction { |
| 292 | Ok(document) => Ok(document), |
| 293 | Err(_error) |
| 294 | if (format == Format::Raw || !is_success) && is_declared_textual(content_type) => |
| 295 | { |
| 296 | let body_text = match decoded_body { |
| 297 | Some(body_text) => body_text.to_string(), |
| 298 | None => { |
| 299 | decode_response_body(bytes, Some(content_type), is_declared_html(content_type))? |
| 300 | } |
| 301 | }; |
| 302 | Ok(ExtractedDocument { |
| 303 | kind: DocumentKind::Text, |
| 304 | title: None, |
| 305 | text: body_text.clone(), |
| 306 | markdown: body_text, |
| 307 | cleaned_html: None, |
| 308 | pdf_pages: None, |
| 309 | media_extension: None, |
| 310 | }) |
| 311 | } |
| 312 | Err(error) => Err(error), |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | fn is_declared_textual(content_type: &str) -> bool { |
| 317 | let content_type = content_type |
| 318 | .split(';') |
| 319 | .next() |
| 320 | .unwrap_or(content_type) |
| 321 | .trim() |
| 322 | .to_ascii_lowercase(); |
| 323 | content_type.starts_with("text/") |
| 324 | || content_type.contains("html") |
| 325 | || content_type.contains("json") |
| 326 | || content_type.contains("xml") |
| 327 | || content_type.contains("yaml") |
| 328 | || content_type.contains("javascript") |
| 329 | } |
| 330 | |
| 331 | fn is_declared_html(content_type: &str) -> bool { |
| 332 | matches!( |
| 333 | content_type |
| 334 | .split(';') |
| 335 | .next() |
| 336 | .unwrap_or(content_type) |
| 337 | .trim() |
| 338 | .to_ascii_lowercase() |
| 339 | .as_str(), |
| 340 | "text/html" | "application/xhtml+xml" |
| 341 | ) |
| 342 | } |
| 343 | |
| 344 | fn render_extracted( |
| 345 | url: &str, |
| 346 | content_type: &str, |
| 347 | format: Format, |
| 348 | document: ExtractedDocument, |
| 349 | bytes: &[u8], |
| 350 | context: &ToolContext, |
| 351 | ) -> Result<(String, Option<ArtifactWrite>), ToolError> { |
| 352 | if document.kind == DocumentKind::Pdf && format != Format::Raw { |
| 353 | let extracted = match format { |
| 354 | Format::Text => document.text, |
| 355 | Format::Markdown => document.markdown, |
| 356 | Format::Raw => unreachable!("raw PDF handled below"), |
| 357 | }; |
| 358 | return bound_text(url, extracted, context); |
| 359 | } |
| 360 | |
| 361 | if document.kind == DocumentKind::Media || document.kind == DocumentKind::Pdf { |
| 362 | let extension = document |
| 363 | .media_extension |
| 364 | .unwrap_or(if document.kind == DocumentKind::Pdf { |
| 365 | "pdf" |
| 366 | } else { |
| 367 | "bin" |
| 368 | }); |
| 369 | let artifact = write_binary_artifact(url, extension, bytes, context)?; |
| 370 | let relative = crate::artifacts::format_artifact_relative_path(&artifact.relative_path); |
| 371 | let label = if document.kind == DocumentKind::Pdf { |
| 372 | "PDF" |
| 373 | } else { |
| 374 | "media" |
| 375 | }; |
| 376 | let content = |
| 377 | format!("[{label} response saved to {relative}; content type: {content_type}.]"); |
| 378 | return Ok((content, Some(artifact))); |
| 379 | } |
| 380 | |
| 381 | let content = match format { |
| 382 | Format::Raw => decode_response_body( |
| 383 | bytes, |
| 384 | Some(content_type), |
| 385 | document.kind == DocumentKind::Html, |
| 386 | )?, |
| 387 | Format::Text => document.text, |
| 388 | Format::Markdown => document.markdown, |
| 389 | }; |
| 390 | bound_text(url, content, context) |
| 391 | } |
| 392 | |
| 393 | fn bound_text( |
| 394 | url: &str, |
| 395 | content: String, |
| 396 | context: &ToolContext, |
| 397 | ) -> Result<(String, Option<ArtifactWrite>), ToolError> { |
| 398 | let bounded = bound_web_text( |
| 399 | content, |
| 400 | context, |
| 401 | |body| fetch_artifact_id(url, body.as_bytes()), |
| 402 | "page", |
| 403 | )?; |
| 404 | let artifact = bounded.artifact.map(|artifact| ArtifactWrite { |
| 405 | session_id: artifact.session_id, |
| 406 | absolute_path: artifact.absolute_path, |
| 407 | relative_path: artifact.relative_path, |
| 408 | byte_size: artifact.byte_size, |
| 409 | preview: artifact.preview, |
| 410 | }); |
| 411 | Ok((bounded.content, artifact)) |
| 412 | } |
| 413 | |
| 414 | fn write_binary_artifact( |
| 415 | url: &str, |
| 416 | extension: &str, |
| 417 | bytes: &[u8], |
| 418 | context: &ToolContext, |
| 419 | ) -> Result<ArtifactWrite, ToolError> { |
| 420 | let artifact_id = fetch_artifact_id(url, bytes); |
| 421 | let (absolute_path, relative_path) = crate::artifacts::write_session_artifact_bytes( |
| 422 | &context.state_namespace, |
| 423 | &artifact_id, |
| 424 | extension, |
| 425 | bytes, |
| 426 | ) |
| 427 | .map_err(|error| { |
| 428 | ToolError::execution_failed(format!( |
| 429 | "failed to preserve fetched media artifact: {error}" |
| 430 | )) |
| 431 | })?; |
| 432 | Ok(ArtifactWrite { |
| 433 | session_id: context.state_namespace.clone(), |
| 434 | absolute_path, |
| 435 | relative_path, |
| 436 | byte_size: bytes.len() as u64, |
| 437 | preview: format!("Fetched {extension} artifact from {url}"), |
| 438 | }) |
| 439 | } |
| 440 | |
| 441 | fn fetch_artifact_id(url: &str, bytes: &[u8]) -> String { |
| 442 | let mut identity = Vec::with_capacity(url.len() + bytes.len()); |
| 443 | identity.extend_from_slice(url.as_bytes()); |
| 444 | identity.extend_from_slice(bytes); |
| 445 | let digest = crate::hashing::sha256_hex(&identity); |
| 446 | format!("fetch_{}", &digest[..16]) |
| 447 | } |
| 448 | |
| 449 | fn artifact_metadata(write: ArtifactWrite) -> Value { |
| 450 | json!({ |
| 451 | "spillover_path": write.absolute_path.display().to_string(), |
| 452 | "artifact_session_id": write.session_id, |
| 453 | "artifact_relative_path": crate::artifacts::format_artifact_relative_path(&write.relative_path), |
| 454 | "artifact_byte_size": write.byte_size, |
| 455 | "artifact_preview": write.preview, |
| 456 | }) |
| 457 | } |
| 458 | |
| 459 | fn parse_fields(input: &Value) -> Result<Vec<String>, ToolError> { |
| 460 | let Some(values) = input.get("fields") else { |
| 461 | return Ok(Vec::new()); |
| 462 | }; |
| 463 | let Some(values) = values.as_array() else { |
| 464 | return Err(ToolError::invalid_input("`fields` must be an array")); |
| 465 | }; |
| 466 | let mut fields = Vec::new(); |
| 467 | for value in values { |
| 468 | let Some(field) = value.as_str() else { |
| 469 | return Err(ToolError::invalid_input( |
| 470 | "`fields` entries must be JSONPath strings", |
| 471 | )); |
| 472 | }; |
| 473 | let field = field.trim(); |
| 474 | if !field.is_empty() { |
| 475 | fields.push(field.to_string()); |
| 476 | } |
| 477 | } |
| 478 | Ok(fields) |
| 479 | } |
| 480 | |
| 481 | fn project_json_fields( |
| 482 | body_text: &str, |
| 483 | content_type: &str, |
| 484 | fields: &[String], |
| 485 | ) -> Result<Option<BTreeMap<String, Vec<Value>>>, ToolError> { |
| 486 | if fields.is_empty() { |
| 487 | return Ok(None); |
| 488 | } |
| 489 | if !content_type.to_ascii_lowercase().contains("json") { |
| 490 | return Err(ToolError::invalid_input( |
| 491 | "`fields` can only be used with JSON responses", |
| 492 | )); |
| 493 | } |
| 494 | let body_json: Value = serde_json::from_str(body_text).map_err(|e| { |
| 495 | ToolError::execution_failed(format!("response body is not valid JSON for `fields`: {e}")) |
| 496 | })?; |
| 497 | let mut out = BTreeMap::new(); |
| 498 | for field in fields { |
| 499 | let matches = query_jsonpath(&body_json, field).map_err(|e| { |
| 500 | ToolError::invalid_input(format!("invalid JSONPath `{field}` in `fields`: {e}")) |
| 501 | })?; |
| 502 | out.insert(field.clone(), matches); |
| 503 | } |
| 504 | Ok(Some(out)) |
| 505 | } |
| 506 | |
| 507 | #[cfg(test)] |
| 508 | #[path = "fetch_url/tests.rs"] |
| 509 | mod pdf_tests; |
| 510 | |
| 511 | #[cfg(test)] |
| 512 | mod tests { |
| 513 | use super::*; |
| 514 | use crate::tools::spec::ToolContext; |
| 515 | use std::path::PathBuf; |
| 516 | |
| 517 | struct ArtifactRootRestore(Option<PathBuf>); |
| 518 | |
| 519 | impl Drop for ArtifactRootRestore { |
| 520 | fn drop(&mut self) { |
| 521 | crate::artifacts::set_test_artifact_sessions_root(self.0.take()); |
| 522 | } |
| 523 | } |
| 524 | |
| 525 | fn ctx() -> ToolContext { |
| 526 | ToolContext::new(PathBuf::from(".")) |
| 527 | } |
| 528 | |
| 529 | #[test] |
| 530 | fn format_parse_accepts_aliases_and_rejects_unknown() { |
| 531 | assert_eq!(Format::parse(Some("markdown")).unwrap(), Format::Markdown); |
| 532 | assert_eq!(Format::parse(Some("MD")).unwrap(), Format::Markdown); |
| 533 | assert_eq!(Format::parse(Some("text")).unwrap(), Format::Text); |
| 534 | assert_eq!(Format::parse(Some("raw")).unwrap(), Format::Raw); |
| 535 | assert_eq!(Format::parse(None).unwrap(), Format::Markdown); |
| 536 | assert!(Format::parse(Some("yaml")).is_err()); |
| 537 | } |
| 538 | |
| 539 | #[test] |
| 540 | fn raw_text_uses_declared_charset_and_rejects_nul_binary() { |
| 541 | let document = || ExtractedDocument { |
| 542 | kind: DocumentKind::Text, |
| 543 | title: None, |
| 544 | text: String::new(), |
| 545 | markdown: String::new(), |
| 546 | cleaned_html: None, |
| 547 | pdf_pages: None, |
| 548 | media_extension: None, |
| 549 | }; |
| 550 | let (bytes, _, _) = encoding_rs::WINDOWS_1252.encode("café"); |
| 551 | let (content, artifact) = render_extracted( |
| 552 | "https://example.com/plain", |
| 553 | "text/plain; charset=windows-1252", |
| 554 | Format::Raw, |
| 555 | document(), |
| 556 | &bytes, |
| 557 | &ctx(), |
| 558 | ) |
| 559 | .expect("decode raw text"); |
| 560 | assert_eq!(content, "café"); |
| 561 | assert!(artifact.is_none()); |
| 562 | |
| 563 | let error = render_extracted( |
| 564 | "https://example.com/not-text", |
| 565 | "text/plain", |
| 566 | Format::Raw, |
| 567 | document(), |
| 568 | b"binary\0payload", |
| 569 | &ctx(), |
| 570 | ) |
| 571 | .expect_err("raw text must retain the binary guard"); |
| 572 | assert!(error.to_string().contains("NUL bytes")); |
| 573 | } |
| 574 | |
| 575 | #[test] |
| 576 | fn textual_and_html_fallback_classification_is_exact() { |
| 577 | assert!(is_declared_textual("Application/JSON; charset=utf-8")); |
| 578 | assert!(is_declared_html("TEXT/HTML; charset=gbk")); |
| 579 | assert!(!is_declared_html("text/plain; note=text/html")); |
| 580 | assert!(!is_declared_textual("application/octet-stream")); |
| 581 | } |
| 582 | |
| 583 | #[test] |
| 584 | fn route_budget_overflow_round_trips_through_session_artifact() { |
| 585 | let _lock = crate::artifacts::TEST_ARTIFACT_SESSIONS_GUARD |
| 586 | .lock() |
| 587 | .unwrap_or_else(|error| error.into_inner()); |
| 588 | let tmp = tempfile::tempdir().unwrap(); |
| 589 | let prior = |
| 590 | crate::artifacts::set_test_artifact_sessions_root(Some(tmp.path().join("sessions"))); |
| 591 | let _restore = ArtifactRootRestore(prior); |
| 592 | let context = ToolContext::new(".") |
| 593 | .with_state_namespace("fetch-overflow") |
| 594 | .with_route_context_window(10_000); |
| 595 | let full = "Whale content. ".repeat(200); |
| 596 | |
| 597 | let (inline, artifact) = |
| 598 | bound_text("https://example.com/large", full.clone(), &context).unwrap(); |
| 599 | let artifact = artifact.expect("overflow artifact"); |
| 600 | |
| 601 | assert!(inline.contains("retrieve_tool_result")); |
| 602 | assert!(inline.chars().count() <= inline_char_budget(&context)); |
| 603 | assert_eq!( |
| 604 | std::fs::read_to_string(artifact.absolute_path).unwrap(), |
| 605 | full |
| 606 | ); |
| 607 | } |
| 608 | |
| 609 | #[test] |
| 610 | fn project_json_fields_returns_requested_jsonpath_matches() { |
| 611 | let fields = vec!["$.items[*].name".to_string(), "$.count".to_string()]; |
| 612 | let projected = project_json_fields( |
| 613 | r#"{"items":[{"name":"alpha"},{"name":"beta"}],"count":2}"#, |
| 614 | "application/json", |
| 615 | &fields, |
| 616 | ) |
| 617 | .expect("project") |
| 618 | .expect("some"); |
| 619 | |
| 620 | assert_eq!( |
| 621 | projected.get("$.items[*].name").unwrap(), |
| 622 | &vec![json!("alpha"), json!("beta")] |
| 623 | ); |
| 624 | assert_eq!(projected.get("$.count").unwrap(), &vec![json!(2)]); |
| 625 | } |
| 626 | |
| 627 | #[test] |
| 628 | fn project_json_fields_rejects_non_json_content_type() { |
| 629 | let fields = vec!["$.name".to_string()]; |
| 630 | let err = project_json_fields("{}", "text/plain", &fields).expect_err("must reject"); |
| 631 | assert!(format!("{err}").contains("JSON responses")); |
| 632 | } |
| 633 | |
| 634 | #[tokio::test] |
| 635 | async fn rejects_non_http_schemes() { |
| 636 | let tool = FetchUrlTool; |
| 637 | let res = tool |
| 638 | .execute(json!({"url": "file:///etc/passwd"}), &ctx()) |
| 639 | .await; |
| 640 | let err = res.unwrap_err(); |
| 641 | assert!(format!("{err:?}").contains("http")); |
| 642 | } |
| 643 | |
| 644 | #[tokio::test] |
| 645 | async fn rejects_empty_url() { |
| 646 | let tool = FetchUrlTool; |
| 647 | let res = tool.execute(json!({"url": " "}), &ctx()).await; |
| 648 | assert!(res.is_err()); |
| 649 | } |
| 650 | |
| 651 | #[tokio::test] |
| 652 | async fn rejects_missing_url() { |
| 653 | let tool = FetchUrlTool; |
| 654 | let res = tool.execute(json!({}), &ctx()).await; |
| 655 | assert!(res.is_err()); |
| 656 | } |
| 657 | |
| 658 | #[tokio::test] |
| 659 | async fn rejects_localhost_hostname() { |
| 660 | let tool = FetchUrlTool; |
| 661 | let res = tool |
| 662 | .execute(json!({"url": "http://localhost:8080/admin"}), &ctx()) |
| 663 | .await; |
| 664 | let err = res.unwrap_err(); |
| 665 | assert!(format!("{err}").contains("localhost")); |
| 666 | } |
| 667 | |
| 668 | #[tokio::test] |
| 669 | async fn network_policy_denies_blocked_host() { |
| 670 | use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider}; |
| 671 | let policy = NetworkPolicy { |
| 672 | default: Decision::Deny.into(), |
| 673 | allow: vec!["api.deepseek.com".to_string()], |
| 674 | deny: vec![], |
| 675 | proxy: Vec::new(), |
| 676 | proxy_fake_ip_cidrs: Vec::new(), |
| 677 | audit: false, |
| 678 | }; |
| 679 | let decider = NetworkPolicyDecider::new(policy, None); |
| 680 | let ctx = ToolContext::new(PathBuf::from(".")).with_network_policy(decider); |
| 681 | let tool = FetchUrlTool; |
| 682 | let res = tool |
| 683 | .execute(json!({"url": "https://example.com/foo"}), &ctx) |
| 684 | .await; |
| 685 | let err = res.expect_err("blocked host should fail"); |
| 686 | assert!(format!("{err}").contains("blocked")); |
| 687 | } |
| 688 | |
| 689 | #[tokio::test] |
| 690 | async fn proxy_opt_in_does_not_allow_restricted_ip_literal() { |
| 691 | use crate::network_policy::{Decision, NetworkPolicy, NetworkPolicyDecider}; |
| 692 | |
| 693 | let policy = NetworkPolicy { |
| 694 | default: Decision::Allow.into(), |
| 695 | allow: Vec::new(), |
| 696 | deny: Vec::new(), |
| 697 | proxy: vec!["198.18.0.1".to_string()], |
| 698 | proxy_fake_ip_cidrs: vec!["198.18.0.0/15".to_string()], |
| 699 | audit: false, |
| 700 | }; |
| 701 | let decider = NetworkPolicyDecider::new(policy, None); |
| 702 | let ctx = ToolContext::new(PathBuf::from(".")).with_network_policy(decider); |
| 703 | let tool = FetchUrlTool; |
| 704 | |
| 705 | let err = tool |
| 706 | .execute(json!({"url": "http://198.18.0.1/status"}), &ctx) |
| 707 | .await |
| 708 | .expect_err("literal restricted IP URLs must stay blocked"); |
| 709 | |
| 710 | assert!(format!("{err}").contains("IP 198.18.0.1 is a restricted address")); |
| 711 | } |
| 712 | } |
| 713 |