| 1 | //! Legacy parser for text-based tool calls from DeepSeek models. |
| 2 | //! |
| 3 | //! Structured tool-call items are preferred, so the engine no longer invokes |
| 4 | //! this parser. It is kept for reference/debugging. |
| 5 | //! |
| 6 | //! Some DeepSeek outputs tool calls as text in various formats: |
| 7 | //! ```text |
| 8 | //! [TOOL_CALL] |
| 9 | //! {tool => "tool_name", args => {...}} |
| 10 | //! [/TOOL_CALL] |
| 11 | //! ``` |
| 12 | //! |
| 13 | //! Or XML-style format: |
| 14 | //! ```text |
| 15 | //! <codewhale:tool_call> |
| 16 | //! <invoke name="tool_name"> |
| 17 | //! <parameter name="arg">value</parameter> |
| 18 | //! </invoke> |
| 19 | //! </codewhale:tool_call> |
| 20 | //! ``` |
| 21 | //! |
| 22 | //! This module parses these text patterns into structured tool calls. |
| 23 | |
| 24 | use regex::Regex; |
| 25 | use serde_json::{Value, json}; |
| 26 | use std::sync::OnceLock; |
| 27 | |
| 28 | /// A parsed tool call from text content. |
| 29 | #[derive(Debug, Clone)] |
| 30 | pub struct ParsedToolCall { |
| 31 | /// Tool name |
| 32 | pub name: String, |
| 33 | /// Tool arguments as JSON |
| 34 | pub args: Value, |
| 35 | /// Generated ID for the tool call |
| 36 | pub id: String, |
| 37 | } |
| 38 | |
| 39 | /// Result of parsing text for tool calls. |
| 40 | #[derive(Debug)] |
| 41 | pub struct ParseResult { |
| 42 | /// The text with tool call markers removed (for display) |
| 43 | pub clean_text: String, |
| 44 | /// Parsed tool calls found in the text |
| 45 | pub tool_calls: Vec<ParsedToolCall>, |
| 46 | } |
| 47 | |
| 48 | static TOOL_CALL_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 49 | static XML_TOOL_CALL_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 50 | static INVOKE_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 51 | static THINKING_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 52 | static FAKE_TOOL_WRAPPER_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 53 | |
| 54 | const FAKE_TOOL_CALL_MARKERS: &[&str] = &[ |
| 55 | "<function_calls>", |
| 56 | "<｜DSML｜tool_calls>", |
| 57 | "<｜DSML｜invoke ", |
| 58 | "<|DSML|tool_calls>", |
| 59 | "<|DSML|invoke ", |
| 60 | "<|dsml|tool_calls>", |
| 61 | "<|dsml|invoke ", |
| 62 | "<|tool_calls>", |
| 63 | // DeepSeek native tool-call tokens (#3880). See |
| 64 | // `engine::streaming::TOOL_CALL_MARKER_PAIRS` for why the `▁` (U+2581) |
| 65 | // separator matters: these match no DSML entry, so they used to reach the |
| 66 | // user as visible text. |
| 67 | "<｜tool▁calls▁begin｜>", |
| 68 | "<｜tool▁call▁begin｜>", |
| 69 | "<|tool▁calls▁begin|>", |
| 70 | "<|tool▁call▁begin|>", |
| 71 | "<｜tool_calls_begin｜>", |
| 72 | "<｜tool_call_begin｜>", |
| 73 | "<|tool_calls_begin|>", |
| 74 | "<|tool_call_begin|>", |
| 75 | ]; |
| 76 | |
| 77 | /// Tool-call wrapper pairs whose start and end markers are plain literals, so |
| 78 | /// their regex alternative is built by escaping rather than hand-written. The |
| 79 | /// DSML entries stay hand-written above because they carry attributes |
| 80 | /// (`invoke name="…"`) and need `\b[^>]*>` rather than a literal match. |
| 81 | const LITERAL_FAKE_WRAPPER_PAIRS: &[(&str, &str)] = &[ |
| 82 | ("<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>"), |
| 83 | ("<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>"), |
| 84 | ("<｜tool▁outputs▁begin｜>", "<｜tool▁outputs▁end｜>"), |
| 85 | ("<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>"), |
| 86 | ("<|tool▁calls▁begin|>", "<|tool▁calls▁end|>"), |
| 87 | ("<|tool▁call▁begin|>", "<|tool▁call▁end|>"), |
| 88 | ("<|tool▁outputs▁begin|>", "<|tool▁outputs▁end|>"), |
| 89 | ("<|tool▁output▁begin|>", "<|tool▁output▁end|>"), |
| 90 | ("<｜tool_calls_begin｜>", "<｜tool_calls_end｜>"), |
| 91 | ("<｜tool_call_begin｜>", "<｜tool_call_end｜>"), |
| 92 | ("<｜tool_outputs_begin｜>", "<｜tool_outputs_end｜>"), |
| 93 | ("<｜tool_output_begin｜>", "<｜tool_output_end｜>"), |
| 94 | ("<|tool_calls_begin|>", "<|tool_calls_end|>"), |
| 95 | ("<|tool_call_begin|>", "<|tool_call_end|>"), |
| 96 | ("<|tool_outputs_begin|>", "<|tool_outputs_end|>"), |
| 97 | ("<|tool_output_begin|>", "<|tool_output_end|>"), |
| 98 | ]; |
| 99 | |
| 100 | fn get_tool_call_regex() -> &'static Regex { |
| 101 | TOOL_CALL_REGEX.get_or_init(|| { |
| 102 | // Match [TOOL_CALL] ... [/TOOL_CALL] blocks |
| 103 | Regex::new(r"(?s)\[TOOL_CALL\]\s*(.*?)\s*\[/TOOL_CALL\]") |
| 104 | .expect("TOOL_CALL regex pattern is valid") |
| 105 | }) |
| 106 | } |
| 107 | |
| 108 | fn get_xml_tool_call_regex() -> &'static Regex { |
| 109 | XML_TOOL_CALL_REGEX.get_or_init(|| { |
| 110 | // Match <codewhale:tool_call>...</codewhale:tool_call> or similar XML patterns |
| 111 | Regex::new(r"(?s)<(?:codewhale:)?tool_call[^>]*>\s*(.*?)\s*</(?:codewhale:)?tool_call>") |
| 112 | .expect("XML tool_call regex pattern is valid") |
| 113 | }) |
| 114 | } |
| 115 | |
| 116 | fn get_invoke_regex() -> &'static Regex { |
| 117 | INVOKE_REGEX.get_or_init(|| { |
| 118 | // Match <invoke name="tool_name">...</invoke> patterns |
| 119 | Regex::new(r#"(?s)<invoke\s+name\s*=\s*"([^"]+)"[^>]*>(.*?)</invoke>"#) |
| 120 | .expect("invoke regex pattern is valid") |
| 121 | }) |
| 122 | } |
| 123 | |
| 124 | fn get_thinking_regex() -> &'static Regex { |
| 125 | THINKING_REGEX.get_or_init(|| { |
| 126 | // Match thinking blocks including partial closing tags |
| 127 | Regex::new(r"(?s)</?(?:think|thinking)[^>]*>").expect("thinking regex pattern is valid") |
| 128 | }) |
| 129 | } |
| 130 | |
| 131 | fn get_fake_tool_wrapper_regex() -> &'static Regex { |
| 132 | FAKE_TOOL_WRAPPER_REGEX.get_or_init(|| { |
| 133 | let mut alternatives = vec![ |
| 134 | r"<function_calls>.*?</function_calls>".to_string(), |
| 135 | r"<｜DSML｜tool_calls>.*?</｜DSML｜tool_calls>".to_string(), |
| 136 | r"<｜DSML｜invoke\b[^>]*>.*?</｜DSML｜invoke>".to_string(), |
| 137 | r"<\|DSML\|tool_calls>.*?</\|DSML\|tool_calls>".to_string(), |
| 138 | r"<\|DSML\|invoke\b[^>]*>.*?</\|DSML\|invoke>".to_string(), |
| 139 | r"<\|dsml\|tool_calls>.*?</\|dsml\|tool_calls>".to_string(), |
| 140 | r"<\|dsml\|invoke\b[^>]*>.*?</\|dsml\|invoke>".to_string(), |
| 141 | r"<\|tool_calls>.*?</\|tool_calls>".to_string(), |
| 142 | ]; |
| 143 | alternatives.extend( |
| 144 | LITERAL_FAKE_WRAPPER_PAIRS |
| 145 | .iter() |
| 146 | .map(|(start, end)| format!("{}.*?{}", regex::escape(start), regex::escape(end))), |
| 147 | ); |
| 148 | Regex::new(&format!("(?s){}", alternatives.join("|"))) |
| 149 | .expect("fake tool wrapper regex pattern is valid") |
| 150 | }) |
| 151 | } |
| 152 | |
| 153 | /// Parse tool calls from text content. |
| 154 | /// Returns the clean text (with markers removed) and any parsed tool calls. |
| 155 | pub fn parse_tool_calls(text: &str) -> ParseResult { |
| 156 | let mut tool_calls = Vec::new(); |
| 157 | let mut clean_text = text.to_string(); |
| 158 | let mut id_counter = 0; |
| 159 | |
| 160 | // First, remove thinking tags |
| 161 | let thinking_regex = get_thinking_regex(); |
| 162 | clean_text = thinking_regex.replace_all(&clean_text, "").to_string(); |
| 163 | |
| 164 | // Parse [TOOL_CALL] format |
| 165 | let regex = get_tool_call_regex(); |
| 166 | for cap in regex.captures_iter(text) { |
| 167 | let (Some(full_match), Some(inner)) = (cap.get(0), cap.get(1)) else { |
| 168 | continue; |
| 169 | }; |
| 170 | let full_match = full_match.as_str(); |
| 171 | let inner = inner.as_str().trim(); |
| 172 | |
| 173 | if let Some(parsed) = parse_tool_call_inner(inner, &mut id_counter) { |
| 174 | tool_calls.push(parsed); |
| 175 | } |
| 176 | |
| 177 | clean_text = clean_text.replace(full_match, ""); |
| 178 | } |
| 179 | |
| 180 | // Parse XML-style <codewhale:tool_call> or <tool_call> format |
| 181 | let xml_regex = get_xml_tool_call_regex(); |
| 182 | for cap in xml_regex.captures_iter(text) { |
| 183 | let (Some(full_match), Some(inner)) = (cap.get(0), cap.get(1)) else { |
| 184 | continue; |
| 185 | }; |
| 186 | let full_match = full_match.as_str(); |
| 187 | let inner = inner.as_str().trim(); |
| 188 | |
| 189 | // Parse invoke blocks inside |
| 190 | if let Some(parsed) = parse_invoke_block(inner, &mut id_counter) { |
| 191 | tool_calls.push(parsed); |
| 192 | } else if let Some(parsed) = parse_tool_call_inner(inner, &mut id_counter) { |
| 193 | tool_calls.push(parsed); |
| 194 | } |
| 195 | |
| 196 | clean_text = clean_text.replace(full_match, ""); |
| 197 | } |
| 198 | |
| 199 | // Also parse standalone <invoke> blocks that might not be wrapped |
| 200 | let invoke_regex = get_invoke_regex(); |
| 201 | for cap in invoke_regex.captures_iter(&clean_text.clone()) { |
| 202 | let (Some(full_match), Some(tool_name), Some(inner)) = (cap.get(0), cap.get(1), cap.get(2)) |
| 203 | else { |
| 204 | continue; |
| 205 | }; |
| 206 | let full_match = full_match.as_str(); |
| 207 | let tool_name = tool_name.as_str(); |
| 208 | let inner = inner.as_str(); |
| 209 | |
| 210 | let args = parse_xml_parameters(inner); |
| 211 | id_counter += 1; |
| 212 | tool_calls.push(ParsedToolCall { |
| 213 | name: tool_name.to_string(), |
| 214 | args, |
| 215 | id: format!("xml_tool_{id_counter}"), |
| 216 | }); |
| 217 | |
| 218 | clean_text = clean_text.replace(full_match, ""); |
| 219 | } |
| 220 | |
| 221 | clean_text = get_fake_tool_wrapper_regex() |
| 222 | .replace_all(&clean_text, "") |
| 223 | .to_string(); |
| 224 | |
| 225 | // Clean up extra whitespace and empty lines |
| 226 | clean_text = clean_text |
| 227 | .lines() |
| 228 | .filter(|line| !line.trim().is_empty()) |
| 229 | .collect::<Vec<_>>() |
| 230 | .join("\n") |
| 231 | .trim() |
| 232 | .to_string(); |
| 233 | |
| 234 | ParseResult { |
| 235 | clean_text, |
| 236 | tool_calls, |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | /// Parse an `<invoke>` block into a tool call. |
| 241 | fn parse_invoke_block(content: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 242 | let invoke_regex = get_invoke_regex(); |
| 243 | let cap = invoke_regex.captures(content)?; |
| 244 | |
| 245 | let tool_name = cap.get(1)?.as_str(); |
| 246 | let inner = cap.get(2)?.as_str(); |
| 247 | |
| 248 | let args = parse_xml_parameters(inner); |
| 249 | |
| 250 | *id_counter += 1; |
| 251 | Some(ParsedToolCall { |
| 252 | name: tool_name.to_string(), |
| 253 | args, |
| 254 | id: format!("xml_tool_{id_counter}"), |
| 255 | }) |
| 256 | } |
| 257 | |
| 258 | /// Parse XML-style parameters like <parameter name="foo">value</parameter> |
| 259 | fn parse_xml_parameters(content: &str) -> Value { |
| 260 | let param_regex = Regex::new( |
| 261 | "<(?:parameter|param)\\s+name\\s*=\\s*\"([^\"]+)\"[^>]*>(.*?)</(?:parameter|param)>", |
| 262 | ) |
| 263 | .ok(); |
| 264 | let simple_tag_regex = |
| 265 | Regex::new("<([a-zA-Z_][a-zA-Z0-9_]*)>(.*?)</([a-zA-Z_][a-zA-Z0-9_]*)>").ok(); |
| 266 | |
| 267 | let mut map = serde_json::Map::new(); |
| 268 | |
| 269 | // Try parsing <parameter name="...">value</parameter> |
| 270 | if let Some(regex) = param_regex { |
| 271 | for cap in regex.captures_iter(content) { |
| 272 | if let (Some(name), Some(value)) = (cap.get(1), cap.get(2)) { |
| 273 | let name_str = name.as_str(); |
| 274 | let value_str = value.as_str().trim(); |
| 275 | |
| 276 | // Try to parse as JSON, otherwise use as string |
| 277 | let json_value = serde_json::from_str(value_str) |
| 278 | .unwrap_or_else(|_| Value::String(value_str.to_string())); |
| 279 | map.insert(name_str.to_string(), json_value); |
| 280 | } |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | // Also try parsing <tagname>value</tagname> format |
| 285 | if let Some(regex) = simple_tag_regex { |
| 286 | for cap in regex.captures_iter(content) { |
| 287 | if let (Some(name), Some(value), Some(close)) = (cap.get(1), cap.get(2), cap.get(3)) { |
| 288 | if name.as_str() != close.as_str() { |
| 289 | continue; |
| 290 | } |
| 291 | let name_str = name.as_str(); |
| 292 | // Skip known wrapper tags |
| 293 | if ["invoke", "tool_call", "parameter", "param"].contains(&name_str) { |
| 294 | continue; |
| 295 | } |
| 296 | let value_str = value.as_str().trim(); |
| 297 | if !map.contains_key(name_str) { |
| 298 | let json_value = serde_json::from_str(value_str) |
| 299 | .unwrap_or_else(|_| Value::String(value_str.to_string())); |
| 300 | map.insert(name_str.to_string(), json_value); |
| 301 | } |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | Value::Object(map) |
| 307 | } |
| 308 | |
| 309 | /// Parse the inner content of a `TOOL_CALL` block. |
| 310 | fn parse_tool_call_inner(inner: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 311 | // Try to parse as JSON first |
| 312 | if let Ok(json) = serde_json::from_str::<Value>(inner) { |
| 313 | return parse_from_json(&json, id_counter); |
| 314 | } |
| 315 | |
| 316 | // Try the arrow syntax: {tool => "name", args => {...}} |
| 317 | if let Some(parsed) = parse_arrow_syntax(inner, id_counter) { |
| 318 | return Some(parsed); |
| 319 | } |
| 320 | |
| 321 | // Try to extract tool name and args from any format |
| 322 | parse_flexible_format(inner, id_counter) |
| 323 | } |
| 324 | |
| 325 | /// Parse from JSON object. |
| 326 | fn parse_from_json(json: &Value, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 327 | let obj = json.as_object()?; |
| 328 | |
| 329 | // Try different field names for the tool name |
| 330 | let name = obj |
| 331 | .get("tool") |
| 332 | .or_else(|| obj.get("name")) |
| 333 | .or_else(|| obj.get("function")) |
| 334 | .and_then(|v| v.as_str())? |
| 335 | .to_string(); |
| 336 | |
| 337 | // Try different field names for the arguments |
| 338 | let args = obj |
| 339 | .get("args") |
| 340 | .or_else(|| obj.get("arguments")) |
| 341 | .or_else(|| obj.get("input")) |
| 342 | .or_else(|| obj.get("parameters")) |
| 343 | .cloned() |
| 344 | .unwrap_or(json!({})); |
| 345 | |
| 346 | *id_counter += 1; |
| 347 | Some(ParsedToolCall { |
| 348 | name, |
| 349 | args, |
| 350 | id: format!("text_tool_{id_counter}"), |
| 351 | }) |
| 352 | } |
| 353 | |
| 354 | /// Parse the arrow syntax: {tool => "name", args => {...}} |
| 355 | fn parse_arrow_syntax(inner: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 356 | // Extract tool name |
| 357 | let tool_regex = Regex::new(r#"tool\s*=>\s*"([^"]+)""#).ok()?; |
| 358 | let name = tool_regex.captures(inner)?.get(1)?.as_str().to_string(); |
| 359 | |
| 360 | // Extract args - try to find the JSON object after "args =>" |
| 361 | let args = if let Some(args_start) = inner.find("args =>") { |
| 362 | let args_str = inner[args_start + 7..].trim(); |
| 363 | // Try to parse as JSON first |
| 364 | if let Ok(args_json) = serde_json::from_str::<Value>(args_str) { |
| 365 | args_json |
| 366 | } else if let Some(brace_start) = args_str.find('{') { |
| 367 | // Try to extract the content between braces |
| 368 | let mut brace_count = 0; |
| 369 | let mut end_idx = brace_start; |
| 370 | for (i, c) in args_str[brace_start..].chars().enumerate() { |
| 371 | match c { |
| 372 | '{' => brace_count += 1, |
| 373 | '}' => { |
| 374 | brace_count -= 1; |
| 375 | if brace_count == 0 { |
| 376 | end_idx = brace_start + i + 1; |
| 377 | break; |
| 378 | } |
| 379 | } |
| 380 | _ => {} |
| 381 | } |
| 382 | } |
| 383 | let content = &args_str[brace_start + 1..end_idx - 1]; |
| 384 | |
| 385 | // Try to parse as JSON |
| 386 | if let Ok(json) = serde_json::from_str::<Value>(&format!("{{{content}}}")) { |
| 387 | json |
| 388 | } else { |
| 389 | // Try CLI-style args: --arg_name "value" or --arg_name value |
| 390 | parse_cli_style_args(content) |
| 391 | } |
| 392 | } else { |
| 393 | json!({}) |
| 394 | } |
| 395 | } else { |
| 396 | json!({}) |
| 397 | }; |
| 398 | |
| 399 | *id_counter += 1; |
| 400 | Some(ParsedToolCall { |
| 401 | name, |
| 402 | args, |
| 403 | id: format!("text_tool_{id_counter}"), |
| 404 | }) |
| 405 | } |
| 406 | |
| 407 | /// Parse CLI-style arguments: --`arg_name` "value" or --`arg_name` value |
| 408 | fn parse_cli_style_args(content: &str) -> Value { |
| 409 | let mut map = serde_json::Map::new(); |
| 410 | |
| 411 | // Pattern: --arg_name "value" or --arg_name 'value' or --arg_name value |
| 412 | let arg_regex = |
| 413 | Regex::new(r#"--([a-zA-Z_][a-zA-Z0-9_]*)\s+(?:"([^"]*)"|'([^']*)'|(\S+))"#).ok(); |
| 414 | |
| 415 | if let Some(regex) = arg_regex { |
| 416 | for cap in regex.captures_iter(content) { |
| 417 | if let Some(arg_name) = cap.get(1) { |
| 418 | let arg_name = arg_name.as_str(); |
| 419 | // Get the value from whichever capture group matched |
| 420 | let value = cap |
| 421 | .get(2) |
| 422 | .or_else(|| cap.get(3)) |
| 423 | .or_else(|| cap.get(4)) |
| 424 | .map_or("", |m| m.as_str()); |
| 425 | |
| 426 | // Try to parse as JSON value, otherwise use as string |
| 427 | let json_value = serde_json::from_str(value) |
| 428 | .unwrap_or_else(|_| Value::String(value.to_string())); |
| 429 | map.insert(arg_name.to_string(), json_value); |
| 430 | } |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | // Also try simple key=value format |
| 435 | let kv_regex = |
| 436 | Regex::new(r#"([a-zA-Z_][a-zA-Z0-9_]*)\s*[:=]\s*(?:"([^"]*)"|'([^']*)'|(\S+))"#).ok(); |
| 437 | if let Some(regex) = kv_regex { |
| 438 | for cap in regex.captures_iter(content) { |
| 439 | if let Some(key) = cap.get(1) { |
| 440 | let key = key.as_str(); |
| 441 | if !map.contains_key(key) { |
| 442 | let value = cap |
| 443 | .get(2) |
| 444 | .or_else(|| cap.get(3)) |
| 445 | .or_else(|| cap.get(4)) |
| 446 | .map_or("", |m| m.as_str()); |
| 447 | let json_value = serde_json::from_str(value) |
| 448 | .unwrap_or_else(|_| Value::String(value.to_string())); |
| 449 | map.insert(key.to_string(), json_value); |
| 450 | } |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | Value::Object(map) |
| 456 | } |
| 457 | |
| 458 | /// Try to parse a flexible format. |
| 459 | fn parse_flexible_format(inner: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 460 | // Look for common patterns like: |
| 461 | // tool: list_dir |
| 462 | // name: "list_dir" |
| 463 | // function: list_dir |
| 464 | |
| 465 | let patterns = [( |
| 466 | r#"(?:tool|name|function)\s*[:=]\s*"?([a-zA-Z_][a-zA-Z0-9_]*)"?"#, |
| 467 | 1, |
| 468 | )]; |
| 469 | |
| 470 | for (pattern, group) in patterns { |
| 471 | if let Ok(regex) = Regex::new(pattern) |
| 472 | && let Some(cap) = regex.captures(inner) |
| 473 | && let Some(name_match) = cap.get(group) |
| 474 | { |
| 475 | let name = name_match.as_str().to_string(); |
| 476 | |
| 477 | // Try to extract args/input as JSON |
| 478 | let args = extract_json_object(inner).unwrap_or(json!({})); |
| 479 | |
| 480 | *id_counter += 1; |
| 481 | return Some(ParsedToolCall { |
| 482 | name, |
| 483 | args, |
| 484 | id: format!("text_tool_{id_counter}"), |
| 485 | }); |
| 486 | } |
| 487 | } |
| 488 | |
| 489 | None |
| 490 | } |
| 491 | |
| 492 | /// Extract the first JSON object from a string. |
| 493 | fn extract_json_object(text: &str) -> Option<Value> { |
| 494 | let start = text.find('{')?; |
| 495 | let mut brace_count = 0; |
| 496 | let mut end_idx = start; |
| 497 | |
| 498 | for (i, c) in text[start..].chars().enumerate() { |
| 499 | match c { |
| 500 | '{' => brace_count += 1, |
| 501 | '}' => { |
| 502 | brace_count -= 1; |
| 503 | if brace_count == 0 { |
| 504 | end_idx = start + i + 1; |
| 505 | break; |
| 506 | } |
| 507 | } |
| 508 | _ => {} |
| 509 | } |
| 510 | } |
| 511 | |
| 512 | let json_str = &text[start..end_idx]; |
| 513 | serde_json::from_str(json_str).ok() |
| 514 | } |
| 515 | |
| 516 | /// Check if text contains tool call markers (either format). |
| 517 | pub fn has_tool_call_markers(text: &str) -> bool { |
| 518 | text.contains("[TOOL_CALL]") |
| 519 | || text.contains("<codewhale:tool_call") |
| 520 | || text.contains("<tool_call") |
| 521 | || text.contains("<invoke ") |
| 522 | || FAKE_TOOL_CALL_MARKERS |
| 523 | .iter() |
| 524 | .any(|marker| text.contains(marker)) |
| 525 | } |
| 526 | |
| 527 | #[cfg(test)] |
| 528 | mod tests { |
| 529 | use super::*; |
| 530 | |
| 531 | #[test] |
| 532 | fn test_parse_arrow_syntax() { |
| 533 | let text = r#"I'll list the directory. |
| 534 | [TOOL_CALL] |
| 535 | {tool => "list_dir", args => {}} |
| 536 | [/TOOL_CALL]"#; |
| 537 | |
| 538 | let result = parse_tool_calls(text); |
| 539 | assert_eq!(result.tool_calls.len(), 1); |
| 540 | assert_eq!(result.tool_calls[0].name, "list_dir"); |
| 541 | assert_eq!(result.clean_text, "I'll list the directory."); |
| 542 | } |
| 543 | |
| 544 | #[test] |
| 545 | fn test_parse_json_syntax() { |
| 546 | let text = r#"Let me check. |
| 547 | [TOOL_CALL] |
| 548 | {"tool": "read_file", "args": {"path": "test.txt"}} |
| 549 | [/TOOL_CALL]"#; |
| 550 | |
| 551 | let result = parse_tool_calls(text); |
| 552 | assert_eq!(result.tool_calls.len(), 1); |
| 553 | assert_eq!(result.tool_calls[0].name, "read_file"); |
| 554 | assert_eq!(result.tool_calls[0].args["path"], "test.txt"); |
| 555 | } |
| 556 | |
| 557 | #[test] |
| 558 | fn test_parse_multiple_tool_calls() { |
| 559 | let text = r#"First I'll list, then read. |
| 560 | [TOOL_CALL] |
| 561 | {tool => "list_dir", args => {}} |
| 562 | [/TOOL_CALL] |
| 563 | [TOOL_CALL] |
| 564 | {tool => "read_file", args => {"path": "file.txt"}} |
| 565 | [/TOOL_CALL]"#; |
| 566 | |
| 567 | let result = parse_tool_calls(text); |
| 568 | assert_eq!(result.tool_calls.len(), 2); |
| 569 | assert_eq!(result.tool_calls[0].name, "list_dir"); |
| 570 | assert_eq!(result.tool_calls[1].name, "read_file"); |
| 571 | } |
| 572 | |
| 573 | #[test] |
| 574 | fn test_no_tool_calls() { |
| 575 | let text = "Just some regular text without any tool calls."; |
| 576 | let result = parse_tool_calls(text); |
| 577 | assert!(result.tool_calls.is_empty()); |
| 578 | assert_eq!(result.clean_text, text); |
| 579 | } |
| 580 | |
| 581 | #[test] |
| 582 | fn test_dsml_wrappers_are_stripped_without_execution() { |
| 583 | let text = "before\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"read_file\">\n<｜DSML｜parameter name=\"path\" string=\"true\">secret.txt</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\nafter"; |
| 584 | |
| 585 | assert!(has_tool_call_markers(text)); |
| 586 | let result = parse_tool_calls(text); |
| 587 | |
| 588 | assert!(result.tool_calls.is_empty()); |
| 589 | assert!(result.clean_text.contains("before")); |
| 590 | assert!(result.clean_text.contains("after")); |
| 591 | assert!(!result.clean_text.contains("DSML")); |
| 592 | assert!(!result.clean_text.contains("read_file")); |
| 593 | assert!(!result.clean_text.contains("secret.txt")); |
| 594 | } |
| 595 | |
| 596 | #[test] |
| 597 | fn test_ascii_dsml_wrappers_are_stripped_without_execution() { |
| 598 | let text = "before <|DSML|invoke name=\"grep_files\"><|DSML|parameter name=\"pattern\">SECRET</|DSML|parameter></|DSML|invoke> after"; |
| 599 | |
| 600 | assert!(has_tool_call_markers(text)); |
| 601 | let result = parse_tool_calls(text); |
| 602 | |
| 603 | assert!(result.tool_calls.is_empty()); |
| 604 | assert!(result.clean_text.contains("before")); |
| 605 | assert!(result.clean_text.contains("after")); |
| 606 | assert!(!result.clean_text.contains("DSML")); |
| 607 | assert!(!result.clean_text.contains("grep_files")); |
| 608 | assert!(!result.clean_text.contains("SECRET")); |
| 609 | } |
| 610 | |
| 611 | #[test] |
| 612 | fn test_deepseek_native_tool_tokens_are_stripped_without_execution() { |
| 613 | // #3880: DeepSeek's own tool-call tokens use `▁` (U+2581) as the word |
| 614 | // separator, so they matched none of the DSML shapes and survived into |
| 615 | // the text shown to the user. |
| 616 | for (start, end) in [ |
| 617 | ("<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>"), |
| 618 | ("<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>"), |
| 619 | ("<|tool▁calls▁begin|>", "<|tool▁calls▁end|>"), |
| 620 | ("<｜tool_calls_begin｜>", "<｜tool_calls_end｜>"), |
| 621 | ("<|tool_call_begin|>", "<|tool_call_end|>"), |
| 622 | ] { |
| 623 | let text = format!( |
| 624 | "before {start}function<｜tool▁sep｜>grep_files\n```json\n{{\"pattern\":\"SECRET\"}}\n```{end} after" |
| 625 | ); |
| 626 | |
| 627 | assert!(has_tool_call_markers(&text), "not detected: {start}"); |
| 628 | let result = parse_tool_calls(&text); |
| 629 | |
| 630 | // The wrapper is scrubbed, never executed: a model forging a tool |
| 631 | // call in plain text must not become a real invocation. |
| 632 | assert!(result.tool_calls.is_empty(), "{start} was executed"); |
| 633 | assert!( |
| 634 | result.clean_text.contains("before"), |
| 635 | "{:?}", |
| 636 | result.clean_text |
| 637 | ); |
| 638 | assert!( |
| 639 | result.clean_text.contains("after"), |
| 640 | "{:?}", |
| 641 | result.clean_text |
| 642 | ); |
| 643 | assert!( |
| 644 | !result.clean_text.contains("grep_files") |
| 645 | && !result.clean_text.contains("SECRET") |
| 646 | && !result.clean_text.contains("tool▁") |
| 647 | && !result.clean_text.contains("tool_calls"), |
| 648 | "leaked {start}: {:?}", |
| 649 | result.clean_text |
| 650 | ); |
| 651 | } |
| 652 | } |
| 653 | |
| 654 | #[test] |
| 655 | fn test_has_markers() { |
| 656 | assert!(has_tool_call_markers("[TOOL_CALL]test[/TOOL_CALL]")); |
| 657 | assert!(has_tool_call_markers( |
| 658 | "<｜DSML｜tool_calls>...</｜DSML｜tool_calls>" |
| 659 | )); |
| 660 | assert!(!has_tool_call_markers("no markers here")); |
| 661 | } |
| 662 | } |
| 663 |