| 1 | //! Legacy parser for text-based tool calls from DeepSeek models. |
| 2 | //! |
| 3 | //! The engine prefers structured tool-call items and uses this fallback when |
| 4 | //! a response has tool-call markers but no structured calls. Unbalanced argument |
| 5 | //! objects are rejected; they must not become calls with invented empty args. |
| 6 | //! |
| 7 | //! Some DeepSeek outputs tool calls as text in various formats: |
| 8 | //! ```text |
| 9 | //! [TOOL_CALL] |
| 10 | //! {tool => "tool_name", args => {...}} |
| 11 | //! [/TOOL_CALL] |
| 12 | //! ``` |
| 13 | //! |
| 14 | //! Or XML-style format: |
| 15 | //! ```text |
| 16 | //! <codewhale:tool_call> |
| 17 | //! <invoke name="tool_name"> |
| 18 | //! <parameter name="arg">value</parameter> |
| 19 | //! </invoke> |
| 20 | //! </codewhale:tool_call> |
| 21 | //! ``` |
| 22 | //! |
| 23 | //! This module parses these text patterns into structured tool calls. |
| 24 | |
| 25 | use regex::Regex; |
| 26 | use serde_json::{Value, json}; |
| 27 | use std::sync::OnceLock; |
| 28 | |
| 29 | /// A parsed tool call from text content. |
| 30 | #[derive(Debug, Clone)] |
| 31 | pub struct ParsedToolCall { |
| 32 | /// Tool name |
| 33 | pub name: String, |
| 34 | /// Tool arguments as JSON |
| 35 | pub args: Value, |
| 36 | /// Generated ID for the tool call |
| 37 | pub id: String, |
| 38 | } |
| 39 | |
| 40 | /// Result of parsing text for tool calls. |
| 41 | #[derive(Debug)] |
| 42 | pub struct ParseResult { |
| 43 | /// The text with tool call markers removed (for display) |
| 44 | pub clean_text: String, |
| 45 | /// Parsed tool calls found in the text |
| 46 | pub tool_calls: Vec<ParsedToolCall>, |
| 47 | } |
| 48 | |
| 49 | static TOOL_CALL_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 50 | static XML_TOOL_CALL_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 51 | static INVOKE_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 52 | static THINKING_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 53 | static FAKE_TOOL_WRAPPER_REGEX: OnceLock<Regex> = OnceLock::new(); |
| 54 | |
| 55 | const FAKE_TOOL_CALL_MARKERS: &[&str] = &[ |
| 56 | "<function_calls>", |
| 57 | "<｜DSML｜tool_calls>", |
| 58 | "<｜DSML｜invoke ", |
| 59 | "<|DSML|tool_calls>", |
| 60 | "<|DSML|invoke ", |
| 61 | "<|dsml|tool_calls>", |
| 62 | "<|dsml|invoke ", |
| 63 | "<|tool_calls>", |
| 64 | // DeepSeek native tool-call tokens (#3880). See |
| 65 | // `engine::streaming::TOOL_CALL_MARKER_PAIRS` for why the `▁` (U+2581) |
| 66 | // separator matters: these match no DSML entry, so they used to reach the |
| 67 | // user as visible text. |
| 68 | "<｜tool▁calls▁begin｜>", |
| 69 | "<｜tool▁call▁begin｜>", |
| 70 | "<|tool▁calls▁begin|>", |
| 71 | "<|tool▁call▁begin|>", |
| 72 | "<｜tool_calls_begin｜>", |
| 73 | "<｜tool_call_begin｜>", |
| 74 | "<|tool_calls_begin|>", |
| 75 | "<|tool_call_begin|>", |
| 76 | ]; |
| 77 | |
| 78 | /// Tool-call wrapper pairs whose start and end markers are plain literals, so |
| 79 | /// their regex alternative is built by escaping rather than hand-written. The |
| 80 | /// DSML entries stay hand-written above because they carry attributes |
| 81 | /// (`invoke name="…"`) and need `\b[^>]*>` rather than a literal match. |
| 82 | const LITERAL_FAKE_WRAPPER_PAIRS: &[(&str, &str)] = &[ |
| 83 | ("<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>"), |
| 84 | ("<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>"), |
| 85 | ("<｜tool▁outputs▁begin｜>", "<｜tool▁outputs▁end｜>"), |
| 86 | ("<｜tool▁output▁begin｜>", "<｜tool▁output▁end｜>"), |
| 87 | ("<|tool▁calls▁begin|>", "<|tool▁calls▁end|>"), |
| 88 | ("<|tool▁call▁begin|>", "<|tool▁call▁end|>"), |
| 89 | ("<|tool▁outputs▁begin|>", "<|tool▁outputs▁end|>"), |
| 90 | ("<|tool▁output▁begin|>", "<|tool▁output▁end|>"), |
| 91 | ("<｜tool_calls_begin｜>", "<｜tool_calls_end｜>"), |
| 92 | ("<｜tool_call_begin｜>", "<｜tool_call_end｜>"), |
| 93 | ("<｜tool_outputs_begin｜>", "<｜tool_outputs_end｜>"), |
| 94 | ("<｜tool_output_begin｜>", "<｜tool_output_end｜>"), |
| 95 | ("<|tool_calls_begin|>", "<|tool_calls_end|>"), |
| 96 | ("<|tool_call_begin|>", "<|tool_call_end|>"), |
| 97 | ("<|tool_outputs_begin|>", "<|tool_outputs_end|>"), |
| 98 | ("<|tool_output_begin|>", "<|tool_output_end|>"), |
| 99 | ]; |
| 100 | |
| 101 | fn get_tool_call_regex() -> &'static Regex { |
| 102 | TOOL_CALL_REGEX.get_or_init(|| { |
| 103 | // Match [TOOL_CALL] ... [/TOOL_CALL] blocks |
| 104 | Regex::new(r"(?s)\[TOOL_CALL\]\s*(.*?)\s*\[/TOOL_CALL\]") |
| 105 | .expect("TOOL_CALL regex pattern is valid") |
| 106 | }) |
| 107 | } |
| 108 | |
| 109 | fn get_xml_tool_call_regex() -> &'static Regex { |
| 110 | XML_TOOL_CALL_REGEX.get_or_init(|| { |
| 111 | // Match <codewhale:tool_call>...</codewhale:tool_call> or similar XML patterns |
| 112 | Regex::new(r"(?s)<(?:codewhale:)?tool_call[^>]*>\s*(.*?)\s*</(?:codewhale:)?tool_call>") |
| 113 | .expect("XML tool_call regex pattern is valid") |
| 114 | }) |
| 115 | } |
| 116 | |
| 117 | fn get_invoke_regex() -> &'static Regex { |
| 118 | INVOKE_REGEX.get_or_init(|| { |
| 119 | // Match <invoke name="tool_name">...</invoke> patterns |
| 120 | Regex::new(r#"(?s)<invoke\s+name\s*=\s*"([^"]+)"[^>]*>(.*?)</invoke>"#) |
| 121 | .expect("invoke regex pattern is valid") |
| 122 | }) |
| 123 | } |
| 124 | |
| 125 | fn get_thinking_regex() -> &'static Regex { |
| 126 | THINKING_REGEX.get_or_init(|| { |
| 127 | // Match thinking blocks including partial closing tags |
| 128 | Regex::new(r"(?s)</?(?:think|thinking)[^>]*>").expect("thinking regex pattern is valid") |
| 129 | }) |
| 130 | } |
| 131 | |
| 132 | fn get_fake_tool_wrapper_regex() -> &'static Regex { |
| 133 | FAKE_TOOL_WRAPPER_REGEX.get_or_init(|| { |
| 134 | let mut alternatives = vec![ |
| 135 | r"<function_calls>.*?</function_calls>".to_string(), |
| 136 | r"<｜DSML｜tool_calls>.*?</｜DSML｜tool_calls>".to_string(), |
| 137 | r"<｜DSML｜invoke\b[^>]*>.*?</｜DSML｜invoke>".to_string(), |
| 138 | r"<\|DSML\|tool_calls>.*?</\|DSML\|tool_calls>".to_string(), |
| 139 | r"<\|DSML\|invoke\b[^>]*>.*?</\|DSML\|invoke>".to_string(), |
| 140 | r"<\|dsml\|tool_calls>.*?</\|dsml\|tool_calls>".to_string(), |
| 141 | r"<\|dsml\|invoke\b[^>]*>.*?</\|dsml\|invoke>".to_string(), |
| 142 | r"<\|tool_calls>.*?</\|tool_calls>".to_string(), |
| 143 | ]; |
| 144 | alternatives.extend( |
| 145 | LITERAL_FAKE_WRAPPER_PAIRS |
| 146 | .iter() |
| 147 | .map(|(start, end)| format!("{}.*?{}", regex::escape(start), regex::escape(end))), |
| 148 | ); |
| 149 | Regex::new(&format!("(?s){}", alternatives.join("|"))) |
| 150 | .expect("fake tool wrapper regex pattern is valid") |
| 151 | }) |
| 152 | } |
| 153 | |
| 154 | /// Parse tool calls from text content. |
| 155 | /// Returns the clean text (with markers removed) and any parsed tool calls. |
| 156 | pub fn parse_tool_calls(text: &str) -> ParseResult { |
| 157 | let mut tool_calls = Vec::new(); |
| 158 | let mut clean_text = text.to_string(); |
| 159 | let mut id_counter = 0; |
| 160 | |
| 161 | // First, remove thinking tags |
| 162 | let thinking_regex = get_thinking_regex(); |
| 163 | clean_text = thinking_regex.replace_all(&clean_text, "").to_string(); |
| 164 | |
| 165 | // Parse [TOOL_CALL] format |
| 166 | let regex = get_tool_call_regex(); |
| 167 | for cap in regex.captures_iter(text) { |
| 168 | let (Some(full_match), Some(inner)) = (cap.get(0), cap.get(1)) else { |
| 169 | continue; |
| 170 | }; |
| 171 | let full_match = full_match.as_str(); |
| 172 | let inner = inner.as_str().trim(); |
| 173 | |
| 174 | if let Some(parsed) = parse_tool_call_inner(inner, &mut id_counter) { |
| 175 | tool_calls.push(parsed); |
| 176 | } |
| 177 | |
| 178 | clean_text = clean_text.replace(full_match, ""); |
| 179 | } |
| 180 | |
| 181 | // Parse XML-style <codewhale:tool_call> or <tool_call> format |
| 182 | let xml_regex = get_xml_tool_call_regex(); |
| 183 | for cap in xml_regex.captures_iter(text) { |
| 184 | let (Some(full_match), Some(inner)) = (cap.get(0), cap.get(1)) else { |
| 185 | continue; |
| 186 | }; |
| 187 | let full_match = full_match.as_str(); |
| 188 | let inner = inner.as_str().trim(); |
| 189 | |
| 190 | // Parse invoke blocks inside |
| 191 | if let Some(parsed) = parse_invoke_block(inner, &mut id_counter) { |
| 192 | tool_calls.push(parsed); |
| 193 | } else if let Some(parsed) = parse_tool_call_inner(inner, &mut id_counter) { |
| 194 | tool_calls.push(parsed); |
| 195 | } |
| 196 | |
| 197 | clean_text = clean_text.replace(full_match, ""); |
| 198 | } |
| 199 | |
| 200 | // Also parse standalone <invoke> blocks that might not be wrapped |
| 201 | let invoke_regex = get_invoke_regex(); |
| 202 | for cap in invoke_regex.captures_iter(&clean_text.clone()) { |
| 203 | let (Some(full_match), Some(tool_name), Some(inner)) = (cap.get(0), cap.get(1), cap.get(2)) |
| 204 | else { |
| 205 | continue; |
| 206 | }; |
| 207 | let full_match = full_match.as_str(); |
| 208 | let tool_name = tool_name.as_str(); |
| 209 | let inner = inner.as_str(); |
| 210 | |
| 211 | let args = parse_xml_parameters(inner); |
| 212 | id_counter += 1; |
| 213 | tool_calls.push(ParsedToolCall { |
| 214 | name: tool_name.to_string(), |
| 215 | args, |
| 216 | id: format!("xml_tool_{id_counter}"), |
| 217 | }); |
| 218 | |
| 219 | clean_text = clean_text.replace(full_match, ""); |
| 220 | } |
| 221 | |
| 222 | clean_text = get_fake_tool_wrapper_regex() |
| 223 | .replace_all(&clean_text, "") |
| 224 | .to_string(); |
| 225 | |
| 226 | // Clean up extra whitespace and empty lines |
| 227 | clean_text = clean_text |
| 228 | .lines() |
| 229 | .filter(|line| !line.trim().is_empty()) |
| 230 | .collect::<Vec<_>>() |
| 231 | .join("\n") |
| 232 | .trim() |
| 233 | .to_string(); |
| 234 | |
| 235 | ParseResult { |
| 236 | clean_text, |
| 237 | tool_calls, |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | /// Parse an `<invoke>` block into a tool call. |
| 242 | fn parse_invoke_block(content: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 243 | let invoke_regex = get_invoke_regex(); |
| 244 | let cap = invoke_regex.captures(content)?; |
| 245 | |
| 246 | let tool_name = cap.get(1)?.as_str(); |
| 247 | let inner = cap.get(2)?.as_str(); |
| 248 | |
| 249 | let args = parse_xml_parameters(inner); |
| 250 | |
| 251 | *id_counter += 1; |
| 252 | Some(ParsedToolCall { |
| 253 | name: tool_name.to_string(), |
| 254 | args, |
| 255 | id: format!("xml_tool_{id_counter}"), |
| 256 | }) |
| 257 | } |
| 258 | |
| 259 | /// Parse XML-style parameters like <parameter name="foo">value</parameter> |
| 260 | fn parse_xml_parameters(content: &str) -> Value { |
| 261 | let param_regex = Regex::new( |
| 262 | "<(?:parameter|param)\\s+name\\s*=\\s*\"([^\"]+)\"[^>]*>(.*?)</(?:parameter|param)>", |
| 263 | ) |
| 264 | .ok(); |
| 265 | let simple_tag_regex = |
| 266 | Regex::new("<([a-zA-Z_][a-zA-Z0-9_]*)>(.*?)</([a-zA-Z_][a-zA-Z0-9_]*)>").ok(); |
| 267 | |
| 268 | let mut map = serde_json::Map::new(); |
| 269 | |
| 270 | // Try parsing <parameter name="...">value</parameter> |
| 271 | if let Some(regex) = param_regex { |
| 272 | for cap in regex.captures_iter(content) { |
| 273 | if let (Some(name), Some(value)) = (cap.get(1), cap.get(2)) { |
| 274 | let name_str = name.as_str(); |
| 275 | let value_str = value.as_str().trim(); |
| 276 | |
| 277 | // Try to parse as JSON, otherwise use as string |
| 278 | let json_value = serde_json::from_str(value_str) |
| 279 | .unwrap_or_else(|_| Value::String(value_str.to_string())); |
| 280 | map.insert(name_str.to_string(), json_value); |
| 281 | } |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | // Also try parsing <tagname>value</tagname> format |
| 286 | if let Some(regex) = simple_tag_regex { |
| 287 | for cap in regex.captures_iter(content) { |
| 288 | if let (Some(name), Some(value), Some(close)) = (cap.get(1), cap.get(2), cap.get(3)) { |
| 289 | if name.as_str() != close.as_str() { |
| 290 | continue; |
| 291 | } |
| 292 | let name_str = name.as_str(); |
| 293 | // Skip known wrapper tags |
| 294 | if ["invoke", "tool_call", "parameter", "param"].contains(&name_str) { |
| 295 | continue; |
| 296 | } |
| 297 | let value_str = value.as_str().trim(); |
| 298 | if !map.contains_key(name_str) { |
| 299 | let json_value = serde_json::from_str(value_str) |
| 300 | .unwrap_or_else(|_| Value::String(value_str.to_string())); |
| 301 | map.insert(name_str.to_string(), json_value); |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | Value::Object(map) |
| 308 | } |
| 309 | |
| 310 | /// Parse the inner content of a `TOOL_CALL` block. |
| 311 | fn parse_tool_call_inner(inner: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 312 | // Try to parse as JSON first |
| 313 | if let Ok(json) = serde_json::from_str::<Value>(inner) { |
| 314 | return parse_from_json(&json, id_counter); |
| 315 | } |
| 316 | |
| 317 | // Once the arrow format is recognized, invalid arguments must not fall |
| 318 | // through to the looser name search inside the argument text. |
| 319 | let tool_regex = Regex::new(r#"tool\s*=>\s*"([^"]+)""#).ok()?; |
| 320 | if let Some(cap) = tool_regex.captures(inner) { |
| 321 | let name = cap.get(1)?.as_str().to_string(); |
| 322 | return parse_arrow_syntax(inner, name, id_counter); |
| 323 | } |
| 324 | |
| 325 | if inner.starts_with('{') && extract_braced_object(inner, ArgumentSyntax::Json)? != inner { |
| 326 | return None; |
| 327 | } |
| 328 | |
| 329 | // Try to extract tool name and args from any format |
| 330 | parse_flexible_format(inner, id_counter) |
| 331 | } |
| 332 | |
| 333 | /// Parse from JSON object. |
| 334 | fn parse_from_json(json: &Value, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 335 | let obj = json.as_object()?; |
| 336 | |
| 337 | // Try different field names for the tool name |
| 338 | let name = obj |
| 339 | .get("tool") |
| 340 | .or_else(|| obj.get("name")) |
| 341 | .or_else(|| obj.get("function")) |
| 342 | .and_then(|v| v.as_str())? |
| 343 | .to_string(); |
| 344 | |
| 345 | // Try different field names for the arguments |
| 346 | let args = obj |
| 347 | .get("args") |
| 348 | .or_else(|| obj.get("arguments")) |
| 349 | .or_else(|| obj.get("input")) |
| 350 | .or_else(|| obj.get("parameters")) |
| 351 | .cloned() |
| 352 | .unwrap_or(json!({})); |
| 353 | |
| 354 | *id_counter += 1; |
| 355 | Some(ParsedToolCall { |
| 356 | name, |
| 357 | args, |
| 358 | id: format!("text_tool_{id_counter}"), |
| 359 | }) |
| 360 | } |
| 361 | |
| 362 | /// Parse the arrow syntax: {tool => "name", args => {...}} |
| 363 | fn parse_arrow_syntax(inner: &str, name: String, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 364 | // Match the same whitespace-tolerant arrow form as the tool name. |
| 365 | let args_regex = Regex::new(r"\bargs\s*=>").ok()?; |
| 366 | let args = if let Some(args_start) = args_regex.find(inner) { |
| 367 | let args_str = inner[args_start.end()..].trim(); |
| 368 | let syntax = if args_str.strip_prefix('{').is_some_and(|content| { |
| 369 | let content = content.trim_start(); |
| 370 | !content.starts_with('"') && !content.starts_with('}') |
| 371 | }) { |
| 372 | ArgumentSyntax::Cli |
| 373 | } else { |
| 374 | ArgumentSyntax::Json |
| 375 | }; |
| 376 | // The enclosing arrow object must close too. Its arguments decide |
| 377 | // the quote rules: CLI backslashes are literal, unlike JSON escapes. |
| 378 | if inner.starts_with('{') && extract_braced_object(inner, syntax)? != inner { |
| 379 | return None; |
| 380 | } |
| 381 | // Try to parse as JSON first |
| 382 | if let Ok(args_json) = serde_json::from_str::<Value>(args_str) { |
| 383 | args_json |
| 384 | } else { |
| 385 | let object = extract_braced_object(args_str, syntax)?; |
| 386 | if let Ok(json) = serde_json::from_str::<Value>(object) { |
| 387 | json |
| 388 | } else { |
| 389 | let content = &object[1..object.len() - 1]; |
| 390 | // A broken JSON object is not a CLI argument list, even if a |
| 391 | // string inside it happens to contain `key=value` text. |
| 392 | if matches!(syntax, ArgumentSyntax::Json) { |
| 393 | return None; |
| 394 | } |
| 395 | // Try CLI-style args: --arg_name "value" or --arg_name value |
| 396 | let args = parse_cli_style_args(content); |
| 397 | if args.as_object()?.is_empty() { |
| 398 | return None; |
| 399 | } |
| 400 | args |
| 401 | } |
| 402 | } |
| 403 | } else { |
| 404 | if inner.starts_with('{') && extract_braced_object(inner, ArgumentSyntax::Json)? != inner { |
| 405 | return None; |
| 406 | } |
| 407 | json!({}) |
| 408 | }; |
| 409 | |
| 410 | *id_counter += 1; |
| 411 | Some(ParsedToolCall { |
| 412 | name, |
| 413 | args, |
| 414 | id: format!("text_tool_{id_counter}"), |
| 415 | }) |
| 416 | } |
| 417 | |
| 418 | /// Parse CLI-style arguments: --`arg_name` "value" or --`arg_name` value |
| 419 | fn parse_cli_style_args(content: &str) -> Value { |
| 420 | let mut map = serde_json::Map::new(); |
| 421 | |
| 422 | // Pattern: --arg_name "value" or --arg_name 'value' or --arg_name value |
| 423 | let arg_regex = |
| 424 | Regex::new(r#"--([a-zA-Z_][a-zA-Z0-9_]*)\s+(?:"([^"]*)"|'([^']*)'|(\S+))"#).ok(); |
| 425 | |
| 426 | if let Some(regex) = arg_regex { |
| 427 | for cap in regex.captures_iter(content) { |
| 428 | if let Some(arg_name) = cap.get(1) { |
| 429 | let arg_name = arg_name.as_str(); |
| 430 | // Get the value from whichever capture group matched |
| 431 | let value = cap |
| 432 | .get(2) |
| 433 | .or_else(|| cap.get(3)) |
| 434 | .or_else(|| cap.get(4)) |
| 435 | .map_or("", |m| m.as_str()); |
| 436 | |
| 437 | // Try to parse as JSON value, otherwise use as string |
| 438 | let json_value = serde_json::from_str(value) |
| 439 | .unwrap_or_else(|_| Value::String(value.to_string())); |
| 440 | map.insert(arg_name.to_string(), json_value); |
| 441 | } |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | // Also try simple key=value format |
| 446 | let kv_regex = |
| 447 | Regex::new(r#"([a-zA-Z_][a-zA-Z0-9_]*)\s*[:=]\s*(?:"([^"]*)"|'([^']*)'|(\S+))"#).ok(); |
| 448 | if let Some(regex) = kv_regex { |
| 449 | for cap in regex.captures_iter(content) { |
| 450 | if let Some(key) = cap.get(1) { |
| 451 | let key = key.as_str(); |
| 452 | if !map.contains_key(key) { |
| 453 | let value = cap |
| 454 | .get(2) |
| 455 | .or_else(|| cap.get(3)) |
| 456 | .or_else(|| cap.get(4)) |
| 457 | .map_or("", |m| m.as_str()); |
| 458 | let json_value = serde_json::from_str(value) |
| 459 | .unwrap_or_else(|_| Value::String(value.to_string())); |
| 460 | map.insert(key.to_string(), json_value); |
| 461 | } |
| 462 | } |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | Value::Object(map) |
| 467 | } |
| 468 | |
| 469 | /// Try to parse a flexible format. |
| 470 | fn parse_flexible_format(inner: &str, id_counter: &mut u32) -> Option<ParsedToolCall> { |
| 471 | // Look for common patterns like: |
| 472 | // tool: list_dir |
| 473 | // name: "list_dir" |
| 474 | // function: list_dir |
| 475 | |
| 476 | let patterns = [( |
| 477 | r#"(?:tool|name|function)\s*[:=]\s*"?([a-zA-Z_][a-zA-Z0-9_]*)"?"#, |
| 478 | 1, |
| 479 | )]; |
| 480 | |
| 481 | for (pattern, group) in patterns { |
| 482 | if let Ok(regex) = Regex::new(pattern) |
| 483 | && let Some(cap) = regex.captures(inner) |
| 484 | && let Some(name_match) = cap.get(group) |
| 485 | { |
| 486 | let name = name_match.as_str().to_string(); |
| 487 | |
| 488 | // Missing arguments may be empty; a present but malformed object |
| 489 | // must not silently turn into an empty-argument tool call. |
| 490 | let args = if inner.contains('{') { |
| 491 | extract_json_object(inner)? |
| 492 | } else { |
| 493 | json!({}) |
| 494 | }; |
| 495 | |
| 496 | *id_counter += 1; |
| 497 | return Some(ParsedToolCall { |
| 498 | name, |
| 499 | args, |
| 500 | id: format!("text_tool_{id_counter}"), |
| 501 | }); |
| 502 | } |
| 503 | } |
| 504 | |
| 505 | None |
| 506 | } |
| 507 | |
| 508 | /// Extract the first JSON object from a string. |
| 509 | fn extract_json_object(text: &str) -> Option<Value> { |
| 510 | serde_json::from_str(extract_braced_object(text, ArgumentSyntax::Json)?).ok() |
| 511 | } |
| 512 | |
| 513 | #[derive(Clone, Copy)] |
| 514 | enum ArgumentSyntax { |
| 515 | Json, |
| 516 | Cli, |
| 517 | } |
| 518 | |
| 519 | /// Extract one balanced object without treating quoted braces as delimiters. |
| 520 | /// Both JSON and the legacy CLI argument form use this byte-safe boundary scan. |
| 521 | fn extract_braced_object(text: &str, syntax: ArgumentSyntax) -> Option<&str> { |
| 522 | let start = text.find('{')?; |
| 523 | let mut depth = 0usize; |
| 524 | let mut quote = None; |
| 525 | let mut escaped = false; |
| 526 | let mut at_value_start = true; |
| 527 | |
| 528 | for (offset, ch) in text[start..].char_indices() { |
| 529 | if let Some(delimiter) = quote { |
| 530 | if escaped { |
| 531 | escaped = false; |
| 532 | } else if matches!(syntax, ArgumentSyntax::Json) && ch == '\\' { |
| 533 | escaped = true; |
| 534 | } else if ch == delimiter { |
| 535 | quote = None; |
| 536 | at_value_start = false; |
| 537 | } |
| 538 | continue; |
| 539 | } |
| 540 | match ch { |
| 541 | '"' if matches!(syntax, ArgumentSyntax::Json) || at_value_start => quote = Some(ch), |
| 542 | '\'' if matches!(syntax, ArgumentSyntax::Cli) && at_value_start => quote = Some(ch), |
| 543 | '{' => depth += 1, |
| 544 | '}' => { |
| 545 | depth -= 1; |
| 546 | if depth == 0 { |
| 547 | return Some(&text[start..start + offset + ch.len_utf8()]); |
| 548 | } |
| 549 | } |
| 550 | _ => {} |
| 551 | } |
| 552 | // The existing CLI regex recognizes quotes only at the start of a |
| 553 | // value; an apostrophe inside O'Brien.txt is ordinary filename data. |
| 554 | at_value_start = ch.is_whitespace() || matches!(ch, '{' | ':' | '=' | '>'); |
| 555 | } |
| 556 | None |
| 557 | } |
| 558 | |
| 559 | /// Check if text contains tool call markers (either format). |
| 560 | pub fn has_tool_call_markers(text: &str) -> bool { |
| 561 | text.contains("[TOOL_CALL]") |
| 562 | || text.contains("<codewhale:tool_call") |
| 563 | || text.contains("<tool_call") |
| 564 | || text.contains("<invoke ") |
| 565 | || FAKE_TOOL_CALL_MARKERS |
| 566 | .iter() |
| 567 | .any(|marker| text.contains(marker)) |
| 568 | } |
| 569 | |
| 570 | #[cfg(test)] |
| 571 | mod tests { |
| 572 | use super::*; |
| 573 | |
| 574 | #[test] |
| 575 | fn test_parse_arrow_syntax() { |
| 576 | let text = r#"I'll list the directory. |
| 577 | [TOOL_CALL] |
| 578 | {tool => "list_dir", args => {}} |
| 579 | [/TOOL_CALL]"#; |
| 580 | |
| 581 | let result = parse_tool_calls(text); |
| 582 | assert_eq!(result.tool_calls.len(), 1); |
| 583 | assert_eq!(result.tool_calls[0].name, "list_dir"); |
| 584 | assert_eq!(result.clean_text, "I'll list the directory."); |
| 585 | } |
| 586 | |
| 587 | #[test] |
| 588 | fn test_parse_json_syntax() { |
| 589 | let text = r#"Let me check. |
| 590 | [TOOL_CALL] |
| 591 | {"tool": "read_file", "args": {"path": "test.txt"}} |
| 592 | [/TOOL_CALL]"#; |
| 593 | |
| 594 | let result = parse_tool_calls(text); |
| 595 | assert_eq!(result.tool_calls.len(), 1); |
| 596 | assert_eq!(result.tool_calls[0].name, "read_file"); |
| 597 | assert_eq!(result.tool_calls[0].args["path"], "test.txt"); |
| 598 | } |
| 599 | |
| 600 | #[test] |
| 601 | fn unicode_and_quoted_braces_preserve_json_arguments() { |
| 602 | for args in [ |
| 603 | json!({"content": "日本語"}), |
| 604 | json!({"content": "你好世界，这是一个测试"}), |
| 605 | json!({"content": "café 🐋"}), |
| 606 | json!({ |
| 607 | "content": "literal } then {, escaped \"quote }\", and \\ slash", |
| 608 | "nested": {"items": [{"text": "{深い}"}]}, |
| 609 | }), |
| 610 | ] { |
| 611 | for body in [ |
| 612 | format!(r#"{{tool => "write_file", args => {args}}}"#), |
| 613 | format!(r#"{{tool=>"write_file", args=>{args}}}"#), |
| 614 | format!("{{tool\t=>\"write_file\", args\t\n=>{args}}}"), |
| 615 | format!("tool: write_file {args}"), |
| 616 | ] { |
| 617 | let result = parse_tool_calls(&format!("[TOOL_CALL]{body}[/TOOL_CALL]")); |
| 618 | assert_eq!(result.tool_calls.len(), 1, "{body}"); |
| 619 | assert_eq!(result.tool_calls[0].name, "write_file"); |
| 620 | assert_eq!(result.tool_calls[0].args, args, "{body}"); |
| 621 | } |
| 622 | } |
| 623 | } |
| 624 | |
| 625 | #[test] |
| 626 | fn unicode_and_quoted_braces_preserve_cli_arguments() { |
| 627 | let result = parse_tool_calls( |
| 628 | r#"[TOOL_CALL] |
| 629 | {tool => "write_file", args => { |
| 630 | --path "鲸鱼.md" |
| 631 | --content '你好 } { 世界' |
| 632 | }} |
| 633 | [/TOOL_CALL]"#, |
| 634 | ); |
| 635 | assert_eq!(result.tool_calls.len(), 1); |
| 636 | assert_eq!( |
| 637 | result.tool_calls[0].args, |
| 638 | json!({"path": "鲸鱼.md", "content": "你好 } { 世界"}) |
| 639 | ); |
| 640 | |
| 641 | // The legacy CLI grammar treats quoted backslashes literally, |
| 642 | // including one immediately before the closing quote. |
| 643 | let result = parse_tool_calls( |
| 644 | r#"[TOOL_CALL]{tool => "write_file", args => {--path '目录\' --content 'literal } {\'}}[/TOOL_CALL]"#, |
| 645 | ); |
| 646 | assert_eq!(result.tool_calls.len(), 1); |
| 647 | assert_eq!( |
| 648 | result.tool_calls[0].args, |
| 649 | json!({"path": "目录\\", "content": "literal } {\\"}) |
| 650 | ); |
| 651 | |
| 652 | for (arguments, path) in [ |
| 653 | (r#"--path "C:\""#, "C:\\"), |
| 654 | (r#"--path 'C:\'"#, "C:\\"), |
| 655 | ("--path O'Brien.txt", "O'Brien.txt"), |
| 656 | (r#"path="C:\""#, "C:\\"), |
| 657 | ("path=O'Brien.txt", "O'Brien.txt"), |
| 658 | (r#"--path "O'Brien {界}.txt""#, "O'Brien {界}.txt"), |
| 659 | ] { |
| 660 | let text = |
| 661 | format!(r#"[TOOL_CALL]{{tool=>"read_file", args=>{{{arguments}}}}}[/TOOL_CALL]"#); |
| 662 | let result = parse_tool_calls(&text); |
| 663 | assert_eq!(result.tool_calls.len(), 1, "{text}"); |
| 664 | assert_eq!(result.tool_calls[0].name, "read_file"); |
| 665 | assert_eq!(result.tool_calls[0].args["path"], path, "{text}"); |
| 666 | } |
| 667 | } |
| 668 | |
| 669 | #[test] |
| 670 | fn malformed_objects_do_not_become_empty_or_nested_tool_calls() { |
| 671 | for body in [ |
| 672 | r#"{tool => "write_file", args => {"#, |
| 673 | // The argument object closes, but the enclosing arrow object does not. |
| 674 | r#"{tool => "write_file", args => {"content":"日本語"}"#, |
| 675 | r#"{tool => "write_file", args => {"content":"日本語",}}"#, |
| 676 | r#"{tool => "write_file", args=> {"content":"x",}}"#, |
| 677 | r#"{tool => "write_file", args => {"content":"x",}}"#, |
| 678 | "{tool => \"write_file\", args\t\n=> {\"content\":\"x\",}}", |
| 679 | // Neither the tool name nor CLI-looking text in a broken JSON |
| 680 | // string may be reinterpreted as a different argument format. |
| 681 | r#"{tool => "write_file", args => {"content":"tool: exec_shell key=value",}}"#, |
| 682 | r#"{tool => "write_file", args => {--content "unterminated}}"#, |
| 683 | r#"{tool => "read_file", args => {--path "C:\"}"#, |
| 684 | r#"{tool => "read_file", args => {--path O'Brien.txt}"#, |
| 685 | r#"tool: write_file {"#, |
| 686 | r#"tool: write_file {"content":"日本語""#, |
| 687 | r#"tool: write_file {"content":"日本語",}"#, |
| 688 | r#"tool: write_file {"content":"escaped quote \"}"#, |
| 689 | ] { |
| 690 | let result = parse_tool_calls(&format!("[TOOL_CALL]{body}[/TOOL_CALL]")); |
| 691 | assert!(result.tool_calls.is_empty(), "{body}: {result:?}"); |
| 692 | } |
| 693 | } |
| 694 | |
| 695 | #[test] |
| 696 | fn flexible_call_without_an_argument_object_still_parses() { |
| 697 | for body in ["tool: list_dir", r#"{tool => "list_dir"}"#] { |
| 698 | let result = parse_tool_calls(&format!("[TOOL_CALL]{body}[/TOOL_CALL]")); |
| 699 | assert_eq!(result.tool_calls.len(), 1); |
| 700 | assert_eq!(result.tool_calls[0].name, "list_dir"); |
| 701 | assert_eq!(result.tool_calls[0].args, json!({})); |
| 702 | } |
| 703 | } |
| 704 | |
| 705 | #[test] |
| 706 | fn test_parse_multiple_tool_calls() { |
| 707 | let text = r#"First I'll list, then read. |
| 708 | [TOOL_CALL] |
| 709 | {tool => "list_dir", args => {}} |
| 710 | [/TOOL_CALL] |
| 711 | [TOOL_CALL] |
| 712 | {tool => "read_file", args => {"path": "file.txt"}} |
| 713 | [/TOOL_CALL]"#; |
| 714 | |
| 715 | let result = parse_tool_calls(text); |
| 716 | assert_eq!(result.tool_calls.len(), 2); |
| 717 | assert_eq!(result.tool_calls[0].name, "list_dir"); |
| 718 | assert_eq!(result.tool_calls[1].name, "read_file"); |
| 719 | } |
| 720 | |
| 721 | #[test] |
| 722 | fn test_no_tool_calls() { |
| 723 | let text = "Just some regular text without any tool calls."; |
| 724 | let result = parse_tool_calls(text); |
| 725 | assert!(result.tool_calls.is_empty()); |
| 726 | assert_eq!(result.clean_text, text); |
| 727 | } |
| 728 | |
| 729 | #[test] |
| 730 | fn test_dsml_wrappers_are_stripped_without_execution() { |
| 731 | let text = "before\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"read_file\">\n<｜DSML｜parameter name=\"path\" string=\"true\">secret.txt</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\nafter"; |
| 732 | |
| 733 | assert!(has_tool_call_markers(text)); |
| 734 | let result = parse_tool_calls(text); |
| 735 | |
| 736 | assert!(result.tool_calls.is_empty()); |
| 737 | assert!(result.clean_text.contains("before")); |
| 738 | assert!(result.clean_text.contains("after")); |
| 739 | assert!(!result.clean_text.contains("DSML")); |
| 740 | assert!(!result.clean_text.contains("read_file")); |
| 741 | assert!(!result.clean_text.contains("secret.txt")); |
| 742 | } |
| 743 | |
| 744 | #[test] |
| 745 | fn test_ascii_dsml_wrappers_are_stripped_without_execution() { |
| 746 | let text = "before <|DSML|invoke name=\"grep_files\"><|DSML|parameter name=\"pattern\">SECRET</|DSML|parameter></|DSML|invoke> after"; |
| 747 | |
| 748 | assert!(has_tool_call_markers(text)); |
| 749 | let result = parse_tool_calls(text); |
| 750 | |
| 751 | assert!(result.tool_calls.is_empty()); |
| 752 | assert!(result.clean_text.contains("before")); |
| 753 | assert!(result.clean_text.contains("after")); |
| 754 | assert!(!result.clean_text.contains("DSML")); |
| 755 | assert!(!result.clean_text.contains("grep_files")); |
| 756 | assert!(!result.clean_text.contains("SECRET")); |
| 757 | } |
| 758 | |
| 759 | #[test] |
| 760 | fn test_deepseek_native_tool_tokens_are_stripped_without_execution() { |
| 761 | // #3880: DeepSeek's own tool-call tokens use `▁` (U+2581) as the word |
| 762 | // separator, so they matched none of the DSML shapes and survived into |
| 763 | // the text shown to the user. |
| 764 | for (start, end) in [ |
| 765 | ("<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>"), |
| 766 | ("<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>"), |
| 767 | ("<|tool▁calls▁begin|>", "<|tool▁calls▁end|>"), |
| 768 | ("<｜tool_calls_begin｜>", "<｜tool_calls_end｜>"), |
| 769 | ("<|tool_call_begin|>", "<|tool_call_end|>"), |
| 770 | ] { |
| 771 | let text = format!( |
| 772 | "before {start}function<｜tool▁sep｜>grep_files\n```json\n{{\"pattern\":\"SECRET\"}}\n```{end} after" |
| 773 | ); |
| 774 | |
| 775 | assert!(has_tool_call_markers(&text), "not detected: {start}"); |
| 776 | let result = parse_tool_calls(&text); |
| 777 | |
| 778 | // The wrapper is scrubbed, never executed: a model forging a tool |
| 779 | // call in plain text must not become a real invocation. |
| 780 | assert!(result.tool_calls.is_empty(), "{start} was executed"); |
| 781 | assert!( |
| 782 | result.clean_text.contains("before"), |
| 783 | "{:?}", |
| 784 | result.clean_text |
| 785 | ); |
| 786 | assert!( |
| 787 | result.clean_text.contains("after"), |
| 788 | "{:?}", |
| 789 | result.clean_text |
| 790 | ); |
| 791 | assert!( |
| 792 | !result.clean_text.contains("grep_files") |
| 793 | && !result.clean_text.contains("SECRET") |
| 794 | && !result.clean_text.contains("tool▁") |
| 795 | && !result.clean_text.contains("tool_calls"), |
| 796 | "leaked {start}: {:?}", |
| 797 | result.clean_text |
| 798 | ); |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | #[test] |
| 803 | fn test_has_markers() { |
| 804 | assert!(has_tool_call_markers("[TOOL_CALL]test[/TOOL_CALL]")); |
| 805 | assert!(has_tool_call_markers( |
| 806 | "<｜DSML｜tool_calls>...</｜DSML｜tool_calls>" |
| 807 | )); |
| 808 | assert!(!has_tool_call_markers("no markers here")); |
| 809 | } |
| 810 | } |
| 811 |