返回 CodeWhale
speech.rs
根目录 / crates / tui / src / tools / speech.rs
1 //! Model-visible Xiaomi MiMo speech/TTS generation tool.
2 //!
3 //! This mirrors the CLI `speech` / `tts` command as a first-class API tool so
4 //! the TUI model can generate narrated audio without shelling out to a nested
5 //! CodeWhale process.
6
7 use std::path::{Path, PathBuf};
8
9 use anyhow::Context as _;
10 use async_trait::async_trait;
11 use base64::{Engine as _, engine::general_purpose};
12 use serde_json::{Value, json};
13
14 use crate::client::{DeepSeekClient, SpeechSynthesisRequest};
15 use crate::config::{ApiProvider, normalize_model_name_for_provider};
16 use crate::network_policy::{Decision, host_from_url};
17
18 use super::spec::{
19 ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
20 optional_bool, optional_str, required_str,
21 };
22
23 pub(crate) const DEFAULT_FORMAT: &str = "wav";
24 pub(crate) const DEFAULT_VOICE: &str = "mimo_default";
25 const VOICE_CLONE_BASE64_MAX_BYTES: usize = 10 * 1024 * 1024;
26 pub(crate) const SUPPORTED_SPEECH_FORMATS: &[&str] = &["wav", "mp3", "pcm16"];
27
28 pub const SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS: &[&str] = &[
29 "mimo-v2.5-tts-voiceclone",
30 "mimo-v2.5-tts-voicedesign",
31 "mimo-v2.5-tts",
32 "mimo-v2-tts",
33 ];
34
35 pub(crate) const SPEECH_MODEL_EXAMPLES: &[&str] = &[
36 "mimo-v2.5-tts",
37 "mimo-v2.5-tts-voicedesign",
38 "mimo-v2.5-tts-voiceclone",
39 "mimo-v2-tts",
40 ];
41
42 pub struct SpeechTool {
43 name: &'static str,
44 client: Option<DeepSeekClient>,
45 output_dir: Option<PathBuf>,
46 }
47
48 impl SpeechTool {
49 #[must_use]
50 pub fn new(
51 name: &'static str,
52 client: Option<DeepSeekClient>,
53 output_dir: Option<PathBuf>,
54 ) -> Self {
55 Self {
56 name,
57 client,
58 output_dir,
59 }
60 }
61 }
62
63 #[async_trait]
64 impl ToolSpec for SpeechTool {
65 fn name(&self) -> &str {
66 self.name
67 }
68
69 fn description(&self) -> &str {
70 "Generate speech/audio directly through the configured Xiaomi MiMo OpenAI-compatible API. Use this when the user asks for speech, TTS, narration, read-aloud, voice design, or voice cloning."
71 }
72
73 fn input_schema(&self) -> Value {
74 json!({
75 "type": "object",
76 "properties": {
77 "text": {
78 "type": "string",
79 "description": "Text to synthesize. This is sent as the assistant message and is the spoken content; MiMo TTS style/audio tags may be included here."
80 },
81 "output": {
82 "type": "string",
83 "description": "Audio file path to write, relative to the workspace unless absolute. Default: speech.<format> in output_dir, configured [speech].output_dir, or the workspace."
84 },
85 "output_dir": {
86 "type": "string",
87 "description": "Directory for the default speech.<format> output file when output is omitted. Relative paths stay inside the workspace."
88 },
89 "model": {
90 "type": "string",
91 "description": "TTS model. Defaults to mimo-v2.5-tts, or infers voice-design/voice-clone models from voice_prompt/clone_voice.",
92 "enum": SPEECH_MODEL_EXAMPLES
93 },
94 "voice": {
95 "type": "string",
96 "description": "Built-in voice ID (for example mimo_default, 冰糖, 茉莉, 苏打, 白桦, Mia, Chloe, Milo, Dean) or a data:audio/...;base64,... URI for voice clone."
97 },
98 "instruction": {
99 "type": "string",
100 "description": "Natural-language style, emotion, speed, scene, or performance instruction. It is not spoken verbatim."
101 },
102 "voice_prompt": {
103 "type": "string",
104 "description": "Voice design prompt. When model is omitted this uses mimo-v2.5-tts-voicedesign."
105 },
106 "clone_voice": {
107 "type": "string",
108 "description": "Path to a .mp3 or .wav voice sample for cloning. When model is omitted this uses mimo-v2.5-tts-voiceclone."
109 },
110 "format": {
111 "type": "string",
112 "description": "Requested audio format. Default: wav. MiMo-V2.5-TTS documentation examples use wav and pcm16; mp3 is accepted when the API returns it.",
113 "enum": SUPPORTED_SPEECH_FORMATS
114 },
115 "stream": {
116 "type": "boolean",
117 "description": "Low-latency streaming request. The direct tool currently writes complete audio files only, so leave this false."
118 }
119 },
120 "required": ["text"]
121 })
122 }
123
124 fn capabilities(&self) -> Vec<ToolCapability> {
125 vec![
126 ToolCapability::WritesFiles,
127 ToolCapability::Network,
128 ToolCapability::Sandboxable,
129 ]
130 }
131
132 fn approval_requirement(&self) -> ApprovalRequirement {
133 // Speech generation is an explicit user-facing generation action.
134 // Path resolution still enforces workspace/trusted-root boundaries.
135 ApprovalRequirement::Auto
136 }
137
138 async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
139 let text = required_str(&input, "text")?.trim().to_string();
140 if text.is_empty() {
141 return Err(ToolError::invalid_input("speech text cannot be empty"));
142 }
143
144 let client = self.client.clone().ok_or_else(|| {
145 ToolError::not_available(
146 "speech tool requires an active Xiaomi MiMo API client; configure provider = \"xiaomi-mimo\" and an API key first",
147 )
148 })?;
149
150 let requested_format_raw = optional_str(&input, "format")?
151 .map(str::trim)
152 .filter(|value| !value.is_empty())
153 .unwrap_or(DEFAULT_FORMAT);
154 let requested_format = normalize_speech_format(requested_format_raw).ok_or_else(|| {
155 ToolError::invalid_input(format!(
156 "unsupported speech format '{requested_format_raw}' (allowed: {})",
157 SUPPORTED_SPEECH_FORMATS.join(", ")
158 ))
159 })?;
160 if optional_bool(&input, "stream", false)? {
161 return Err(ToolError::invalid_input(
162 "stream=true low-latency speech output is not implemented in the direct tool yet; use stream=false to generate a complete audio file",
163 ));
164 }
165 let output_raw = optional_str(&input, "output")?
166 .map(str::trim)
167 .filter(|value| !value.is_empty());
168 let output_path = resolve_speech_output_path(
169 &input,
170 context,
171 output_raw,
172 &requested_format,
173 self.output_dir.as_ref(),
174 )?;
175 let output_label = output_raw
176 .map(str::to_string)
177 .unwrap_or_else(|| output_path.display().to_string());
178
179 let raw_voice = optional_str(&input, "voice")?
180 .map(str::trim)
181 .filter(|value| !value.is_empty())
182 .map(str::to_string);
183 let raw_instruction = optional_str(&input, "instruction")?
184 .map(str::trim)
185 .filter(|value| !value.is_empty())
186 .map(str::to_string);
187 let voice_prompt = optional_str(&input, "voice_prompt")?
188 .map(str::trim)
189 .filter(|value| !value.is_empty())
190 .map(str::to_string);
191 let clone_voice = optional_str(&input, "clone_voice")?
192 .map(str::trim)
193 .filter(|value| !value.is_empty())
194 .map(str::to_string);
195
196 let voice_is_data_uri = raw_voice
197 .as_deref()
198 .is_some_and(|value| value.starts_with("data:audio/"));
199 if clone_voice.is_some() && raw_voice.is_some() {
200 return Err(ToolError::invalid_input(
201 "use either clone_voice or voice for cloned voice data, not both",
202 ));
203 }
204 let model = infer_speech_model(
205 optional_str(&input, "model")?,
206 clone_voice.is_some() || voice_is_data_uri,
207 voice_prompt.is_some(),
208 );
209 let model_lower = model.to_ascii_lowercase();
210 if !model_lower.contains("tts") {
211 return Err(ToolError::invalid_input(format!(
212 "speech tool requires a TTS model (examples: {}), got '{model}'",
213 SPEECH_MODEL_EXAMPLES.join(", ")
214 )));
215 }
216
217 let is_voice_design = model_lower.contains("voicedesign");
218 let is_voice_clone = model_lower.contains("voiceclone");
219 let instruction = combine_speech_instructions(raw_instruction, voice_prompt);
220 if is_voice_design
221 && instruction
222 .as_deref()
223 .is_none_or(|value| value.trim().is_empty())
224 {
225 return Err(ToolError::invalid_input(
226 "mimo-v2.5-tts-voicedesign requires voice_prompt or instruction",
227 ));
228 }
229
230 let voice = if let Some(clone_path) = clone_voice {
231 let clone_path = context.resolve_path(&clone_path)?;
232 Some(encode_voice_clone_data_uri(&clone_path).await?)
233 } else if is_voice_design {
234 None
235 } else if let Some(value) = raw_voice {
236 Some(value)
237 } else if is_voice_clone {
238 return Err(ToolError::invalid_input(
239 "mimo-v2.5-tts-voiceclone requires clone_voice <mp3|wav> or voice <data-uri>",
240 ));
241 } else {
242 Some(DEFAULT_VOICE.to_string())
243 };
244
245 check_network_policy(context, client.base_url())?;
246
247 let response = client
248 .synthesize_speech(SpeechSynthesisRequest {
249 model: model.clone(),
250 text,
251 instruction,
252 audio_format: requested_format,
253 voice,
254 })
255 .await
256 .map_err(|err| {
257 ToolError::execution_failed(format!("speech synthesis failed: {err}"))
258 })?;
259
260 if let Some(parent) = output_path
261 .parent()
262 .filter(|path| !path.as_os_str().is_empty())
263 {
264 tokio::fs::create_dir_all(parent).await.map_err(|err| {
265 ToolError::execution_failed(format!(
266 "failed to create output directory {}: {err}",
267 parent.display()
268 ))
269 })?;
270 }
271 tokio::fs::write(&output_path, &response.audio_bytes)
272 .await
273 .map_err(|err| {
274 ToolError::execution_failed(format!(
275 "failed to write audio file {}: {err}",
276 output_path.display()
277 ))
278 })?;
279
280 let result = json!({
281 "mode": "speech",
282 "success": true,
283 "api": "Xiaomi MiMo OpenAI-compatible chat/completions speech synthesis",
284 "base_url": openai_compatible_base_url(client.base_url()),
285 "model": response.model,
286 "format": response.audio_format,
287 "stream": false,
288 "output": output_label,
289 "absolute_output": output_path.display().to_string(),
290 "bytes": response.audio_bytes.len(),
291 "voice": response.voice.as_deref().map(describe_speech_voice),
292 "transcript": response.transcript,
293 "supported_formats": SUPPORTED_SPEECH_FORMATS,
294 "supported_xiaomi_mimo_models": SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS,
295 });
296 ToolResult::json(&result).map_err(|err| {
297 ToolError::execution_failed(format!("failed to serialize result: {err}"))
298 })
299 }
300 }
301
302 pub(crate) fn infer_speech_model(
303 model: Option<&str>,
304 has_clone_voice: bool,
305 has_voice_prompt: bool,
306 ) -> String {
307 match model.map(str::trim).filter(|value| !value.is_empty()) {
308 Some(value) => normalize_model_name_for_provider(ApiProvider::XiaomiMimo, value)
309 .unwrap_or_else(|| value.into()),
310 None if has_clone_voice => "mimo-v2.5-tts-voiceclone".to_string(),
311 None if has_voice_prompt => "mimo-v2.5-tts-voicedesign".to_string(),
312 None => "mimo-v2.5-tts".to_string(),
313 }
314 }
315
316 pub(crate) fn combine_speech_instructions(
317 instruction: Option<String>,
318 voice_prompt: Option<String>,
319 ) -> Option<String> {
320 match (instruction, voice_prompt) {
321 (Some(instruction), Some(voice_prompt)) => {
322 let instruction = instruction.trim();
323 let voice_prompt = voice_prompt.trim();
324 if instruction.is_empty() {
325 Some(voice_prompt.to_string()).filter(|value| !value.is_empty())
326 } else if voice_prompt.is_empty() {
327 Some(instruction.to_string()).filter(|value| !value.is_empty())
328 } else {
329 Some(format!("{voice_prompt}\n\n{instruction}"))
330 }
331 }
332 (Some(value), None) | (None, Some(value)) => {
333 let value = value.trim().to_string();
334 if value.is_empty() { None } else { Some(value) }
335 }
336 (None, None) => None,
337 }
338 }
339
340 pub(crate) fn normalize_speech_format(format: &str) -> Option<String> {
341 let normalized = format.trim().to_ascii_lowercase();
342 match normalized.as_str() {
343 "wav" | "mp3" | "pcm16" => Some(normalized),
344 "pcm" => Some("pcm16".to_string()),
345 _ => None,
346 }
347 }
348
349 pub(crate) fn default_speech_output_name(format: &str) -> String {
350 format!(
351 "speech.{}",
352 normalize_speech_format(format)
353 .as_deref()
354 .unwrap_or(DEFAULT_FORMAT)
355 )
356 }
357
358 fn resolve_speech_output_path(
359 input: &Value,
360 context: &ToolContext,
361 output_raw: Option<&str>,
362 format: &str,
363 configured_output_dir: Option<&PathBuf>,
364 ) -> Result<PathBuf, ToolError> {
365 if let Some(output) = output_raw {
366 return context.resolve_path(output);
367 }
368
369 let filename = default_speech_output_name(format);
370 if let Some(output_dir) = optional_str(input, "output_dir")?
371 .map(str::trim)
372 .filter(|value| !value.is_empty())
373 {
374 return Ok(context.resolve_path(output_dir)?.join(filename));
375 }
376
377 if let Some(output_dir) = configured_output_dir {
378 return Ok(output_dir.join(filename));
379 }
380
381 Ok(context.workspace.join(filename))
382 }
383
384 async fn encode_voice_clone_data_uri(path: &Path) -> Result<String, ToolError> {
385 let bytes = tokio::fs::read(path).await.map_err(|err| {
386 ToolError::execution_failed(format!(
387 "failed to read voice clone sample {}: {err}",
388 path.display()
389 ))
390 })?;
391
392 voice_clone_data_uri_from_bytes(path, &bytes)
393 .map_err(|err| ToolError::invalid_input(err.to_string()))
394 }
395
396 pub(crate) fn encode_voice_clone_sample_data_uri(path: &Path) -> anyhow::Result<String> {
397 let bytes = std::fs::read(path)
398 .with_context(|| format!("Failed to read voice clone sample {}", path.display()))?;
399
400 voice_clone_data_uri_from_bytes(path, &bytes)
401 }
402
403 fn voice_clone_data_uri_from_bytes(path: &Path, bytes: &[u8]) -> anyhow::Result<String> {
404 let base64_audio = general_purpose::STANDARD.encode(bytes);
405 if base64_audio.len() > VOICE_CLONE_BASE64_MAX_BYTES {
406 anyhow::bail!(
407 "voice clone sample is too large after base64 encoding ({} bytes > 10 MB)",
408 base64_audio.len()
409 );
410 }
411
412 let extension = path
413 .extension()
414 .and_then(|value| value.to_str())
415 .unwrap_or_default()
416 .to_ascii_lowercase();
417 let mime = match extension.as_str() {
418 "mp3" => "audio/mpeg",
419 "wav" => "audio/wav",
420 other => {
421 anyhow::bail!("unsupported voice clone sample extension '{other}'. Use .mp3 or .wav.");
422 }
423 };
424
425 Ok(format!("data:{mime};base64,{base64_audio}"))
426 }
427
428 pub(crate) fn describe_speech_voice(voice: &str) -> String {
429 if voice.starts_with("data:") {
430 "embedded voice clone sample".to_string()
431 } else {
432 voice.to_string()
433 }
434 }
435
436 fn openai_compatible_base_url(base_url: &str) -> String {
437 let trimmed = base_url.trim_end_matches('/');
438 if trimmed.ends_with("/v1") || trimmed.ends_with("/beta") {
439 trimmed.to_string()
440 } else {
441 format!("{trimmed}/v1")
442 }
443 }
444
445 fn check_network_policy(context: &ToolContext, base_url: &str) -> Result<(), ToolError> {
446 let Some(decider) = context.network_policy.as_ref() else {
447 return Ok(());
448 };
449 let display_url = openai_compatible_base_url(base_url);
450 let Some(host) = host_from_url(&display_url) else {
451 return Ok(());
452 };
453 match decider.evaluate(&host, "speech") {
454 Decision::Allow => Ok(()),
455 Decision::Deny => Err(ToolError::permission_denied(format!(
456 "speech network call to '{host}' blocked by network policy"
457 ))),
458 Decision::Prompt => Err(ToolError::permission_denied(format!(
459 "speech network call to '{host}' requires approval; re-run after `/network allow {host}` or set network.default = \"allow\" in config"
460 ))),
461 }
462 }
463
464 #[cfg(test)]
465 mod tests {
466 use super::*;
467
468 #[test]
469 fn infers_speech_model_from_requested_mode() {
470 assert_eq!(infer_speech_model(None, false, false), "mimo-v2.5-tts");
471 assert_eq!(
472 infer_speech_model(None, false, true),
473 "mimo-v2.5-tts-voicedesign"
474 );
475 assert_eq!(
476 infer_speech_model(None, true, false),
477 "mimo-v2.5-tts-voiceclone"
478 );
479 assert_eq!(
480 infer_speech_model(Some("mimo-tts"), false, false),
481 "mimo-v2.5-tts"
482 );
483 assert_eq!(
484 infer_speech_model(Some("mimo-v2-tts"), false, false),
485 "mimo-v2-tts"
486 );
487 }
488
489 #[test]
490 fn combines_voice_prompt_before_instruction() {
491 assert_eq!(
492 combine_speech_instructions(
493 Some("Speak warmly.".to_string()),
494 Some("Young Chinese female voice".to_string())
495 )
496 .as_deref(),
497 Some("Young Chinese female voice\n\nSpeak warmly.")
498 );
499 assert_eq!(
500 combine_speech_instructions(Some(" calm ".to_string()), None).as_deref(),
501 Some("calm")
502 );
503 }
504
505 #[test]
506 fn normalizes_documented_speech_formats() {
507 assert_eq!(normalize_speech_format("WAV").as_deref(), Some("wav"));
508 assert_eq!(normalize_speech_format("pcm16").as_deref(), Some("pcm16"));
509 assert_eq!(normalize_speech_format("pcm").as_deref(), Some("pcm16"));
510 assert_eq!(normalize_speech_format("flac"), None);
511 }
512
513 #[test]
514 fn supported_xiaomi_mimo_speech_models_are_tts_only() {
515 assert!(
516 SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS
517 .iter()
518 .all(|model| model.to_ascii_lowercase().contains("tts")),
519 "model-visible speech list must not include chat-only MiMo models"
520 );
521 assert!(SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5-tts"));
522 assert!(!SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5-pro"));
523 assert!(!SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5"));
524 }
525
526 #[test]
527 fn configured_output_dir_is_used_for_default_tool_output() {
528 let tmp = tempfile::tempdir().expect("tempdir");
529 let context = ToolContext::new(tmp.path().to_path_buf());
530 let configured = tmp.path().join("speech-artifacts");
531
532 let output = resolve_speech_output_path(
533 &json!({"text": "hello"}),
534 &context,
535 None,
536 "pcm",
537 Some(&configured),
538 )
539 .expect("output path");
540
541 assert_eq!(output, configured.join("speech.pcm16"));
542 }
543
544 #[test]
545 fn displays_openai_compatible_base_url() {
546 assert_eq!(
547 openai_compatible_base_url("https://api.xiaomimimo.com"),
548 "https://api.xiaomimimo.com/v1"
549 );
550 assert_eq!(
551 openai_compatible_base_url("https://api.xiaomimimo.com/v1"),
552 "https://api.xiaomimimo.com/v1"
553 );
554 }
555
556 #[test]
557 fn speech_tool_is_auto_approved_but_not_read_only() {
558 let tool = SpeechTool::new("speech", None, None);
559 assert_eq!(tool.name(), "speech");
560 assert_eq!(tool.approval_requirement(), ApprovalRequirement::Auto);
561 assert!(!tool.is_read_only());
562 let schema = tool.input_schema();
563 assert!(schema.to_string().contains("mimo-v2.5-tts-voiceclone"));
564 assert!(schema.to_string().contains("pcm16"));
565 assert!(schema.to_string().contains("stream"));
566 }
567 }
568
568 lines RUST