| 1 | import asyncio |
| 2 | import base64 |
| 3 | import io |
| 4 | import inspect |
| 5 | import json |
| 6 | import math |
| 7 | import os |
| 8 | import queue |
| 9 | import re |
| 10 | import subprocess |
| 11 | import threading |
| 12 | import time |
| 13 | import unicodedata |
| 14 | from datetime import datetime |
| 15 | from typing import Union |
| 16 | from xml.sax.saxutils import unescape |
| 17 | |
| 18 | import edge_tts |
| 19 | import requests |
| 20 | from edge_tts import SubMaker |
| 21 | from loguru import logger |
| 22 | from moviepy.video.tools import subtitles |
| 23 | from moviepy.audio.io.AudioFileClip import AudioFileClip |
| 24 | from openai import OpenAI |
| 25 | |
| 26 | from app.config import config |
| 27 | from app.utils import utils |
| 28 | |
| 29 | _DEFAULT_EDGE_TTS_TIMEOUT_SECONDS = 30.0 |
| 30 | _MIMO_DEFAULT_BASE_URL = "https://api.xiaomimimo.com/v1" |
| 31 | _MIMO_DEFAULT_TTS_MODEL = "mimo-v2.5-tts" |
| 32 | NO_VOICE_NAME = "no-voice" |
| 33 | # `none` 是 PR #981 里曾使用过的无配音标识。这里短期兼容这个值，避免 |
| 34 | # 已经手动调用过该分支的 API 用户升级后立即失效；WebUI 和新代码统一使用 |
| 35 | # 更明确的 `no-voice`。 |
| 36 | _NO_VOICE_ALIASES = {NO_VOICE_NAME, "none"} |
| 37 | |
| 38 | |
| 39 | def _configure_pydub_ffmpeg(audio_segment_cls): |
| 40 | configured_ffmpeg = utils.get_ffmpeg_binary() |
| 41 | if configured_ffmpeg: |
| 42 | audio_segment_cls.converter = configured_ffmpeg |
| 43 | |
| 44 | |
| 45 | def mktimestamp(time_unit: float) -> str: |
| 46 | """ |
| 47 | 将 edge_tts 使用的 100 纳秒时间单位转换为字幕时间戳。 |
| 48 | |
| 49 | edge_tts 7.x 不再导出旧版本里的 `mktimestamp`，但项目里旧字幕链路 |
| 50 | 还需要这个格式化函数来兼容 Azure v2、Gemini、SiliconFlow 这些 |
| 51 | 手工构造的字幕时间轴，因此这里内置一个等价实现。 |
| 52 | """ |
| 53 | hour = math.floor(time_unit / 10**7 / 3600) |
| 54 | minute = math.floor((time_unit / 10**7 / 60) % 60) |
| 55 | seconds = (time_unit / 10**7) % 60 |
| 56 | return f"{hour:02d}:{minute:02d}:{seconds:06.3f}" |
| 57 | |
| 58 | |
| 59 | def get_siliconflow_voices() -> list[str]: |
| 60 | """ |
| 61 | 获取硅基流动的声音列表 |
| 62 | |
| 63 | Returns: |
| 64 | 声音列表，格式为 ["siliconflow:FunAudioLLM/CosyVoice2-0.5B:alex", ...] |
| 65 | """ |
| 66 | # 硅基流动的声音列表和对应的性别（用于显示） |
| 67 | voices_with_gender = [ |
| 68 | ("FunAudioLLM/CosyVoice2-0.5B", "alex", "Male"), |
| 69 | ("FunAudioLLM/CosyVoice2-0.5B", "anna", "Female"), |
| 70 | ("FunAudioLLM/CosyVoice2-0.5B", "bella", "Female"), |
| 71 | ("FunAudioLLM/CosyVoice2-0.5B", "benjamin", "Male"), |
| 72 | ("FunAudioLLM/CosyVoice2-0.5B", "charles", "Male"), |
| 73 | ("FunAudioLLM/CosyVoice2-0.5B", "claire", "Female"), |
| 74 | ("FunAudioLLM/CosyVoice2-0.5B", "david", "Male"), |
| 75 | ("FunAudioLLM/CosyVoice2-0.5B", "diana", "Female"), |
| 76 | ] |
| 77 | |
| 78 | # 添加siliconflow:前缀，并格式化为显示名称 |
| 79 | return [ |
| 80 | f"siliconflow:{model}:{voice}-{gender}" |
| 81 | for model, voice, gender in voices_with_gender |
| 82 | ] |
| 83 | |
| 84 | |
| 85 | def get_gemini_voices() -> list[str]: |
| 86 | """ |
| 87 | 获取Gemini TTS的声音列表 |
| 88 | |
| 89 | Returns: |
| 90 | 声音列表，格式为 ["gemini:Zephyr-Female", "gemini:Puck-Male", ...] |
| 91 | """ |
| 92 | # Gemini TTS支持的语音列表 |
| 93 | voices_with_gender = [ |
| 94 | ("Zephyr", "Female"), |
| 95 | ("Puck", "Male"), |
| 96 | ("Charon", "Male"), |
| 97 | ("Kore", "Female"), |
| 98 | ("Fenrir", "Male"), |
| 99 | ("Aoede", "Female"), |
| 100 | ("Thalia", "Female"), |
| 101 | ("Sage", "Male"), |
| 102 | ("Echo", "Female"), |
| 103 | ("Harmony", "Female"), |
| 104 | ("Lux", "Female"), |
| 105 | ("Nova", "Female"), |
| 106 | ("Vale", "Male"), |
| 107 | ("Orion", "Male"), |
| 108 | ("Atlas", "Male"), |
| 109 | ] |
| 110 | |
| 111 | # 添加gemini:前缀，并格式化为显示名称 |
| 112 | return [ |
| 113 | f"gemini:{voice}-{gender}" |
| 114 | for voice, gender in voices_with_gender |
| 115 | ] |
| 116 | |
| 117 | |
| 118 | def get_mimo_voices() -> list[str]: |
| 119 | """ |
| 120 | 获取 Xiaomi MiMo V2.5 TTS 的预置音色列表。 |
| 121 | |
| 122 | 当前只接入官方文档里的 `mimo-v2.5-tts` 预置音色模式。音色设计 |
| 123 | `mimo-v2.5-tts-voicedesign` 和音色复刻 `mimo-v2.5-tts-voiceclone` |
| 124 | 需要额外的输入表单和素材上传流程，先不混入普通 TTS 下拉框，避免 |
| 125 | 用户误以为选择一个 voice id 就能完成所有高级能力。 |
| 126 | """ |
| 127 | voices_with_gender = [ |
| 128 | ("mimo_default", "Female"), |
| 129 | ("冰糖", "Female"), |
| 130 | ("茉莉", "Female"), |
| 131 | ("苏打", "Male"), |
| 132 | ("白桦", "Male"), |
| 133 | ("Mia", "Female"), |
| 134 | ("Chloe", "Female"), |
| 135 | ("Milo", "Male"), |
| 136 | ("Dean", "Male"), |
| 137 | ] |
| 138 | |
| 139 | return [f"mimo:{voice}-{gender}" for voice, gender in voices_with_gender] |
| 140 | |
| 141 | |
| 142 | def get_elevenlabs_voices(api_key: str) -> list[str]: |
| 143 | if not api_key: |
| 144 | return [] |
| 145 | try: |
| 146 | url = "https://api.elevenlabs.io/v2/voices" |
| 147 | params = {"is_favorite": "true", "page_size": 100} |
| 148 | headers = {"xi-api-key": api_key} |
| 149 | response = requests.get(url, params=params, headers=headers, timeout=10) |
| 150 | if response.status_code != 200: |
| 151 | logger.warning( |
| 152 | f"ElevenLabs voices fetch failed with status {response.status_code}: {response.text}" |
| 153 | ) |
| 154 | return [] |
| 155 | data = response.json() |
| 156 | voices = data.get("voices", []) |
| 157 | return [ |
| 158 | f"elevenlabs:{v['voice_id']}:{v['name']}" |
| 159 | for v in voices |
| 160 | if v.get("voice_id") and v.get("name") and v.get("status") != "disabled" |
| 161 | ] |
| 162 | except Exception as e: |
| 163 | logger.warning(f"ElevenLabs voices fetch failed: {str(e)}") |
| 164 | return [] |
| 165 | |
| 166 | |
| 167 | def get_chatterbox_voices() -> list[str]: |
| 168 | """Return the configured Chatterbox voices. |
| 169 | |
| 170 | Chatterbox is self-hosted, so there is no global voice catalog. Operators |
| 171 | list the voice names exposed by their server via ``[chatterbox] voices`` |
| 172 | (a TOML array, or a comma-separated string). Each entry is normalised to |
| 173 | the ``chatterbox:<name>`` format used by the TTS dispatcher. |
| 174 | """ |
| 175 | voices = config.chatterbox.get("voices", []) or [] |
| 176 | if isinstance(voices, str): |
| 177 | voices = [v.strip() for v in voices.split(",") if v.strip()] |
| 178 | result = [] |
| 179 | for v in voices: |
| 180 | v = str(v).strip() |
| 181 | if not v: |
| 182 | continue |
| 183 | result.append(v if v.startswith("chatterbox:") else f"chatterbox:{v}") |
| 184 | if not result: |
| 185 | # keep the dropdown usable even before any voice is configured |
| 186 | result = ["chatterbox:default-Female"] |
| 187 | return result |
| 188 | |
| 189 | |
| 190 | _AZURE_VOICES_DATA_FILE = os.path.join( |
| 191 | os.path.dirname(__file__), "data", "azure_voices.json" |
| 192 | ) |
| 193 | _azure_voices_cache = None |
| 194 | |
| 195 | |
| 196 | def _load_azure_voices() -> list[dict]: |
| 197 | global _azure_voices_cache |
| 198 | if _azure_voices_cache is None: |
| 199 | with open(_AZURE_VOICES_DATA_FILE, "r", encoding="utf-8") as f: |
| 200 | _azure_voices_cache = json.load(f) |
| 201 | return _azure_voices_cache |
| 202 | |
| 203 | |
| 204 | def get_all_azure_voices(filter_locals=None) -> list[str]: |
| 205 | voices = [] |
| 206 | for item in _load_azure_voices(): |
| 207 | name = item["name"] |
| 208 | gender = item["gender"] |
| 209 | # 应用过滤条件 |
| 210 | if filter_locals and any( |
| 211 | name.lower().startswith(fl.lower()) for fl in filter_locals |
| 212 | ): |
| 213 | voices.append(f"{name}-{gender}") |
| 214 | elif not filter_locals: |
| 215 | voices.append(f"{name}-{gender}") |
| 216 | |
| 217 | voices.sort() |
| 218 | return voices |
| 219 | |
| 220 | |
| 221 | def parse_voice_name(name: str): |
| 222 | # zh-CN-XiaoyiNeural-Female |
| 223 | # zh-CN-YunxiNeural-Male |
| 224 | # zh-CN-XiaoxiaoMultilingualNeural-V2-Female |
| 225 | name = name.replace("-Female", "").replace("-Male", "").strip() |
| 226 | return name |
| 227 | |
| 228 | |
| 229 | def is_azure_v2_voice(voice_name: str): |
| 230 | voice_name = parse_voice_name(voice_name) |
| 231 | if voice_name.endswith("-V2"): |
| 232 | return voice_name.replace("-V2", "").strip() |
| 233 | return "" |
| 234 | |
| 235 | |
| 236 | def is_siliconflow_voice(voice_name: str): |
| 237 | """检查是否是硅基流动的声音""" |
| 238 | return voice_name.startswith("siliconflow:") |
| 239 | |
| 240 | |
| 241 | def is_gemini_voice(voice_name: str): |
| 242 | """检查是否是Gemini TTS的声音""" |
| 243 | return voice_name.startswith("gemini:") |
| 244 | |
| 245 | |
| 246 | def is_mimo_voice(voice_name: str): |
| 247 | """检查是否是 Xiaomi MiMo TTS 的声音""" |
| 248 | return voice_name.startswith("mimo:") |
| 249 | |
| 250 | |
| 251 | def is_elevenlabs_voice(voice_name: str) -> bool: |
| 252 | return (voice_name or "").startswith("elevenlabs:") |
| 253 | |
| 254 | |
| 255 | def is_chatterbox_voice(voice_name: str) -> bool: |
| 256 | return (voice_name or "").startswith("chatterbox:") |
| 257 | |
| 258 | |
| 259 | def is_no_voice(voice_name: str | None) -> bool: |
| 260 | """ |
| 261 | 判断用户是否明确选择了“无配音”模式。 |
| 262 | |
| 263 | 这里刻意不把空字符串当成无配音：空 voice 更可能是配置损坏、旧版本 |
| 264 | WebUI 状态丢失或接口参数缺失。只有明确的 sentinel 才进入静音分支， |
| 265 | 这样可以避免把真实错误伪装成正常生成。 |
| 266 | """ |
| 267 | return str(voice_name or "").strip().lower() in _NO_VOICE_ALIASES |
| 268 | |
| 269 | |
| 270 | def estimate_no_voice_duration(text: str) -> float: |
| 271 | """ |
| 272 | 为无配音模式估算一个稳定的视频时间轴长度。 |
| 273 | |
| 274 | 无配音仍需要一个音频占位来驱动现有素材裁剪、字幕时间轴和最终合成。 |
| 275 | 估算策略尽量简单： |
| 276 | 1. 中文等 CJK 字符按约 4.2 字/秒估算； |
| 277 | 2. 英文/数字按约 2.7 词/秒估算； |
| 278 | 3. 其他语种文字按约 4.0 字符/秒兜底估算，覆盖俄语、阿拉伯语、 |
| 279 | 日文假名、韩文等非 ASCII 文本； |
| 280 | 4. 每个断句补一点停顿，让字幕切换不至于过于紧凑； |
| 281 | 5. 最少 3 秒，避免极短脚本生成 0 秒音频。 |
| 282 | """ |
| 283 | normalized_text = (text or "").strip() |
| 284 | if not normalized_text: |
| 285 | return 3.0 |
| 286 | |
| 287 | cjk_chars = len(re.findall(r"[\u4e00-\u9fff]", normalized_text)) |
| 288 | words = len(re.findall(r"[A-Za-z0-9]+", normalized_text)) |
| 289 | ascii_word_chars = sum(len(word) for word in re.findall(r"[A-Za-z0-9]+", normalized_text)) |
| 290 | other_text_chars = 0 |
| 291 | for char in normalized_text: |
| 292 | # Unicode category 以 L 开头表示各语种字母，N 表示数字。前面已经单独 |
| 293 | # 统计了 CJK 和 ASCII 单词，这里只统计剩余文字，避免英文被重复计时。 |
| 294 | category = unicodedata.category(char) |
| 295 | if category.startswith(("L", "N")): |
| 296 | other_text_chars += 1 |
| 297 | other_text_chars = max(other_text_chars - cjk_chars - ascii_word_chars, 0) |
| 298 | sentence_count = max(len(utils.split_string_by_punctuations(normalized_text)), 1) |
| 299 | |
| 300 | cjk_duration = cjk_chars / 4.2 |
| 301 | word_duration = words / 2.7 |
| 302 | other_text_duration = other_text_chars / 4.0 |
| 303 | pause_duration = max(sentence_count - 1, 0) * 0.35 |
| 304 | return max(3.0, cjk_duration + word_duration + other_text_duration + pause_duration) |
| 305 | |
| 306 | |
| 307 | def generate_silent_audio(duration_seconds: float, output_file: str) -> bool: |
| 308 | """ |
| 309 | 生成 MP3 静音音频，作为“无配音”模式的时间轴占位。 |
| 310 | |
| 311 | 使用 FFmpeg 的 anullsrc 直接生成静音，比先构造临时 WAV 再转码更少中间 |
| 312 | 文件。失败时返回 False，让上层按普通 TTS 失败路径处理并记录日志。 |
| 313 | """ |
| 314 | ensure_file_path_exists(output_file) |
| 315 | duration_seconds = max(float(duration_seconds or 0), 0.1) |
| 316 | ffmpeg_binary = utils.get_ffmpeg_binary() |
| 317 | command = [ |
| 318 | ffmpeg_binary, |
| 319 | "-y", |
| 320 | "-f", |
| 321 | "lavfi", |
| 322 | "-i", |
| 323 | "anullsrc=r=44100:cl=mono", |
| 324 | "-t", |
| 325 | f"{duration_seconds:.3f}", |
| 326 | "-codec:a", |
| 327 | "libmp3lame", |
| 328 | "-q:a", |
| 329 | "4", |
| 330 | output_file, |
| 331 | ] |
| 332 | |
| 333 | logger.info( |
| 334 | f"generating silent audio for no-voice mode, duration: {duration_seconds:.2f}s" |
| 335 | ) |
| 336 | result = subprocess.run( |
| 337 | command, |
| 338 | capture_output=True, |
| 339 | text=True, |
| 340 | check=False, |
| 341 | ) |
| 342 | if result.returncode != 0: |
| 343 | logger.error( |
| 344 | "failed to generate silent audio: " |
| 345 | f"{(result.stderr or result.stdout or '').strip()}" |
| 346 | ) |
| 347 | return False |
| 348 | if not os.path.exists(output_file) or os.path.getsize(output_file) <= 0: |
| 349 | logger.error( |
| 350 | "silent audio output file is missing or empty, " |
| 351 | f"file: {output_file}, duration: {duration_seconds:.2f}s" |
| 352 | ) |
| 353 | return False |
| 354 | return True |
| 355 | |
| 356 | |
| 357 | def tts( |
| 358 | text: str, |
| 359 | voice_name: str, |
| 360 | voice_rate: float, |
| 361 | voice_file: str, |
| 362 | voice_volume: float = 1.0, |
| 363 | ) -> Union[SubMaker, None]: |
| 364 | if is_no_voice(voice_name): |
| 365 | duration_seconds = estimate_no_voice_duration(text) |
| 366 | if not generate_silent_audio(duration_seconds, voice_file): |
| 367 | return None |
| 368 | |
| 369 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 370 | return populate_legacy_submaker_with_full_text( |
| 371 | sub_maker=sub_maker, |
| 372 | text=text, |
| 373 | audio_duration_seconds=duration_seconds, |
| 374 | ) |
| 375 | |
| 376 | if is_azure_v2_voice(voice_name): |
| 377 | return azure_tts_v2(text, voice_name, voice_file) |
| 378 | elif is_siliconflow_voice(voice_name): |
| 379 | # 从voice_name中提取模型和声音 |
| 380 | # 格式: siliconflow:model:voice-Gender |
| 381 | parts = voice_name.split(":") |
| 382 | if len(parts) >= 3: |
| 383 | model = parts[1] |
| 384 | # 移除性别后缀，例如 "alex-Male" -> "alex" |
| 385 | voice_with_gender = parts[2] |
| 386 | voice = voice_with_gender.split("-")[0] |
| 387 | # 构建完整的voice参数，格式为 "model:voice" |
| 388 | full_voice = f"{model}:{voice}" |
| 389 | return siliconflow_tts( |
| 390 | text, model, full_voice, voice_rate, voice_file, voice_volume |
| 391 | ) |
| 392 | else: |
| 393 | logger.error(f"Invalid siliconflow voice name format: {voice_name}") |
| 394 | return None |
| 395 | elif is_gemini_voice(voice_name): |
| 396 | # 从voice_name中提取声音名称 |
| 397 | # 格式: gemini:voice-Gender |
| 398 | parts = voice_name.split(":") |
| 399 | if len(parts) >= 2: |
| 400 | # 移除性别后缀，例如 "Zephyr-Female" -> "Zephyr" |
| 401 | voice_with_gender = parts[1] |
| 402 | voice = voice_with_gender.split("-")[0] |
| 403 | return gemini_tts(text, voice, voice_rate, voice_file, voice_volume) |
| 404 | else: |
| 405 | logger.error(f"Invalid gemini voice name format: {voice_name}") |
| 406 | return None |
| 407 | elif is_mimo_voice(voice_name): |
| 408 | # 从voice_name中提取声音名称 |
| 409 | # 格式: mimo:voice-Gender；如果调用方已执行 parse_voice_name， |
| 410 | # 则可能是 mimo:voice。两种格式都兼容。 |
| 411 | parts = voice_name.split(":") |
| 412 | if len(parts) >= 2: |
| 413 | voice_with_gender = parts[1] |
| 414 | voice = voice_with_gender.split("-")[0] |
| 415 | return mimo_tts(text, voice, voice_rate, voice_file, voice_volume) |
| 416 | else: |
| 417 | logger.error(f"Invalid mimo voice name format: {voice_name}") |
| 418 | return None |
| 419 | elif is_elevenlabs_voice(voice_name): |
| 420 | # 格式: elevenlabs:{voice_id}:{name} |
| 421 | parts = voice_name.split(":") |
| 422 | if len(parts) >= 2: |
| 423 | voice_id = parts[1] |
| 424 | return elevenlabs_tts(text, voice_id, voice_file, voice_rate, voice_volume) |
| 425 | else: |
| 426 | logger.error(f"Invalid elevenlabs voice name format: {voice_name}") |
| 427 | return None |
| 428 | elif is_chatterbox_voice(voice_name): |
| 429 | # 格式: chatterbox:<voice>，voice 可带显示用的 -Female/-Male 后缀 |
| 430 | parts = voice_name.split(":", 1) |
| 431 | if len(parts) >= 2 and parts[1].strip(): |
| 432 | chatterbox_voice = parts[1].strip() |
| 433 | if chatterbox_voice.endswith(("-Female", "-Male")): |
| 434 | chatterbox_voice = chatterbox_voice.rsplit("-", 1)[0] |
| 435 | return chatterbox_tts( |
| 436 | text, chatterbox_voice, voice_file, voice_rate, voice_volume |
| 437 | ) |
| 438 | else: |
| 439 | logger.error(f"Invalid chatterbox voice name format: {voice_name}") |
| 440 | return None |
| 441 | return azure_tts_v1(text, voice_name, voice_rate, voice_file) |
| 442 | |
| 443 | |
| 444 | def convert_rate_to_percent(rate: float) -> str: |
| 445 | # edge-tts requires a sign-prefixed percentage (e.g. "+0%", "-20%"). |
| 446 | # Rounding can yield 0 for rates near but not equal to 1.0 (e.g. 1.004, |
| 447 | # 0.997); those must still be returned as "+0%", not the unsigned "0%" |
| 448 | # which edge-tts rejects with ValueError: Invalid rate '0%'. |
| 449 | # API 或批处理调用可能传入 0、0.0、None 或无法转换的空值；这些值不代表 |
| 450 | # 合法语速，直接计算会变成 -100% 或抛异常。这里统一回退到正常语速， |
| 451 | # 避免生成极慢音频或让 TTS 流程在边界输入下失败。 |
| 452 | try: |
| 453 | rate = float(rate) |
| 454 | except (TypeError, ValueError): |
| 455 | rate = 1.0 |
| 456 | if rate <= 0: |
| 457 | rate = 1.0 |
| 458 | percent = round((rate - 1.0) * 100) |
| 459 | if percent >= 0: |
| 460 | return f"+{percent}%" |
| 461 | return f"{percent}%" |
| 462 | |
| 463 | |
| 464 | def ensure_file_path_exists(file_path: str) -> None: |
| 465 | """ |
| 466 | 确保输出文件所在目录一定存在。 |
| 467 | |
| 468 | 这里单独做一层兜底，是因为 edge_tts 7.x 在真正发起网络请求之前， |
| 469 | 就会先打开目标音频文件；如果目录不存在，会直接因为本地文件路径报错， |
| 470 | 从而掩盖真正的 TTS 行为结果。 |
| 471 | """ |
| 472 | dir_path = os.path.dirname(file_path) |
| 473 | if dir_path: |
| 474 | os.makedirs(dir_path, exist_ok=True) |
| 475 | |
| 476 | |
| 477 | def ensure_legacy_submaker_fields(sub_maker: SubMaker) -> SubMaker: |
| 478 | """ |
| 479 | 为项目里仍然沿用旧字幕结构的调用方补齐兼容字段。 |
| 480 | |
| 481 | edge_tts 7.x 的 `SubMaker` 主要暴露 `cues/get_srt()`，但项目里 Azure v2、 |
| 482 | Gemini、SiliconFlow 这些路径仍然会直接读写 `subs/offset`。这里统一补齐， |
| 483 | 避免升级 edge_tts 后这些非 edge 路径被连带破坏。 |
| 484 | """ |
| 485 | if not hasattr(sub_maker, "subs"): |
| 486 | sub_maker.subs = [] |
| 487 | if not hasattr(sub_maker, "offset"): |
| 488 | sub_maker.offset = [] |
| 489 | return sub_maker |
| 490 | |
| 491 | |
| 492 | def populate_legacy_submaker_with_full_text( |
| 493 | sub_maker: SubMaker, text: str, audio_duration_seconds: float |
| 494 | ) -> SubMaker: |
| 495 | """ |
| 496 | 用整段文本填充项目历史沿用的 `subs/offset` 字幕结构。 |
| 497 | |
| 498 | 背景： |
| 499 | 1. edge_tts 7.x 的 `SubMaker` 不再提供旧版本里的 `create_sub()`； |
| 500 | 2. 项目里 Gemini、SiliconFlow 等非 edge 路径依然需要返回一个 |
| 501 | 带 `subs/offset` 的对象，供后续统一计算音频时长和生成字幕； |
| 502 | 3. 对于拿不到逐词边界的 TTS 服务，需要至少按脚本断句切成多个片段， |
| 503 | 这样后续 `subtitle_provider=edge` 的聚合逻辑才能继续工作，而不是 |
| 504 | 因为整段文本无法和脚本断句逐行匹配而回退 Whisper。 |
| 505 | |
| 506 | Args: |
| 507 | sub_maker: 需要写入兼容字段的字幕对象 |
| 508 | text: 原始脚本文本 |
| 509 | audio_duration_seconds: 音频总时长，单位秒 |
| 510 | |
| 511 | Returns: |
| 512 | 已填充兼容字幕数据的 SubMaker 对象 |
| 513 | """ |
| 514 | sub_maker = ensure_legacy_submaker_fields(sub_maker) |
| 515 | |
| 516 | # 清空旧值，避免调用方重复复用对象时出现脏数据叠加。 |
| 517 | sub_maker.subs = [] |
| 518 | sub_maker.offset = [] |
| 519 | |
| 520 | normalized_text = (text or "").strip() |
| 521 | if not normalized_text: |
| 522 | return sub_maker |
| 523 | |
| 524 | audio_duration_100ns = max(int(audio_duration_seconds * 10000000), 1) |
| 525 | |
| 526 | # Gemini / SiliconFlow 这类路径拿不到逐词边界时，仍然尽量沿用项目 |
| 527 | # 原来的“按标点断句 + 按字符数比例分配时长”的策略。这样既能让 |
| 528 | # create_subtitle() 匹配脚本断句，也能避免再次回退 Whisper。 |
| 529 | sentences = utils.split_string_by_punctuations(normalized_text) |
| 530 | if not sentences: |
| 531 | sentences = [normalized_text] |
| 532 | |
| 533 | total_chars = sum(len(sentence) for sentence in sentences) |
| 534 | if total_chars <= 0: |
| 535 | sub_maker.subs.append(normalized_text) |
| 536 | sub_maker.offset.append((0, audio_duration_100ns)) |
| 537 | return sub_maker |
| 538 | |
| 539 | current_offset = 0 |
| 540 | for index, sentence in enumerate(sentences): |
| 541 | cleaned_sentence = sentence.strip() |
| 542 | if not cleaned_sentence: |
| 543 | continue |
| 544 | |
| 545 | # 前面的句子按字符数比例分配时长，最后一句兜底吃掉剩余时长， |
| 546 | # 避免整数取整导致总时长丢失或字幕结束时间短于音频。 |
| 547 | if index == len(sentences) - 1: |
| 548 | sentence_end = audio_duration_100ns |
| 549 | else: |
| 550 | sentence_chars = len(cleaned_sentence) |
| 551 | sentence_duration = max( |
| 552 | int(audio_duration_100ns * (sentence_chars / total_chars)), |
| 553 | 1, |
| 554 | ) |
| 555 | sentence_end = min(current_offset + sentence_duration, audio_duration_100ns) |
| 556 | |
| 557 | sub_maker.subs.append(cleaned_sentence) |
| 558 | sub_maker.offset.append((current_offset, sentence_end)) |
| 559 | current_offset = sentence_end |
| 560 | |
| 561 | return sub_maker |
| 562 | |
| 563 | |
| 564 | def create_edge_tts_communicate( |
| 565 | text: str, voice_name: str, rate_str: str |
| 566 | ) -> edge_tts.Communicate: |
| 567 | """ |
| 568 | 按当前已安装的 edge_tts 版本构造 Communicate 对象。 |
| 569 | |
| 570 | 背景： |
| 571 | 1. 主线代码已经升级到 edge_tts 7.x，并使用 `boundary` 参数拿到更细的边界事件； |
| 572 | 2. 但 Windows 便携包如果更新失败，现场环境可能仍然停留在旧版 edge_tts； |
| 573 | 3. 旧版 `Communicate.__init__()` 不接受 `boundary`，会直接抛出 |
| 574 | `unexpected keyword argument 'boundary'`，导致整个 TTS 链路失败。 |
| 575 | |
| 576 | 因此这里先根据构造函数签名探测当前版本支持的参数，再决定是否传入 |
| 577 | `boundary`，让同一份代码同时兼容旧版和新版依赖。 |
| 578 | """ |
| 579 | communicate_kwargs = {"rate": rate_str} |
| 580 | communicate_signature = inspect.signature(edge_tts.Communicate) |
| 581 | |
| 582 | if "boundary" in communicate_signature.parameters: |
| 583 | communicate_kwargs["boundary"] = "WordBoundary" |
| 584 | |
| 585 | return edge_tts.Communicate(text, voice_name, **communicate_kwargs) |
| 586 | |
| 587 | |
| 588 | def get_edge_tts_timeout_seconds() -> Union[float, None]: |
| 589 | """ |
| 590 | 获取 Azure TTS V1 单次流式请求的超时时间。 |
| 591 | |
| 592 | 背景： |
| 593 | Edge consumer TTS 在网络不通、服务端限流、voice 与文本语言不匹配等场景下， |
| 594 | 可能长时间卡在 `stream_sync()` 内部，日志只停留在 `start`。这里提供一个 |
| 595 | 默认超时，避免 WebUI 任务长期无反馈。 |
| 596 | |
| 597 | 使用方式： |
| 598 | - 默认 30 秒，覆盖常见短视频脚本的首包等待时间； |
| 599 | - 如用户处于慢网络或代理环境，可在 `config.toml` 里设置 |
| 600 | `edge_tts_timeout = 60`； |
| 601 | - 设置为 0 或负数表示显式禁用超时，保留完全向后兼容。 |
| 602 | """ |
| 603 | raw_timeout = config.app.get( |
| 604 | "edge_tts_timeout", _DEFAULT_EDGE_TTS_TIMEOUT_SECONDS |
| 605 | ) |
| 606 | try: |
| 607 | timeout_seconds = float(raw_timeout) |
| 608 | except (TypeError, ValueError): |
| 609 | logger.warning( |
| 610 | "invalid edge_tts_timeout: " |
| 611 | f"{raw_timeout}, fallback to {_DEFAULT_EDGE_TTS_TIMEOUT_SECONDS}s" |
| 612 | ) |
| 613 | timeout_seconds = _DEFAULT_EDGE_TTS_TIMEOUT_SECONDS |
| 614 | |
| 615 | if timeout_seconds <= 0: |
| 616 | return None |
| 617 | |
| 618 | return timeout_seconds |
| 619 | |
| 620 | |
| 621 | def _stream_edge_tts_sync_with_timeout( |
| 622 | communicate, on_chunk, timeout_seconds: float |
| 623 | ) -> None: |
| 624 | """ |
| 625 | 带总超时地消费 edge_tts 7.x 的同步流。 |
| 626 | |
| 627 | 实现原因： |
| 628 | `stream_sync()` 本身是阻塞迭代器，网络层卡住时主线程无法及时恢复。 |
| 629 | 这里把阻塞迭代放到 daemon 线程中，主线程通过 Queue 获取 chunk， |
| 630 | 到达超时时间后直接抛出 TimeoutError，让外层重试和错误日志继续工作。 |
| 631 | |
| 632 | 注意： |
| 633 | daemon 线程只作为兜底保护使用，最多随 Azure TTS V1 的 3 次重试产生 |
| 634 | 少量残留线程；进程退出时会自动回收。相比 WebUI 任务永久卡住，这是 |
| 635 | 更可控的失败模式。 |
| 636 | """ |
| 637 | stream_queue = queue.Queue() |
| 638 | done_marker = object() |
| 639 | |
| 640 | def _produce_chunks(): |
| 641 | try: |
| 642 | for chunk in communicate.stream_sync(): |
| 643 | stream_queue.put(("chunk", chunk)) |
| 644 | stream_queue.put(("done", done_marker)) |
| 645 | except Exception as e: |
| 646 | stream_queue.put(("error", e)) |
| 647 | |
| 648 | thread = threading.Thread(target=_produce_chunks, daemon=True) |
| 649 | thread.start() |
| 650 | |
| 651 | deadline = time.monotonic() + timeout_seconds |
| 652 | while True: |
| 653 | remaining_seconds = deadline - time.monotonic() |
| 654 | if remaining_seconds <= 0: |
| 655 | raise TimeoutError( |
| 656 | f"edge_tts stream timed out after {timeout_seconds:g}s" |
| 657 | ) |
| 658 | |
| 659 | try: |
| 660 | item_type, payload = stream_queue.get( |
| 661 | timeout=min(0.5, remaining_seconds) |
| 662 | ) |
| 663 | except queue.Empty: |
| 664 | continue |
| 665 | |
| 666 | if item_type == "chunk": |
| 667 | on_chunk(payload) |
| 668 | elif item_type == "error": |
| 669 | raise payload |
| 670 | elif item_type == "done": |
| 671 | return |
| 672 | |
| 673 | |
| 674 | def stream_edge_tts_chunks( |
| 675 | communicate, on_chunk, timeout_seconds: Union[float, None] = None |
| 676 | ) -> None: |
| 677 | """ |
| 678 | 统一消费 edge_tts 的同步流和旧版异步流。 |
| 679 | |
| 680 | edge_tts 7.x 提供 `stream_sync()`，可以在同步函数里直接迭代； |
| 681 | 更早的版本通常只有异步 `stream()`。为了让 `azure_tts_v1()` 在 |
| 682 | 旧依赖残留场景下仍能继续工作，这里统一做一层流式兼容。 |
| 683 | |
| 684 | Args: |
| 685 | communicate: edge_tts.Communicate 实例 |
| 686 | on_chunk: 每拿到一个事件块时执行的回调 |
| 687 | timeout_seconds: 单次流式请求总超时；为 None 时不启用超时。 |
| 688 | """ |
| 689 | if hasattr(communicate, "stream_sync"): |
| 690 | if timeout_seconds: |
| 691 | _stream_edge_tts_sync_with_timeout( |
| 692 | communicate, on_chunk, timeout_seconds |
| 693 | ) |
| 694 | return |
| 695 | |
| 696 | for chunk in communicate.stream_sync(): |
| 697 | on_chunk(chunk) |
| 698 | return |
| 699 | |
| 700 | if not hasattr(communicate, "stream"): |
| 701 | raise AttributeError("edge_tts communicate object has no stream method") |
| 702 | |
| 703 | async def _consume_async_stream(): |
| 704 | async for chunk in communicate.stream(): |
| 705 | on_chunk(chunk) |
| 706 | |
| 707 | # 这里显式创建独立事件循环，而不是复用外部上下文，目的是避免 |
| 708 | # 在同步调用栈里遇到“当前线程没有事件循环”或跨线程复用循环的问题。 |
| 709 | loop = asyncio.new_event_loop() |
| 710 | try: |
| 711 | if timeout_seconds: |
| 712 | loop.run_until_complete( |
| 713 | asyncio.wait_for(_consume_async_stream(), timeout=timeout_seconds) |
| 714 | ) |
| 715 | else: |
| 716 | loop.run_until_complete(_consume_async_stream()) |
| 717 | finally: |
| 718 | loop.close() |
| 719 | |
| 720 | |
| 721 | def azure_tts_v1( |
| 722 | text: str, voice_name: str, voice_rate: float, voice_file: str |
| 723 | ) -> Union[SubMaker, None]: |
| 724 | voice_name = parse_voice_name(voice_name) |
| 725 | text = text.strip() |
| 726 | rate_str = convert_rate_to_percent(voice_rate) |
| 727 | for i in range(3): |
| 728 | try: |
| 729 | logger.info(f"start, voice name: {voice_name}, try: {i + 1}") |
| 730 | |
| 731 | # 这里同时兼容 edge_tts 7.x 和旧版便携包里可能残留的老依赖： |
| 732 | # 1. 新版支持 `boundary` + `stream_sync()` |
| 733 | # 2. 旧版不支持 `boundary`，且通常只暴露异步 `stream()` |
| 734 | ensure_file_path_exists(voice_file) |
| 735 | communicate = create_edge_tts_communicate(text, voice_name, rate_str) |
| 736 | sub_maker = edge_tts.SubMaker() |
| 737 | timeout_seconds = get_edge_tts_timeout_seconds() |
| 738 | |
| 739 | with open(voice_file, "wb") as file: |
| 740 | def _handle_chunk(chunk): |
| 741 | chunk_type = chunk["type"] |
| 742 | if chunk_type == "audio": |
| 743 | file.write(chunk["data"]) |
| 744 | elif chunk_type in ["WordBoundary", "SentenceBoundary"]: |
| 745 | # 无论来自 7.x 的同步流，还是旧版异步流，只要事件结构 |
| 746 | # 里仍有边界信息，就统一喂给 SubMaker，保证后续字幕链路 |
| 747 | # 仍然走项目现有逻辑。 |
| 748 | sub_maker.feed(chunk) |
| 749 | |
| 750 | stream_edge_tts_chunks( |
| 751 | communicate, _handle_chunk, timeout_seconds=timeout_seconds |
| 752 | ) |
| 753 | |
| 754 | if not sub_maker.get_srt(): |
| 755 | logger.warning("failed, sub_maker.get_srt() is empty") |
| 756 | continue |
| 757 | |
| 758 | logger.info(f"completed, output file: {voice_file}") |
| 759 | return sub_maker |
| 760 | except Exception as e: |
| 761 | logger.error(f"failed, error: {str(e)}") |
| 762 | # TTS 流式写入如果在首包前超时或网络异常，会留下 0 字节音频文件。 |
| 763 | # 这种文件既不可播放，也可能误导后续排查，因此失败后只清理空文件； |
| 764 | # 如果已经写入了部分数据，则保留现场文件，便于分析服务端返回内容。 |
| 765 | if os.path.exists(voice_file) and os.path.getsize(voice_file) == 0: |
| 766 | try: |
| 767 | os.remove(voice_file) |
| 768 | except Exception as remove_error: |
| 769 | logger.warning( |
| 770 | "failed to remove empty tts file: " |
| 771 | f"{voice_file}, error: {str(remove_error)}" |
| 772 | ) |
| 773 | return None |
| 774 | |
| 775 | |
| 776 | def siliconflow_tts( |
| 777 | text: str, |
| 778 | model: str, |
| 779 | voice: str, |
| 780 | voice_rate: float, |
| 781 | voice_file: str, |
| 782 | voice_volume: float = 1.0, |
| 783 | ) -> Union[SubMaker, None]: |
| 784 | """ |
| 785 | 使用硅基流动的API生成语音 |
| 786 | |
| 787 | Args: |
| 788 | text: 要转换为语音的文本 |
| 789 | model: 模型名称，如 "FunAudioLLM/CosyVoice2-0.5B" |
| 790 | voice: 声音名称，如 "FunAudioLLM/CosyVoice2-0.5B:alex" |
| 791 | voice_rate: 语音速度，范围[0.25, 4.0] |
| 792 | voice_file: 输出的音频文件路径 |
| 793 | voice_volume: 语音音量，范围[0.6, 5.0]，需要转换为硅基流动的增益范围[-10, 10] |
| 794 | |
| 795 | Returns: |
| 796 | SubMaker对象或None |
| 797 | """ |
| 798 | text = text.strip() |
| 799 | api_key = config.siliconflow.get("api_key", "") |
| 800 | |
| 801 | if not api_key: |
| 802 | logger.error("SiliconFlow API key is not set") |
| 803 | return None |
| 804 | |
| 805 | # 将voice_volume转换为硅基流动的增益范围 |
| 806 | # 默认voice_volume为1.0，对应gain为0 |
| 807 | gain = voice_volume - 1.0 |
| 808 | # 确保gain在[-10, 10]范围内 |
| 809 | gain = max(-10, min(10, gain)) |
| 810 | |
| 811 | url = "https://api.siliconflow.cn/v1/audio/speech" |
| 812 | |
| 813 | payload = { |
| 814 | "model": model, |
| 815 | "input": text, |
| 816 | "voice": voice, |
| 817 | "response_format": "mp3", |
| 818 | "sample_rate": 32000, |
| 819 | "stream": False, |
| 820 | "speed": voice_rate, |
| 821 | "gain": gain, |
| 822 | } |
| 823 | |
| 824 | headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} |
| 825 | |
| 826 | for i in range(3): # 尝试3次 |
| 827 | try: |
| 828 | logger.info( |
| 829 | f"start siliconflow tts, model: {model}, voice: {voice}, try: {i + 1}" |
| 830 | ) |
| 831 | |
| 832 | response = requests.post(url, json=payload, headers=headers) |
| 833 | |
| 834 | if response.status_code == 200: |
| 835 | # 保存音频文件 |
| 836 | with open(voice_file, "wb") as f: |
| 837 | f.write(response.content) |
| 838 | |
| 839 | # 这里仍然沿用项目原有的字幕结构，因此需要补齐旧字段。 |
| 840 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 841 | |
| 842 | # 获取音频文件的实际长度 |
| 843 | try: |
| 844 | # 尝试使用moviepy获取音频长度 |
| 845 | from moviepy import AudioFileClip |
| 846 | |
| 847 | audio_clip = AudioFileClip(voice_file) |
| 848 | audio_duration = audio_clip.duration |
| 849 | audio_clip.close() |
| 850 | |
| 851 | # 将音频长度转换为100纳秒单位（与edge_tts兼容） |
| 852 | audio_duration_100ns = int(audio_duration * 10000000) |
| 853 | |
| 854 | # 使用文本分割来创建更准确的字幕 |
| 855 | # 将文本按标点符号分割成句子 |
| 856 | sentences = utils.split_string_by_punctuations(text) |
| 857 | |
| 858 | if sentences: |
| 859 | # 计算每个句子的大致时长（按字符数比例分配） |
| 860 | total_chars = sum(len(s) for s in sentences) |
| 861 | char_duration = ( |
| 862 | audio_duration_100ns / total_chars if total_chars > 0 else 0 |
| 863 | ) |
| 864 | |
| 865 | current_offset = 0 |
| 866 | for sentence in sentences: |
| 867 | if not sentence.strip(): |
| 868 | continue |
| 869 | |
| 870 | # 计算当前句子的时长 |
| 871 | sentence_chars = len(sentence) |
| 872 | sentence_duration = int(sentence_chars * char_duration) |
| 873 | |
| 874 | # 添加到SubMaker |
| 875 | sub_maker.subs.append(sentence) |
| 876 | sub_maker.offset.append( |
| 877 | (current_offset, current_offset + sentence_duration) |
| 878 | ) |
| 879 | |
| 880 | # 更新偏移量 |
| 881 | current_offset += sentence_duration |
| 882 | else: |
| 883 | # 如果无法分割，则使用整个文本作为一个字幕 |
| 884 | sub_maker.subs = [text] |
| 885 | sub_maker.offset = [(0, audio_duration_100ns)] |
| 886 | |
| 887 | except Exception as e: |
| 888 | logger.warning(f"Failed to create accurate subtitles: {str(e)}") |
| 889 | # 回退到简单的字幕 |
| 890 | sub_maker.subs = [text] |
| 891 | # 使用音频文件的实际长度，如果无法获取，则假设为10秒 |
| 892 | sub_maker.offset = [ |
| 893 | ( |
| 894 | 0, |
| 895 | audio_duration_100ns |
| 896 | if "audio_duration_100ns" in locals() |
| 897 | else 10000000, |
| 898 | ) |
| 899 | ] |
| 900 | |
| 901 | logger.success(f"siliconflow tts succeeded: {voice_file}") |
| 902 | logger.debug( |
| 903 | "siliconflow subtitle timeline generated, " |
| 904 | f"subs: {len(sub_maker.subs)}, offsets: {len(sub_maker.offset)}" |
| 905 | ) |
| 906 | return sub_maker |
| 907 | else: |
| 908 | logger.error( |
| 909 | f"siliconflow tts failed with status code {response.status_code}: {response.text}" |
| 910 | ) |
| 911 | except Exception as e: |
| 912 | logger.error(f"siliconflow tts failed: {str(e)}") |
| 913 | |
| 914 | return None |
| 915 | |
| 916 | |
| 917 | def azure_tts_v2(text: str, voice_name: str, voice_file: str) -> Union[SubMaker, None]: |
| 918 | voice_name = is_azure_v2_voice(voice_name) |
| 919 | if not voice_name: |
| 920 | logger.error(f"invalid voice name: {voice_name}") |
| 921 | raise ValueError(f"invalid voice name: {voice_name}") |
| 922 | text = text.strip() |
| 923 | |
| 924 | def _format_duration_to_offset(duration) -> int: |
| 925 | if isinstance(duration, str): |
| 926 | time_obj = datetime.strptime(duration, "%H:%M:%S.%f") |
| 927 | milliseconds = ( |
| 928 | (time_obj.hour * 3600000) |
| 929 | + (time_obj.minute * 60000) |
| 930 | + (time_obj.second * 1000) |
| 931 | + (time_obj.microsecond // 1000) |
| 932 | ) |
| 933 | return milliseconds * 10000 |
| 934 | |
| 935 | if isinstance(duration, int): |
| 936 | return duration |
| 937 | |
| 938 | return 0 |
| 939 | |
| 940 | for i in range(3): |
| 941 | try: |
| 942 | logger.info(f"start, voice name: {voice_name}, try: {i + 1}") |
| 943 | |
| 944 | import azure.cognitiveservices.speech as speechsdk |
| 945 | |
| 946 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 947 | |
| 948 | def speech_synthesizer_word_boundary_cb(evt: speechsdk.SessionEventArgs): |
| 949 | # print('WordBoundary event:') |
| 950 | # print('\tBoundaryType: {}'.format(evt.boundary_type)) |
| 951 | # print('\tAudioOffset: {}ms'.format((evt.audio_offset + 5000))) |
| 952 | # print('\tDuration: {}'.format(evt.duration)) |
| 953 | # print('\tText: {}'.format(evt.text)) |
| 954 | # print('\tTextOffset: {}'.format(evt.text_offset)) |
| 955 | # print('\tWordLength: {}'.format(evt.word_length)) |
| 956 | |
| 957 | duration = _format_duration_to_offset(str(evt.duration)) |
| 958 | offset = _format_duration_to_offset(evt.audio_offset) |
| 959 | sub_maker.subs.append(evt.text) |
| 960 | sub_maker.offset.append((offset, offset + duration)) |
| 961 | |
| 962 | # Creates an instance of a speech config with specified subscription key and service region. |
| 963 | speech_key = config.azure.get("speech_key", "") |
| 964 | service_region = config.azure.get("speech_region", "") |
| 965 | if not speech_key or not service_region: |
| 966 | logger.error("Azure speech key or region is not set") |
| 967 | return None |
| 968 | |
| 969 | audio_config = speechsdk.audio.AudioOutputConfig( |
| 970 | filename=voice_file, use_default_speaker=True |
| 971 | ) |
| 972 | speech_config = speechsdk.SpeechConfig( |
| 973 | subscription=speech_key, region=service_region |
| 974 | ) |
| 975 | speech_config.speech_synthesis_voice_name = voice_name |
| 976 | # speech_config.set_property(property_id=speechsdk.PropertyId.SpeechServiceResponse_RequestSentenceBoundary, |
| 977 | # value='true') |
| 978 | speech_config.set_property( |
| 979 | property_id=speechsdk.PropertyId.SpeechServiceResponse_RequestWordBoundary, |
| 980 | value="true", |
| 981 | ) |
| 982 | |
| 983 | speech_config.set_speech_synthesis_output_format( |
| 984 | speechsdk.SpeechSynthesisOutputFormat.Audio48Khz192KBitRateMonoMp3 |
| 985 | ) |
| 986 | speech_synthesizer = speechsdk.SpeechSynthesizer( |
| 987 | audio_config=audio_config, speech_config=speech_config |
| 988 | ) |
| 989 | speech_synthesizer.synthesis_word_boundary.connect( |
| 990 | speech_synthesizer_word_boundary_cb |
| 991 | ) |
| 992 | |
| 993 | result = speech_synthesizer.speak_text_async(text).get() |
| 994 | if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted: |
| 995 | logger.success(f"azure v2 speech synthesis succeeded: {voice_file}") |
| 996 | return sub_maker |
| 997 | elif result.reason == speechsdk.ResultReason.Canceled: |
| 998 | cancellation_details = result.cancellation_details |
| 999 | logger.error( |
| 1000 | f"azure v2 speech synthesis canceled: {cancellation_details.reason}" |
| 1001 | ) |
| 1002 | if cancellation_details.reason == speechsdk.CancellationReason.Error: |
| 1003 | logger.error( |
| 1004 | f"azure v2 speech synthesis error: {cancellation_details.error_details}" |
| 1005 | ) |
| 1006 | logger.info(f"completed, output file: {voice_file}") |
| 1007 | except Exception as e: |
| 1008 | logger.error(f"failed, error: {str(e)}") |
| 1009 | return None |
| 1010 | |
| 1011 | |
| 1012 | def gemini_tts( |
| 1013 | text: str, |
| 1014 | voice_name: str, |
| 1015 | voice_rate: float, |
| 1016 | voice_file: str, |
| 1017 | voice_volume: float = 1.0, |
| 1018 | ) -> Union[SubMaker, None]: |
| 1019 | """ |
| 1020 | 使用Google Gemini TTS生成语音 |
| 1021 | |
| 1022 | Args: |
| 1023 | text: 要转换的文本 |
| 1024 | voice_name: 语音名称，如 "Zephyr", "Puck" 等 |
| 1025 | voice_rate: 语音速率（当前未使用） |
| 1026 | voice_file: 输出音频文件路径 |
| 1027 | voice_volume: 音频音量（当前未使用） |
| 1028 | |
| 1029 | Returns: |
| 1030 | SubMaker对象或None |
| 1031 | """ |
| 1032 | import base64 |
| 1033 | import io |
| 1034 | from pydub import AudioSegment |
| 1035 | import google.generativeai as genai |
| 1036 | _configure_pydub_ffmpeg(AudioSegment) |
| 1037 | |
| 1038 | try: |
| 1039 | # 配置Gemini API |
| 1040 | api_key = config.app.get("gemini_api_key", "") |
| 1041 | if not api_key: |
| 1042 | logger.error("Gemini API key is not set") |
| 1043 | return None |
| 1044 | |
| 1045 | genai.configure(api_key=api_key) |
| 1046 | |
| 1047 | logger.info(f"start, voice name: {voice_name}, try: 1") |
| 1048 | |
| 1049 | # 使用Gemini TTS API |
| 1050 | model = genai.GenerativeModel("gemini-2.5-flash-preview-tts") |
| 1051 | |
| 1052 | generation_config = { |
| 1053 | "response_modalities": ["AUDIO"], |
| 1054 | "speech_config": { |
| 1055 | "voice_config": { |
| 1056 | "prebuilt_voice_config": { |
| 1057 | "voice_name": voice_name |
| 1058 | } |
| 1059 | } |
| 1060 | } |
| 1061 | } |
| 1062 | |
| 1063 | response = model.generate_content( |
| 1064 | contents=text, |
| 1065 | generation_config=generation_config |
| 1066 | ) |
| 1067 | |
| 1068 | # 检查响应 |
| 1069 | if not response.candidates or not response.candidates[0].content: |
| 1070 | logger.error("No audio content received from Gemini TTS") |
| 1071 | return None |
| 1072 | |
| 1073 | # 获取音频数据 |
| 1074 | audio_data = None |
| 1075 | for part in response.candidates[0].content.parts: |
| 1076 | if hasattr(part, 'inline_data') and part.inline_data: |
| 1077 | audio_data = part.inline_data.data |
| 1078 | break |
| 1079 | |
| 1080 | if not audio_data: |
| 1081 | logger.error("No audio data found in response") |
| 1082 | return None |
| 1083 | |
| 1084 | # 音频数据已经是原始字节，不需要base64解码 |
| 1085 | if isinstance(audio_data, str): |
| 1086 | # 如果是字符串，则需要base64解码 |
| 1087 | audio_bytes = base64.b64decode(audio_data) |
| 1088 | else: |
| 1089 | # 如果已经是字节，直接使用 |
| 1090 | audio_bytes = audio_data |
| 1091 | |
| 1092 | # 尝试不同的音频格式 - Gemini可能返回不同的格式 |
| 1093 | audio_segment = None |
| 1094 | |
| 1095 | # Gemini返回Linear PCM格式，按照文档参数解析 |
| 1096 | try: |
| 1097 | audio_segment = AudioSegment.from_file( |
| 1098 | io.BytesIO(audio_bytes), |
| 1099 | format="raw", |
| 1100 | frame_rate=24000, # Gemini TTS默认采样率 |
| 1101 | channels=1, # 单声道 |
| 1102 | sample_width=2 # 16-bit |
| 1103 | ) |
| 1104 | except Exception as e: |
| 1105 | logger.error(f"Failed to load PCM audio: {e}") |
| 1106 | return None |
| 1107 | |
| 1108 | # 导出为MP3格式 |
| 1109 | audio_segment.export(voice_file, format="mp3") |
| 1110 | |
| 1111 | logger.info(f"completed, output file: {voice_file}") |
| 1112 | |
| 1113 | # Gemini 拿不到 edge_tts 那种逐词边界事件，因此这里退回到 |
| 1114 | # 项目原有的 `subs/offset` 兼容结构，至少保证后续字幕与时长 |
| 1115 | # 计算链路可继续工作。 |
| 1116 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 1117 | audio_duration = len(audio_segment) / 1000.0 # 转换为秒 |
| 1118 | return populate_legacy_submaker_with_full_text( |
| 1119 | sub_maker=sub_maker, |
| 1120 | text=text, |
| 1121 | audio_duration_seconds=audio_duration, |
| 1122 | ) |
| 1123 | |
| 1124 | except ImportError as e: |
| 1125 | logger.error(f"Missing required package for Gemini TTS: {str(e)}. Please install: pip install pydub") |
| 1126 | return None |
| 1127 | except Exception as e: |
| 1128 | logger.error(f"Gemini TTS failed, error: {str(e)}") |
| 1129 | return None |
| 1130 | |
| 1131 | |
| 1132 | def mimo_tts( |
| 1133 | text: str, |
| 1134 | voice_name: str, |
| 1135 | voice_rate: float, |
| 1136 | voice_file: str, |
| 1137 | voice_volume: float = 1.0, |
| 1138 | ) -> Union[SubMaker, None]: |
| 1139 | """ |
| 1140 | 使用 Xiaomi MiMo V2.5 TTS 生成语音。 |
| 1141 | |
| 1142 | 官方接口兼容 OpenAI Chat Completions，但 TTS 有两个关键差异： |
| 1143 | 1. 待合成文本必须放在 `assistant` 消息里； |
| 1144 | 2. 音频以 `message.audio.data` 的 base64 字符串返回。 |
| 1145 | |
| 1146 | MiMo 当前没有返回逐词时间轴，因此这里复用项目已有的 legacy |
| 1147 | SubMaker 兜底方案：根据最终音频时长和脚本文本断句生成字幕时间轴。 |
| 1148 | """ |
| 1149 | from pydub import AudioSegment |
| 1150 | |
| 1151 | text = (text or "").strip() |
| 1152 | if not text: |
| 1153 | logger.error("MiMo TTS text is empty") |
| 1154 | return None |
| 1155 | |
| 1156 | api_key = config.app.get("mimo_api_key", "") |
| 1157 | if not api_key: |
| 1158 | logger.error("MiMo API key is not set") |
| 1159 | return None |
| 1160 | |
| 1161 | base_url = config.app.get("mimo_base_url", "") or _MIMO_DEFAULT_BASE_URL |
| 1162 | model_name = config.app.get("mimo_tts_model_name", "") or _MIMO_DEFAULT_TTS_MODEL |
| 1163 | style_prompt = config.app.get( |
| 1164 | "mimo_tts_style_prompt", |
| 1165 | "请用自然、清晰、适合短视频旁白的语气朗读。", |
| 1166 | ) |
| 1167 | |
| 1168 | _configure_pydub_ffmpeg(AudioSegment) |
| 1169 | |
| 1170 | for i in range(3): |
| 1171 | try: |
| 1172 | logger.info( |
| 1173 | f"start mimo tts, model: {model_name}, voice: {voice_name}, try: {i + 1}" |
| 1174 | ) |
| 1175 | ensure_file_path_exists(voice_file) |
| 1176 | |
| 1177 | client = OpenAI(api_key=api_key, base_url=base_url) |
| 1178 | completion = client.chat.completions.create( |
| 1179 | model=model_name, |
| 1180 | messages=[ |
| 1181 | {"role": "user", "content": style_prompt}, |
| 1182 | {"role": "assistant", "content": text}, |
| 1183 | ], |
| 1184 | audio={ |
| 1185 | "format": "wav", |
| 1186 | "voice": voice_name, |
| 1187 | }, |
| 1188 | ) |
| 1189 | |
| 1190 | if not completion or not getattr(completion, "choices", None): |
| 1191 | raise ValueError("MiMo TTS returned empty response") |
| 1192 | |
| 1193 | message = completion.choices[0].message |
| 1194 | audio = getattr(message, "audio", None) |
| 1195 | audio_data = None |
| 1196 | if isinstance(audio, dict): |
| 1197 | audio_data = audio.get("data") |
| 1198 | elif audio is not None: |
| 1199 | audio_data = getattr(audio, "data", None) |
| 1200 | |
| 1201 | if not audio_data: |
| 1202 | raise ValueError("MiMo TTS returned empty audio data") |
| 1203 | |
| 1204 | audio_bytes = base64.b64decode(audio_data) |
| 1205 | audio_segment = AudioSegment.from_file(io.BytesIO(audio_bytes), format="wav") |
| 1206 | |
| 1207 | output_format = utils.parse_extension(voice_file) or "mp3" |
| 1208 | if output_format == "wav": |
| 1209 | with open(voice_file, "wb") as f: |
| 1210 | f.write(audio_bytes) |
| 1211 | else: |
| 1212 | audio_segment.export(voice_file, format=output_format) |
| 1213 | |
| 1214 | audio_duration = len(audio_segment) / 1000.0 |
| 1215 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 1216 | logger.success(f"mimo tts succeeded: {voice_file}") |
| 1217 | logger.debug( |
| 1218 | "mimo subtitle timeline generated, " |
| 1219 | f"duration: {audio_duration:.3f}s, output_format: {output_format}" |
| 1220 | ) |
| 1221 | return populate_legacy_submaker_with_full_text( |
| 1222 | sub_maker=sub_maker, |
| 1223 | text=text, |
| 1224 | audio_duration_seconds=audio_duration, |
| 1225 | ) |
| 1226 | except Exception as e: |
| 1227 | logger.error(f"mimo tts failed: {str(e)}") |
| 1228 | |
| 1229 | return None |
| 1230 | |
| 1231 | |
| 1232 | def elevenlabs_tts( |
| 1233 | text: str, |
| 1234 | voice_id: str, |
| 1235 | voice_file: str, |
| 1236 | voice_rate: float = 1.0, |
| 1237 | voice_volume: float = 1.0, |
| 1238 | model_id: str = "", |
| 1239 | ) -> Union[SubMaker, None]: |
| 1240 | text = (text or "").strip() |
| 1241 | if not text: |
| 1242 | logger.error("ElevenLabs TTS text is empty") |
| 1243 | return None |
| 1244 | |
| 1245 | api_key = config.elevenlabs.get("api_key", "") |
| 1246 | if not api_key: |
| 1247 | logger.error("ElevenLabs API key is not set") |
| 1248 | return None |
| 1249 | |
| 1250 | if not model_id: |
| 1251 | model_id = config.elevenlabs.get("model_id", "eleven_multilingual_v2") |
| 1252 | |
| 1253 | url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}" |
| 1254 | headers = { |
| 1255 | "xi-api-key": api_key, |
| 1256 | "Content-Type": "application/json", |
| 1257 | } |
| 1258 | payload = { |
| 1259 | "text": text, |
| 1260 | "model_id": model_id, |
| 1261 | "voice_settings": { |
| 1262 | "stability": 0.5, |
| 1263 | "similarity_boost": 0.75, |
| 1264 | "style": 0.0, |
| 1265 | "use_speaker_boost": True, |
| 1266 | }, |
| 1267 | } |
| 1268 | |
| 1269 | # Errors where retrying will never help (auth/access/validation failures). |
| 1270 | _NON_RETRYABLE_CODES = {401, 403, 422} |
| 1271 | _NON_RETRYABLE_STATUSES = {"voice_disabled", "voice_access_denied", "unauthorized"} |
| 1272 | |
| 1273 | for i in range(3): |
| 1274 | try: |
| 1275 | logger.info(f"start elevenlabs tts, voice_id: {voice_id}, try: {i + 1}") |
| 1276 | ensure_file_path_exists(voice_file) |
| 1277 | |
| 1278 | response = requests.post(url, json=payload, headers=headers, timeout=60) |
| 1279 | if response.status_code != 200: |
| 1280 | error_status = "" |
| 1281 | try: |
| 1282 | detail = response.json().get("detail", {}) |
| 1283 | if isinstance(detail, dict): |
| 1284 | error_status = detail.get("status", "") |
| 1285 | except Exception: |
| 1286 | pass |
| 1287 | |
| 1288 | if response.status_code in _NON_RETRYABLE_CODES or error_status in _NON_RETRYABLE_STATUSES: |
| 1289 | logger.error( |
| 1290 | f"ElevenLabs TTS failed (non-retryable) — voice_id: {voice_id}, " |
| 1291 | f"status: {response.status_code}, error: {error_status or response.text[:200]}. " |
| 1292 | "Please select a different ElevenLabs voice." |
| 1293 | ) |
| 1294 | return None |
| 1295 | |
| 1296 | logger.error( |
| 1297 | f"elevenlabs tts failed with status {response.status_code}: {response.text[:200]}" |
| 1298 | ) |
| 1299 | continue |
| 1300 | |
| 1301 | with open(voice_file, "wb") as f: |
| 1302 | f.write(response.content) |
| 1303 | |
| 1304 | audio_clip = AudioFileClip(voice_file) |
| 1305 | audio_duration = audio_clip.duration |
| 1306 | audio_clip.close() |
| 1307 | |
| 1308 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 1309 | logger.success(f"elevenlabs tts succeeded: {voice_file}") |
| 1310 | return populate_legacy_submaker_with_full_text( |
| 1311 | sub_maker=sub_maker, |
| 1312 | text=text, |
| 1313 | audio_duration_seconds=audio_duration, |
| 1314 | ) |
| 1315 | except Exception as e: |
| 1316 | logger.error(f"elevenlabs tts failed: {str(e)}") |
| 1317 | |
| 1318 | return None |
| 1319 | |
| 1320 | |
| 1321 | def chatterbox_tts( |
| 1322 | text: str, |
| 1323 | voice: str, |
| 1324 | voice_file: str, |
| 1325 | voice_rate: float = 1.0, |
| 1326 | voice_volume: float = 1.0, |
| 1327 | model_id: str = "", |
| 1328 | ) -> Union[SubMaker, None]: |
| 1329 | """Generate speech with a self-hosted Chatterbox TTS server. |
| 1330 | |
| 1331 | Chatterbox (Resemble AI, MIT) is an open-source, locally hosted TTS model |
| 1332 | with zero-shot voice cloning — a self-hostable alternative to ElevenLabs. |
| 1333 | This talks to an OpenAI-compatible ``/audio/speech`` endpoint, so it works |
| 1334 | with the common community servers (e.g. devnen/Chatterbox-TTS-Server, |
| 1335 | travisvn/chatterbox-tts-api). Configure ``[chatterbox] base_url`` (and an |
| 1336 | optional ``api_key``). |
| 1337 | |
| 1338 | Like ElevenLabs, Chatterbox does not return word-level timestamps, so the |
| 1339 | subtitle path falls back to the full-text SubMaker. For tighter subtitle |
| 1340 | sync set ``subtitle_provider = "whisper"``. |
| 1341 | """ |
| 1342 | text = (text or "").strip() |
| 1343 | if not text: |
| 1344 | logger.error("Chatterbox TTS text is empty") |
| 1345 | return None |
| 1346 | |
| 1347 | base_url = (config.chatterbox.get("base_url", "") or "").strip().rstrip("/") |
| 1348 | if not base_url: |
| 1349 | logger.error( |
| 1350 | "Chatterbox base_url is not set, please configure [chatterbox] base_url in config.toml" |
| 1351 | ) |
| 1352 | return None |
| 1353 | |
| 1354 | api_key = config.chatterbox.get("api_key", "") |
| 1355 | if not model_id: |
| 1356 | model_id = config.chatterbox.get("model_id", "chatterbox") or "chatterbox" |
| 1357 | |
| 1358 | url = f"{base_url}/audio/speech" |
| 1359 | headers = {"Content-Type": "application/json"} |
| 1360 | if api_key: |
| 1361 | headers["Authorization"] = f"Bearer {api_key}" |
| 1362 | payload = { |
| 1363 | "model": model_id, |
| 1364 | "input": text, |
| 1365 | "voice": voice, |
| 1366 | "response_format": "mp3", |
| 1367 | # OpenAI speech API accepts speed 0.25-4.0; MoneyPrinterTurbo's rate is a |
| 1368 | # 1.0-centred multiplier, so it maps directly (clamped to the valid range). |
| 1369 | "speed": max(0.25, min(4.0, float(voice_rate or 1.0))), |
| 1370 | } |
| 1371 | # voice_volume is accepted for parity with the other TTS providers but is |
| 1372 | # intentionally not sent: the OpenAI /audio/speech contract has no volume |
| 1373 | # field, so Chatterbox servers ignore it. Adjust loudness via voice_rate |
| 1374 | # (speed) or in post-processing instead. |
| 1375 | |
| 1376 | for i in range(3): |
| 1377 | try: |
| 1378 | logger.info(f"start chatterbox tts, voice: {voice}, try: {i + 1}") |
| 1379 | ensure_file_path_exists(voice_file) |
| 1380 | |
| 1381 | response = requests.post(url, json=payload, headers=headers, timeout=120) |
| 1382 | if response.status_code != 200: |
| 1383 | logger.error( |
| 1384 | f"chatterbox tts failed with status {response.status_code}: {response.text[:200]}" |
| 1385 | ) |
| 1386 | continue |
| 1387 | |
| 1388 | with open(voice_file, "wb") as f: |
| 1389 | f.write(response.content) |
| 1390 | |
| 1391 | audio_clip = AudioFileClip(voice_file) |
| 1392 | audio_duration = audio_clip.duration |
| 1393 | audio_clip.close() |
| 1394 | |
| 1395 | sub_maker = ensure_legacy_submaker_fields(SubMaker()) |
| 1396 | logger.success(f"chatterbox tts succeeded: {voice_file}") |
| 1397 | return populate_legacy_submaker_with_full_text( |
| 1398 | sub_maker=sub_maker, |
| 1399 | text=text, |
| 1400 | audio_duration_seconds=audio_duration, |
| 1401 | ) |
| 1402 | except Exception as e: |
| 1403 | logger.error(f"chatterbox tts failed: {str(e)}") |
| 1404 | |
| 1405 | return None |
| 1406 | |
| 1407 | |
| 1408 | def _format_text(text: str) -> str: |
| 1409 | """ |
| 1410 | 清理字幕对齐前的脚本文本。 |
| 1411 | |
| 1412 | 这里不能只在 LLM 生成阶段处理，因为用户也可能手动粘贴脚本，或通过 |
| 1413 | API 直接传入包含 Markdown 标记的文本。TTS 通常不会朗读 `---`、 |
| 1414 | `___`、`***` 这类分隔符行，也不会朗读 `_` 这种强调标记；如果字幕 |
| 1415 | 对齐仍保留这些字符，`create_subtitle()` 会一直等待不存在的 cue， |
| 1416 | 最终导致字幕文件缺失并在 Whisper fallback 校正时补出全 0 时间轴。 |
| 1417 | """ |
| 1418 | text = text.replace("[", " ") |
| 1419 | text = text.replace("]", " ") |
| 1420 | text = text.replace("(", " ") |
| 1421 | text = text.replace(")", " ") |
| 1422 | text = text.replace("{", " ") |
| 1423 | text = text.replace("}", " ") |
| 1424 | return utils.normalize_script_for_subtitle_matching(text) |
| 1425 | |
| 1426 | |
| 1427 | def _build_subtitle_formatter(): |
| 1428 | """ |
| 1429 | 返回统一的 SRT 行格式化函数。 |
| 1430 | |
| 1431 | 这里单独拆成一个小工具，是为了让 edge_tts 7.x 的 cues 路径 |
| 1432 | 和项目原有的 legacy `subs/offset` 路径共用同一套字幕落盘格式， |
| 1433 | 避免两套逻辑各自产生细微格式差异。 |
| 1434 | """ |
| 1435 | |
| 1436 | def formatter(idx: int, start_time: float, end_time: float, sub_text: str) -> str: |
| 1437 | start_t = mktimestamp(start_time).replace(".", ",") |
| 1438 | end_t = mktimestamp(end_time).replace(".", ",") |
| 1439 | return f"{idx}\n{start_t} --> {end_t}\n{sub_text}\n" |
| 1440 | |
| 1441 | return formatter |
| 1442 | |
| 1443 | |
| 1444 | # 阿拉伯语变音符号和 Tatweel 拉长符在 edge-tts 返回文本中可能出现， |
| 1445 | # 这些字符不影响语义，但会导致脚本文本和字幕 cue 字符串精确匹配失败。 |
| 1446 | _ARABIC_DIACRITICS = re.compile("[\u0610-\u061A\u064B-\u065F\u0670\u0640\u06D6-\u06ED]") |
| 1447 | |
| 1448 | |
| 1449 | def _normalize_arabic(text: str) -> str: |
| 1450 | """统一阿拉伯语常见字母变体，提升字幕 cue 与脚本行的匹配容错率。 |
| 1451 | |
| 1452 | edge-tts 对阿拉伯语可能返回与原脚本不同的字母形态，例如把 أ/إ/آ |
| 1453 | 归一成 ا，或者携带变音符号。这里仅在最后一层匹配兜底中使用， |
| 1454 | 不改变原始字幕文本，避免影响最终展示内容。 |
| 1455 | """ |
| 1456 | text = _ARABIC_DIACRITICS.sub("", text) |
| 1457 | for src, dst in ( |
| 1458 | ("أإآٱ", "ا"), |
| 1459 | ("ىئ", "ي"), |
| 1460 | ("ة", "ه"), |
| 1461 | ("ؤ", "و"), |
| 1462 | ): |
| 1463 | for ch in src: |
| 1464 | text = text.replace(ch, dst) |
| 1465 | return text |
| 1466 | |
| 1467 | |
| 1468 | def _match_script_line(script_lines: list[str], current_text: str, sub_index: int) -> str: |
| 1469 | """ |
| 1470 | 尝试把当前累计的字幕文本，与脚本中的某一条标准断句匹配起来。 |
| 1471 | |
| 1472 | 这里复用了项目原有的“按标点拆脚本，再逐段比对”的思路： |
| 1473 | 1. 优先精确匹配； |
| 1474 | 2. 再做一次去标点和 Markdown `_` 格式符后的匹配； |
| 1475 | 3. 最后做一次阿拉伯语字符形态归一化匹配。 |
| 1476 | |
| 1477 | 这样可以兼容： |
| 1478 | - TTS 返回里可能缺失或单独拆分的标点； |
| 1479 | - 中文场景下词边界和脚本文本不完全一一对应的情况。 |
| 1480 | """ |
| 1481 | if len(script_lines) <= sub_index: |
| 1482 | return "" |
| 1483 | |
| 1484 | target_line = script_lines[sub_index] |
| 1485 | if current_text == target_line: |
| 1486 | return target_line.strip() |
| 1487 | |
| 1488 | current_text_normalized = re.sub(r"[_\W]+", "", current_text) |
| 1489 | target_line_normalized = re.sub(r"[_\W]+", "", target_line) |
| 1490 | if current_text_normalized == target_line_normalized: |
| 1491 | return target_line.strip() |
| 1492 | |
| 1493 | # 最后一层阿拉伯语容错：edge-tts 返回的字母形态、变音符号或 Tatweel |
| 1494 | # 可能和脚本不同。只在常规匹配失败后归一化比较，非阿拉伯语文本不会受影响。 |
| 1495 | current_ar = re.sub(r"[_\W]+", "", _normalize_arabic(current_text)) |
| 1496 | target_ar = re.sub(r"[_\W]+", "", _normalize_arabic(target_line)) |
| 1497 | if current_ar and current_ar == target_ar: |
| 1498 | return target_line.strip() |
| 1499 | |
| 1500 | return "" |
| 1501 | |
| 1502 | |
| 1503 | def _write_subtitle_items(sub_items: list[str], subtitle_file: str) -> bool: |
| 1504 | """ |
| 1505 | 将已经聚合好的字幕段写入到 SRT 文件，并做一次基本可读性验证。 |
| 1506 | |
| 1507 | 返回值： |
| 1508 | - `True`：字幕文件成功落盘且可被 moviepy 解析； |
| 1509 | - `False`：字幕文件写入或解析失败。 |
| 1510 | """ |
| 1511 | try: |
| 1512 | ensure_file_path_exists(subtitle_file) |
| 1513 | with open(subtitle_file, "w", encoding="utf-8") as file: |
| 1514 | file.write("\n".join(sub_items) + "\n") |
| 1515 | |
| 1516 | sbs = subtitles.file_to_subtitles(subtitle_file, encoding="utf-8") |
| 1517 | duration = max([tb for ((ta, tb), txt) in sbs]) if sbs else 0 |
| 1518 | logger.info( |
| 1519 | f"completed, subtitle file created: {subtitle_file}, duration: {duration}" |
| 1520 | ) |
| 1521 | return True |
| 1522 | except Exception as e: |
| 1523 | logger.error(f"failed, error: {str(e)}") |
| 1524 | if os.path.exists(subtitle_file): |
| 1525 | os.remove(subtitle_file) |
| 1526 | return False |
| 1527 | |
| 1528 | |
| 1529 | def _build_subtitle_items_from_edge_cues( |
| 1530 | sub_maker: SubMaker, script_lines: list[str] |
| 1531 | ) -> list[str]: |
| 1532 | """ |
| 1533 | 将 edge_tts 7.x 的细粒度 `cues` 聚合为按脚本断句的 SRT 片段。 |
| 1534 | |
| 1535 | 背景： |
| 1536 | edge_tts 7.x 的 `SubMaker.get_srt()` 更偏向逐词/逐短语的时间轴。 |
| 1537 | 对英文做逐词高亮尚可，但中文短视频字幕如果直接照搬，会出现 |
| 1538 | “金钱 / 是 / 一种 / 社会 / 工具” 这种阅读体验很差的效果。 |
| 1539 | |
| 1540 | 实现策略： |
| 1541 | 1. 逐个消费 cues 中的 `content`； |
| 1542 | 2. 累积成一段候选文本； |
| 1543 | 3. 当候选文本与脚本里当前目标断句匹配时，收敛为一个完整字幕段； |
| 1544 | 4. 使用第一条 cue 的开始时间和最后一条 cue 的结束时间，保证时间轴连续。 |
| 1545 | """ |
| 1546 | formatter = _build_subtitle_formatter() |
| 1547 | sub_items = [] |
| 1548 | sub_index = 0 |
| 1549 | current_text = "" |
| 1550 | current_start_time = None |
| 1551 | |
| 1552 | for cue in sub_maker.cues: |
| 1553 | cue_text = unescape(cue.content) |
| 1554 | if current_start_time is None: |
| 1555 | current_start_time = int(cue.start.total_seconds() * 10000000) |
| 1556 | |
| 1557 | current_end_time = int(cue.end.total_seconds() * 10000000) |
| 1558 | current_text += cue_text |
| 1559 | |
| 1560 | matched_text = _match_script_line(script_lines, current_text, sub_index) |
| 1561 | if not matched_text: |
| 1562 | continue |
| 1563 | |
| 1564 | sub_index += 1 |
| 1565 | sub_items.append( |
| 1566 | formatter( |
| 1567 | idx=sub_index, |
| 1568 | start_time=current_start_time, |
| 1569 | end_time=current_end_time, |
| 1570 | sub_text=matched_text, |
| 1571 | ) |
| 1572 | ) |
| 1573 | current_text = "" |
| 1574 | current_start_time = None |
| 1575 | |
| 1576 | if current_text.strip(): |
| 1577 | logger.warning( |
| 1578 | f"edge cues still have unmatched text after aggregation: {current_text}" |
| 1579 | ) |
| 1580 | |
| 1581 | return sub_items |
| 1582 | |
| 1583 | |
| 1584 | def _build_subtitle_items_from_legacy_submaker( |
| 1585 | sub_maker: SubMaker, script_lines: list[str] |
| 1586 | ) -> list[str]: |
| 1587 | """ |
| 1588 | 将项目原有 `subs/offset` 结构聚合为按脚本断句的 SRT 片段。 |
| 1589 | |
| 1590 | 这部分保留了原来的核心思路，只是拆成独立函数，便于与 edge_tts 7.x |
| 1591 | 的 cues 聚合逻辑共享同一套断句匹配与落盘流程。 |
| 1592 | """ |
| 1593 | formatter = _build_subtitle_formatter() |
| 1594 | start_time = -1.0 |
| 1595 | sub_items = [] |
| 1596 | sub_index = 0 |
| 1597 | sub_line = "" |
| 1598 | |
| 1599 | legacy_offsets = getattr(sub_maker, "offset", []) |
| 1600 | legacy_subs = getattr(sub_maker, "subs", []) |
| 1601 | for _, (offset, sub) in enumerate(zip(legacy_offsets, legacy_subs)): |
| 1602 | current_start_time, current_end_time = offset |
| 1603 | if start_time < 0: |
| 1604 | start_time = current_start_time |
| 1605 | |
| 1606 | sub_line += unescape(sub) |
| 1607 | matched_text = _match_script_line(script_lines, sub_line, sub_index) |
| 1608 | if not matched_text: |
| 1609 | continue |
| 1610 | |
| 1611 | sub_index += 1 |
| 1612 | sub_items.append( |
| 1613 | formatter( |
| 1614 | idx=sub_index, |
| 1615 | start_time=start_time, |
| 1616 | end_time=current_end_time, |
| 1617 | sub_text=matched_text, |
| 1618 | ) |
| 1619 | ) |
| 1620 | start_time = -1.0 |
| 1621 | sub_line = "" |
| 1622 | |
| 1623 | if sub_line.strip(): |
| 1624 | logger.warning( |
| 1625 | f"legacy subtitle items still have unmatched text after aggregation: {sub_line}" |
| 1626 | ) |
| 1627 | |
| 1628 | return sub_items |
| 1629 | |
| 1630 | |
| 1631 | def create_subtitle(sub_maker: SubMaker, text: str, subtitle_file: str): |
| 1632 | """ |
| 1633 | 优化字幕文件 |
| 1634 | 1. 将字幕文件按照标点符号分割成多行 |
| 1635 | 2. 逐行匹配字幕文件中的文本 |
| 1636 | 3. 生成新的字幕文件 |
| 1637 | """ |
| 1638 | text = _format_text(text) |
| 1639 | script_lines = utils.split_string_by_punctuations(text) |
| 1640 | try: |
| 1641 | if hasattr(sub_maker, "cues") and sub_maker.cues: |
| 1642 | sub_items = _build_subtitle_items_from_edge_cues(sub_maker, script_lines) |
| 1643 | else: |
| 1644 | sub_items = _build_subtitle_items_from_legacy_submaker( |
| 1645 | sub_maker, script_lines |
| 1646 | ) |
| 1647 | |
| 1648 | if len(sub_items) != len(script_lines): |
| 1649 | logger.warning( |
| 1650 | f"failed, sub_items len: {len(sub_items)}, script_lines len: {len(script_lines)}" |
| 1651 | ) |
| 1652 | return |
| 1653 | |
| 1654 | _write_subtitle_items(sub_items, subtitle_file) |
| 1655 | except Exception as e: |
| 1656 | logger.error(f"failed, error: {str(e)}") |
| 1657 | |
| 1658 | |
| 1659 | def _get_audio_duration_from_submaker(sub_maker: SubMaker): |
| 1660 | """ |
| 1661 | 获取音频时长 |
| 1662 | """ |
| 1663 | # 优先兼容 edge_tts 7.x 的 cues 结构； |
| 1664 | # 如果是项目里其他 TTS 手工填充的旧结构，则继续读取 offset。 |
| 1665 | if hasattr(sub_maker, "cues") and sub_maker.cues: |
| 1666 | return sub_maker.cues[-1].end.total_seconds() |
| 1667 | |
| 1668 | legacy_offsets = getattr(sub_maker, "offset", []) |
| 1669 | if not legacy_offsets: |
| 1670 | return 0.0 |
| 1671 | return legacy_offsets[-1][1] / 10000000 |
| 1672 | |
| 1673 | def _get_audio_duration_from_mp3(mp3_file: str) -> float: |
| 1674 | """ |
| 1675 | 获取MP3音频时长 |
| 1676 | """ |
| 1677 | if not os.path.exists(mp3_file): |
| 1678 | logger.error(f"MP3 file does not exist: {mp3_file}") |
| 1679 | return 0.0 |
| 1680 | |
| 1681 | try: |
| 1682 | # Use moviepy to get the duration of the MP3 file |
| 1683 | with AudioFileClip(mp3_file) as audio: |
| 1684 | return audio.duration # Duration in seconds |
| 1685 | except Exception as e: |
| 1686 | logger.error(f"Failed to get audio duration from MP3: {str(e)}") |
| 1687 | return 0.0 |
| 1688 | |
| 1689 | def get_audio_duration(target: Union[str, SubMaker]) -> float: |
| 1690 | """ |
| 1691 | 获取音频时长 |
| 1692 | 如果是SubMaker对象，则从SubMaker中获取时长 |
| 1693 | 如果是MP3文件，则从MP3文件中获取时长 |
| 1694 | """ |
| 1695 | if isinstance(target, SubMaker): |
| 1696 | return _get_audio_duration_from_submaker(target) |
| 1697 | elif isinstance(target, str) and target.endswith(".mp3"): |
| 1698 | return _get_audio_duration_from_mp3(target) |
| 1699 | else: |
| 1700 | logger.error(f"Invalid target type: {type(target)}") |
| 1701 | return 0.0 |
| 1702 | |
| 1703 | if __name__ == "__main__": |
| 1704 | voice_name = "zh-CN-XiaoxiaoMultilingualNeural-V2-Female" |
| 1705 | voice_name = parse_voice_name(voice_name) |
| 1706 | voice_name = is_azure_v2_voice(voice_name) |
| 1707 | print(voice_name) |
| 1708 | |
| 1709 | voices = get_all_azure_voices() |
| 1710 | print(len(voices)) |
| 1711 | |
| 1712 | async def _do(): |
| 1713 | temp_dir = utils.storage_dir("temp") |
| 1714 | |
| 1715 | voice_names = [ |
| 1716 | "zh-CN-XiaoxiaoMultilingualNeural", |
| 1717 | # 女性 |
| 1718 | "zh-CN-XiaoxiaoNeural", |
| 1719 | "zh-CN-XiaoyiNeural", |
| 1720 | # 男性 |
| 1721 | "zh-CN-YunyangNeural", |
| 1722 | "zh-CN-YunxiNeural", |
| 1723 | ] |
| 1724 | text = """ |
| 1725 | 静夜思是唐代诗人李白创作的一首五言古诗。这首诗描绘了诗人在寂静的夜晚，看到窗前的明月，不禁想起远方的家乡和亲人，表达了他对家乡和亲人的深深思念之情。全诗内容是：“床前明月光，疑是地上霜。举头望明月，低头思故乡。”在这短短的四句诗中，诗人通过“明月”和“思故乡”的意象，巧妙地表达了离乡背井人的孤独与哀愁。首句“床前明月光”设景立意，通过明亮的月光引出诗人的遐想；“疑是地上霜”增添了夜晚的寒冷感，加深了诗人的孤寂之情；“举头望明月”和“低头思故乡”则是情感的升华，展现了诗人内心深处的乡愁和对家的渴望。这首诗简洁明快，情感真挚，是中国古典诗歌中非常著名的一首，也深受后人喜爱和推崇。 |
| 1726 | """ |
| 1727 | |
| 1728 | text = """ |
| 1729 | What is the meaning of life? This question has puzzled philosophers, scientists, and thinkers of all kinds for centuries. Throughout history, various cultures and individuals have come up with their interpretations and beliefs around the purpose of life. Some say it's to seek happiness and self-fulfillment, while others believe it's about contributing to the welfare of others and making a positive impact in the world. Despite the myriad of perspectives, one thing remains clear: the meaning of life is a deeply personal concept that varies from one person to another. It's an existential inquiry that encourages us to reflect on our values, desires, and the essence of our existence. |
| 1730 | """ |
| 1731 | |
| 1732 | text = """ |
| 1733 | 预计未来3天深圳冷空气活动频繁，未来两天持续阴天有小雨，出门带好雨具； |
| 1734 | 10-11日持续阴天有小雨，日温差小，气温在13-17℃之间，体感阴凉； |
| 1735 | 12日天气短暂好转，早晚清凉； |
| 1736 | """ |
| 1737 | |
| 1738 | text = "[Opening scene: A sunny day in a suburban neighborhood. A young boy named Alex, around 8 years old, is playing in his front yard with his loyal dog, Buddy.]\n\n[Camera zooms in on Alex as he throws a ball for Buddy to fetch. Buddy excitedly runs after it and brings it back to Alex.]\n\nAlex: Good boy, Buddy! You're the best dog ever!\n\n[Buddy barks happily and wags his tail.]\n\n[As Alex and Buddy continue playing, a series of potential dangers loom nearby, such as a stray dog approaching, a ball rolling towards the street, and a suspicious-looking stranger walking by.]\n\nAlex: Uh oh, Buddy, look out!\n\n[Buddy senses the danger and immediately springs into action. He barks loudly at the stray dog, scaring it away. Then, he rushes to retrieve the ball before it reaches the street and gently nudges it back towards Alex. Finally, he stands protectively between Alex and the stranger, growling softly to warn them away.]\n\nAlex: Wow, Buddy, you're like my superhero!\n\n[Just as Alex and Buddy are about to head inside, they hear a loud crash from a nearby construction site. They rush over to investigate and find a pile of rubble blocking the path of a kitten trapped underneath.]\n\nAlex: Oh no, Buddy, we have to help!\n\n[Buddy barks in agreement and together they work to carefully move the rubble aside, allowing the kitten to escape unharmed. The kitten gratefully nuzzles against Buddy, who responds with a friendly lick.]\n\nAlex: We did it, Buddy! We saved the day again!\n\n[As Alex and Buddy walk home together, the sun begins to set, casting a warm glow over the neighborhood.]\n\nAlex: Thanks for always being there to watch over me, Buddy. You're not just my dog, you're my best friend.\n\n[Buddy barks happily and nuzzles against Alex as they disappear into the sunset, ready to face whatever adventures tomorrow may bring.]\n\n[End scene.]" |
| 1739 | |
| 1740 | text = "大家好，我是乔哥，一个想帮你把信用卡全部还清的家伙！\n今天我们要聊的是信用卡的取现功能。\n你是不是也曾经因为一时的资金紧张，而拿着信用卡到ATM机取现？如果是，那你得好好看看这个视频了。\n现在都2024年了，我以为现在不会再有人用信用卡取现功能了。前几天一个粉丝发来一张图片，取现1万。\n信用卡取现有三个弊端。\n一，信用卡取现功能代价可不小。会先收取一个取现手续费，比如这个粉丝，取现1万，按2.5%收取手续费，收取了250元。\n二，信用卡正常消费有最长56天的免息期，但取现不享受免息期。从取现那一天开始，每天按照万5收取利息，这个粉丝用了11天，收取了55元利息。\n三，频繁的取现行为，银行会认为你资金紧张，会被标记为高风险用户，影响你的综合评分和额度。\n那么，如果你资金紧张了，该怎么办呢？\n乔哥给你支一招，用破思机摩擦信用卡，只需要少量的手续费，而且还可以享受最长56天的免息期。\n最后，如果你对玩卡感兴趣，可以找乔哥领取一本《卡神秘籍》，用卡过程中遇到任何疑惑，也欢迎找乔哥交流。\n别忘了，关注乔哥，回复用卡技巧，免费领取《2024用卡技巧》，让我们一起成为用卡高手！" |
| 1741 | |
| 1742 | text = """ |
| 1743 | 2023全年业绩速览 |
| 1744 | 公司全年累计实现营业收入1476.94亿元，同比增长19.01%，归母净利润747.34亿元，同比增长19.16%。EPS达到59.49元。第四季度单季，营业收入444.25亿元，同比增长20.26%，环比增长31.86%；归母净利润218.58亿元，同比增长19.33%，环比增长29.37%。这一阶段 |
| 1745 | 的业绩表现不仅突显了公司的增长动力和盈利能力，也反映出公司在竞争激烈的市场环境中保持了良好的发展势头。 |
| 1746 | 2023年Q4业绩速览 |
| 1747 | 第四季度，营业收入贡献主要增长点；销售费用高增致盈利能力承压；税金同比上升27%，扰动净利率表现。 |
| 1748 | 业绩解读 |
| 1749 | 利润方面，2023全年贵州茅台，>归母净利润增速为19%，其中营业收入正贡献18%，营业成本正贡献百分之一，管理费用正贡献百分之一点四。(注：归母净利润增速值=营业收入增速+各科目贡献，展示贡献/拖累的前四名科目，且要求贡献值/净利润增速>15%) |
| 1750 | """ |
| 1751 | text = "静夜思是唐代诗人李白创作的一首五言古诗。这首诗描绘了诗人在寂静的夜晚，看到窗前的明月，不禁想起远方的家乡和亲人" |
| 1752 | |
| 1753 | text = _format_text(text) |
| 1754 | lines = utils.split_string_by_punctuations(text) |
| 1755 | print(lines) |
| 1756 | |
| 1757 | for voice_name in voice_names: |
| 1758 | voice_file = f"{temp_dir}/tts-{voice_name}.mp3" |
| 1759 | subtitle_file = f"{temp_dir}/tts.mp3.srt" |
| 1760 | sub_maker = azure_tts_v2( |
| 1761 | text=text, voice_name=voice_name, voice_file=voice_file |
| 1762 | ) |
| 1763 | create_subtitle(sub_maker=sub_maker, text=text, subtitle_file=subtitle_file) |
| 1764 | audio_duration = get_audio_duration(sub_maker) |
| 1765 | print(f"voice: {voice_name}, audio duration: {audio_duration}s") |
| 1766 | |
| 1767 | loop = asyncio.get_event_loop_policy().get_event_loop() |
| 1768 | try: |
| 1769 | loop.run_until_complete(_do()) |
| 1770 | finally: |
| 1771 | loop.close() |
| 1772 |