| 1 | import * as fs from 'node:fs' |
| 2 | import * as os from 'node:os' |
| 3 | import * as path from 'node:path' |
| 4 | import { createSdkMcpServer, McpSdkServerConfigWithInstance } from '@anthropic-ai/claude-agent-sdk' |
| 5 | import { Injectable, Logger } from '@nestjs/common' |
| 6 | import { AssetsService, VideoMetadataService } from '@yikart/assets' |
| 7 | import { FileUtil, UserType } from '@yikart/common' |
| 8 | import { AssetType } from '@yikart/mongodb' |
| 9 | import { execa } from 'execa' |
| 10 | import * as subtitle from 'subtitle' |
| 11 | import { z } from 'zod' |
| 12 | import { AiAvailabilityService } from '../../ai-availability' |
| 13 | import { ChatService } from '../../ai/chat' |
| 14 | import { VolcengineService } from '../../ai/libs/volcengine' |
| 15 | import { McpServerName } from '../agent.constants' |
| 16 | import { errorResult, srtTimestampToMs, successResult, wrapTool } from './mcp.utils' |
| 17 | |
| 18 | const getVolcVideoInfoSchema = z.object({ |
| 19 | vids: z.array(z.string()).min(1).describe('Volcengine video VID list, supports vid://xxx format or plain VID string'), |
| 20 | }) |
| 21 | |
| 22 | const uploadAndGetVidSchema = z.object({ |
| 23 | videoUrl: FileUtil.zodBuildUrl().nonoptional().describe('Video URL'), |
| 24 | }) |
| 25 | |
| 26 | const probeVideoMetadataSchema = z.object({ |
| 27 | videoUrl: FileUtil.zodBuildUrl().nonoptional().describe('Video URL'), |
| 28 | }) |
| 29 | |
| 30 | const extractThumbnailSchema = z.object({ |
| 31 | videoUrl: FileUtil.zodBuildUrl().nonoptional().describe('Video URL'), |
| 32 | timeInSeconds: z.number().min(0).default(1).describe('Time point in seconds to extract frame, default 1 second'), |
| 33 | }) |
| 34 | |
| 35 | const generateSubtitleSchema = z.object({ |
| 36 | mediaUrl: FileUtil.zodBuildUrl().nonoptional().describe('Video or audio URL to transcribe'), |
| 37 | language: z.string().optional().describe('Target language code, e.g., "zh-CN", "en-US"'), |
| 38 | }) |
| 39 | |
| 40 | const srtNodesSchema = z.object({ |
| 41 | start: z.string().describe('Start time in SRT format: HH:MM:SS,sss'), |
| 42 | end: z.string().describe('End time in SRT format: HH:MM:SS,sss'), |
| 43 | text: z.string().describe('Subtitle text'), |
| 44 | }).array() |
| 45 | |
| 46 | export enum VideoUtilsToolName { |
| 47 | GetVolcVideoInfo = 'getVolcVideoInfo', |
| 48 | UploadAndGetVid = 'uploadAndGetVid', |
| 49 | ProbeVideoMetadata = 'probeVideoMetadata', |
| 50 | ExtractThumbnail = 'extractThumbnail', |
| 51 | GenerateSubtitle = 'generateSubtitle', |
| 52 | } |
| 53 | |
| 54 | interface VideoInfoSuccess { |
| 55 | vid: string |
| 56 | duration: number |
| 57 | width: number |
| 58 | height: number |
| 59 | fileSize: number |
| 60 | format: string |
| 61 | bitrate: number |
| 62 | frameRate: number |
| 63 | codecName: string |
| 64 | } |
| 65 | |
| 66 | interface VideoInfoError { |
| 67 | vid: string |
| 68 | error: string |
| 69 | } |
| 70 | |
| 71 | @Injectable() |
| 72 | export class VideoUtilsMcp { |
| 73 | private readonly logger = new Logger(VideoUtilsMcp.name) |
| 74 | |
| 75 | constructor( |
| 76 | private readonly volcengineService: VolcengineService, |
| 77 | private readonly assetsService: AssetsService, |
| 78 | private readonly aiAvailability: AiAvailabilityService, |
| 79 | private readonly videoMetadataService: VideoMetadataService, |
| 80 | private readonly chatService: ChatService, |
| 81 | ) {} |
| 82 | |
| 83 | /** |
| 84 | * 获取火山引擎视频信息（仅支持 VID） |
| 85 | */ |
| 86 | createGetVolcVideoInfoTool(_userId: string, _userType: UserType) { |
| 87 | return wrapTool( |
| 88 | this.logger, |
| 89 | VideoUtilsToolName.GetVolcVideoInfo, |
| 90 | `Get video information from Volcengine by VID. Use this after uploading video to Volcengine. |
| 91 | |
| 92 | Use this tool to: |
| 93 | - Get video duration for time calculations |
| 94 | - Get video dimensions for canvas setup |
| 95 | - Get video metadata (format, bitrate, fps) |
| 96 | |
| 97 | Input: Array of VIDs (supports "vid://xxx" format or plain VID string) |
| 98 | Returns: Array of { vid, duration, width, height, format, ... }`, |
| 99 | getVolcVideoInfoSchema.shape, |
| 100 | async ({ vids }) => { |
| 101 | const normalizedVids = vids.map((vid) => { |
| 102 | if (vid.startsWith('vid://')) { |
| 103 | return vid.replace('vid://', '') |
| 104 | } |
| 105 | return vid |
| 106 | }) |
| 107 | |
| 108 | const mediaInfos = await this.volcengineService.getMediaInfos({ |
| 109 | Vids: normalizedVids.join(','), |
| 110 | }) |
| 111 | |
| 112 | const successResults: VideoInfoSuccess[] = [] |
| 113 | const errorResults: VideoInfoError[] = [] |
| 114 | |
| 115 | for (let index = 0; index < normalizedVids.length; index++) { |
| 116 | const vid = normalizedVids[index] |
| 117 | const mediaInfo = mediaInfos.MediaInfoList?.[index] |
| 118 | |
| 119 | if (!mediaInfo) { |
| 120 | errorResults.push({ vid, error: 'Video not found or inaccessible' }) |
| 121 | continue |
| 122 | } |
| 123 | |
| 124 | const sourceInfo = mediaInfo.SourceInfo |
| 125 | successResults.push({ |
| 126 | vid, |
| 127 | duration: sourceInfo?.Duration || 0, |
| 128 | width: sourceInfo?.Width || 0, |
| 129 | height: sourceInfo?.Height || 0, |
| 130 | fileSize: sourceInfo?.Size || 0, |
| 131 | format: sourceInfo?.Format || 'unknown', |
| 132 | bitrate: sourceInfo?.Bitrate || 0, |
| 133 | frameRate: sourceInfo?.Fps || 0, |
| 134 | codecName: sourceInfo?.Codec || 'unknown', |
| 135 | }) |
| 136 | } |
| 137 | |
| 138 | if (successResults.length === 0) { |
| 139 | return errorResult(`All videos failed:\n${errorResults.map(r => `- vid://${r.vid}: ${r.error}`).join('\n')}`) |
| 140 | } |
| 141 | |
| 142 | const infoMessages = successResults.map((videoInfo) => { |
| 143 | const durationMs = videoInfo.duration * 1000 |
| 144 | |
| 145 | return `**Video: vid://${videoInfo.vid}** |
| 146 | - Duration: ${videoInfo.duration} seconds (${durationMs} milliseconds) |
| 147 | - Resolution: ${videoInfo.width}x${videoInfo.height} |
| 148 | - File Size: ${(videoInfo.fileSize / 1024 / 1024).toFixed(2)} MB |
| 149 | - Format: ${videoInfo.format} |
| 150 | - Bitrate: ${(videoInfo.bitrate / 1000).toFixed(0)} kbps |
| 151 | - Frame Rate: ${videoInfo.frameRate} fps |
| 152 | - Codec: ${videoInfo.codecName} |
| 153 | |
| 154 | **For video editing:** |
| 155 | - Use Source: "vid://${videoInfo.vid}" |
| 156 | - Video dimensions: ${videoInfo.width}x${videoInfo.height} (Canvas will be auto-detected if omitted in submitDirectEditTask) |
| 157 | - Full video TargetTime: [0, ${durationMs}]` |
| 158 | }).join('\n\n---\n\n') |
| 159 | |
| 160 | let message = `Video Information Retrieved (${successResults.length}/${normalizedVids.length} successful):\n\n${infoMessages}` |
| 161 | |
| 162 | if (errorResults.length > 0) { |
| 163 | message += `\n\n**Failed videos:**\n${errorResults.map(r => `- vid://${r.vid}: ${r.error}`).join('\n')}` |
| 164 | } |
| 165 | |
| 166 | return successResult(message) |
| 167 | }, |
| 168 | this.aiAvailability, |
| 169 | ) |
| 170 | } |
| 171 | |
| 172 | /** |
| 173 | * 上传视频到火山引擎并获取 VID |
| 174 | */ |
| 175 | createUploadAndGetVidTool(_userId: string, _userType: UserType) { |
| 176 | return wrapTool( |
| 177 | this.logger, |
| 178 | VideoUtilsToolName.UploadAndGetVid, |
| 179 | `Upload a video from URL to Volcengine cloud and get VID. Required before video editing. |
| 180 | |
| 181 | Use this tool to: |
| 182 | - Convert HTTP URL to VID for Volcengine video editing |
| 183 | - Get video information after upload |
| 184 | |
| 185 | Returns: { vid, duration, width, height, ... }`, |
| 186 | uploadAndGetVidSchema.shape, |
| 187 | async ({ videoUrl }) => { |
| 188 | this.logger.debug({ url: videoUrl }, '[UploadAndGetVid] Uploading video URL to Volcengine') |
| 189 | |
| 190 | const vid = await this.volcengineService.downloadUrlAndUploadAsStream(videoUrl) |
| 191 | this.logger.debug({ vid }, '[UploadAndGetVid] Video uploaded successfully') |
| 192 | |
| 193 | const mediaInfos = await this.volcengineService.getMediaInfos({ |
| 194 | Vids: vid, |
| 195 | }) |
| 196 | |
| 197 | const mediaInfo = mediaInfos.MediaInfoList?.[0] |
| 198 | if (!mediaInfo) { |
| 199 | return errorResult('Video uploaded but failed to retrieve information') |
| 200 | } |
| 201 | |
| 202 | const sourceInfo = mediaInfo.SourceInfo |
| 203 | const duration = sourceInfo?.Duration || 0 |
| 204 | const durationMs = duration * 1000 |
| 205 | |
| 206 | return successResult(`Video uploaded successfully! |
| 207 | |
| 208 | **VID:** vid://${vid} |
| 209 | **Duration:** ${duration} seconds (${durationMs} milliseconds) |
| 210 | **Resolution:** ${sourceInfo?.Width || 0}x${sourceInfo?.Height || 0} |
| 211 | **Format:** ${sourceInfo?.Format || 'unknown'} |
| 212 | **Bitrate:** ${((sourceInfo?.Bitrate || 0) / 1000).toFixed(0)} kbps |
| 213 | **Frame Rate:** ${sourceInfo?.Fps || 0} fps |
| 214 | |
| 215 | **For video editing:** |
| 216 | - Use Source: "vid://${vid}" |
| 217 | - Video dimensions: ${sourceInfo?.Width || 0}x${sourceInfo?.Height || 0} (Canvas will be auto-detected if omitted in submitDirectEditTask) |
| 218 | - Full video TargetTime: [0, ${durationMs}]`) |
| 219 | }, |
| 220 | this.aiAvailability, |
| 221 | ) |
| 222 | } |
| 223 | |
| 224 | /** |
| 225 | * 使用 ffprobe 获取视频元数据（不上传） |
| 226 | */ |
| 227 | createProbeVideoMetadataTool(_userId: string, _userType: UserType) { |
| 228 | return wrapTool( |
| 229 | this.logger, |
| 230 | VideoUtilsToolName.ProbeVideoMetadata, |
| 231 | `Probe video metadata from URL using ffprobe. Does NOT upload the video. |
| 232 | |
| 233 | Use this tool when you only need video metadata (duration, dimensions) without editing: |
| 234 | - Display video information to user |
| 235 | - Check video properties before processing |
| 236 | - Validate video format |
| 237 | |
| 238 | Returns: { width, height, duration, bitrate, frameRate }`, |
| 239 | probeVideoMetadataSchema.shape, |
| 240 | async ({ videoUrl }) => { |
| 241 | this.logger.debug({ url: videoUrl }, '[ProbeVideoMetadata] Probing video metadata') |
| 242 | |
| 243 | const metadata = await this.videoMetadataService.probeVideoMetadata(videoUrl) |
| 244 | |
| 245 | return successResult(`Video Metadata: |
| 246 | - Resolution: ${metadata.width}x${metadata.height} |
| 247 | - Duration: ${metadata.duration} seconds (${metadata.duration * 1000} milliseconds) |
| 248 | - Bitrate: ${(metadata.bitrate / 1000).toFixed(0)} kbps |
| 249 | - Frame Rate: ${metadata.frameRate} fps |
| 250 | |
| 251 | Note: This video has NOT been uploaded to Volcengine. Use uploadAndGetVid if you need to edit it.`) |
| 252 | }, |
| 253 | this.aiAvailability, |
| 254 | ) |
| 255 | } |
| 256 | |
| 257 | /** |
| 258 | * 提取视频缩略图 |
| 259 | */ |
| 260 | createExtractThumbnailTool(userId: string, _userType: UserType) { |
| 261 | return wrapTool( |
| 262 | this.logger, |
| 263 | VideoUtilsToolName.ExtractThumbnail, |
| 264 | 'Extract a thumbnail from a video URL at a specified time point. Uses ffmpeg to directly stream from the URL without downloading the full video. Returns the uploaded thumbnail URL.', |
| 265 | extractThumbnailSchema.shape, |
| 266 | async ({ videoUrl, timeInSeconds }) => { |
| 267 | this.logger.debug({ userId, videoUrl, timeInSeconds }, '[extractThumbnail] Starting thumbnail extraction') |
| 268 | |
| 269 | const thumbnailBuffer = await this.videoMetadataService.extractThumbnailFromUrl(videoUrl, timeInSeconds) |
| 270 | this.logger.debug({ size: thumbnailBuffer.length }, '[extractThumbnail] Frame extracted') |
| 271 | |
| 272 | const result = await this.assetsService.uploadFromBuffer(userId, thumbnailBuffer, { |
| 273 | type: AssetType.VideoThumbnail, |
| 274 | mimeType: 'image/png', |
| 275 | filename: 'thumbnail.png', |
| 276 | }) |
| 277 | |
| 278 | this.logger.debug({ thumbnailUrl: result.url }, '[extractThumbnail] Thumbnail uploaded') |
| 279 | |
| 280 | return successResult(`Thumbnail extracted and uploaded successfully. URL: ${result.url}`) |
| 281 | }, |
| 282 | this.aiAvailability, |
| 283 | ) |
| 284 | } |
| 285 | |
| 286 | /** |
| 287 | * 使用 FFmpeg 从 URL 提取音频 |
| 288 | */ |
| 289 | private async extractAudioWithFFmpeg(mediaUrl: string): Promise<Buffer> { |
| 290 | const tempDir = os.tmpdir() |
| 291 | const outputPath = path.join(tempDir, `audio-${Date.now()}.mp3`) |
| 292 | try { |
| 293 | await execa('ffmpeg', [ |
| 294 | '-i', |
| 295 | mediaUrl, |
| 296 | '-map', |
| 297 | '0:a:0', |
| 298 | '-vn', |
| 299 | '-acodec', |
| 300 | 'libmp3lame', |
| 301 | '-b:a', |
| 302 | '128k', |
| 303 | '-y', |
| 304 | outputPath, |
| 305 | ]) |
| 306 | |
| 307 | return fs.readFileSync(outputPath) |
| 308 | } |
| 309 | finally { |
| 310 | if (fs.existsSync(outputPath)) { |
| 311 | fs.unlinkSync(outputPath) |
| 312 | } |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | /** |
| 317 | * 构建字幕转录提示词 |
| 318 | */ |
| 319 | private buildSubtitlePrompt(language?: string): string { |
| 320 | const languageInstruction = language |
| 321 | ? `Transcribe the audio and translate into ${language}. All subtitle text must be in ${language}.` |
| 322 | : `Transcribe the audio in its original spoken language.` |
| 323 | |
| 324 | return `${languageInstruction} |
| 325 | |
| 326 | ## Rules |
| 327 | |
| 328 | 1. **Accuracy**: Only transcribe speech actually present in the audio. Never add or fabricate content. |
| 329 | |
| 330 | 2. **Timestamps**: |
| 331 | - Use SRT format: HH:MM:SS,sss (e.g., 00:00:01,500) |
| 332 | - Timestamps must accurately match when speech occurs in the audio |
| 333 | - The last subtitle should end near the actual end of speech |
| 334 | - Minimum subtitle duration: 1.5 seconds for readability |
| 335 | |
| 336 | 3. **Segmentation**: Split at natural speech pauses. Each subtitle should be a complete phrase. |
| 337 | |
| 338 | 4. **Translation** (if applicable): Translate naturally while preserving the original meaning. |
| 339 | |
| 340 | ## Output Format |
| 341 | |
| 342 | JSON array with SRT timestamps: |
| 343 | [ |
| 344 | {"start": "00:00:00,500", "end": "00:00:03,000", "text": "Hello everyone."}, |
| 345 | {"start": "00:00:03,500", "end": "00:00:06,000", "text": "Welcome to today's video."} |
| 346 | ]` |
| 347 | } |
| 348 | |
| 349 | /** |
| 350 | * 生成字幕 |
| 351 | */ |
| 352 | createGenerateSubtitleTool(userId: string, userType: UserType) { |
| 353 | return wrapTool( |
| 354 | this.logger, |
| 355 | VideoUtilsToolName.GenerateSubtitle, |
| 356 | `Generate SRT subtitles from video or audio using AI transcription. |
| 357 | |
| 358 | Parameters: |
| 359 | - mediaUrl: URL of the video or audio file to transcribe |
| 360 | - language (optional): Target language code (e.g., "zh-CN", "en-US", "ja-JP") |
| 361 | |
| 362 | Returns the generated SRT subtitle file URL. |
| 363 | |
| 364 | Use this tool when user wants to: |
| 365 | - 给视频加字幕 / Add subtitles to video |
| 366 | - 生成字幕 / Generate subtitles |
| 367 | - 视频转录 / Transcribe video |
| 368 | - 提取视频文字 / Extract text from video`, |
| 369 | generateSubtitleSchema.shape, |
| 370 | async ({ mediaUrl, language }) => { |
| 371 | this.logger.log({ mediaUrl, language }, 'Starting subtitle generation') |
| 372 | |
| 373 | const audioBuffer = await this.extractAudioWithFFmpeg(mediaUrl) |
| 374 | |
| 375 | const prompt = this.buildSubtitlePrompt(language) |
| 376 | |
| 377 | this.logger.debug({ mediaUrl, language }, 'Calling Gemini for transcription') |
| 378 | const response = await this.chatService.userGeminiGenerateContent({ |
| 379 | userId, |
| 380 | userType, |
| 381 | model: 'gemini-3.5-flash', |
| 382 | contents: [{ |
| 383 | role: 'user', |
| 384 | parts: [ |
| 385 | { |
| 386 | inlineData: { |
| 387 | mimeType: 'audio/mp3', |
| 388 | data: audioBuffer.toString('base64'), |
| 389 | }, |
| 390 | }, |
| 391 | { text: prompt }, |
| 392 | ], |
| 393 | }], |
| 394 | config: { |
| 395 | responseMimeType: 'application/json', |
| 396 | responseJsonSchema: z.toJSONSchema(srtNodesSchema), |
| 397 | }, |
| 398 | }) |
| 399 | |
| 400 | const responseText = response.text |
| 401 | if (!responseText) { |
| 402 | throw new Error('No response from Gemini') |
| 403 | } |
| 404 | |
| 405 | const result = z.safeParse(srtNodesSchema, JSON.parse(responseText)) |
| 406 | if (!result.success) { |
| 407 | throw new Error(`Invalid subtitle data: ${z.prettifyError(result.error)}`) |
| 408 | } |
| 409 | const subtitleData = result.data.map(node => ({ |
| 410 | type: 'cue', |
| 411 | data: { |
| 412 | ...node, |
| 413 | start: srtTimestampToMs(node.start), |
| 414 | end: srtTimestampToMs(node.end), |
| 415 | }, |
| 416 | } as const)) |
| 417 | |
| 418 | const srtContent = subtitle.stringifySync(subtitleData, { format: 'SRT' }) |
| 419 | |
| 420 | this.logger.debug('Uploading SRT file') |
| 421 | const srtBuffer = Buffer.from(srtContent, 'utf-8') |
| 422 | const uploadResult = await this.assetsService.uploadFromBuffer(userId, srtBuffer, { |
| 423 | type: AssetType.Subtitle, |
| 424 | mimeType: 'text/plain', |
| 425 | filename: `subtitle-${Date.now()}.srt`, |
| 426 | }) |
| 427 | |
| 428 | return successResult(`Subtitle generated successfully! |
| 429 | **SRT File URL:** ${uploadResult.url} |
| 430 | You can use this SRT URL in video editing to add subtitles.`) |
| 431 | }, |
| 432 | this.aiAvailability, |
| 433 | ) |
| 434 | } |
| 435 | |
| 436 | createServer(userId: string, userType: UserType): McpSdkServerConfigWithInstance { |
| 437 | return createSdkMcpServer({ |
| 438 | name: McpServerName.VideoUtils, |
| 439 | version: '1.0.0', |
| 440 | tools: [ |
| 441 | this.createGetVolcVideoInfoTool(userId, userType), |
| 442 | this.createUploadAndGetVidTool(userId, userType), |
| 443 | this.createProbeVideoMetadataTool(userId, userType), |
| 444 | this.createExtractThumbnailTool(userId, userType), |
| 445 | // this.createGenerateSubtitleTool(userId, userType), |
| 446 | ], |
| 447 | }) |
| 448 | } |
| 449 | } |
| 450 |