| 1 | # Copyright (C) 2025 AIDC-AI |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 7 | |
| 8 | """ |
| 9 | API VLM-based asset analysis service. |
| 10 | |
| 11 | This service mirrors the text description contract of ImageAnalysisService and |
| 12 | VideoAnalysisService, but uses direct provider VLM APIs instead of ComfyUI or |
| 13 | RunningHub workflows. |
| 14 | """ |
| 15 | |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import asyncio |
| 19 | from pathlib import Path |
| 20 | from typing import Optional |
| 21 | |
| 22 | from loguru import logger |
| 23 | |
| 24 | |
| 25 | class APIAssetAnalysisService: |
| 26 | """Analyze image/video assets with a direct VLM API provider.""" |
| 27 | |
| 28 | VLM_MODELS = { |
| 29 | "dashscope": [ |
| 30 | "qwen3.7-plus", |
| 31 | "qwen3.6-plus", |
| 32 | "qwen3.6-flash", |
| 33 | "qwen3.5-omni-plus", |
| 34 | ], |
| 35 | } |
| 36 | |
| 37 | VLM_PROVIDER_LABELS = { |
| 38 | "dashscope": "DashScope", |
| 39 | } |
| 40 | |
| 41 | IMAGE_PROMPT = """请分析这张素材图片，用中文给出适合短视频脚本创作的简洁描述。 |
| 42 | |
| 43 | 请重点说明： |
| 44 | 1. 画面主体、人物/商品/场景 |
| 45 | 2. 可用于营销或叙事的关键信息 |
| 46 | 3. 画面风格、氛围、颜色和构图 |
| 47 | |
| 48 | 输出 2-5 句话，不要编造图片中不存在的信息。""" |
| 49 | |
| 50 | VIDEO_PROMPT = """请分析这个上传的视频素材，用中文概括视频内容。 |
| 51 | |
| 52 | 请重点说明： |
| 53 | 1. 视频中的主体、场景和动作变化 |
| 54 | 2. 可用于短视频脚本的卖点或叙事信息 |
| 55 | 3. 整体风格、节奏和氛围 |
| 56 | |
| 57 | 输出 3-6 句话，不要编造关键帧中看不到的信息。""" |
| 58 | |
| 59 | def __init__(self, config: dict, core=None): |
| 60 | self.config = config |
| 61 | self.core = core |
| 62 | |
| 63 | def list_models(self, configured_only: bool = True) -> list[dict]: |
| 64 | """Return VLM models available for API-backed asset analysis.""" |
| 65 | providers = self.config.get("api_providers", {}) or {} |
| 66 | models = [] |
| 67 | |
| 68 | for provider, provider_models in self.VLM_MODELS.items(): |
| 69 | provider_config = providers.get(provider, {}) or {} |
| 70 | if configured_only and not provider_config.get("api_key"): |
| 71 | continue |
| 72 | |
| 73 | provider_label = self.VLM_PROVIDER_LABELS.get(provider, provider.title()) |
| 74 | for model in provider_models: |
| 75 | key = f"api/vlm/{provider}/{model}" |
| 76 | models.append({ |
| 77 | "key": key, |
| 78 | "name": model, |
| 79 | "display_name": f"{model} - API {provider_label}", |
| 80 | "source": "api", |
| 81 | "provider": provider, |
| 82 | "model": model, |
| 83 | "media_type": "asset_analysis", |
| 84 | "ability_type": "vlm_asset_analysis", |
| 85 | "ability_types": ["vlm_asset_analysis"], |
| 86 | }) |
| 87 | |
| 88 | return models |
| 89 | |
| 90 | async def analyze_image( |
| 91 | self, |
| 92 | image_path: str, |
| 93 | model: Optional[str] = None, |
| 94 | prompt: Optional[str] = None, |
| 95 | **_: object, |
| 96 | ) -> str: |
| 97 | image_file = Path(image_path) |
| 98 | if not image_file.exists(): |
| 99 | raise FileNotFoundError(f"Image file not found: {image_path}") |
| 100 | |
| 101 | return await self._query_vlm( |
| 102 | prompt=prompt or self.IMAGE_PROMPT, |
| 103 | image_paths=[str(image_file)], |
| 104 | model=model, |
| 105 | ) |
| 106 | |
| 107 | async def analyze_video( |
| 108 | self, |
| 109 | video_path: str, |
| 110 | model: Optional[str] = None, |
| 111 | prompt: Optional[str] = None, |
| 112 | **_: object, |
| 113 | ) -> str: |
| 114 | video_file = Path(video_path) |
| 115 | if not video_file.exists(): |
| 116 | raise FileNotFoundError(f"Video file not found: {video_path}") |
| 117 | |
| 118 | return await self._query_vlm( |
| 119 | prompt=prompt or self.VIDEO_PROMPT, |
| 120 | image_paths=[], |
| 121 | video_paths=[str(video_file)], |
| 122 | model=model, |
| 123 | ) |
| 124 | |
| 125 | async def __call__(self, asset_path: str, asset_type: Optional[str] = None, **kwargs) -> str: |
| 126 | path = Path(asset_path) |
| 127 | resolved_type = asset_type or self._get_asset_type(path) |
| 128 | if resolved_type == "image": |
| 129 | return await self.analyze_image(asset_path, **kwargs) |
| 130 | if resolved_type == "video": |
| 131 | return await self.analyze_video(asset_path, **kwargs) |
| 132 | raise ValueError(f"Unsupported asset type for VLM analysis: {asset_path}") |
| 133 | |
| 134 | async def _query_vlm( |
| 135 | self, |
| 136 | prompt: str, |
| 137 | image_paths: list[str], |
| 138 | model: Optional[str], |
| 139 | video_paths: Optional[list[str]] = None, |
| 140 | ) -> str: |
| 141 | from pixelle_video.services.api_services.vlm_client import VLM |
| 142 | |
| 143 | selected_model = (model or "").strip() |
| 144 | if not selected_model: |
| 145 | raise RuntimeError( |
| 146 | "API VLM analysis requires an explicitly selected VLM model. " |
| 147 | "Please choose one in the asset analysis service settings." |
| 148 | ) |
| 149 | |
| 150 | logger.info( |
| 151 | f"Analyzing asset via API VLM model={selected_model}, " |
| 152 | f"images={len(image_paths)}, videos={len(video_paths or [])}" |
| 153 | ) |
| 154 | |
| 155 | providers = self.config.get("api_providers", {}) or {} |
| 156 | dashscope = providers.get("dashscope", {}) or {} |
| 157 | |
| 158 | client = VLM( |
| 159 | dashscope_api_key=dashscope.get("api_key"), |
| 160 | dashscope_base_url=dashscope.get("base_url"), |
| 161 | ) |
| 162 | result = await asyncio.to_thread( |
| 163 | client.query, |
| 164 | prompt, |
| 165 | image_paths, |
| 166 | selected_model, |
| 167 | None, |
| 168 | video_paths, |
| 169 | ) |
| 170 | description = str(result or "").strip() |
| 171 | if not description: |
| 172 | raise RuntimeError("API VLM analysis returned empty description") |
| 173 | return description |
| 174 | |
| 175 | def _get_asset_type(self, path: Path) -> str: |
| 176 | image_exts = {".jpg", ".jpeg", ".png", ".gif", ".webp"} |
| 177 | video_exts = {".mp4", ".mov", ".avi", ".mkv", ".webm"} |
| 178 | ext = path.suffix.lower() |
| 179 | if ext in image_exts: |
| 180 | return "image" |
| 181 | if ext in video_exts: |
| 182 | return "video" |
| 183 | return "unknown" |
| 184 |