| 1 | import os |
| 2 | import random |
| 3 | import threading |
| 4 | from typing import List |
| 5 | from urllib.parse import urlencode |
| 6 | |
| 7 | import requests |
| 8 | from loguru import logger |
| 9 | from moviepy.video.io.VideoFileClip import VideoFileClip |
| 10 | |
| 11 | from app.config import config |
| 12 | from app.models.schema import MaterialInfo, VideoAspect, VideoConcatMode |
| 13 | from app.utils import utils |
| 14 | |
| 15 | # Thread-safe counter for API key rotation |
| 16 | _api_key_counter = 0 |
| 17 | _api_key_lock = threading.Lock() |
| 18 | |
| 19 | |
| 20 | def _get_tls_verify() -> bool: |
| 21 | # 默认开启 TLS 证书校验，防止素材搜索和下载过程被中间人篡改。 |
| 22 | # 仅在企业代理、自签证书等明确需要的场景下，允许用户通过 |
| 23 | # `config.toml` 显式设置 `tls_verify = false` 临时关闭。 |
| 24 | tls_verify = config.app.get("tls_verify", True) |
| 25 | if isinstance(tls_verify, str): |
| 26 | tls_verify = tls_verify.strip().lower() not in ("0", "false", "no", "off") |
| 27 | |
| 28 | if not tls_verify: |
| 29 | logger.warning( |
| 30 | "TLS certificate verification is disabled by config.app.tls_verify=false. " |
| 31 | "Only use this in trusted proxy environments." |
| 32 | ) |
| 33 | |
| 34 | return bool(tls_verify) |
| 35 | |
| 36 | |
| 37 | def get_api_key(cfg_key: str): |
| 38 | api_keys = config.app.get(cfg_key) |
| 39 | if not api_keys: |
| 40 | raise ValueError( |
| 41 | f"\n\n##### {cfg_key} is not set #####\n\nPlease set it in the config.toml file: {config.config_file}\n\n" |
| 42 | f"{utils.to_json(config.app)}" |
| 43 | ) |
| 44 | |
| 45 | # if only one key is provided, return it |
| 46 | if isinstance(api_keys, str): |
| 47 | return api_keys |
| 48 | |
| 49 | global _api_key_counter |
| 50 | with _api_key_lock: |
| 51 | _api_key_counter += 1 |
| 52 | return api_keys[_api_key_counter % len(api_keys)] |
| 53 | |
| 54 | |
| 55 | def search_videos_pexels( |
| 56 | search_term: str, |
| 57 | minimum_duration: int, |
| 58 | video_aspect: VideoAspect = VideoAspect.portrait, |
| 59 | ) -> List[MaterialInfo]: |
| 60 | aspect = VideoAspect(video_aspect) |
| 61 | video_orientation = aspect.name |
| 62 | video_width, video_height = aspect.to_resolution() |
| 63 | api_key = get_api_key("pexels_api_keys") |
| 64 | headers = { |
| 65 | "Authorization": api_key, |
| 66 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36", |
| 67 | } |
| 68 | # Build URL |
| 69 | params = {"query": search_term, "per_page": 20, "orientation": video_orientation} |
| 70 | query_url = f"https://api.pexels.com/videos/search?{urlencode(params)}" |
| 71 | logger.info(f"searching videos: {query_url}, with proxies: {config.proxy}") |
| 72 | |
| 73 | try: |
| 74 | r = requests.get( |
| 75 | query_url, |
| 76 | headers=headers, |
| 77 | proxies=config.proxy, |
| 78 | verify=_get_tls_verify(), |
| 79 | timeout=(30, 60), |
| 80 | ) |
| 81 | response = r.json() |
| 82 | video_items = [] |
| 83 | if "videos" not in response: |
| 84 | logger.error(f"search videos failed: {response}") |
| 85 | return video_items |
| 86 | videos = response["videos"] |
| 87 | # loop through each video in the result |
| 88 | for v in videos: |
| 89 | duration = v["duration"] |
| 90 | # check if video has desired minimum duration |
| 91 | if duration < minimum_duration: |
| 92 | continue |
| 93 | video_files = v["video_files"] |
| 94 | # loop through each url to determine the best quality |
| 95 | for video in video_files: |
| 96 | w = int(video["width"]) |
| 97 | h = int(video["height"]) |
| 98 | if w == video_width and h == video_height: |
| 99 | item = MaterialInfo() |
| 100 | item.provider = "pexels" |
| 101 | item.url = video["link"] |
| 102 | item.duration = duration |
| 103 | video_items.append(item) |
| 104 | break |
| 105 | return video_items |
| 106 | except Exception as e: |
| 107 | logger.error(f"search videos failed: {str(e)}") |
| 108 | |
| 109 | return [] |
| 110 | |
| 111 | |
| 112 | def search_videos_pixabay( |
| 113 | search_term: str, |
| 114 | minimum_duration: int, |
| 115 | video_aspect: VideoAspect = VideoAspect.portrait, |
| 116 | ) -> List[MaterialInfo]: |
| 117 | aspect = VideoAspect(video_aspect) |
| 118 | |
| 119 | video_width, video_height = aspect.to_resolution() |
| 120 | |
| 121 | api_key = get_api_key("pixabay_api_keys") |
| 122 | # Build URL |
| 123 | params = { |
| 124 | "q": search_term, |
| 125 | "video_type": "all", # Accepted values: "all", "film", "animation" |
| 126 | "per_page": 50, |
| 127 | "key": api_key, |
| 128 | } |
| 129 | query_url = f"https://pixabay.com/api/videos/?{urlencode(params)}" |
| 130 | logger.info(f"searching videos: {query_url}, with proxies: {config.proxy}") |
| 131 | |
| 132 | try: |
| 133 | r = requests.get( |
| 134 | query_url, proxies=config.proxy, verify=_get_tls_verify(), timeout=(30, 60) |
| 135 | ) |
| 136 | response = r.json() |
| 137 | video_items = [] |
| 138 | if "hits" not in response: |
| 139 | logger.error(f"search videos failed: {response}") |
| 140 | return video_items |
| 141 | videos = response["hits"] |
| 142 | # loop through each video in the result |
| 143 | for v in videos: |
| 144 | duration = v["duration"] |
| 145 | # check if video has desired minimum duration |
| 146 | if duration < minimum_duration: |
| 147 | continue |
| 148 | video_files = v["videos"] |
| 149 | # loop through each url to determine the best quality |
| 150 | for video_type in video_files: |
| 151 | video = video_files[video_type] |
| 152 | w = int(video["width"]) |
| 153 | # h = int(video["height"]) |
| 154 | if w >= video_width: |
| 155 | item = MaterialInfo() |
| 156 | item.provider = "pixabay" |
| 157 | item.url = video["url"] |
| 158 | item.duration = duration |
| 159 | video_items.append(item) |
| 160 | break |
| 161 | return video_items |
| 162 | except Exception as e: |
| 163 | logger.error(f"search videos failed: {str(e)}") |
| 164 | |
| 165 | return [] |
| 166 | |
| 167 | |
| 168 | def search_videos_coverr( |
| 169 | search_term: str, |
| 170 | minimum_duration: int, |
| 171 | video_aspect: VideoAspect = VideoAspect.portrait, |
| 172 | ) -> List[MaterialInfo]: |
| 173 | """ |
| 174 | Coverr (https://coverr.co) - free HD/4K stock videos, |
| 175 | subject to Coverr license terms (https://coverr.co/license). |
| 176 | |
| 177 | Coverr API notes (based on official docs at api.coverr.co/docs/): |
| 178 | - 鉴权: Authorization: Bearer <api_key> |
| 179 | - 搜索端点: GET /videos?query=...,响应结构 {"hits": [...], ...} |
| 180 | - 加 ?urls=true 在搜索响应里直接返回 mp4 直链 |
| 181 | - URL 是 signed JWT(绑定 API key,无过期时间) |
| 182 | - Coverr 库以 16:9 横屏为主,9:16 portrait 占比极低(约 1%) |
| 183 | 因此本函数不做 aspect_ratio 过滤,由下游 video.py 的 |
| 184 | resize + letterbox 逻辑统一处理 |
| 185 | - duration 字段同时存在 number 和 string 两种形态,本函数都接受 |
| 186 | |
| 187 | 本函数使用 urls.mp4_download 字段作为下载地址 —— 按 Coverr 官方文档 |
| 188 | (https://api.coverr.co/docs/videos/#download-a-video) 的说法, |
| 189 | GET 这个 URL 本身就被 Coverr 当作一次合法的 download 事件计入统计, |
| 190 | 无需再调用 PATCH /videos/:id/stats/downloads。 |
| 191 | """ |
| 192 | api_key = get_api_key("coverr_api_keys") |
| 193 | headers = {"Authorization": f"Bearer {api_key}"} |
| 194 | params = { |
| 195 | "query": search_term, |
| 196 | "page_size": 20, |
| 197 | "urls": "true", |
| 198 | "sort": "popular", |
| 199 | } |
| 200 | query_url = f"https://api.coverr.co/videos?{urlencode(params)}" |
| 201 | logger.info(f"searching videos: {query_url}, with proxies: {config.proxy}") |
| 202 | |
| 203 | try: |
| 204 | r = requests.get( |
| 205 | query_url, |
| 206 | headers=headers, |
| 207 | proxies=config.proxy, |
| 208 | verify=_get_tls_verify(), |
| 209 | timeout=(30, 60), |
| 210 | ) |
| 211 | response = r.json() |
| 212 | video_items: List[MaterialInfo] = [] |
| 213 | |
| 214 | if not isinstance(response, dict) or "hits" not in response: |
| 215 | logger.error(f"search videos failed: {response}") |
| 216 | return video_items |
| 217 | |
| 218 | for v in response["hits"]: |
| 219 | # duration 在不同响应里可能是 number(11.625) 或 string("10.500000") |
| 220 | try: |
| 221 | duration = int(float(v.get("duration") or 0)) |
| 222 | except (TypeError, ValueError): |
| 223 | continue |
| 224 | if duration < minimum_duration: |
| 225 | continue |
| 226 | |
| 227 | video_id = v.get("id") |
| 228 | mp4_download_url = (v.get("urls") or {}).get("mp4_download") |
| 229 | if not video_id or not mp4_download_url: |
| 230 | continue |
| 231 | |
| 232 | item = MaterialInfo() |
| 233 | item.provider = "coverr" |
| 234 | item.url = mp4_download_url |
| 235 | item.duration = duration |
| 236 | video_items.append(item) |
| 237 | return video_items |
| 238 | except Exception as e: |
| 239 | logger.error(f"search videos failed: {str(e)}") |
| 240 | |
| 241 | return [] |
| 242 | |
| 243 | |
| 244 | def save_video(video_url: str, save_dir: str = "") -> str: |
| 245 | if not save_dir: |
| 246 | save_dir = utils.storage_dir("cache_videos") |
| 247 | |
| 248 | if not os.path.exists(save_dir): |
| 249 | os.makedirs(save_dir) |
| 250 | |
| 251 | url_without_query = video_url.split("?")[0] |
| 252 | url_hash = utils.md5(url_without_query) |
| 253 | video_id = f"vid-{url_hash}" |
| 254 | video_path = f"{save_dir}/{video_id}.mp4" |
| 255 | |
| 256 | # if video already exists, return the path |
| 257 | if os.path.exists(video_path) and os.path.getsize(video_path) > 0: |
| 258 | logger.info(f"video already exists: {video_path}") |
| 259 | return video_path |
| 260 | |
| 261 | headers = { |
| 262 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" |
| 263 | } |
| 264 | |
| 265 | # if video does not exist, download it |
| 266 | with open(video_path, "wb") as f: |
| 267 | f.write( |
| 268 | requests.get( |
| 269 | video_url, |
| 270 | headers=headers, |
| 271 | proxies=config.proxy, |
| 272 | verify=_get_tls_verify(), |
| 273 | timeout=(60, 240), |
| 274 | ).content |
| 275 | ) |
| 276 | |
| 277 | if os.path.exists(video_path) and os.path.getsize(video_path) > 0: |
| 278 | clip = None |
| 279 | try: |
| 280 | clip = VideoFileClip(video_path) |
| 281 | duration = clip.duration |
| 282 | fps = clip.fps |
| 283 | if duration > 0 and fps > 0: |
| 284 | return video_path |
| 285 | except Exception as e: |
| 286 | logger.warning(f"invalid video file: {video_path} => {str(e)}") |
| 287 | try: |
| 288 | os.remove(video_path) |
| 289 | except Exception as remove_error: |
| 290 | logger.warning( |
| 291 | f"failed to remove invalid video file: {video_path}, error: {str(remove_error)}" |
| 292 | ) |
| 293 | finally: |
| 294 | if clip is not None: |
| 295 | try: |
| 296 | clip.close() |
| 297 | except Exception as close_error: |
| 298 | logger.warning( |
| 299 | f"failed to close video clip: {video_path}, error: {str(close_error)}" |
| 300 | ) |
| 301 | return "" |
| 302 | |
| 303 | |
| 304 | def download_videos( |
| 305 | task_id: str, |
| 306 | search_terms: List[str], |
| 307 | source: str = "pexels", |
| 308 | video_aspect: VideoAspect = VideoAspect.portrait, |
| 309 | video_concat_mode: VideoConcatMode = VideoConcatMode.random, |
| 310 | audio_duration: float = 0.0, |
| 311 | max_clip_duration: int = 5, |
| 312 | match_script_order: bool = False, |
| 313 | ) -> List[str]: |
| 314 | search_videos = search_videos_pexels |
| 315 | if source == "pixabay": |
| 316 | search_videos = search_videos_pixabay |
| 317 | elif source == "coverr": |
| 318 | search_videos = search_videos_coverr |
| 319 | |
| 320 | material_directory = config.app.get("material_directory", "").strip() |
| 321 | if material_directory == "task": |
| 322 | material_directory = utils.task_dir(task_id) |
| 323 | elif material_directory and not os.path.isdir(material_directory): |
| 324 | material_directory = "" |
| 325 | |
| 326 | if match_script_order: |
| 327 | return _download_videos_by_script_order( |
| 328 | task_id=task_id, |
| 329 | search_terms=search_terms, |
| 330 | search_videos=search_videos, |
| 331 | video_aspect=video_aspect, |
| 332 | audio_duration=audio_duration, |
| 333 | max_clip_duration=max_clip_duration, |
| 334 | material_directory=material_directory, |
| 335 | ) |
| 336 | |
| 337 | valid_video_items = [] |
| 338 | valid_video_urls = [] |
| 339 | found_duration = 0.0 |
| 340 | for search_term in search_terms: |
| 341 | video_items = search_videos( |
| 342 | search_term=search_term, |
| 343 | minimum_duration=max_clip_duration, |
| 344 | video_aspect=video_aspect, |
| 345 | ) |
| 346 | logger.info(f"found {len(video_items)} videos for '{search_term}'") |
| 347 | |
| 348 | for item in video_items: |
| 349 | if item.url not in valid_video_urls: |
| 350 | valid_video_items.append(item) |
| 351 | valid_video_urls.append(item.url) |
| 352 | found_duration += item.duration |
| 353 | |
| 354 | logger.info( |
| 355 | f"found total videos: {len(valid_video_items)}, required duration: {audio_duration} seconds, found duration: {found_duration} seconds" |
| 356 | ) |
| 357 | video_paths = [] |
| 358 | |
| 359 | concat_mode_value = getattr(video_concat_mode, "value", video_concat_mode) |
| 360 | if concat_mode_value == VideoConcatMode.random.value: |
| 361 | random.shuffle(valid_video_items) |
| 362 | |
| 363 | total_duration = 0.0 |
| 364 | for item in valid_video_items: |
| 365 | try: |
| 366 | logger.info(f"downloading video: {item.url}") |
| 367 | saved_video_path = save_video( |
| 368 | video_url=item.url, save_dir=material_directory |
| 369 | ) |
| 370 | if saved_video_path: |
| 371 | logger.info(f"video saved: {saved_video_path}") |
| 372 | video_paths.append(saved_video_path) |
| 373 | seconds = min(max_clip_duration, item.duration) |
| 374 | total_duration += seconds |
| 375 | if total_duration > audio_duration: |
| 376 | logger.info( |
| 377 | f"total duration of downloaded videos: {total_duration} seconds, skip downloading more" |
| 378 | ) |
| 379 | break |
| 380 | except Exception as e: |
| 381 | logger.error(f"failed to download video: {utils.to_json(item)} => {str(e)}") |
| 382 | logger.success(f"downloaded {len(video_paths)} videos") |
| 383 | return video_paths |
| 384 | |
| 385 | |
| 386 | def _download_videos_by_script_order( |
| 387 | task_id: str, |
| 388 | search_terms: List[str], |
| 389 | search_videos, |
| 390 | video_aspect: VideoAspect, |
| 391 | audio_duration: float, |
| 392 | max_clip_duration: int, |
| 393 | material_directory: str, |
| 394 | ) -> List[str]: |
| 395 | """ |
| 396 | 按脚本文案顺序下载素材。 |
| 397 | |
| 398 | 默认下载逻辑会把所有关键词的候选素材合并成一个大列表；如果第一个 |
| 399 | 关键词返回很多结果，最终下载时可能一直消耗这个关键词的素材，后续 |
| 400 | 脚本主题就排不上时间线。这里按关键词分组后轮询下载： |
| 401 | 第 1 轮取每个关键词的第 1 个候选，第 2 轮取每个关键词的第 2 个候选。 |
| 402 | 这样在不重写视频合成引擎的前提下，尽量保证素材顺序贴近文案顺序。 |
| 403 | """ |
| 404 | logger.info("downloading videos with script-order material matching") |
| 405 | candidate_groups = [] |
| 406 | valid_video_urls = set() |
| 407 | found_duration = 0.0 |
| 408 | |
| 409 | for search_term in search_terms: |
| 410 | video_items = search_videos( |
| 411 | search_term=search_term, |
| 412 | minimum_duration=max_clip_duration, |
| 413 | video_aspect=video_aspect, |
| 414 | ) |
| 415 | logger.info(f"found {len(video_items)} videos for '{search_term}'") |
| 416 | |
| 417 | term_items = [] |
| 418 | for item in video_items: |
| 419 | if item.url in valid_video_urls: |
| 420 | continue |
| 421 | term_items.append(item) |
| 422 | valid_video_urls.add(item.url) |
| 423 | found_duration += item.duration |
| 424 | |
| 425 | if term_items: |
| 426 | candidate_groups.append((search_term, term_items)) |
| 427 | |
| 428 | logger.info( |
| 429 | f"found total ordered video candidates: {sum(len(items) for _, items in candidate_groups)}, " |
| 430 | f"required duration: {audio_duration} seconds, found duration: {found_duration} seconds" |
| 431 | ) |
| 432 | |
| 433 | video_paths = [] |
| 434 | total_duration = 0.0 |
| 435 | candidate_index = 0 |
| 436 | while candidate_groups and total_duration <= audio_duration: |
| 437 | has_candidate = False |
| 438 | for search_term, term_items in candidate_groups: |
| 439 | if candidate_index >= len(term_items): |
| 440 | continue |
| 441 | |
| 442 | has_candidate = True |
| 443 | item = term_items[candidate_index] |
| 444 | try: |
| 445 | logger.info( |
| 446 | f"downloading ordered video for '{search_term}': {item.url}" |
| 447 | ) |
| 448 | saved_video_path = save_video( |
| 449 | video_url=item.url, save_dir=material_directory |
| 450 | ) |
| 451 | if saved_video_path: |
| 452 | logger.info(f"video saved: {saved_video_path}") |
| 453 | video_paths.append(saved_video_path) |
| 454 | total_duration += min(max_clip_duration, item.duration) |
| 455 | if total_duration > audio_duration: |
| 456 | logger.info( |
| 457 | f"total duration of downloaded videos: {total_duration} seconds, skip downloading more" |
| 458 | ) |
| 459 | break |
| 460 | except Exception as e: |
| 461 | logger.error( |
| 462 | f"failed to download ordered video: {utils.to_json(item)} => {str(e)}" |
| 463 | ) |
| 464 | |
| 465 | if not has_candidate: |
| 466 | break |
| 467 | candidate_index += 1 |
| 468 | |
| 469 | logger.success(f"downloaded {len(video_paths)} ordered videos") |
| 470 | return video_paths |
| 471 | |
| 472 | |
| 473 | if __name__ == "__main__": |
| 474 | download_videos( |
| 475 | "test123", ["Money Exchange Medium"], audio_duration=100, source="pixabay" |
| 476 | ) |
| 477 |