| 1 | import json |
| 2 | import re |
| 3 | from abc import ABC, abstractmethod |
| 4 | from datetime import datetime |
| 5 | from pathlib import Path |
| 6 | from typing import Any, Dict, List, Optional, Protocol, Tuple |
| 7 | from urllib.parse import urlparse |
| 8 | |
| 9 | from auth import CookieManager |
| 10 | from config import ConfigLoader |
| 11 | from control import QueueManager, RateLimiter, RetryHandler |
| 12 | from core.api_client import DouyinAPIClient |
| 13 | from core.metadata import extract_author_sec_uid |
| 14 | from core.transcript_manager import TranscriptManager |
| 15 | from storage import Database, FileManager, MetadataHandler |
| 16 | from utils.logger import setup_logger |
| 17 | from utils.naming import ( |
| 18 | DEFAULT_FILE_TEMPLATE, |
| 19 | DEFAULT_FOLDER_TEMPLATE, |
| 20 | build_aweme_context, |
| 21 | render_template, |
| 22 | ) |
| 23 | |
| 24 | logger = setup_logger("BaseDownloader") |
| 25 | |
| 26 | |
| 27 | class ProgressReporter(Protocol): |
| 28 | def update_step(self, step: str, detail: str = "") -> None: ... |
| 29 | |
| 30 | def set_item_total(self, total: int, detail: str = "") -> None: ... |
| 31 | |
| 32 | def advance_item(self, status: str, detail: str = "") -> None: ... |
| 33 | |
| 34 | |
| 35 | class DownloadResult: |
| 36 | def __init__(self): |
| 37 | self.total = 0 |
| 38 | self.success = 0 |
| 39 | self.failed = 0 |
| 40 | self.skipped = 0 |
| 41 | |
| 42 | def __str__(self): |
| 43 | return f"Total: {self.total}, Success: {self.success}, Failed: {self.failed}, Skipped: {self.skipped}" |
| 44 | |
| 45 | |
| 46 | class BaseDownloader(ABC): |
| 47 | def __init__( |
| 48 | self, |
| 49 | config: ConfigLoader, |
| 50 | api_client: DouyinAPIClient, |
| 51 | file_manager: FileManager, |
| 52 | cookie_manager: CookieManager, |
| 53 | database: Optional[Database] = None, |
| 54 | rate_limiter: Optional[RateLimiter] = None, |
| 55 | retry_handler: Optional[RetryHandler] = None, |
| 56 | queue_manager: Optional[QueueManager] = None, |
| 57 | progress_reporter: Optional[ProgressReporter] = None, |
| 58 | ): |
| 59 | self.config = config |
| 60 | self.api_client = api_client |
| 61 | self.file_manager = file_manager |
| 62 | self.cookie_manager = cookie_manager |
| 63 | self.database = database |
| 64 | self.rate_limiter = rate_limiter or RateLimiter() |
| 65 | self.retry_handler = retry_handler or RetryHandler() |
| 66 | thread_count = int(self.config.get("thread", 5) or 5) |
| 67 | self.queue_manager = queue_manager or QueueManager(max_workers=thread_count) |
| 68 | self.progress_reporter = progress_reporter |
| 69 | self.metadata_handler = MetadataHandler() |
| 70 | self.transcript_manager = TranscriptManager(self.config, self.file_manager, self.database) |
| 71 | self._local_aweme_ids: Optional[set[str]] = None |
| 72 | self._aweme_id_pattern = re.compile(r"(?<!\d)(\d{15,20})(?!\d)") |
| 73 | self._local_media_suffixes = { |
| 74 | ".mp4", |
| 75 | ".jpg", |
| 76 | ".jpeg", |
| 77 | ".png", |
| 78 | ".webp", |
| 79 | ".gif", |
| 80 | ".mp3", |
| 81 | ".m4a", |
| 82 | } |
| 83 | # 控制终端错误日志量，避免进度条被大量日志打断后出现重复重绘。 |
| 84 | self._download_error_log_count = 0 |
| 85 | self._download_error_log_limit = 5 |
| 86 | |
| 87 | def _progress_update_step(self, step: str, detail: str = "") -> None: |
| 88 | if not self.progress_reporter: |
| 89 | return |
| 90 | try: |
| 91 | self.progress_reporter.update_step(step, detail) |
| 92 | except Exception as exc: |
| 93 | logger.debug("Progress update_step failed: %s", exc) |
| 94 | |
| 95 | def _progress_set_item_total(self, total: int, detail: str = "") -> None: |
| 96 | if not self.progress_reporter: |
| 97 | return |
| 98 | try: |
| 99 | self.progress_reporter.set_item_total(total, detail) |
| 100 | except Exception as exc: |
| 101 | logger.debug("Progress set_item_total failed: %s", exc) |
| 102 | |
| 103 | def _progress_advance_item(self, status: str, detail: str = "") -> None: |
| 104 | if not self.progress_reporter: |
| 105 | return |
| 106 | try: |
| 107 | self.progress_reporter.advance_item(status, detail) |
| 108 | except Exception as exc: |
| 109 | logger.debug("Progress advance_item failed: %s", exc) |
| 110 | |
| 111 | def _progress_report_author( |
| 112 | self, |
| 113 | nickname: Optional[str] = None, |
| 114 | sec_uid: Optional[str] = None, |
| 115 | ) -> None: |
| 116 | """Surface author metadata to the reporter so the hosting job can |
| 117 | cache it for retry and display. |
| 118 | |
| 119 | Downloaders call this as soon as author info is known (user_info |
| 120 | lookup for batch jobs, aweme_data.author for single-video jobs). |
| 121 | Safe to call with `None` values — the reporter drops empty payloads. |
| 122 | """ |
| 123 | if not self.progress_reporter: |
| 124 | return |
| 125 | try: |
| 126 | fn = getattr(self.progress_reporter, "on_author", None) |
| 127 | if callable(fn): |
| 128 | fn(nickname=nickname, sec_uid=sec_uid) |
| 129 | except Exception as exc: # pragma: no cover — defensive |
| 130 | logger.debug("Progress on_author failed: %s", exc) |
| 131 | |
| 132 | def _log_download_error(self, log_fn, message: str) -> None: |
| 133 | if self._download_error_log_count < self._download_error_log_limit: |
| 134 | log_fn(message) |
| 135 | elif self._download_error_log_count == self._download_error_log_limit: |
| 136 | logger.error("Too many download errors, suppressing further per-file logs...") |
| 137 | self._download_error_log_count += 1 |
| 138 | |
| 139 | def _download_headers(self, user_agent: Optional[str] = None) -> Dict[str, str]: |
| 140 | headers = { |
| 141 | "Referer": f"{self.api_client.BASE_URL}/", |
| 142 | "Origin": self.api_client.BASE_URL, |
| 143 | "Accept": "*/*", |
| 144 | } |
| 145 | |
| 146 | headers["User-Agent"] = user_agent or self.api_client.headers.get("User-Agent", "") |
| 147 | return headers |
| 148 | |
| 149 | @abstractmethod |
| 150 | async def download(self, parsed_url: Dict[str, Any]) -> DownloadResult: |
| 151 | pass |
| 152 | |
| 153 | async def _should_download(self, aweme_id: str) -> bool: |
| 154 | in_local = self._is_locally_downloaded(aweme_id) |
| 155 | in_db = False |
| 156 | if self.database: |
| 157 | in_db = await self.database.is_downloaded(aweme_id) |
| 158 | |
| 159 | if in_db and in_local: |
| 160 | return False |
| 161 | |
| 162 | if in_db and not in_local: |
| 163 | logger.info( |
| 164 | "Aweme %s exists in database but media file not found locally, retry download", |
| 165 | aweme_id, |
| 166 | ) |
| 167 | return True |
| 168 | |
| 169 | if in_local: |
| 170 | logger.info("Aweme %s already exists locally, skipping", aweme_id) |
| 171 | return False |
| 172 | |
| 173 | return True |
| 174 | |
| 175 | def _is_locally_downloaded(self, aweme_id: str) -> bool: |
| 176 | if not aweme_id: |
| 177 | return False |
| 178 | |
| 179 | if self._local_aweme_ids is None: |
| 180 | self._build_local_aweme_index() |
| 181 | |
| 182 | if self._local_aweme_ids is None: |
| 183 | return False |
| 184 | return aweme_id in self._local_aweme_ids |
| 185 | |
| 186 | def _build_local_aweme_index(self): |
| 187 | base_path = self.file_manager.base_path |
| 188 | aweme_ids: set[str] = set() |
| 189 | |
| 190 | if base_path.exists(): |
| 191 | for path in base_path.rglob("*"): |
| 192 | if not path.is_file(): |
| 193 | continue |
| 194 | if path.suffix.lower() not in self._local_media_suffixes: |
| 195 | continue |
| 196 | try: |
| 197 | if path.stat().st_size <= 0: |
| 198 | continue |
| 199 | except OSError: |
| 200 | continue |
| 201 | for match in self._aweme_id_pattern.finditer(path.name): |
| 202 | aweme_ids.add(match.group(1)) |
| 203 | |
| 204 | self._local_aweme_ids = aweme_ids |
| 205 | |
| 206 | def _mark_local_aweme_downloaded(self, aweme_id: str): |
| 207 | if not aweme_id: |
| 208 | return |
| 209 | |
| 210 | if self._local_aweme_ids is None: |
| 211 | self._local_aweme_ids = set() |
| 212 | self._local_aweme_ids.add(aweme_id) |
| 213 | |
| 214 | def _filter_by_time(self, aweme_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]: |
| 215 | start_time = self.config.get("start_time") |
| 216 | end_time = self.config.get("end_time") |
| 217 | |
| 218 | if not start_time and not end_time: |
| 219 | return aweme_list |
| 220 | |
| 221 | start_ts = ( |
| 222 | int(datetime.strptime(start_time, "%Y-%m-%d").timestamp()) if start_time else None |
| 223 | ) |
| 224 | end_ts = int(datetime.strptime(end_time, "%Y-%m-%d").timestamp()) if end_time else None |
| 225 | |
| 226 | filtered: List[Dict[str, Any]] = [] |
| 227 | for aweme in aweme_list: |
| 228 | create_time = aweme.get("create_time", 0) |
| 229 | if start_ts is not None and create_time < start_ts: |
| 230 | continue |
| 231 | if end_ts is not None and create_time > end_ts: |
| 232 | continue |
| 233 | filtered.append(aweme) |
| 234 | |
| 235 | return filtered |
| 236 | |
| 237 | def _limit_count(self, aweme_list: List[Dict[str, Any]], mode: str) -> List[Dict[str, Any]]: |
| 238 | number_config = self.config.get("number", {}) |
| 239 | limit = number_config.get(mode, 0) |
| 240 | |
| 241 | if limit > 0: |
| 242 | return aweme_list[:limit] |
| 243 | return aweme_list |
| 244 | |
| 245 | async def _download_aweme_assets( |
| 246 | self, |
| 247 | aweme_data: Dict[str, Any], |
| 248 | author_name: str, |
| 249 | mode: Optional[str] = None, |
| 250 | *, |
| 251 | db_batch: Optional[List[Dict[str, Any]]] = None, |
| 252 | ) -> bool: |
| 253 | aweme_id = aweme_data.get("aweme_id") |
| 254 | if not aweme_id: |
| 255 | logger.error("Missing aweme_id in aweme data") |
| 256 | return False |
| 257 | |
| 258 | desc = (aweme_data.get("desc", "no_title") or "").strip() or "no_title" |
| 259 | publish_ts, publish_date = self._resolve_publish_time(aweme_data.get("create_time")) |
| 260 | if not publish_date: |
| 261 | publish_date = datetime.now().strftime("%Y-%m-%d") |
| 262 | logger.warning( |
| 263 | "Aweme %s missing/invalid create_time, fallback to current date %s", |
| 264 | aweme_id, |
| 265 | publish_date, |
| 266 | ) |
| 267 | media_type = self._detect_media_type(aweme_data) |
| 268 | template_context = build_aweme_context( |
| 269 | aweme_id=str(aweme_id), |
| 270 | title=desc, |
| 271 | author_name=author_name, |
| 272 | author_sec_uid=extract_author_sec_uid(aweme_data), |
| 273 | publish_date=publish_date, |
| 274 | publish_ts=publish_ts, |
| 275 | media_type=media_type, |
| 276 | mode=mode, |
| 277 | ) |
| 278 | filename_template = self.config.get("filename_template") or DEFAULT_FILE_TEMPLATE |
| 279 | folder_template = self.config.get("folder_template") or DEFAULT_FOLDER_TEMPLATE |
| 280 | file_stem = render_template( |
| 281 | filename_template, |
| 282 | template_context, |
| 283 | fallback=f"{publish_date}_{aweme_id}", |
| 284 | ) |
| 285 | folder_name = render_template( |
| 286 | folder_template, |
| 287 | template_context, |
| 288 | fallback=f"{publish_date}_{aweme_id}", |
| 289 | ) |
| 290 | |
| 291 | save_dir = self.file_manager.get_save_path( |
| 292 | author_name=author_name, |
| 293 | mode=mode, |
| 294 | aweme_title=desc, |
| 295 | aweme_id=aweme_id, |
| 296 | folderstyle=self.config.get("folderstyle", True), |
| 297 | download_date=publish_date, |
| 298 | folder_name=folder_name, |
| 299 | author_sec_uid=extract_author_sec_uid(aweme_data), |
| 300 | author_dir_style=self.config.get("author_dir") or "nickname", |
| 301 | ) |
| 302 | downloaded_files: List[Path] = [] |
| 303 | |
| 304 | session = await self.api_client.get_session() |
| 305 | video_path: Optional[Path] = None |
| 306 | |
| 307 | if media_type == "video": |
| 308 | video_info = self._build_no_watermark_url(aweme_data) |
| 309 | if not video_info: |
| 310 | logger.error("No playable video URL found for aweme %s", aweme_id) |
| 311 | return False |
| 312 | |
| 313 | video_url, video_headers = video_info |
| 314 | video_path = save_dir / f"{file_stem}.mp4" |
| 315 | if not await self._download_with_retry( |
| 316 | video_url, video_path, session, headers=video_headers |
| 317 | ): |
| 318 | return False |
| 319 | downloaded_files.append(video_path) |
| 320 | |
| 321 | if self.config.get("cover"): |
| 322 | cover_url = self._extract_first_url(aweme_data.get("video", {}).get("cover")) |
| 323 | if cover_url: |
| 324 | cover_path = save_dir / f"{file_stem}_cover.jpg" |
| 325 | if await self._download_with_retry( |
| 326 | cover_url, |
| 327 | cover_path, |
| 328 | session, |
| 329 | headers=self._download_headers(), |
| 330 | optional=True, |
| 331 | ): |
| 332 | downloaded_files.append(cover_path) |
| 333 | |
| 334 | if self.config.get("music"): |
| 335 | music_url = self._extract_first_url(aweme_data.get("music", {}).get("play_url")) |
| 336 | if music_url: |
| 337 | music_path = save_dir / f"{file_stem}_music.mp3" |
| 338 | if await self._download_with_retry( |
| 339 | music_url, |
| 340 | music_path, |
| 341 | session, |
| 342 | headers=self._download_headers(), |
| 343 | optional=True, |
| 344 | ): |
| 345 | downloaded_files.append(music_path) |
| 346 | |
| 347 | elif media_type == "gallery": |
| 348 | image_url_candidates = self._collect_image_url_candidates(aweme_data) |
| 349 | image_live_urls = self._collect_image_live_urls(aweme_data) |
| 350 | logger.info( |
| 351 | "Gallery aweme %s: %d image(s), %d live photo(s)", |
| 352 | aweme_id, |
| 353 | len(image_url_candidates), |
| 354 | len(image_live_urls), |
| 355 | ) |
| 356 | if not image_url_candidates and not image_live_urls: |
| 357 | logger.error( |
| 358 | "No gallery assets found for aweme %s (aweme_type=%s, " |
| 359 | "has image_post_info=%s, has images=%s)", |
| 360 | aweme_id, |
| 361 | aweme_data.get("aweme_type"), |
| 362 | "image_post_info" in aweme_data, |
| 363 | "images" in aweme_data, |
| 364 | ) |
| 365 | return False |
| 366 | |
| 367 | for index, candidates in enumerate(image_url_candidates, start=1): |
| 368 | download_result: bool | Path = False |
| 369 | for image_url in candidates: |
| 370 | suffix = self._infer_image_extension(image_url) |
| 371 | image_path = save_dir / f"{file_stem}_{index}{suffix}" |
| 372 | download_result = await self._download_with_retry( |
| 373 | image_url, |
| 374 | image_path, |
| 375 | session, |
| 376 | headers=self._download_headers(), |
| 377 | prefer_response_content_type=True, |
| 378 | return_saved_path=True, |
| 379 | ) |
| 380 | if download_result: |
| 381 | downloaded_files.append( |
| 382 | download_result if isinstance(download_result, Path) else image_path |
| 383 | ) |
| 384 | break |
| 385 | if not download_result: |
| 386 | logger.error(f"Failed downloading image {index} for aweme {aweme_id}") |
| 387 | return False |
| 388 | |
| 389 | for index, live_url in enumerate(image_live_urls, start=1): |
| 390 | suffix = Path(urlparse(live_url).path).suffix or ".mp4" |
| 391 | live_path = save_dir / f"{file_stem}_live_{index}{suffix}" |
| 392 | success = await self._download_with_retry( |
| 393 | live_url, |
| 394 | live_path, |
| 395 | session, |
| 396 | headers=self._download_headers(), |
| 397 | ) |
| 398 | if not success: |
| 399 | logger.error(f"Failed downloading live image {index} for aweme {aweme_id}") |
| 400 | return False |
| 401 | downloaded_files.append(live_path) |
| 402 | else: |
| 403 | logger.error("Unsupported media type for aweme %s: %s", aweme_id, media_type) |
| 404 | return False |
| 405 | |
| 406 | if self.config.get("avatar"): |
| 407 | author = aweme_data.get("author", {}) |
| 408 | avatar_url = self._extract_first_url(author.get("avatar_larger")) |
| 409 | if avatar_url: |
| 410 | avatar_path = save_dir / f"{file_stem}_avatar.jpg" |
| 411 | if await self._download_with_retry( |
| 412 | avatar_url, |
| 413 | avatar_path, |
| 414 | session, |
| 415 | headers=self._download_headers(), |
| 416 | optional=True, |
| 417 | ): |
| 418 | downloaded_files.append(avatar_path) |
| 419 | |
| 420 | if self.config.get("json"): |
| 421 | json_path = save_dir / f"{file_stem}_data.json" |
| 422 | if await self.metadata_handler.save_metadata(aweme_data, json_path): |
| 423 | downloaded_files.append(json_path) |
| 424 | |
| 425 | comments_cfg = self.config.get("comments") or {} |
| 426 | if isinstance(comments_cfg, dict) and comments_cfg.get("enabled"): |
| 427 | from core.comments_collector import CommentsCollector |
| 428 | |
| 429 | collector = CommentsCollector( |
| 430 | self.api_client, |
| 431 | self.metadata_handler, |
| 432 | include_replies=bool(comments_cfg.get("include_replies", False)), |
| 433 | max_comments=int(comments_cfg.get("max_comments", 0) or 0), |
| 434 | page_size=int(comments_cfg.get("page_size", 20) or 20), |
| 435 | ) |
| 436 | comments_path = save_dir / f"{file_stem}_comments.json" |
| 437 | saved = await collector.collect_and_save(aweme_id, comments_path) |
| 438 | if saved is not None: |
| 439 | downloaded_files.append(comments_path) |
| 440 | |
| 441 | author = aweme_data.get("author", {}) |
| 442 | if self.database: |
| 443 | metadata_json = json.dumps(aweme_data, ensure_ascii=False) |
| 444 | record = { |
| 445 | "aweme_id": aweme_id, |
| 446 | "aweme_type": media_type, |
| 447 | "title": desc, |
| 448 | "author_id": author.get("uid"), |
| 449 | "author_name": author.get("nickname", author_name), |
| 450 | "create_time": aweme_data.get("create_time"), |
| 451 | "file_path": str(save_dir), |
| 452 | "metadata": metadata_json, |
| 453 | # Attach sec_uid onto the payload so both the batched path |
| 454 | # (add_aweme_batch iterates `record["author_sec_uid"]`) and |
| 455 | # the single-write path (add_aweme reads the payload as |
| 456 | # fallback when the kwarg is None) pick it up identically. |
| 457 | "author_sec_uid": extract_author_sec_uid(aweme_data), |
| 458 | } |
| 459 | # Caller may opt into batched DB writes by passing a list; we just |
| 460 | # accumulate the record and let the caller commit them all at once. |
| 461 | if db_batch is not None: |
| 462 | db_batch.append(record) |
| 463 | else: |
| 464 | await self.database.add_aweme(record) |
| 465 | |
| 466 | manifest_record = { |
| 467 | "date": publish_date, |
| 468 | "aweme_id": aweme_id, |
| 469 | "author_name": author.get("nickname", author_name), |
| 470 | "desc": desc, |
| 471 | "media_type": media_type, |
| 472 | "tags": self._extract_tags(aweme_data), |
| 473 | "file_names": [path.name for path in downloaded_files], |
| 474 | "file_paths": [self._to_manifest_path(path) for path in downloaded_files], |
| 475 | } |
| 476 | if publish_ts: |
| 477 | manifest_record["publish_timestamp"] = publish_ts |
| 478 | await self.metadata_handler.append_download_manifest( |
| 479 | self.file_manager.base_path, manifest_record |
| 480 | ) |
| 481 | |
| 482 | if media_type == "video" and video_path is not None: |
| 483 | transcript_result = await self.transcript_manager.process_video( |
| 484 | video_path, aweme_id=aweme_id |
| 485 | ) |
| 486 | transcript_status = transcript_result.get("status") |
| 487 | if transcript_status == "skipped": |
| 488 | logger.info( |
| 489 | "Transcript skipped for aweme %s: %s", |
| 490 | aweme_id, |
| 491 | transcript_result.get("reason", "unknown"), |
| 492 | ) |
| 493 | elif transcript_status == "failed": |
| 494 | logger.warning( |
| 495 | "Transcript failed for aweme %s: %s", |
| 496 | aweme_id, |
| 497 | transcript_result.get("error", "unknown"), |
| 498 | ) |
| 499 | |
| 500 | self._mark_local_aweme_downloaded(aweme_id) |
| 501 | logger.info("Downloaded %s: %s (%s)", media_type, desc, aweme_id) |
| 502 | return True |
| 503 | |
| 504 | async def _download_with_retry( |
| 505 | self, |
| 506 | url: str, |
| 507 | save_path: Path, |
| 508 | session, |
| 509 | *, |
| 510 | headers: Optional[Dict[str, str]] = None, |
| 511 | optional: bool = False, |
| 512 | prefer_response_content_type: bool = False, |
| 513 | return_saved_path: bool = False, |
| 514 | ) -> bool | Path: |
| 515 | async def _task(): |
| 516 | download_result = await self.file_manager.download_file( |
| 517 | url, |
| 518 | save_path, |
| 519 | session, |
| 520 | headers=headers, |
| 521 | proxy=getattr(self.api_client, "proxy", None), |
| 522 | prefer_response_content_type=prefer_response_content_type, |
| 523 | return_saved_path=return_saved_path, |
| 524 | ) |
| 525 | if not download_result: |
| 526 | raise RuntimeError(f"Download failed for {url}") |
| 527 | return download_result |
| 528 | |
| 529 | try: |
| 530 | return await self.retry_handler.execute_with_retry(_task) |
| 531 | except Exception as error: |
| 532 | log_fn = logger.warning if optional else logger.error |
| 533 | self._log_download_error( |
| 534 | log_fn, |
| 535 | f"Download error for {save_path.name}: {error}", |
| 536 | ) |
| 537 | return False |
| 538 | |
| 539 | # aweme_type codes that indicate image/note content |
| 540 | _GALLERY_AWEME_TYPES = {2, 68, 150} |
| 541 | _PLAY_ADDR_KEYS = ( |
| 542 | "play_addr_h264", |
| 543 | "play_addr_265", |
| 544 | "play_addr_256", |
| 545 | "play_addr", |
| 546 | ) |
| 547 | |
| 548 | def _detect_media_type(self, aweme_data: Dict[str, Any]) -> str: |
| 549 | if self._iter_gallery_items(aweme_data): |
| 550 | return "gallery" |
| 551 | aweme_type = aweme_data.get("aweme_type") |
| 552 | if isinstance(aweme_type, int) and aweme_type in self._GALLERY_AWEME_TYPES: |
| 553 | video = aweme_data.get("video") if isinstance(aweme_data.get("video"), dict) else {} |
| 554 | if self._has_video_source(video): |
| 555 | logger.info( |
| 556 | "Detected note video via aweme_type=%s for aweme %s", |
| 557 | aweme_type, |
| 558 | aweme_data.get("aweme_id"), |
| 559 | ) |
| 560 | return "video" |
| 561 | logger.info( |
| 562 | "Detected gallery via aweme_type=%s for aweme %s", |
| 563 | aweme_type, |
| 564 | aweme_data.get("aweme_id"), |
| 565 | ) |
| 566 | return "gallery" |
| 567 | return "video" |
| 568 | |
| 569 | def _build_no_watermark_url( |
| 570 | self, aweme_data: Dict[str, Any] |
| 571 | ) -> Optional[Tuple[str, Dict[str, str]]]: |
| 572 | video = aweme_data.get("video", {}) |
| 573 | quality = str(self.config.get("video_quality") or "highest") |
| 574 | play_addr = self._pick_preferred_play_addr(video, quality) or {} |
| 575 | url_candidates = [c for c in (play_addr.get("url_list") or []) if c] |
| 576 | url_candidates.sort(key=lambda u: 0 if "watermark=0" in u else 1) |
| 577 | |
| 578 | fallback_candidate: Optional[Tuple[str, Dict[str, str]]] = None |
| 579 | watermarked_candidate: Optional[Tuple[str, Dict[str, str]]] = None |
| 580 | |
| 581 | for candidate in url_candidates: |
| 582 | parsed = urlparse(candidate) |
| 583 | headers = self._download_headers() |
| 584 | is_watermarked = self._is_watermarked_media_url(candidate) |
| 585 | |
| 586 | if parsed.netloc.endswith("douyin.com"): |
| 587 | if "X-Bogus=" not in candidate: |
| 588 | signed_url, ua = self.api_client.sign_url(candidate) |
| 589 | headers = self._download_headers(user_agent=ua) |
| 590 | if is_watermarked: |
| 591 | watermarked_candidate = watermarked_candidate or ( |
| 592 | signed_url, |
| 593 | headers, |
| 594 | ) |
| 595 | continue |
| 596 | return signed_url, headers |
| 597 | if is_watermarked: |
| 598 | watermarked_candidate = watermarked_candidate or (candidate, headers) |
| 599 | continue |
| 600 | return candidate, headers |
| 601 | |
| 602 | if is_watermarked: |
| 603 | watermarked_candidate = watermarked_candidate or (candidate, headers) |
| 604 | else: |
| 605 | fallback_candidate = fallback_candidate or (candidate, headers) |
| 606 | |
| 607 | # Prefer direct CDN URLs (e.g. douyinvod.com) over the /aweme/v1/play/ |
| 608 | # signed endpoint: the latter redirects to a URL that returns 403 Forbidden. |
| 609 | if fallback_candidate: |
| 610 | return fallback_candidate |
| 611 | |
| 612 | uri = play_addr.get("uri") or video.get("vid") or video.get("download_addr", {}).get("uri") |
| 613 | if uri: |
| 614 | # Douyin /aweme/v1/play/ accepts a limited set of ratio strings. |
| 615 | # Map "highest"/"lowest" to 1080p/540p respectively; pass through |
| 616 | # recognised <N>p values; fall back to 1080p for unknowns so the |
| 617 | # legacy default is preserved. |
| 618 | ratio_map = { |
| 619 | "highest": "1080p", |
| 620 | "lowest": "540p", |
| 621 | } |
| 622 | ratio = ratio_map.get(quality, quality if quality in self._QUALITY_TARGET_WIDTH else "1080p") |
| 623 | params = { |
| 624 | "video_id": uri, |
| 625 | "ratio": ratio, |
| 626 | "line": "0", |
| 627 | "is_play_url": "1", |
| 628 | "watermark": "0", |
| 629 | "source": "PackSourceEnum_PUBLISH", |
| 630 | } |
| 631 | signed_url, ua = self.api_client.build_signed_path("/aweme/v1/play/", params) |
| 632 | return signed_url, self._download_headers(user_agent=ua) |
| 633 | |
| 634 | if watermarked_candidate: |
| 635 | return watermarked_candidate |
| 636 | |
| 637 | return None |
| 638 | |
| 639 | # 视频画质选择支持的规格名称。用于 `_pick_play_addr_by_quality` 按 |
| 640 | # 分辨率（width）匹配最接近的档位；匹配不到时自动降级到最高可用档。 |
| 641 | _QUALITY_TARGET_WIDTH: Dict[str, int] = { |
| 642 | "1440p": 2560, |
| 643 | "1080p": 1920, |
| 644 | "720p": 1280, |
| 645 | "540p": 960, |
| 646 | "480p": 854, |
| 647 | "360p": 640, |
| 648 | } |
| 649 | |
| 650 | @staticmethod |
| 651 | def _pick_highest_quality_play_addr(video: Dict[str, Any]) -> Optional[Dict[str, Any]]: |
| 652 | """Shortcut for highest-quality selection (backward-compatible). |
| 653 | |
| 654 | Kept for older callers; new code should use |
| 655 | :meth:`_pick_play_addr_by_quality` with an explicit quality string. |
| 656 | """ |
| 657 | return BaseDownloader._pick_play_addr_by_quality(video, "highest") |
| 658 | |
| 659 | @staticmethod |
| 660 | def _pick_play_addr_by_quality( |
| 661 | video: Dict[str, Any], quality: str = "highest" |
| 662 | ) -> Optional[Dict[str, Any]]: |
| 663 | """从 video.bit_rate 多档率中按目标画质挑选 play_addr。 |
| 664 | |
| 665 | 抖音 API 的 ``video.bit_rate`` 是按质量排序的字典列表，每项含 |
| 666 | ``bit_rate``（码率）与 ``play_addr``（内含 ``url_list`` 与 ``width``）。 |
| 667 | |
| 668 | - ``highest`` / 未知值 / 空：取 bit_rate 最大的档；同码率按 width 决胜。 |
| 669 | - ``lowest``：取 bit_rate 最小的档；同码率按 width 小者优先。 |
| 670 | - ``<N>p``（如 ``1080p``）：按 width 最接近目标值的档；完全没有 bit_rate |
| 671 | 数据时返回 ``None``，让调用方回退到 ``video.play_addr``。 |
| 672 | """ |
| 673 | bit_rates = video.get("bit_rate") if isinstance(video, dict) else None |
| 674 | if not isinstance(bit_rates, list) or not bit_rates: |
| 675 | return None |
| 676 | |
| 677 | # Normalise + collect valid (bit_rate, width, play_addr) triples. |
| 678 | entries: List[Tuple[int, int, Dict[str, Any]]] = [] |
| 679 | for entry in bit_rates: |
| 680 | if not isinstance(entry, dict): |
| 681 | continue |
| 682 | play_addr = entry.get("play_addr") |
| 683 | if not isinstance(play_addr, dict): |
| 684 | continue |
| 685 | try: |
| 686 | br = int(entry.get("bit_rate") or 0) |
| 687 | except (TypeError, ValueError): |
| 688 | br = 0 |
| 689 | width = int(play_addr.get("width") or entry.get("width") or 0) |
| 690 | entries.append((br, width, play_addr)) |
| 691 | if not entries: |
| 692 | return None |
| 693 | |
| 694 | normalised = (quality or "highest").strip().lower() |
| 695 | |
| 696 | if normalised == "lowest": |
| 697 | # Lowest bit_rate; tie-break by smaller width. |
| 698 | entries.sort(key=lambda t: (t[0], t[1])) |
| 699 | return entries[0][2] |
| 700 | |
| 701 | target_width = BaseDownloader._QUALITY_TARGET_WIDTH.get(normalised) |
| 702 | if target_width is not None: |
| 703 | # Closest width to target; tie-break by higher bit_rate so we |
| 704 | # don't accidentally pick a re-encoded low-bitrate copy. |
| 705 | entries.sort(key=lambda t: (abs(t[1] - target_width), -t[0])) |
| 706 | return entries[0][2] |
| 707 | |
| 708 | # Default / "highest": highest bit_rate, tie-break by higher width. |
| 709 | entries.sort(key=lambda t: (-t[0], -t[1])) |
| 710 | return entries[0][2] |
| 711 | |
| 712 | @staticmethod |
| 713 | def _pick_preferred_play_addr( |
| 714 | video: Dict[str, Any], quality: str = "highest" |
| 715 | ) -> Optional[Dict[str, Any]]: |
| 716 | preferred = BaseDownloader._pick_play_addr_by_quality(video, quality) |
| 717 | if preferred: |
| 718 | return preferred |
| 719 | if not isinstance(video, dict): |
| 720 | return None |
| 721 | primary = video.get("play_addr") |
| 722 | if isinstance(primary, dict) and primary.get("uri"): |
| 723 | return primary |
| 724 | for key in BaseDownloader._PLAY_ADDR_KEYS: |
| 725 | candidate = video.get(key) |
| 726 | if isinstance(candidate, dict) and ( |
| 727 | BaseDownloader._extract_first_url(candidate) or candidate.get("uri") |
| 728 | ): |
| 729 | return candidate |
| 730 | return None |
| 731 | |
| 732 | @staticmethod |
| 733 | def _has_video_source(video: Dict[str, Any]) -> bool: |
| 734 | if BaseDownloader._pick_preferred_play_addr(video): |
| 735 | return True |
| 736 | if not isinstance(video, dict): |
| 737 | return False |
| 738 | return bool( |
| 739 | video.get("vid") |
| 740 | or (isinstance(video.get("download_addr"), dict) and video["download_addr"].get("uri")) |
| 741 | ) |
| 742 | |
| 743 | def _collect_image_urls(self, aweme_data: Dict[str, Any]) -> List[str]: |
| 744 | return [ |
| 745 | candidates[0] |
| 746 | for candidates in self._collect_image_url_candidates(aweme_data) |
| 747 | if candidates |
| 748 | ] |
| 749 | |
| 750 | def _collect_image_url_candidates(self, aweme_data: Dict[str, Any]) -> List[List[str]]: |
| 751 | image_urls = [] |
| 752 | gallery_items = self._iter_gallery_items(aweme_data) |
| 753 | for item in gallery_items: |
| 754 | if not isinstance(item, dict): |
| 755 | continue |
| 756 | candidates = self._collect_media_urls( |
| 757 | item.get("watermark_free_download_url_list"), |
| 758 | item, |
| 759 | item.get("origin_image"), |
| 760 | item.get("display_image"), |
| 761 | item.get("download_url"), |
| 762 | item.get("download_addr"), |
| 763 | item.get("download_url_list"), |
| 764 | item.get("owner_watermark_image"), |
| 765 | ) |
| 766 | if candidates: |
| 767 | image_urls.append(candidates) |
| 768 | if not image_urls: |
| 769 | logger.warning( |
| 770 | "No image URLs extracted for aweme %s; gallery items count=%d", |
| 771 | aweme_data.get("aweme_id"), |
| 772 | len(gallery_items), |
| 773 | ) |
| 774 | return image_urls |
| 775 | |
| 776 | def _collect_image_live_urls(self, aweme_data: Dict[str, Any]) -> List[str]: |
| 777 | live_urls: List[str] = [] |
| 778 | quality = str(self.config.get("video_quality") or "highest") |
| 779 | for item in self._iter_gallery_items(aweme_data): |
| 780 | if not isinstance(item, dict): |
| 781 | continue |
| 782 | video = item.get("video") if isinstance(item.get("video"), dict) else {} |
| 783 | # 实况图同样会有 bit_rate 多档，按配置的画质偏好选择对应档位。 |
| 784 | preferred_play_addr = self._pick_preferred_play_addr(video, quality) |
| 785 | live_url = self._pick_first_media_url( |
| 786 | preferred_play_addr, |
| 787 | video.get("play_addr_h264"), |
| 788 | video.get("play_addr_265"), |
| 789 | video.get("play_addr_256"), |
| 790 | video.get("play_addr"), |
| 791 | video.get("download_addr"), |
| 792 | item.get("video_play_addr"), |
| 793 | item.get("video_download_addr"), |
| 794 | ) |
| 795 | if live_url: |
| 796 | live_urls.append(live_url) |
| 797 | return self._deduplicate_urls(live_urls) |
| 798 | |
| 799 | @staticmethod |
| 800 | def _iter_gallery_items(aweme_data: Dict[str, Any]) -> List[Any]: |
| 801 | image_post = aweme_data.get("image_post_info") |
| 802 | if isinstance(image_post, dict): |
| 803 | for key in ("images", "image_list"): |
| 804 | candidate = image_post.get(key) |
| 805 | if isinstance(candidate, list) and candidate: |
| 806 | return candidate |
| 807 | images = aweme_data.get("images") or aweme_data.get("image_list") or [] |
| 808 | if isinstance(images, list): |
| 809 | return images |
| 810 | return [] |
| 811 | |
| 812 | @staticmethod |
| 813 | def _deduplicate_urls(urls: List[str]) -> List[str]: |
| 814 | deduped: List[str] = [] |
| 815 | seen: set[str] = set() |
| 816 | for url in urls: |
| 817 | if not url or url in seen: |
| 818 | continue |
| 819 | seen.add(url) |
| 820 | deduped.append(url) |
| 821 | return deduped |
| 822 | |
| 823 | @staticmethod |
| 824 | def _pick_first_media_url(*sources: Any) -> Optional[str]: |
| 825 | for source in sources: |
| 826 | candidate = BaseDownloader._extract_first_url(source) |
| 827 | if candidate: |
| 828 | return candidate |
| 829 | return None |
| 830 | |
| 831 | @staticmethod |
| 832 | def _collect_media_urls(*sources: Any) -> List[str]: |
| 833 | urls: List[str] = [] |
| 834 | seen: set[str] = set() |
| 835 | for source in sources: |
| 836 | for candidate in sorted( |
| 837 | BaseDownloader._extract_urls(source), |
| 838 | key=BaseDownloader._media_url_priority, |
| 839 | ): |
| 840 | if candidate in seen: |
| 841 | continue |
| 842 | seen.add(candidate) |
| 843 | urls.append(candidate) |
| 844 | return urls |
| 845 | |
| 846 | @staticmethod |
| 847 | def _media_url_priority(url: str) -> int: |
| 848 | normalized = url.lower() |
| 849 | path = (urlparse(url).path or "").lower() |
| 850 | score = 100 if BaseDownloader._is_watermarked_media_url(normalized) else 0 |
| 851 | return score + (1 if ".webp" in path else 0) |
| 852 | |
| 853 | @staticmethod |
| 854 | def _is_watermarked_media_url(url: str) -> bool: |
| 855 | normalized = url.lower() |
| 856 | watermark_hints = ( |
| 857 | "tplv-dy-water", |
| 858 | "dy-water", |
| 859 | "owner_watermark", |
| 860 | "watermark_image", |
| 861 | "watermark=1", |
| 862 | "playwm", |
| 863 | ) |
| 864 | return any(hint in normalized for hint in watermark_hints) |
| 865 | |
| 866 | @staticmethod |
| 867 | def _extract_first_url(source: Any) -> Optional[str]: |
| 868 | urls = BaseDownloader._extract_urls(source) |
| 869 | return urls[0] if urls else None |
| 870 | |
| 871 | @staticmethod |
| 872 | def _extract_urls(source: Any) -> List[str]: |
| 873 | if isinstance(source, dict): |
| 874 | url_list = source.get("url_list") or source.get("urlList") |
| 875 | if isinstance(url_list, list) and url_list: |
| 876 | return [item for item in url_list if isinstance(item, str) and item] |
| 877 | elif isinstance(source, list) and source: |
| 878 | return [item for item in source if isinstance(item, str) and item] |
| 879 | elif isinstance(source, str) and source: |
| 880 | return [source] |
| 881 | return [] |
| 882 | |
| 883 | @staticmethod |
| 884 | def _infer_image_extension(image_url: str) -> str: |
| 885 | allowed_exts = {".jpg", ".jpeg", ".png", ".webp", ".gif"} |
| 886 | if not image_url: |
| 887 | return ".jpg" |
| 888 | |
| 889 | image_path = (urlparse(image_url).path or "").lower() |
| 890 | raw_suffix = Path(image_path).suffix.lower() |
| 891 | if raw_suffix in allowed_exts: |
| 892 | return raw_suffix |
| 893 | |
| 894 | matches = re.findall(r"\.(?:jpe?g|png|webp|gif)(?=[^a-z0-9]|$)", image_path) |
| 895 | if matches: |
| 896 | return matches[-1].lower() |
| 897 | |
| 898 | return ".jpg" |
| 899 | |
| 900 | @staticmethod |
| 901 | def _resolve_publish_time(create_time: Any) -> Tuple[Optional[int], str]: |
| 902 | if create_time in (None, ""): |
| 903 | return None, "" |
| 904 | |
| 905 | try: |
| 906 | publish_ts = int(create_time) |
| 907 | if publish_ts <= 0: |
| 908 | return None, "" |
| 909 | return publish_ts, datetime.fromtimestamp(publish_ts).strftime("%Y-%m-%d") |
| 910 | except (TypeError, ValueError, OSError, OverflowError): |
| 911 | return None, "" |
| 912 | |
| 913 | @staticmethod |
| 914 | def _extract_tags(aweme_data: Dict[str, Any]) -> List[str]: |
| 915 | tags: List[str] = [] |
| 916 | |
| 917 | def _append_tag(raw_tag: Any): |
| 918 | if not raw_tag: |
| 919 | return |
| 920 | normalized_tag = str(raw_tag).strip().lstrip("#") |
| 921 | if normalized_tag and normalized_tag not in tags: |
| 922 | tags.append(normalized_tag) |
| 923 | |
| 924 | for item in aweme_data.get("text_extra") or []: |
| 925 | if not isinstance(item, dict): |
| 926 | continue |
| 927 | _append_tag(item.get("hashtag_name")) |
| 928 | _append_tag(item.get("tag_name")) |
| 929 | |
| 930 | for item in aweme_data.get("cha_list") or []: |
| 931 | if not isinstance(item, dict): |
| 932 | continue |
| 933 | _append_tag(item.get("cha_name")) |
| 934 | _append_tag(item.get("name")) |
| 935 | |
| 936 | desc = aweme_data.get("desc") or "" |
| 937 | for hashtag in re.findall(r"#([^\s#]+)", desc): |
| 938 | _append_tag(hashtag) |
| 939 | |
| 940 | return tags |
| 941 | |
| 942 | def _to_manifest_path(self, path: Path) -> str: |
| 943 | try: |
| 944 | return str(path.relative_to(self.file_manager.base_path)) |
| 945 | except ValueError: |
| 946 | return str(path) |
| 947 |