| 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*- |
| 3 | """ |
| 4 | web_to_md.py - Web Page to Markdown Converter (Python Version) |
| 5 | |
| 6 | Usage: |
| 7 | python scripts/source_to_md/web_to_md.py <url> |
| 8 | python scripts/source_to_md/web_to_md.py <url1> <url2> ... |
| 9 | python scripts/source_to_md/web_to_md.py -f urls.txt |
| 10 | python scripts/source_to_md/web_to_md.py <url> -o output.md |
| 11 | |
| 12 | Dependencies: |
| 13 | pip install requests beautifulsoup4 |
| 14 | |
| 15 | TLS fingerprint handling: |
| 16 | Some sites (e.g., WeChat mp.weixin.qq.com) block Python's default 'requests' |
| 17 | library based on TLS fingerprints (JA3). If 'curl_cffi' is installed, this script |
| 18 | uses it to impersonate a modern Chrome fingerprint and bypass such blocks. If |
| 19 | 'curl_cffi' is unavailable, it silently falls back to plain 'requests' — so |
| 20 | non-blocking sites still work without the extra dependency. |
| 21 | |
| 22 | Install for WeChat / Chinese-portal coverage: |
| 23 | pip install curl_cffi |
| 24 | |
| 25 | If curl_cffi is unavailable on your platform, the Node.js counterpart |
| 26 | (scripts/source_to_md/web_to_md.cjs) remains available as a fallback. |
| 27 | """ |
| 28 | |
| 29 | import argparse |
| 30 | import codecs |
| 31 | import datetime |
| 32 | import io |
| 33 | import json |
| 34 | import os |
| 35 | import re |
| 36 | import sys |
| 37 | import time |
| 38 | from pathlib import Path |
| 39 | from urllib.parse import urljoin, urlparse |
| 40 | |
| 41 | _SCRIPTS_DIR = Path(__file__).resolve().parents[1] |
| 42 | if str(_SCRIPTS_DIR) not in sys.path: |
| 43 | sys.path.insert(0, str(_SCRIPTS_DIR)) |
| 44 | |
| 45 | from console_encoding import configure_utf8_stdio # noqa: E402 |
| 46 | from _conversion_profile import ( # noqa: E402 |
| 47 | profile_path_for, |
| 48 | write_conversion_profile_best_effort, |
| 49 | ) |
| 50 | |
| 51 | configure_utf8_stdio() |
| 52 | |
| 53 | try: |
| 54 | import requests |
| 55 | from bs4 import BeautifulSoup, NavigableString, Tag |
| 56 | except ImportError: |
| 57 | print("Error: This script requires 'requests' and 'beautifulsoup4'.") |
| 58 | print("Please run: pip install requests beautifulsoup4") |
| 59 | sys.exit(1) |
| 60 | |
| 61 | # Prefer curl_cffi for TLS-fingerprint impersonation (bypasses JA3 blocking on |
| 62 | # sites like WeChat). Fall back to plain requests when it's not installed. |
| 63 | try: |
| 64 | from curl_cffi import requests as curl_requests # type: ignore |
| 65 | _CURL_IMPERSONATE = "chrome120" |
| 66 | except ImportError: |
| 67 | curl_requests = None |
| 68 | _CURL_IMPERSONATE = None |
| 69 | |
| 70 | |
| 71 | def _http_get(url: str, *, headers: dict | None = None, timeout: int | None = None, |
| 72 | verify: bool = False, stream: bool = False): |
| 73 | """HTTP GET with curl_cffi preferred, requests fallback. |
| 74 | |
| 75 | Using curl_cffi lets this script fetch sites that reject Python's default |
| 76 | TLS fingerprint (notably mp.weixin.qq.com). Signature mirrors the subset of |
| 77 | requests.get() this script actually uses. |
| 78 | """ |
| 79 | if curl_requests is not None: |
| 80 | return curl_requests.get( |
| 81 | url, headers=headers, timeout=timeout, |
| 82 | verify=verify, impersonate=_CURL_IMPERSONATE, stream=stream, |
| 83 | ) |
| 84 | return requests.get(url, headers=headers, timeout=timeout, |
| 85 | verify=verify, stream=stream) |
| 86 | |
| 87 | |
| 88 | def _normalize_charset(charset: str | None) -> str: |
| 89 | """Return a Python codec name when the declared charset is usable.""" |
| 90 | if not charset: |
| 91 | return "" |
| 92 | charset = charset.strip().strip('"').strip("'").lower() |
| 93 | if not charset: |
| 94 | return "" |
| 95 | try: |
| 96 | return codecs.lookup(charset).name |
| 97 | except LookupError: |
| 98 | return "" |
| 99 | |
| 100 | |
| 101 | def _charset_from_headers(headers: dict) -> str: |
| 102 | content_type = headers.get("Content-Type") or headers.get("content-type") or "" |
| 103 | match = re.search(r"charset\s*=\s*([^;\s]+)", content_type, re.I) |
| 104 | return _normalize_charset(match.group(1)) if match else "" |
| 105 | |
| 106 | |
| 107 | def _charset_from_html(raw: bytes) -> str: |
| 108 | """Extract a charset declaration from the first chunk of HTML bytes.""" |
| 109 | head = raw[:8192] |
| 110 | patterns = [ |
| 111 | rb"<meta[^>]+charset=[\"']?\s*([a-zA-Z0-9_\-]+)", |
| 112 | rb"<meta[^>]+content=[\"'][^\"']*charset=\s*([a-zA-Z0-9_\-]+)", |
| 113 | ] |
| 114 | for pattern in patterns: |
| 115 | match = re.search(pattern, head, re.I) |
| 116 | if match: |
| 117 | return _normalize_charset(match.group(1).decode("ascii", "ignore")) |
| 118 | return "" |
| 119 | |
| 120 | |
| 121 | def _decode_quality_score(text: str) -> int: |
| 122 | """Score obvious decode artifacts; lower is better.""" |
| 123 | mojibake_markers = [ |
| 124 | "�", "锟", "Ã", "Â", "â€", "’", "“", "â€\x9d", |
| 125 | "琚", "佸", "鍦", "涓", "鏄", "寤", "骞", "鏈", "鏃", "鈥", |
| 126 | ] |
| 127 | marker_hits = sum(text.count(marker) for marker in mojibake_markers) |
| 128 | control_hits = sum(1 for ch in text if ord(ch) < 32 and ch not in "\t\n\r") |
| 129 | return marker_hits * 20 + control_hits * 10 + text.count("\ufffd") * 50 |
| 130 | |
| 131 | |
| 132 | def _decode_response_text(response) -> str: |
| 133 | """Decode HTTP response bytes without letting guessed encodings override declarations.""" |
| 134 | raw = response.content |
| 135 | declared = [ |
| 136 | _charset_from_headers(response.headers), |
| 137 | _charset_from_html(raw), |
| 138 | ] |
| 139 | if raw.startswith(codecs.BOM_UTF8): |
| 140 | declared.insert(0, "utf-8-sig") |
| 141 | |
| 142 | seen = set() |
| 143 | declared = [enc for enc in declared if enc and not (enc in seen or seen.add(enc))] |
| 144 | for enc in declared: |
| 145 | try: |
| 146 | return raw.decode(enc) |
| 147 | except UnicodeDecodeError: |
| 148 | continue |
| 149 | |
| 150 | candidates = [] |
| 151 | for enc in [ |
| 152 | getattr(response, "encoding", None), |
| 153 | getattr(response, "apparent_encoding", None), |
| 154 | "utf-8", |
| 155 | "gb18030", |
| 156 | "big5", |
| 157 | ]: |
| 158 | enc = _normalize_charset(enc) |
| 159 | if enc and enc not in candidates: |
| 160 | candidates.append(enc) |
| 161 | |
| 162 | decoded = [] |
| 163 | for enc in candidates: |
| 164 | try: |
| 165 | text = raw.decode(enc) |
| 166 | except UnicodeDecodeError: |
| 167 | continue |
| 168 | decoded.append((_decode_quality_score(text), enc, text)) |
| 169 | |
| 170 | if decoded: |
| 171 | decoded.sort(key=lambda item: item[0]) |
| 172 | return decoded[0][2] |
| 173 | |
| 174 | return raw.decode("utf-8", errors="replace") |
| 175 | |
| 176 | try: |
| 177 | from PIL import Image |
| 178 | PILLOW_AVAILABLE = True |
| 179 | except ImportError: |
| 180 | PILLOW_AVAILABLE = False |
| 181 | print("[WARN] Pillow not installed. WebP images will not be converted to PNG.") |
| 182 | print(" Run: pip install Pillow") |
| 183 | |
| 184 | # ============ Config ============ |
| 185 | CONFIG = { |
| 186 | "output_dir": "./projects", |
| 187 | "timeout": 30, |
| 188 | "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", |
| 189 | # Specific content identifiers often found in Chinese CMS (Gov/News) |
| 190 | "content_selectors": [ |
| 191 | {"class_": re.compile(r"tys-main-zt-show", re.I)}, |
| 192 | {"class_": re.compile(r"tys-main", re.I)}, |
| 193 | {"class_": "TRS_Editor"}, |
| 194 | {"class_": "TRS_UEDITOR"}, |
| 195 | {"class_": "ucontent"}, |
| 196 | {"class_": "article-content"}, |
| 197 | {"class_": "news-content"}, |
| 198 | {"class_": "detail-content"}, |
| 199 | {"class_": "content-text"}, |
| 200 | {"class_": "pages_content"}, |
| 201 | {"class_": "zwgk_content"}, |
| 202 | {"class_": "content_detail"}, |
| 203 | {"class_": "text_content"}, |
| 204 | {"class_": "main-content"}, |
| 205 | {"class_": "main_content"}, |
| 206 | {"class_": "view-content"}, |
| 207 | {"class_": "info-content"}, |
| 208 | {"id": "Zoom"}, |
| 209 | {"id": "content"}, |
| 210 | {"id": "article"}, |
| 211 | {"class_": "content"}, |
| 212 | {"name": "article"}, # tag name |
| 213 | {"name": "main"}, # tag name |
| 214 | ] |
| 215 | } |
| 216 | |
| 217 | |
| 218 | def fetch_url(url: str) -> str: |
| 219 | """Fetch a web page with explicit headers and encoding detection. |
| 220 | |
| 221 | Args: |
| 222 | url: Target URL. |
| 223 | |
| 224 | Returns: |
| 225 | The response body as text. |
| 226 | """ |
| 227 | headers = { |
| 228 | "User-Agent": CONFIG["user_agent"], |
| 229 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", |
| 230 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8" |
| 231 | } |
| 232 | |
| 233 | try: |
| 234 | response = _http_get(url, headers=headers, |
| 235 | timeout=CONFIG["timeout"], verify=False) |
| 236 | response.raise_for_status() |
| 237 | |
| 238 | return _decode_response_text(response) |
| 239 | except Exception as e: |
| 240 | raise Exception(f"Failed to fetch {url}: {str(e)}") |
| 241 | |
| 242 | |
| 243 | def clean_title(title: str) -> str: |
| 244 | """Remove common site suffixes from a title.""" |
| 245 | if not title: |
| 246 | return "" |
| 247 | # Remove site name suffixes often found in Chinese titles |
| 248 | clean = re.sub(r"[-_|].*?(政府|门户|网站|委员会).*$", "", title) |
| 249 | return clean.strip() |
| 250 | |
| 251 | |
| 252 | def sanitize_filename(name: str) -> str: |
| 253 | """Sanitize a string for filesystem-safe filenames.""" |
| 254 | # Replace whitespace with underscore first |
| 255 | clean = re.sub(r'\s+', '_', name) |
| 256 | # Remove all except Chinese, English, Numbers, Underscore |
| 257 | clean = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9_]', '', clean) |
| 258 | # Collapse repeating underscores |
| 259 | clean = re.sub(r'_+', '_', clean) |
| 260 | return clean[:80] # Truncate |
| 261 | |
| 262 | |
| 263 | def derive_base_name(title: str, url: str) -> str: |
| 264 | """Derive a safe, non-empty basename from a title or URL.""" |
| 265 | base = sanitize_filename(title or "") |
| 266 | if base: |
| 267 | return base |
| 268 | |
| 269 | parsed = urlparse(url) |
| 270 | path = parsed.path.strip('/') |
| 271 | if path: |
| 272 | candidate = f"{parsed.netloc}_{path}" |
| 273 | else: |
| 274 | candidate = parsed.netloc or "untitled" |
| 275 | base = sanitize_filename(candidate) |
| 276 | if base: |
| 277 | return base |
| 278 | |
| 279 | ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") |
| 280 | return f"untitled_{ts}" |
| 281 | |
| 282 | |
| 283 | def build_image_filename(abs_url: str, seq: int, content_type: str | None = None) -> str: |
| 284 | """Build a safe image filename from URL metadata.""" |
| 285 | parsed = urlparse(abs_url) |
| 286 | basename = os.path.basename(parsed.path).split('?')[0] |
| 287 | stem, ext = os.path.splitext(basename) |
| 288 | if not ext or len(ext) > 5 or '/' in ext: |
| 289 | ext = "" |
| 290 | if not ext and content_type: |
| 291 | ctype = content_type.split(';')[0].lower() |
| 292 | ext_map = { |
| 293 | "image/jpeg": ".jpg", |
| 294 | "image/jpg": ".jpg", |
| 295 | "image/png": ".png", |
| 296 | "image/gif": ".gif", |
| 297 | "image/webp": ".webp", |
| 298 | } |
| 299 | ext = ext_map.get(ctype, "") |
| 300 | if not ext: |
| 301 | ext = ".jpg" |
| 302 | stem = sanitize_filename(stem) if stem else f"image_{seq}" |
| 303 | return f"{stem}{ext}" |
| 304 | |
| 305 | |
| 306 | def download_and_rewrite_images( |
| 307 | content_element: Tag | None, |
| 308 | page_url: str, |
| 309 | image_dir: str, |
| 310 | rel_prefix: str, |
| 311 | ) -> int: |
| 312 | """Download images under the main content node and rewrite `src` paths.""" |
| 313 | if content_element is None: |
| 314 | return 0 |
| 315 | images = list(content_element.find_all("img")) |
| 316 | if not images: |
| 317 | return 0 |
| 318 | |
| 319 | os.makedirs(image_dir, exist_ok=True) |
| 320 | downloaded = {} |
| 321 | manifest_by_filename: dict[str, dict[str, object]] = {} |
| 322 | saved = 0 |
| 323 | |
| 324 | for idx, img in enumerate(images): |
| 325 | # Prefer lazy-load attributes — WeChat, Zhihu, and many CMSes keep the |
| 326 | # real image URL in data-src / data-original / data-lazy-src, with |
| 327 | # `src` pointing at a 1x1 placeholder or a template literal. |
| 328 | candidates = [ |
| 329 | img.get("data-src"), |
| 330 | img.get("data-original"), |
| 331 | img.get("data-lazy-src"), |
| 332 | img.get("data-actualsrc"), |
| 333 | img.get("src"), |
| 334 | ] |
| 335 | src = next((s for s in candidates |
| 336 | if s and not s.startswith("data:") |
| 337 | and s.startswith(("http://", "https://", "//", "/"))), None) |
| 338 | if not src: |
| 339 | continue |
| 340 | |
| 341 | # Promote the chosen URL into the element's src so downstream rewrite |
| 342 | # (which matches on src) can retarget it to the local file. |
| 343 | img["src"] = src |
| 344 | |
| 345 | abs_url = urljoin(page_url, src) |
| 346 | content_type = "" |
| 347 | converted_from = "" |
| 348 | if abs_url in downloaded: |
| 349 | saved_name = downloaded[abs_url] |
| 350 | else: |
| 351 | try: |
| 352 | resp = _http_get( |
| 353 | abs_url, |
| 354 | headers={"User-Agent": CONFIG["user_agent"]}, |
| 355 | timeout=CONFIG["timeout"], |
| 356 | verify=False, |
| 357 | ) |
| 358 | resp.raise_for_status() |
| 359 | filename = build_image_filename( |
| 360 | abs_url, idx, resp.headers.get("Content-Type")) |
| 361 | |
| 362 | # Check if image is webp and convert to png |
| 363 | stem, ext = os.path.splitext(filename) |
| 364 | content_type = resp.headers.get("Content-Type", "").lower() |
| 365 | is_webp = ext.lower() == ".webp" or "webp" in content_type |
| 366 | |
| 367 | if is_webp and PILLOW_AVAILABLE: |
| 368 | # Convert webp to png (optimized) |
| 369 | try: |
| 370 | img_data = io.BytesIO(resp.content) |
| 371 | pil_image = Image.open(img_data) |
| 372 | |
| 373 | # Update filename to .png |
| 374 | converted_from = filename |
| 375 | filename = f"{stem}.png" |
| 376 | local_path = os.path.join(image_dir, filename) |
| 377 | |
| 378 | # Avoid accidental overwrites if filenames collide |
| 379 | counter = 1 |
| 380 | while os.path.exists(local_path): |
| 381 | local_path = os.path.join( |
| 382 | image_dir, f"{stem}_{counter}.png") |
| 383 | filename = os.path.basename(local_path) |
| 384 | counter += 1 |
| 385 | |
| 386 | # Save as PNG directly (Pillow auto-converts, no need for explicit mode conversion) |
| 387 | pil_image.save(local_path, 'PNG', optimize=False) |
| 388 | pil_image.close() |
| 389 | print(f" [INFO] Converted webp to png: {filename}") |
| 390 | except Exception as convert_err: |
| 391 | print( |
| 392 | f" [WARN] Failed to convert webp: {convert_err}, saving as-is") |
| 393 | local_path = os.path.join(image_dir, filename) |
| 394 | counter = 1 |
| 395 | stem, ext = os.path.splitext(filename) |
| 396 | while os.path.exists(local_path): |
| 397 | local_path = os.path.join( |
| 398 | image_dir, f"{stem}_{counter}{ext}") |
| 399 | filename = os.path.basename(local_path) |
| 400 | counter += 1 |
| 401 | with open(local_path, "wb") as f: |
| 402 | f.write(resp.content) |
| 403 | else: |
| 404 | local_path = os.path.join(image_dir, filename) |
| 405 | |
| 406 | # Avoid accidental overwrites if filenames collide |
| 407 | counter = 1 |
| 408 | stem, ext = os.path.splitext(filename) |
| 409 | while os.path.exists(local_path): |
| 410 | local_path = os.path.join( |
| 411 | image_dir, f"{stem}_{counter}{ext}") |
| 412 | filename = os.path.basename(local_path) |
| 413 | counter += 1 |
| 414 | |
| 415 | with open(local_path, "wb") as f: |
| 416 | f.write(resp.content) |
| 417 | downloaded[abs_url] = filename |
| 418 | saved_name = filename |
| 419 | manifest_by_filename[saved_name] = { |
| 420 | "index": len(manifest_by_filename) + 1, |
| 421 | "filename": saved_name, |
| 422 | "original_filename": converted_from or saved_name, |
| 423 | "asset_kind": "bitmap", |
| 424 | "svg_renderable": True, |
| 425 | "pptx_native_supported": True, |
| 426 | "source_kind": "web_image", |
| 427 | "source_url": abs_url, |
| 428 | "source_page_url": page_url, |
| 429 | "content_type": content_type.split(";")[0] if content_type else "", |
| 430 | "occurrences": [], |
| 431 | } |
| 432 | saved += 1 |
| 433 | except Exception as e: |
| 434 | print(f" [WARN] Skip image {abs_url}: {e}") |
| 435 | continue |
| 436 | |
| 437 | rel_path = os.path.join( |
| 438 | rel_prefix, saved_name) if rel_prefix else saved_name |
| 439 | img["src"] = rel_path |
| 440 | manifest_item = manifest_by_filename.get(saved_name) |
| 441 | if manifest_item is not None: |
| 442 | occurrences = manifest_item.setdefault("occurrences", []) |
| 443 | if isinstance(occurrences, list): |
| 444 | occurrences.append({ |
| 445 | "occurrence_index": idx + 1, |
| 446 | "source_url": abs_url, |
| 447 | "alt_text": img.get("alt", ""), |
| 448 | }) |
| 449 | manifest_item["usage_count"] = len(occurrences) |
| 450 | |
| 451 | if manifest_by_filename: |
| 452 | manifest_path = os.path.join(image_dir, "image_manifest.json") |
| 453 | with open(manifest_path, "w", encoding="utf-8") as f: |
| 454 | json.dump( |
| 455 | list(manifest_by_filename.values()), |
| 456 | f, |
| 457 | ensure_ascii=False, |
| 458 | indent=2, |
| 459 | ) |
| 460 | f.write("\n") |
| 461 | |
| 462 | return saved |
| 463 | |
| 464 | |
| 465 | def extract_metadata(soup: BeautifulSoup, url: str) -> dict[str, str]: |
| 466 | """Extract page metadata such as title, date, description, and author.""" |
| 467 | |
| 468 | # 1. Title |
| 469 | title_tag = soup.title |
| 470 | title = clean_title(title_tag.string if title_tag else "") |
| 471 | |
| 472 | # 2. Meta tags |
| 473 | metas = {} |
| 474 | for meta in soup.find_all("meta"): |
| 475 | name = meta.get("name") or meta.get("property") |
| 476 | content = meta.get("content") |
| 477 | if name and content: |
| 478 | metas[name.lower()] = content.strip() |
| 479 | |
| 480 | # 3. Date Extraction Strategies |
| 481 | date = ( |
| 482 | metas.get("article:published_time") or |
| 483 | metas.get("og:published_time") or |
| 484 | metas.get("pubdate") or |
| 485 | metas.get("publishdate") or |
| 486 | metas.get("date") |
| 487 | ) |
| 488 | |
| 489 | if not date: |
| 490 | # Try matching date patterns in the text |
| 491 | text_content = soup.get_text() |
| 492 | date_patterns = [ |
| 493 | r"发布[时日]间[：:]\s*(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)", |
| 494 | r"日期[：:]\s*(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)", |
| 495 | r"(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)\s*(?:发布|来源)", |
| 496 | r"时间[：:]\s*(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})" |
| 497 | ] |
| 498 | for pattern in date_patterns: |
| 499 | match = re.search(pattern, text_content) |
| 500 | if match: |
| 501 | date = match.group(1).replace( |
| 502 | "年", "-").replace("月", "-").replace("日", "") |
| 503 | break |
| 504 | |
| 505 | if not date: |
| 506 | # Try URL matching |
| 507 | match = re.search(r"(\d{4})(\d{2})[\/_](?:t\d+_)?", url) |
| 508 | if match: |
| 509 | date = f"{match.group(1)}-{match.group(2)}" |
| 510 | else: |
| 511 | match = re.search(r"(\d{4})[-\/](\d{2})[-\/](\d{2})", url) |
| 512 | if match: |
| 513 | date = f"{match.group(1)}-{match.group(2)}-{match.group(3)}" |
| 514 | |
| 515 | # 4. Description |
| 516 | description = ( |
| 517 | metas.get("description") or |
| 518 | metas.get("og:description") or |
| 519 | metas.get("twitter:description") or |
| 520 | "" |
| 521 | ) |
| 522 | |
| 523 | # 5. Author/Source |
| 524 | author = metas.get("author") or metas.get("article:author") |
| 525 | if not author: |
| 526 | # Try common patterns |
| 527 | source_patterns = [ |
| 528 | r"来源[：:]\s*([^\s<]+)", |
| 529 | r"发布(?:单位|机构)[：:]\s*([^\s<]+)" |
| 530 | ] |
| 531 | for pattern in source_patterns: |
| 532 | match = re.search(pattern, soup.get_text()) |
| 533 | if match: |
| 534 | author = match.group(1) |
| 535 | break |
| 536 | |
| 537 | return { |
| 538 | "title": title or metas.get("og:title") or "Untitled", |
| 539 | "date": date or "", |
| 540 | "description": description, |
| 541 | "author": author or "", |
| 542 | "source_url": url |
| 543 | } |
| 544 | |
| 545 | |
| 546 | def find_main_content(soup: BeautifulSoup) -> Tag | None: |
| 547 | """Find the most likely main content container in a page.""" |
| 548 | # 1. Clean up first (remove known clutter) |
| 549 | for tag in soup(["script", "style", "nav", "header", "footer", "aside", "noscript", "iframe"]): |
| 550 | tag.decompose() |
| 551 | |
| 552 | best_element = None |
| 553 | max_score = 0 |
| 554 | |
| 555 | # 2. Strategy A: Check specific classes/ids |
| 556 | for selector in CONFIG["content_selectors"]: |
| 557 | if "name" in selector: |
| 558 | # Tag name match (article, main) |
| 559 | elements = soup.find_all(selector["name"]) |
| 560 | else: |
| 561 | # Class or ID match |
| 562 | elements = soup.find_all(attrs=selector) |
| 563 | |
| 564 | for el in elements: |
| 565 | # Score based on text length and chinese character count |
| 566 | text = el.get_text(strip=True) |
| 567 | length = len(text) |
| 568 | if length < 100: |
| 569 | continue |
| 570 | |
| 571 | chinese_count = len(re.findall(r'[\u4e00-\u9fa5]', text)) |
| 572 | score = length + (chinese_count * 2) |
| 573 | |
| 574 | if score > max_score: |
| 575 | max_score = score |
| 576 | best_element = el |
| 577 | |
| 578 | # 3. Strategy B: If no specific container found, look for dense text areas with paragraphs |
| 579 | if not best_element or max_score < 200: |
| 580 | for div in soup.find_all("div"): |
| 581 | p_count = len(div.find_all("p", recursive=False)) |
| 582 | # recursive=False ensures we don't just pick the top-level body by accident |
| 583 | # but sometimes content is nested deep |
| 584 | if p_count == 0: |
| 585 | # Check if it has lots of text even without p tags (br tags?) |
| 586 | pass |
| 587 | |
| 588 | text = div.get_text(strip=True) |
| 589 | if len(text) > 200 and p_count >= 1: |
| 590 | # Recalculate deep score |
| 591 | chinese_count = len(re.findall(r'[\u4e00-\u9fa5]', text)) |
| 592 | score = len(text) + (chinese_count * 2) + (p_count * 50) |
| 593 | if score > max_score: |
| 594 | max_score = score |
| 595 | best_element = div |
| 596 | |
| 597 | # Fallback to body |
| 598 | return best_element if best_element else soup.body |
| 599 | |
| 600 | |
| 601 | def element_to_markdown(element: Tag | NavigableString | None) -> str: |
| 602 | """Recursively convert a BeautifulSoup node to Markdown.""" |
| 603 | if element is None: |
| 604 | return "" |
| 605 | |
| 606 | if isinstance(element, NavigableString): |
| 607 | text = str(element).strip() |
| 608 | return text if text else "" |
| 609 | |
| 610 | tag_name = element.name.lower() |
| 611 | |
| 612 | # Skip hidden/unwanted tags |
| 613 | if tag_name in ['script', 'style', 'meta', 'link', 'input', 'button', 'select']: |
| 614 | return "" |
| 615 | |
| 616 | content = "" |
| 617 | for child in element.children: |
| 618 | content += element_to_markdown(child) |
| 619 | # Add spacing logic here if needed, but usually block elements handle it |
| 620 | |
| 621 | # Block handlers |
| 622 | if tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: |
| 623 | level = int(tag_name[1]) |
| 624 | return f"\n{'#' * level} {content}\n\n" |
| 625 | |
| 626 | elif tag_name == 'p': |
| 627 | # Clean up internal whitespace |
| 628 | content = re.sub(r'\s+', ' ', content).strip() |
| 629 | return f"\n{content}\n\n" if content else "" |
| 630 | |
| 631 | elif tag_name == 'br': |
| 632 | return " \n" |
| 633 | |
| 634 | elif tag_name == 'hr': |
| 635 | return "\n---\n" |
| 636 | |
| 637 | elif tag_name == 'div': |
| 638 | return f"\n{content}\n" |
| 639 | |
| 640 | elif tag_name == 'blockquote': |
| 641 | lines = content.strip().split('\n') |
| 642 | quoted = '\n'.join([f"> {line}" for line in lines if line.strip()]) |
| 643 | return f"\n{quoted}\n\n" |
| 644 | |
| 645 | elif tag_name in ['ul', 'ol']: |
| 646 | # This is tricky without "state" (knowing we are in a list) |
| 647 | # For simplicity in this recursive version, we rely on LI handling |
| 648 | return f"\n{content}\n" |
| 649 | |
| 650 | elif tag_name == 'li': |
| 651 | # Simple list handling |
| 652 | clean_content = content.strip() |
| 653 | return f"- {clean_content}\n" |
| 654 | |
| 655 | elif tag_name == 'pre': |
| 656 | return f"\n```\n{content}\n```\n\n" |
| 657 | |
| 658 | elif tag_name == 'code': |
| 659 | # If parent is pre, handle in pre. If inline: |
| 660 | parent = element.parent |
| 661 | if parent and parent.name == 'pre': |
| 662 | return content |
| 663 | return f"`{content}`" |
| 664 | |
| 665 | elif tag_name == 'a': |
| 666 | href = element.get('href', '') |
| 667 | if href and not href.startswith('javascript:'): |
| 668 | return f"[{content}]({href})" |
| 669 | return content |
| 670 | |
| 671 | elif tag_name == 'img': |
| 672 | src = element.get('src', '') |
| 673 | alt = element.get('alt', '') |
| 674 | if src: |
| 675 | return f"" |
| 676 | return "" |
| 677 | |
| 678 | elif tag_name == 'table': |
| 679 | # Basic table text extraction, full markdown table support is complex |
| 680 | # Leaving as raw text or simplistic conversion for now |
| 681 | # Ideally, we'd parse TRs and TDs |
| 682 | return f"\n{content}\n" |
| 683 | |
| 684 | elif tag_name == 'tr': |
| 685 | return f"{content}|\n" |
| 686 | |
| 687 | elif tag_name in ['td', 'th']: |
| 688 | return f"| {content.strip()} " |
| 689 | |
| 690 | # Style formatting |
| 691 | elif tag_name in ['strong', 'b']: |
| 692 | return f"**{content}**" |
| 693 | elif tag_name in ['em', 'i']: |
| 694 | return f"*{content}*" |
| 695 | elif tag_name in ['del', 's', 'strike']: |
| 696 | return f"~~{content}~~" |
| 697 | |
| 698 | # Default for span, section, etc. |
| 699 | return f"{content} " |
| 700 | |
| 701 | |
| 702 | def simple_html_to_markdown_traversal(soup: Tag | BeautifulSoup | None) -> str: |
| 703 | """Convert HTML content to Markdown using BeautifulSoup traversal.""" |
| 704 | lines = [] |
| 705 | |
| 706 | def traverse(node: Tag | NavigableString) -> str: |
| 707 | if isinstance(node, NavigableString): |
| 708 | text = str(node) |
| 709 | # Normalize whitespace but keep single spaces |
| 710 | text = re.sub(r'\s+', ' ', text) |
| 711 | if text.strip(): |
| 712 | return text |
| 713 | return "" |
| 714 | |
| 715 | if node.name in ['script', 'style', 'comment', 'meta', 'link']: |
| 716 | return "" |
| 717 | |
| 718 | # Handle Block Elements |
| 719 | is_block = node.name in ['p', 'div', 'h1', 'h2', 'h3', 'h4', |
| 720 | 'h5', 'h6', 'li', 'blockquote', 'pre', 'hr', 'table', 'tr'] |
| 721 | |
| 722 | # Pre-processing |
| 723 | prefix = "" |
| 724 | suffix = "" |
| 725 | |
| 726 | if node.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: |
| 727 | level = int(node.name[1]) |
| 728 | prefix = f"\n\n{'#' * level} " |
| 729 | suffix = "\n\n" |
| 730 | elif node.name == 'p': |
| 731 | prefix = "\n\n" |
| 732 | suffix = "\n\n" |
| 733 | elif node.name == 'li': |
| 734 | prefix = "\n- " |
| 735 | elif node.name == 'blockquote': |
| 736 | prefix = "\n> " |
| 737 | suffix = "\n" |
| 738 | elif node.name == 'hr': |
| 739 | return "\n\n---\n\n" |
| 740 | elif node.name == 'br': |
| 741 | return " \n" |
| 742 | elif node.name == 'pre': |
| 743 | # Extract raw text from pre to preserve formatting |
| 744 | return f"\n\n```\n{node.get_text()}\n```\n\n" |
| 745 | |
| 746 | # Inline formatting |
| 747 | if node.name in ['strong', 'b']: |
| 748 | prefix, suffix = "**", "**" |
| 749 | elif node.name in ['em', 'i']: |
| 750 | prefix, suffix = "*", "*" |
| 751 | elif node.name == 'code' and node.parent.name != 'pre': |
| 752 | prefix, suffix = "`", "`" |
| 753 | elif node.name == 'a': |
| 754 | href = node.get('href') |
| 755 | if href and not href.startswith('javascript:'): |
| 756 | prefix = "[" |
| 757 | suffix = f"]({href})" |
| 758 | else: |
| 759 | prefix, suffix = "", "" |
| 760 | elif node.name == 'img': |
| 761 | src = node.get('src') |
| 762 | alt = node.get('alt', '') |
| 763 | if src: |
| 764 | return f"" |
| 765 | return "" |
| 766 | |
| 767 | # Recurse |
| 768 | inner_text = "" |
| 769 | for child in node.children: |
| 770 | res = traverse(child) |
| 771 | if res: |
| 772 | inner_text += res |
| 773 | |
| 774 | # Post-processing for tables (simplified) |
| 775 | if node.name == 'tr': |
| 776 | # count tds |
| 777 | cells = [c.get_text(strip=True) for c in node.find_all( |
| 778 | ['td', 'th'], recursive=False)] |
| 779 | return f"| {' | '.join(cells)} |\n" |
| 780 | if node.name == 'table': |
| 781 | # Try to add a separator line after first row if it looks like a header |
| 782 | rows = inner_text.strip().split('\n') |
| 783 | if rows: |
| 784 | cols_count = rows[0].count('|') - 1 |
| 785 | if cols_count > 0: |
| 786 | # rough approx |
| 787 | sep = "| " + " | ".join(["---"] * int(cols_count/2)) + " |" |
| 788 | # Actually, the traverse of TR returns newline terminated strings. |
| 789 | # Let's just return what we gathered. |
| 790 | pass |
| 791 | return f"\n\n{inner_text}\n\n" |
| 792 | |
| 793 | return f"{prefix}{inner_text}{suffix}" |
| 794 | |
| 795 | # Actually, a simpler approach for this script is "just get string" but with markers? |
| 796 | # Let's use a simplified approach: use get_text but with 'separator' logic? |
| 797 | # text = soup.get_text(separator='\n\n') |
| 798 | # But that loses links and boldness. |
| 799 | |
| 800 | # Recommendation: Let's stick to the traversal above which constructs a string. |
| 801 | md = traverse(soup) |
| 802 | |
| 803 | # Cleanup Markdown |
| 804 | if md: |
| 805 | # Remove excessive newlines |
| 806 | md = re.sub(r'\n{3,}', '\n\n', md) |
| 807 | md = md.strip() |
| 808 | return md or "" |
| 809 | |
| 810 | |
| 811 | def process_url(url: str, output_file: str | None = None) -> tuple[bool, str, str | None, str | None]: |
| 812 | """Fetch, convert, and save one web page as Markdown. |
| 813 | |
| 814 | Returns (success, url, error, output_path). output_path is the actual saved |
| 815 | Markdown path (derived from the article title when no output_file is given), |
| 816 | so a caller can locate a title-named file it did not choose upfront. |
| 817 | """ |
| 818 | print(f"\n[Fetching] {url}") |
| 819 | try: |
| 820 | html = fetch_url(url) |
| 821 | soup = BeautifulSoup(html, 'html.parser') |
| 822 | |
| 823 | # Extract Metadata |
| 824 | metadata = extract_metadata(soup, url) |
| 825 | print(f" [OK] Title: {metadata['title']}") |
| 826 | if metadata['date']: |
| 827 | print(f" [OK] Date: {metadata['date']}") |
| 828 | |
| 829 | # Determine output path and image directory upfront |
| 830 | if output_file: |
| 831 | output_path = output_file |
| 832 | else: |
| 833 | base_name = derive_base_name(metadata['title'], url) |
| 834 | filename = f"{base_name}.md" |
| 835 | output_path = os.path.join(CONFIG["output_dir"], filename) |
| 836 | |
| 837 | output_dirname = os.path.dirname(output_path) or "." |
| 838 | os.makedirs(output_dirname, exist_ok=True) |
| 839 | base_name = os.path.splitext(os.path.basename(output_path))[0] |
| 840 | image_dir = os.path.join(output_dirname, f"{base_name}_files") |
| 841 | rel_image_prefix = os.path.relpath(image_dir, output_dirname) |
| 842 | |
| 843 | # Extract Content |
| 844 | content_div = find_main_content(soup) |
| 845 | |
| 846 | # Download images and rewrite src before markdown conversion |
| 847 | image_count = download_and_rewrite_images( |
| 848 | content_div, url, image_dir, rel_image_prefix) |
| 849 | if image_count: |
| 850 | print(f" [OK] Images: {image_count} saved to {image_dir}") |
| 851 | |
| 852 | # Convert to MD |
| 853 | # Note: We pass the element to our traversal function |
| 854 | markdown_text = simple_html_to_markdown_traversal(content_div) |
| 855 | print(f" [OK] Content: {len(markdown_text)} chars") |
| 856 | |
| 857 | # Construct content |
| 858 | final_output = [] |
| 859 | final_output.append("<!--") |
| 860 | final_output.append(f" Source: {url}") |
| 861 | final_output.append( |
| 862 | f" Crawled: {datetime.datetime.now().isoformat()}") |
| 863 | if metadata['date']: |
| 864 | final_output.append(f" Published: {metadata['date']}") |
| 865 | if metadata['author']: |
| 866 | final_output.append(f" Author: {metadata['author']}") |
| 867 | final_output.append("-->\n") |
| 868 | |
| 869 | if metadata['title']: |
| 870 | final_output.append(f"# {metadata['title']}\n") |
| 871 | |
| 872 | if metadata['description']: |
| 873 | final_output.append(f"> {metadata['description']}\n") |
| 874 | |
| 875 | final_output.append(markdown_text) |
| 876 | |
| 877 | full_content = "\n".join(final_output) |
| 878 | |
| 879 | with open(output_path, 'w', encoding='utf-8') as f: |
| 880 | f.write(full_content) |
| 881 | profile_path = write_conversion_profile_best_effort( |
| 882 | input_path=url, |
| 883 | markdown_path=output_path, |
| 884 | converter="web_to_md.py", |
| 885 | conversion_type="web", |
| 886 | asset_dir=image_dir, |
| 887 | ) |
| 888 | |
| 889 | print(f" [OK] Saved: {output_path}") |
| 890 | if profile_path: |
| 891 | print(f" [OK] Conversion profile: {profile_path}") |
| 892 | return True, url, None, output_path |
| 893 | |
| 894 | except Exception as e: |
| 895 | print(f" [ERROR] {str(e)}") |
| 896 | return False, url, str(e), None |
| 897 | |
| 898 | |
| 899 | def _write_emit_result(result_file: str, url: str, markdown_path: str) -> None: |
| 900 | """Write the actual saved path as JSON so a caller can locate the output.""" |
| 901 | md = Path(markdown_path).resolve() |
| 902 | profile = profile_path_for(md) |
| 903 | payload = { |
| 904 | "input": url, |
| 905 | "markdown": str(md), |
| 906 | "conversion_profile": str(profile) if profile.is_file() else "", |
| 907 | } |
| 908 | try: |
| 909 | Path(result_file).write_text( |
| 910 | json.dumps(payload, ensure_ascii=False), encoding="utf-8") |
| 911 | except OSError as exc: |
| 912 | print(f" [WARN] Could not write --emit-result: {exc}") |
| 913 | |
| 914 | |
| 915 | def main(argv: list[str] | None = None) -> int: |
| 916 | """Run the CLI entry point.""" |
| 917 | parser = argparse.ArgumentParser( |
| 918 | description="Web to Markdown Converter (Python)") |
| 919 | parser.add_argument("urls", nargs="*", help="URLs to process") |
| 920 | parser.add_argument( |
| 921 | "-f", "--file", help="File containing URLs (one per line)") |
| 922 | parser.add_argument("-o", "--output", help="Output file (single URL only)") |
| 923 | parser.add_argument("-d", "--dir", help="Output directory") |
| 924 | parser.add_argument( |
| 925 | "--emit-result", |
| 926 | help="On success, write the saved output path as JSON to this file " |
| 927 | "(single-URL dispatcher use, so a title-named file can be located)") |
| 928 | |
| 929 | args = parser.parse_args(argv) |
| 930 | |
| 931 | if args.dir: |
| 932 | CONFIG["output_dir"] = args.dir |
| 933 | |
| 934 | targets = [] |
| 935 | if args.urls: |
| 936 | targets.extend(args.urls) |
| 937 | |
| 938 | if args.file: |
| 939 | if os.path.exists(args.file): |
| 940 | with open(args.file, 'r', encoding='utf-8') as f: |
| 941 | lines = [l.strip() for l in f if l.strip() |
| 942 | and not l.strip().startswith("#")] |
| 943 | targets.extend(lines) |
| 944 | else: |
| 945 | print(f"Error: File {args.file} not found", file=sys.stderr) |
| 946 | return 1 |
| 947 | |
| 948 | if not targets: |
| 949 | parser.print_usage(sys.stderr) |
| 950 | print( |
| 951 | "web_to_md.py: error: at least one URL or --file is required", |
| 952 | file=sys.stderr, |
| 953 | ) |
| 954 | return 2 |
| 955 | |
| 956 | results = [] |
| 957 | for i, url in enumerate(targets): |
| 958 | # Allow specific output file only if 1 URL |
| 959 | out = args.output if (len(targets) == 1 and args.output) else None |
| 960 | success, url, err, out_path = process_url(url, out) |
| 961 | results.append((success, url, err)) |
| 962 | if args.emit_result and success and out_path: |
| 963 | _write_emit_result(args.emit_result, url, out_path) |
| 964 | |
| 965 | # Summary |
| 966 | success_count = sum(1 for r in results if r[0]) |
| 967 | fail_count = len(results) - success_count |
| 968 | |
| 969 | print("\n" + "="*50) |
| 970 | print( |
| 971 | f"[Done] Success: {success_count}/{len(results)}, Failed: {fail_count}") |
| 972 | |
| 973 | if fail_count > 0: |
| 974 | print("\n[Failed URLs]:") |
| 975 | for r in results: |
| 976 | if not r[0]: |
| 977 | print(f" - {r[1]}: {r[2]}") |
| 978 | return 1 |
| 979 | return 0 |
| 980 | |
| 981 | |
| 982 | if __name__ == "__main__": |
| 983 | # Disable warnings for verify=False if needed, though often useful to see |
| 984 | import urllib3 |
| 985 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) |
| 986 | raise SystemExit(main()) |
| 987 |