| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | PDF to Markdown Converter |
| 4 | Uses PyMuPDF to extract PDF text content and convert to Markdown format. |
| 5 | Supports heading levels, bold, italic, and list detection. |
| 6 | """ |
| 7 | |
| 8 | import argparse |
| 9 | import hashlib |
| 10 | import json |
| 11 | import os |
| 12 | import re |
| 13 | import sys |
| 14 | from pathlib import Path |
| 15 | from collections import Counter |
| 16 | |
| 17 | _SCRIPTS_DIR = Path(__file__).resolve().parents[1] |
| 18 | if str(_SCRIPTS_DIR) not in sys.path: |
| 19 | sys.path.insert(0, str(_SCRIPTS_DIR)) |
| 20 | |
| 21 | from console_encoding import configure_utf8_stdio # noqa: E402 |
| 22 | from _batch import run_path_batch # noqa: E402 |
| 23 | from _conversion_profile import write_conversion_profile_best_effort # noqa: E402 |
| 24 | |
| 25 | configure_utf8_stdio() |
| 26 | |
| 27 | try: |
| 28 | import fitz # PyMuPDF |
| 29 | except ImportError: |
| 30 | print("[ERROR] PyMuPDF not installed. Run: pip install PyMuPDF", file=sys.stderr) |
| 31 | sys.exit(1) |
| 32 | |
| 33 | FONT_BODY_SIZE = 12 |
| 34 | FONT_H1_SIZE = 24 |
| 35 | FONT_H2_SIZE = 18 |
| 36 | FONT_H3_SIZE = 14 |
| 37 | HEADER_FOOTER_SAMPLE_LIMIT = 40 |
| 38 | HEADER_FOOTER_EDGE_SAMPLE_SIZE = 20 |
| 39 | CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]') |
| 40 | |
| 41 | |
| 42 | def analyze_font_sizes(doc: fitz.Document) -> dict[str, float]: |
| 43 | """Analyze font size distribution to infer heading levels. |
| 44 | |
| 45 | Args: |
| 46 | doc: Open PDF document. |
| 47 | |
| 48 | Returns: |
| 49 | A size mapping containing body and inferred heading sizes. |
| 50 | """ |
| 51 | size_counter = Counter() |
| 52 | |
| 53 | for page in doc: |
| 54 | blocks = page.get_text("dict")["blocks"] |
| 55 | for block in blocks: |
| 56 | if block["type"] == 0: |
| 57 | for line in block["lines"]: |
| 58 | for span in line["spans"]: |
| 59 | size = round(span["size"], 1) |
| 60 | text = span["text"].strip() |
| 61 | if text: |
| 62 | size_counter[size] += len(text) |
| 63 | |
| 64 | if not size_counter: |
| 65 | return { |
| 66 | "body": FONT_BODY_SIZE, |
| 67 | "h1": FONT_H1_SIZE, |
| 68 | "h2": FONT_H2_SIZE, |
| 69 | "h3": FONT_H3_SIZE, |
| 70 | } |
| 71 | |
| 72 | sorted_sizes = sorted(size_counter.items(), key=lambda x: x[1], reverse=True) |
| 73 | body_size = sorted_sizes[0][0] |
| 74 | |
| 75 | all_sizes = sorted(size_counter.keys(), reverse=True) |
| 76 | larger_sizes = [s for s in all_sizes if s > body_size + 1] |
| 77 | |
| 78 | size_map = {"body": body_size} |
| 79 | if len(larger_sizes) >= 1: |
| 80 | size_map["h1"] = larger_sizes[0] |
| 81 | if len(larger_sizes) >= 2: |
| 82 | size_map["h2"] = larger_sizes[1] |
| 83 | if len(larger_sizes) >= 3: |
| 84 | size_map["h3"] = larger_sizes[2] |
| 85 | |
| 86 | return size_map |
| 87 | |
| 88 | |
| 89 | def get_heading_level(size: float, size_map: dict, text: str = "", |
| 90 | flags: int = 0, strict: bool = True) -> int: |
| 91 | """ |
| 92 | Determine heading level using multiple heuristics. |
| 93 | |
| 94 | Args: |
| 95 | size: Font size |
| 96 | size_map: Font size mapping |
| 97 | text: Text content (used for additional heuristics) |
| 98 | flags: Font flags (bit 4 = bold) |
| 99 | strict: Strict mode, requires more conditions to be met |
| 100 | |
| 101 | Returns: |
| 102 | Heading level (0 = body text, 1-3 = H1-H3) |
| 103 | """ |
| 104 | # Initial determination based on font size |
| 105 | level = 0 |
| 106 | if "h1" in size_map and size >= size_map["h1"] - 0.5: |
| 107 | level = 1 |
| 108 | elif "h2" in size_map and size >= size_map["h2"] - 0.5: |
| 109 | level = 2 |
| 110 | elif "h3" in size_map and size >= size_map["h3"] - 0.5: |
| 111 | level = 3 |
| 112 | |
| 113 | if level == 0: |
| 114 | return 0 |
| 115 | |
| 116 | # Non-strict mode returns directly (backward compatible) |
| 117 | if not strict or not text: |
| 118 | return level |
| 119 | |
| 120 | # Strict mode: additional validation conditions |
| 121 | text = text.strip() |
| 122 | |
| 123 | # Exclusion: text too long is unlikely to be a heading |
| 124 | if len(text) > 80: |
| 125 | return 0 |
| 126 | |
| 127 | # Exclusion: complete sentences ending with punctuation |
| 128 | sentence_endings = '.。!！?？' |
| 129 | if text and text[-1] in sentence_endings: |
| 130 | # But keep numbered headings like "1. Overview" or "Chapter 1." |
| 131 | if not re.match(r'^[\d第]+[.、章节]', text): |
| 132 | return 0 |
| 133 | |
| 134 | # Bonus: bold text is more likely to be a heading |
| 135 | is_bold = flags & 16 |
| 136 | if not is_bold and level >= 2: |
| 137 | # Non-bold subheadings require a larger font size difference |
| 138 | body_size = size_map.get("body", 12) |
| 139 | if size < body_size + 2: |
| 140 | return 0 |
| 141 | |
| 142 | return level |
| 143 | |
| 144 | def is_monospace_font(font_name: str) -> bool: |
| 145 | """ |
| 146 | Determine if the font is monospace (typically used for code). |
| 147 | """ |
| 148 | if not font_name: |
| 149 | return False |
| 150 | font_lower = font_name.lower() |
| 151 | mono_fonts = [ |
| 152 | 'courier', 'consolas', 'monaco', 'menlo', 'monospace', |
| 153 | 'source code', 'fira code', 'jetbrains', 'inconsolata', |
| 154 | 'dejavu sans mono', 'liberation mono', 'ubuntu mono', |
| 155 | 'roboto mono', 'robotomono', 'sf mono', 'cascadia', 'hack' |
| 156 | ] |
| 157 | return any(f in font_lower for f in mono_fonts) |
| 158 | |
| 159 | |
| 160 | def format_span_text(text: str, flags: int) -> str: |
| 161 | """Format text based on font flags (bold, italic).""" |
| 162 | text = CONTROL_CHARS_RE.sub('', text) |
| 163 | text = text.strip() |
| 164 | if not text: |
| 165 | return "" |
| 166 | |
| 167 | is_bold = flags & 16 |
| 168 | is_italic = flags & 2 |
| 169 | |
| 170 | if is_bold and is_italic: |
| 171 | return f"***{text}***" |
| 172 | elif is_bold: |
| 173 | return f"**{text}**" |
| 174 | elif is_italic: |
| 175 | return f"*{text}*" |
| 176 | return text |
| 177 | |
| 178 | |
| 179 | def detect_list_item(text: str) -> tuple: |
| 180 | """Detect if the text is a list item. Returns (is_list, list_type, content).""" |
| 181 | text = text.strip() |
| 182 | |
| 183 | ul_patterns = [ |
| 184 | (r'^[•●○◦▪▸►]\s*', '-'), |
| 185 | (r'^[-–—]\s+', '-'), |
| 186 | (r'^\*\s+', '-'), |
| 187 | ] |
| 188 | for pattern, marker in ul_patterns: |
| 189 | match = re.match(pattern, text) |
| 190 | if match: |
| 191 | return (True, 'ul', marker + ' ' + text[match.end():]) |
| 192 | |
| 193 | ol_pattern = r'^(\d+)[.、)]\s*' |
| 194 | match = re.match(ol_pattern, text) |
| 195 | if match: |
| 196 | num = match.group(1) |
| 197 | return (True, 'ol', f"{num}. " + text[match.end():]) |
| 198 | |
| 199 | return (False, None, text) |
| 200 | |
| 201 | |
| 202 | def remove_page_footer(text: str) -> str: |
| 203 | """ |
| 204 | Remove page number patterns from footers, e.g. 'November 2025 8' or '2025年11月 8'. |
| 205 | """ |
| 206 | # English month + year + page number |
| 207 | months_en = r'(?:January|February|March|April|May|June|July|August|September|October|November|December)' |
| 208 | pattern_en = rf'\s*{months_en}\s+\d{{4}}\s+\d{{1,3}}\s*$' |
| 209 | text = re.sub(pattern_en, '', text, flags=re.IGNORECASE) |
| 210 | |
| 211 | # Chinese format: 2025年11月 8 |
| 212 | pattern_cn = r'\s*\d{4}年\d{1,2}月\s+\d{1,3}\s*$' |
| 213 | text = re.sub(pattern_cn, '', text) |
| 214 | |
| 215 | return text.rstrip() |
| 216 | |
| 217 | |
| 218 | def detect_headers_footers(doc: fitz.Document, threshold_ratio: float = 0.6) -> set[str]: |
| 219 | """ |
| 220 | Detect headers and footers statistically. |
| 221 | |
| 222 | Principle: Headers and footers typically appear at fixed positions (top or bottom) |
| 223 | on each page with the same content. We collect top and bottom text from all pages, |
| 224 | and if certain text appears more frequently than the threshold, it is treated as noise. |
| 225 | """ |
| 226 | if len(doc) < 3: |
| 227 | return set() |
| 228 | |
| 229 | headers = [] |
| 230 | footers = [] |
| 231 | |
| 232 | # Sample first 20 and last 20 pages (avoid processing too slowly) |
| 233 | pages_to_scan = list(range(len(doc))) |
| 234 | if len(doc) > HEADER_FOOTER_SAMPLE_LIMIT: |
| 235 | pages_to_scan = ( |
| 236 | pages_to_scan[:HEADER_FOOTER_EDGE_SAMPLE_SIZE] |
| 237 | + pages_to_scan[-HEADER_FOOTER_EDGE_SAMPLE_SIZE:] |
| 238 | ) |
| 239 | |
| 240 | for i in pages_to_scan: |
| 241 | page = doc[i] |
| 242 | rect = page.rect |
| 243 | h = rect.height |
| 244 | |
| 245 | # Define top and bottom regions (15% each) |
| 246 | top_rect = fitz.Rect(0, 0, rect.width, h * 0.15) |
| 247 | bottom_rect = fitz.Rect(0, h * 0.85, rect.width, h) |
| 248 | |
| 249 | # Extract text blocks |
| 250 | blocks = page.get_text("blocks") |
| 251 | for b in blocks: |
| 252 | b_rect = fitz.Rect(b[:4]) |
| 253 | text = b[4].strip() |
| 254 | if not text: |
| 255 | continue |
| 256 | |
| 257 | # Simple spatial determination |
| 258 | if b_rect.intersects(top_rect): |
| 259 | headers.append(text) |
| 260 | elif b_rect.intersects(bottom_rect): |
| 261 | footers.append(text) |
| 262 | |
| 263 | # Count frequencies |
| 264 | noise_texts = set() |
| 265 | total_scanned = len(pages_to_scan) |
| 266 | |
| 267 | for collection in [headers, footers]: |
| 268 | counter = Counter(collection) |
| 269 | for text, count in counter.items(): |
| 270 | # if text appears in > 60% of scanned pages, mark as noise |
| 271 | if count / total_scanned > threshold_ratio: |
| 272 | noise_texts.add(text) |
| 273 | |
| 274 | return noise_texts |
| 275 | |
| 276 | |
| 277 | def merge_adjacent_headings(elements: list) -> list: |
| 278 | """ |
| 279 | Merge adjacent same-level short headings. |
| 280 | Example: '# Agent Tools &' + '# Interoperability' -> '# Agent Tools & Interoperability' |
| 281 | """ |
| 282 | if not elements: |
| 283 | return elements |
| 284 | |
| 285 | merged = [] |
| 286 | i = 0 |
| 287 | |
| 288 | while i < len(elements): |
| 289 | el = elements[i] |
| 290 | |
| 291 | # Only process heading elements |
| 292 | if el.get("type") != 0 or not el.get("is_heading"): |
| 293 | merged.append(el) |
| 294 | i += 1 |
| 295 | continue |
| 296 | |
| 297 | content = el["content"] |
| 298 | # Extract heading level |
| 299 | match = re.match(r'^(#{1,6})\s+(.+)$', content) |
| 300 | if not match: |
| 301 | merged.append(el) |
| 302 | i += 1 |
| 303 | continue |
| 304 | |
| 305 | level = match.group(1) |
| 306 | title_text = match.group(2) |
| 307 | |
| 308 | # If heading is short and the next one is also the same level, try to merge |
| 309 | j = i + 1 |
| 310 | while j < len(elements) and len(title_text) < 60: |
| 311 | next_el = elements[j] |
| 312 | if next_el.get("type") != 0 or not next_el.get("is_heading"): |
| 313 | break |
| 314 | |
| 315 | next_match = re.match(r'^(#{1,6})\s+(.+)$', next_el["content"]) |
| 316 | if not next_match or next_match.group(1) != level: |
| 317 | break |
| 318 | |
| 319 | next_text = next_match.group(2) |
| 320 | # Only merge short heading fragments |
| 321 | if len(next_text) > 40: |
| 322 | break |
| 323 | |
| 324 | # Merge |
| 325 | title_text += " " + next_text |
| 326 | j += 1 |
| 327 | |
| 328 | # Create merged element |
| 329 | merged_el = el.copy() |
| 330 | merged_el["content"] = f"{level} {title_text}" |
| 331 | merged.append(merged_el) |
| 332 | i = j |
| 333 | |
| 334 | return merged |
| 335 | |
| 336 | |
| 337 | # Image filtering thresholds |
| 338 | MIN_IMAGE_PIXELS = 100 # Minimum pixel dimension (width AND height) |
| 339 | MIN_IMAGE_AREA = 30000 # Minimum pixel area (e.g. 200x150) |
| 340 | MIN_IMAGE_BYTES = 2048 # Minimum image data size (2KB) |
| 341 | MIN_PAGE_RATIO = 0.05 # Minimum render size relative to page (5%) |
| 342 | MIN_VISIBLE_IMAGE_WIDTH = 40 |
| 343 | MIN_VISIBLE_IMAGE_HEIGHT = 40 |
| 344 | MIN_VISIBLE_IMAGE_AREA_RATIO = 0.01 |
| 345 | MAX_ASPECT_RATIO = 12 # Maximum aspect ratio (filters decorative bars) |
| 346 | MAX_LOW_INFO_BPP = 0.08 # Bytes-per-pixel threshold for low-info images |
| 347 | MAX_LOW_INFO_AREA = 500000 # Area threshold: only apply bpp filter below this |
| 348 | MIN_VECTOR_FIGURE_WIDTH = 100 |
| 349 | MIN_VECTOR_FIGURE_HEIGHT = 80 |
| 350 | MIN_VECTOR_FIGURE_AREA = 30000 |
| 351 | MAX_VECTOR_FIGURE_ASPECT_RATIO = 8 |
| 352 | VECTOR_FIGURE_PADDING = 4 |
| 353 | VECTOR_FIGURE_DPI = 180 |
| 354 | VECTOR_CAPTION_SEARCH_HEIGHT = 380 |
| 355 | VECTOR_CAPTION_HORIZONTAL_GAP = 90 |
| 356 | MAX_VECTOR_BACKGROUND_AREA_RATIO = 1.9 |
| 357 | # Caption delimiters: ``Figure 1:`` / ``Figure 1.`` (classic) and ``Figure 1 |`` |
| 358 | # (the DeepMind / Distill / Nature house style used by many ML papers, incl. |
| 359 | # full-width ``｜``). Without the pipe variants, captioned vector figures route |
| 360 | # to the generic per-drawing fallback, which discards fine-grained line plots |
| 361 | # (each plot is hundreds of tiny primitives, none large enough on its own). |
| 362 | FIGURE_CAPTION_RE = re.compile(r'^(?:Figure|Fig\.?)\s*\d+\s*[:.|｜]', re.IGNORECASE) |
| 363 | |
| 364 | |
| 365 | TABLE_CAPTION_RE = re.compile( |
| 366 | r'^\u8868\s*\d+(?:\.\d+)*\s+(?!(?:\u7684|\u5217\u793a|\u6240\u793a|\u4e3a)).+' |
| 367 | ) |
| 368 | TABLE_REFERENCE_PROSE_RE = re.compile( |
| 369 | r'^\u8868\s*\d+(?:\.\d+)*\s*(?:\u7684|\u5217\u793a|\u6240\u793a|\u4e3a)' |
| 370 | ) |
| 371 | SECTION_HEADING_RE = re.compile(r'^\d+(?:\.\d+){1,3}\s+\S') |
| 372 | NUMBER_RE = re.compile(r'[-+]?\d+(?:\.\d+)?') |
| 373 | MODEL_COLUMN_RE = re.compile(r'^[（(]\d+[）)]$') |
| 374 | TABLE_NOTE_PREFIX = '\u6ce8' |
| 375 | TABLE_CONTINUATION_Y_RATIO = 0.88 |
| 376 | TABLE_SCAN_BOTTOM_RATIO = 0.92 |
| 377 | |
| 378 | |
| 379 | def should_keep_image( |
| 380 | block: dict[str, object], |
| 381 | page_rect: fitz.Rect, |
| 382 | seen_hashes: set[str] | None = None, |
| 383 | ) -> bool: |
| 384 | """Filter out small, decorative, or duplicate images. |
| 385 | |
| 386 | Args: |
| 387 | block: Image block extracted from PyMuPDF. |
| 388 | page_rect: Current page rectangle. |
| 389 | seen_hashes: Optional set used to deduplicate image payloads. |
| 390 | |
| 391 | Returns: |
| 392 | Whether the image should be kept in the Markdown output. |
| 393 | """ |
| 394 | w, h = block.get("width", 0), block.get("height", 0) |
| 395 | bbox = block.get("bbox", (0, 0, 0, 0)) |
| 396 | render_w = bbox[2] - bbox[0] |
| 397 | render_h = bbox[3] - bbox[1] |
| 398 | page_area = page_rect.width * page_rect.height |
| 399 | render_area_ratio = (render_w * render_h) / page_area if page_area > 0 else 0 |
| 400 | visibly_placed = ( |
| 401 | render_w >= MIN_VISIBLE_IMAGE_WIDTH |
| 402 | and render_h >= MIN_VISIBLE_IMAGE_HEIGHT |
| 403 | and render_area_ratio >= MIN_VISIBLE_IMAGE_AREA_RATIO |
| 404 | ) |
| 405 | |
| 406 | # Pixel dimension filter |
| 407 | if not visibly_placed and (w < MIN_IMAGE_PIXELS or h < MIN_IMAGE_PIXELS): |
| 408 | return False |
| 409 | |
| 410 | # Pixel area filter |
| 411 | area = w * h |
| 412 | if not visibly_placed and area < MIN_IMAGE_AREA: |
| 413 | return False |
| 414 | |
| 415 | image_data = block.get("image", b"") |
| 416 | if not visibly_placed and len(image_data) < MIN_IMAGE_BYTES: |
| 417 | return False |
| 418 | |
| 419 | # Deduplicate tiny repeats, but preserve visibly placed logos / charts on |
| 420 | # distinct pages. Academic PDFs often reuse a small logo on title pages. |
| 421 | if seen_hashes is not None: |
| 422 | img_hash = hashlib.md5(image_data).hexdigest() |
| 423 | if img_hash in seen_hashes and not visibly_placed: |
| 424 | return False |
| 425 | seen_hashes.add(img_hash) |
| 426 | |
| 427 | # Check render size relative to page |
| 428 | page_w = page_rect.width |
| 429 | page_h = page_rect.height |
| 430 | if page_w > 0 and page_h > 0: |
| 431 | if render_w / page_w < MIN_PAGE_RATIO and render_h / page_h < MIN_PAGE_RATIO: |
| 432 | return False |
| 433 | |
| 434 | # Filter extreme aspect ratios (decorative bars/separators) |
| 435 | aspect = max(w, h) / max(min(w, h), 1) |
| 436 | if aspect > MAX_ASPECT_RATIO: |
| 437 | return False |
| 438 | |
| 439 | # Filter low-info images: solid color blocks / gradients have very high |
| 440 | # compression ratios (low bytes-per-pixel). Only apply to smaller images |
| 441 | # to avoid filtering large photos with dark/uniform backgrounds. |
| 442 | bpp = len(image_data) / area |
| 443 | if bpp < MAX_LOW_INFO_BPP and area < MAX_LOW_INFO_AREA and not visibly_placed: |
| 444 | return False |
| 445 | |
| 446 | return True |
| 447 | |
| 448 | |
| 449 | def _clip_rect_to_page(rect: fitz.Rect, page_rect: fitz.Rect) -> fitz.Rect: |
| 450 | """Clamp a rectangle to the current PDF page.""" |
| 451 | return fitz.Rect( |
| 452 | max(page_rect.x0, rect.x0), |
| 453 | max(page_rect.y0, rect.y0), |
| 454 | min(page_rect.x1, rect.x1), |
| 455 | min(page_rect.y1, rect.y1), |
| 456 | ) |
| 457 | |
| 458 | |
| 459 | def _is_rect_contained(inner: fitz.Rect, outer: fitz.Rect, threshold: float = 0.9) -> bool: |
| 460 | """Return whether ``inner`` is mostly covered by ``outer``.""" |
| 461 | inner_area = inner.get_area() |
| 462 | if inner_area <= 0: |
| 463 | return False |
| 464 | return ((inner & outer).get_area() / inner_area) >= threshold |
| 465 | |
| 466 | |
| 467 | def _overlaps_table(rect: fitz.Rect, tab_rects: list[fitz.Rect]) -> bool: |
| 468 | """Skip vector regions that are already handled as extracted tables.""" |
| 469 | rect_area = rect.get_area() |
| 470 | if rect_area <= 0: |
| 471 | return False |
| 472 | for tab_rect in tab_rects: |
| 473 | intersection = rect & tab_rect |
| 474 | if intersection.get_area() > 0.5 * min(rect_area, tab_rect.get_area()): |
| 475 | return True |
| 476 | return False |
| 477 | |
| 478 | |
| 479 | def _is_white(color: tuple[float, ...] | None) -> bool: |
| 480 | """Return whether a PDF drawing color is visually white.""" |
| 481 | if color is None: |
| 482 | return False |
| 483 | return all(channel >= 0.98 for channel in color[:3]) |
| 484 | |
| 485 | |
| 486 | def _is_background_drawing(drawing: dict[str, object]) -> bool: |
| 487 | """Identify white background rectangles that should not drive crops.""" |
| 488 | return _is_white(drawing.get("fill")) and drawing.get("color") is None |
| 489 | |
| 490 | |
| 491 | def find_figure_caption_rects(page: fitz.Page) -> list[fitz.Rect]: |
| 492 | """Return text-line rectangles that look like figure captions.""" |
| 493 | caption_rects = [] |
| 494 | for block in page.get_text("dict")["blocks"]: |
| 495 | if block.get("type") != 0: |
| 496 | continue |
| 497 | for line in block["lines"]: |
| 498 | text = "".join(span["text"] for span in line["spans"]).strip() |
| 499 | if FIGURE_CAPTION_RE.match(text): |
| 500 | caption_rects.append(fitz.Rect(line["bbox"])) |
| 501 | return caption_rects |
| 502 | |
| 503 | |
| 504 | def _expand_rect(rect: fitz.Rect, padding: float, page_rect: fitz.Rect) -> fitz.Rect: |
| 505 | """Pad a rectangle and clamp it to the current PDF page.""" |
| 506 | return _clip_rect_to_page( |
| 507 | fitz.Rect( |
| 508 | rect.x0 - padding, |
| 509 | rect.y0 - padding, |
| 510 | rect.x1 + padding, |
| 511 | rect.y1 + padding, |
| 512 | ), |
| 513 | page_rect, |
| 514 | ) |
| 515 | |
| 516 | |
| 517 | def _union_rects(rects: list[fitz.Rect]) -> fitz.Rect: |
| 518 | """Return the bounding union for one or more rectangles.""" |
| 519 | result = fitz.Rect(rects[0]) |
| 520 | for rect in rects[1:]: |
| 521 | result |= rect |
| 522 | return result |
| 523 | |
| 524 | |
| 525 | def _find_captioned_vector_figures( |
| 526 | page: fitz.Page, |
| 527 | drawing_rects: list[fitz.Rect], |
| 528 | background_rects: list[fitz.Rect], |
| 529 | caption_rects: list[fitz.Rect], |
| 530 | ) -> list[fitz.Rect]: |
| 531 | """Build tight figure crops from non-background drawings above captions.""" |
| 532 | figure_rects = [] |
| 533 | page_rect = page.rect |
| 534 | |
| 535 | for caption_rect in caption_rects: |
| 536 | related = [] |
| 537 | for rect in drawing_rects: |
| 538 | horizontal_gap = max(caption_rect.x0 - rect.x1, rect.x0 - caption_rect.x1, 0) |
| 539 | if horizontal_gap > VECTOR_CAPTION_HORIZONTAL_GAP: |
| 540 | continue |
| 541 | if rect.y0 > caption_rect.y0: |
| 542 | continue |
| 543 | if caption_rect.y0 - rect.y1 > VECTOR_CAPTION_SEARCH_HEIGHT: |
| 544 | continue |
| 545 | related.append(rect) |
| 546 | |
| 547 | if not related: |
| 548 | continue |
| 549 | |
| 550 | content_rect = _union_rects(related) |
| 551 | rect = content_rect |
| 552 | for background_rect in background_rects: |
| 553 | background_rect = _clip_rect_to_page(background_rect, page_rect) |
| 554 | if not _is_rect_contained(content_rect, background_rect, threshold=0.95): |
| 555 | continue |
| 556 | if background_rect.get_area() > content_rect.get_area() * MAX_VECTOR_BACKGROUND_AREA_RATIO: |
| 557 | continue |
| 558 | rect = background_rect |
| 559 | break |
| 560 | |
| 561 | rect = _expand_rect(rect, 10, page_rect) |
| 562 | rect.y1 = min(rect.y1, caption_rect.y0 - 2) |
| 563 | if rect.width >= MIN_VECTOR_FIGURE_WIDTH and rect.height >= MIN_VECTOR_FIGURE_HEIGHT: |
| 564 | figure_rects.append(rect) |
| 565 | |
| 566 | return figure_rects |
| 567 | |
| 568 | |
| 569 | def detect_vector_figure_rects(page: fitz.Page, tab_rects: list[fitz.Rect]) -> list[fitz.Rect]: |
| 570 | """Detect large vector drawing regions that should be rasterized as figures. |
| 571 | |
| 572 | Some academic PDFs store charts and diagrams as vector drawing commands, |
| 573 | not image XObjects. ``page.get_text("dict")`` exposes only raster image |
| 574 | blocks, so those figures need a separate drawing-region fallback. |
| 575 | """ |
| 576 | candidates = [] |
| 577 | page_rect = page.rect |
| 578 | caption_rects = find_figure_caption_rects(page) |
| 579 | drawing_rects = [] |
| 580 | background_rects = [] |
| 581 | |
| 582 | for drawing in page.get_drawings(): |
| 583 | rect = drawing.get("rect") |
| 584 | if not rect: |
| 585 | continue |
| 586 | |
| 587 | rect = fitz.Rect(rect) |
| 588 | if rect.is_empty: |
| 589 | continue |
| 590 | |
| 591 | if _is_background_drawing(drawing): |
| 592 | background_rects.append(rect) |
| 593 | continue |
| 594 | |
| 595 | drawing_rects.append(rect) |
| 596 | |
| 597 | if caption_rects: |
| 598 | return _find_captioned_vector_figures(page, drawing_rects, background_rects, caption_rects) |
| 599 | |
| 600 | for rect in drawing_rects: |
| 601 | rect = _expand_rect(rect, VECTOR_FIGURE_PADDING, page_rect) |
| 602 | rect = _clip_rect_to_page(rect, page_rect) |
| 603 | |
| 604 | width = rect.width |
| 605 | height = rect.height |
| 606 | if width < MIN_VECTOR_FIGURE_WIDTH or height < MIN_VECTOR_FIGURE_HEIGHT: |
| 607 | continue |
| 608 | |
| 609 | area = rect.get_area() |
| 610 | if area < MIN_VECTOR_FIGURE_AREA: |
| 611 | continue |
| 612 | |
| 613 | aspect = max(width, height) / max(min(width, height), 1) |
| 614 | if aspect > MAX_VECTOR_FIGURE_ASPECT_RATIO: |
| 615 | continue |
| 616 | |
| 617 | if _overlaps_table(rect, tab_rects): |
| 618 | continue |
| 619 | |
| 620 | candidates.append(rect) |
| 621 | |
| 622 | candidates.sort(key=lambda r: r.get_area(), reverse=True) |
| 623 | |
| 624 | kept = [] |
| 625 | for rect in candidates: |
| 626 | if any(_is_rect_contained(rect, existing) for existing in kept): |
| 627 | continue |
| 628 | kept.append(rect) |
| 629 | |
| 630 | return sorted(kept, key=lambda r: (r.y0, r.x0)) |
| 631 | |
| 632 | |
| 633 | def clean_text(text: str) -> str: |
| 634 | """Clean extracted text.""" |
| 635 | lines = text.split('\n') |
| 636 | cleaned_lines = [] |
| 637 | prev_empty = False |
| 638 | |
| 639 | for line in lines: |
| 640 | line = line.rstrip() |
| 641 | is_empty = len(line.strip()) == 0 |
| 642 | |
| 643 | if is_empty: |
| 644 | if not prev_empty: |
| 645 | cleaned_lines.append('') |
| 646 | prev_empty = True |
| 647 | else: |
| 648 | cleaned_lines.append(line) |
| 649 | prev_empty = False |
| 650 | |
| 651 | return '\n'.join(cleaned_lines) |
| 652 | |
| 653 | |
| 654 | def _extract_text_lines(page: fitz.Page) -> list[tuple[fitz.Rect, str]]: |
| 655 | """Return text lines with their page rectangles in reading order.""" |
| 656 | lines = [] |
| 657 | for block in page.get_text("dict")["blocks"]: |
| 658 | if block.get("type") != 0: |
| 659 | continue |
| 660 | for line in block["lines"]: |
| 661 | text = "".join(span["text"] for span in line["spans"]).strip() |
| 662 | if text: |
| 663 | lines.append((fitz.Rect(line["bbox"]), text)) |
| 664 | return sorted(lines, key=lambda item: (item[0].y0, item[0].x0)) |
| 665 | |
| 666 | |
| 667 | def _is_table_caption(text: str) -> bool: |
| 668 | """Return whether a line is a real table caption, not prose citing a table.""" |
| 669 | return bool(TABLE_CAPTION_RE.match(text.strip())) |
| 670 | |
| 671 | |
| 672 | def _caption_table_start_y( |
| 673 | caption_rect: fitz.Rect, |
| 674 | lines: list[tuple[fitz.Rect, str]], |
| 675 | ) -> float: |
| 676 | """Start below a caption and its adjacent English translation line.""" |
| 677 | start_y = caption_rect.y1 + 1 |
| 678 | for rect, text in lines: |
| 679 | if rect.y0 < caption_rect.y1 - 1 or rect.y0 > caption_rect.y1 + 35: |
| 680 | continue |
| 681 | if text.startswith("Table"): |
| 682 | start_y = max(start_y, rect.y1 + 1) |
| 683 | return start_y |
| 684 | |
| 685 | |
| 686 | def _looks_like_numeric_table_line(text: str) -> bool: |
| 687 | """Detect long data rows so they are not mistaken for prose boundaries.""" |
| 688 | numbers = NUMBER_RE.findall(text) |
| 689 | if len(numbers) >= 3: |
| 690 | return True |
| 691 | tokens = [token for token in re.split(r'\s+', text.strip()) if token] |
| 692 | return len(tokens) >= 4 and len(numbers) >= 2 |
| 693 | |
| 694 | |
| 695 | def _is_table_region_boundary( |
| 696 | rect: fitz.Rect, |
| 697 | text: str, |
| 698 | page: fitz.Page, |
| 699 | start_y: float, |
| 700 | ) -> bool: |
| 701 | """Return whether a line likely starts prose after a text-detected table.""" |
| 702 | text = text.strip() |
| 703 | if rect.y0 < start_y + 45: |
| 704 | return False |
| 705 | if text.startswith(TABLE_NOTE_PREFIX): |
| 706 | return True |
| 707 | if SECTION_HEADING_RE.match(text): |
| 708 | return True |
| 709 | if TABLE_REFERENCE_PROSE_RE.match(text): |
| 710 | return True |
| 711 | if _looks_like_numeric_table_line(text): |
| 712 | return False |
| 713 | |
| 714 | width_ratio = rect.width / page.rect.width if page.rect.width > 0 else 0 |
| 715 | return len(text) >= 26 and width_ratio > 0.52 and rect.x0 < page.rect.width * 0.25 |
| 716 | |
| 717 | |
| 718 | def _table_region_bottom( |
| 719 | page: fitz.Page, |
| 720 | lines: list[tuple[fitz.Rect, str]], |
| 721 | start_y: float, |
| 722 | ) -> float: |
| 723 | """Find a conservative bottom edge for a caption-guided text table scan.""" |
| 724 | for rect, text in lines: |
| 725 | if rect.y0 <= start_y: |
| 726 | continue |
| 727 | if _is_table_region_boundary(rect, text, page, start_y): |
| 728 | return max(start_y + 20, rect.y0 - 2) |
| 729 | return page.rect.height * TABLE_SCAN_BOTTOM_RATIO |
| 730 | |
| 731 | |
| 732 | def _normalize_table_cell(value: object) -> str: |
| 733 | """Normalize one extracted table cell for Markdown output.""" |
| 734 | if value is None: |
| 735 | return "" |
| 736 | text = CONTROL_CHARS_RE.sub('', str(value)) |
| 737 | text = re.sub(r'\s*\n\s*', '<br>', text.strip()) |
| 738 | text = re.sub(r'[ \t]+', ' ', text) |
| 739 | return text.replace('|', r'\|') |
| 740 | |
| 741 | |
| 742 | def _clean_table_rows(rows: list[list[object]]) -> list[list[str]]: |
| 743 | """Remove empty rows / columns from PyMuPDF table extraction output.""" |
| 744 | normalized = [[_normalize_table_cell(cell) for cell in row] for row in rows] |
| 745 | normalized = [row for row in normalized if any(cell for cell in row)] |
| 746 | if not normalized: |
| 747 | return [] |
| 748 | |
| 749 | max_cols = max(len(row) for row in normalized) |
| 750 | padded = [row + [""] * (max_cols - len(row)) for row in normalized] |
| 751 | keep_cols = [ |
| 752 | idx |
| 753 | for idx in range(max_cols) |
| 754 | if any(row[idx] for row in padded) |
| 755 | ] |
| 756 | if len(keep_cols) < 2: |
| 757 | return [] |
| 758 | return _postprocess_table_rows([[row[idx] for idx in keep_cols] for row in padded]) |
| 759 | |
| 760 | |
| 761 | def _nonempty_cell_indexes(row: list[str]) -> list[int]: |
| 762 | """Return indexes of non-empty cells in a row.""" |
| 763 | return [idx for idx, cell in enumerate(row) if cell] |
| 764 | |
| 765 | |
| 766 | def _merge_label_underscore_rows(rows: list[list[str]]) -> list[list[str]]: |
| 767 | """Join rows where PDF extraction split a leading underscore from a label.""" |
| 768 | merged = [] |
| 769 | for row in rows: |
| 770 | nonempty = _nonempty_cell_indexes(row) |
| 771 | if nonempty == [0] and row[0] == "_" and merged and merged[-1][0]: |
| 772 | merged[-1][0] = f"_{merged[-1][0]}" |
| 773 | continue |
| 774 | merged.append(row) |
| 775 | return merged |
| 776 | |
| 777 | |
| 778 | def _merge_single_cell_continuations(rows: list[list[str]]) -> list[list[str]]: |
| 779 | """Merge wrapped single-cell table rows into the previous row.""" |
| 780 | merged: list[list[str]] = [] |
| 781 | for row in rows: |
| 782 | nonempty = _nonempty_cell_indexes(row) |
| 783 | if ( |
| 784 | len(nonempty) == 1 |
| 785 | and nonempty[0] > 0 |
| 786 | and merged |
| 787 | and not MODEL_COLUMN_RE.match(row[nonempty[0]]) |
| 788 | ): |
| 789 | index = nonempty[0] |
| 790 | separator = "<br>" if merged[-1][index] else "" |
| 791 | merged[-1][index] = f"{merged[-1][index]}{separator}{row[index]}" |
| 792 | continue |
| 793 | merged.append(row) |
| 794 | return merged |
| 795 | |
| 796 | |
| 797 | def _looks_like_model_row(row: list[str]) -> bool: |
| 798 | """Return whether a row contains model-number table headings.""" |
| 799 | values = [cell for cell in row[1:] if cell] |
| 800 | return len(values) >= 2 and all(MODEL_COLUMN_RE.match(cell) for cell in values) |
| 801 | |
| 802 | |
| 803 | def _looks_like_outcome_row(row: list[str]) -> bool: |
| 804 | """Return whether a row contains regression outcome labels.""" |
| 805 | values = [cell for cell in row[1:] if cell] |
| 806 | if len(values) < 2: |
| 807 | return False |
| 808 | short_values = [cell for cell in values if len(cell) <= 12 and not NUMBER_RE.search(cell)] |
| 809 | return len(short_values) == len(values) |
| 810 | |
| 811 | |
| 812 | def _regression_group_labels(row: list[str], data_cols: int) -> list[str]: |
| 813 | """Infer repeated group labels for common regression-table headings.""" |
| 814 | compact = "".join(row) |
| 815 | if "总样本" in compact and "国有" in compact and "非国有" in compact and data_cols == 7: |
| 816 | return ["总样本"] * 3 + ["国有企业"] * 2 + ["非国有企业"] * 2 |
| 817 | return [""] * data_cols |
| 818 | |
| 819 | |
| 820 | def _flatten_regression_header(rows: list[list[str]]) -> list[list[str]]: |
| 821 | """Flatten multi-line regression headings into one Markdown header row.""" |
| 822 | if len(rows) < 3: |
| 823 | return rows |
| 824 | |
| 825 | if ( |
| 826 | len(rows) >= 2 |
| 827 | and rows[0][0] |
| 828 | and not any(rows[0][1:]) |
| 829 | and _looks_like_model_row(rows[1]) |
| 830 | ): |
| 831 | outcome = rows[0][0] |
| 832 | header = ["变量"] |
| 833 | header.extend( |
| 834 | f"{model} {outcome}".strip() |
| 835 | for model in rows[1][1:] |
| 836 | ) |
| 837 | return [header] + rows[2:] |
| 838 | |
| 839 | header_offset = 0 |
| 840 | groups = [""] * (len(rows[0]) - 1) |
| 841 | if not _looks_like_model_row(rows[0]) and _looks_like_model_row(rows[1]): |
| 842 | header_offset = 1 |
| 843 | groups = _regression_group_labels(rows[0], len(rows[1]) - 1) |
| 844 | |
| 845 | if not _looks_like_model_row(rows[header_offset]): |
| 846 | return rows |
| 847 | if len(rows) <= header_offset + 1 or not _looks_like_outcome_row(rows[header_offset + 1]): |
| 848 | return rows |
| 849 | |
| 850 | model_row = rows[header_offset] |
| 851 | outcome_row = rows[header_offset + 1] |
| 852 | header = ["变量"] |
| 853 | for idx, model in enumerate(model_row[1:]): |
| 854 | pieces = [] |
| 855 | if idx < len(groups) and groups[idx]: |
| 856 | pieces.append(groups[idx]) |
| 857 | if model: |
| 858 | pieces.append(model) |
| 859 | if idx + 1 < len(outcome_row) and outcome_row[idx + 1]: |
| 860 | pieces.append(outcome_row[idx + 1]) |
| 861 | header.append(" ".join(pieces).strip()) |
| 862 | return [header] + rows[header_offset + 2:] |
| 863 | |
| 864 | |
| 865 | def _fix_paired_sample_t_table(rows: list[list[str]]) -> list[list[str]]: |
| 866 | """Collapse multi-row paired-sample T-test headings into readable columns.""" |
| 867 | if not rows or not any("成对差分" in cell for cell in rows[0]): |
| 868 | return rows |
| 869 | body = [row for row in rows if row and row[0].startswith("对")] |
| 870 | if len(body) < 1: |
| 871 | return rows |
| 872 | header = [ |
| 873 | "配对", |
| 874 | "变量", |
| 875 | "均值", |
| 876 | "标准差", |
| 877 | "均值的标准误", |
| 878 | "差分95%置信区间下限", |
| 879 | "差分95%置信区间上限", |
| 880 | "t", |
| 881 | "Df", |
| 882 | "Sig.(双侧)", |
| 883 | ] |
| 884 | fixed_rows = [header] |
| 885 | for row in body: |
| 886 | fixed_rows.append(row[:len(header)] + [""] * max(0, len(header) - len(row))) |
| 887 | return fixed_rows |
| 888 | |
| 889 | |
| 890 | def _fix_variable_definition_table(rows: list[list[str]]) -> list[list[str]]: |
| 891 | """Repeat variable-category labels for common variable definition tables.""" |
| 892 | if not rows or rows[0] != ["变量类型", "变量名称", "符号", "变量说明"]: |
| 893 | return rows |
| 894 | |
| 895 | fixed = [rows[0]] |
| 896 | for row in rows[1:]: |
| 897 | if not any(row): |
| 898 | continue |
| 899 | name = row[1] if len(row) > 1 else "" |
| 900 | symbol = row[2] if len(row) > 2 else "" |
| 901 | description = row[3] if len(row) > 3 else "" |
| 902 | if not name or not symbol: |
| 903 | continue |
| 904 | |
| 905 | if symbol in {"R＆D", "Fixed", "Hc"}: |
| 906 | category = "被解释变量" |
| 907 | elif symbol == "Vat": |
| 908 | category = "解释变量" |
| 909 | else: |
| 910 | category = "控制变量" |
| 911 | fixed.append([category, name, symbol, description]) |
| 912 | return fixed |
| 913 | |
| 914 | |
| 915 | def _fix_correlation_triangle(rows: list[list[str]]) -> list[list[str]]: |
| 916 | """Restore the missing last self-correlation column in triangular tables.""" |
| 917 | if len(rows) < 4 or not rows[0] or rows[0][0] != "变量": |
| 918 | return rows |
| 919 | body_names = [row[0] for row in rows[1:] if row and row[0]] |
| 920 | header_names = rows[0][1:] |
| 921 | if len(body_names) != len(header_names) + 1: |
| 922 | return rows |
| 923 | missing_name = body_names[-1] |
| 924 | fixed = [rows[0] + [missing_name]] |
| 925 | for row in rows[1:-1]: |
| 926 | fixed.append(row + [""]) |
| 927 | fixed.append(rows[-1] + ["1"]) |
| 928 | return fixed |
| 929 | |
| 930 | |
| 931 | def _postprocess_table_rows(rows: list[list[str]]) -> list[list[str]]: |
| 932 | """Apply Markdown-oriented cleanup to extracted table rows.""" |
| 933 | rows = _merge_label_underscore_rows(rows) |
| 934 | rows = _merge_single_cell_continuations(rows) |
| 935 | rows = _fix_variable_definition_table(rows) |
| 936 | rows = _fix_paired_sample_t_table(rows) |
| 937 | rows = _flatten_regression_header(rows) |
| 938 | rows = _fix_correlation_triangle(rows) |
| 939 | return rows |
| 940 | |
| 941 | |
| 942 | def _rows_to_markdown(rows: list[list[str]]) -> str: |
| 943 | """Convert cleaned table rows to GitHub-flavored Markdown.""" |
| 944 | if len(rows) < 2: |
| 945 | return "" |
| 946 | col_count = max(len(row) for row in rows) |
| 947 | padded = [row + [""] * (col_count - len(row)) for row in rows] |
| 948 | header = padded[0] |
| 949 | body = padded[1:] |
| 950 | lines = [ |
| 951 | "|" + "|".join(header) + "|", |
| 952 | "|" + "|".join(["---"] * col_count) + "|", |
| 953 | ] |
| 954 | lines.extend("|" + "|".join(row) + "|" for row in body) |
| 955 | return "\n".join(lines) |
| 956 | |
| 957 | |
| 958 | def _table_to_markdown(tab: object) -> str: |
| 959 | """Convert a PyMuPDF table object to cleaned Markdown.""" |
| 960 | try: |
| 961 | rows = tab.extract() or [] |
| 962 | except Exception: |
| 963 | return "" |
| 964 | return _rows_to_markdown(_clean_table_rows(rows)) |
| 965 | |
| 966 | |
| 967 | def _is_valid_table_markdown(markdown: str) -> bool: |
| 968 | """Return whether generated Markdown contains a minimally useful table.""" |
| 969 | return markdown.count("\n") >= 2 and markdown.startswith("|") |
| 970 | |
| 971 | |
| 972 | def _markdown_col_count(markdown: str) -> int: |
| 973 | """Return the column count implied by the first Markdown table row.""" |
| 974 | first_line = markdown.splitlines()[0] if markdown else "" |
| 975 | return max(0, first_line.count("|") - 1) |
| 976 | |
| 977 | |
| 978 | def _append_table_markdown_candidate( |
| 979 | candidates: list[dict[str, object]], |
| 980 | bbox: fitz.Rect, |
| 981 | markdown: str, |
| 982 | method: str, |
| 983 | replace_narrow: bool = False, |
| 984 | ) -> None: |
| 985 | """Append or replace a table candidate after overlap deduplication.""" |
| 986 | if not _is_valid_table_markdown(markdown): |
| 987 | return |
| 988 | |
| 989 | for index, candidate in enumerate(candidates): |
| 990 | existing = candidate["bbox"] |
| 991 | if not isinstance(existing, fitz.Rect): |
| 992 | continue |
| 993 | overlap = (bbox & existing).get_area() |
| 994 | if overlap <= 0.8 * min(bbox.get_area(), existing.get_area()): |
| 995 | continue |
| 996 | |
| 997 | existing_markdown = str(candidate.get("content", "")) |
| 998 | can_replace = ( |
| 999 | replace_narrow |
| 1000 | and bbox.width > existing.width * 1.4 |
| 1001 | and _markdown_col_count(markdown) >= _markdown_col_count(existing_markdown) |
| 1002 | ) |
| 1003 | if can_replace: |
| 1004 | candidates[index] = { |
| 1005 | "bbox": bbox, |
| 1006 | "content": markdown, |
| 1007 | "method": method, |
| 1008 | } |
| 1009 | return |
| 1010 | |
| 1011 | candidates.append({ |
| 1012 | "bbox": bbox, |
| 1013 | "content": markdown, |
| 1014 | "method": method, |
| 1015 | }) |
| 1016 | |
| 1017 | |
| 1018 | def _add_table_candidate( |
| 1019 | candidates: list[dict[str, object]], |
| 1020 | tab: object, |
| 1021 | method: str, |
| 1022 | ) -> None: |
| 1023 | """Append a table candidate if it has useful Markdown and is not duplicate.""" |
| 1024 | markdown = _table_to_markdown(tab) |
| 1025 | if not _is_valid_table_markdown(markdown): |
| 1026 | return |
| 1027 | |
| 1028 | bbox = fitz.Rect(tab.bbox) |
| 1029 | _append_table_markdown_candidate(candidates, bbox, markdown, method) |
| 1030 | |
| 1031 | |
| 1032 | def _merge_word_runs(words: list[tuple]) -> list[dict[str, object]]: |
| 1033 | """Group PyMuPDF words into row-level text runs.""" |
| 1034 | rows: list[list[tuple]] = [] |
| 1035 | for word in sorted(words, key=lambda item: (item[1], item[0])): |
| 1036 | if not rows or abs(rows[-1][0][1] - word[1]) > 4: |
| 1037 | rows.append([word]) |
| 1038 | else: |
| 1039 | rows[-1].append(word) |
| 1040 | |
| 1041 | runs = [] |
| 1042 | for row in rows: |
| 1043 | row_runs = [] |
| 1044 | for word in sorted(row, key=lambda item: item[0]): |
| 1045 | x0, y0, x1, y1, text = word[:5] |
| 1046 | if row_runs and x0 - row_runs[-1]["x1"] <= 8: |
| 1047 | row_runs[-1]["x1"] = x1 |
| 1048 | row_runs[-1]["y0"] = min(row_runs[-1]["y0"], y0) |
| 1049 | row_runs[-1]["y1"] = max(row_runs[-1]["y1"], y1) |
| 1050 | row_runs[-1]["text"] = f"{row_runs[-1]['text']} {text}" |
| 1051 | else: |
| 1052 | row_runs.append({ |
| 1053 | "x0": x0, |
| 1054 | "y0": y0, |
| 1055 | "x1": x1, |
| 1056 | "y1": y1, |
| 1057 | "text": text, |
| 1058 | }) |
| 1059 | runs.extend(row_runs) |
| 1060 | return runs |
| 1061 | |
| 1062 | |
| 1063 | def _cluster_word_columns(runs: list[dict[str, object]]) -> list[float]: |
| 1064 | """Infer stable table columns from word-run centers.""" |
| 1065 | centers = sorted((float(run["x0"]) + float(run["x1"])) / 2 for run in runs) |
| 1066 | columns: list[float] = [] |
| 1067 | for center in centers: |
| 1068 | if not columns or abs(center - columns[-1]) > 14: |
| 1069 | columns.append(center) |
| 1070 | else: |
| 1071 | columns[-1] = (columns[-1] + center) / 2 |
| 1072 | return columns |
| 1073 | |
| 1074 | |
| 1075 | def _word_runs_to_rows( |
| 1076 | runs: list[dict[str, object]], |
| 1077 | columns: list[float], |
| 1078 | ) -> list[list[str]]: |
| 1079 | """Place word runs into inferred columns and return table rows.""" |
| 1080 | row_groups: list[list[dict[str, object]]] = [] |
| 1081 | for run in sorted(runs, key=lambda item: (float(item["y0"]), float(item["x0"]))): |
| 1082 | if not row_groups or abs(float(row_groups[-1][0]["y0"]) - float(run["y0"])) > 4: |
| 1083 | row_groups.append([run]) |
| 1084 | else: |
| 1085 | row_groups[-1].append(run) |
| 1086 | |
| 1087 | rows = [] |
| 1088 | for group in row_groups: |
| 1089 | row = [""] * len(columns) |
| 1090 | for run in group: |
| 1091 | center = (float(run["x0"]) + float(run["x1"])) / 2 |
| 1092 | col_index = min(range(len(columns)), key=lambda idx: abs(columns[idx] - center)) |
| 1093 | text = _normalize_table_cell(run["text"]) |
| 1094 | row[col_index] = f"{row[col_index]} {text}".strip() if row[col_index] else text |
| 1095 | rows.append(row) |
| 1096 | return rows |
| 1097 | |
| 1098 | |
| 1099 | def _words_to_markdown_table( |
| 1100 | page: fitz.Page, |
| 1101 | clip: fitz.Rect, |
| 1102 | ) -> tuple[fitz.Rect, str] | None: |
| 1103 | """Build a simple table from word coordinates inside a clipped region.""" |
| 1104 | words = page.get_text("words", clip=clip) |
| 1105 | if len(words) < 6: |
| 1106 | return None |
| 1107 | |
| 1108 | runs = _merge_word_runs(words) |
| 1109 | if len(runs) < 6: |
| 1110 | return None |
| 1111 | |
| 1112 | columns = _cluster_word_columns(runs) |
| 1113 | if len(columns) < 3: |
| 1114 | return None |
| 1115 | |
| 1116 | rows = _word_runs_to_rows(runs, columns) |
| 1117 | cleaned_rows = _clean_table_rows(rows) |
| 1118 | markdown = _rows_to_markdown(cleaned_rows) |
| 1119 | if not _is_valid_table_markdown(markdown): |
| 1120 | return None |
| 1121 | |
| 1122 | x0 = min(float(run["x0"]) for run in runs) |
| 1123 | y0 = min(float(run["y0"]) for run in runs) |
| 1124 | x1 = max(float(run["x1"]) for run in runs) |
| 1125 | y1 = max(float(run["y1"]) for run in runs) |
| 1126 | return fitz.Rect(x0, y0, x1, y1), markdown |
| 1127 | |
| 1128 | |
| 1129 | def _find_tables_in_clip( |
| 1130 | page: fitz.Page, |
| 1131 | clip: fitz.Rect, |
| 1132 | ) -> list[object]: |
| 1133 | """Find text-strategy tables inside a clipped page region.""" |
| 1134 | if clip.height < 20 or clip.width < 80: |
| 1135 | return [] |
| 1136 | try: |
| 1137 | return list(page.find_tables(strategy="text", clip=clip)) |
| 1138 | except Exception: |
| 1139 | return [] |
| 1140 | |
| 1141 | |
| 1142 | def find_page_tables( |
| 1143 | page: fitz.Page, |
| 1144 | include_top_continuation: bool = False, |
| 1145 | ) -> tuple[list[dict[str, object]], bool]: |
| 1146 | """Find line-detected tables plus caption-guided text tables on one page.""" |
| 1147 | candidates: list[dict[str, object]] = [] |
| 1148 | try: |
| 1149 | for tab in page.find_tables(): |
| 1150 | _add_table_candidate(candidates, tab, "lines") |
| 1151 | except Exception: |
| 1152 | pass |
| 1153 | |
| 1154 | lines = _extract_text_lines(page) |
| 1155 | for rect, text in lines: |
| 1156 | if not _is_table_caption(text): |
| 1157 | continue |
| 1158 | start_y = _caption_table_start_y(rect, lines) |
| 1159 | bottom_y = _table_region_bottom(page, lines, start_y) |
| 1160 | clip = fitz.Rect(0, start_y, page.rect.width, bottom_y) |
| 1161 | for tab in _find_tables_in_clip(page, clip): |
| 1162 | _add_table_candidate(candidates, tab, "caption-text") |
| 1163 | word_table = _words_to_markdown_table(page, clip) |
| 1164 | if word_table: |
| 1165 | bbox, markdown = word_table |
| 1166 | _append_table_markdown_candidate( |
| 1167 | candidates, |
| 1168 | bbox, |
| 1169 | markdown, |
| 1170 | "caption-words", |
| 1171 | replace_narrow=True, |
| 1172 | ) |
| 1173 | |
| 1174 | if include_top_continuation: |
| 1175 | start_y = page.rect.height * 0.08 |
| 1176 | bottom_y = _table_region_bottom(page, lines, start_y) |
| 1177 | clip = fitz.Rect(0, start_y, page.rect.width, bottom_y) |
| 1178 | for tab in _find_tables_in_clip(page, clip): |
| 1179 | _add_table_candidate(candidates, tab, "continuation-text") |
| 1180 | word_table = _words_to_markdown_table(page, clip) |
| 1181 | if word_table: |
| 1182 | bbox, markdown = word_table |
| 1183 | _append_table_markdown_candidate( |
| 1184 | candidates, |
| 1185 | bbox, |
| 1186 | markdown, |
| 1187 | "continuation-words", |
| 1188 | replace_narrow=True, |
| 1189 | ) |
| 1190 | |
| 1191 | candidates.sort(key=lambda candidate: candidate["bbox"].y0) |
| 1192 | table_continues = any( |
| 1193 | isinstance(candidate["bbox"], fitz.Rect) |
| 1194 | and candidate["bbox"].y1 >= page.rect.height * TABLE_CONTINUATION_Y_RATIO |
| 1195 | for candidate in candidates |
| 1196 | ) |
| 1197 | return candidates, table_continues |
| 1198 | |
| 1199 | |
| 1200 | def _is_markdown_table_line(line: str) -> bool: |
| 1201 | """Return whether a Markdown line belongs to a pipe table.""" |
| 1202 | return line.startswith("|") |
| 1203 | |
| 1204 | |
| 1205 | def _is_markdown_separator_line(line: str) -> bool: |
| 1206 | """Return whether a Markdown table line is the separator row.""" |
| 1207 | cells = [cell.strip() for cell in line.strip("|").split("|")] |
| 1208 | return bool(cells) and all(cell and set(cell) <= {"-", ":"} for cell in cells) |
| 1209 | |
| 1210 | |
| 1211 | def _compatible_table_headers(first: list[str], second: list[str]) -> bool: |
| 1212 | """Return whether two Markdown table blocks have the same flattened header.""" |
| 1213 | if len(first) < 2 or len(second) < 2: |
| 1214 | return False |
| 1215 | if not _is_markdown_separator_line(first[1]) or not _is_markdown_separator_line(second[1]): |
| 1216 | return False |
| 1217 | return first[0] == second[0] and _markdown_col_count(first[0]) > 2 |
| 1218 | |
| 1219 | |
| 1220 | def _is_table_continuation_noise(line: str) -> bool: |
| 1221 | """Allow only page/header noise between split table parts.""" |
| 1222 | text = line.strip() |
| 1223 | if not text: |
| 1224 | return True |
| 1225 | if text.startswith("<!-- Page ") and text.endswith("-->"): |
| 1226 | return True |
| 1227 | if re.fullmatch(r'\d+', text): |
| 1228 | return True |
| 1229 | if "重庆大学硕士学位论文" in text: |
| 1230 | return True |
| 1231 | continuation_labels = [ |
| 1232 | "营改增", |
| 1233 | "深化增值税改革", |
| 1234 | "国有企业", |
| 1235 | "非国有企业", |
| 1236 | ] |
| 1237 | return any(label in text for label in continuation_labels) |
| 1238 | |
| 1239 | |
| 1240 | def _read_markdown_table_block(lines: list[str], start: int) -> tuple[list[str], int]: |
| 1241 | """Read a contiguous Markdown table block from ``start``.""" |
| 1242 | end = start |
| 1243 | while end < len(lines) and _is_markdown_table_line(lines[end]): |
| 1244 | end += 1 |
| 1245 | return lines[start:end], end |
| 1246 | |
| 1247 | |
| 1248 | def merge_markdown_continuation_tables(markdown: str) -> str: |
| 1249 | """Merge split cross-page Markdown tables with repeated headers.""" |
| 1250 | lines = markdown.splitlines() |
| 1251 | result = [] |
| 1252 | index = 0 |
| 1253 | |
| 1254 | while index < len(lines): |
| 1255 | if not _is_markdown_table_line(lines[index]): |
| 1256 | result.append(lines[index]) |
| 1257 | index += 1 |
| 1258 | continue |
| 1259 | |
| 1260 | table, table_end = _read_markdown_table_block(lines, index) |
| 1261 | search = table_end |
| 1262 | while True: |
| 1263 | between_start = search |
| 1264 | while search < len(lines) and not _is_markdown_table_line(lines[search]): |
| 1265 | if not _is_table_continuation_noise(lines[search]): |
| 1266 | break |
| 1267 | search += 1 |
| 1268 | |
| 1269 | if search >= len(lines) or not _is_markdown_table_line(lines[search]): |
| 1270 | break |
| 1271 | if any( |
| 1272 | not _is_table_continuation_noise(line) |
| 1273 | for line in lines[between_start:search] |
| 1274 | ): |
| 1275 | break |
| 1276 | |
| 1277 | next_table, next_end = _read_markdown_table_block(lines, search) |
| 1278 | if not _compatible_table_headers(table, next_table): |
| 1279 | break |
| 1280 | |
| 1281 | table.extend(next_table[2:]) |
| 1282 | search = next_end |
| 1283 | |
| 1284 | result.extend(table) |
| 1285 | index = search if search != table_end else table_end |
| 1286 | |
| 1287 | return "\n".join(result) |
| 1288 | |
| 1289 | |
| 1290 | def merge_adjacent_formatting(text: str) -> str: |
| 1291 | """Merge adjacent same-style formatted spans split across PDF tokens. |
| 1292 | |
| 1293 | PyMuPDF often emits a phrase as several spans, so per-span wrapping in |
| 1294 | ``format_span_text`` produces ``**X****Y**`` (bold) or ``***X******Y***`` |
| 1295 | (bold-italic) where one phrase is intended. Collapse the abutting markers |
| 1296 | so the run reads as a single phrase: ``**X Y**`` / ``***X Y***``. |
| 1297 | |
| 1298 | Italic-italic adjacency (``*X**Y*``) is indistinguishable from a plain |
| 1299 | bold span's open/close pair and is left alone — merging it would corrupt |
| 1300 | every legitimate ``**bold**`` phrase on the page. The previous regexes |
| 1301 | ``\\*\\s*\\*`` and ``\\*\\*\\*\\s*\\*\\*\\*`` did exactly that, deleting |
| 1302 | all bold formatting from the converted Markdown. |
| 1303 | """ |
| 1304 | # Bold-italic adjacency first so the inner ``******`` isn't half-eaten |
| 1305 | # by the bold pass. |
| 1306 | text = re.sub(r'\*{6}', ' ', text) |
| 1307 | text = re.sub(r'\*{4}', ' ', text) |
| 1308 | return text |
| 1309 | |
| 1310 | |
| 1311 | def is_sentence_end(text: str) -> bool: |
| 1312 | """Check if the text ends with sentence-ending punctuation.""" |
| 1313 | text = text.rstrip() |
| 1314 | if not text: |
| 1315 | return True |
| 1316 | end_puncts = '.。!！?？:：;；' |
| 1317 | return text[-1] in end_puncts |
| 1318 | |
| 1319 | |
| 1320 | def should_merge_lines(current: dict, next_line: dict) -> bool: |
| 1321 | """Determine if two lines should be merged into the same paragraph.""" |
| 1322 | if current.get("is_heading") or next_line.get("is_heading"): |
| 1323 | return False |
| 1324 | if current.get("is_list") or next_line.get("is_list"): |
| 1325 | return False |
| 1326 | if is_sentence_end(current.get("content", "")): |
| 1327 | return False |
| 1328 | return True |
| 1329 | |
| 1330 | |
| 1331 | def extract_pdf_to_markdown( |
| 1332 | pdf_path: str, |
| 1333 | output_path: str = None, |
| 1334 | images: str = "filtered", |
| 1335 | render_vector_figures: bool = False, |
| 1336 | vector_figure_dpi: int = VECTOR_FIGURE_DPI, |
| 1337 | ) -> str: |
| 1338 | """Extract text, images, and tables from a PDF and convert to Markdown. |
| 1339 | |
| 1340 | Args: |
| 1341 | pdf_path: Path to the PDF file. |
| 1342 | output_path: Optional output path for the Markdown file. |
| 1343 | images: Image extraction mode. |
| 1344 | "filtered" = apply size/quality filters (default), |
| 1345 | "all" = extract all images without filtering, |
| 1346 | "none" = skip all images. |
| 1347 | render_vector_figures: Rasterize large vector drawing regions as PNGs. |
| 1348 | vector_figure_dpi: DPI used for rendered vector figure PNGs. |
| 1349 | """ |
| 1350 | try: |
| 1351 | doc = fitz.open(pdf_path) |
| 1352 | except Exception as e: |
| 1353 | print(f"[ERROR] Failed to open PDF file: {e}") |
| 1354 | return "" |
| 1355 | |
| 1356 | if len(doc) >= 200: |
| 1357 | print(f"[HINT] {len(doc)} pages — for very large PDFs, consider splitting " |
| 1358 | f"the source by chapter beforehand (e.g. with pdftk / qpdf / PyPDF2) " |
| 1359 | f"and converting each part individually.") |
| 1360 | |
| 1361 | filename = Path(pdf_path).stem |
| 1362 | title = re.sub(r'^\d+-', '', filename).strip() |
| 1363 | |
| 1364 | print(f"[INFO] Analyzing document structure...") |
| 1365 | size_map = analyze_font_sizes(doc) |
| 1366 | print(f" Font size mapping: body={size_map.get('body', 'N/A')}, " + |
| 1367 | f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}") |
| 1368 | |
| 1369 | print(f"[INFO] Detecting repeated headers/footers...") |
| 1370 | noise_texts = detect_headers_footers(doc) |
| 1371 | if noise_texts: |
| 1372 | print(f" Found {len(noise_texts)} repeated noise texts (will be removed):") |
| 1373 | for t in list(noise_texts)[:3]: |
| 1374 | print(f" - {t[:30]}...") |
| 1375 | |
| 1376 | markdown_content = f"# {title}\n\n" |
| 1377 | seen_image_hashes = set() # Track seen image hashes for deduplication |
| 1378 | |
| 1379 | img_dir = None |
| 1380 | rel_img_dir = None |
| 1381 | if output_path: |
| 1382 | output_path = Path(output_path) |
| 1383 | rel_img_dir = f"{output_path.stem}_files" |
| 1384 | img_dir = output_path.parent / rel_img_dir |
| 1385 | |
| 1386 | img_count = 0 |
| 1387 | image_manifest: list[dict[str, object]] = [] |
| 1388 | previous_table_continues = False |
| 1389 | |
| 1390 | for page_num, page in enumerate(doc, 1): |
| 1391 | if page_num > 1: |
| 1392 | # Add page break marker to help LLM understand context segmentation |
| 1393 | markdown_content += f"\n\n<!-- Page {page_num} -->\n\n" |
| 1394 | |
| 1395 | table_candidates, previous_table_continues = find_page_tables( |
| 1396 | page, |
| 1397 | include_top_continuation=previous_table_continues, |
| 1398 | ) |
| 1399 | tab_rects = [ |
| 1400 | candidate["bbox"] |
| 1401 | for candidate in table_candidates |
| 1402 | if isinstance(candidate["bbox"], fitz.Rect) |
| 1403 | ] |
| 1404 | |
| 1405 | page_elements = [] |
| 1406 | |
| 1407 | for table in table_candidates: |
| 1408 | bbox = table["bbox"] |
| 1409 | if not isinstance(bbox, fitz.Rect): |
| 1410 | continue |
| 1411 | page_elements.append({ |
| 1412 | "y0": bbox.y0, |
| 1413 | "type": 2, |
| 1414 | "content": table["content"] |
| 1415 | }) |
| 1416 | print(f" [OK] Found table: P{page_num} ({table['method']})") |
| 1417 | |
| 1418 | if render_vector_figures: |
| 1419 | for figure_rect in detect_vector_figure_rects(page, tab_rects): |
| 1420 | page_elements.append({ |
| 1421 | "y0": figure_rect.y0, |
| 1422 | "type": 3, |
| 1423 | "content": figure_rect, |
| 1424 | }) |
| 1425 | print(f" [OK] Found vector figure region: P{page_num} {tuple(round(v, 1) for v in figure_rect)}") |
| 1426 | |
| 1427 | blocks = page.get_text("dict")["blocks"] |
| 1428 | |
| 1429 | for block in blocks: |
| 1430 | block_rect = fitz.Rect(block["bbox"]) |
| 1431 | |
| 1432 | # Check if this is table content |
| 1433 | is_in_table = False |
| 1434 | for tab_rect in tab_rects: |
| 1435 | intersect = block_rect & tab_rect |
| 1436 | if intersect.get_area() > 0.6 * block_rect.get_area(): |
| 1437 | is_in_table = True |
| 1438 | break |
| 1439 | |
| 1440 | if is_in_table: |
| 1441 | continue |
| 1442 | |
| 1443 | if block["type"] == 0: |
| 1444 | # Check if this is noise text to be filtered (whole block match) |
| 1445 | block_text_full = "".join([span["text"] for line in block["lines"] for span in line["spans"]]).strip() |
| 1446 | if block_text_full in noise_texts: |
| 1447 | continue |
| 1448 | |
| 1449 | for line in block["lines"]: |
| 1450 | line_text = "" |
| 1451 | line_size = 0 |
| 1452 | line_flags = 0 |
| 1453 | span_count = 0 |
| 1454 | is_code_line = False |
| 1455 | |
| 1456 | formatted_spans = [] |
| 1457 | for span in line["spans"]: |
| 1458 | span_text = CONTROL_CHARS_RE.sub('', span["text"]) |
| 1459 | if not span_text.strip(): |
| 1460 | if span_text: |
| 1461 | formatted_spans.append(span_text) |
| 1462 | continue |
| 1463 | |
| 1464 | span_size = span["size"] |
| 1465 | span_flags = span["flags"] |
| 1466 | |
| 1467 | line_size = max(line_size, span_size) |
| 1468 | line_flags |= span_flags |
| 1469 | span_count += 1 |
| 1470 | |
| 1471 | heading_level = get_heading_level(span_size, size_map, span_text, span_flags) |
| 1472 | |
| 1473 | # Detect code font |
| 1474 | font_name = span.get("font", "") |
| 1475 | if is_monospace_font(font_name): |
| 1476 | is_code_line = True |
| 1477 | formatted_spans.append(span_text) # No formatting for code |
| 1478 | elif heading_level > 0: |
| 1479 | formatted_spans.append(span_text.strip()) |
| 1480 | else: |
| 1481 | formatted_spans.append(format_span_text(span_text, span_flags)) |
| 1482 | |
| 1483 | line_text = ''.join(formatted_spans).strip() |
| 1484 | if not line_text: |
| 1485 | continue |
| 1486 | |
| 1487 | # Secondary check: line-level noise match (sometimes blocks are split) |
| 1488 | if line_text in noise_texts: |
| 1489 | continue |
| 1490 | |
| 1491 | line_text = merge_adjacent_formatting(line_text) |
| 1492 | |
| 1493 | heading_level = get_heading_level(line_size, size_map, line_text, line_flags) |
| 1494 | |
| 1495 | is_list, list_type, list_content = detect_list_item(line_text) |
| 1496 | |
| 1497 | if heading_level > 0: |
| 1498 | prefix = '#' * heading_level + ' ' |
| 1499 | clean_line = re.sub(r'\*+([^*]+)\*+', r'\1', line_text) |
| 1500 | final_text = prefix + clean_line |
| 1501 | elif is_list: |
| 1502 | final_text = list_content |
| 1503 | else: |
| 1504 | final_text = line_text |
| 1505 | |
| 1506 | page_elements.append({ |
| 1507 | "y0": line["bbox"][1], |
| 1508 | "type": 0, |
| 1509 | "content": final_text, |
| 1510 | "is_heading": heading_level > 0, |
| 1511 | "is_list": is_list, |
| 1512 | "is_code": is_code_line |
| 1513 | }) |
| 1514 | |
| 1515 | elif block["type"] == 1: |
| 1516 | if images == "none": |
| 1517 | pass |
| 1518 | elif images == "all" or should_keep_image(block, page.rect, seen_image_hashes): |
| 1519 | page_elements.append({ |
| 1520 | "y0": block["bbox"][1], |
| 1521 | "type": 1, |
| 1522 | "content": block |
| 1523 | }) |
| 1524 | else: |
| 1525 | w, h = block.get("width", 0), block.get("height", 0) |
| 1526 | print(f" [SKIP] Filtered small/decorative image: {w}x{h}px, {len(block.get('image', b''))} bytes") |
| 1527 | |
| 1528 | page_elements.sort(key=lambda x: x["y0"]) |
| 1529 | |
| 1530 | # Merge adjacent same-level short headings |
| 1531 | page_elements = merge_adjacent_headings(page_elements) |
| 1532 | |
| 1533 | merged_elements = [] |
| 1534 | i = 0 |
| 1535 | while i < len(page_elements): |
| 1536 | el = page_elements[i] |
| 1537 | if el["type"] == 0 and not el.get("is_heading") and not el.get("is_list"): |
| 1538 | merged_content = el["content"] |
| 1539 | j = i + 1 |
| 1540 | while j < len(page_elements): |
| 1541 | next_el = page_elements[j] |
| 1542 | if next_el["type"] != 0: |
| 1543 | break |
| 1544 | if not should_merge_lines({"content": merged_content, "is_heading": False, "is_list": False}, next_el): |
| 1545 | break |
| 1546 | merged_content += " " + next_el["content"] |
| 1547 | j += 1 |
| 1548 | merged_elements.append({ |
| 1549 | "type": 0, |
| 1550 | "content": remove_page_footer(merged_content), |
| 1551 | "is_heading": False, |
| 1552 | "is_list": False |
| 1553 | }) |
| 1554 | i = j |
| 1555 | else: |
| 1556 | merged_elements.append(el) |
| 1557 | i += 1 |
| 1558 | |
| 1559 | prev_was_list = False |
| 1560 | prev_was_code = False |
| 1561 | code_block_lines = [] |
| 1562 | |
| 1563 | def flush_code_block(): |
| 1564 | """Flush accumulated code block.""" |
| 1565 | nonlocal code_block_lines, markdown_content |
| 1566 | if code_block_lines: |
| 1567 | markdown_content += "```\n" |
| 1568 | markdown_content += "\n".join(code_block_lines) + "\n" |
| 1569 | markdown_content += "```\n\n" |
| 1570 | code_block_lines = [] |
| 1571 | |
| 1572 | for el in merged_elements: |
| 1573 | if el["type"] == 0: |
| 1574 | is_list = el.get("is_list", False) |
| 1575 | is_heading = el.get("is_heading", False) |
| 1576 | is_code = el.get("is_code", False) |
| 1577 | |
| 1578 | if is_code: |
| 1579 | # Accumulate code lines |
| 1580 | if prev_was_list: |
| 1581 | markdown_content += "\n" |
| 1582 | prev_was_list = False |
| 1583 | code_block_lines.append(el["content"]) |
| 1584 | prev_was_code = True |
| 1585 | else: |
| 1586 | # Non-code line, flush accumulated code block first |
| 1587 | if prev_was_code: |
| 1588 | flush_code_block() |
| 1589 | prev_was_code = False |
| 1590 | |
| 1591 | if is_heading: |
| 1592 | if prev_was_list: |
| 1593 | markdown_content += "\n" |
| 1594 | markdown_content += el["content"] + "\n\n" |
| 1595 | prev_was_list = False |
| 1596 | elif is_list: |
| 1597 | markdown_content += el["content"] + "\n" |
| 1598 | prev_was_list = True |
| 1599 | else: |
| 1600 | if prev_was_list: |
| 1601 | markdown_content += "\n" |
| 1602 | markdown_content += el["content"] + "\n\n" |
| 1603 | prev_was_list = False |
| 1604 | |
| 1605 | elif el["type"] == 2: |
| 1606 | if prev_was_code: |
| 1607 | flush_code_block() |
| 1608 | prev_was_code = False |
| 1609 | if prev_was_list: |
| 1610 | markdown_content += "\n" |
| 1611 | markdown_content += el["content"] + "\n\n" |
| 1612 | prev_was_list = False |
| 1613 | |
| 1614 | elif el["type"] == 1: |
| 1615 | if prev_was_code: |
| 1616 | flush_code_block() |
| 1617 | prev_was_code = False |
| 1618 | if img_dir: |
| 1619 | block = el["content"] |
| 1620 | ext = block["ext"] |
| 1621 | image_data = block["image"] |
| 1622 | safe_filename = filename.replace(" ", "_") |
| 1623 | image_name = f"{safe_filename}_p{page_num}_{img_count}.{ext}" |
| 1624 | image_path = img_dir / image_name |
| 1625 | |
| 1626 | try: |
| 1627 | img_dir.mkdir(parents=True, exist_ok=True) |
| 1628 | with open(image_path, "wb") as f: |
| 1629 | f.write(image_data) |
| 1630 | |
| 1631 | if prev_was_list: |
| 1632 | markdown_content += "\n" |
| 1633 | markdown_content += f"\n\n" |
| 1634 | width = int(block.get("width", 0) or 0) |
| 1635 | height = int(block.get("height", 0) or 0) |
| 1636 | ratio = width / height if width > 0 and height > 0 else None |
| 1637 | image_manifest.append({ |
| 1638 | "index": len(image_manifest) + 1, |
| 1639 | "filename": image_name, |
| 1640 | "original_filename": image_name, |
| 1641 | "asset_kind": "bitmap", |
| 1642 | "svg_renderable": True, |
| 1643 | "pptx_native_supported": True, |
| 1644 | "source_kind": "pdf_image", |
| 1645 | "source_ext": f".{ext}", |
| 1646 | "page_index": page_num, |
| 1647 | "occurrence_index": img_count + 1, |
| 1648 | "pixel_width": width or None, |
| 1649 | "pixel_height": height or None, |
| 1650 | "pixel_ratio": round(ratio, 6) if ratio else None, |
| 1651 | "display_ratio": round(ratio, 6) if ratio else None, |
| 1652 | "source_sha256": hashlib.sha256(image_data).hexdigest(), |
| 1653 | "bbox": list(block.get("bbox", [])), |
| 1654 | }) |
| 1655 | img_count += 1 |
| 1656 | prev_was_list = False |
| 1657 | print(f" [OK] Extracted image: {image_name}") |
| 1658 | except Exception as e: |
| 1659 | print(f" [WARN] Failed to save image: {e}") |
| 1660 | |
| 1661 | elif el["type"] == 3: |
| 1662 | if prev_was_code: |
| 1663 | flush_code_block() |
| 1664 | prev_was_code = False |
| 1665 | if img_dir: |
| 1666 | figure_rect = el["content"] |
| 1667 | safe_filename = filename.replace(" ", "_") |
| 1668 | image_name = f"{safe_filename}_p{page_num}_figure_{img_count}.png" |
| 1669 | image_path = img_dir / image_name |
| 1670 | |
| 1671 | try: |
| 1672 | img_dir.mkdir(parents=True, exist_ok=True) |
| 1673 | scale = vector_figure_dpi / 72 |
| 1674 | pix = page.get_pixmap( |
| 1675 | matrix=fitz.Matrix(scale, scale), |
| 1676 | clip=figure_rect, |
| 1677 | alpha=False, |
| 1678 | ) |
| 1679 | pix.save(str(image_path)) |
| 1680 | |
| 1681 | if prev_was_list: |
| 1682 | markdown_content += "\n" |
| 1683 | markdown_content += f"\n\n" |
| 1684 | ratio = pix.width / pix.height if pix.width > 0 and pix.height > 0 else None |
| 1685 | image_manifest.append({ |
| 1686 | "index": len(image_manifest) + 1, |
| 1687 | "filename": image_name, |
| 1688 | "original_filename": image_name, |
| 1689 | "asset_kind": "bitmap", |
| 1690 | "svg_renderable": True, |
| 1691 | "pptx_native_supported": True, |
| 1692 | "source_kind": "pdf_vector_figure", |
| 1693 | "source_ext": ".png", |
| 1694 | "page_index": page_num, |
| 1695 | "occurrence_index": img_count + 1, |
| 1696 | "pixel_width": pix.width, |
| 1697 | "pixel_height": pix.height, |
| 1698 | "pixel_ratio": round(ratio, 6) if ratio else None, |
| 1699 | "display_ratio": round(ratio, 6) if ratio else None, |
| 1700 | "bbox": [ |
| 1701 | figure_rect.x0, |
| 1702 | figure_rect.y0, |
| 1703 | figure_rect.x1, |
| 1704 | figure_rect.y1, |
| 1705 | ], |
| 1706 | }) |
| 1707 | img_count += 1 |
| 1708 | prev_was_list = False |
| 1709 | print(f" [OK] Rendered vector figure: {image_name}") |
| 1710 | except Exception as e: |
| 1711 | print(f" [WARN] Failed to render vector figure: {e}") |
| 1712 | |
| 1713 | # Flush code block at end of page |
| 1714 | if prev_was_code: |
| 1715 | flush_code_block() |
| 1716 | |
| 1717 | doc.close() |
| 1718 | |
| 1719 | markdown_content = merge_markdown_continuation_tables(markdown_content) |
| 1720 | markdown_content = CONTROL_CHARS_RE.sub('', markdown_content) |
| 1721 | markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content) |
| 1722 | markdown_content = markdown_content.strip() + "\n" |
| 1723 | |
| 1724 | if output_path: |
| 1725 | os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True) |
| 1726 | with open(output_path, 'w', encoding='utf-8') as f: |
| 1727 | f.write(markdown_content) |
| 1728 | if img_dir and image_manifest: |
| 1729 | (img_dir / "image_manifest.json").write_text( |
| 1730 | json.dumps(image_manifest, ensure_ascii=False, indent=2) + "\n", |
| 1731 | encoding="utf-8", |
| 1732 | ) |
| 1733 | profile_path = write_conversion_profile_best_effort( |
| 1734 | input_path=pdf_path, |
| 1735 | markdown_path=output_path, |
| 1736 | converter="pdf_to_md.py", |
| 1737 | conversion_type="pdf", |
| 1738 | asset_dir=img_dir, |
| 1739 | ) |
| 1740 | print(f"[OK] Saved Markdown to: {output_path}") |
| 1741 | if profile_path: |
| 1742 | print(f" Wrote conversion profile -> {profile_path}") |
| 1743 | |
| 1744 | return markdown_content |
| 1745 | |
| 1746 | |
| 1747 | def main() -> int: |
| 1748 | """Run the CLI entry point.""" |
| 1749 | parser = argparse.ArgumentParser( |
| 1750 | description='PDF to Markdown converter (with structure detection and LLM optimization)', |
| 1751 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 1752 | epilog=''' |
| 1753 | Examples: |
| 1754 | python pdf_to_md.py book.pdf # Convert a single file |
| 1755 | python pdf_to_md.py book.pdf appendix.pdf # Convert multiple files |
| 1756 | python pdf_to_md.py ./pdfs -o ./markdown # Convert PDFs in a directory |
| 1757 | python pdf_to_md.py book.pdf -o output.md # Specify output file |
| 1758 | python pdf_to_md.py book.pdf --render-vector-figures |
| 1759 | |
| 1760 | Structure detection features: |
| 1761 | - Auto-detect heading levels (based on font size) |
| 1762 | - Detect bold and italic text |
| 1763 | - Detect ordered and unordered lists |
| 1764 | - Extract tables and convert to Markdown format (with deduplication) |
| 1765 | - [New] Smart detection and removal of repeated page headers/footers |
| 1766 | - [New] Add <!-- Page N --> page break markers to help LLM understanding |
| 1767 | ''' |
| 1768 | ) |
| 1769 | |
| 1770 | parser.add_argument('inputs', nargs='+', help='PDF file(s) or directories') |
| 1771 | parser.add_argument( |
| 1772 | '-o', |
| 1773 | '--output', |
| 1774 | help='Output Markdown file for one input, or output directory for multiple inputs/directories', |
| 1775 | ) |
| 1776 | parser.add_argument( |
| 1777 | '--images', |
| 1778 | choices=['all', 'filtered', 'none'], |
| 1779 | default='filtered', |
| 1780 | help='Image extraction mode: filtered=apply size/quality filters (default), all=no filtering, none=skip images', |
| 1781 | ) |
| 1782 | parser.add_argument( |
| 1783 | '--render-vector-figures', |
| 1784 | action='store_true', |
| 1785 | help='Render large vector drawing regions as PNG figure assets', |
| 1786 | ) |
| 1787 | parser.add_argument( |
| 1788 | '--vector-figure-dpi', |
| 1789 | type=int, |
| 1790 | default=VECTOR_FIGURE_DPI, |
| 1791 | help=f'DPI for --render-vector-figures output (default: {VECTOR_FIGURE_DPI})', |
| 1792 | ) |
| 1793 | |
| 1794 | args = parser.parse_args() |
| 1795 | |
| 1796 | return run_path_batch( |
| 1797 | args.inputs, |
| 1798 | {'.pdf'}, |
| 1799 | args.output, |
| 1800 | lambda source, output: bool( |
| 1801 | extract_pdf_to_markdown( |
| 1802 | str(source), |
| 1803 | str(output), |
| 1804 | images=args.images, |
| 1805 | render_vector_figures=args.render_vector_figures, |
| 1806 | vector_figure_dpi=args.vector_figure_dpi, |
| 1807 | ) |
| 1808 | ), |
| 1809 | ) |
| 1810 | |
| 1811 | |
| 1812 | if __name__ == '__main__': |
| 1813 | raise SystemExit(main()) |
| 1814 |