| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | PPT Master - Unified Markdown Converter |
| 4 | |
| 5 | Auto-detect source type and dispatch to the existing source_to_md converters. |
| 6 | |
| 7 | Usage: |
| 8 | python3 scripts/source_to_md.py <file_or_url_or_dir> [<file_or_url_or_dir> ...] [options] |
| 9 | |
| 10 | Examples: |
| 11 | python3 scripts/source_to_md.py paper.pdf |
| 12 | python3 scripts/source_to_md.py paper.pdf report.docx deck.pptx |
| 13 | python3 scripts/source_to_md.py ./sources -o ./markdown |
| 14 | python3 scripts/source_to_md.py report.docx -o report.md |
| 15 | python3 scripts/source_to_md.py deck.pptx --json |
| 16 | |
| 17 | Dependencies: |
| 18 | Same as the backend converter selected for the input. |
| 19 | """ |
| 20 | |
| 21 | from __future__ import annotations |
| 22 | |
| 23 | import argparse |
| 24 | import json |
| 25 | import subprocess |
| 26 | import sys |
| 27 | import tempfile |
| 28 | from pathlib import Path |
| 29 | |
| 30 | _SCRIPTS_DIR = Path(__file__).resolve().parent |
| 31 | if str(_SCRIPTS_DIR) not in sys.path: |
| 32 | sys.path.insert(0, str(_SCRIPTS_DIR)) |
| 33 | |
| 34 | from console_encoding import configure_utf8_stdio # noqa: E402 |
| 35 | |
| 36 | _SOURCE_TO_MD_DIR = _SCRIPTS_DIR / "source_to_md" |
| 37 | if str(_SOURCE_TO_MD_DIR) not in sys.path: |
| 38 | sys.path.insert(0, str(_SOURCE_TO_MD_DIR)) |
| 39 | |
| 40 | from _conversion_profile import ( # noqa: E402 |
| 41 | build_result_payload, |
| 42 | profile_path_for, |
| 43 | write_conversion_profile, |
| 44 | ) |
| 45 | from _batch import expand_directory_inputs, unique_output_path # noqa: E402 |
| 46 | from _dispatcher import ( # noqa: E402 |
| 47 | build_conversion_command, |
| 48 | default_markdown_path, |
| 49 | detect_source_type, |
| 50 | is_url, |
| 51 | ) |
| 52 | |
| 53 | configure_utf8_stdio() |
| 54 | |
| 55 | |
| 56 | def resolve_output(output: str | None, input_arg: str) -> Path: |
| 57 | return Path(output) if output else default_markdown_path(input_arg) |
| 58 | |
| 59 | |
| 60 | def _print_status(message: str) -> None: |
| 61 | print(message, file=sys.stderr) |
| 62 | |
| 63 | |
| 64 | def _is_supported_directory_item(path: Path) -> bool: |
| 65 | return detect_source_type(str(path)) in { |
| 66 | "pdf", "doc", "excel", "pptx", "markdown", "text", |
| 67 | } |
| 68 | |
| 69 | |
| 70 | def _dispatch_output_arg( |
| 71 | input_arg: str, |
| 72 | conversion_type: str, |
| 73 | output_arg: str | None, |
| 74 | batch_mode: bool, |
| 75 | used_outputs: set[Path], |
| 76 | ) -> str | None: |
| 77 | if output_arg and batch_mode and conversion_type == "web": |
| 78 | return None |
| 79 | if output_arg and batch_mode: |
| 80 | return str( |
| 81 | unique_output_path( |
| 82 | Path(output_arg), |
| 83 | default_markdown_path(input_arg).stem, |
| 84 | used_outputs, |
| 85 | ) |
| 86 | ) |
| 87 | if output_arg: |
| 88 | return output_arg |
| 89 | if batch_mode and conversion_type != "web": |
| 90 | return str(default_markdown_path(input_arg)) |
| 91 | return None |
| 92 | |
| 93 | |
| 94 | def run_backend(command: list[str], script_name: str) -> int: |
| 95 | _print_status(f"[>>] {script_name} {' '.join(command[2:])}") |
| 96 | try: |
| 97 | result = subprocess.run( |
| 98 | command, |
| 99 | check=False, |
| 100 | capture_output=True, |
| 101 | text=True, |
| 102 | encoding="utf-8", |
| 103 | errors="replace", |
| 104 | ) |
| 105 | except KeyboardInterrupt: |
| 106 | return 130 |
| 107 | if result.stdout.strip(): |
| 108 | print(result.stdout.strip(), file=sys.stderr) |
| 109 | if result.stderr.strip(): |
| 110 | print(result.stderr.strip(), file=sys.stderr) |
| 111 | return result.returncode |
| 112 | |
| 113 | |
| 114 | def print_output(path: Path) -> None: |
| 115 | print(f"OUTPUT: {path.resolve()}") |
| 116 | |
| 117 | |
| 118 | def write_passthrough( |
| 119 | input_arg: str, |
| 120 | output: Path, |
| 121 | conversion_type: str, |
| 122 | json_output: bool = False, |
| 123 | ) -> int: |
| 124 | """Copy text-like input to Markdown and write the profile sidecar.""" |
| 125 | source = Path(input_arg) |
| 126 | try: |
| 127 | text = source.read_text(encoding="utf-8", errors="replace") |
| 128 | except OSError as exc: |
| 129 | print(f"[ERROR] Cannot read {source}: {exc}", file=sys.stderr) |
| 130 | return 1 |
| 131 | |
| 132 | output.parent.mkdir(parents=True, exist_ok=True) |
| 133 | if output.resolve() != source.resolve(): |
| 134 | output.write_text(text, encoding="utf-8") |
| 135 | profile = write_conversion_profile( |
| 136 | input_path=input_arg, |
| 137 | markdown_path=output, |
| 138 | converter="source_to_md.py", |
| 139 | conversion_type=conversion_type, |
| 140 | ) |
| 141 | _print_status(f"[OK] Saved Markdown to: {output}") |
| 142 | _print_status(f" Wrote conversion profile -> {profile}") |
| 143 | print_output(output) |
| 144 | if json_output: |
| 145 | payload = build_result_payload( |
| 146 | input_path=input_arg, |
| 147 | markdown_path=output, |
| 148 | converter="source_to_md.py", |
| 149 | conversion_type=conversion_type, |
| 150 | profile_path=profile, |
| 151 | ) |
| 152 | print(json.dumps(payload, ensure_ascii=False)) |
| 153 | return 0 |
| 154 | |
| 155 | |
| 156 | def ensure_profile( |
| 157 | input_arg: str, |
| 158 | output: Path, |
| 159 | converter: str, |
| 160 | conversion_type: str, |
| 161 | ) -> Path: |
| 162 | """Return an existing profile path, writing one if the backend did not.""" |
| 163 | profile = profile_path_for(output) |
| 164 | if profile.is_file(): |
| 165 | return profile |
| 166 | return write_conversion_profile( |
| 167 | input_path=input_arg, |
| 168 | markdown_path=output, |
| 169 | converter=converter, |
| 170 | conversion_type=conversion_type, |
| 171 | ) |
| 172 | |
| 173 | |
| 174 | def print_json_result( |
| 175 | input_arg: str, |
| 176 | output: Path, |
| 177 | converter: str, |
| 178 | conversion_type: str, |
| 179 | profile: Path, |
| 180 | ) -> None: |
| 181 | payload = build_result_payload( |
| 182 | input_path=input_arg, |
| 183 | markdown_path=output, |
| 184 | converter=converter, |
| 185 | conversion_type=conversion_type, |
| 186 | profile_path=profile, |
| 187 | ) |
| 188 | print(json.dumps(payload, ensure_ascii=False)) |
| 189 | |
| 190 | |
| 191 | def _read_emit_result(path: Path) -> Path | None: |
| 192 | try: |
| 193 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 194 | except (OSError, json.JSONDecodeError): |
| 195 | return None |
| 196 | markdown = payload.get("markdown") |
| 197 | return Path(markdown) if isinstance(markdown, str) and markdown else None |
| 198 | |
| 199 | |
| 200 | def _pdf_image_mode(args: argparse.Namespace) -> str | None: |
| 201 | image_mode = args.images |
| 202 | if args.no_images: |
| 203 | image_mode = "none" |
| 204 | if args.filter_images: |
| 205 | image_mode = "filtered" |
| 206 | return image_mode |
| 207 | |
| 208 | |
| 209 | def _validate_image_options(args: argparse.Namespace) -> bool: |
| 210 | selected = sum(bool(value) for value in (args.images, args.no_images, args.filter_images)) |
| 211 | if selected > 1: |
| 212 | print( |
| 213 | "[ERROR] --images, --no-images, and --filter-images are mutually exclusive", |
| 214 | file=sys.stderr, |
| 215 | ) |
| 216 | return False |
| 217 | return True |
| 218 | |
| 219 | |
| 220 | def dispatch_single( |
| 221 | input_arg: str, |
| 222 | conversion_type: str, |
| 223 | output_arg: str | None, |
| 224 | args: argparse.Namespace, |
| 225 | unknown_args: list[str], |
| 226 | web_output_dir: str | None = None, |
| 227 | ) -> int: |
| 228 | """Dispatch one source to the matching existing converter.""" |
| 229 | if conversion_type == "auto": |
| 230 | conversion_type = detect_source_type(input_arg) |
| 231 | |
| 232 | if conversion_type == "markdown": |
| 233 | output = resolve_output(output_arg, input_arg) |
| 234 | return write_passthrough(input_arg, output, "markdown", args.json) |
| 235 | if conversion_type == "text": |
| 236 | output = resolve_output(output_arg, input_arg) |
| 237 | return write_passthrough(input_arg, output, "text", args.json) |
| 238 | |
| 239 | if conversion_type == "web": |
| 240 | output = Path(output_arg) if output_arg else None |
| 241 | emit_result: Path | None = None |
| 242 | extra_args = list(unknown_args) |
| 243 | if output is None: |
| 244 | emit_file = tempfile.NamedTemporaryFile( |
| 245 | prefix="ppt-master-web-result-", |
| 246 | suffix=".json", |
| 247 | delete=False, |
| 248 | ) |
| 249 | emit_file.close() |
| 250 | emit_result = Path(emit_file.name) |
| 251 | extra_args.extend(["--emit-result", str(emit_result)]) |
| 252 | if web_output_dir: |
| 253 | extra_args.extend(["--dir", web_output_dir]) |
| 254 | try: |
| 255 | route = build_conversion_command( |
| 256 | input_arg, |
| 257 | output, |
| 258 | forced_type="web", |
| 259 | extra_args=extra_args, |
| 260 | ) |
| 261 | except ValueError as exc: |
| 262 | if emit_result: |
| 263 | emit_result.unlink(missing_ok=True) |
| 264 | print(f"[ERROR] {exc}", file=sys.stderr) |
| 265 | return 1 |
| 266 | rc = run_backend(route.command, route.script_name) |
| 267 | if rc != 0: |
| 268 | if emit_result: |
| 269 | emit_result.unlink(missing_ok=True) |
| 270 | return rc |
| 271 | output_path = route.output_path |
| 272 | if output_path is None and emit_result is not None: |
| 273 | output_path = _read_emit_result(emit_result) |
| 274 | emit_result.unlink(missing_ok=True) |
| 275 | if output_path and output_path.is_file(): |
| 276 | profile = ensure_profile(input_arg, output_path, route.script_name, "web") |
| 277 | print_output(output_path) |
| 278 | if args.json: |
| 279 | print_json_result(input_arg, output_path, route.script_name, "web", profile) |
| 280 | return 0 |
| 281 | if output is not None: |
| 282 | print(f"[ERROR] Expected Markdown output not found: {output}", file=sys.stderr) |
| 283 | else: |
| 284 | print("[ERROR] Web conversion did not report a Markdown output path", file=sys.stderr) |
| 285 | return 1 |
| 286 | |
| 287 | if conversion_type not in {"pdf", "doc", "excel", "pptx"}: |
| 288 | print( |
| 289 | f"[ERROR] Could not determine conversion type for {input_arg!r}. " |
| 290 | "Use -t pdf|doc|excel|pptx|web|markdown|text.", |
| 291 | file=sys.stderr, |
| 292 | ) |
| 293 | return 1 |
| 294 | |
| 295 | if not is_url(input_arg) and not Path(input_arg).exists(): |
| 296 | print(f"[ERROR] File not found: {input_arg}", file=sys.stderr) |
| 297 | return 1 |
| 298 | |
| 299 | output = resolve_output(output_arg, input_arg) |
| 300 | try: |
| 301 | route = build_conversion_command( |
| 302 | input_arg, |
| 303 | output, |
| 304 | forced_type=conversion_type, |
| 305 | extra_args=unknown_args, |
| 306 | pdf_image_mode=_pdf_image_mode(args), |
| 307 | render_vector_figures=args.render_vector_figures, |
| 308 | ) |
| 309 | except ValueError as exc: |
| 310 | print(f"[ERROR] {exc}", file=sys.stderr) |
| 311 | return 1 |
| 312 | rc = run_backend(route.command, route.script_name) |
| 313 | if rc != 0: |
| 314 | return rc |
| 315 | if not output.is_file(): |
| 316 | print(f"[ERROR] Expected Markdown output not found: {output}", file=sys.stderr) |
| 317 | return 1 |
| 318 | |
| 319 | profile = ensure_profile(input_arg, output, route.script_name, conversion_type) |
| 320 | print_output(output) |
| 321 | if args.json: |
| 322 | print_json_result(input_arg, output, route.script_name, conversion_type, profile) |
| 323 | return 0 |
| 324 | |
| 325 | |
| 326 | def build_parser() -> argparse.ArgumentParser: |
| 327 | parser = argparse.ArgumentParser( |
| 328 | description="Auto-detect source type and convert to Markdown via source_to_md backends.", |
| 329 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 330 | epilog=""" |
| 331 | Examples: |
| 332 | python3 scripts/source_to_md.py paper.pdf |
| 333 | python3 scripts/source_to_md.py paper.pdf report.docx deck.pptx |
| 334 | python3 scripts/source_to_md.py ./sources -o ./markdown |
| 335 | python3 scripts/source_to_md.py report.docx -o output.md |
| 336 | python3 scripts/source_to_md.py deck.pptx --json |
| 337 | python3 scripts/source_to_md.py https://example.com/article -o article.md |
| 338 | |
| 339 | Backend-specific flags not listed here are passed through to the selected |
| 340 | converter, so existing converter behavior remains the source of truth. |
| 341 | """, |
| 342 | ) |
| 343 | parser.add_argument("inputs", nargs="+", help="Input file(s), directories, or URL(s)") |
| 344 | parser.add_argument( |
| 345 | "-t", |
| 346 | "--type", |
| 347 | choices=["auto", "pdf", "doc", "excel", "pptx", "web", "markdown", "text"], |
| 348 | default="auto", |
| 349 | help="Force a conversion type (default: auto)", |
| 350 | ) |
| 351 | parser.add_argument( |
| 352 | "-o", |
| 353 | "--output", |
| 354 | help="Output Markdown file for one input, or output directory for multiple inputs/directories", |
| 355 | ) |
| 356 | parser.add_argument( |
| 357 | "--images", |
| 358 | choices=["all", "filtered", "none"], |
| 359 | help="PDF image extraction mode; maps to pdf_to_md.py --images", |
| 360 | ) |
| 361 | parser.add_argument( |
| 362 | "--no-images", |
| 363 | action="store_true", |
| 364 | help="Alias for --images none on PDF inputs", |
| 365 | ) |
| 366 | parser.add_argument( |
| 367 | "--filter-images", |
| 368 | action="store_true", |
| 369 | help="Alias for --images filtered on PDF inputs", |
| 370 | ) |
| 371 | parser.add_argument( |
| 372 | "--render-vector-figures", |
| 373 | action="store_true", |
| 374 | help="Pass through to pdf_to_md.py for PDF vector figure rendering", |
| 375 | ) |
| 376 | parser.add_argument( |
| 377 | "--json", |
| 378 | action="store_true", |
| 379 | help="Print a machine-readable result after successful conversion", |
| 380 | ) |
| 381 | return parser |
| 382 | |
| 383 | |
| 384 | def _has_pdf_image_flags(args: argparse.Namespace) -> bool: |
| 385 | return bool(args.images or args.no_images or args.filter_images or args.render_vector_figures) |
| 386 | |
| 387 | |
| 388 | def _conversion_type_for_input(input_arg: str, requested_type: str) -> str: |
| 389 | if requested_type == "auto": |
| 390 | return detect_source_type(input_arg) |
| 391 | return requested_type |
| 392 | |
| 393 | |
| 394 | def _validate_pdf_image_flags(args: argparse.Namespace, conversion_types: list[str]) -> bool: |
| 395 | if not _has_pdf_image_flags(args): |
| 396 | return True |
| 397 | if any(conversion_type != "pdf" for conversion_type in conversion_types): |
| 398 | print("[ERROR] Image extraction flags are currently supported only for PDFs", file=sys.stderr) |
| 399 | return False |
| 400 | return True |
| 401 | |
| 402 | |
| 403 | def dispatch_many( |
| 404 | inputs: list[str], |
| 405 | args: argparse.Namespace, |
| 406 | unknown_args: list[str], |
| 407 | conversion_types: list[str], |
| 408 | batch_mode: bool = False, |
| 409 | initial_failures: list[str] | None = None, |
| 410 | ) -> int: |
| 411 | success_count = 0 |
| 412 | failed: list[str] = [] |
| 413 | skipped: list[str] = list(initial_failures or []) |
| 414 | batch_mode = batch_mode or len(inputs) > 1 |
| 415 | if args.output and batch_mode: |
| 416 | output_dir = Path(args.output) |
| 417 | if output_dir.exists() and not output_dir.is_dir(): |
| 418 | print(f"[ERROR] Batch output path is not a directory: {args.output}", file=sys.stderr) |
| 419 | return 1 |
| 420 | output_dir.mkdir(parents=True, exist_ok=True) |
| 421 | |
| 422 | used_outputs: set[Path] = set() |
| 423 | for input_arg, conversion_type in zip(inputs, conversion_types): |
| 424 | output_arg = _dispatch_output_arg( |
| 425 | input_arg, |
| 426 | conversion_type, |
| 427 | args.output, |
| 428 | batch_mode, |
| 429 | used_outputs, |
| 430 | ) |
| 431 | web_output_dir = ( |
| 432 | args.output |
| 433 | if args.output and batch_mode and conversion_type == "web" |
| 434 | else None |
| 435 | ) |
| 436 | if batch_mode: |
| 437 | _print_status(f"\n==> {input_arg}") |
| 438 | |
| 439 | rc = dispatch_single( |
| 440 | input_arg, |
| 441 | conversion_type, |
| 442 | output_arg, |
| 443 | args, |
| 444 | unknown_args, |
| 445 | web_output_dir=web_output_dir, |
| 446 | ) |
| 447 | if rc == 0: |
| 448 | success_count += 1 |
| 449 | else: |
| 450 | failed.append(f"{input_arg}: exit {rc}") |
| 451 | |
| 452 | if batch_mode: |
| 453 | _print_status(f"\n[Done] Success: {success_count}/{len(inputs)}, Failed: {len(failed)}") |
| 454 | if skipped: |
| 455 | _print_status("\n[Skipped directories]:") |
| 456 | for item in skipped: |
| 457 | _print_status(f" - {item}") |
| 458 | if failed: |
| 459 | _print_status("\n[Failed inputs]:") |
| 460 | for item in failed: |
| 461 | _print_status(f" - {item}") |
| 462 | if not inputs: |
| 463 | return 1 |
| 464 | return 0 if not failed and not skipped else 1 |
| 465 | |
| 466 | |
| 467 | def main(argv: list[str] | None = None) -> int: |
| 468 | parser = build_parser() |
| 469 | args, unknown_args = parser.parse_known_args(argv) |
| 470 | |
| 471 | if not _validate_image_options(args): |
| 472 | return 2 |
| 473 | |
| 474 | inputs, expansion_errors, saw_directory = expand_directory_inputs( |
| 475 | args.inputs, |
| 476 | _is_supported_directory_item, |
| 477 | is_external_ref=is_url, |
| 478 | ) |
| 479 | batch_mode = saw_directory or len(inputs) > 1 |
| 480 | |
| 481 | conversion_types = [_conversion_type_for_input(item, args.type) for item in inputs] |
| 482 | if not _validate_pdf_image_flags(args, conversion_types): |
| 483 | return 2 |
| 484 | |
| 485 | if unknown_args and any( |
| 486 | conversion_type in {"markdown", "text"} for conversion_type in conversion_types |
| 487 | ): |
| 488 | print( |
| 489 | "[ERROR] Backend-specific flags cannot be used with markdown/text passthrough inputs", |
| 490 | file=sys.stderr, |
| 491 | ) |
| 492 | return 2 |
| 493 | |
| 494 | return dispatch_many( |
| 495 | inputs, |
| 496 | args, |
| 497 | unknown_args, |
| 498 | conversion_types, |
| 499 | batch_mode=batch_mode, |
| 500 | initial_failures=expansion_errors, |
| 501 | ) |
| 502 | |
| 503 | |
| 504 | if __name__ == "__main__": |
| 505 | raise SystemExit(main()) |
| 506 |