| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | PPT Master - Large Vector Asset Extractor |
| 4 | |
| 5 | Factor large inline vector groups (complex illustrations) out of working SVGs |
| 6 | into project icon assets, leaving a one-line `<use data-icon="namespace/id"/>` |
| 7 | placeholder behind — so the working SVG stays readable (structure, not a wall of |
| 8 | `<path>`). Visually lossless and reversible: the existing icon embedding path |
| 9 | re-inlines each asset before export, so the exported PPTX remains native shapes, |
| 10 | not an embedded picture. |
| 11 | |
| 12 | Because re-inlining restores the extracted vector subtree, the detection |
| 13 | threshold is a readability convenience — it changes which blobs are factored |
| 14 | out, not whether the export stays editable. |
| 15 | |
| 16 | Usage: |
| 17 | python3 scripts/extract_svg_assets.py <svg_dir> [options] |
| 18 | |
| 19 | Examples: |
| 20 | python3 scripts/extract_svg_assets.py import_ws/authoring-svg \ |
| 21 | --icons-dir import_ws/icons --icon-namespace imported \ |
| 22 | --inplace --id-prefix layered --clean-stale |
| 23 | python3 scripts/extract_svg_assets.py import_ws/authoring-svg-flat \ |
| 24 | --icons-dir import_ws/icons --icon-namespace imported \ |
| 25 | --reuse-inventory import_ws/authoring-svg_vector_asset_inventory.json \ |
| 26 | --inplace --id-prefix flat --clean-stale |
| 27 | python3 scripts/extract_svg_assets.py project/svg_output --inplace --min-drawables 40 |
| 28 | |
| 29 | Dependencies: |
| 30 | None (standard library only). |
| 31 | |
| 32 | See workflows/create-template.md and svg_finalize/embed_icons.py. |
| 33 | """ |
| 34 | |
| 35 | from __future__ import annotations |
| 36 | |
| 37 | import argparse |
| 38 | import copy |
| 39 | import hashlib |
| 40 | import json |
| 41 | import re |
| 42 | import sys |
| 43 | from pathlib import Path |
| 44 | from typing import Optional |
| 45 | from xml.etree import ElementTree as ET |
| 46 | |
| 47 | from console_encoding import configure_utf8_stdio |
| 48 | from svg_authoring_view import ( |
| 49 | AUTHORING_MANIFEST_NAME, |
| 50 | write_authoring_summary, |
| 51 | ) |
| 52 | |
| 53 | configure_utf8_stdio() |
| 54 | |
| 55 | SVG_NS = "http://www.w3.org/2000/svg" |
| 56 | DRAWABLE = {"path", "polygon", "polyline", "rect", "circle", "ellipse", "line"} |
| 57 | SEMANTIC_CONTENT = {"text", "tspan", "foreignObject"} |
| 58 | DEFAULT_MIN_DRAWABLES = 20 |
| 59 | DEFAULT_MIN_BYTES = 3000 |
| 60 | DEFAULT_MIN_DECORATION_BYTES = 3000 |
| 61 | SOURCE_REF_ATTRIBUTE = "data-pptx-source-ref" |
| 62 | ICON_NAMESPACE_RE = re.compile(r"^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?$") |
| 63 | URL_REF_RE = re.compile(r"url\(\s*(['\"]?)#([^)'\"]\S*?)\1\s*\)") |
| 64 | |
| 65 | |
| 66 | def _local(tag: object) -> str: |
| 67 | return tag.rsplit("}", 1)[-1] if isinstance(tag, str) else "" |
| 68 | |
| 69 | |
| 70 | def _drawable_count(elem: ET.Element) -> int: |
| 71 | return sum(1 for e in elem.iter() if _local(e.tag) in DRAWABLE) |
| 72 | |
| 73 | |
| 74 | def _xml_size(elem: ET.Element) -> int: |
| 75 | if not any(item.get(SOURCE_REF_ATTRIBUTE) for item in elem.iter()): |
| 76 | return len(ET.tostring(elem, encoding="utf-8")) |
| 77 | measured = copy.deepcopy(elem) |
| 78 | for item in measured.iter(): |
| 79 | item.attrib.pop(SOURCE_REF_ATTRIBUTE, None) |
| 80 | return len(ET.tostring(measured, encoding="utf-8")) |
| 81 | |
| 82 | |
| 83 | def _large_enough(elem: ET.Element, min_drawables: int, min_bytes: int) -> bool: |
| 84 | return _drawable_count(elem) >= min_drawables or _xml_size(elem) >= min_bytes |
| 85 | |
| 86 | |
| 87 | def _has_semantic_content(elem: ET.Element) -> bool: |
| 88 | """Text-bearing groups must stay readable/editable in the working SVG.""" |
| 89 | return any(_local(e.tag) in SEMANTIC_CONTENT for e in elem.iter()) |
| 90 | |
| 91 | |
| 92 | def _is_existing_placeholder(elem: ET.Element) -> bool: |
| 93 | return _local(elem.tag) == "use" and elem.get("data-icon") is not None |
| 94 | |
| 95 | |
| 96 | def _has_icon_placeholder(elem: ET.Element) -> bool: |
| 97 | return any(_is_existing_placeholder(item) for item in elem.iter()) |
| 98 | |
| 99 | |
| 100 | def _is_extractable_subtree(elem: ET.Element) -> bool: |
| 101 | """Pure vector subtrees can be moved; semantic content must stay inline.""" |
| 102 | if _has_icon_placeholder(elem) or _is_chart_group(elem) or _has_semantic_content(elem): |
| 103 | return False |
| 104 | return _drawable_count(elem) > 0 |
| 105 | |
| 106 | |
| 107 | def _is_chart_group(elem: ET.Element) -> bool: |
| 108 | """Charts are handled separately (data + calibration) — never extract them.""" |
| 109 | gid = (elem.get("id") or "").lower() |
| 110 | if "chart" in gid: |
| 111 | return True |
| 112 | return any("chart" in (e.get("id") or "").lower() for e in elem.iter()) |
| 113 | |
| 114 | |
| 115 | def _tag_histogram(elem: ET.Element) -> dict[str, int]: |
| 116 | hist: dict[str, int] = {} |
| 117 | for e in elem.iter(): |
| 118 | name = _local(e.tag) |
| 119 | if name in DRAWABLE: |
| 120 | hist[name] = hist.get(name, 0) + 1 |
| 121 | return hist |
| 122 | |
| 123 | |
| 124 | def _source_references(elem: ET.Element) -> list[str]: |
| 125 | return sorted({ |
| 126 | source_ref |
| 127 | for item in elem.iter() |
| 128 | if (source_ref := item.get(SOURCE_REF_ATTRIBUTE)) |
| 129 | }) |
| 130 | |
| 131 | |
| 132 | def _is_descendant(container: ET.Element, candidate: ET.Element) -> bool: |
| 133 | return any(elem is candidate for elem in container.iter()) |
| 134 | |
| 135 | |
| 136 | def _id_index(root: ET.Element) -> dict[str, ET.Element]: |
| 137 | return {elem_id: elem for elem in root.iter() if (elem_id := elem.get("id"))} |
| 138 | |
| 139 | |
| 140 | def _referenced_ids(elem: ET.Element) -> set[str]: |
| 141 | refs: set[str] = set() |
| 142 | for item in elem.iter(): |
| 143 | for attr_name, value in item.attrib.items(): |
| 144 | refs.update(match.group(2) for match in URL_REF_RE.finditer(value)) |
| 145 | if _local(attr_name) == "href" and value.startswith("#") and len(value) > 1: |
| 146 | refs.add(value[1:]) |
| 147 | return refs |
| 148 | |
| 149 | |
| 150 | def _dependency_elements(root: ET.Element, asset_group: ET.Element) -> list[ET.Element]: |
| 151 | """ |
| 152 | Return external definition elements referenced by the extracted subtree. |
| 153 | |
| 154 | Gradients, patterns, filters, clip paths, and markers often live in the |
| 155 | source SVG root <defs>. The extracted asset must carry these dependencies |
| 156 | itself; otherwise the standalone icon and later re-inline can lose styling. |
| 157 | """ |
| 158 | by_id = _id_index(root) |
| 159 | dependencies: list[ET.Element] = [] |
| 160 | seen: set[str] = set() |
| 161 | queue = sorted(_referenced_ids(asset_group)) |
| 162 | |
| 163 | while queue: |
| 164 | ref_id = queue.pop(0) |
| 165 | if ref_id in seen: |
| 166 | continue |
| 167 | seen.add(ref_id) |
| 168 | |
| 169 | target = by_id.get(ref_id) |
| 170 | if target is None or _is_descendant(asset_group, target): |
| 171 | continue |
| 172 | |
| 173 | dependencies.append(target) |
| 174 | for nested_ref in sorted(_referenced_ids(target)): |
| 175 | if nested_ref not in seen: |
| 176 | queue.append(nested_ref) |
| 177 | |
| 178 | return dependencies |
| 179 | |
| 180 | |
| 181 | def _collect_id_mapping(asset_id: str, group: ET.Element, dependencies: list[ET.Element]) -> dict[str, str]: |
| 182 | mapping: dict[str, str] = {} |
| 183 | for root in [group, *dependencies]: |
| 184 | for elem in root.iter(): |
| 185 | elem_id = elem.get("id") |
| 186 | if elem_id and elem_id not in mapping: |
| 187 | mapping[elem_id] = f"{asset_id}_{elem_id}" |
| 188 | return mapping |
| 189 | |
| 190 | |
| 191 | def _rewrite_references(elem: ET.Element, id_mapping: dict[str, str]) -> None: |
| 192 | def rewrite_url(match: re.Match[str]) -> str: |
| 193 | quote, ref_id = match.group(1), match.group(2) |
| 194 | new_id = id_mapping.get(ref_id, ref_id) |
| 195 | return f"url({quote}#{new_id}{quote})" |
| 196 | |
| 197 | for item in elem.iter(): |
| 198 | elem_id = item.get("id") |
| 199 | if elem_id in id_mapping: |
| 200 | item.set("id", id_mapping[elem_id]) |
| 201 | |
| 202 | for attr_name, value in list(item.attrib.items()): |
| 203 | rewritten = URL_REF_RE.sub(rewrite_url, value) |
| 204 | if _local(attr_name) == "href" and value.startswith("#") and value[1:] in id_mapping: |
| 205 | rewritten = f"#{id_mapping[value[1:]]}" |
| 206 | if rewritten != value: |
| 207 | item.set(attr_name, rewritten) |
| 208 | |
| 209 | |
| 210 | def _find_extractable(root: ET.Element, min_drawables: int, min_bytes: int) -> list[ET.Element]: |
| 211 | """Outermost <g> groups whose drawable count clears the threshold (no nesting).""" |
| 212 | found: list[ET.Element] = [] |
| 213 | |
| 214 | def walk(elem: ET.Element) -> None: |
| 215 | for child in list(elem): |
| 216 | if _local(child.tag) != "g": |
| 217 | walk(child) |
| 218 | continue |
| 219 | if ( |
| 220 | not _is_chart_group(child) |
| 221 | and not _has_semantic_content(child) |
| 222 | and not _has_icon_placeholder(child) |
| 223 | and _large_enough(child, min_drawables, min_bytes) |
| 224 | ): |
| 225 | found.append(child) # outermost qualifying — do not descend |
| 226 | else: |
| 227 | walk(child) |
| 228 | |
| 229 | walk(root) |
| 230 | return found |
| 231 | |
| 232 | |
| 233 | def _find_extractable_runs( |
| 234 | root: ET.Element, |
| 235 | min_drawables: int, |
| 236 | min_bytes: int, |
| 237 | min_decoration_bytes: int, |
| 238 | ) -> list[tuple[ET.Element, list[ET.Element]]]: |
| 239 | """ |
| 240 | Consecutive pure-vector children inside mixed groups. |
| 241 | |
| 242 | PPT exports often flatten text and large vector decorations as siblings |
| 243 | under the same parent. Whole-group extraction would hide text, so only the |
| 244 | contiguous vector runs are factored out. |
| 245 | """ |
| 246 | found: list[tuple[ET.Element, list[ET.Element]]] = [] |
| 247 | |
| 248 | def flush(parent: ET.Element, run: list[ET.Element]) -> None: |
| 249 | byte_threshold = min_decoration_bytes if _has_semantic_content(parent) else min_bytes |
| 250 | if run and ( |
| 251 | sum(_drawable_count(child) for child in run) >= min_drawables |
| 252 | or sum(_xml_size(child) for child in run) >= byte_threshold |
| 253 | ): |
| 254 | found.append((parent, list(run))) |
| 255 | |
| 256 | def walk(elem: ET.Element) -> None: |
| 257 | run: list[ET.Element] = [] |
| 258 | for child in list(elem): |
| 259 | if _is_extractable_subtree(child): |
| 260 | run.append(child) |
| 261 | continue |
| 262 | flush(elem, run) |
| 263 | run = [] |
| 264 | walk(child) |
| 265 | flush(elem, run) |
| 266 | |
| 267 | walk(root) |
| 268 | return found |
| 269 | |
| 270 | |
| 271 | def _asset_svg( |
| 272 | group: ET.Element, |
| 273 | dependencies: list[ET.Element], |
| 274 | view_box: str | None, |
| 275 | width: str | None, |
| 276 | height: str | None, |
| 277 | ) -> bytes: |
| 278 | """Standalone, independently-viewable SVG carrying the group in page coords.""" |
| 279 | svg = ET.Element(f"{{{SVG_NS}}}svg") |
| 280 | svg.set("data-icon-style", "preserve-color") |
| 281 | if view_box: |
| 282 | svg.set("viewBox", view_box) |
| 283 | if width: |
| 284 | svg.set("width", width) |
| 285 | if height: |
| 286 | svg.set("height", height) |
| 287 | if dependencies: |
| 288 | defs = ET.SubElement(svg, f"{{{SVG_NS}}}defs") |
| 289 | for dependency in dependencies: |
| 290 | defs.append(dependency) |
| 291 | svg.append(group) |
| 292 | return ET.tostring(svg, encoding="utf-8", xml_declaration=True) |
| 293 | |
| 294 | |
| 295 | def _source_sha256( |
| 296 | group: ET.Element, |
| 297 | dependencies: list[ET.Element], |
| 298 | view_box: str | None, |
| 299 | width: str | None, |
| 300 | height: str | None, |
| 301 | ) -> str: |
| 302 | """Fingerprint an extracted subtree before asset-id namespacing.""" |
| 303 | payload = _asset_svg( |
| 304 | copy.deepcopy(group), |
| 305 | [copy.deepcopy(dependency) for dependency in dependencies], |
| 306 | view_box, |
| 307 | width, |
| 308 | height, |
| 309 | ) |
| 310 | return hashlib.sha256(payload).hexdigest() |
| 311 | |
| 312 | |
| 313 | def _asset_group(nodes: list[ET.Element]) -> ET.Element: |
| 314 | group = ET.Element(f"{{{SVG_NS}}}g") |
| 315 | for node in nodes: |
| 316 | group.append(node) |
| 317 | return group |
| 318 | |
| 319 | |
| 320 | def _asset_id(svg_path: Path, index: int, id_prefix: str) -> str: |
| 321 | prefix = f"{id_prefix}_" if id_prefix else "" |
| 322 | return f"{prefix}{svg_path.stem}_ill{index:02d}" |
| 323 | |
| 324 | |
| 325 | def _icon_reference(icon_namespace: str, asset_id: str) -> str: |
| 326 | return f"{icon_namespace}/{asset_id}" if icon_namespace else asset_id |
| 327 | |
| 328 | |
| 329 | def _asset_relative_path(icon_namespace: str, asset_id: str) -> str: |
| 330 | return f"{_icon_reference(icon_namespace, asset_id)}.svg" |
| 331 | |
| 332 | |
| 333 | def _icon_asset_for_namespace(icon_name: str, icon_namespace: str) -> str | None: |
| 334 | """Map one local placeholder to its asset path, excluding other libraries.""" |
| 335 | if icon_namespace: |
| 336 | prefix = f"{icon_namespace}/" |
| 337 | if not icon_name.startswith(prefix): |
| 338 | return None |
| 339 | asset_id = icon_name[len(prefix):] |
| 340 | if not asset_id or "/" in asset_id: |
| 341 | return None |
| 342 | return f"{icon_name}.svg" |
| 343 | if "/" in icon_name: |
| 344 | return None |
| 345 | return f"{icon_name}.svg" |
| 346 | |
| 347 | |
| 348 | def _has_namespace_placeholder(root: ET.Element, icon_namespace: str) -> bool: |
| 349 | if not icon_namespace: |
| 350 | return False |
| 351 | return any( |
| 352 | _local(elem.tag) == "use" |
| 353 | and (icon_name := elem.get("data-icon")) is not None |
| 354 | and _icon_asset_for_namespace(icon_name, icon_namespace) is not None |
| 355 | for elem in root.iter() |
| 356 | ) |
| 357 | |
| 358 | |
| 359 | def _generated_asset_re(svg_stems: list[str], id_prefix: str) -> re.Pattern[str] | None: |
| 360 | if not svg_stems: |
| 361 | return None |
| 362 | prefix = f"{re.escape(id_prefix)}_" if id_prefix else "" |
| 363 | stems = "|".join(re.escape(stem) for stem in svg_stems) |
| 364 | return re.compile(rf"^{prefix}(?:{stems})_ill\d+\.svg$") |
| 365 | |
| 366 | |
| 367 | def _clean_stale_assets( |
| 368 | icons_dir: Path, |
| 369 | icon_namespace: str, |
| 370 | svg_paths: list[Path], |
| 371 | id_prefix: str, |
| 372 | keep_assets: set[str], |
| 373 | ) -> list[str]: |
| 374 | pattern = _generated_asset_re([path.stem for path in svg_paths], id_prefix) |
| 375 | if pattern is None: |
| 376 | return [] |
| 377 | |
| 378 | removed: list[str] = [] |
| 379 | asset_dir = icons_dir / icon_namespace if icon_namespace else icons_dir |
| 380 | for asset_path in sorted(asset_dir.glob("*.svg")): |
| 381 | relative_asset = asset_path.relative_to(icons_dir).as_posix() |
| 382 | if relative_asset in keep_assets or not pattern.match(asset_path.name): |
| 383 | continue |
| 384 | asset_path.unlink() |
| 385 | removed.append(relative_asset) |
| 386 | return removed |
| 387 | |
| 388 | |
| 389 | def _referenced_icon_assets(svg_paths: list[Path], icon_namespace: str) -> set[str]: |
| 390 | assets: set[str] = set() |
| 391 | for svg_path in svg_paths: |
| 392 | try: |
| 393 | root = ET.parse(svg_path).getroot() |
| 394 | except ET.ParseError: |
| 395 | continue |
| 396 | for elem in root.iter(): |
| 397 | if _local(elem.tag) != "use": |
| 398 | continue |
| 399 | icon_name = elem.get("data-icon") |
| 400 | if icon_name and (asset := _icon_asset_for_namespace(icon_name, icon_namespace)): |
| 401 | assets.add(asset) |
| 402 | return assets |
| 403 | |
| 404 | |
| 405 | def _existing_placeholder_entries( |
| 406 | svg_paths: list[Path], |
| 407 | icons_dir: Path, |
| 408 | icon_namespace: str, |
| 409 | known_assets: set[str], |
| 410 | ) -> list[dict]: |
| 411 | by_asset: dict[str, dict] = {} |
| 412 | for svg_path in svg_paths: |
| 413 | try: |
| 414 | root = ET.parse(svg_path).getroot() |
| 415 | except ET.ParseError: |
| 416 | continue |
| 417 | |
| 418 | for elem in root.iter(): |
| 419 | if _local(elem.tag) != "use": |
| 420 | continue |
| 421 | icon_name = elem.get("data-icon") |
| 422 | if not icon_name: |
| 423 | continue |
| 424 | |
| 425 | asset = _icon_asset_for_namespace(icon_name, icon_namespace) |
| 426 | if asset is None: |
| 427 | continue |
| 428 | if asset in known_assets: |
| 429 | continue |
| 430 | |
| 431 | entry = by_asset.setdefault( |
| 432 | asset, |
| 433 | { |
| 434 | "svg": svg_path.name, |
| 435 | "svgs": [], |
| 436 | "id": icon_name, |
| 437 | "icon": icon_name, |
| 438 | "asset": asset, |
| 439 | "source": "existing-placeholder", |
| 440 | "asset_exists": (icons_dir / asset).exists(), |
| 441 | }, |
| 442 | ) |
| 443 | if svg_path.name not in entry["svgs"]: |
| 444 | entry["svgs"].append(svg_path.name) |
| 445 | |
| 446 | entries: list[dict] = [] |
| 447 | for asset, entry in sorted(by_asset.items()): |
| 448 | asset_path = icons_dir / asset |
| 449 | if asset_path.exists(): |
| 450 | entry["asset_sha256"] = hashlib.sha256( |
| 451 | asset_path.read_bytes() |
| 452 | ).hexdigest() |
| 453 | try: |
| 454 | root = ET.parse(asset_path).getroot() |
| 455 | except ET.ParseError: |
| 456 | pass |
| 457 | else: |
| 458 | entry["drawable_count"] = _drawable_count(root) |
| 459 | entry["byte_count"] = _xml_size(root) |
| 460 | entry["elements"] = _tag_histogram(root) |
| 461 | entry["source_refs"] = _source_references(root) |
| 462 | entry["dependencies"] = sorted( |
| 463 | elem_id |
| 464 | for elem in root.iter() |
| 465 | if _local(elem.tag) == "defs" |
| 466 | for child in elem |
| 467 | if (elem_id := child.get("id")) |
| 468 | ) |
| 469 | entries.append(entry) |
| 470 | return entries |
| 471 | |
| 472 | |
| 473 | def _load_reusable_assets(inventory_path: Path, icons_dir: Path) -> dict[str, dict]: |
| 474 | """Load fingerprinted assets from an earlier extraction inventory.""" |
| 475 | try: |
| 476 | payload = json.loads(inventory_path.read_text(encoding="utf-8")) |
| 477 | except FileNotFoundError as exc: |
| 478 | raise ValueError(f"reuse inventory not found: {inventory_path}") from exc |
| 479 | except json.JSONDecodeError as exc: |
| 480 | raise ValueError(f"invalid reuse inventory JSON: {inventory_path}: {exc}") from exc |
| 481 | |
| 482 | entries = payload.get("assets") |
| 483 | if not isinstance(entries, list): |
| 484 | raise ValueError(f"reuse inventory has no assets list: {inventory_path}") |
| 485 | |
| 486 | reusable: dict[str, dict] = {} |
| 487 | fingerprinted = 0 |
| 488 | for entry in entries: |
| 489 | if not isinstance(entry, dict): |
| 490 | continue |
| 491 | source_sha256 = entry.get("source_sha256") |
| 492 | asset_sha256 = entry.get("asset_sha256") |
| 493 | asset = entry.get("asset") |
| 494 | icon = entry.get("icon") |
| 495 | if not all( |
| 496 | isinstance(value, str) and value |
| 497 | for value in (source_sha256, asset_sha256, asset, icon) |
| 498 | ): |
| 499 | continue |
| 500 | fingerprinted += 1 |
| 501 | asset_path = icons_dir / asset |
| 502 | if not asset_path.is_file(): |
| 503 | raise ValueError( |
| 504 | f"reusable asset is missing from the target icons directory: {asset_path}" |
| 505 | ) |
| 506 | actual_asset_sha256 = hashlib.sha256(asset_path.read_bytes()).hexdigest() |
| 507 | if actual_asset_sha256 != asset_sha256: |
| 508 | raise ValueError( |
| 509 | f"reusable asset hash does not match its inventory: {asset_path}" |
| 510 | ) |
| 511 | current = reusable.get(source_sha256) |
| 512 | if current is None or asset < str(current["asset"]): |
| 513 | reusable[source_sha256] = entry |
| 514 | |
| 515 | extracted_count = payload.get("extracted_count", 0) |
| 516 | if isinstance(extracted_count, int) and extracted_count > fingerprinted: |
| 517 | raise ValueError( |
| 518 | "reuse inventory predates source fingerprints; rerun the source extraction " |
| 519 | f"with the current tool: {inventory_path}" |
| 520 | ) |
| 521 | return reusable |
| 522 | |
| 523 | |
| 524 | def _rewritten_path(svg_path: Path, rewritten_dir: Path | None, inplace: bool) -> Path: |
| 525 | if inplace: |
| 526 | return svg_path |
| 527 | if rewritten_dir is None: |
| 528 | return svg_path.parent.parent / f"{svg_path.parent.name}-rewritten" / svg_path.name |
| 529 | return rewritten_dir / svg_path.name |
| 530 | |
| 531 | |
| 532 | def extract_file( |
| 533 | svg_path: Path, |
| 534 | icons_dir: Path, |
| 535 | icon_namespace: str, |
| 536 | min_drawables: int, |
| 537 | min_bytes: int, |
| 538 | min_decoration_bytes: int, |
| 539 | inplace: bool, |
| 540 | id_prefix: str = "", |
| 541 | rewritten_dir: Path | None = None, |
| 542 | reusable_assets: dict[str, dict] | None = None, |
| 543 | ) -> list[dict]: |
| 544 | """Extract qualifying groups from one SVG. Returns inventory entries.""" |
| 545 | ET.register_namespace("", SVG_NS) |
| 546 | tree = ET.parse(svg_path) |
| 547 | root = tree.getroot() |
| 548 | view_box = root.get("viewBox") |
| 549 | width = root.get("width") |
| 550 | height = root.get("height") |
| 551 | |
| 552 | # A namespaced projection is an all-at-once readability pass. Once it owns |
| 553 | # an asset reference, reruns inventory the existing placeholders instead of |
| 554 | # progressively factoring their remaining parent/sibling geometry. |
| 555 | if _has_namespace_placeholder(root, icon_namespace): |
| 556 | if not inplace: |
| 557 | rewritten = _rewritten_path(svg_path, rewritten_dir, inplace) |
| 558 | rewritten.parent.mkdir(parents=True, exist_ok=True) |
| 559 | tree.write(rewritten, encoding="utf-8", xml_declaration=True) |
| 560 | return [] |
| 561 | |
| 562 | targets: list[tuple[ET.Element, list[ET.Element]]] = [] |
| 563 | parents = {child: parent for parent in root.iter() for child in parent} |
| 564 | group_targets = _find_extractable(root, min_drawables, min_bytes) |
| 565 | selected_groups = set(group_targets) |
| 566 | |
| 567 | def inside_selected_group(elem: ET.Element) -> bool: |
| 568 | current = elem |
| 569 | while current in parents: |
| 570 | current = parents[current] |
| 571 | if current in selected_groups: |
| 572 | return True |
| 573 | return False |
| 574 | |
| 575 | for group in group_targets: |
| 576 | parent = parents.get(group) |
| 577 | if parent is not None: |
| 578 | targets.append((parent, [group])) |
| 579 | |
| 580 | for parent, run in _find_extractable_runs(root, min_drawables, min_bytes, min_decoration_bytes): |
| 581 | if ( |
| 582 | parent not in selected_groups |
| 583 | and not inside_selected_group(parent) |
| 584 | and not any(child in selected_groups for child in run) |
| 585 | and all(child in parent for child in run) |
| 586 | ): |
| 587 | targets.append((parent, run)) |
| 588 | |
| 589 | if not targets: |
| 590 | if not inplace: |
| 591 | rewritten = _rewritten_path(svg_path, rewritten_dir, inplace) |
| 592 | rewritten.parent.mkdir(parents=True, exist_ok=True) |
| 593 | tree.write(rewritten, encoding="utf-8", xml_declaration=True) |
| 594 | return [] |
| 595 | |
| 596 | asset_dir = icons_dir / icon_namespace if icon_namespace else icons_dir |
| 597 | asset_dir.mkdir(parents=True, exist_ok=True) |
| 598 | entries = [] |
| 599 | |
| 600 | for index, (parent, nodes) in enumerate(targets, start=1): |
| 601 | if not nodes or not all(node in parent for node in nodes): |
| 602 | continue |
| 603 | asset_id = _asset_id(svg_path, index, id_prefix) |
| 604 | icon_reference = _icon_reference(icon_namespace, asset_id) |
| 605 | asset = _asset_relative_path(icon_namespace, asset_id) |
| 606 | pos = list(parent).index(nodes[0]) |
| 607 | group = nodes[0] if len(nodes) == 1 and _local(nodes[0].tag) == "g" else _asset_group(nodes) |
| 608 | |
| 609 | dependencies = [copy.deepcopy(elem) for elem in _dependency_elements(root, group)] |
| 610 | dependency_source_ids = sorted({ |
| 611 | elem_id |
| 612 | for dependency in dependencies |
| 613 | for elem in dependency.iter() |
| 614 | if (elem_id := elem.get("id")) |
| 615 | }) |
| 616 | source_sha256 = _source_sha256(group, dependencies, view_box, width, height) |
| 617 | source_refs = _source_references(group) |
| 618 | reusable = (reusable_assets or {}).get(source_sha256) |
| 619 | if reusable is not None: |
| 620 | reused_icon = str(reusable["icon"]) |
| 621 | reused_asset = str(reusable["asset"]) |
| 622 | placeholder = ET.Element(f"{{{SVG_NS}}}use") |
| 623 | placeholder.set("data-icon", reused_icon) |
| 624 | for node in nodes: |
| 625 | if node in parent: |
| 626 | parent.remove(node) |
| 627 | parent.insert(pos, placeholder) |
| 628 | entries.append({ |
| 629 | "svg": svg_path.name, |
| 630 | "id": reused_icon, |
| 631 | "icon": reused_icon, |
| 632 | "asset": reused_asset, |
| 633 | "source": "reused-inventory", |
| 634 | "source_sha256": source_sha256, |
| 635 | "asset_sha256": reusable["asset_sha256"], |
| 636 | "reused_from_svg": reusable.get("svg"), |
| 637 | "drawable_count": _drawable_count(group), |
| 638 | "byte_count": _xml_size(group), |
| 639 | "source_refs": source_refs, |
| 640 | "dependencies": dependency_source_ids, |
| 641 | "elements": _tag_histogram(group), |
| 642 | }) |
| 643 | continue |
| 644 | |
| 645 | id_mapping = _collect_id_mapping(asset_id, group, dependencies) |
| 646 | _rewrite_references(group, id_mapping) |
| 647 | for dependency in dependencies: |
| 648 | _rewrite_references(dependency, id_mapping) |
| 649 | |
| 650 | # Asset keeps the group in original page coordinates and carries its defs. |
| 651 | asset_bytes = _asset_svg(group, dependencies, view_box, width, height) |
| 652 | (icons_dir / asset).write_bytes(asset_bytes) |
| 653 | |
| 654 | placeholder = ET.Element(f"{{{SVG_NS}}}use") |
| 655 | placeholder.set("data-icon", icon_reference) |
| 656 | for node in nodes: |
| 657 | if node in parent: |
| 658 | parent.remove(node) |
| 659 | parent.insert(pos, placeholder) |
| 660 | |
| 661 | entries.append({ |
| 662 | "svg": svg_path.name, |
| 663 | "id": icon_reference, |
| 664 | "icon": icon_reference, |
| 665 | "asset": asset, |
| 666 | "source": "extracted", |
| 667 | "source_sha256": source_sha256, |
| 668 | "asset_sha256": hashlib.sha256(asset_bytes).hexdigest(), |
| 669 | "drawable_count": _drawable_count(group), |
| 670 | "byte_count": _xml_size(group), |
| 671 | "source_refs": source_refs, |
| 672 | "dependencies": [id_mapping.get(elem_id, elem_id) for elem_id in dependency_source_ids], |
| 673 | "elements": _tag_histogram(group), |
| 674 | }) |
| 675 | |
| 676 | rewritten = _rewritten_path(svg_path, rewritten_dir, inplace) |
| 677 | rewritten.parent.mkdir(parents=True, exist_ok=True) |
| 678 | tree.write(rewritten, encoding="utf-8", xml_declaration=True) |
| 679 | return entries |
| 680 | |
| 681 | |
| 682 | def build_parser() -> argparse.ArgumentParser: |
| 683 | parser = argparse.ArgumentParser( |
| 684 | description="Factor large inline vector groups out of SVGs into reusable assets.", |
| 685 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 686 | ) |
| 687 | parser.add_argument("svg_dir", help="Directory of working SVGs (e.g. import_ws/svg or project/svg_output)") |
| 688 | parser.add_argument("-o", "--output", dest="icons_dir", help="Project icon dir (default: <svg_dir>/../icons)") |
| 689 | parser.add_argument("--icons-dir", dest="icons_dir", help="Project icon dir (default: <svg_dir>/../icons)") |
| 690 | parser.add_argument( |
| 691 | "--icon-namespace", |
| 692 | default="", |
| 693 | help=( |
| 694 | "Optional lower-case subdirectory and data-icon prefix for extracted " |
| 695 | "assets (create-template uses: imported)" |
| 696 | ), |
| 697 | ) |
| 698 | parser.add_argument( |
| 699 | "--rewritten-dir", |
| 700 | help="Directory for rewritten SVGs when not using --inplace (default: <svg_dir>/../<svg_dir-name>-rewritten)", |
| 701 | ) |
| 702 | parser.add_argument( |
| 703 | "--inventory", |
| 704 | help="Inventory JSON path (default: <svg_dir>/../<svg_dir-name>_vector_asset_inventory.json)", |
| 705 | ) |
| 706 | parser.add_argument( |
| 707 | "--reuse-inventory", |
| 708 | help=( |
| 709 | "Reuse fingerprint-matched assets from an earlier extraction inventory; " |
| 710 | "only unmatched vector subtrees create new assets" |
| 711 | ), |
| 712 | ) |
| 713 | parser.add_argument( |
| 714 | "--id-prefix", |
| 715 | default="", |
| 716 | help=( |
| 717 | "Optional prefix for generated asset IDs, useful when processing " |
| 718 | "layered and flat SVG dirs into one icons dir" |
| 719 | ), |
| 720 | ) |
| 721 | parser.add_argument( |
| 722 | "--min-drawables", type=int, default=DEFAULT_MIN_DRAWABLES, |
| 723 | help=f"Min drawable elements for a group to be extracted (default: {DEFAULT_MIN_DRAWABLES})", |
| 724 | ) |
| 725 | parser.add_argument( |
| 726 | "--min-bytes", type=int, default=DEFAULT_MIN_BYTES, |
| 727 | help=f"Min XML bytes for a pure-vector group/run to be extracted (default: {DEFAULT_MIN_BYTES})", |
| 728 | ) |
| 729 | parser.add_argument( |
| 730 | "--min-decoration-bytes", type=int, default=DEFAULT_MIN_DECORATION_BYTES, |
| 731 | help=( |
| 732 | "Min XML bytes for pure-vector decoration runs inside text-bearing groups " |
| 733 | f"(default: {DEFAULT_MIN_DECORATION_BYTES})" |
| 734 | ), |
| 735 | ) |
| 736 | parser.add_argument( |
| 737 | "--inplace", action="store_true", |
| 738 | help="Rewrite the source SVGs in place instead of writing to --rewritten-dir", |
| 739 | ) |
| 740 | parser.add_argument( |
| 741 | "--clean-stale", |
| 742 | action="store_true", |
| 743 | help=( |
| 744 | "Remove stale generated assets for the current svg filenames/id prefix " |
| 745 | "that are not referenced by this run's inventory" |
| 746 | ), |
| 747 | ) |
| 748 | return parser |
| 749 | |
| 750 | |
| 751 | def main(argv: Optional[list[str]] = None) -> int: |
| 752 | args = build_parser().parse_args(argv) |
| 753 | svg_dir = Path(args.svg_dir) |
| 754 | if not svg_dir.is_dir(): |
| 755 | print(f"[ERROR] svg_dir not found: {svg_dir}", file=sys.stderr) |
| 756 | return 1 |
| 757 | |
| 758 | icons_dir = Path(args.icons_dir) if args.icons_dir else svg_dir.parent / "icons" |
| 759 | icons_dir.mkdir(parents=True, exist_ok=True) |
| 760 | icon_namespace = args.icon_namespace.strip() |
| 761 | if icon_namespace and not ICON_NAMESPACE_RE.fullmatch(icon_namespace): |
| 762 | print( |
| 763 | "[ERROR] --icon-namespace must be one lower-case ASCII directory name " |
| 764 | "using only letters, digits, '_' or '-'", |
| 765 | file=sys.stderr, |
| 766 | ) |
| 767 | return 1 |
| 768 | reusable_assets: dict[str, dict] = {} |
| 769 | reuse_inventory_path = Path(args.reuse_inventory) if args.reuse_inventory else None |
| 770 | if reuse_inventory_path is not None: |
| 771 | try: |
| 772 | reusable_assets = _load_reusable_assets(reuse_inventory_path, icons_dir) |
| 773 | except ValueError as exc: |
| 774 | print(f"[ERROR] {exc}", file=sys.stderr) |
| 775 | return 1 |
| 776 | rewritten_dir = Path(args.rewritten_dir) if args.rewritten_dir else None |
| 777 | inventory_path = ( |
| 778 | Path(args.inventory) |
| 779 | if args.inventory |
| 780 | else svg_dir.parent / f"{svg_dir.name}_vector_asset_inventory.json" |
| 781 | ) |
| 782 | svg_paths = sorted(svg_dir.glob("*.svg")) |
| 783 | |
| 784 | inventory: list[dict] = [] |
| 785 | for svg_path in svg_paths: |
| 786 | try: |
| 787 | inventory.extend( |
| 788 | extract_file( |
| 789 | svg_path, |
| 790 | icons_dir, |
| 791 | icon_namespace, |
| 792 | args.min_drawables, |
| 793 | args.min_bytes, |
| 794 | args.min_decoration_bytes, |
| 795 | args.inplace, |
| 796 | args.id_prefix, |
| 797 | rewritten_dir, |
| 798 | reusable_assets, |
| 799 | ) |
| 800 | ) |
| 801 | except ET.ParseError as exc: |
| 802 | print(f"[WARN] skip unparseable {svg_path.name}: {exc}", file=sys.stderr) |
| 803 | |
| 804 | extracted_count = sum(entry.get("source") == "extracted" for entry in inventory) |
| 805 | reused_count = sum(entry.get("source") == "reused-inventory" for entry in inventory) |
| 806 | known_assets = {str(entry["asset"]) for entry in inventory} |
| 807 | inventory.extend( |
| 808 | _existing_placeholder_entries( |
| 809 | svg_paths, |
| 810 | icons_dir, |
| 811 | icon_namespace, |
| 812 | known_assets, |
| 813 | ) |
| 814 | ) |
| 815 | |
| 816 | stale_removed: list[str] = [] |
| 817 | if args.clean_stale: |
| 818 | keep_assets = {str(entry["asset"]) for entry in inventory} |
| 819 | keep_assets.update(_referenced_icon_assets(svg_paths, icon_namespace)) |
| 820 | stale_removed = _clean_stale_assets( |
| 821 | icons_dir, |
| 822 | icon_namespace, |
| 823 | svg_paths, |
| 824 | args.id_prefix, |
| 825 | keep_assets, |
| 826 | ) |
| 827 | |
| 828 | manifest = { |
| 829 | "schema": "vector_asset_inventory.v1", |
| 830 | "svg_dir": str(svg_dir), |
| 831 | "icons_dir": str(icons_dir), |
| 832 | "icon_namespace": icon_namespace or None, |
| 833 | "rewritten_dir": ( |
| 834 | None |
| 835 | if args.inplace |
| 836 | else str(_rewritten_path(svg_dir / "_sample.svg", rewritten_dir, False).parent) |
| 837 | ), |
| 838 | "reuse_inventory": str(reuse_inventory_path) if reuse_inventory_path is not None else None, |
| 839 | "min_drawables": args.min_drawables, |
| 840 | "min_bytes": args.min_bytes, |
| 841 | "min_decoration_bytes": args.min_decoration_bytes, |
| 842 | "extracted_count": extracted_count, |
| 843 | "reused_count": reused_count, |
| 844 | "asset_count": len(inventory), |
| 845 | "stale_removed": stale_removed, |
| 846 | "assets": inventory, |
| 847 | } |
| 848 | inventory_path.parent.mkdir(parents=True, exist_ok=True) |
| 849 | inventory_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") |
| 850 | summary_path: Path | None = None |
| 851 | if args.inplace and (svg_dir / AUTHORING_MANIFEST_NAME).is_file(): |
| 852 | try: |
| 853 | summary_path = write_authoring_summary(svg_dir) |
| 854 | except (OSError, ValueError) as exc: |
| 855 | print( |
| 856 | f"[ERROR] vector extraction succeeded but authoring summary " |
| 857 | f"refresh failed: {exc}", |
| 858 | file=sys.stderr, |
| 859 | ) |
| 860 | return 1 |
| 861 | print( |
| 862 | f"[OK] extracted {extracted_count} new asset(s), reused {reused_count} asset(s), " |
| 863 | f"inventoried {len(inventory)} asset reference(s) -> " |
| 864 | f"{icons_dir / icon_namespace if icon_namespace else icons_dir}", |
| 865 | file=sys.stderr, |
| 866 | ) |
| 867 | if stale_removed: |
| 868 | print(f"[OK] removed {len(stale_removed)} stale generated asset(s)", file=sys.stderr) |
| 869 | if summary_path is not None: |
| 870 | print(f"[OK] refreshed model-readable summary: {summary_path}", file=sys.stderr) |
| 871 | return 0 |
| 872 | |
| 873 | |
| 874 | if __name__ == "__main__": |
| 875 | raise SystemExit(main()) |
| 876 |