| 1 | """Wikimedia Commons provider. |
| 2 | |
| 3 | Zero-config (no API key required). Strong on educational, scientific, |
| 4 | geographic, and historical imagery; weaker on contemporary stock-style |
| 5 | photography and people. |
| 6 | |
| 7 | Uses the MediaWiki API's ``generator=search`` mode to combine fulltext |
| 8 | search with imageinfo/extmetadata in a single round trip. |
| 9 | |
| 10 | API docs: https://www.mediawiki.org/wiki/API:Search |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import sys |
| 16 | from pathlib import Path |
| 17 | |
| 18 | _SCRIPTS_DIR = Path(__file__).resolve().parents[1] |
| 19 | if str(_SCRIPTS_DIR) not in sys.path: |
| 20 | sys.path.insert(0, str(_SCRIPTS_DIR)) |
| 21 | |
| 22 | from console_encoding import configure_utf8_stdio # noqa: E402 |
| 23 | |
| 24 | configure_utf8_stdio() |
| 25 | |
| 26 | if __name__ == "__main__": |
| 27 | print(__doc__) |
| 28 | print("Use via: python3 skills/ppt-master/scripts/image_search.py ...") |
| 29 | raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1) |
| 30 | |
| 31 | import html |
| 32 | import re |
| 33 | |
| 34 | import requests |
| 35 | |
| 36 | from image_sources.provider_common import ( |
| 37 | AssetCandidate, |
| 38 | ImageSearchRequest, |
| 39 | USER_AGENT, |
| 40 | build_query_progression, |
| 41 | classify_license, |
| 42 | normalize_license_name, |
| 43 | normalize_orientation, |
| 44 | ) |
| 45 | |
| 46 | |
| 47 | API_URL = "https://commons.wikimedia.org/w/api.php" |
| 48 | DEFAULT_SEARCH_LIMIT = 20 |
| 49 | DEFAULT_TIMEOUT = 30 |
| 50 | |
| 51 | # File extensions we are willing to embed in a deck. SVG/GIF/audio etc. are |
| 52 | # excluded — Wikimedia returns these freely from a generic search. |
| 53 | _ACCEPTED_EXTENSIONS = frozenset({".jpg", ".jpeg", ".png", ".webp", ".tiff", ".tif"}) |
| 54 | |
| 55 | _TAG_RE = re.compile(r"<[^>]+>") |
| 56 | _WS_RE = re.compile(r"\s+") |
| 57 | |
| 58 | |
| 59 | def _strip_html(value: str) -> str: |
| 60 | """Wikimedia's extmetadata fields contain inline HTML markup. Flatten it.""" |
| 61 | if not value: |
| 62 | return "" |
| 63 | text = html.unescape(str(value)) |
| 64 | text = _TAG_RE.sub(" ", text) |
| 65 | text = _WS_RE.sub(" ", text) |
| 66 | return text.strip() |
| 67 | |
| 68 | |
| 69 | def _ext_value(extmetadata: dict, key: str) -> str: |
| 70 | """Pull ``extmetadata[key].value`` and strip HTML.""" |
| 71 | entry = extmetadata.get(key) or {} |
| 72 | if isinstance(entry, dict): |
| 73 | return _strip_html(entry.get("value", "")) |
| 74 | return _strip_html(entry) |
| 75 | |
| 76 | |
| 77 | def _accept_extension(title: str) -> bool: |
| 78 | """Drop non-image files (svg/gif/audio/video) by extension.""" |
| 79 | lower = (title or "").lower() |
| 80 | return any(lower.endswith(ext) for ext in _ACCEPTED_EXTENSIONS) |
| 81 | |
| 82 | |
| 83 | def _page_label(title: str) -> str: |
| 84 | """``File:Some_image.jpg`` → ``Some_image.jpg``.""" |
| 85 | clean = _strip_html(title) |
| 86 | if clean.lower().startswith("file:"): |
| 87 | return clean.split(":", 1)[1].strip() |
| 88 | return clean |
| 89 | |
| 90 | |
| 91 | def parse_results(payload: dict) -> list[AssetCandidate]: |
| 92 | """Translate a MediaWiki ``query`` payload into a list of candidates.""" |
| 93 | candidates: list[AssetCandidate] = [] |
| 94 | pages = ((payload.get("query") or {}).get("pages") or {}) |
| 95 | |
| 96 | for page in pages.values(): |
| 97 | title = page.get("title") or "" |
| 98 | if not _accept_extension(title): |
| 99 | continue |
| 100 | |
| 101 | info_list = page.get("imageinfo") or [] |
| 102 | if not info_list: |
| 103 | continue |
| 104 | info = info_list[0] |
| 105 | extmetadata = info.get("extmetadata") or {} |
| 106 | |
| 107 | license_name = ( |
| 108 | _ext_value(extmetadata, "LicenseShortName") |
| 109 | or _ext_value(extmetadata, "License") |
| 110 | ) |
| 111 | license_url = _ext_value(extmetadata, "LicenseUrl") |
| 112 | tier = classify_license(license_name, license_url, provider="wikimedia") |
| 113 | if not tier: |
| 114 | continue |
| 115 | |
| 116 | download_url = (info.get("url") or "").strip() |
| 117 | if not download_url: |
| 118 | continue |
| 119 | |
| 120 | candidates.append( |
| 121 | AssetCandidate( |
| 122 | provider="wikimedia", |
| 123 | title=_page_label(title) or "Untitled", |
| 124 | asset_id=str(page.get("pageid") or ""), |
| 125 | source_page_url=(info.get("descriptionurl") or "").strip(), |
| 126 | license_name=normalize_license_name(license_name), |
| 127 | license_url=license_url, |
| 128 | license_tier=tier, |
| 129 | width=int(info.get("width") or 0), |
| 130 | height=int(info.get("height") or 0), |
| 131 | download_url=download_url, |
| 132 | author=_ext_value(extmetadata, "Artist"), |
| 133 | raw=page, |
| 134 | ) |
| 135 | ) |
| 136 | |
| 137 | return candidates |
| 138 | |
| 139 | |
| 140 | def _filter_by_orientation( |
| 141 | candidates: list[AssetCandidate], orientation: str |
| 142 | ) -> list[AssetCandidate]: |
| 143 | """Wikimedia API has no orientation parameter; filter client-side.""" |
| 144 | if not orientation or orientation == "any": |
| 145 | return candidates |
| 146 | matching = [ |
| 147 | c for c in candidates if normalize_orientation(c.width, c.height) == orientation |
| 148 | ] |
| 149 | # Fall back to the unfiltered list if orientation pruning leaves nothing — |
| 150 | # better an off-orientation match than no image at all. |
| 151 | return matching or candidates |
| 152 | |
| 153 | |
| 154 | def search( |
| 155 | request: ImageSearchRequest, |
| 156 | *, |
| 157 | license_tier_filter: str = "no-attribution-only", |
| 158 | search_limit: int = DEFAULT_SEARCH_LIMIT, |
| 159 | timeout: int = DEFAULT_TIMEOUT, |
| 160 | ) -> list[AssetCandidate]: |
| 161 | """Search Wikimedia Commons for candidates. |
| 162 | |
| 163 | Wikimedia returns license info via ``extmetadata`` rather than as a |
| 164 | request parameter, so the ``license_tier_filter`` does its work in |
| 165 | ``parse_results`` (and is honored implicitly because tier classification |
| 166 | happens there). Returning candidates from the first non-empty query. |
| 167 | """ |
| 168 | if license_tier_filter not in {"no-attribution-only", "all"}: |
| 169 | raise ValueError(f"unsupported license_tier_filter: {license_tier_filter!r}") |
| 170 | |
| 171 | orientation = (request.orientation or "").strip().lower() |
| 172 | |
| 173 | for query in build_query_progression(request.query): |
| 174 | params = { |
| 175 | "action": "query", |
| 176 | "format": "json", |
| 177 | "generator": "search", |
| 178 | "gsrnamespace": "6", # File: namespace |
| 179 | "gsrsearch": f"{query} filetype:bitmap", |
| 180 | "gsrlimit": search_limit, |
| 181 | "prop": "imageinfo", |
| 182 | "iiprop": "url|size|extmetadata|mime", |
| 183 | "iiextmetadatafilter": ( |
| 184 | "LicenseShortName|License|LicenseUrl|Artist" |
| 185 | ), |
| 186 | } |
| 187 | |
| 188 | response = requests.get( |
| 189 | API_URL, |
| 190 | params=params, |
| 191 | headers={"User-Agent": USER_AGENT, "Accept": "application/json"}, |
| 192 | timeout=timeout, |
| 193 | ) |
| 194 | response.raise_for_status() |
| 195 | all_candidates = parse_results(response.json()) |
| 196 | |
| 197 | if license_tier_filter == "no-attribution-only": |
| 198 | all_candidates = [ |
| 199 | c for c in all_candidates if c.license_tier == "no-attribution" |
| 200 | ] |
| 201 | |
| 202 | all_candidates = _filter_by_orientation(all_candidates, orientation) |
| 203 | if all_candidates: |
| 204 | return all_candidates |
| 205 | |
| 206 | return [] |
| 207 |