返回 ppt-master
provider_common.py
根目录 / skills / ppt-master / scripts / image_sources / provider_common.py
1 """Shared primitives for web image providers.
2
3 This module is the single home for everything that all four providers
4 (Openverse / Wikimedia / Pexels / Pixabay) need:
5
6 - License tier classification (the central abstraction of this module)
7 - Search request / asset candidate dataclasses
8 - Query simplification for keyword-based image APIs
9 - Candidate scoring
10 - Attribution text builder
11 - Small helpers (orientation, json path, etc.)
12
13 Provider-specific code (API URLs, payload shape, parse_results) lives in
14 the corresponding provider_<name>.py module and only imports from here.
15 """
16
17 from __future__ import annotations
18
19 import sys
20 from pathlib import Path
21
22 _SCRIPTS_DIR = Path(__file__).resolve().parents[1]
23 if str(_SCRIPTS_DIR) not in sys.path:
24 sys.path.insert(0, str(_SCRIPTS_DIR))
25
26 from console_encoding import configure_utf8_stdio # noqa: E402
27
28 configure_utf8_stdio()
29
30 if __name__ == "__main__":
31 print(__doc__)
32 print("This is an internal helper module used by image_search.py and the four web image providers.")
33 raise SystemExit(0 if any(arg in {"-h", "--help", "help"} for arg in sys.argv[1:]) else 1)
34
35 import re
36 from dataclasses import dataclass, field
37 from typing import Any, Optional
38
39
40 # ---------------------------------------------------------------------------
41 # Project-wide constants
42 # ---------------------------------------------------------------------------
43
44 USER_AGENT = "PPTMaster/1.0 (https://github.com/hugohe3/ppt-master)"
45
46
47 # ---------------------------------------------------------------------------
48 # License tier classification
49 # ---------------------------------------------------------------------------
50 #
51 # Every accepted candidate is classified into exactly one of two tiers:
52 #
53 # "no-attribution" -> No on-slide credit needed (CC0, PD, Pexels,
54 # Pixabay). Default search target.
55 # "attribution-required" -> CC BY / CC BY-SA. Executor must add an
56 # inline credit text element on the slide.
57 #
58 # Anything else (CC BY-NC, CC BY-ND, all-rights-reserved, unknown) returns
59 # None and the candidate is rejected outright.
60
61 LICENSE_TIER_NO_ATTRIBUTION = "no-attribution"
62 LICENSE_TIER_ATTRIBUTION_REQUIRED = "attribution-required"
63
64 # Tokens that mark a license as "no attribution required".
65 NO_ATTRIBUTION_TOKENS: tuple[str, ...] = (
66 "cc0",
67 "public domain",
68 "publicdomain",
69 "creativecommons.org/publicdomain/",
70 "pexels license",
71 "pixabay content license",
72 "pixabay license",
73 )
74
75 # Tokens that mark a license as "attribution required".
76 ATTRIBUTION_REQUIRED_TOKENS: tuple[str, ...] = (
77 "cc by",
78 "cc-by",
79 "by-sa",
80 "by sa",
81 "creativecommons.org/licenses/by/",
82 "creativecommons.org/licenses/by-sa/",
83 )
84
85 # Tokens that disqualify a candidate entirely.
86 REJECTED_TOKENS: tuple[str, ...] = (
87 "by-nc",
88 "by nc",
89 "noncommercial",
90 "non-commercial",
91 "by-nd",
92 "by nd",
93 "no derivatives",
94 "noderivatives",
95 "all rights reserved",
96 )
97
98
99 # Canonical display forms for license names. Different providers report
100 # the same license with different capitalization (Openverse: "cc0",
101 # Wikimedia: "Public domain"); the Executor renders these as on-slide
102 # text, so a normalized form prevents inconsistent credits.
103 _LICENSE_NAME_CANON: dict[str, str] = {
104 "cc0": "CC0",
105 "cc 0": "CC0",
106 "public domain": "Public Domain",
107 "publicdomain": "Public Domain",
108 "pdm": "Public Domain",
109 "pexels license": "Pexels License",
110 "pixabay content license": "Pixabay Content License",
111 "pixabay license": "Pixabay Content License",
112 }
113
114 # CC license short-name pattern used to canonicalize "cc by 4.0" → "CC BY 4.0".
115 _CC_PATTERN = re.compile(
116 r"^\s*cc[\s-]+(by(?:[\s-]+(?:sa|nc|nd))*)\s*([0-9.]*)\s*$",
117 re.IGNORECASE,
118 )
119
120
121 def normalize_license_name(name: str) -> str:
122 """Return a canonical display form for a license name.
123
124 Maps common aliases to a consistent capitalization so the on-slide
125 credit text written by the Executor is uniform across providers.
126 Unknown inputs are returned trimmed but otherwise unchanged.
127 """
128 if not name:
129 return ""
130 key = name.strip().lower()
131 if not key:
132 return ""
133
134 if key in _LICENSE_NAME_CANON:
135 return _LICENSE_NAME_CANON[key]
136
137 cc_match = _CC_PATTERN.match(key)
138 if cc_match:
139 suffix_raw, version = cc_match.group(1), cc_match.group(2)
140 suffix = suffix_raw.replace(" ", "-").upper()
141 return f"CC {suffix} {version}".strip()
142
143 return name.strip()
144
145
146 def classify_license(
147 license_name: str,
148 license_url: str = "",
149 provider: str = "",
150 ) -> Optional[str]:
151 """Classify a license string into one of the two tiers, or reject it.
152
153 Returns:
154 ``"no-attribution"`` / ``"attribution-required"`` / ``None``.
155
156 The provider hint lets us treat Pexels and Pixabay's own licenses as
157 ``no-attribution`` even when the upstream API only returns a short
158 label like ``"Pexels"``.
159 """
160 text = " ".join(
161 part.strip().lower()
162 for part in (license_name or "", license_url or "")
163 if part
164 )
165 provider_key = (provider or "").strip().lower()
166
167 if not text and not provider_key:
168 return None
169
170 if any(token in text for token in REJECTED_TOKENS):
171 return None
172
173 if any(token in text for token in NO_ATTRIBUTION_TOKENS):
174 return LICENSE_TIER_NO_ATTRIBUTION
175
176 # Provider-default fallback: pexels / pixabay items often arrive with a
177 # bare "Pexels" / "Pixabay" license string. Their site-wide license is
178 # "free for commercial use, no attribution required".
179 #
180 # Guard: require the license text to actually mention the provider name,
181 # so an empty / missing license field never silently passes as no-attribution.
182 if (
183 provider_key in {"pexels", "pixabay"}
184 and provider_key in text
185 and not any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS)
186 ):
187 return LICENSE_TIER_NO_ATTRIBUTION
188
189 if any(token in text for token in ATTRIBUTION_REQUIRED_TOKENS):
190 return LICENSE_TIER_ATTRIBUTION_REQUIRED
191
192 return None # unknown license -> reject
193
194
195 # ---------------------------------------------------------------------------
196 # Dataclasses
197 # ---------------------------------------------------------------------------
198
199
200 @dataclass
201 class ImageSearchRequest:
202 """A single image search intent passed to a provider."""
203
204 query: str
205 purpose: str = ""
206 orientation: str = "" # "landscape" / "portrait" / "square" / ""
207 min_width: int = 0
208 min_height: int = 0
209 filename: str = ""
210 slide: str = ""
211 required_terms: tuple[str, ...] = ()
212
213
214 @dataclass
215 class AssetCandidate:
216 """One ranked candidate returned by a provider's parse_results."""
217
218 provider: str
219 title: str
220 asset_id: str = ""
221 source_page_url: str = ""
222 license_name: str = ""
223 license_url: str = ""
224 license_tier: str = "" # one of LICENSE_TIER_* constants
225 width: int = 0
226 height: int = 0
227 download_url: str = ""
228 author: str = ""
229 raw: Any = field(default=None)
230
231
232 # ---------------------------------------------------------------------------
233 # Query simplification
234 # ---------------------------------------------------------------------------
235 #
236 # Web image APIs do keyword matching against image metadata, not semantic
237 # search. Long, descriptive queries with brand names, HEX codes, and
238 # composition notes return zero results. We progressively trim the query
239 # down to the most concrete nouns.
240
241 _NOISE_WORDS = frozenset({
242 # Brand / product names
243 "claude", "openai", "gpt", "gemini", "copilot", "chatgpt", "midjourney",
244 "stable", "diffusion", "dall-e", "cursor", "anthropic", "microsoft",
245 "google", "apple", "meta", "nvidia", "tesla",
246 # Generic filler
247 "using", "with", "from", "that", "this", "have", "been", "will",
248 "into", "more", "also", "very", "some", "than", "them", "other",
249 })
250
251 # Words that look generic but are actually useful when they ARE the
252 # subject of the deck (e.g. a deck about AI). We only drop them when
253 # there are still other concrete nouns left.
254 _SOFT_NOISE_WORDS = frozenset({
255 "ai", "code", "software", "system", "digital", "platform", "solution",
256 "application", "interface", "framework", "algorithm", "api", "sdk",
257 "assistant", "tool", "service", "technology", "tech", "program",
258 # Visual-quality / usage terms. These are helpful in the full provider
259 # query, but should not consume the 3-4 keyword fallback budget or
260 # dominate relevance scoring over the real subject.
261 "professional", "editorial", "commercial", "premium", "stock",
262 "photo", "photograph", "photography", "image", "picture", "visual",
263 "background", "hero", "cover", "banner", "wallpaper",
264 "high", "quality", "resolution", "sharp", "clean", "cinematic",
265 "dramatic", "lighting", "light", "modern", "natural", "visible",
266 })
267
268 _TOKEN_STRIP_CHARS = ".,;:!?\"'()[]{},。;:!?、"
269 _MATCH_SEPARATOR_RE = re.compile(r"""[\s\-_./:;,'"()[\]{}]+""")
270
271
272 def simplify_query(query: str, max_words: int = 4) -> str:
273 """Trim a verbose query into a short keyword phrase.
274
275 Strategy:
276 1. Strip HEX color codes and parenthetical asides.
277 2. Drop hard-noise words (brand names, generic filler).
278 3. Drop soft-noise words ONLY if concrete nouns remain.
279 4. If the result would be empty, return the original query
280 (fail-open: better an over-broad search than zero results).
281 5. Cap at ``max_words`` words.
282 """
283 cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query)
284 cleaned = re.sub(r"\([^)]*\)", "", cleaned)
285 words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
286 words = [w for w in words if len(w) > 2]
287
288 after_hard = [w for w in words if w.lower() not in _NOISE_WORDS]
289 after_soft = [w for w in after_hard if w.lower() not in _SOFT_NOISE_WORDS]
290
291 # Only drop soft-noise if there are still concrete nouns left.
292 filtered = after_soft if after_soft else after_hard
293
294 if not filtered:
295 # Everything got filtered. Fail open: return the original query.
296 return query.strip()
297
298 return " ".join(filtered[:max_words])
299
300
301 def build_query_progression(query: str) -> list[str]:
302 """Return a list of progressively simpler queries to try in order.
303
304 Stops as soon as one of them yields candidates upstream. Duplicates
305 are dropped while preserving order.
306 """
307 seen: set[str] = set()
308 out: list[str] = []
309 for candidate in (
310 query,
311 simplify_query(query, max_words=4),
312 simplify_query(query, max_words=3),
313 simplify_query(query, max_words=2),
314 simplify_query(query, max_words=1),
315 ):
316 candidate = candidate.strip()
317 if candidate and candidate not in seen:
318 seen.add(candidate)
319 out.append(candidate)
320 return out
321
322
323 # ---------------------------------------------------------------------------
324 # Scoring
325 # ---------------------------------------------------------------------------
326
327
328 def normalize_orientation(width: int, height: int) -> str:
329 if width <= 0 or height <= 0:
330 return "unknown"
331 if width > height:
332 return "landscape"
333 if height > width:
334 return "portrait"
335 return "square"
336
337
338 def _query_tokens(query: str) -> list[str]:
339 """Extract ASCII keyword tokens from a query for relevance scoring.
340
341 Uses the same noise-word filtering as ``simplify_query`` so the
342 relevance signal lines up with the keywords we actually search by.
343 Non-ASCII tokens (CJK etc.) are dropped — image metadata is mostly
344 English even on multi-language providers, so substring matching CJK
345 against an English title is unreliable. When this leaves no tokens,
346 ``compute_relevance`` falls back to neutral (1.0) and lets the other
347 score dimensions decide.
348 """
349 cleaned = re.sub(r"#[0-9a-fA-F]{3,8}", "", query.lower())
350 cleaned = re.sub(r"\([^)]*\)", "", cleaned)
351 words = [w.strip(_TOKEN_STRIP_CHARS) for w in cleaned.split()]
352 words = [w for w in words if len(w) > 2 and w.isascii()]
353 if not words:
354 return []
355 after_hard = [w for w in words if w not in _NOISE_WORDS]
356 after_soft = [w for w in after_hard if w not in _SOFT_NOISE_WORDS]
357 return after_soft if after_soft else after_hard
358
359
360 def _candidate_text(candidate: AssetCandidate) -> str:
361 """Concatenate the candidate's matchable metadata fields for scoring."""
362 return " ".join(
363 filter(
364 None,
365 (
366 candidate.title,
367 candidate.author,
368 candidate.source_page_url,
369 ),
370 )
371 ).lower()
372
373
374 def _normalize_match_text(text: str) -> str:
375 """Normalize metadata / required terms for conservative substring matching."""
376 lowered = (text or "").lower()
377 return _MATCH_SEPARATOR_RE.sub(" ", lowered).strip()
378
379
380 def _term_group_alternatives(term_group: str) -> list[str]:
381 """Split one required term group into alternatives.
382
383 ``"Jiefangbei|Liberation Monument"`` means either alternative satisfies
384 that required group. Different list items are ANDed by
385 ``missing_required_terms``.
386 """
387 return [
388 _normalize_match_text(part)
389 for part in str(term_group or "").split("|")
390 if _normalize_match_text(part)
391 ]
392
393
394 def missing_required_terms(
395 candidate: AssetCandidate,
396 required_terms: tuple[str, ...] | list[str] | None,
397 ) -> list[str]:
398 """Return required term groups not present in candidate metadata.
399
400 This is an entity-safety gate, not a fuzzy visual classifier. Use it for
401 exact subjects such as landmarks, people, companies, or products where a
402 visually nice but wrong image is worse than no image.
403 """
404 if not required_terms:
405 return []
406
407 text = _normalize_match_text(_candidate_text(candidate))
408 compact_text = text.replace(" ", "")
409 missing: list[str] = []
410 for group in required_terms:
411 alternatives = _term_group_alternatives(group)
412 if not alternatives:
413 continue
414 matched = any(
415 alt in text or alt.replace(" ", "") in compact_text
416 for alt in alternatives
417 )
418 if not matched:
419 missing.append(str(group))
420 return missing
421
422
423 def compute_relevance(candidate: AssetCandidate, query: str) -> float:
424 """Fraction of query tokens that appear in the candidate's metadata.
425
426 Range ``[0.0, 1.0]``. Returns ``1.0`` (neutral) when the query has no
427 ASCII tokens to match — this lets non-English queries fall through
428 to license / size scoring without being unfairly rejected.
429 """
430 tokens = _query_tokens(query)
431 if not tokens:
432 return 1.0
433 text = _candidate_text(candidate)
434 if not text:
435 return 0.0
436 hits = sum(1 for t in tokens if t in text)
437 return hits / len(tokens)
438
439
440 def score_candidate(candidate: AssetCandidate, request: ImageSearchRequest) -> float:
441 """Score a candidate against a request. Higher is better; -inf rejects.
442
443 Relevance dominates: a candidate whose metadata shares no query
444 tokens is rejected outright, so size / license / orientation cannot
445 rescue an irrelevant image from a permissive provider.
446 """
447 if not candidate.license_tier:
448 return float("-inf")
449 if (
450 candidate.license_tier == LICENSE_TIER_ATTRIBUTION_REQUIRED
451 and not candidate.author.strip()
452 ):
453 return float("-inf")
454
455 required_misses = missing_required_terms(candidate, request.required_terms)
456 if required_misses:
457 return float("-inf")
458
459 relevance = compute_relevance(candidate, request.query)
460 if relevance == 0.0 and not request.required_terms:
461 return float("-inf")
462
463 score = relevance * 10000.0
464 title_text = _normalize_match_text(candidate.title)
465 compact_title = title_text.replace(" ", "")
466 for group in request.required_terms or ():
467 alternatives = _term_group_alternatives(group)
468 if any(
469 alt in title_text or alt.replace(" ", "") in compact_title
470 for alt in alternatives
471 ):
472 score += 1500.0
473
474 # Penalize infrastructure/transit metadata if the user didn't explicitly ask for it.
475 # This prevents high-res subway station photos from outranking actual tourist landmarks.
476 text = _candidate_text(candidate)
477 query_lower = request.query.lower()
478 infra_terms = [
479 "station", "subway", "metro", "rail", "transit", "airport", "bus",
480 "地铁", "站", "轨道",
481 ]
482
483 if not any(t in query_lower for t in infra_terms):
484 if any(t in text for t in infra_terms):
485 score -= 5000.0
486
487 candidate_orientation = normalize_orientation(candidate.width, candidate.height)
488 requested = (request.orientation or "").strip().lower()
489 if requested:
490 if candidate_orientation == requested:
491 score += 1000.0
492 else:
493 score -= 250.0
494
495 if request.min_width and candidate.width < request.min_width:
496 score -= 500.0
497 if request.min_height and candidate.height < request.min_height:
498 score -= 500.0
499
500 if candidate.license_tier == LICENSE_TIER_NO_ATTRIBUTION:
501 score += 250.0
502
503 # Larger images score higher, but only as a tie-breaker; entity accuracy
504 # and metadata relevance must dominate pixel count.
505 pixel_score = max(candidate.width, 0) * max(candidate.height, 0) / 1000.0
506 score += min(pixel_score, 1500.0)
507 return score
508
509
510 # ---------------------------------------------------------------------------
511 # Attribution text
512 # ---------------------------------------------------------------------------
513
514
515 PROVIDER_DISPLAY_NAMES: dict[str, str] = {
516 "openverse": "Openverse",
517 "wikimedia": "Wikimedia Commons",
518 "pexels": "Pexels",
519 "pixabay": "Pixabay",
520 }
521
522
523 def build_attribution_text(filename: str, candidate: AssetCandidate) -> str:
524 """Render the canonical attribution string for the manifest.
525
526 Format:
527 ``filename — "title" by author, via Provider, license: name (url)``
528
529 Empty fields are gracefully omitted. The text is intended for use by
530 the Executor when generating in-SVG credit elements; it is not meant
531 to be machine-parsed downstream.
532 """
533 provider_name = PROVIDER_DISPLAY_NAMES.get(
534 candidate.provider, candidate.provider or "unknown"
535 )
536
537 parts: list[str] = [filename or candidate.download_url or "image"]
538 middle: list[str] = []
539 if candidate.title:
540 middle.append(f'"{candidate.title}"')
541 if candidate.author:
542 middle.append(f"by {candidate.author}")
543 middle.append(f"via {provider_name}")
544 parts.append(" ".join(middle))
545
546 license_part = candidate.license_name or candidate.license_url
547 if license_part:
548 if candidate.license_url and candidate.license_name:
549 license_part = f"{candidate.license_name} ({candidate.license_url})"
550 parts.append(f"license: {license_part}")
551
552 return " — ".join(parts)
553
554
555 # ---------------------------------------------------------------------------
556 # Small helpers
557 # ---------------------------------------------------------------------------
558
559
560 def ensure_json_parent(path: str | Path) -> Path:
561 """Make sure the parent directory of ``path`` exists; return as Path."""
562 p = Path(path)
563 p.parent.mkdir(parents=True, exist_ok=True)
564 return p
565
565 lines PYTHON