| 1 | #!/usr/bin/env python3 |
| 2 | """Compare two last30days revisions on the v3 ranked candidate output.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import argparse |
| 7 | import hashlib |
| 8 | import json |
| 9 | import math |
| 10 | import os |
| 11 | import subprocess |
| 12 | import sys |
| 13 | import tempfile |
| 14 | from datetime import datetime |
| 15 | from pathlib import Path |
| 16 | from typing import Any |
| 17 | from urllib.error import HTTPError, URLError |
| 18 | from urllib.request import Request, urlopen |
| 19 | |
| 20 | sys.path.insert(0, str(Path(__file__).parent)) |
| 21 | |
| 22 | from lib import env as envlib |
| 23 | from lib import schema |
| 24 | from lib.providers import GEMINI_FLASH_LITE |
| 25 | |
| 26 | |
| 27 | SKILL_ROOT = Path(__file__).resolve().parents[1] |
| 28 | REPO_ROOT = Path(__file__).resolve().parents[3] |
| 29 | EVAL_TOPICS_FILE = REPO_ROOT / "fixtures" / "eval_topics.json" |
| 30 | |
| 31 | |
| 32 | def _load_default_topics() -> list[tuple[str, str]]: |
| 33 | if EVAL_TOPICS_FILE.exists(): |
| 34 | rows = json.loads(EVAL_TOPICS_FILE.read_text()) |
| 35 | return [(row["topic"], row["query_type"]) for row in rows] |
| 36 | return [ |
| 37 | ("nano banana pro prompting", "product"), |
| 38 | ("codex vs claude code", "comparison"), |
| 39 | ("openclaw vs nanoclaw vs ironclaw", "comparison"), |
| 40 | ("anthropic odds", "prediction"), |
| 41 | ("kanye west", "breaking_news"), |
| 42 | ("remotion animations for Claude Code", "how_to"), |
| 43 | ] |
| 44 | |
| 45 | |
| 46 | DEFAULT_TOPICS = _load_default_topics() |
| 47 | DEFAULT_SEARCH = "" |
| 48 | DEFAULT_JUDGE_MODEL = GEMINI_FLASH_LITE |
| 49 | GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}" |
| 50 | EVAL_CREDENTIAL_ENV_KEYS = ( |
| 51 | "GOOGLE_API_KEY", |
| 52 | "GEMINI_API_KEY", |
| 53 | "GOOGLE_GENAI_API_KEY", |
| 54 | "OPENAI_API_KEY", |
| 55 | "XAI_API_KEY", |
| 56 | "SCRAPECREATORS_API_KEY", |
| 57 | "BSKY_HANDLE", |
| 58 | "BSKY_APP_PASSWORD", |
| 59 | "TRUTHSOCIAL_TOKEN", |
| 60 | "AUTH_TOKEN", |
| 61 | "CT0", |
| 62 | ) |
| 63 | |
| 64 | |
| 65 | def stable_item_key(item: dict[str, Any]) -> str: |
| 66 | return str(item.get("candidate_id") or item.get("url") or item.get("title") or "") |
| 67 | |
| 68 | |
| 69 | def row_sources(row: dict[str, Any]) -> list[str]: |
| 70 | candidate = schema.candidate_from_dict(row) |
| 71 | return schema.candidate_sources(candidate) |
| 72 | |
| 73 | |
| 74 | def row_best_date(row: dict[str, Any]) -> str | None: |
| 75 | candidate = schema.candidate_from_dict(row) |
| 76 | return schema.candidate_best_published_at(candidate) |
| 77 | |
| 78 | |
| 79 | V2_SOURCE_KEYS = [ |
| 80 | ("reddit", "title"), |
| 81 | ("x", "text"), |
| 82 | ("youtube", "title"), |
| 83 | ("tiktok", "text"), |
| 84 | ("instagram", "text"), |
| 85 | ("hackernews", "title"), |
| 86 | ("bluesky", "text"), |
| 87 | ("truthsocial", "text"), |
| 88 | ("polymarket", "question"), |
| 89 | ("web", "title"), |
| 90 | ] |
| 91 | |
| 92 | |
| 93 | def build_ranked_items(report: dict[str, Any], limit: int) -> list[dict[str, Any]]: |
| 94 | # v3 format: ranked_candidates list |
| 95 | if report.get("ranked_candidates"): |
| 96 | ranked = [] |
| 97 | for row in report["ranked_candidates"][:limit]: |
| 98 | candidate_sources = row_sources(row) |
| 99 | ranked.append({ |
| 100 | "key": stable_item_key(row), |
| 101 | "source": ", ".join(candidate_sources), |
| 102 | "sources": candidate_sources, |
| 103 | "url": str(row.get("url") or ""), |
| 104 | "text": str(row.get("title") or ""), |
| 105 | "date": row_best_date(row), |
| 106 | "score": float(row.get("final_score") or 0.0), |
| 107 | }) |
| 108 | return ranked |
| 109 | |
| 110 | # v2 format: per-source lists (reddit, x, youtube, etc.) |
| 111 | all_items = [] |
| 112 | for source_key, text_field in V2_SOURCE_KEYS: |
| 113 | for item in report.get(source_key) or []: |
| 114 | if not isinstance(item, dict): |
| 115 | continue |
| 116 | all_items.append({ |
| 117 | "key": str(item.get("url") or item.get("id") or item.get(text_field) or ""), |
| 118 | "source": source_key, |
| 119 | "sources": [source_key], |
| 120 | "url": str(item.get("url") or ""), |
| 121 | "text": str(item.get(text_field) or item.get("title") or ""), |
| 122 | "date": item.get("date"), |
| 123 | "score": float(item.get("score") or 0.0), |
| 124 | }) |
| 125 | all_items.sort(key=lambda x: x["score"], reverse=True) |
| 126 | return all_items[:limit] |
| 127 | |
| 128 | |
| 129 | def source_sets(report: dict[str, Any], limit: int) -> dict[str, set[str]]: |
| 130 | grouped: dict[str, set[str]] = {} |
| 131 | for item in build_ranked_items(report, limit): |
| 132 | for source in item["sources"]: |
| 133 | grouped.setdefault(source, set()).add(item["key"]) |
| 134 | return grouped |
| 135 | |
| 136 | |
| 137 | def jaccard(left: set[str], right: set[str]) -> float: |
| 138 | if not left and not right: |
| 139 | return 1.0 |
| 140 | union = left | right |
| 141 | if not union: |
| 142 | return 1.0 |
| 143 | return len(left & right) / len(union) |
| 144 | |
| 145 | |
| 146 | def retention(left: set[str], right: set[str]) -> float: |
| 147 | if not left: |
| 148 | return 1.0 |
| 149 | return len(left & right) / len(left) |
| 150 | |
| 151 | |
| 152 | def precision_at_k(ranking: list[dict[str, Any]], judgments: dict[str, int], k: int) -> float: |
| 153 | top = ranking[:k] |
| 154 | if not top: |
| 155 | return 0.0 |
| 156 | return sum(1 for item in top if judgments.get(item["key"], 0) >= 2) / len(top) |
| 157 | |
| 158 | |
| 159 | def ndcg_at_k(ranking: list[dict[str, Any]], judgments: dict[str, int], k: int, judged_pool: list[dict[str, Any]]) -> float: |
| 160 | top = ranking[:k] |
| 161 | if not top: |
| 162 | return 0.0 |
| 163 | |
| 164 | def dcg(grades: list[int]) -> float: |
| 165 | total = 0.0 |
| 166 | for index, grade in enumerate(grades, start=1): |
| 167 | total += (2**grade - 1) / math.log2(index + 1) |
| 168 | return total |
| 169 | |
| 170 | actual = [judgments.get(item["key"], 0) for item in top] |
| 171 | ideal = sorted((judgments.get(item["key"], 0) for item in judged_pool), reverse=True)[: len(top)] |
| 172 | ideal_score = dcg(ideal) |
| 173 | if ideal_score == 0: |
| 174 | return 0.0 |
| 175 | return dcg(actual) / ideal_score |
| 176 | |
| 177 | |
| 178 | def source_coverage_recall(ranking: list[dict[str, Any]], judged_pool: list[dict[str, Any]], judgments: dict[str, int]) -> float: |
| 179 | good_sources = { |
| 180 | source |
| 181 | for item in judged_pool |
| 182 | if judgments.get(item["key"], 0) >= 2 |
| 183 | for source in item["sources"] |
| 184 | } |
| 185 | if not good_sources: |
| 186 | return 1.0 |
| 187 | hit_sources = { |
| 188 | source |
| 189 | for item in ranking |
| 190 | if judgments.get(item["key"], 0) >= 2 |
| 191 | for source in item["sources"] |
| 192 | } |
| 193 | return len(hit_sources & good_sources) / len(good_sources) |
| 194 | |
| 195 | |
| 196 | def resolve_google_judge_api_key(config: dict[str, Any]) -> str | None: |
| 197 | return ( |
| 198 | os.environ.get("GOOGLE_API_KEY") |
| 199 | or config.get("GOOGLE_API_KEY") |
| 200 | or os.environ.get("GEMINI_API_KEY") |
| 201 | or config.get("GEMINI_API_KEY") |
| 202 | or os.environ.get("GOOGLE_GENAI_API_KEY") |
| 203 | or config.get("GOOGLE_GENAI_API_KEY") |
| 204 | ) |
| 205 | |
| 206 | |
| 207 | def extract_gemini_text(payload: dict[str, Any]) -> str: |
| 208 | for candidate in payload.get("candidates") or []: |
| 209 | content = candidate.get("content") or {} |
| 210 | for part in content.get("parts") or []: |
| 211 | if part.get("text"): |
| 212 | return part["text"] |
| 213 | raise ValueError("Gemini response did not contain text.") |
| 214 | |
| 215 | |
| 216 | def call_gemini_judge(api_key: str, model: str, prompt: str) -> dict[str, Any]: |
| 217 | body = { |
| 218 | "contents": [{"parts": [{"text": prompt}]}], |
| 219 | "generationConfig": {"temperature": 0, "responseMimeType": "application/json"}, |
| 220 | } |
| 221 | request = Request( |
| 222 | GEMINI_API_URL.format(model=model, api_key=api_key), |
| 223 | data=json.dumps(body).encode("utf-8"), |
| 224 | headers={"Content-Type": "application/json"}, |
| 225 | method="POST", |
| 226 | ) |
| 227 | try: |
| 228 | with urlopen(request, timeout=120) as response: |
| 229 | payload = json.loads(response.read().decode("utf-8")) |
| 230 | except HTTPError as exc: |
| 231 | detail = exc.read().decode("utf-8", errors="replace") |
| 232 | raise RuntimeError(f"Gemini HTTP {exc.code}: {detail}") from exc |
| 233 | except URLError as exc: |
| 234 | raise RuntimeError(f"Gemini request failed: {exc}") from exc |
| 235 | return json.loads(extract_gemini_text(payload)) |
| 236 | |
| 237 | |
| 238 | def build_judge_prompt(topic: str, query_type: str, items: list[dict[str, Any]]) -> str: |
| 239 | item_lines = [] |
| 240 | for item in items: |
| 241 | item_lines.append( |
| 242 | "\n".join([ |
| 243 | f"- id: {item['key']}", |
| 244 | f" source: {item['source']}", |
| 245 | f" title: {item['text'][:220]}", |
| 246 | f" url: {item['url']}", |
| 247 | f" date: {item.get('date') or 'unknown'}", |
| 248 | ]) |
| 249 | ) |
| 250 | return f""" |
| 251 | Judge search-result relevance for a last-30-days research tool. |
| 252 | |
| 253 | Topic: {topic} |
| 254 | Query type: {query_type} |
| 255 | |
| 256 | Score each item on this 0-3 scale: |
| 257 | - 0 = off-topic or clearly bad |
| 258 | - 1 = weak or tangential |
| 259 | - 2 = relevant and useful |
| 260 | - 3 = highly relevant, one of the best results |
| 261 | |
| 262 | Return JSON only: |
| 263 | {{ |
| 264 | "judgments": [ |
| 265 | {{"id": "ITEM_ID", "grade": 0}} |
| 266 | ] |
| 267 | }} |
| 268 | |
| 269 | Items: |
| 270 | {chr(10).join(item_lines)} |
| 271 | """.strip() |
| 272 | |
| 273 | |
| 274 | def get_judgments( |
| 275 | *, |
| 276 | output_dir: Path, |
| 277 | slug: str, |
| 278 | topic: str, |
| 279 | query_type: str, |
| 280 | items: list[dict[str, Any]], |
| 281 | judge_model: str, |
| 282 | gemini_api_key: str | None, |
| 283 | ) -> dict[str, int]: |
| 284 | cache_file = output_dir / "judgments" / f"{slug}.json" |
| 285 | cache_file.parent.mkdir(parents=True, exist_ok=True) |
| 286 | prompt = build_judge_prompt(topic, query_type, items) |
| 287 | input_sha256 = hashlib.sha256(json.dumps([judge_model, prompt]).encode("utf-8")).hexdigest() |
| 288 | stale_cache = False |
| 289 | if cache_file.exists(): |
| 290 | payload = json.loads(cache_file.read_text()) |
| 291 | if payload.get("input_sha256") == input_sha256: |
| 292 | return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} |
| 293 | stale_cache = True |
| 294 | if not gemini_api_key or not items: |
| 295 | if stale_cache and not gemini_api_key: |
| 296 | sys.stderr.write( |
| 297 | f"[Eval] Cached judgments for {slug!r} have different or unverified " |
| 298 | f"judge inputs/model and no Gemini API key is set to re-judge; returning " |
| 299 | f"no grades (metrics for this topic will be zero).\n" |
| 300 | ) |
| 301 | return {} |
| 302 | payload = call_gemini_judge(gemini_api_key, judge_model, prompt) |
| 303 | payload["judge_model"] = judge_model |
| 304 | payload["input_sha256"] = input_sha256 |
| 305 | cache_file.write_text(json.dumps(payload, indent=2)) |
| 306 | return {row["id"]: int(row["grade"]) for row in payload.get("judgments") or []} |
| 307 | |
| 308 | |
| 309 | def create_eval_env() -> dict[str, str]: |
| 310 | config = envlib.get_config() |
| 311 | passthrough = { |
| 312 | "PATH": os.environ.get("PATH", ""), |
| 313 | "LANG": os.environ.get("LANG", "en_US.UTF-8"), |
| 314 | "LC_ALL": os.environ.get("LC_ALL", ""), |
| 315 | "TMPDIR": os.environ.get("TMPDIR", ""), |
| 316 | "PYTHONUTF8": "1", |
| 317 | "LAST30DAYS_CONFIG_DIR": "", |
| 318 | } |
| 319 | for key in EVAL_CREDENTIAL_ENV_KEYS: |
| 320 | value = os.environ.get(key) or config.get(key) |
| 321 | if value: |
| 322 | passthrough[key] = value |
| 323 | return passthrough |
| 324 | |
| 325 | |
| 326 | def run_last30days(repo_dir: Path, topic: str, *, search: str, timeout_seconds: int, quick: bool, mock: bool, env: dict[str, str]) -> dict[str, Any]: |
| 327 | engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py" |
| 328 | if not engine.exists(): |
| 329 | engine = repo_dir / "scripts" / "last30days.py" |
| 330 | cmd = [sys.executable, str(engine), topic, "--emit=json"] |
| 331 | # Current engines default to the stable agent export, while older revisions |
| 332 | # used by the evaluator implicitly emit the raw report and do not recognize |
| 333 | # --json-profile. Request raw explicitly whenever the checked-out engine |
| 334 | # supports the selector. |
| 335 | if not engine.exists() or "--json-profile" in engine.read_text(encoding="utf-8"): |
| 336 | cmd.append("--json-profile=raw") |
| 337 | if search: |
| 338 | cmd.extend(["--search", search]) |
| 339 | if quick: |
| 340 | cmd.append("--quick") |
| 341 | if mock: |
| 342 | cmd.append("--mock") |
| 343 | result = subprocess.run( |
| 344 | cmd, |
| 345 | cwd=repo_dir, |
| 346 | env=env, |
| 347 | capture_output=True, |
| 348 | text=True, |
| 349 | timeout=timeout_seconds, |
| 350 | check=False, |
| 351 | ) |
| 352 | if result.returncode != 0: |
| 353 | raise RuntimeError(f"{repo_dir.name} failed for '{topic}' with exit {result.returncode}\n{result.stderr.strip()}") |
| 354 | payload = json.loads(result.stdout) |
| 355 | # Shape guard: the evaluator compares raw Report fields. If the engine |
| 356 | # emitted the agent profile anyway (flag detection missed a future |
| 357 | # spelling), fail loudly instead of scoring empty ranked_candidates. |
| 358 | if "schema_version" in payload and "ranked_candidates" not in payload: |
| 359 | raise RuntimeError( |
| 360 | f"{repo_dir.name} emitted the agent JSON profile; the evaluator " |
| 361 | "requires the raw Report (--json-profile=raw)." |
| 362 | ) |
| 363 | return payload |
| 364 | |
| 365 | |
| 366 | def create_worktree(rev: str) -> Path: |
| 367 | worktree_dir = Path(tempfile.mkdtemp(prefix="last30days-eval-")) |
| 368 | subprocess.run( |
| 369 | ["git", "worktree", "add", "--detach", str(worktree_dir), rev], |
| 370 | cwd=REPO_ROOT, |
| 371 | check=True, |
| 372 | capture_output=True, |
| 373 | text=True, |
| 374 | ) |
| 375 | return worktree_dir |
| 376 | |
| 377 | |
| 378 | def resolve_repo_dir(label: str) -> tuple[Path, bool]: |
| 379 | """Resolve a benchmark label into a repo directory and whether it is temporary.""" |
| 380 | if label == "WORKTREE": |
| 381 | return REPO_ROOT, False |
| 382 | return create_worktree(label), True |
| 383 | |
| 384 | |
| 385 | def remove_worktree(path: Path) -> None: |
| 386 | subprocess.run( |
| 387 | ["git", "worktree", "remove", "--force", str(path)], |
| 388 | cwd=REPO_ROOT, |
| 389 | check=False, |
| 390 | capture_output=True, |
| 391 | text=True, |
| 392 | ) |
| 393 | try: |
| 394 | os.rmdir(path) |
| 395 | except OSError: |
| 396 | pass |
| 397 | |
| 398 | |
| 399 | def summarize_topic(topic: str, query_type: str, baseline_report: dict[str, Any], candidate_report: dict[str, Any], judgments: dict[str, int], judged_pool: list[dict[str, Any]], limit: int) -> dict[str, Any]: |
| 400 | baseline_ranked = build_ranked_items(baseline_report, limit) |
| 401 | candidate_ranked = build_ranked_items(candidate_report, limit) |
| 402 | baseline_sets = source_sets(baseline_report, limit) |
| 403 | candidate_sets = source_sets(candidate_report, limit) |
| 404 | overall_left = set().union(*baseline_sets.values()) if baseline_sets else set() |
| 405 | overall_right = set().union(*candidate_sets.values()) if candidate_sets else set() |
| 406 | sources = sorted(set(baseline_sets) | set(candidate_sets)) |
| 407 | return { |
| 408 | "topic": topic, |
| 409 | "query_type": query_type, |
| 410 | "baseline": { |
| 411 | "precision_at_5": precision_at_k(baseline_ranked, judgments, 5), |
| 412 | "ndcg_at_5": ndcg_at_k(baseline_ranked, judgments, 5, judged_pool), |
| 413 | "source_coverage_recall": source_coverage_recall(baseline_ranked, judged_pool, judgments), |
| 414 | }, |
| 415 | "candidate": { |
| 416 | "precision_at_5": precision_at_k(candidate_ranked, judgments, 5), |
| 417 | "ndcg_at_5": ndcg_at_k(candidate_ranked, judgments, 5, judged_pool), |
| 418 | "source_coverage_recall": source_coverage_recall(candidate_ranked, judged_pool, judgments), |
| 419 | }, |
| 420 | "stability": { |
| 421 | "overall_jaccard": jaccard(overall_left, overall_right), |
| 422 | "overall_retention_vs_baseline": retention(overall_left, overall_right), |
| 423 | "per_source": { |
| 424 | source: { |
| 425 | "baseline_count": len(baseline_sets.get(source, set())), |
| 426 | "candidate_count": len(candidate_sets.get(source, set())), |
| 427 | "jaccard": jaccard(baseline_sets.get(source, set()), candidate_sets.get(source, set())), |
| 428 | "retention_vs_baseline": retention(baseline_sets.get(source, set()), candidate_sets.get(source, set())), |
| 429 | } |
| 430 | for source in sources |
| 431 | }, |
| 432 | }, |
| 433 | } |
| 434 | |
| 435 | |
| 436 | def write_summary(output_dir: Path, baseline_label: str, candidate_label: str, summaries: list[dict[str, Any]]) -> None: |
| 437 | output_dir.mkdir(parents=True, exist_ok=True) |
| 438 | payload = { |
| 439 | "generated_at": datetime.now().isoformat(timespec="seconds"), |
| 440 | "baseline": baseline_label, |
| 441 | "candidate": candidate_label, |
| 442 | "topics": summaries, |
| 443 | } |
| 444 | (output_dir / "metrics.json").write_text(json.dumps(payload, indent=2)) |
| 445 | |
| 446 | lines = [ |
| 447 | "# Search Quality Evaluation", |
| 448 | "", |
| 449 | f"- Baseline: `{baseline_label}`", |
| 450 | f"- Candidate: `{candidate_label}`", |
| 451 | f"- Generated: {payload['generated_at']}", |
| 452 | "", |
| 453 | "| Topic | Base P@5 | Cand P@5 | Base nDCG@5 | Cand nDCG@5 | Jaccard | Retention |", |
| 454 | "|---|---:|---:|---:|---:|---:|---:|", |
| 455 | ] |
| 456 | for row in summaries: |
| 457 | lines.append( |
| 458 | "| {topic} | {bp:.2f} | {cp:.2f} | {bn:.2f} | {cn:.2f} | {jac:.2f} | {ret:.2f} |".format( |
| 459 | topic=row["topic"], |
| 460 | bp=row["baseline"]["precision_at_5"], |
| 461 | cp=row["candidate"]["precision_at_5"], |
| 462 | bn=row["baseline"]["ndcg_at_5"], |
| 463 | cn=row["candidate"]["ndcg_at_5"], |
| 464 | jac=row["stability"]["overall_jaccard"], |
| 465 | ret=row["stability"]["overall_retention_vs_baseline"], |
| 466 | ) |
| 467 | ) |
| 468 | (output_dir / "summary.md").write_text("\n".join(lines) + "\n") |
| 469 | |
| 470 | |
| 471 | def write_failure_summary( |
| 472 | output_dir: Path, |
| 473 | baseline_label: str, |
| 474 | candidate_label: str, |
| 475 | summaries: list[dict[str, Any]], |
| 476 | failures: list[dict[str, Any]], |
| 477 | ) -> None: |
| 478 | write_summary(output_dir, baseline_label, candidate_label, summaries) |
| 479 | metrics_path = output_dir / "metrics.json" |
| 480 | payload = json.loads(metrics_path.read_text()) if metrics_path.exists() else { |
| 481 | "generated_at": datetime.now().isoformat(timespec="seconds"), |
| 482 | "baseline": baseline_label, |
| 483 | "candidate": candidate_label, |
| 484 | "topics": [], |
| 485 | } |
| 486 | payload["failures"] = failures |
| 487 | metrics_path.write_text(json.dumps(payload, indent=2)) |
| 488 | |
| 489 | summary_path = output_dir / "summary.md" |
| 490 | lines = summary_path.read_text().splitlines() if summary_path.exists() else ["# Search Quality Evaluation", ""] |
| 491 | if failures: |
| 492 | lines.extend([ |
| 493 | "", |
| 494 | "## Failures", |
| 495 | "", |
| 496 | ]) |
| 497 | for failure in failures: |
| 498 | lines.append(f"- `{failure['topic']}`: {failure['error']}") |
| 499 | summary_path.write_text("\n".join(lines).rstrip() + "\n") |
| 500 | |
| 501 | |
| 502 | def parse_topics_file(path: Path) -> list[tuple[str, str]]: |
| 503 | rows = json.loads(path.read_text()) |
| 504 | return [(str(row["topic"]), str(row.get("query_type") or "general")) for row in rows] |
| 505 | |
| 506 | |
| 507 | def build_parser() -> argparse.ArgumentParser: |
| 508 | parser = argparse.ArgumentParser(description="Compare two last30days revisions on ranked candidate quality") |
| 509 | parser.add_argument("--baseline", default="HEAD~1") |
| 510 | parser.add_argument("--candidate", default="WORKTREE") |
| 511 | parser.add_argument("--search", default=DEFAULT_SEARCH) |
| 512 | parser.add_argument("--output-dir", default="tmp/search-quality") |
| 513 | parser.add_argument("--judge-model", default=DEFAULT_JUDGE_MODEL) |
| 514 | parser.add_argument("--timeout", type=int, default=240) |
| 515 | parser.add_argument("--limit", type=int, default=20) |
| 516 | parser.add_argument("--mock", action="store_true") |
| 517 | parser.add_argument("--quick", action="store_true") |
| 518 | parser.add_argument("--topics-file") |
| 519 | return parser |
| 520 | |
| 521 | |
| 522 | def main() -> int: |
| 523 | args = build_parser().parse_args() |
| 524 | topics = parse_topics_file(Path(args.topics_file)) if args.topics_file else DEFAULT_TOPICS |
| 525 | output_dir = Path(args.output_dir).resolve() |
| 526 | config = envlib.get_config() |
| 527 | gemini_api_key = resolve_google_judge_api_key(config) |
| 528 | run_env = create_eval_env() |
| 529 | |
| 530 | baseline_dir, baseline_temp = resolve_repo_dir(args.baseline) |
| 531 | candidate_dir, candidate_temp = resolve_repo_dir(args.candidate) |
| 532 | try: |
| 533 | summaries = [] |
| 534 | failures = [] |
| 535 | for topic, query_type in topics: |
| 536 | try: |
| 537 | baseline_report = run_last30days( |
| 538 | baseline_dir, |
| 539 | topic, |
| 540 | search=args.search, |
| 541 | timeout_seconds=args.timeout, |
| 542 | quick=args.quick, |
| 543 | mock=args.mock, |
| 544 | env=run_env, |
| 545 | ) |
| 546 | candidate_report = run_last30days( |
| 547 | candidate_dir, |
| 548 | topic, |
| 549 | search=args.search, |
| 550 | timeout_seconds=args.timeout, |
| 551 | quick=args.quick, |
| 552 | mock=args.mock, |
| 553 | env=run_env, |
| 554 | ) |
| 555 | judged_pool_map = { |
| 556 | item["key"]: item |
| 557 | for item in build_ranked_items(baseline_report, args.limit) + build_ranked_items(candidate_report, args.limit) |
| 558 | } |
| 559 | judged_pool = list(judged_pool_map.values()) |
| 560 | judgments = get_judgments( |
| 561 | output_dir=output_dir, |
| 562 | slug="".join(char.lower() if char.isalnum() else "-" for char in topic).strip("-"), |
| 563 | topic=topic, |
| 564 | query_type=query_type, |
| 565 | items=judged_pool, |
| 566 | judge_model=args.judge_model, |
| 567 | gemini_api_key=gemini_api_key, |
| 568 | ) |
| 569 | summaries.append(summarize_topic(topic, query_type, baseline_report, candidate_report, judgments, judged_pool, args.limit)) |
| 570 | except Exception as exc: |
| 571 | failures.append({"topic": topic, "query_type": query_type, "error": str(exc)}) |
| 572 | write_failure_summary(output_dir, args.baseline, args.candidate, summaries, failures) |
| 573 | finally: |
| 574 | if baseline_temp: |
| 575 | remove_worktree(baseline_dir) |
| 576 | if candidate_temp: |
| 577 | remove_worktree(candidate_dir) |
| 578 | result = {"output_dir": str(output_dir), "topics": len(topics), "failures": len(failures)} |
| 579 | print(json.dumps(result, indent=2)) |
| 580 | return 1 if failures else 0 |
| 581 | |
| 582 | |
| 583 | if __name__ == "__main__": |
| 584 | raise SystemExit(main()) |
| 585 |