| 1 | #!/usr/bin/env python3 |
| 2 | """Run the v3 verification bundle for last30days.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import argparse |
| 7 | import json |
| 8 | import os |
| 9 | import statistics |
| 10 | import subprocess |
| 11 | import sys |
| 12 | import time |
| 13 | from pathlib import Path |
| 14 | |
| 15 | |
| 16 | SKILL_ROOT = Path(__file__).resolve().parents[1] |
| 17 | REPO_ROOT = Path(__file__).resolve().parents[3] |
| 18 | PYTHON = sys.executable |
| 19 | ENGINE = SKILL_ROOT / "scripts" / "last30days.py" |
| 20 | EVALUATOR = SKILL_ROOT / "scripts" / "evaluate_search_quality.py" |
| 21 | |
| 22 | SMOKE_TOPIC = "openclaw skills" |
| 23 | SMOKE_CASES = [ |
| 24 | ("gemini", ["--quick", "--search=grounding,hackernews"]), |
| 25 | ("openai", ["--quick", "--search=reddit,hackernews"]), |
| 26 | ("xai", ["--quick", "--search=reddit,hackernews"]), |
| 27 | ("auto", ["--quick", "--search=reddit,grounding,hackernews"]), |
| 28 | ] |
| 29 | |
| 30 | LATENCY_TOPICS = [ |
| 31 | "openclaw skills", |
| 32 | "codex vs claude code", |
| 33 | "anthropic odds", |
| 34 | ] |
| 35 | LATENCY_PROFILES = [ |
| 36 | ("quick", ["--quick", "--search=grounding,hackernews"]), |
| 37 | ("default", ["--search=grounding,hackernews"]), |
| 38 | ("deep", ["--deep", "--search=grounding,hackernews"]), |
| 39 | ] |
| 40 | |
| 41 | |
| 42 | def run_command(cmd: list[str], *, env: dict[str, str] | None = None, timeout: int = 600) -> subprocess.CompletedProcess[str]: |
| 43 | return subprocess.run( |
| 44 | cmd, |
| 45 | cwd=REPO_ROOT, |
| 46 | env=env, |
| 47 | text=True, |
| 48 | capture_output=True, |
| 49 | timeout=timeout, |
| 50 | check=True, |
| 51 | ) |
| 52 | |
| 53 | |
| 54 | def verify_unit() -> dict[str, str]: |
| 55 | run_command([PYTHON, "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py"], timeout=600) |
| 56 | run_command( |
| 57 | [ |
| 58 | PYTHON, |
| 59 | "-m", |
| 60 | "py_compile", |
| 61 | *subprocess.run( |
| 62 | [ |
| 63 | "rg", |
| 64 | "--files", |
| 65 | "skills/last30days/scripts", |
| 66 | "tests", |
| 67 | "-g", |
| 68 | "*.py", |
| 69 | "-g", |
| 70 | "!skills/last30days/scripts/lib/vendor/**", |
| 71 | ], |
| 72 | cwd=REPO_ROOT, |
| 73 | text=True, |
| 74 | capture_output=True, |
| 75 | check=True, |
| 76 | ).stdout.split(), |
| 77 | ], |
| 78 | timeout=600, |
| 79 | ) |
| 80 | return {"status": "ok"} |
| 81 | |
| 82 | |
| 83 | def verify_diagnose() -> dict[str, object]: |
| 84 | result = run_command([PYTHON, str(ENGINE), "--diagnose"], timeout=120) |
| 85 | return json.loads(result.stdout) |
| 86 | |
| 87 | |
| 88 | def verify_smoke() -> list[dict[str, object]]: |
| 89 | rows: list[dict[str, object]] = [] |
| 90 | for provider, extra in SMOKE_CASES: |
| 91 | env = os.environ.copy() |
| 92 | env["LAST30DAYS_REASONING_PROVIDER"] = provider |
| 93 | start = time.time() |
| 94 | result = run_command( |
| 95 | [PYTHON, str(ENGINE), SMOKE_TOPIC, "--emit=json", "--json-profile=raw", *extra], |
| 96 | env=env, |
| 97 | timeout=240, |
| 98 | ) |
| 99 | duration = round(time.time() - start, 2) |
| 100 | report = json.loads(result.stdout) |
| 101 | rows.append( |
| 102 | { |
| 103 | "provider": provider, |
| 104 | "duration_seconds": duration, |
| 105 | "reasoning_provider": (report.get("provider_runtime") or {}).get("reasoning_provider"), |
| 106 | "cluster_count": len(report.get("clusters") or []), |
| 107 | "candidate_count": len(report.get("ranked_candidates") or []), |
| 108 | "error_sources": sorted((report.get("errors_by_source") or {}).keys()), |
| 109 | } |
| 110 | ) |
| 111 | return rows |
| 112 | |
| 113 | |
| 114 | def verify_latency() -> dict[str, dict[str, object]]: |
| 115 | results: dict[str, dict[str, object]] = {} |
| 116 | for profile, extra in LATENCY_PROFILES: |
| 117 | timings = [] |
| 118 | for topic in LATENCY_TOPICS: |
| 119 | start = time.time() |
| 120 | run_command( |
| 121 | [PYTHON, str(ENGINE), topic, "--emit=json", "--json-profile=raw", *extra], |
| 122 | timeout=300, |
| 123 | ) |
| 124 | timings.append(time.time() - start) |
| 125 | results[profile] = { |
| 126 | "times": [round(value, 2) for value in timings], |
| 127 | "median_seconds": round(statistics.median(timings), 2), |
| 128 | "max_seconds": round(max(timings), 2), |
| 129 | } |
| 130 | return results |
| 131 | |
| 132 | |
| 133 | def verify_eval( |
| 134 | *, |
| 135 | baseline: str, |
| 136 | candidate: str, |
| 137 | output_dir: str, |
| 138 | quick: bool, |
| 139 | limit: int, |
| 140 | timeout: int, |
| 141 | ) -> dict[str, object]: |
| 142 | cmd = [ |
| 143 | PYTHON, |
| 144 | str(EVALUATOR), |
| 145 | f"--baseline={baseline}", |
| 146 | f"--candidate={candidate}", |
| 147 | f"--output-dir={output_dir}", |
| 148 | f"--limit={limit}", |
| 149 | f"--timeout={timeout}", |
| 150 | ] |
| 151 | if quick: |
| 152 | cmd.append("--quick") |
| 153 | run_command(cmd, timeout=max(timeout * 8, 600)) |
| 154 | output = Path(output_dir) |
| 155 | metrics = json.loads((output / "metrics.json").read_text()) |
| 156 | summary = (output / "summary.md").read_text() |
| 157 | return {"metrics": metrics, "summary": summary} |
| 158 | |
| 159 | |
| 160 | def build_parser() -> argparse.ArgumentParser: |
| 161 | parser = argparse.ArgumentParser(description="Run the v3 verification bundle") |
| 162 | parser.add_argument("--skip-eval", action="store_true", help="Skip the judged evaluator") |
| 163 | parser.add_argument("--skip-latency", action="store_true", help="Skip live latency sampling") |
| 164 | parser.add_argument("--baseline", default="HEAD~1") |
| 165 | parser.add_argument("--candidate", default="WORKTREE") |
| 166 | parser.add_argument("--output-dir", default="/tmp/last30days-v3-verify") |
| 167 | parser.add_argument("--quick-eval", action="store_true", help="Use evaluator quick mode") |
| 168 | parser.add_argument("--eval-limit", type=int, default=20) |
| 169 | parser.add_argument("--eval-timeout", type=int, default=240) |
| 170 | return parser |
| 171 | |
| 172 | |
| 173 | def main() -> int: |
| 174 | args = build_parser().parse_args() |
| 175 | summary: dict[str, object] = {} |
| 176 | summary["unit"] = verify_unit() |
| 177 | summary["diagnose"] = verify_diagnose() |
| 178 | summary["smoke"] = verify_smoke() |
| 179 | if not args.skip_latency: |
| 180 | summary["latency"] = verify_latency() |
| 181 | if not args.skip_eval: |
| 182 | summary["eval"] = verify_eval( |
| 183 | baseline=args.baseline, |
| 184 | candidate=args.candidate, |
| 185 | output_dir=args.output_dir, |
| 186 | quick=args.quick_eval, |
| 187 | limit=args.eval_limit, |
| 188 | timeout=args.eval_timeout, |
| 189 | ) |
| 190 | print(json.dumps(summary, indent=2, sort_keys=True)) |
| 191 | return 0 |
| 192 | |
| 193 | |
| 194 | if __name__ == "__main__": |
| 195 | raise SystemExit(main()) |
| 196 |