| 1 | #!/usr/bin/env python3 |
| 2 | """E2E comparison: run sample queries on both v3 (current branch) and v2.9.5 (main). |
| 3 | |
| 4 | Usage: |
| 5 | python3 tests/e2e_comparison.py [--v2-script PATH] |
| 6 | |
| 7 | Outputs a markdown comparison table with per-query metrics. |
| 8 | """ |
| 9 | |
| 10 | import json |
| 11 | import subprocess |
| 12 | import sys |
| 13 | import time |
| 14 | from pathlib import Path |
| 15 | |
| 16 | REPO = Path(__file__).resolve().parents[1] |
| 17 | V3_SCRIPT = str(REPO / "scripts" / "last30days.py") |
| 18 | |
| 19 | # v2.9.5 from plugin cache (main branch equivalent) |
| 20 | V2_SCRIPT = str( |
| 21 | Path.home() |
| 22 | / ".claude/plugins/cache/last30days/last30days/2.9.5/scripts/last30days.py" |
| 23 | ) |
| 24 | |
| 25 | EVAL_TOPICS_FILE = REPO / "fixtures" / "eval_topics.json" |
| 26 | |
| 27 | |
| 28 | def _load_queries() -> list[tuple[str, str]]: |
| 29 | if EVAL_TOPICS_FILE.exists(): |
| 30 | rows = json.loads(EVAL_TOPICS_FILE.read_text()) |
| 31 | return [(row["topic"], row["query_type"]) for row in rows] |
| 32 | return [ |
| 33 | ("openclaw vs nanoclaw vs ironclaw", "comparison"), |
| 34 | ("how to deploy on Fly.io", "how_to"), |
| 35 | ("kanye west", "breaking_news"), |
| 36 | ("odds of recession", "prediction"), |
| 37 | ("explain transformer architecture", "concept"), |
| 38 | ] |
| 39 | |
| 40 | |
| 41 | QUERIES = _load_queries() |
| 42 | |
| 43 | |
| 44 | def run_query(script: str, topic: str, timeout: int = 180) -> dict: |
| 45 | """Run a query and return parsed JSON + timing.""" |
| 46 | start = time.time() |
| 47 | try: |
| 48 | result = subprocess.run( |
| 49 | [sys.executable, script, topic, "--emit=json", "--json-profile=raw"], |
| 50 | capture_output=True, |
| 51 | text=True, |
| 52 | timeout=timeout, |
| 53 | ) |
| 54 | elapsed = time.time() - start |
| 55 | if result.returncode != 0: |
| 56 | return { |
| 57 | "error": result.stderr[:200], |
| 58 | "elapsed": elapsed, |
| 59 | "sources": 0, |
| 60 | "candidates": 0, |
| 61 | "intent": "error", |
| 62 | "subqueries": 0, |
| 63 | } |
| 64 | data = json.loads(result.stdout) |
| 65 | |
| 66 | # v3 shape |
| 67 | if "query_plan" in data: |
| 68 | items_by_source = data.get("items_by_source", {}) |
| 69 | return { |
| 70 | "elapsed": elapsed, |
| 71 | "sources": sum(1 for v in items_by_source.values() if v), |
| 72 | "total_items": sum(len(v) for v in items_by_source.values()), |
| 73 | "candidates": len(data.get("ranked_candidates", [])), |
| 74 | "clusters": len(data.get("clusters", [])), |
| 75 | "intent": data["query_plan"].get("intent", "?"), |
| 76 | "subqueries": len(data["query_plan"].get("subqueries", [])), |
| 77 | "errors": list(data.get("errors_by_source", {}).keys()), |
| 78 | } |
| 79 | |
| 80 | # v2 shape |
| 81 | sources_with_items = 0 |
| 82 | total_items = 0 |
| 83 | for key in ["reddit", "x", "youtube", "tiktok", "instagram", "hackernews", |
| 84 | "bluesky", "truthsocial", "polymarket", "web"]: |
| 85 | items = data.get(key, []) |
| 86 | if items: |
| 87 | sources_with_items += 1 |
| 88 | total_items += len(items) |
| 89 | return { |
| 90 | "elapsed": elapsed, |
| 91 | "sources": sources_with_items, |
| 92 | "total_items": total_items, |
| 93 | "candidates": total_items, |
| 94 | "clusters": 0, |
| 95 | "intent": data.get("mode", "?"), |
| 96 | "subqueries": 0, |
| 97 | "errors": [k for k in ["reddit_error", "x_error", "youtube_error", |
| 98 | "tiktok_error", "instagram_error"] |
| 99 | if data.get(k)], |
| 100 | } |
| 101 | except subprocess.TimeoutExpired: |
| 102 | return { |
| 103 | "error": "timeout", |
| 104 | "elapsed": timeout, |
| 105 | "sources": 0, |
| 106 | "candidates": 0, |
| 107 | "intent": "timeout", |
| 108 | "subqueries": 0, |
| 109 | } |
| 110 | except Exception as exc: |
| 111 | return { |
| 112 | "error": str(exc)[:200], |
| 113 | "elapsed": time.time() - start, |
| 114 | "sources": 0, |
| 115 | "candidates": 0, |
| 116 | "intent": "error", |
| 117 | "subqueries": 0, |
| 118 | } |
| 119 | |
| 120 | |
| 121 | def main(): |
| 122 | v2_script = V2_SCRIPT |
| 123 | if len(sys.argv) > 2 and sys.argv[1] == "--v2-script": |
| 124 | v2_script = sys.argv[2] |
| 125 | |
| 126 | if not Path(v2_script).exists(): |
| 127 | print(f"v2 script not found at {v2_script}", file=sys.stderr) |
| 128 | print("Use --v2-script PATH to specify", file=sys.stderr) |
| 129 | sys.exit(1) |
| 130 | |
| 131 | print("# E2E Comparison: v3.0.0 (branch) vs v2.9.5 (main)") |
| 132 | print() |
| 133 | print(f"- v3 script: {V3_SCRIPT}") |
| 134 | print(f"- v2 script: {v2_script}") |
| 135 | print(f"- Queries: {len(QUERIES)}") |
| 136 | print() |
| 137 | |
| 138 | results = [] |
| 139 | for i, (topic, expected_intent) in enumerate(QUERIES, 1): |
| 140 | print(f"[{i}/{len(QUERIES)}] {topic}", file=sys.stderr) |
| 141 | sys.stderr.flush() |
| 142 | |
| 143 | print(f" v3...", end="", file=sys.stderr) |
| 144 | sys.stderr.flush() |
| 145 | v3 = run_query(V3_SCRIPT, topic) |
| 146 | print(f" {v3.get('elapsed', 0):.1f}s", file=sys.stderr) |
| 147 | sys.stderr.flush() |
| 148 | |
| 149 | print(f" v2...", end="", file=sys.stderr) |
| 150 | sys.stderr.flush() |
| 151 | v2 = run_query(v2_script, topic) |
| 152 | print(f" {v2.get('elapsed', 0):.1f}s", file=sys.stderr) |
| 153 | sys.stderr.flush() |
| 154 | |
| 155 | results.append({ |
| 156 | "topic": topic, |
| 157 | "expected_intent": expected_intent, |
| 158 | "v3": v3, |
| 159 | "v2": v2, |
| 160 | }) |
| 161 | |
| 162 | # Print comparison table |
| 163 | print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 errors | v2 errors |") |
| 164 | print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|") |
| 165 | for r in results: |
| 166 | v3, v2 = r["v3"], r["v2"] |
| 167 | v3_err = ", ".join(v3.get("errors", [])) or "-" |
| 168 | v2_err = ", ".join(v2.get("errors", [])) or "-" |
| 169 | print( |
| 170 | f"| {r['topic'][:45]} | {v3.get('intent', '?')} | " |
| 171 | f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | " |
| 172 | f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | " |
| 173 | f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | " |
| 174 | f"{v3_err} | {v2_err} |" |
| 175 | ) |
| 176 | |
| 177 | # Summary |
| 178 | print() |
| 179 | v3_total_sources = sum(r["v3"].get("sources", 0) for r in results) |
| 180 | v2_total_sources = sum(r["v2"].get("sources", 0) for r in results) |
| 181 | v3_total_items = sum(r["v3"].get("total_items", 0) for r in results) |
| 182 | v2_total_items = sum(r["v2"].get("total_items", 0) for r in results) |
| 183 | v3_total_time = sum(r["v3"].get("elapsed", 0) for r in results) |
| 184 | v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results) |
| 185 | v3_errors = sum(len(r["v3"].get("errors", [])) for r in results) |
| 186 | v2_errors = sum(len(r["v2"].get("errors", [])) for r in results) |
| 187 | |
| 188 | print("## Summary") |
| 189 | print() |
| 190 | print(f"| Metric | v3.0.0 | v2.9.5 | Delta |") |
| 191 | print(f"|--------|--------|--------|-------|") |
| 192 | print(f"| Total sources with items | {v3_total_sources} | {v2_total_sources} | {v3_total_sources - v2_total_sources:+d} |") |
| 193 | print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |") |
| 194 | print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |") |
| 195 | print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |") |
| 196 | print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |") |
| 197 | print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |") |
| 198 | print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |") |
| 199 | |
| 200 | |
| 201 | if __name__ == "__main__": |
| 202 | main() |
| 203 |