| 1 | """Offline research-quality eval harness built on recorded source responses.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import itertools |
| 6 | import json |
| 7 | import sys |
| 8 | from dataclasses import dataclass |
| 9 | from datetime import datetime, timezone |
| 10 | from pathlib import Path |
| 11 | from typing import Any |
| 12 | from unittest import mock |
| 13 | |
| 14 | ROOT = Path(__file__).resolve().parents[2] |
| 15 | SCRIPTS = ROOT / "skills" / "last30days" / "scripts" |
| 16 | if str(SCRIPTS) not in sys.path: |
| 17 | sys.path.insert(0, str(SCRIPTS)) |
| 18 | |
| 19 | from lib import entity_extract, http, pipeline, schema # noqa: E402 |
| 20 | |
| 21 | FIXTURES_DIR = Path(__file__).with_name("fixtures") |
| 22 | BASELINE_PATH = Path(__file__).with_name("baseline.json") |
| 23 | METRIC_NAMES = ( |
| 24 | "citation_grounding", |
| 25 | "recency_compliance", |
| 26 | "cluster_coherence", |
| 27 | "coverage", |
| 28 | "determinism", |
| 29 | ) |
| 30 | ENTITY_OVERLAP_FLOOR = 0.45 |
| 31 | |
| 32 | |
| 33 | class _FrozenDateTime(datetime): |
| 34 | @classmethod |
| 35 | def now(cls, tz=None): |
| 36 | fixed = cls(2026, 7, 10, 12, 0, 0, tzinfo=timezone.utc) |
| 37 | return fixed if tz is not None else fixed.replace(tzinfo=None) |
| 38 | |
| 39 | |
| 40 | @dataclass(frozen=True) |
| 41 | class EvalFixture: |
| 42 | name: str |
| 43 | path: Path |
| 44 | manifest: dict[str, Any] |
| 45 | input_urls: frozenset[str] |
| 46 | |
| 47 | |
| 48 | @dataclass |
| 49 | class EvalResult: |
| 50 | fixture: EvalFixture |
| 51 | report: schema.Report |
| 52 | scores: dict[str, float] |
| 53 | |
| 54 | |
| 55 | def _http_urls(value: Any) -> set[str]: |
| 56 | urls: set[str] = set() |
| 57 | if isinstance(value, dict): |
| 58 | for child in value.values(): |
| 59 | urls.update(_http_urls(child)) |
| 60 | elif isinstance(value, list): |
| 61 | for child in value: |
| 62 | urls.update(_http_urls(child)) |
| 63 | elif isinstance(value, str) and value.startswith(("https://", "http://")): |
| 64 | urls.add(value) |
| 65 | return urls |
| 66 | |
| 67 | |
| 68 | def load_fixtures(root: Path = FIXTURES_DIR) -> list[EvalFixture]: |
| 69 | fixtures: list[EvalFixture] = [] |
| 70 | for manifest_path in sorted(root.glob("*/manifest.json")): |
| 71 | manifest = json.loads(manifest_path.read_text(encoding="utf-8")) |
| 72 | http_payload = json.loads((manifest_path.parent / "http.json").read_text(encoding="utf-8")) |
| 73 | input_urls: set[str] = set() |
| 74 | for exchange in http_payload.get("exchanges") or []: |
| 75 | input_urls.update(_http_urls((exchange.get("response") or {}).get("value"))) |
| 76 | for exchange in http_payload.get("source_exchanges") or []: |
| 77 | input_urls.update(_http_urls(exchange.get("value"))) |
| 78 | fixtures.append( |
| 79 | EvalFixture( |
| 80 | name=manifest.get("name") or manifest_path.parent.name, |
| 81 | path=manifest_path.parent, |
| 82 | manifest=manifest, |
| 83 | input_urls=frozenset(input_urls), |
| 84 | ) |
| 85 | ) |
| 86 | return fixtures |
| 87 | |
| 88 | |
| 89 | def _run_once(fixture: EvalFixture) -> schema.Report: |
| 90 | manifest = fixture.manifest |
| 91 | config = dict(manifest.get("config") or {}) |
| 92 | network_error = AssertionError(f"Network attempted while replaying {fixture.name}") |
| 93 | with mock.patch.object(pipeline, "datetime", _FrozenDateTime), \ |
| 94 | mock.patch.object(schema, "datetime", _FrozenDateTime), \ |
| 95 | mock.patch.object( |
| 96 | pipeline, |
| 97 | "available_sources", |
| 98 | return_value=list(manifest["fixture_sources"]), |
| 99 | ), \ |
| 100 | mock.patch.object(http.urllib.request, "urlopen", side_effect=network_error), \ |
| 101 | http.replaying_requests(fixture.path): |
| 102 | return pipeline.run( |
| 103 | topic=manifest["topic"], |
| 104 | config=config, |
| 105 | depth=manifest.get("depth", "quick"), |
| 106 | requested_sources=list(manifest["fixture_sources"]), |
| 107 | mock=False, |
| 108 | web_backend=manifest.get("web_backend", "none"), |
| 109 | external_plan=manifest["plan"], |
| 110 | lookback_days=int(manifest.get("lookback_days", 30)), |
| 111 | as_of_date=manifest["as_of_date"], |
| 112 | ) |
| 113 | |
| 114 | |
| 115 | def _citation_grounding(report: schema.Report, fixture: EvalFixture) -> float: |
| 116 | results = schema.to_agent_export(report)["results"] |
| 117 | if not results: |
| 118 | return 0.0 |
| 119 | grounded = sum(bool(result.get("url")) and result["url"] in fixture.input_urls for result in results) |
| 120 | return grounded / len(results) |
| 121 | |
| 122 | |
| 123 | def _recency_compliance(report: schema.Report) -> float: |
| 124 | ranked_items = [ |
| 125 | item |
| 126 | for candidate in report.ranked_candidates |
| 127 | for item in candidate.source_items |
| 128 | ] |
| 129 | if not ranked_items: |
| 130 | return 0.0 |
| 131 | compliant = sum( |
| 132 | not item.published_at |
| 133 | or report.range_from <= item.published_at[:10] <= report.range_to |
| 134 | for item in ranked_items |
| 135 | ) |
| 136 | return compliant / len(ranked_items) |
| 137 | |
| 138 | |
| 139 | def _cluster_coherence(report: schema.Report, fixture: EvalFixture) -> float: |
| 140 | candidates = {candidate.candidate_id: candidate for candidate in report.ranked_candidates} |
| 141 | pair_scores: list[float] = [] |
| 142 | for report_cluster in report.clusters: |
| 143 | members = [candidates[candidate_id] for candidate_id in report_cluster.candidate_ids if candidate_id in candidates] |
| 144 | for left, right in itertools.combinations(members, 2): |
| 145 | left_entities = entity_extract.extract_text_entities(f"{left.title} {left.snippet}") |
| 146 | right_entities = entity_extract.extract_text_entities(f"{right.title} {right.snippet}") |
| 147 | overlap = entity_extract.entity_overlap(left_entities, right_entities) |
| 148 | pair_scores.append(1.0 if overlap >= ENTITY_OVERLAP_FLOOR else 0.0) |
| 149 | if pair_scores: |
| 150 | return sum(pair_scores) / len(pair_scores) |
| 151 | # No multi-member clusters formed. For fixtures that historically cluster |
| 152 | # (expects_clusters: true in the manifest), that means cluster formation |
| 153 | # regressed and must fail rather than score a vacuous 1.0. Sparse topics |
| 154 | # that legitimately produce singletons declare expects_clusters: false. |
| 155 | return 0.0 if fixture.manifest.get("expects_clusters") else 1.0 |
| 156 | |
| 157 | |
| 158 | def _coverage(report: schema.Report, fixture: EvalFixture) -> float: |
| 159 | sources = list(fixture.manifest["fixture_sources"]) |
| 160 | if not sources: |
| 161 | return 0.0 |
| 162 | covered = sum( |
| 163 | bool(report.items_by_source.get(source)) or source in report.source_status |
| 164 | for source in sources |
| 165 | ) |
| 166 | return covered / len(sources) |
| 167 | |
| 168 | |
| 169 | def score_report( |
| 170 | report: schema.Report, |
| 171 | fixture: EvalFixture, |
| 172 | *, |
| 173 | deterministic: bool, |
| 174 | ) -> dict[str, float]: |
| 175 | return { |
| 176 | "citation_grounding": _citation_grounding(report, fixture), |
| 177 | "recency_compliance": _recency_compliance(report), |
| 178 | "cluster_coherence": _cluster_coherence(report, fixture), |
| 179 | "coverage": _coverage(report, fixture), |
| 180 | "determinism": 1.0 if deterministic else 0.0, |
| 181 | } |
| 182 | |
| 183 | |
| 184 | def evaluate_fixture(fixture: EvalFixture) -> EvalResult: |
| 185 | first = _run_once(fixture) |
| 186 | second = _run_once(fixture) |
| 187 | deterministic = schema.to_dict(first) == schema.to_dict(second) |
| 188 | return EvalResult( |
| 189 | fixture=fixture, |
| 190 | report=first, |
| 191 | scores=score_report(first, fixture, deterministic=deterministic), |
| 192 | ) |
| 193 | |
| 194 | |
| 195 | def evaluate_all(fixtures: list[EvalFixture] | None = None) -> list[EvalResult]: |
| 196 | selected = fixtures if fixtures is not None else load_fixtures() |
| 197 | if not selected: |
| 198 | raise AssertionError(f"No eval fixtures found under {FIXTURES_DIR}") |
| 199 | return [evaluate_fixture(fixture) for fixture in selected] |
| 200 | |
| 201 | |
| 202 | def aggregate_scores(results: list[EvalResult]) -> dict[str, float]: |
| 203 | return { |
| 204 | metric: sum(result.scores[metric] for result in results) / len(results) |
| 205 | for metric in METRIC_NAMES |
| 206 | } |
| 207 | |
| 208 | |
| 209 | def load_baseline(path: Path = BASELINE_PATH) -> dict[str, float]: |
| 210 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 211 | return {metric: float(payload["metrics"][metric]) for metric in METRIC_NAMES} |
| 212 | |
| 213 | |
| 214 | def baseline_failures( |
| 215 | scores: dict[str, float], |
| 216 | baseline: dict[str, float] | None = None, |
| 217 | ) -> list[str]: |
| 218 | floors = baseline if baseline is not None else load_baseline() |
| 219 | return [ |
| 220 | f"{metric}: {scores[metric]:.3f} < {floors[metric]:.3f}" |
| 221 | for metric in METRIC_NAMES |
| 222 | if scores[metric] + 1e-12 < floors[metric] |
| 223 | ] |
| 224 | |
| 225 | |
| 226 | def per_fixture_failures(results: list["EvalResult"]) -> list[str]: |
| 227 | """Enforce per-fixture floors so one broken archetype cannot hide in the |
| 228 | cross-fixture average (a total clustering failure on breaking-event scores |
| 229 | 0.0 but averages to 0.857 across seven fixtures).""" |
| 230 | raw = json.loads(BASELINE_PATH.read_text()) |
| 231 | floors = raw.get("per_fixture_floors") or {} |
| 232 | failures: list[str] = [] |
| 233 | for result in results: |
| 234 | for metric, floor in floors.items(): |
| 235 | value = result.scores.get(metric) |
| 236 | if value is not None and value + 1e-12 < float(floor): |
| 237 | failures.append( |
| 238 | f"{result.fixture.name}/{metric}: {value:.3f} < {float(floor):.3f}" |
| 239 | ) |
| 240 | return failures |
| 241 | |
| 242 | |
| 243 | def format_score_table(results: list[EvalResult]) -> str: |
| 244 | aggregate = aggregate_scores(results) |
| 245 | headers = ["fixture", *METRIC_NAMES] |
| 246 | rows = [ |
| 247 | [result.fixture.name, *(f"{result.scores[name]:.3f}" for name in METRIC_NAMES)] |
| 248 | for result in results |
| 249 | ] |
| 250 | rows.append(["AVERAGE", *(f"{aggregate[name]:.3f}" for name in METRIC_NAMES)]) |
| 251 | widths = [max(len(str(row[index])) for row in [headers, *rows]) for index in range(len(headers))] |
| 252 | rendered = [" | ".join(str(value).ljust(widths[index]) for index, value in enumerate(headers))] |
| 253 | rendered.append("-+-".join("-" * width for width in widths)) |
| 254 | rendered.extend( |
| 255 | " | ".join(str(value).ljust(widths[index]) for index, value in enumerate(row)) |
| 256 | for row in rows |
| 257 | ) |
| 258 | return "\n".join(rendered) |
| 259 | |
| 260 | |
| 261 | def main() -> int: |
| 262 | results = evaluate_all() |
| 263 | print(format_score_table(results)) |
| 264 | failures = baseline_failures(aggregate_scores(results)) |
| 265 | if failures: |
| 266 | print("\nBaseline failures:", file=sys.stderr) |
| 267 | for failure in failures: |
| 268 | print(f"- {failure}", file=sys.stderr) |
| 269 | return 1 |
| 270 | return 0 |
| 271 | |
| 272 | |
| 273 | if __name__ == "__main__": |
| 274 | raise SystemExit(main()) |
| 275 |