| 1 | #!/usr/bin/env python3 |
| 2 | """Topic watchlist management for last30days.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import argparse |
| 7 | import json |
| 8 | import subprocess |
| 9 | import sys |
| 10 | import time |
| 11 | import urllib.parse |
| 12 | from pathlib import Path |
| 13 | |
| 14 | SCRIPT_DIR = Path(__file__).parent.resolve() |
| 15 | sys.path.insert(0, str(SCRIPT_DIR)) |
| 16 | |
| 17 | import store |
| 18 | from lib import http, schema |
| 19 | |
| 20 | |
| 21 | # --- Webhook Delivery Functions --- |
| 22 | |
| 23 | def _deliver_findings(topic_name: str, counts: dict) -> None: |
| 24 | """Send webhook notification if delivery is configured and there are new findings.""" |
| 25 | channel = store.get_setting("delivery_channel", "") |
| 26 | if not channel or counts.get("new", 0) == 0: |
| 27 | return |
| 28 | |
| 29 | mode = store.get_setting("delivery_mode", "announce") |
| 30 | message = _format_delivery_message(topic_name, counts, mode) |
| 31 | |
| 32 | # Require https before routing. The old "hooks.slack.com" in channel |
| 33 | # substring test ran before any scheme check, so a channel like |
| 34 | # http://evil.example/hooks.slack.com was treated as Slack and POSTed in |
| 35 | # cleartext to the wrong host. Match Slack on the exact hostname instead. |
| 36 | parsed = urllib.parse.urlparse(channel) |
| 37 | if parsed.scheme != "https": |
| 38 | print( |
| 39 | f"Delivery skipped: delivery_channel must be an https:// URL, got {channel!r}", |
| 40 | file=sys.stderr, |
| 41 | ) |
| 42 | return |
| 43 | |
| 44 | try: |
| 45 | if parsed.hostname == "hooks.slack.com": |
| 46 | _send_slack_webhook(channel, message) |
| 47 | else: |
| 48 | _send_generic_webhook(channel, message) |
| 49 | except Exception as e: |
| 50 | # Don't fail the research run if delivery fails |
| 51 | print(f"Delivery failed: {e}", file=sys.stderr) |
| 52 | |
| 53 | |
| 54 | def _format_delivery_message(topic: str, counts: dict, mode: str) -> str: |
| 55 | """Format notification message based on delivery mode.""" |
| 56 | new = counts.get("new", 0) |
| 57 | updated = counts.get("updated", 0) |
| 58 | |
| 59 | if mode == "announce": |
| 60 | return f"📰 *last30days update: {topic}*\n{new} new, {updated} updated" |
| 61 | elif mode == "silent": |
| 62 | return f"last30days: {new} new findings for '{topic}'" |
| 63 | else: |
| 64 | return f"last30days: Research complete for '{topic}'" |
| 65 | |
| 66 | |
| 67 | def _send_slack_webhook(url: str, text: str) -> None: |
| 68 | """POST to Slack incoming webhook.""" |
| 69 | http.post(url, json_data={"text": text}, timeout=10, retries=1) |
| 70 | |
| 71 | |
| 72 | def _send_generic_webhook(url: str, text: str) -> None: |
| 73 | """POST JSON payload to generic webhook.""" |
| 74 | http.post( |
| 75 | url, |
| 76 | json_data={ |
| 77 | "message": text, |
| 78 | "source": "last30days", |
| 79 | "timestamp": time.time(), |
| 80 | }, |
| 81 | timeout=10, |
| 82 | retries=1, |
| 83 | ) |
| 84 | |
| 85 | |
| 86 | # --- Command Handlers --- |
| 87 | |
| 88 | def cmd_add(args): |
| 89 | schedule = "0 8 * * 1" if args.weekly else (args.schedule or "0 8 * * *") |
| 90 | queries = [query.strip() for query in (args.queries or "").split(",") if query.strip()] or None |
| 91 | topic = store.add_topic(args.topic, search_queries=queries, schedule=schedule) |
| 92 | sched_desc = "weekly (Mondays 8am)" if args.weekly else f"daily ({schedule})" |
| 93 | print(json.dumps({ |
| 94 | "action": "added", |
| 95 | "topic": topic["name"], |
| 96 | "schedule": sched_desc, |
| 97 | "message": f'Added "{topic["name"]}" to watchlist. Schedule: {sched_desc}.', |
| 98 | }, default=str)) |
| 99 | |
| 100 | |
| 101 | def cmd_remove(args): |
| 102 | removed = store.remove_topic(args.topic) |
| 103 | if not removed: |
| 104 | print(json.dumps({"action": "not_found", "topic": args.topic, "message": f'Topic not found: "{args.topic}"'})) |
| 105 | return |
| 106 | remaining = store.list_topics() |
| 107 | print(json.dumps({ |
| 108 | "action": "removed", |
| 109 | "topic": args.topic, |
| 110 | "message": f'Removed "{args.topic}" from watchlist.', |
| 111 | "remaining": len(remaining), |
| 112 | })) |
| 113 | |
| 114 | |
| 115 | def cmd_list(args): |
| 116 | del args |
| 117 | topics = store.list_topics() |
| 118 | budget_used = store.get_daily_cost() |
| 119 | budget_limit = float(store.get_setting("daily_budget", "5.00")) |
| 120 | print(json.dumps({ |
| 121 | "topics": topics, |
| 122 | "budget_used": budget_used, |
| 123 | "budget_limit": budget_limit, |
| 124 | }, default=str)) |
| 125 | |
| 126 | |
| 127 | def cmd_delta(args): |
| 128 | topic = store.get_topic(args.topic) |
| 129 | if not topic: |
| 130 | print(json.dumps({"error": f'Topic not found: "{args.topic}"'})) |
| 131 | sys.exit(1) |
| 132 | print(json.dumps(store.compute_topic_delta(topic["id"]), default=str)) |
| 133 | |
| 134 | |
| 135 | def cmd_run_one(args): |
| 136 | topic = store.get_topic(args.topic) |
| 137 | if not topic: |
| 138 | print(json.dumps({"error": f'Topic not found: "{args.topic}"'})) |
| 139 | sys.exit(1) |
| 140 | print(json.dumps(_run_topic(topic), default=str)) |
| 141 | |
| 142 | |
| 143 | def cmd_run_all(args): |
| 144 | del args |
| 145 | topics = [topic for topic in store.list_topics() if topic["enabled"]] |
| 146 | if not topics: |
| 147 | print(json.dumps({"message": "No enabled topics to research."})) |
| 148 | return |
| 149 | |
| 150 | budget_limit = float(store.get_setting("daily_budget", "5.00")) |
| 151 | results = [] |
| 152 | for topic in topics: |
| 153 | if store.get_daily_cost() >= budget_limit: |
| 154 | results.append({ |
| 155 | "topic": topic["name"], |
| 156 | "status": "skipped", |
| 157 | "reason": f"Budget exceeded: ${store.get_daily_cost():.2f}/${budget_limit:.2f}", |
| 158 | }) |
| 159 | continue |
| 160 | results.append(_run_topic(topic)) |
| 161 | |
| 162 | print(json.dumps({ |
| 163 | "action": "run_all", |
| 164 | "results": results, |
| 165 | "budget_used": store.get_daily_cost(), |
| 166 | "budget_limit": budget_limit, |
| 167 | }, default=str)) |
| 168 | |
| 169 | |
| 170 | def _run_topic(topic: dict) -> dict: |
| 171 | start_time = time.time() |
| 172 | topic_id = topic["id"] |
| 173 | run_id = store.record_run(topic_id, source_mode="v3", status="running") |
| 174 | |
| 175 | try: |
| 176 | search_queries = json.loads(topic["search_queries"]) if topic.get("search_queries") else None |
| 177 | search_term = search_queries[0] if search_queries else topic["name"] |
| 178 | result = subprocess.run( |
| 179 | [ |
| 180 | sys.executable, |
| 181 | str(SCRIPT_DIR / "last30days.py"), |
| 182 | search_term, |
| 183 | "--emit=json", |
| 184 | "--json-profile=raw", |
| 185 | "--quick", |
| 186 | "--lookback-days", |
| 187 | "90", |
| 188 | # Watchlist is an unattended cron host: never probe browser |
| 189 | # cookies (matches the MCP server). Avoids a silent Chromium |
| 190 | # read / unattended macOS Keychain prompt when a user has set |
| 191 | # FROM_BROWSER=auto for interactive use. |
| 192 | "--no-browser-cookies", |
| 193 | ], |
| 194 | capture_output=True, |
| 195 | text=True, |
| 196 | timeout=300, |
| 197 | ) |
| 198 | duration = time.time() - start_time |
| 199 | if result.returncode != 0: |
| 200 | store.update_run( |
| 201 | run_id, |
| 202 | status="failed", |
| 203 | error_message=result.stderr[:500], |
| 204 | duration_seconds=duration, |
| 205 | ) |
| 206 | return { |
| 207 | "topic": topic["name"], |
| 208 | "status": "failed", |
| 209 | "error": result.stderr[:200], |
| 210 | "duration": duration, |
| 211 | } |
| 212 | |
| 213 | report = schema.report_from_dict(json.loads(result.stdout)) |
| 214 | findings = store.findings_from_report(report, limit=25) |
| 215 | counts = store.store_findings(run_id, topic_id, findings) |
| 216 | store.update_run( |
| 217 | run_id, |
| 218 | status="completed", |
| 219 | duration_seconds=duration, |
| 220 | findings_new=counts["new"], |
| 221 | findings_updated=counts["updated"], |
| 222 | ) |
| 223 | |
| 224 | # Deliver webhook notification if configured |
| 225 | _deliver_findings(topic["name"], counts) |
| 226 | |
| 227 | return { |
| 228 | "topic": topic["name"], |
| 229 | "status": "completed", |
| 230 | "new": counts["new"], |
| 231 | "updated": counts["updated"], |
| 232 | "duration": duration, |
| 233 | } |
| 234 | except subprocess.TimeoutExpired: |
| 235 | duration = time.time() - start_time |
| 236 | store.update_run( |
| 237 | run_id, |
| 238 | status="failed", |
| 239 | error_message="Research timed out after 300s", |
| 240 | duration_seconds=duration, |
| 241 | ) |
| 242 | return {"topic": topic["name"], "status": "failed", "error": "timeout"} |
| 243 | except json.JSONDecodeError as exc: |
| 244 | duration = time.time() - start_time |
| 245 | store.update_run( |
| 246 | run_id, |
| 247 | status="failed", |
| 248 | error_message=f"Invalid JSON output: {exc}", |
| 249 | duration_seconds=duration, |
| 250 | ) |
| 251 | return {"topic": topic["name"], "status": "failed", "error": f"parse error: {exc}"} |
| 252 | def cmd_config(args): |
| 253 | if args.key == "budget": |
| 254 | store.set_setting("daily_budget", str(args.value)) |
| 255 | print(json.dumps({"action": "config", "key": "daily_budget", "value": str(args.value)})) |
| 256 | return |
| 257 | if args.key == "delivery": |
| 258 | value = str(args.value) |
| 259 | # Reject a non-https channel at write time so the operator gets |
| 260 | # immediate feedback, rather than discovering it via a stderr line |
| 261 | # buried in a research run hours later. Matches the delivery-time guard |
| 262 | # in _deliver_findings. |
| 263 | if value and urllib.parse.urlparse(value).scheme != "https": |
| 264 | raise SystemExit(f"delivery_channel must be an https:// URL, got {value!r}") |
| 265 | store.set_setting("delivery_channel", value) |
| 266 | print(json.dumps({"action": "config", "key": "delivery_channel", "value": value})) |
| 267 | return |
| 268 | raise SystemExit(f"Unknown config key: {args.key}") |
| 269 | |
| 270 | |
| 271 | def build_parser() -> argparse.ArgumentParser: |
| 272 | parser = argparse.ArgumentParser(description="Manage the last30days watchlist") |
| 273 | sub = parser.add_subparsers(dest="command") |
| 274 | |
| 275 | add = sub.add_parser("add") |
| 276 | add.add_argument("topic") |
| 277 | add.add_argument("--schedule") |
| 278 | add.add_argument("--weekly", action="store_true") |
| 279 | add.add_argument("--queries") |
| 280 | add.set_defaults(func=cmd_add) |
| 281 | |
| 282 | remove = sub.add_parser("remove") |
| 283 | remove.add_argument("topic") |
| 284 | remove.set_defaults(func=cmd_remove) |
| 285 | |
| 286 | list_parser = sub.add_parser("list") |
| 287 | list_parser.set_defaults(func=cmd_list) |
| 288 | |
| 289 | delta = sub.add_parser("delta") |
| 290 | delta.add_argument("topic") |
| 291 | delta.set_defaults(func=cmd_delta) |
| 292 | |
| 293 | run_one = sub.add_parser("run-one") |
| 294 | run_one.add_argument("topic") |
| 295 | run_one.set_defaults(func=cmd_run_one) |
| 296 | |
| 297 | run_all = sub.add_parser("run-all") |
| 298 | run_all.set_defaults(func=cmd_run_all) |
| 299 | |
| 300 | config = sub.add_parser("config") |
| 301 | config.add_argument("key", choices=["delivery", "budget"]) |
| 302 | config.add_argument("value") |
| 303 | config.set_defaults(func=cmd_config) |
| 304 | |
| 305 | return parser |
| 306 | |
| 307 | |
| 308 | def main() -> int: |
| 309 | parser = build_parser() |
| 310 | args = parser.parse_args() |
| 311 | if not getattr(args, "command", None): |
| 312 | parser.print_help() |
| 313 | return 1 |
| 314 | args.func(args) |
| 315 | return 0 |
| 316 | |
| 317 | |
| 318 | if __name__ == "__main__": |
| 319 | raise SystemExit(main()) |
| 320 |