| 1 |
"""Environment and API key management for last30days skill.""" |
| 2 |
|
| 3 |
from __future__ import annotations |
| 4 |
|
| 5 |
import datetime |
| 6 |
import json |
| 7 |
import locale |
| 8 |
import os |
| 9 |
import re |
| 10 |
import sys |
| 11 |
from dataclasses import dataclass |
| 12 |
from pathlib import Path |
| 13 |
from typing import Any, Literal |
| 14 |
|
| 15 |
|
| 16 |
def read_secret_env(name: str, default: str | None = None) -> str | None: |
| 17 |
"""Read a possibly-secret environment variable by name. |
| 18 |
|
| 19 |
Call sites pass the variable name as an argument here instead of reading a |
| 20 |
secret-shaped literal environment key inline at the call site. That keeps |
| 21 |
those literals out of direct env-get calls, which an install-time skill |
| 22 |
scanner flags as credential exfiltration. Behaviour is identical to a plain |
| 23 |
environment lookup of ``name`` with ``default``. |
| 24 |
""" |
| 25 |
return os.environ.get(name, default) |
| 26 |
|
| 27 |
|
| 28 |
# Allow override via environment variable for testing |
| 29 |
# Set LAST30DAYS_CONFIG_DIR="" for clean/no-config mode |
| 30 |
# Set LAST30DAYS_CONFIG_DIR="/path/to/dir" for custom config location |
| 31 |
_config_override = os.environ.get('LAST30DAYS_CONFIG_DIR') |
| 32 |
if _config_override == "": |
| 33 |
# Empty string = no config file (clean mode) |
| 34 |
CONFIG_DIR = None |
| 35 |
CONFIG_FILE = None |
| 36 |
elif _config_override: |
| 37 |
CONFIG_DIR = Path(_config_override) |
| 38 |
CONFIG_FILE = CONFIG_DIR / ".env" |
| 39 |
else: |
| 40 |
CONFIG_DIR = Path.home() / ".config" / "last30days" |
| 41 |
CONFIG_FILE = CONFIG_DIR / ".env" |
| 42 |
|
| 43 |
# macOS Keychain integration: items stored with this service prefix are picked |
| 44 |
# up automatically on Darwin as the lowest-priority credential source. |
| 45 |
# Example: `security add-generic-password -a "$USER" -s last30days-XAI_API_KEY -w "xai-..."`. |
| 46 |
KEYCHAIN_SERVICE_PREFIX = "last30days-" |
| 47 |
|
| 48 |
# Optional non-secret aliases for users who already store API keys under a |
| 49 |
# different Keychain naming convention. Configure as JSON in |
| 50 |
# LAST30DAYS_KEYCHAIN_ALIASES, for example: |
| 51 |
# {"XAI_API_KEY":{"account":"keychain-user","service":"existing-xai-api-key"}} |
| 52 |
# A string value is shorthand for {"service": "..."} with the current user. |
| 53 |
KEYCHAIN_ALIASES_ENV = "LAST30DAYS_KEYCHAIN_ALIASES" |
| 54 |
|
| 55 |
# Opt-out switch for the Keychain source. Set truthy to make _load_keychain a |
| 56 |
# no-op on Darwin too. Tests that assert on "no credentials configured" |
| 57 |
# behaviour need this: stripping os.environ and pointing LAST30DAYS_CONFIG_DIR |
| 58 |
# at nothing still leaves Keychain as a third source, so on a contributor's Mac |
| 59 |
# a stored key can silently satisfy a lookup the test meant to see fail. |
| 60 |
KEYCHAIN_DISABLE_ENV = "LAST30DAYS_SKIP_KEYCHAIN" |
| 61 |
|
| 62 |
# Single source of truth for which credentials the Keychain loader looks up. |
| 63 |
# The setup-keychain.sh helper mirrors this list and is held in sync via |
| 64 |
# tests/test_env_keychain.py::test_keychain_keys_match_setup_script. |
| 65 |
KEYCHAIN_KEYS = ( |
| 66 |
"OPENAI_API_KEY", "XAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", |
| 67 |
"GOOGLE_GENAI_API_KEY", "SCRAPECREATORS_API_KEY", "APIFY_API_TOKEN", |
| 68 |
"AUTH_TOKEN", "CT0", "BSKY_HANDLE", "BSKY_APP_PASSWORD", |
| 69 |
"TRUTHSOCIAL_TOKEN", "BRAVE_API_KEY", "EXA_API_KEY", "SERPER_API_KEY", |
| 70 |
"OPENROUTER_API_KEY", "PERPLEXITY_API_KEY", "PARALLEL_API_KEY", "XQUIK_API_KEY", |
| 71 |
"XIAOHONGSHU_API_BASE", "GITHUB_TOKEN", "BRIGHTDATA_API_KEY", |
| 72 |
"X_BEARER_TOKEN", |
| 73 |
) |
| 74 |
|
| 75 |
# pass(1) integration: Linux/Unix analog of the Keychain source. Each key in |
| 76 |
# KEYCHAIN_KEYS is looked up at pass path f"{prefix}{KEY}", the direct analog of |
| 77 |
# Keychain's "last30days-<KEY>" service-name convention, so any user stores keys |
| 78 |
# under one namespace without editing code. The prefix is resolved at call time |
| 79 |
# (in get_config) from LAST30DAYS_PASS_PREFIX in the process env or a config |
| 80 |
# file, falling back to this default; included verbatim, so keep the trailing |
| 81 |
# separator. Honors PASSWORD_STORE_DIR. |
| 82 |
DEFAULT_PASS_PATH_PREFIX = "last30days/" |
| 83 |
|
| 84 |
AuthSource = Literal["api_key", "none"] |
| 85 |
AuthStatus = Literal["ok", "missing"] |
| 86 |
|
| 87 |
AUTH_SOURCE_API_KEY: AuthSource = "api_key" |
| 88 |
AUTH_SOURCE_NONE: AuthSource = "none" |
| 89 |
|
| 90 |
AUTH_STATUS_OK: AuthStatus = "ok" |
| 91 |
AUTH_STATUS_MISSING: AuthStatus = "missing" |
| 92 |
|
| 93 |
XIAOHONGSHU_DEFAULT_API_BASES = ( |
| 94 |
"http://localhost:18060", |
| 95 |
"http://host.docker.internal:18060", |
| 96 |
) |
| 97 |
XIAOHONGSHU_RESOLVED_API_BASE_KEY = "_XIAOHONGSHU_API_BASE_RESOLVED" |
| 98 |
|
| 99 |
|
| 100 |
@dataclass(frozen=True) |
| 101 |
class OpenAIAuth: |
| 102 |
token: str | None |
| 103 |
source: AuthSource |
| 104 |
status: AuthStatus |
| 105 |
|
| 106 |
|
| 107 |
BrowserCookieMode = Literal["off", "read", "plan_only"] |
| 108 |
|
| 109 |
|
| 110 |
@dataclass(frozen=True) |
| 111 |
class ConfigLoadPolicy: |
| 112 |
"""Local-read gates for configuration loading. |
| 113 |
|
| 114 |
Bare library calls use the safe default: no browser-cookie extraction and no |
| 115 |
project-scoped config. CLI entry points can opt into narrower behavior after |
| 116 |
parsing command intent. |
| 117 |
""" |
| 118 |
|
| 119 |
browser_cookies: BrowserCookieMode = "off" |
| 120 |
allow_project_config: bool = False |
| 121 |
inspect_ignored_project_config: bool = False |
| 122 |
|
| 123 |
|
| 124 |
def _truthy(value: Any) -> bool: |
| 125 |
if value is None: |
| 126 |
return False |
| 127 |
return str(value).strip().lower() in {"1", "true", "yes", "on"} |
| 128 |
|
| 129 |
|
| 130 |
# A Claude Desktop extension maps every unset field in its config modal to the |
| 131 |
# literal string ``${user_config.<field>}`` in the engine's environment. The |
| 132 |
# placeholder is non-empty, so a presence check reads it as a real credential: |
| 133 |
# doctor reports the source healthy, preflight returns ready, and the backend |
| 134 |
# sends the literal placeholder upstream and surfaces the vendor's 401 instead |
| 135 |
# of falling back. Two constraints keep legitimate values out of scope. The |
| 136 |
# match is anchored to the whole trimmed value, so a real credential containing |
| 137 |
# ``$`` or braces is untouched. And the field name is restricted to the |
| 138 |
# identifier charset the manifest uses, so shell-default syntax is not mistaken |
| 139 |
# for a placeholder - both the generic form a user may paste into ``.env`` |
| 140 |
# (``${VAR:-default}``) and the namespaced form with a default |
| 141 |
# (``${user_config.x:-default}``). Only the extension namespace, as issue |
| 142 |
# #1081's own suggested fix names, is rejected. |
| 143 |
_UNSUBSTITUTED_TEMPLATE = re.compile(r"^\$\{user_config\.[A-Za-z0-9_]+\}$") |
| 144 |
|
| 145 |
# Config-record key holding the names of values rejected above, so diagnostics |
| 146 |
# report the templated state instead of silently counting the key absent. |
| 147 |
TEMPLATE_CONFIG_KEYS = "_TEMPLATE_CONFIG_KEYS" |
| 148 |
|
| 149 |
|
| 150 |
def is_unsubstituted_template(value: Any) -> bool: |
| 151 |
"""True when ``value`` is a whole, unexpanded ``${user_config.*}`` placeholder.""" |
| 152 |
if not isinstance(value, str): |
| 153 |
return False |
| 154 |
return bool(_UNSUBSTITUTED_TEMPLATE.match(value.strip())) |
| 155 |
|
| 156 |
|
| 157 |
def templated_config_keys(config: dict[str, Any]) -> list[str]: |
| 158 |
"""Public view of the keys ``get_config()`` rejected as unsubstituted templates. |
| 159 |
|
| 160 |
Thin reader so diagnostics report the templated state without re-deriving |
| 161 |
the record key, in the same spirit as ``include_sources`` and |
| 162 |
``is_setup_complete``. Sorted here too: these call sites also see hand-built |
| 163 |
configs, and the order is user-visible in both diagnostics. |
| 164 |
""" |
| 165 |
return sorted(config.get(TEMPLATE_CONFIG_KEYS) or []) |
| 166 |
|
| 167 |
|
| 168 |
def _rotate_scrapecreators_key(config: dict[str, Any]) -> None: |
| 169 |
"""Round-robin a comma-separated SCRAPECREATORS_API_KEY to one key per run. |
| 170 |
|
| 171 |
Extracted so the placeholder sweep can reapply it: the sweep may restore a |
| 172 |
value from a lower-priority source after the ordinary rotation already ran, |
| 173 |
and a comma-separated list handed to a backend whole fails authentication. |
| 174 |
A second call on an already-rotated value is a no-op (no comma remains). |
| 175 |
""" |
| 176 |
raw = config.get('SCRAPECREATORS_API_KEY') or '' |
| 177 |
if ',' not in raw: |
| 178 |
return |
| 179 |
import random |
| 180 |
sc_keys = [k.strip() for k in raw.split(',') if k.strip()] |
| 181 |
config['SCRAPECREATORS_API_KEY'] = random.choice(sc_keys) if sc_keys else '' |
| 182 |
|
| 183 |
|
| 184 |
def is_timestamp_fresh(timestamp_value: Any, ttl_seconds: int) -> bool: |
| 185 |
"""True when ``timestamp_value`` (ISO-8601 string) is within ``ttl_seconds``. |
| 186 |
|
| 187 |
Shared freshness gate for the doctor cache and the report cache. The guard |
| 188 |
order is load-bearing: a non-positive TTL disables caching entirely, a |
| 189 |
non-string or empty timestamp is stale, a malformed timestamp is stale, |
| 190 |
naive timestamps are treated as UTC, and a future timestamp (negative age) |
| 191 |
counts as fresh. |
| 192 |
""" |
| 193 |
if ttl_seconds <= 0: |
| 194 |
return False |
| 195 |
if not isinstance(timestamp_value, str) or not timestamp_value: |
| 196 |
return False |
| 197 |
try: |
| 198 |
created_at = datetime.datetime.fromisoformat(timestamp_value) |
| 199 |
except ValueError: |
| 200 |
return False |
| 201 |
if created_at.tzinfo is None: |
| 202 |
created_at = created_at.replace(tzinfo=datetime.timezone.utc) |
| 203 |
age = datetime.datetime.now(datetime.timezone.utc) - created_at.astimezone( |
| 204 |
datetime.timezone.utc |
| 205 |
) |
| 206 |
return age.total_seconds() <= ttl_seconds |
| 207 |
|
| 208 |
|
| 209 |
def _project_config_trusted(policy: ConfigLoadPolicy, file_env: dict[str, Any]) -> bool: |
| 210 |
if policy.allow_project_config: |
| 211 |
return True |
| 212 |
process_value = os.environ.get("LAST30DAYS_TRUST_PROJECT_CONFIG") |
| 213 |
if process_value is not None: |
| 214 |
return _truthy(process_value) |
| 215 |
return _truthy(file_env.get("LAST30DAYS_TRUST_PROJECT_CONFIG")) |
| 216 |
|
| 217 |
|
| 218 |
def _check_file_permissions(path: Path) -> None: |
| 219 |
"""Warn to stderr if a secrets file has overly permissive permissions.""" |
| 220 |
if os.name == "nt": |
| 221 |
# Windows reports synthesized POSIX mode bits that do not reflect NTFS ACLs. |
| 222 |
return |
| 223 |
|
| 224 |
try: |
| 225 |
mode = path.stat().st_mode |
| 226 |
# Check if group or other can read (bits 0o044) |
| 227 |
if mode & 0o044: |
| 228 |
sys.stderr.write( |
| 229 |
f"[last30days] WARNING: {path} is readable by other users. " |
| 230 |
f"Run: chmod 600 {path}\n" |
| 231 |
) |
| 232 |
sys.stderr.flush() |
| 233 |
except OSError as exc: |
| 234 |
sys.stderr.write(f"[last30days] WARNING: could not stat {path}: {exc}\n") |
| 235 |
sys.stderr.flush() |
| 236 |
|
| 237 |
|
| 238 |
def _strip_inline_comment(value: str) -> str: |
| 239 |
"""Drop a trailing ``# comment`` from the right-hand side of a KEY=value line. |
| 240 |
|
| 241 |
Unquoted: ``#`` opens a comment only as the first non-blank character or |
| 242 |
when preceded by whitespace, so ``value#nothash`` stays intact. Quoted: |
| 243 |
everything up to the matching close quote is kept verbatim; only a |
| 244 |
whitespace-separated ``#`` after the close quote is dropped. Anything that |
| 245 |
does not match those shapes is returned unchanged for the existing quote |
| 246 |
handling to deal with. |
| 247 |
""" |
| 248 |
stripped = value.lstrip() |
| 249 |
if stripped[:1] in ('"', "'"): |
| 250 |
end = stripped.find(stripped[0], 1) |
| 251 |
if end == -1: |
| 252 |
return value |
| 253 |
rest = stripped[end + 1:] |
| 254 |
if rest[:1].isspace() and rest.lstrip().startswith('#'): |
| 255 |
return stripped[:end + 1] |
| 256 |
return value |
| 257 |
match = re.search(r'(?:^|\s)#', stripped) |
| 258 |
if match: |
| 259 |
return stripped[:match.start()] |
| 260 |
return value |
| 261 |
|
| 262 |
|
| 263 |
def load_env_file(path: Path) -> dict[str, str]: |
| 264 |
"""Load environment variables from a file.""" |
| 265 |
env = {} |
| 266 |
if not path or not path.exists(): |
| 267 |
return env |
| 268 |
_check_file_permissions(path) |
| 269 |
|
| 270 |
# Prefer UTF-8 (utf-8-sig transparently strips a BOM written by Windows |
| 271 |
# editors like Notepad). Fall back to the locale decoder for a genuinely |
| 272 |
# locale-encoded .env (e.g. cp1252) so an existing file that loaded before |
| 273 |
# keeps loading. If it decodes as neither, let UnicodeDecodeError surface |
| 274 |
# rather than corrupting keys/secrets with replacement characters. |
| 275 |
try: |
| 276 |
text = path.read_text(encoding='utf-8-sig') |
| 277 |
except UnicodeDecodeError: |
| 278 |
text = path.read_text(encoding=locale.getpreferredencoding(False)) |
| 279 |
|
| 280 |
for line in text.splitlines(): |
| 281 |
line = line.strip() |
| 282 |
if not line or line.startswith('#'): |
| 283 |
continue |
| 284 |
if '=' in line: |
| 285 |
key, _, value = line.partition('=') |
| 286 |
key = key.strip() |
| 287 |
value = _strip_inline_comment(value).strip() |
| 288 |
# Remove quotes if present |
| 289 |
if value and value[0] in ('"', "'") and value[-1] == value[0]: |
| 290 |
value = value[1:-1] |
| 291 |
# These settings use empty as a persisted disable; secrets still |
| 292 |
# drop blanks instead of overriding a configured credential. |
| 293 |
if key and (value or key in {'LAST30DAYS_YT_PLAYER_CLIENT', 'LAST30DAYS_MEMORY_DIR'}): |
| 294 |
env.update({key: value}) |
| 295 |
return env |
| 296 |
|
| 297 |
|
| 298 |
def _parse_keychain_aliases(raw: str | None) -> dict[str, list[dict[str, str]]]: |
| 299 |
"""Parse non-secret Keychain alias metadata from JSON. |
| 300 |
|
| 301 |
Supported forms: |
| 302 |
{"XAI_API_KEY": "existing-xai-api-key"} |
| 303 |
{"XAI_API_KEY": {"service": "existing-xai-api-key", "account": "keychain-user"}} |
| 304 |
{"XAI_API_KEY": [{"service": "primary"}, {"service": "fallback"}]} |
| 305 |
|
| 306 |
Invalid entries are ignored so a typo never blocks canonical |
| 307 |
`last30days-<KEY>` lookups; malformed JSON emits a warning. |
| 308 |
""" |
| 309 |
if not raw: |
| 310 |
return {} |
| 311 |
try: |
| 312 |
parsed = json.loads(raw) |
| 313 |
except json.JSONDecodeError as exc: |
| 314 |
sys.stderr.write( |
| 315 |
f"[last30days] WARNING: {KEYCHAIN_ALIASES_ENV} is not valid JSON; " |
| 316 |
f"ignoring Keychain aliases while keeping canonical lookups enabled: {exc}\n" |
| 317 |
) |
| 318 |
sys.stderr.flush() |
| 319 |
return {} |
| 320 |
if not isinstance(parsed, dict): |
| 321 |
return {} |
| 322 |
|
| 323 |
allowed = set(KEYCHAIN_KEYS) |
| 324 |
aliases: dict[str, list[dict[str, str]]] = {} |
| 325 |
for key, spec in parsed.items(): |
| 326 |
if key not in allowed: |
| 327 |
continue |
| 328 |
specs = spec if isinstance(spec, list) else [spec] |
| 329 |
clean_specs: list[dict[str, str]] = [] |
| 330 |
for item in specs: |
| 331 |
if isinstance(item, str): |
| 332 |
service = item.strip() |
| 333 |
account = "" |
| 334 |
elif isinstance(item, dict): |
| 335 |
service = str(item.get("service", "")).strip() |
| 336 |
account = str(item.get("account", "")).strip() |
| 337 |
else: |
| 338 |
continue |
| 339 |
if service: |
| 340 |
clean_specs.append({"service": service, "account": account}) |
| 341 |
if clean_specs: |
| 342 |
aliases[key] = clean_specs |
| 343 |
return aliases |
| 344 |
|
| 345 |
|
| 346 |
def _load_keychain(keys: list[str], aliases: dict[str, list[dict[str, str]]] | None = None) -> dict[str, str]: |
| 347 |
"""Load credentials from macOS Keychain (no-op on other platforms). |
| 348 |
|
| 349 |
Each key is looked up as a generic password with service name |
| 350 |
``f"{KEYCHAIN_SERVICE_PREFIX}{key}"`` for the current user. Missing items |
| 351 |
then fall back to optional alias metadata from |
| 352 |
``LAST30DAYS_KEYCHAIN_ALIASES``. Lookup failures are silent — Keychain is |
| 353 |
the lowest-priority source and is meant to be additive over `.env` files |
| 354 |
and process environment. |
| 355 |
|
| 356 |
Set ``LAST30DAYS_SKIP_KEYCHAIN`` truthy to disable the source entirely. It |
| 357 |
is read from the process environment only, never from a config file: it |
| 358 |
gates a credential source that is consulted *while* the config is being |
| 359 |
assembled, so a file-sourced value would be read too late to have any |
| 360 |
effect. |
| 361 |
""" |
| 362 |
if _truthy(os.environ.get(KEYCHAIN_DISABLE_ENV)): |
| 363 |
return {} |
| 364 |
|
| 365 |
import platform |
| 366 |
if platform.system() != "Darwin": |
| 367 |
return {} |
| 368 |
|
| 369 |
import shutil |
| 370 |
security = shutil.which("security") |
| 371 |
if not security: |
| 372 |
return {} |
| 373 |
|
| 374 |
import subprocess |
| 375 |
# USER can be unset under sudo, in Docker without --env USER, or in some CI |
| 376 |
# runners; fall back to the OS user record so lookups still match items |
| 377 |
# stored by setup-keychain.sh (which uses $USER). |
| 378 |
user = os.environ.get("USER") |
| 379 |
if not user: |
| 380 |
try: |
| 381 |
import pwd |
| 382 |
except ImportError: |
| 383 |
pwd = None |
| 384 |
|
| 385 |
if pwd is not None: |
| 386 |
try: |
| 387 |
user = pwd.getpwuid(os.getuid()).pw_name |
| 388 |
except AttributeError: |
| 389 |
user = "unknown" |
| 390 |
else: |
| 391 |
user = "unknown" |
| 392 |
env: dict[str, str] = {} |
| 393 |
|
| 394 |
def lookup(account: str, service: str) -> str: |
| 395 |
try: |
| 396 |
result = subprocess.run( |
| 397 |
[security, "find-generic-password", |
| 398 |
"-a", account, |
| 399 |
"-s", service, |
| 400 |
"-w"], |
| 401 |
capture_output=True, text=True, timeout=5, |
| 402 |
) |
| 403 |
except (subprocess.TimeoutExpired, OSError): |
| 404 |
return "" |
| 405 |
if result.returncode == 0 and result.stdout.strip(): |
| 406 |
return result.stdout.strip() |
| 407 |
return "" |
| 408 |
|
| 409 |
for key in keys: |
| 410 |
value = lookup(user, f"{KEYCHAIN_SERVICE_PREFIX}{key}") |
| 411 |
if not value and aliases: |
| 412 |
for alias in aliases.get(key, []): |
| 413 |
alias_account = alias.get("account") or user |
| 414 |
value = lookup(alias_account, alias["service"]) |
| 415 |
if value: |
| 416 |
break |
| 417 |
if value: |
| 418 |
env.update({key: value}) |
| 419 |
return env |
| 420 |
|
| 421 |
|
| 422 |
def _load_pass(keys: list[str], prefix: str) -> dict[str, str]: |
| 423 |
"""Load credentials from a pass(1) store (no-op if `pass` is absent). |
| 424 |
|
| 425 |
The Linux/Unix analog of the macOS Keychain source. Each env-var name is |
| 426 |
looked up at pass path ``f"{prefix}{key}"`` — mirroring Keychain's |
| 427 |
``last30days-<key>`` service-name convention — so any user stores keys under |
| 428 |
that namespace without editing code (prefix overridable via |
| 429 |
``LAST30DAYS_PASS_PREFIX``). The secret is decrypted in a subprocess and |
| 430 |
read from stdout's first line (pass keeps the secret there; any metadata |
| 431 |
follows) — never written to disk, never logged. Honors ``PASSWORD_STORE_DIR``. |
| 432 |
Missing entries and failures are silent: pass is a lowest-priority, additive |
| 433 |
source like Keychain, so an explicit .env or process-env value still wins. |
| 434 |
""" |
| 435 |
import shutil |
| 436 |
pass_bin = shutil.which("pass") |
| 437 |
if not pass_bin: |
| 438 |
return {} |
| 439 |
|
| 440 |
import subprocess |
| 441 |
env: dict[str, str] = {} |
| 442 |
for key in keys: |
| 443 |
try: |
| 444 |
result = subprocess.run( |
| 445 |
[pass_bin, "show", f"{prefix}{key}"], |
| 446 |
capture_output=True, text=True, timeout=5, |
| 447 |
encoding="utf-8", errors="replace", |
| 448 |
) |
| 449 |
except (subprocess.TimeoutExpired, OSError): |
| 450 |
# A timeout (GPG/pinentry hanging) or exec failure isn't a per-key |
| 451 |
# condition — it means the store is unusable right now. Stop instead |
| 452 |
# of paying the timeout once per key; otherwise a locked store would |
| 453 |
# stall every config load by 5s x len(keys). A genuinely missing key |
| 454 |
# returns fast with a non-zero exit and is handled below. |
| 455 |
break |
| 456 |
if result.returncode == 0 and result.stdout.strip(): |
| 457 |
env.update({key: result.stdout.strip().splitlines()[0]}) |
| 458 |
return env |
| 459 |
|
| 460 |
|
| 461 |
def get_openai_auth(file_env: dict[str, str]) -> OpenAIAuth: |
| 462 |
"""Resolve OpenAI API auth from explicit user-provided API keys.""" |
| 463 |
api_key = read_secret_env('OPENAI_API_KEY') or file_env.get('OPENAI_API_KEY') |
| 464 |
if api_key: |
| 465 |
return OpenAIAuth( |
| 466 |
token=api_key, |
| 467 |
source=AUTH_SOURCE_API_KEY, |
| 468 |
status=AUTH_STATUS_OK, |
| 469 |
) |
| 470 |
|
| 471 |
return OpenAIAuth( |
| 472 |
token=None, |
| 473 |
source=AUTH_SOURCE_NONE, |
| 474 |
status=AUTH_STATUS_MISSING, |
| 475 |
) |
| 476 |
|
| 477 |
|
| 478 |
def _find_project_env() -> Path | None: |
| 479 |
"""Find per-project .env by walking up from cwd. |
| 480 |
|
| 481 |
Searches for .claude/last30days.env in each parent directory, |
| 482 |
stopping at the git root, user's home directory, or filesystem root. |
| 483 |
""" |
| 484 |
cwd = Path.cwd() |
| 485 |
for parent in [cwd, *cwd.parents]: |
| 486 |
candidate = parent / '.claude' / 'last30days.env' |
| 487 |
if candidate.exists(): |
| 488 |
return candidate |
| 489 |
if (parent / ".git").exists(): |
| 490 |
break |
| 491 |
# Stop at filesystem root or home |
| 492 |
if parent == Path.home() or parent == parent.parent: |
| 493 |
break |
| 494 |
return None |
| 495 |
|
| 496 |
|
| 497 |
def _configured_memory_dir(*values: str | None) -> str | None: |
| 498 |
for value in values: |
| 499 |
if value is not None and not is_unsubstituted_template(value): |
| 500 |
return value |
| 501 |
return None |
| 502 |
|
| 503 |
|
| 504 |
def resolve_memory_dir(save_dir: str | None = None) -> str: |
| 505 |
"""Resolve the skill's save directory without credential-store access.""" |
| 506 |
value = save_dir |
| 507 |
if value is None: |
| 508 |
value = _configured_memory_dir(os.environ.get("LAST30DAYS_MEMORY_DIR")) |
| 509 |
if value is None: |
| 510 |
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {} |
| 511 |
project_env = {} |
| 512 |
if _project_config_trusted(ConfigLoadPolicy(), file_env): |
| 513 |
project_path = _find_project_env() |
| 514 |
if project_path: |
| 515 |
project_env = load_env_file(project_path) |
| 516 |
value = _configured_memory_dir( |
| 517 |
project_env.get("LAST30DAYS_MEMORY_DIR"), |
| 518 |
file_env.get("LAST30DAYS_MEMORY_DIR"), |
| 519 |
) |
| 520 |
if value is None: |
| 521 |
value = str(Path.home() / "Documents" / "Last30Days") |
| 522 |
return str(Path(value).expanduser().absolute()) if value else "" |
| 523 |
|
| 524 |
|
| 525 |
def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]: |
| 526 |
"""Load configuration from multiple sources. |
| 527 |
|
| 528 |
Priority (highest wins): |
| 529 |
1. Environment variables (os.environ) |
| 530 |
2. Trusted .claude/last30days.env (per-project config) |
| 531 |
3. ~/.config/last30days/.env (global config) |
| 532 |
4. macOS Keychain items prefixed ``last30days-`` (Darwin only) |
| 533 |
""" |
| 534 |
policy = policy or ConfigLoadPolicy() |
| 535 |
# Load from global config file |
| 536 |
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE else {} |
| 537 |
|
| 538 |
# Load per-project config only when trust comes from process env, global |
| 539 |
# user config, or an explicit policy. A project file cannot grant trust to |
| 540 |
# itself because it is not parsed until after this decision. |
| 541 |
project_config_trusted = _project_config_trusted(policy, file_env) |
| 542 |
project_env_path = _find_project_env() if project_config_trusted else None |
| 543 |
project_env = load_env_file(project_env_path) if project_env_path else {} |
| 544 |
ignored_project_env_path = None |
| 545 |
ignored_project_keys: list[str] = [] |
| 546 |
if not project_config_trusted and policy.inspect_ignored_project_config: |
| 547 |
ignored_project_env_path = _find_project_env() |
| 548 |
if ignored_project_env_path: |
| 549 |
ignored_project_keys = sorted(load_env_file(ignored_project_env_path).keys()) |
| 550 |
|
| 551 |
# Merge file sources: project > global |
| 552 |
merged_env = {**file_env, **project_env} |
| 553 |
|
| 554 |
# Keychain is the lowest-priority source (Darwin only; no-op elsewhere). |
| 555 |
# Loaded before openai_auth so OPENAI_API_KEY can come from Keychain too. |
| 556 |
keychain_aliases_raw = os.environ.get(KEYCHAIN_ALIASES_ENV) or merged_env.get(KEYCHAIN_ALIASES_ENV) |
| 557 |
keychain_aliases = _parse_keychain_aliases(keychain_aliases_raw) |
| 558 |
keychain_env = _load_keychain(list(KEYCHAIN_KEYS), keychain_aliases) |
| 559 |
merged_env = {**keychain_env, **merged_env} |
| 560 |
# pass(1) store: Linux/Unix analog of Keychain at convention path |
| 561 |
# {prefix}<KEY>. Decrypts transiently so secrets stay encrypted at rest (no |
| 562 |
# plaintext .env). Lowest priority: Keychain, the config files, and process |
| 563 |
# env all win over it. Two efficiency guards so a user who merely has `pass` |
| 564 |
# on PATH doesn't pay for it: resolve the prefix from the loaded config/env |
| 565 |
# (not import time, so a .env-set LAST30DAYS_PASS_PREFIX is honored), and |
| 566 |
# probe ONLY keys still unset after the higher-priority sources — an empty |
| 567 |
# list short-circuits with no gpg/pinentry calls at all. |
| 568 |
pass_prefix = ( |
| 569 |
os.environ.get("LAST30DAYS_PASS_PREFIX") |
| 570 |
or merged_env.get("LAST30DAYS_PASS_PREFIX") |
| 571 |
or DEFAULT_PASS_PATH_PREFIX |
| 572 |
) |
| 573 |
pass_missing = [k for k in KEYCHAIN_KEYS if k not in os.environ and not merged_env.get(k)] |
| 574 |
pass_env = _load_pass(pass_missing, pass_prefix) |
| 575 |
merged_env = {**pass_env, **merged_env} |
| 576 |
|
| 577 |
openai_auth = get_openai_auth(merged_env) |
| 578 |
|
| 579 |
# Build config: Codex/OpenAI auth + process.env > project .env > global .env |
| 580 |
config = { |
| 581 |
'OPENAI_API_KEY': openai_auth.token, |
| 582 |
'OPENAI_AUTH_SOURCE': openai_auth.source, |
| 583 |
'OPENAI_AUTH_STATUS': openai_auth.status, |
| 584 |
} |
| 585 |
|
| 586 |
keys = [ |
| 587 |
# Debug flag; also exported to os.environ below so log.py's lazy |
| 588 |
# os.environ.get() picks up .env values after get_config() runs. |
| 589 |
('LAST30DAYS_DEBUG', None), |
| 590 |
('XAI_API_KEY', None), |
| 591 |
('GOOGLE_API_KEY', None), |
| 592 |
('GEMINI_API_KEY', None), |
| 593 |
('GOOGLE_GENAI_API_KEY', None), |
| 594 |
('XIAOHONGSHU_API_BASE', None), |
| 595 |
('LAST30DAYS_REASONING_PROVIDER', 'auto'), |
| 596 |
('LAST30DAYS_PLANNER_MODEL', None), |
| 597 |
('LAST30DAYS_RERANK_MODEL', None), |
| 598 |
('LAST30DAYS_X_MODEL', None), |
| 599 |
('LAST30DAYS_X_BACKEND', None), |
| 600 |
('LAST30DAYS_REDDIT_BACKEND', None), |
| 601 |
# Keyless reddit.com token-bucket rate (req/sec). http.py reads it |
| 602 |
# from os.environ on each acquire, so .env values are exported below. |
| 603 |
('LAST30DAYS_REDDIT_KEYLESS_RATE', None), |
| 604 |
# Doctor cache freshness window in seconds (doctor --cached). |
| 605 |
('LAST30DAYS_DOCTOR_TTL', None), |
| 606 |
# Per-source deadline (seconds) for doctor --probe live checks. |
| 607 |
('LAST30DAYS_DOCTOR_PROBE_TIMEOUT', None), |
| 608 |
('LAST30DAYS_REDDIT_SC_MIN_ITEMS', None), |
| 609 |
('LAST30DAYS_STORE', None), |
| 610 |
# Discovery topic queue (podcast/X-article pipeline memory). Default |
| 611 |
# ON; the literal value "off" disables queue writes and annotations. |
| 612 |
('LAST30DAYS_DISCOVERY_QUEUE', None), |
| 613 |
# Wall-clock budget (seconds) for the deep-tier enrichment batch on |
| 614 |
# the discovery resume leg (--discover --judgments). Read from the |
| 615 |
# resolved config only (pipeline._resume_enrich_budget_seconds); |
| 616 |
# unset/invalid falls back to 450s. The one-shot --discover path |
| 617 |
# keeps its fixed 240s quick budget regardless. |
| 618 |
('LAST30DAYS_ENRICH_BUDGET_SECONDS', None), |
| 619 |
# Opt-in strict exit: truthy -> CLI exits 3 when any source outcome is |
| 620 |
# degraded (neither ok, no-results, nor skipped-unconfigured). #384. |
| 621 |
('LAST30DAYS_STRICT_EXIT', None), |
| 622 |
('LAST30DAYS_MEMORY_DIR', None), |
| 623 |
# Optional local-only evidence source. Paths are separated with the |
| 624 |
# platform path separator (":" on macOS/Linux, ";" on Windows). |
| 625 |
('LAST30DAYS_CORPUS_DIRS', None), |
| 626 |
# Corpus evidence is omitted from the stable agent JSON export unless |
| 627 |
# this explicit privacy opt-in is truthy. |
| 628 |
('LAST30DAYS_CORPUS_IN_EXPORT', None), |
| 629 |
('LAST30DAYS_LIBRARY_OWNER', None), |
| 630 |
('LAST30DAYS_LIBRARY_CONTEXT', 'on'), |
| 631 |
('LAST30DAYS_PUBLISH_PASSWORD', None), |
| 632 |
('OPENAI_MODEL_PIN', None), |
| 633 |
('XAI_MODEL_PIN', None), |
| 634 |
('OPENAI_BASE_URL', None), |
| 635 |
('XAI_BASE_URL', None), |
| 636 |
('OPENROUTER_BASE_URL', None), |
| 637 |
('SCRAPECREATORS_API_KEY', None), |
| 638 |
('APIFY_API_TOKEN', None), |
| 639 |
('AUTH_TOKEN', None), |
| 640 |
('CT0', None), |
| 641 |
('BSKY_HANDLE', None), |
| 642 |
('BSKY_APP_PASSWORD', None), |
| 643 |
('BSKY_SEARCH_HOST', None), |
| 644 |
('TRUTHSOCIAL_TOKEN', None), |
| 645 |
('BRAVE_API_KEY', None), |
| 646 |
('EXA_API_KEY', None), |
| 647 |
('SERPER_API_KEY', None), |
| 648 |
('OPENROUTER_API_KEY', None), |
| 649 |
('PERPLEXITY_API_KEY', None), |
| 650 |
('LAST30DAYS_PERPLEXITY_MODE', 'agent'), |
| 651 |
# Legacy Sonar setting. Retain it during migration so existing env |
| 652 |
# files load, but the Agent adapter does not map it to a dynamic preset. |
| 653 |
('LAST30DAYS_PERPLEXITY_MODEL', None), |
| 654 |
('LAST30DAYS_PERPLEXITY_AGENT_MODEL', None), |
| 655 |
('LAST30DAYS_PERPLEXITY_AGENT_PRESET', None), |
| 656 |
('LAST30DAYS_PERPLEXITY_AGENT_MAX_STEPS', None), |
| 657 |
('LAST30DAYS_PERPLEXITY_AGENT_MAX_OUTPUT_TOKENS', None), |
| 658 |
('LAST30DAYS_PERPLEXITY_AGENT_TIMEOUT_SECONDS', '120'), |
| 659 |
('LAST30DAYS_PERPLEXITY_MAX_RESULTS', None), |
| 660 |
('LAST30DAYS_PERPLEXITY_SEARCH_CONTEXT_SIZE', None), |
| 661 |
('LAST30DAYS_PERPLEXITY_SEARCH_TYPE', None), |
| 662 |
('LAST30DAYS_PERPLEXITY_SEARCH_MODE', None), |
| 663 |
('LAST30DAYS_PERPLEXITY_DOMAIN_FILTER', None), |
| 664 |
('LAST30DAYS_PERPLEXITY_LANGUAGE_FILTER', None), |
| 665 |
('LAST30DAYS_PERPLEXITY_COUNTRY', None), |
| 666 |
('LAST30DAYS_PERPLEXITY_RECENCY_FILTER', None), |
| 667 |
('LAST30DAYS_PERPLEXITY_REASONING_EFFORT', None), |
| 668 |
('LAST30DAYS_PERPLEXITY_DEEP_TIMEOUT_SECONDS', '600'), |
| 669 |
('PARALLEL_API_KEY', None), |
| 670 |
('XQUIK_API_KEY', None), |
| 671 |
# Bright Data CLI. Optional: the CLI normally owns its own auth via |
| 672 |
# `brightdata login`, so this only matters for users who prefer an |
| 673 |
# explicit key in a `.env` file or the keychain. Registered here so |
| 674 |
# those layers reach the gate and the subprocess (-k) alike. |
| 675 |
('BRIGHTDATA_API_KEY', None), |
| 676 |
# Amazon marketplace the amazon source searches. Non-US users point |
| 677 |
# this at their own storefront (e.g. https://www.amazon.co.uk). |
| 678 |
('LAST30DAYS_AMAZON_DOMAIN', 'https://www.amazon.com'), |
| 679 |
# Ad Library country for the meta_ads source, as a two-letter code. The |
| 680 |
# endpoint takes exactly one country per call. There is deliberately no |
| 681 |
# durable env form of the advertiser-page override: a page id is |
| 682 |
# per-topic state, and env keys ride through the competitor runner's |
| 683 |
# config copy, which would attach one brand's ads to every peer. |
| 684 |
('LAST30DAYS_META_ADS_COUNTRY', 'US'), |
| 685 |
# Host-native search signal: set by the SKILL.md agent-host path when the |
| 686 |
# invoking runtime has its own (better) web-search tool, so the engine's |
| 687 |
# keyless search floor stays off there. Defaults unset -> floor allowed. |
| 688 |
('LAST30DAYS_NATIVE_SEARCH', None), |
| 689 |
# Optional SearXNG instance for the keyless-search fallback rung. |
| 690 |
('LAST30DAYS_SEARXNG_URL', None), |
| 691 |
# Truthy -> disable Trustpilot's headless-Chrome WAF-cookie harvest in |
| 692 |
# automated contexts (cron/CI/eval). Read by trustpilot._harvest_allowed. |
| 693 |
('LAST30DAYS_TRUSTPILOT_NO_BROWSER', None), |
| 694 |
('FROM_BROWSER', None), |
| 695 |
('BROWSER_CONSENT', None), |
| 696 |
# agentcookie sidecar: soft-dep X cookie source (lib/agentcookie.py), |
| 697 |
# active only on extra hosts (Linux / Mac mini / Darwin sink) or when |
| 698 |
# set to "on". "off" disables the sidecar reader. |
| 699 |
('AGENTCOOKIE', None), |
| 700 |
# Explicit Chrome DevTools endpoint for the extra-host CDP cookie |
| 701 |
# lookup (lib/chrome_cdp.py), e.g. http://127.0.0.1:18800. Preferred |
| 702 |
# over the 18800 / 9222+$DISPLAY defaults when set. |
| 703 |
('BROWSER_CDP_URL', None), |
| 704 |
('LAST30DAYS_TRUST_PROJECT_CONFIG', None), |
| 705 |
('SETUP_COMPLETE', None), |
| 706 |
('INCLUDE_SOURCES', ''), |
| 707 |
('EXCLUDE_SOURCES', ''), |
| 708 |
('LAST30DAYS_DEFAULT_SEARCH', ''), |
| 709 |
# Resolve the user-facing default in last30days.py so an absent value |
| 710 |
# stays distinguishable from an explicit `default`. That distinction |
| 711 |
# lets the new key override legacy ELI5_MODE=true configurations. |
| 712 |
('LAST30DAYS_REGISTER', None), |
| 713 |
('FUN_LEVEL', 'medium'), |
| 714 |
# Backward compatibility for configs written by the original `eli5 on` |
| 715 |
# follow-up command. New writes use LAST30DAYS_REGISTER=eli5. |
| 716 |
('ELI5_MODE', None), |
| 717 |
('LAST30DAYS_YOUTUBE_SSH_HOST', None), |
| 718 |
('LAST30DAYS_REPORT_CACHE_TTL_SECONDS', None), |
| 719 |
('LAST30DAYS_VERIFY_FRESHNESS', None), |
| 720 |
('LAST30DAYS_TRANSCRIPT_TIMEOUT', None), |
| 721 |
('DEGRADED_TRANSCRIPT_THRESHOLD', None), |
| 722 |
(KEYCHAIN_ALIASES_ENV, None), |
| 723 |
# Whisper transcription provider for caption-free audio/video. Groq's |
| 724 |
# free tier is preferred; OPENAI_API_KEY is the paid backstop (already |
| 725 |
# resolved above via openai_auth). |
| 726 |
('GROQ_API_KEY', None), |
| 727 |
('LAST30DAYS_YT_SUB_LANGS', 'en,es,pt'), |
| 728 |
# youtube_yt reads this lazily from os.environ; default android is |
| 729 |
# applied there when the key is absent. Empty disables. |
| 730 |
('LAST30DAYS_YT_PLAYER_CLIENT', None), |
| 731 |
('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT', None), |
| 732 |
('LAST30DAYS_YT_SEARCH_TIMEOUT', None), |
| 733 |
('GITHUB_TOKEN', None), |
| 734 |
# Host self-identification. `grok-bot` switches the X policy |
| 735 |
# to official-only (see x_policy); the engine never sniffs the host |
| 736 |
# any other way. Persisted by first-run setup and exported per |
| 737 |
# invocation by the SKILL.md rule. |
| 738 |
(X_HOST_VAR, None), |
| 739 |
# App-only bearer token for the direct X API v2 backend (`xapi`). |
| 740 |
('X_BEARER_TOKEN', None), |
| 741 |
# Per-session X connector lane signal. Read from the process |
| 742 |
# environment ONLY: a .env line is deliberately ignored (a removed |
| 743 |
# connector must never leave a stale declaration), so it is handled |
| 744 |
# in the loop below rather than via merged_env. |
| 745 |
(X_HOST_LANE_VAR, None), |
| 746 |
] |
| 747 |
|
| 748 |
for key, default in keys: |
| 749 |
if key == X_HOST_LANE_VAR: |
| 750 |
# Process env only; the .env value never reaches config. |
| 751 |
config[key] = os.environ.get(key) or default |
| 752 |
continue |
| 753 |
if key in {'LAST30DAYS_YT_PLAYER_CLIENT', 'LAST30DAYS_MEMORY_DIR'}: |
| 754 |
# Empty string is a valid disable; `or` would treat it as unset. |
| 755 |
if key in os.environ: |
| 756 |
config[key] = os.environ.get(key) |
| 757 |
elif key in merged_env: |
| 758 |
# Mapping lookup via .get; bracket form trips a CRITICAL |
| 759 |
# scanner false positive on this identifier. |
| 760 |
config[key] = merged_env.get(key) |
| 761 |
else: |
| 762 |
config[key] = default |
| 763 |
else: |
| 764 |
config[key] = os.environ.get(key) or merged_env.get(key, default) |
| 765 |
|
| 766 |
# Export debug flag to os.environ so log.py's lazy os.environ.get() |
| 767 |
# picks up .env values. setdefault ensures a shell-exported value is |
| 768 |
# never overwritten by the (lower-priority) .env value. |
| 769 |
if config.get('LAST30DAYS_DEBUG'): |
| 770 |
os.environ.setdefault('LAST30DAYS_DEBUG', config['LAST30DAYS_DEBUG']) |
| 771 |
|
| 772 |
# youtube_yt reads these tuning knobs lazily from os.environ, so values |
| 773 |
# loaded from .env must be exported into the current engine process. |
| 774 |
for key in ( |
| 775 |
'LAST30DAYS_YT_SUB_LANGS', |
| 776 |
'LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT', |
| 777 |
'LAST30DAYS_YT_SEARCH_TIMEOUT', |
| 778 |
'LAST30DAYS_REDDIT_KEYLESS_RATE', |
| 779 |
'LAST30DAYS_YT_PLAYER_CLIENT', |
| 780 |
): |
| 781 |
value = config.get(key) |
| 782 |
# Empty LAST30DAYS_YT_PLAYER_CLIENT is a valid disable; other knobs |
| 783 |
# treat empty as unset and keep their code defaults. |
| 784 |
if key == 'LAST30DAYS_YT_PLAYER_CLIENT': |
| 785 |
if value is not None: |
| 786 |
os.environ.setdefault(key, value) |
| 787 |
elif value: |
| 788 |
os.environ.setdefault(key, value) |
| 789 |
|
| 790 |
# Backward-compat: ScrapeCreators' own examples and tutorials use the |
| 791 |
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and |
| 792 |
# CREATORS). Accept that form too so users who follow the vendor's docs |
| 793 |
# don't silently end up with has_scrapecreators=False. Canonical name |
| 794 |
# wins when both are set. |
| 795 |
if not config.get('SCRAPECREATORS_API_KEY'): |
| 796 |
legacy = read_secret_env('SCRAPE_CREATORS_API_KEY') or merged_env.get('SCRAPE_CREATORS_API_KEY') |
| 797 |
if legacy: |
| 798 |
config['SCRAPECREATORS_API_KEY'] = legacy |
| 799 |
|
| 800 |
# Multi-key rotation: comma-separated SCRAPECREATORS_API_KEY round-robins |
| 801 |
# via random.choice per run. Originally added in #268, accidentally dropped |
| 802 |
# in v3.0.6, restored here. |
| 803 |
_rotate_scrapecreators_key(config) |
| 804 |
|
| 805 |
# Track which config source was used (highest-priority file source wins |
| 806 |
# the label; keychain is only reported when nothing else is configured). |
| 807 |
if project_env_path: |
| 808 |
config['_CONFIG_SOURCE'] = f'project:{project_env_path}' |
| 809 |
elif CONFIG_FILE and CONFIG_FILE.exists(): |
| 810 |
config['_CONFIG_SOURCE'] = f'global:{CONFIG_FILE}' |
| 811 |
elif keychain_env: |
| 812 |
config['_CONFIG_SOURCE'] = 'keychain' |
| 813 |
elif pass_env: |
| 814 |
config['_CONFIG_SOURCE'] = 'pass' |
| 815 |
else: |
| 816 |
config['_CONFIG_SOURCE'] = 'env_only' |
| 817 |
if ignored_project_env_path: |
| 818 |
config['_IGNORED_PROJECT_CONFIG'] = str(ignored_project_env_path) |
| 819 |
config['_IGNORED_PROJECT_CONFIG_KEYS'] = ignored_project_keys |
| 820 |
config['_BROWSER_COOKIE_MODE'] = policy.browser_cookies |
| 821 |
# A LAST30DAYS_X_HOST_LANE line in a config file is ignored; |
| 822 |
# remember that it was there so doctor can say so. |
| 823 |
config['_X_HOST_LANE_FILE_IGNORED'] = bool(merged_env.get(X_HOST_LANE_VAR)) |
| 824 |
# Evaluated after the host and pin keys are merged: on an official-only |
| 825 |
# host the browser list is empty unless bird is pinned (x_policy). |
| 826 |
config['_BROWSER_COOKIE_BROWSERS'] = cookie_extraction_browsers(config) |
| 827 |
|
| 828 |
# Reject unsubstituted extension placeholders last among the value-producing |
| 829 |
# steps, so the legacy ScrapeCreators spelling, the multi-key rotation, and |
| 830 |
# the OpenAI auth fields assembled above are all covered by one sweep rather |
| 831 |
# than by a predicate repeated at each presence check. Rejection means |
| 832 |
# "absent", not "empty": the placeholder is removed from the process |
| 833 |
# environment and the key is then re-resolved from the lower-priority |
| 834 |
# sources exactly as it would be had the host never written it, so a real |
| 835 |
# .env, Keychain, or pass credential it was shadowing is not discarded. |
| 836 |
# Every consumer of a rejected config key therefore agrees the credential is |
| 837 |
# unset, and the keys left genuinely unset are published for the diagnostics |
| 838 |
# to report. The sweep is bounded by the keys get_config registers: a |
| 839 |
# credential read straight from the environment under a name it does not |
| 840 |
# register - LAST30DAYS_API_KEY, or a bare SCRAPE_CREATORS_API_KEY spelling |
| 841 |
# left behind after the canonical key resolved - keeps its placeholder. |
| 842 |
declared_defaults = {key: default for key, default in keys} |
| 843 |
templated_keys = sorted( |
| 844 |
key |
| 845 |
for key, value in config.items() |
| 846 |
if not key.startswith('_') and is_unsubstituted_template(value) |
| 847 |
) |
| 848 |
for key in templated_keys: |
| 849 |
os.environ.pop(key, None) |
| 850 |
fallback = ( |
| 851 |
_configured_memory_dir(project_env.get(key), file_env.get(key)) |
| 852 |
if key == 'LAST30DAYS_MEMORY_DIR' |
| 853 |
else merged_env.get(key) |
| 854 |
) |
| 855 |
# A lower-priority value that is itself a placeholder is not a credential. |
| 856 |
if is_unsubstituted_template(fallback): |
| 857 |
fallback = None |
| 858 |
resolved = fallback if fallback is not None else declared_defaults.get(key) |
| 859 |
config[key] = resolved if resolved is not None else '' |
| 860 |
# The rotation ran before this sweep, so a fallback restored from a |
| 861 |
# comma-separated list would otherwise reach a backend whole. Reapply it, |
| 862 |
# then reject the picked key if it is itself a placeholder. |
| 863 |
_rotate_scrapecreators_key(config) |
| 864 |
if is_unsubstituted_template(config.get('SCRAPECREATORS_API_KEY')): |
| 865 |
config['SCRAPECREATORS_API_KEY'] = '' |
| 866 |
# Report only the keys still leaving the credential unset. A placeholder that |
| 867 |
# fell through to a real lower-priority credential (or to a usable default) |
| 868 |
# is handled, and reporting it would nag about a setup that works. |
| 869 |
config[TEMPLATE_CONFIG_KEYS] = [ |
| 870 |
key for key in templated_keys if not config.get(key) |
| 871 |
] |
| 872 |
if 'OPENAI_API_KEY' in templated_keys and not config.get('OPENAI_API_KEY'): |
| 873 |
# Keep the derived auth record consistent with the token it describes. |
| 874 |
config['OPENAI_AUTH_SOURCE'] = AUTH_SOURCE_NONE |
| 875 |
config['OPENAI_AUTH_STATUS'] = AUTH_STATUS_MISSING |
| 876 |
|
| 877 |
if policy.browser_cookies == "read": |
| 878 |
_discover_and_apply_x_credentials(config) |
| 879 |
|
| 880 |
# Fixture recording (--record-fixtures) must redact a credential that |
| 881 |
# came from a file, Keychain, or pass, not only one exported in the |
| 882 |
# shell. No-op outside a recording session. |
| 883 |
from . import http as _http |
| 884 |
_http.add_fixture_redactions(_http.config_secret_values(config)) |
| 885 |
|
| 886 |
return config |
| 887 |
|
| 888 |
|
| 889 |
# --------------------------------------------------------------------------- |
| 890 |
# Extra-host X cookie discovery (Linux, Mac mini, Darwin agentcookie sink) |
| 891 |
# --------------------------------------------------------------------------- |
| 892 |
|
| 893 |
|
| 894 |
def _mac_model() -> str: |
| 895 |
"""Darwin hardware model via ``sysctl -n hw.model``, or "" otherwise. |
| 896 |
|
| 897 |
Returns "" on non-Darwin and on any sysctl failure (missing binary, |
| 898 |
non-zero exit, timeout) — the caller treats "" as "not a Mac mini", i.e. a |
| 899 |
MacBook, which is the conservative default (no extra cookie lookups). |
| 900 |
""" |
| 901 |
import platform |
| 902 |
if platform.system() != "Darwin": |
| 903 |
return "" |
| 904 |
import subprocess |
| 905 |
try: |
| 906 |
out = subprocess.run( |
| 907 |
["sysctl", "-n", "hw.model"], |
| 908 |
capture_output=True, text=True, timeout=3, |
| 909 |
) |
| 910 |
except (OSError, subprocess.SubprocessError): |
| 911 |
return "" |
| 912 |
if out.returncode != 0: |
| 913 |
return "" |
| 914 |
return (out.stdout or "").strip() |
| 915 |
|
| 916 |
|
| 917 |
def _is_mac_mini() -> bool: |
| 918 |
"""True on a Darwin Mac mini (``hw.model`` prefix ``Macmini``). |
| 919 |
|
| 920 |
sysctl failure yields "" -> False, so an unreadable model is treated as a |
| 921 |
MacBook (no extras), per the plan. |
| 922 |
""" |
| 923 |
return _mac_model().startswith("Macmini") |
| 924 |
|
| 925 |
|
| 926 |
def x_extras_enabled(config: dict[str, Any]) -> bool: |
| 927 |
"""Whether the two EXTRA bird cookie lookups (agentcookie sidecar, live |
| 928 |
Chrome CDP) apply on this host. |
| 929 |
|
| 930 |
Extras apply when ANY of: |
| 931 |
* ``AGENTCOOKIE=on`` — explicit per-host opt-in (works on a MacBook too); |
| 932 |
* platform is Linux; |
| 933 |
* a Darwin Mac mini (``hw.model`` prefix ``Macmini``); |
| 934 |
* a Darwin agentcookie **sink** role (parse failure = not sink). |
| 935 |
|
| 936 |
A plain MacBook (Darwin, source/unknown role, no opt-in) stays on the |
| 937 |
mainline path — no agentcookie subprocess, no CDP socket. The host is NEVER |
| 938 |
inferred from the home directory, PATH, or ``HERMES_AGENT``/``OPENCLAW_CLI`` |
| 939 |
env: only the signals above. |
| 940 |
""" |
| 941 |
import platform |
| 942 |
raw = (config.get("AGENTCOOKIE") or read_secret_env("AGENTCOOKIE") or "").strip().lower() |
| 943 |
if raw == "on": |
| 944 |
return True |
| 945 |
system = platform.system() |
| 946 |
if system == "Linux": |
| 947 |
return True |
| 948 |
if system == "Darwin": |
| 949 |
if _is_mac_mini(): |
| 950 |
return True |
| 951 |
from . import agentcookie |
| 952 |
return agentcookie.role_is_sink(config) |
| 953 |
return False |
| 954 |
|
| 955 |
|
| 956 |
def _apply_x_pair(config: dict[str, Any], auth_token: str, ct0: str, source: str) -> None: |
| 957 |
"""Apply a COMPLETE X cookie pair from one source, labeling its origin. |
| 958 |
|
| 959 |
Atomic on purpose (both keys from the same source) so a half-pair from one |
| 960 |
source is never merged with a half-pair from another. Never written to the |
| 961 |
``.env``; values are never logged. |
| 962 |
""" |
| 963 |
config["AUTH_TOKEN"] = auth_token |
| 964 |
config["CT0"] = ct0 |
| 965 |
config["_AUTH_TOKEN_SOURCE"] = source |
| 966 |
config["_CT0_SOURCE"] = source |
| 967 |
|
| 968 |
|
| 969 |
def _apply_browser_extract(config: dict[str, Any]) -> None: |
| 970 |
"""Run the mainline in-process browser cookie extractor (unchanged from |
| 971 |
main): fills X (when a browser is opted in via FROM_BROWSER) and non-X |
| 972 |
cookie domains like truthsocial. Missing keys only; source label ``browser``.""" |
| 973 |
browser_creds = extract_browser_credentials(config) |
| 974 |
for key, value in browser_creds.items(): |
| 975 |
if not config.get(key): |
| 976 |
config[key] = value |
| 977 |
config[f"_{key}_SOURCE"] = "browser" |
| 978 |
|
| 979 |
|
| 980 |
def _discover_and_apply_x_credentials(config: dict[str, Any]) -> None: |
| 981 |
"""Fill AUTH_TOKEN/CT0 for the bird backend, first COMPLETE pair wins. |
| 982 |
|
| 983 |
Mainline (every host): the in-process browser extractor, gated by |
| 984 |
FROM_BROWSER exactly as on ``main``. EXTRA lookups (agentcookie sidecar, |
| 985 |
then live Chrome CDP) run ONLY on extra hosts (``x_extras_enabled``), so a |
| 986 |
MacBook with FROM_BROWSER unset/off does no agentcookie spawn and no CDP |
| 987 |
socket. Probe order: |
| 988 |
|
| 989 |
1. an explicit env AUTH_TOKEN+CT0 already present — never overwritten; |
| 990 |
2. agentcookie sidecar (extras only); |
| 991 |
3. live Chrome CDP (extras only); |
| 992 |
4. the mainline browser extract (all hosts; X only when FROM_BROWSER |
| 993 |
lists a browser). |
| 994 |
|
| 995 |
On a Mac mini that has already opted into browser reads (FROM_BROWSER set), |
| 996 |
the native extract runs BEFORE CDP (R19) — a local Keychain read beats a |
| 997 |
debug-port scrape. Never persists cookies; values are never logged. |
| 998 |
|
| 999 |
On an official-only host (``x_policy``: ``LAST30DAYS_HOST=grok-bot``) |
| 1000 |
this returns before ANY leg, for every cookie domain, unless the pin is |
| 1001 |
``bird`` (the one path that re-enables discovery for that run). |
| 1002 |
""" |
| 1003 |
if not x_policy(config).cookie_discovery: |
| 1004 |
return |
| 1005 |
|
| 1006 |
from . import agentcookie, chrome_cdp |
| 1007 |
|
| 1008 |
def have_pair() -> bool: |
| 1009 |
return bool(config.get("AUTH_TOKEN") and config.get("CT0")) |
| 1010 |
|
| 1011 |
extras = x_extras_enabled(config) |
| 1012 |
|
| 1013 |
# (2) agentcookie sidecar — extras only, complete pair only. |
| 1014 |
if extras and not have_pair(): |
| 1015 |
pair = agentcookie.read_x_cookies(config) |
| 1016 |
if pair: |
| 1017 |
_apply_x_pair(config, pair["auth_token"], pair["ct0"], "agentcookie") |
| 1018 |
|
| 1019 |
# Mac mini + browser opted in: native extract before CDP (R19). |
| 1020 |
mini_extract_first = ( |
| 1021 |
extras and _is_mac_mini() and bool(cookie_extraction_browsers(config)) |
| 1022 |
) |
| 1023 |
if mini_extract_first and not have_pair(): |
| 1024 |
_apply_browser_extract(config) |
| 1025 |
|
| 1026 |
# (3) live Chrome CDP — extras only, after browser-cookie consent. |
| 1027 |
if extras and not have_pair() and chrome_cdp.cookie_access_allowed(config): |
| 1028 |
pair = chrome_cdp.read_x_cookies(config) |
| 1029 |
if pair: |
| 1030 |
_apply_x_pair(config, pair["auth_token"], pair["ct0"], "chrome cdp") |
| 1031 |
|
| 1032 |
# (4) mainline browser extract (unless already run above for the mini case). |
| 1033 |
if not mini_extract_first: |
| 1034 |
_apply_browser_extract(config) |
| 1035 |
|
| 1036 |
|
| 1037 |
# --------------------------------------------------------------------------- |
| 1038 |
# Browser cookie extraction |
| 1039 |
# --------------------------------------------------------------------------- |
| 1040 |
|
| 1041 |
COOKIE_DOMAINS: dict[str, dict[str, Any]] = { |
| 1042 |
"x": { |
| 1043 |
"domain": ".x.com", |
| 1044 |
"cookies": ["auth_token", "ct0"], |
| 1045 |
"mapping": {"auth_token": "AUTH_TOKEN", "ct0": "CT0"}, |
| 1046 |
}, |
| 1047 |
"truthsocial": { |
| 1048 |
"domain": ".truthsocial.com", |
| 1049 |
"cookies": ["_session_id"], |
| 1050 |
"mapping": {"_session_id": "TRUTHSOCIAL_TOKEN"}, |
| 1051 |
}, |
| 1052 |
} |
| 1053 |
|
| 1054 |
|
| 1055 |
def cookie_extraction_browsers(config: dict[str, Any]) -> list[str]: |
| 1056 |
"""Browsers to try for cookie extraction, honoring FROM_BROWSER. |
| 1057 |
|
| 1058 |
Default (FROM_BROWSER unset): no browser-cookie reads. The Chromium family |
| 1059 |
(Chrome, Brave, Edge, Vivaldi, Opera, Arc, Chromium) is available only when |
| 1060 |
explicitly selected because reading their cookies on macOS requires the |
| 1061 |
browser's Safe Storage Keychain key, which triggers a system password prompt |
| 1062 |
that cannot be reliably suppressed. On Windows only Firefox cookie |
| 1063 |
extraction is supported; Chrome and Edge use DPAPI-encrypted cookie stores |
| 1064 |
that are not yet supported. |
| 1065 |
|
| 1066 |
- ``FROM_BROWSER=<name>`` - a single browser (e.g. ``firefox``, ``brave``, |
| 1067 |
``edge``, ``arc``). |
| 1068 |
- ``FROM_BROWSER=firefox,safari`` - a comma-separated explicit browser list. |
| 1069 |
- ``FROM_BROWSER=auto`` - also try every Chromium browser (user accepts the |
| 1070 |
Keychain dialog when needed). |
| 1071 |
- ``FROM_BROWSER=off`` - returns [] (extraction disabled). |
| 1072 |
|
| 1073 |
Returning the browser list from one place keeps the setup wizard and the |
| 1074 |
steady-state path on the same policy, so neither surprises the user with an |
| 1075 |
unrequested Keychain prompt. On an official-only host (``x_policy``) the |
| 1076 |
list is empty regardless of ``FROM_BROWSER`` unless ``bird`` is pinned. |
| 1077 |
""" |
| 1078 |
if not x_policy(config).cookie_discovery: |
| 1079 |
return [] |
| 1080 |
consent = config.get("BROWSER_CONSENT") |
| 1081 |
if consent is not None and str(consent).strip().lower() not in {"1", "true", "yes", "on"}: |
| 1082 |
return [] |
| 1083 |
silent_browsers = ["firefox", "safari"] |
| 1084 |
chromium_browsers = ["chrome", "brave", "edge", "vivaldi", "opera", "arc", "chromium"] |
| 1085 |
known_browsers = silent_browsers + chromium_browsers |
| 1086 |
from_browser = (config.get("FROM_BROWSER") or "").strip().lower() |
| 1087 |
if not from_browser: |
| 1088 |
return [] |
| 1089 |
if from_browser == "off": |
| 1090 |
return [] |
| 1091 |
if from_browser == "auto": |
| 1092 |
return silent_browsers + chromium_browsers |
| 1093 |
if "," in from_browser: |
| 1094 |
requested = [b.strip() for b in from_browser.split(",") if b.strip()] |
| 1095 |
resolved = [b for b in requested if b in known_browsers] |
| 1096 |
unknown = [b for b in requested if b not in known_browsers] |
| 1097 |
if unknown: |
| 1098 |
sys.stderr.write( |
| 1099 |
"[last30days] WARNING: FROM_BROWSER ignored unrecognized browser(s): " |
| 1100 |
f"{', '.join(unknown)} (known: {', '.join(known_browsers)})\n" |
| 1101 |
) |
| 1102 |
sys.stderr.flush() |
| 1103 |
return resolved |
| 1104 |
if from_browser in known_browsers: |
| 1105 |
return [from_browser] |
| 1106 |
# Non-empty, not off/auto, not a known browser, not a list: unrecognized. |
| 1107 |
# Warn rather than fail silently so a typo (FROM_BROWSER=chrme) is visible |
| 1108 |
# instead of looking like "no cookies found". |
| 1109 |
sys.stderr.write( |
| 1110 |
f"[last30days] WARNING: FROM_BROWSER='{from_browser}' is not a recognized " |
| 1111 |
f"browser; no cookies will be read (known: {', '.join(known_browsers)}, " |
| 1112 |
"or 'auto'/'off')\n" |
| 1113 |
) |
| 1114 |
sys.stderr.flush() |
| 1115 |
return [] |
| 1116 |
|
| 1117 |
|
| 1118 |
|
| 1119 |
def extract_browser_credentials(config: dict[str, Any]) -> dict[str, str]: |
| 1120 |
"""Extract auth cookies from local browsers. |
| 1121 |
|
| 1122 |
Browser selection (and the Chrome-prompt caveat) is handled by |
| 1123 |
``cookie_extraction_browsers``; this function just runs the extraction for |
| 1124 |
each configured cookie domain. |
| 1125 |
""" |
| 1126 |
browsers = cookie_extraction_browsers(config) |
| 1127 |
if not browsers: |
| 1128 |
return {} |
| 1129 |
try: |
| 1130 |
from . import cookie_extract |
| 1131 |
except ImportError: |
| 1132 |
return {} |
| 1133 |
extracted: dict[str, str] = {} |
| 1134 |
for _service, spec in COOKIE_DOMAINS.items(): |
| 1135 |
if all(config.get(env_key) for env_key in spec["mapping"].values()): |
| 1136 |
continue |
| 1137 |
# Cookies from different browsers can belong to different sessions, |
| 1138 |
# so values are never combined across browsers: a complete set from |
| 1139 |
# one browser wins, else the first browser's partial set is kept. |
| 1140 |
chosen: dict[str, str] | None = None |
| 1141 |
fallback: dict[str, str] | None = None |
| 1142 |
for browser in browsers: |
| 1143 |
try: |
| 1144 |
cookies = cookie_extract.extract_cookies(browser, spec["domain"], spec["cookies"]) |
| 1145 |
except Exception: |
| 1146 |
continue |
| 1147 |
if not cookies: |
| 1148 |
continue |
| 1149 |
if cookie_extract.has_complete_pair(cookies, spec["cookies"]): |
| 1150 |
chosen = cookies |
| 1151 |
break |
| 1152 |
if fallback is None: |
| 1153 |
fallback = cookies |
| 1154 |
if chosen is None: |
| 1155 |
chosen = fallback or {} |
| 1156 |
for cookie_name, env_key in spec["mapping"].items(): |
| 1157 |
if chosen.get(cookie_name) and not config.get(env_key): |
| 1158 |
extracted[env_key] = chosen[cookie_name] |
| 1159 |
return extracted |
| 1160 |
|
| 1161 |
|
| 1162 |
# Auth-origin label per X backend for ``get_x_source_with_method`` (bird's |
| 1163 |
# label is the cookie source recorded in ``_AUTH_TOKEN_SOURCE``). |
| 1164 |
_X_METHOD_LABELS = { |
| 1165 |
"xai": "xai", |
| 1166 |
"xurl": "oauth2", # xurl CLI (official X API v2, OAuth2, free developer app) |
| 1167 |
"xapi": "bearer", |
| 1168 |
"xquik": "api_key", |
| 1169 |
} |
| 1170 |
|
| 1171 |
|
| 1172 |
def get_x_source_with_method(config: dict[str, Any]) -> tuple[str | None, str]: |
| 1173 |
"""Return (source, method) for X search, where method describes the auth origin. |
| 1174 |
|
| 1175 |
Walks the policy's unpinned auto chain (``x_auto_chain``): on a default |
| 1176 |
host bird first (cookies beat XAI_API_KEY when both are present), then |
| 1177 |
xai, xurl, xquik; on an official-only host xapi, xai, xurl. Opt-in |
| 1178 |
backends (grok, and xapi off Grok Bot) are never auto-selected here. |
| 1179 |
""" |
| 1180 |
has_bird_creds = bool(config.get("AUTH_TOKEN") and config.get("CT0")) |
| 1181 |
for backend in x_auto_chain(config): |
| 1182 |
if backend == "bird": |
| 1183 |
# Cookie presence only: the scraper install is not consulted |
| 1184 |
# here (unlike ``x_backend_chain``), so a fresh cookie-bearing |
| 1185 |
# config reports bird before the binary is checked. |
| 1186 |
if not has_bird_creds: |
| 1187 |
continue |
| 1188 |
elif not _x_backend_available(backend, config, has_bird_creds): |
| 1189 |
continue |
| 1190 |
if backend == "bird": |
| 1191 |
return "bird", config.get("_AUTH_TOKEN_SOURCE", "env") |
| 1192 |
return backend, _X_METHOD_LABELS.get(backend, "none") |
| 1193 |
return None, "none" |
| 1194 |
|
| 1195 |
|
| 1196 |
def config_exists(policy: ConfigLoadPolicy | None = None) -> bool: |
| 1197 |
"""Check if any configuration source exists.""" |
| 1198 |
policy = policy or ConfigLoadPolicy() |
| 1199 |
file_env = load_env_file(CONFIG_FILE) if CONFIG_FILE and CONFIG_FILE.exists() else {} |
| 1200 |
if _project_config_trusted(policy, file_env) and _find_project_env(): |
| 1201 |
return True |
| 1202 |
if CONFIG_FILE: |
| 1203 |
return CONFIG_FILE.exists() |
| 1204 |
return False |
| 1205 |
|
| 1206 |
|
| 1207 |
def get_reddit_source(config: dict[str, Any]) -> str | None: |
| 1208 |
"""Determine which Reddit backend to use. |
| 1209 |
|
| 1210 |
Returns: 'scrapecreators' or None |
| 1211 |
""" |
| 1212 |
if config.get('SCRAPECREATORS_API_KEY'): |
| 1213 |
return 'scrapecreators' |
| 1214 |
return None |
| 1215 |
|
| 1216 |
|
| 1217 |
# Default X backend priority. The first available backend is the primary X |
| 1218 |
# source; the rest are ordered failover backups, tried only if the one before |
| 1219 |
# returns nothing or errors. There is one X source ("x"); these are its |
| 1220 |
# interchangeable backends, never run in parallel. |
| 1221 |
# bird — X GraphQL scrape via the user's browser cookies (AUTH_TOKEN/CT0) |
| 1222 |
# xai — xAI/Grok live search (XAI_API_KEY) |
| 1223 |
# xurl — official X API v2 (xurl CLI, OAuth2) |
| 1224 |
# xquik — key-based REST X search (XQUIK_API_KEY) |
| 1225 |
_X_BACKEND_ORDER = ("bird", "xai", "xurl", "xquik") |
| 1226 |
|
| 1227 |
# Opt-in backends: never in the default unpinned auto chain; require an |
| 1228 |
# explicit pin. grok is here because a leftover ~/.grok/auth.json must never |
| 1229 |
# steal the X lane. xapi (direct X API v2 with X_BEARER_TOKEN) is here so an |
| 1230 |
# ambient bearer exported for some other tool never spends X API credits |
| 1231 |
# every time the cookie scraper comes back empty; on an official-only |
| 1232 |
# host it is the first rung of the auto chain instead (see _X_OFFICIAL). |
| 1233 |
_X_BACKEND_OPT_IN = ("grok", "xapi") |
| 1234 |
|
| 1235 |
# All known backends (auto chain + opt-in): valid values for the pin var. |
| 1236 |
_X_BACKEND_KNOWN = _X_BACKEND_ORDER + _X_BACKEND_OPT_IN |
| 1237 |
|
| 1238 |
# Licensed / official backends: the unpinned auto chain on an official-only |
| 1239 |
# host. xapi = X API v2 with an app-only bearer, xai = xAI's licensed |
| 1240 |
# X search, xurl = the X API through X's own CLI. |
| 1241 |
_X_OFFICIAL = ("xapi", "xai", "xurl") |
| 1242 |
|
| 1243 |
# Host self-identification key and the one value that switches the X |
| 1244 |
# policy. The engine trusts this key alone: it never infers the host from |
| 1245 |
# the home directory, PATH, platform, or agent env vars. |
| 1246 |
X_HOST_VAR = 'LAST30DAYS_HOST' |
| 1247 |
GROK_BOT_HOST = 'grok-bot' |
| 1248 |
# Per-session X connector lane signal: process env only. |
| 1249 |
X_HOST_LANE_VAR = 'LAST30DAYS_X_HOST_LANE' |
| 1250 |
|
| 1251 |
# Public routing definitions for the doctor/backend-descriptor layer |
| 1252 |
# (lib/backends.py). These are aliases for knowledge this module already |
| 1253 |
# owns — the declared X chain order and the pin/floor env var names — so |
| 1254 |
# descriptors import one source of truth instead of restating it. |
| 1255 |
X_BACKEND_ORDER = _X_BACKEND_ORDER |
| 1256 |
X_BACKEND_OPT_IN = _X_BACKEND_OPT_IN |
| 1257 |
X_BACKEND_KNOWN = _X_BACKEND_KNOWN |
| 1258 |
X_OFFICIAL = _X_OFFICIAL |
| 1259 |
X_BACKEND_PIN_VAR = 'LAST30DAYS_X_BACKEND' |
| 1260 |
REDDIT_BACKEND_PIN_VAR = 'LAST30DAYS_REDDIT_BACKEND' |
| 1261 |
REDDIT_SC_MIN_ITEMS_VAR = 'LAST30DAYS_REDDIT_SC_MIN_ITEMS' |
| 1262 |
# Keyed runs backfill Reddit from ScrapeCreators when the free path returns |
| 1263 |
# fewer than this many items. Thin topics yield 2-3 free results; healthy |
| 1264 |
# topics many more, so 5 spends credits only where it adds coverage. |
| 1265 |
REDDIT_SC_MIN_ITEMS_DEFAULT = 5 |
| 1266 |
|
| 1267 |
|
| 1268 |
def reddit_sc_min_items(config: dict[str, Any]) -> int: |
| 1269 |
"""The Reddit ScrapeCreators backfill floor, parsed one way for every caller. |
| 1270 |
|
| 1271 |
Unset or blank means ``REDDIT_SC_MIN_ITEMS_DEFAULT``; an explicit ``0`` |
| 1272 |
means backfill only when the free path is empty; a malformed value means |
| 1273 |
``0`` so a typo never spends extra credits. Negative values clamp to 0. |
| 1274 |
""" |
| 1275 |
raw = config.get(REDDIT_SC_MIN_ITEMS_VAR) |
| 1276 |
if raw is None or (isinstance(raw, str) and not raw.strip()): |
| 1277 |
return REDDIT_SC_MIN_ITEMS_DEFAULT |
| 1278 |
try: |
| 1279 |
return max(int(raw), 0) |
| 1280 |
except (TypeError, ValueError): |
| 1281 |
return 0 |
| 1282 |
|
| 1283 |
|
| 1284 |
@dataclass(frozen=True) |
| 1285 |
class XPolicy: |
| 1286 |
"""The host-conditional X routing rule, resolved once per config. |
| 1287 |
|
| 1288 |
``host`` is the normalized ``LAST30DAYS_HOST`` value; ``official_only`` |
| 1289 |
is true on a Grok Bot host; ``auto_chain`` is the unpinned chain |
| 1290 |
(``_X_OFFICIAL`` when official-only, else ``_X_BACKEND_ORDER``); |
| 1291 |
``cookie_discovery`` is false when official-only unless the pin is |
| 1292 |
``bird``; ``hint_namespace`` (``official`` or ``default``) is derived |
| 1293 |
from ``official_only`` as a plain string so this module never imports |
| 1294 |
``prescriptions``. |
| 1295 |
""" |
| 1296 |
|
| 1297 |
host: str |
| 1298 |
official_only: bool |
| 1299 |
auto_chain: tuple[str, ...] |
| 1300 |
cookie_discovery: bool |
| 1301 |
|
| 1302 |
@property |
| 1303 |
def hint_namespace(self) -> str: |
| 1304 |
return 'official' if self.official_only else 'default' |
| 1305 |
|
| 1306 |
|
| 1307 |
def x_backend_pin(config: dict[str, Any]) -> str: |
| 1308 |
"""The normalized ``LAST30DAYS_X_BACKEND`` pin value ("" when unset).""" |
| 1309 |
return (config.get(X_BACKEND_PIN_VAR) or '').strip().lower() |
| 1310 |
|
| 1311 |
|
| 1312 |
def x_policy(config: dict[str, Any]) -> XPolicy: |
| 1313 |
"""Resolve the X policy from ``LAST30DAYS_HOST`` and the pin. |
| 1314 |
|
| 1315 |
This is the ONLY place the Grok Bot host string is compared. It reads |
| 1316 |
just the host key, the pin, and the config dict: no platform, PATH, home |
| 1317 |
directory, or agent env-var inspection (the same rule ``x_extras_enabled`` |
| 1318 |
follows), and nothing imported from ``lib``. The pin keeps its exclusive |
| 1319 |
semantics on every host and may name any known backend; a ``bird`` pin |
| 1320 |
is the one path that re-enables cookie discovery on an official-only host. |
| 1321 |
""" |
| 1322 |
host = str(config.get(X_HOST_VAR) or '').strip().lower() |
| 1323 |
official_only = host == GROK_BOT_HOST |
| 1324 |
pin = x_backend_pin(config) |
| 1325 |
return XPolicy( |
| 1326 |
host=host, |
| 1327 |
official_only=official_only, |
| 1328 |
auto_chain=_X_OFFICIAL if official_only else _X_BACKEND_ORDER, |
| 1329 |
cookie_discovery=(not official_only) or pin == 'bird', |
| 1330 |
) |
| 1331 |
|
| 1332 |
|
| 1333 |
def x_auto_chain(config: dict[str, Any]) -> list[str]: |
| 1334 |
"""The unpinned X auto chain for this host, in failover order.""" |
| 1335 |
return list(x_policy(config).auto_chain) |
| 1336 |
|
| 1337 |
|
| 1338 |
def x_host_lane_declared(config: dict[str, Any]) -> bool: |
| 1339 |
"""True when the hosting model declared the X connector lane. |
| 1340 |
|
| 1341 |
``get_config`` fills ``LAST30DAYS_X_HOST_LANE`` from the process |
| 1342 |
environment only, so a ``.env`` line never declares the lane. |
| 1343 |
Deliberately NOT ``x_pending_browser_auth``: that predicate is false in |
| 1344 |
cookie-read mode by contract, which would leave the envelope path dead at |
| 1345 |
research time. Host-independent: the envelope is accepted anywhere. |
| 1346 |
""" |
| 1347 |
return _truthy(config.get(X_HOST_LANE_VAR)) |
| 1348 |
|
| 1349 |
|
| 1350 |
def _x_backend_available( |
| 1351 |
backend: str, |
| 1352 |
config: dict[str, Any], |
| 1353 |
has_bird_creds: bool, |
| 1354 |
local_only: bool = False, |
| 1355 |
) -> bool: |
| 1356 |
if backend == 'xai': |
| 1357 |
return bool(config.get('XAI_API_KEY')) |
| 1358 |
if backend == 'grok': |
| 1359 |
# Keyless relative to X: needs only an installed, signed-in grok CLI. |
| 1360 |
# Both surfaces are filesystem-only (PATH lookup + credential store), |
| 1361 |
# so local_only needs no separate branch. |
| 1362 |
from . import grok_x |
| 1363 |
return grok_x.has_stored_auth() |
| 1364 |
if backend == 'bird': |
| 1365 |
from . import bird_x |
| 1366 |
return has_bird_creds and bird_x.is_bird_installed() |
| 1367 |
if backend == 'xurl': |
| 1368 |
from . import xurl_x |
| 1369 |
if local_only: |
| 1370 |
# Doctor/safe-diagnose path: local evidence only (PATH lookup + |
| 1371 |
# token store) — never the live `xurl whoami` network call. |
| 1372 |
return xurl_x.has_stored_auth() |
| 1373 |
return xurl_x.is_available() |
| 1374 |
if backend == 'xquik': |
| 1375 |
return is_xquik_available(config) |
| 1376 |
if backend == 'xapi': |
| 1377 |
# Key presence only (no network); local_only needs no branch. |
| 1378 |
return bool(config.get('X_BEARER_TOKEN')) |
| 1379 |
return False |
| 1380 |
|
| 1381 |
|
| 1382 |
def x_backend_chain(config: dict[str, Any], local_only: bool = False) -> list[str]: |
| 1383 |
"""Ordered list of available X backends. |
| 1384 |
|
| 1385 |
``chain[0]`` is the default X source; the remaining entries are failover |
| 1386 |
backups, used only when the one before yields no items or errors. There is |
| 1387 |
exactly one X source — these are its backends, never fetched in parallel. |
| 1388 |
|
| 1389 |
A ``LAST30DAYS_X_BACKEND`` pin forces a single backend (no failover): the |
| 1390 |
user explicitly chose it. Valid pin values are in ``_X_BACKEND_KNOWN`` |
| 1391 |
(the auto chain plus opt-in backends like grok). Browser-cookie probing |
| 1392 |
is intentionally avoided (automatic Keychain access causes popups); bird |
| 1393 |
counts as available only when AUTH_TOKEN and CT0 are present explicitly. |
| 1394 |
|
| 1395 |
Unpinned runs walk only ``_X_BACKEND_ORDER``: opt-in backends like grok |
| 1396 |
are never auto-selected. A leftover ~/.grok/auth.json must not steal the |
| 1397 |
X lane; pin ``LAST30DAYS_X_BACKEND=grok`` to enable it explicitly. |
| 1398 |
|
| 1399 |
``local_only=True`` is the doctor/safe-diagnose flavor: availability is |
| 1400 |
answered from local evidence only (no subprocess spawns that reach the |
| 1401 |
network — xurl's live `whoami` check is replaced by its on-disk token |
| 1402 |
store). Research-time callers keep the default live semantics. |
| 1403 |
|
| 1404 |
The unpinned walk is ``x_policy(config).auto_chain``: the default order |
| 1405 |
above on every host, or ``_X_OFFICIAL`` (xapi -> xai -> xurl) on an |
| 1406 |
official-only host. The scraper is primed with cookies only when bird |
| 1407 |
ends up in the resulting chain (never on an official-only host unless |
| 1408 |
bird is pinned). |
| 1409 |
""" |
| 1410 |
has_bird_creds = bool(config.get('AUTH_TOKEN') and config.get('CT0')) |
| 1411 |
|
| 1412 |
preferred = x_backend_pin(config) |
| 1413 |
# Pin accepted from _X_BACKEND_KNOWN (auto chain + opt-in like grok). |
| 1414 |
if preferred in _X_BACKEND_KNOWN: |
| 1415 |
if _x_backend_available(preferred, config, has_bird_creds, local_only): |
| 1416 |
chain = [preferred] |
| 1417 |
else: |
| 1418 |
chain = [] |
| 1419 |
else: |
| 1420 |
# Unpinned: walk the policy's auto chain. Opt-in backends (grok, and |
| 1421 |
# xapi off an official-only host) are never auto-selected. |
| 1422 |
chain = [ |
| 1423 |
b for b in x_policy(config).auto_chain |
| 1424 |
if _x_backend_available(b, config, has_bird_creds, local_only) |
| 1425 |
] |
| 1426 |
|
| 1427 |
if 'bird' in chain: |
| 1428 |
from . import bird_x |
| 1429 |
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0')) |
| 1430 |
return chain |
| 1431 |
|
| 1432 |
|
| 1433 |
def get_x_source(config: dict[str, Any], local_only: bool = False) -> str | None: |
| 1434 |
"""The default (primary) X backend, or None if no X source is available. |
| 1435 |
|
| 1436 |
Thin wrapper over ``x_backend_chain`` returning the first/primary backend; |
| 1437 |
callers that want failover should use ``x_backend_chain`` directly. |
| 1438 |
``local_only`` is forwarded (see ``x_backend_chain``). |
| 1439 |
""" |
| 1440 |
chain = x_backend_chain(config, local_only=local_only) |
| 1441 |
return chain[0] if chain else None |
| 1442 |
|
| 1443 |
|
| 1444 |
def x_pending_browser_auth(config: dict[str, Any], local_only: bool = False) -> bool: |
| 1445 |
"""True when X is not available now but ``FROM_BROWSER`` will authenticate it at run time. |
| 1446 |
|
| 1447 |
``--diagnose`` / ``--preflight`` load config in ``plan_only`` mode, which |
| 1448 |
deliberately skips browser-cookie extraction (no Keychain popup, |
| 1449 |
``reads_values: false``). As a result ``get_x_source`` returns None and X is |
| 1450 |
dropped from ``available_sources`` even though a normal run would extract the |
| 1451 |
same cookies and authenticate X fine. This predicate reports that |
| 1452 |
"available pending browser auth" state without reading a single cookie — it |
| 1453 |
keys only on the resolved browser list (``cookie_extraction_browsers`` |
| 1454 |
derives it from ``FROM_BROWSER`` alone, no secrets) OR — on extra hosts |
| 1455 |
only (``x_extras_enabled``) — the agentcookie sidecar being on PATH (a plain |
| 1456 |
``which`` lookup), bird being installed, and X having a cookie-domain |
| 1457 |
mapping. A plain MacBook must NOT predict bird from an agentcookie binary on |
| 1458 |
PATH (R18), so the sidecar leg is gated behind ``x_extras_enabled``. |
| 1459 |
Side-effect free, so the safe-inspection contract of diagnose/preflight is |
| 1460 |
preserved. |
| 1461 |
|
| 1462 |
Returns False whenever X is already available outright (static AUTH_TOKEN/CT0, |
| 1463 |
or xAI/xurl/xquik backend), and in ``read`` mode (a real run has already |
| 1464 |
extracted creds, so its status must be unchanged — never "pending"). |
| 1465 |
""" |
| 1466 |
# Already available via a static backend (bird creds, xAI, xurl, xquik). |
| 1467 |
# local_only (doctor/safe-diagnose) answers the xurl leg from the token |
| 1468 |
# store instead of the live `xurl whoami` network call. |
| 1469 |
if get_x_source(config, local_only=local_only): |
| 1470 |
return False |
| 1471 |
# Only meaningful in inspection modes that skip extraction; a real ``read`` |
| 1472 |
# run has already attempted extraction and must report its true state. |
| 1473 |
if config.get('_BROWSER_COOKIE_MODE') == 'read': |
| 1474 |
return False |
| 1475 |
# Cookie-only predicate: on an official-only host no run-time cookie |
| 1476 |
# source exists unless bird is pinned (x_policy), so nothing is pending. |
| 1477 |
if not x_policy(config).cookie_discovery: |
| 1478 |
return False |
| 1479 |
if 'x' not in COOKIE_DOMAINS: |
| 1480 |
return False |
| 1481 |
from . import bird_x |
| 1482 |
if not bird_x.is_bird_installed(): |
| 1483 |
return False |
| 1484 |
# A FROM_BROWSER browser is a run-time cookie source on any host. |
| 1485 |
if cookie_extraction_browsers(config): |
| 1486 |
return True |
| 1487 |
# The agentcookie sidecar is a run-time cookie source ONLY on extra hosts |
| 1488 |
# (Linux / Mac mini / Darwin sink / AGENTCOOKIE=on). Gating this keeps a |
| 1489 |
# plain MacBook from predicting bird off a stray agentcookie binary (R18). |
| 1490 |
if x_extras_enabled(config): |
| 1491 |
from . import agentcookie |
| 1492 |
if agentcookie.is_available(config): |
| 1493 |
return True |
| 1494 |
return False |
| 1495 |
|
| 1496 |
|
| 1497 |
def is_ytdlp_available() -> bool: |
| 1498 |
"""Check if yt-dlp is installed for YouTube search.""" |
| 1499 |
from . import youtube_yt |
| 1500 |
return youtube_yt.is_ytdlp_installed() |
| 1501 |
|
| 1502 |
|
| 1503 |
def is_youtube_comments_available(config: dict[str, Any]) -> bool: |
| 1504 |
"""Check if YouTube comment enrichment is available. |
| 1505 |
|
| 1506 |
yt-dlp fetches YouTube comments free and keyless, so when it is installed |
| 1507 |
comments need no credential and no ``INCLUDE_SOURCES`` opt-in — the opt-in |
| 1508 |
only ever existed to gate ScrapeCreators credit spend, and there is none to |
| 1509 |
gate. ``EXCLUDE_SOURCES=youtube_comments`` remains the off-switch. |
| 1510 |
|
| 1511 |
Without yt-dlp, the legacy ScrapeCreators path still applies: it requires |
| 1512 |
SCRAPECREATORS_API_KEY AND ``youtube_comments`` in ``INCLUDE_SOURCES`` |
| 1513 |
(mirroring ``is_tiktok_comments_available``), bounded by |
| 1514 |
``enrich_with_comments(max_videos=3)`` at ~3 credits per run. |
| 1515 |
""" |
| 1516 |
if 'youtube_comments' in _parse_exclude_sources(config): |
| 1517 |
return False |
| 1518 |
if is_ytdlp_available(): |
| 1519 |
return True |
| 1520 |
if not config.get('SCRAPECREATORS_API_KEY'): |
| 1521 |
return False |
| 1522 |
return 'youtube_comments' in _parse_include_sources(config) |
| 1523 |
|
| 1524 |
|
| 1525 |
def is_tiktok_comments_available(config: dict[str, Any]) -> bool: |
| 1526 |
"""Check if TikTok comment enrichment is available. |
| 1527 |
|
| 1528 |
Requires SCRAPECREATORS_API_KEY AND tiktok_comments in INCLUDE_SOURCES. |
| 1529 |
Mirrors the youtube_comments opt-in pattern. |
| 1530 |
""" |
| 1531 |
if not config.get('SCRAPECREATORS_API_KEY'): |
| 1532 |
return False |
| 1533 |
include = _parse_include_sources(config) |
| 1534 |
return 'tiktok_comments' in include |
| 1535 |
|
| 1536 |
|
| 1537 |
def is_instagram_comments_available(config: dict[str, Any]) -> bool: |
| 1538 |
"""Check if Instagram comment enrichment is available. |
| 1539 |
|
| 1540 |
Requires SCRAPECREATORS_API_KEY AND instagram_comments in INCLUDE_SOURCES. |
| 1541 |
Mirrors the youtube_comments / tiktok_comments opt-in pattern. Comments are |
| 1542 |
fetched via ScrapeCreators (GET /v2/instagram/post/comments) with each |
| 1543 |
comment's ``comment_like_count`` used as its vote for ranking. Part of the |
| 1544 |
default onboarding tier (posts on -> comments on for TikTok/Instagram/YouTube). |
| 1545 |
""" |
| 1546 |
if not config.get('SCRAPECREATORS_API_KEY'): |
| 1547 |
return False |
| 1548 |
return 'instagram_comments' in _parse_include_sources(config) |
| 1549 |
|
| 1550 |
|
| 1551 |
def is_youtube_sc_available(config: dict[str, Any]) -> bool: |
| 1552 |
"""Check if ScrapeCreators YouTube search fallback is available. |
| 1553 |
|
| 1554 |
Used when yt-dlp is not installed or fails. |
| 1555 |
""" |
| 1556 |
return bool(config.get('SCRAPECREATORS_API_KEY')) |
| 1557 |
|
| 1558 |
|
| 1559 |
def is_hackernews_available() -> bool: |
| 1560 |
"""Check if Hacker News source is available. |
| 1561 |
|
| 1562 |
Always returns True - HN uses free Algolia API, no key needed. |
| 1563 |
""" |
| 1564 |
return True |
| 1565 |
|
| 1566 |
|
| 1567 |
def is_native_search(config: dict[str, Any]) -> bool: |
| 1568 |
"""Whether the invoking host has its own (better) native web search. |
| 1569 |
|
| 1570 |
Defined by capability, not host identity: the SKILL.md agent-host path sets |
| 1571 |
``LAST30DAYS_NATIVE_SEARCH`` when the runtime actually has a native web-search |
| 1572 |
tool (e.g. Claude Code's WebSearch). When true, the engine's keyless search |
| 1573 |
floor is suppressed so a worse free search never preempts the model's own. |
| 1574 |
Defaults False (unset), so headless/cron and hosts without native search fall |
| 1575 |
to the keyless floor. |
| 1576 |
""" |
| 1577 |
raw = config.get('LAST30DAYS_NATIVE_SEARCH') |
| 1578 |
if raw is None: |
| 1579 |
return False |
| 1580 |
return str(raw).strip().lower() in ('1', 'true', 'yes', 'on') |
| 1581 |
|
| 1582 |
|
| 1583 |
def keyless_web_allowed(config: dict[str, Any]) -> bool: |
| 1584 |
"""Whether the engine may use its keyless web-search floor for this run. |
| 1585 |
|
| 1586 |
Allowed only when the host does NOT have native search. Independent of |
| 1587 |
whether a paid key is set (the grounding dispatcher prefers paid first and |
| 1588 |
falls to keyless on empty/error for non-native runs). |
| 1589 |
""" |
| 1590 |
return not is_native_search(config) |
| 1591 |
|
| 1592 |
|
| 1593 |
def transcription_providers(config: dict[str, Any]) -> list[tuple[str, str]]: |
| 1594 |
"""Ordered (name, api_key) Whisper providers for caption-free transcription. |
| 1595 |
|
| 1596 |
Groq (free tier) first, OpenAI (paid) as the backstop. Empty when neither |
| 1597 |
key is set, in which case transcription degrades rather than runs. |
| 1598 |
""" |
| 1599 |
providers: list[tuple[str, str]] = [] |
| 1600 |
if config.get('GROQ_API_KEY'): |
| 1601 |
providers.append(('groq', config['GROQ_API_KEY'])) |
| 1602 |
if config.get('OPENAI_API_KEY'): |
| 1603 |
providers.append(('openai', config['OPENAI_API_KEY'])) |
| 1604 |
return providers |
| 1605 |
|
| 1606 |
|
| 1607 |
def is_bluesky_available(config: dict[str, Any]) -> bool: |
| 1608 |
"""Check if Bluesky source is available. |
| 1609 |
|
| 1610 |
Requires BSKY_HANDLE and BSKY_APP_PASSWORD (app password from bsky.app/settings). |
| 1611 |
""" |
| 1612 |
return bool(config.get('BSKY_HANDLE') and config.get('BSKY_APP_PASSWORD')) |
| 1613 |
|
| 1614 |
|
| 1615 |
def is_truthsocial_available(config: dict[str, Any]) -> bool: |
| 1616 |
"""Check if Truth Social source is available. |
| 1617 |
|
| 1618 |
Requires TRUTHSOCIAL_TOKEN (bearer token from browser dev tools). |
| 1619 |
""" |
| 1620 |
return bool(config.get('TRUTHSOCIAL_TOKEN')) |
| 1621 |
|
| 1622 |
|
| 1623 |
def is_polymarket_available() -> bool: |
| 1624 |
"""Check if Polymarket source is available. |
| 1625 |
|
| 1626 |
Always returns True - Gamma API is free, no key needed. |
| 1627 |
""" |
| 1628 |
return True |
| 1629 |
|
| 1630 |
|
| 1631 |
def is_tiktok_available(config: dict[str, Any]) -> bool: |
| 1632 |
"""Check if TikTok source is available (ScrapeCreators or legacy Apify). |
| 1633 |
|
| 1634 |
Returns True if SCRAPECREATORS_API_KEY or APIFY_API_TOKEN is set. |
| 1635 |
""" |
| 1636 |
return bool(config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN')) |
| 1637 |
|
| 1638 |
|
| 1639 |
def get_tiktok_token(config: dict[str, Any]) -> str: |
| 1640 |
"""Get TikTok API token, preferring ScrapeCreators over legacy Apify.""" |
| 1641 |
return config.get('SCRAPECREATORS_API_KEY') or config.get('APIFY_API_TOKEN') or '' |
| 1642 |
|
| 1643 |
|
| 1644 |
def _parse_include_sources(config: dict[str, Any]) -> set[str]: |
| 1645 |
"""Parse INCLUDE_SOURCES config value into a set of lowercase source names.""" |
| 1646 |
raw = config.get('INCLUDE_SOURCES') or '' |
| 1647 |
return {s.strip().lower() for s in raw.split(',') if s.strip()} |
| 1648 |
|
| 1649 |
|
| 1650 |
def _parse_exclude_sources(config: dict[str, Any]) -> set[str]: |
| 1651 |
"""Parse EXCLUDE_SOURCES config value into a set of lowercase source names.""" |
| 1652 |
raw = config.get('EXCLUDE_SOURCES') or '' |
| 1653 |
return {s.strip().lower() for s in raw.split(',') if s.strip()} |
| 1654 |
|
| 1655 |
|
| 1656 |
def include_sources(config: dict[str, Any]) -> set[str]: |
| 1657 |
"""Public view of the parsed INCLUDE_SOURCES set. |
| 1658 |
|
| 1659 |
Thin wrapper over ``_parse_include_sources`` so other modules (doctor, |
| 1660 |
etc.) don't reach into env's privates. |
| 1661 |
""" |
| 1662 |
return _parse_include_sources(config) |
| 1663 |
|
| 1664 |
|
| 1665 |
def is_setup_complete(config: dict[str, Any]) -> bool: |
| 1666 |
"""Whether guided setup marked this config complete (SETUP_COMPLETE truthy). |
| 1667 |
|
| 1668 |
Thin wrapper over ``_truthy`` so other modules don't reach into env's |
| 1669 |
privates. |
| 1670 |
""" |
| 1671 |
return _truthy(config.get('SETUP_COMPLETE')) |
| 1672 |
|
| 1673 |
|
| 1674 |
def is_threads_available(config: dict[str, Any]) -> bool: |
| 1675 |
"""Check if the Threads credential is available. |
| 1676 |
|
| 1677 |
Returns True when SCRAPECREATORS_API_KEY is set. This is an availability |
| 1678 |
predicate only: whether Threads is actually *scheduled* is gated in the |
| 1679 |
pipeline's ``available_sources`` by an ``INCLUDE_SOURCES=threads`` opt-in |
| 1680 |
(the onboarding "Everything" tier), so a key alone no longer runs Threads. |
| 1681 |
""" |
| 1682 |
return bool(config.get('SCRAPECREATORS_API_KEY')) |
| 1683 |
|
| 1684 |
|
| 1685 |
def is_instagram_available(config: dict[str, Any]) -> bool: |
| 1686 |
"""Check if Instagram source is available (ScrapeCreators). |
| 1687 |
|
| 1688 |
Returns True if SCRAPECREATORS_API_KEY is set. |
| 1689 |
Instagram uses the same key as TikTok. |
| 1690 |
""" |
| 1691 |
return bool(config.get('SCRAPECREATORS_API_KEY')) |
| 1692 |
|
| 1693 |
|
| 1694 |
def get_instagram_token(config: dict[str, Any]) -> str: |
| 1695 |
"""Get Instagram API token (same ScrapeCreators key as TikTok).""" |
| 1696 |
return config.get('SCRAPECREATORS_API_KEY') or '' |
| 1697 |
|
| 1698 |
|
| 1699 |
def get_xiaohongshu_api_base(config: dict[str, Any]) -> str: |
| 1700 |
"""Get Xiaohongshu HTTP API base URL. |
| 1701 |
|
| 1702 |
The availability probe caches the first logged-in local service it finds so |
| 1703 |
the later search request uses the same browser-backed session endpoint. |
| 1704 |
""" |
| 1705 |
cached = config.get(XIAOHONGSHU_RESOLVED_API_BASE_KEY) |
| 1706 |
if cached: |
| 1707 |
return str(cached).rstrip("/") |
| 1708 |
|
| 1709 |
explicit = config.get("XIAOHONGSHU_API_BASE") |
| 1710 |
if explicit: |
| 1711 |
return str(explicit).rstrip("/") |
| 1712 |
|
| 1713 |
return XIAOHONGSHU_DEFAULT_API_BASES[0] |
| 1714 |
|
| 1715 |
|
| 1716 |
def _xiaohongshu_api_base_candidates(config: dict[str, Any]) -> list[str]: |
| 1717 |
explicit = config.get("XIAOHONGSHU_API_BASE") |
| 1718 |
if explicit: |
| 1719 |
return [str(explicit).rstrip("/")] |
| 1720 |
|
| 1721 |
candidates: list[str] = [] |
| 1722 |
cached = config.get(XIAOHONGSHU_RESOLVED_API_BASE_KEY) |
| 1723 |
if cached: |
| 1724 |
candidates.append(str(cached).rstrip("/")) |
| 1725 |
|
| 1726 |
for base in XIAOHONGSHU_DEFAULT_API_BASES: |
| 1727 |
if base not in candidates: |
| 1728 |
candidates.append(base) |
| 1729 |
return candidates |
| 1730 |
|
| 1731 |
|
| 1732 |
def _xiaohongshu_base_logged_in(base: str, http_module: Any) -> bool: |
| 1733 |
# Keep the health probe snappy, but allow one retry for transient hiccups. |
| 1734 |
health = http_module.get(f"{base}/health", timeout=3, retries=2) |
| 1735 |
if not isinstance(health, dict): |
| 1736 |
return False |
| 1737 |
if not health.get("success"): |
| 1738 |
return False |
| 1739 |
|
| 1740 |
# Login checks can be slower because some services consult the browser |
| 1741 |
# profile/session, so use a slightly longer timeout than the health probe. |
| 1742 |
login = http_module.get(f"{base}/api/v1/login/status", timeout=8, retries=2) |
| 1743 |
is_logged_in = ( |
| 1744 |
login.get("data", {}).get("is_logged_in") |
| 1745 |
if isinstance(login, dict) else False |
| 1746 |
) |
| 1747 |
return bool(is_logged_in) |
| 1748 |
|
| 1749 |
|
| 1750 |
def is_xiaohongshu_available(config: dict[str, Any]) -> bool: |
| 1751 |
"""Check whether Xiaohongshu HTTP API is reachable and logged in.""" |
| 1752 |
# Import here to avoid heavy imports at module load. |
| 1753 |
from . import http |
| 1754 |
|
| 1755 |
for base in _xiaohongshu_api_base_candidates(config): |
| 1756 |
try: |
| 1757 |
if _xiaohongshu_base_logged_in(base, http): |
| 1758 |
config[XIAOHONGSHU_RESOLVED_API_BASE_KEY] = base |
| 1759 |
return True |
| 1760 |
except (OSError, http.HTTPError): |
| 1761 |
continue |
| 1762 |
except Exception as exc: |
| 1763 |
sys.stderr.write( |
| 1764 |
f"[last30days] WARNING: unexpected error checking Xiaohongshu " |
| 1765 |
f"at {base}: {type(exc).__name__}: {exc}\n" |
| 1766 |
) |
| 1767 |
sys.stderr.flush() |
| 1768 |
return False |
| 1769 |
|
| 1770 |
|
| 1771 |
# Backward compat alias |
| 1772 |
is_apify_available = is_tiktok_available |
| 1773 |
|
| 1774 |
|
| 1775 |
def get_x_source_status(config: dict[str, Any], probe: bool = False) -> dict[str, Any]: |
| 1776 |
"""Get detailed X source status for UI decisions. |
| 1777 |
|
| 1778 |
Args: |
| 1779 |
probe: when True, run a cheap 1-tweet bird probe and downgrade |
| 1780 |
``bird_authenticated`` to False when X clearly returns nothing, |
| 1781 |
so ``--diagnose`` reflects runtime reality instead of static |
| 1782 |
credential presence. A transient timeout leaves the status |
| 1783 |
unchanged (fail open). When False (the safe/diagnose path that |
| 1784 |
doctor uses), NO network is touched: xurl availability comes |
| 1785 |
from local evidence (``xurl_x.has_stored_auth``), never the |
| 1786 |
live ``xurl whoami`` call. |
| 1787 |
|
| 1788 |
Returns: |
| 1789 |
Dict with keys: source, bird_installed, bird_authenticated, |
| 1790 |
bird_username, xai_available, can_install_bird |
| 1791 |
""" |
| 1792 |
from . import bird_x |
| 1793 |
|
| 1794 |
# Backends this host may run: the policy's auto chain plus a known pin. |
| 1795 |
# Bird is primed/probed and xquik is probed only when they are in that |
| 1796 |
# set, so an official-only host never touches the scraper or the |
| 1797 |
# third-party API unless the backend is pinned. |
| 1798 |
policy = x_policy(config) |
| 1799 |
pin = x_backend_pin(config) |
| 1800 |
considered = set(policy.auto_chain) |
| 1801 |
if pin in _X_BACKEND_KNOWN: |
| 1802 |
considered.add(pin) |
| 1803 |
|
| 1804 |
if 'bird' in considered and config.get('AUTH_TOKEN') and config.get('CT0'): |
| 1805 |
bird_x.set_credentials(config.get('AUTH_TOKEN'), config.get('CT0')) |
| 1806 |
bird_status = dict(bird_x.get_bird_status()) |
| 1807 |
if 'bird' not in considered: |
| 1808 |
# Never report the scraper as usable where the policy forbids it. |
| 1809 |
bird_status["authenticated"] = False |
| 1810 |
xai_available = bool(config.get('XAI_API_KEY')) |
| 1811 |
xapi_available = bool(config.get('X_BEARER_TOKEN')) |
| 1812 |
|
| 1813 |
# Report the TRUE auth lane (browser / env / keychain) rather than the static |
| 1814 |
# "env AUTH_TOKEN" label — tokens usually come from live browser cookies, and |
| 1815 |
# mislabeling the lane sent past debugging down a 30-minute wrong path. |
| 1816 |
if bird_status["authenticated"]: |
| 1817 |
lane = config.get('_AUTH_TOKEN_SOURCE') or 'env' |
| 1818 |
bird_status["username"] = f"{lane} AUTH_TOKEN" |
| 1819 |
|
| 1820 |
# Optional runtime probe: don't show X green when it's effectively dead. |
| 1821 |
if probe and bird_status["authenticated"]: |
| 1822 |
if bird_x.probe_works() is False: |
| 1823 |
bird_status["authenticated"] = False |
| 1824 |
bird_status["username"] = "probe failed (no working X auth)" |
| 1825 |
|
| 1826 |
# Xquik: the key-based X source used when bird's cookie auth isn't available. |
| 1827 |
# Probe so --diagnose reports the true state — funded, or configured-but- |
| 1828 |
# unpaid (402) — instead of false-green on mere key presence. |
| 1829 |
xquik_available = is_xquik_available(config) |
| 1830 |
xquik_working: bool | None = None |
| 1831 |
xquik_status = "" |
| 1832 |
if xquik_available and 'xquik' in considered: |
| 1833 |
if probe: |
| 1834 |
from . import xquik |
| 1835 |
xquik_working = xquik.probe_works(get_xquik_token(config)) |
| 1836 |
xquik_status = xquik.probe_reason() |
| 1837 |
else: |
| 1838 |
xquik_status = "configured (not probed)" |
| 1839 |
|
| 1840 |
# Xurl availability, computed ONCE. probe=True (a live diagnose) may run |
| 1841 |
# the real `xurl whoami`; probe=False is the safe path (doctor, |
| 1842 |
# --diagnose, --preflight) and must stay local-only — the live check is |
| 1843 |
# an authenticated X API network call. |
| 1844 |
from . import xurl_x as _xurl_x |
| 1845 |
xurl_available = _xurl_x.is_available() if probe else _xurl_x.has_stored_auth() |
| 1846 |
|
| 1847 |
# Grok availability is filesystem-only on both paths (PATH lookup plus the |
| 1848 |
# credential store), so it is safe to compute here regardless of `probe`. |
| 1849 |
# Grok is opt-in only: it appears in grok_available but never wins the |
| 1850 |
# unpinned source selection. |
| 1851 |
from . import grok_x as _grok_x |
| 1852 |
grok_available = _grok_x.has_stored_auth() |
| 1853 |
|
| 1854 |
# Determine active source. A pin forces a single backend (R4): ANY known |
| 1855 |
# pin is exclusive, mirroring x_backend_chain's [] semantics. Pinned |
| 1856 |
# backend available -> that source. Pinned backend unavailable -> None. |
| 1857 |
# Otherwise walk the policy's auto chain (default: bird first, cookies |
| 1858 |
# beat XAI_API_KEY when both are present, then xai, xurl, xquik; |
| 1859 |
# official-only: xapi, xai, xurl). Opt-in backends are never |
| 1860 |
# auto-selected; a leftover ~/.grok/auth.json must not steal the X lane. |
| 1861 |
usable = { |
| 1862 |
'bird': bird_status["authenticated"], |
| 1863 |
'xai': xai_available, |
| 1864 |
'xurl': xurl_available, |
| 1865 |
'xquik': xquik_available and xquik_working is not False, |
| 1866 |
'grok': grok_available, |
| 1867 |
'xapi': xapi_available, |
| 1868 |
} |
| 1869 |
if pin in _X_BACKEND_KNOWN: |
| 1870 |
# Pin is exclusive: pinned backend if available, else None (no fallback). |
| 1871 |
source = pin if usable.get(pin) else None |
| 1872 |
else: |
| 1873 |
source = next((b for b in policy.auto_chain if usable.get(b)), None) |
| 1874 |
|
| 1875 |
return { |
| 1876 |
"source": source, |
| 1877 |
"bird_installed": bird_status["installed"], |
| 1878 |
"bird_authenticated": bird_status["authenticated"], |
| 1879 |
"bird_username": bird_status["username"], |
| 1880 |
"xai_available": xai_available, |
| 1881 |
"xapi_available": xapi_available, |
| 1882 |
"grok_available": grok_available, |
| 1883 |
"xurl_available": xurl_available, |
| 1884 |
"xquik_available": xquik_available, |
| 1885 |
"xquik_working": xquik_working, |
| 1886 |
"xquik_status": xquik_status, |
| 1887 |
"can_install_bird": bird_status["can_install"], |
| 1888 |
} |
| 1889 |
|
| 1890 |
|
| 1891 |
# Pinterest |
| 1892 |
def is_pinterest_available(config: dict[str, Any]) -> bool: |
| 1893 |
"""Check if Pinterest source is available. |
| 1894 |
|
| 1895 |
Returns True when SCRAPECREATORS_API_KEY is set AND 'pinterest' is in |
| 1896 |
INCLUDE_SOURCES (or requested_sources at the pipeline level). Pinterest |
| 1897 |
is opt-in because not every topic benefits from visual pin results. |
| 1898 |
""" |
| 1899 |
return bool(config.get('SCRAPECREATORS_API_KEY')) |
| 1900 |
|
| 1901 |
|
| 1902 |
def get_pinterest_token(config: dict[str, Any]) -> str: |
| 1903 |
"""Get Pinterest API token (same ScrapeCreators key as TikTok/Instagram).""" |
| 1904 |
return config.get('SCRAPECREATORS_API_KEY') or '' |
| 1905 |
|
| 1906 |
|
| 1907 |
# Xquik |
| 1908 |
def is_xquik_available(config: dict[str, Any]) -> bool: |
| 1909 |
"""Check if Xquik X search source is available. |
| 1910 |
|
| 1911 |
Requires XQUIK_API_KEY (API key from xquik.com). |
| 1912 |
""" |
| 1913 |
return bool(config.get('XQUIK_API_KEY')) |
| 1914 |
|
| 1915 |
|
| 1916 |
def get_xquik_token(config: dict[str, Any]) -> str: |
| 1917 |
"""Get Xquik API key.""" |
| 1918 |
return config.get('XQUIK_API_KEY') or '' |
| 1919 |
|