| 1 | """Regression guard: the Hermes install-time scanner must find zero CRITICAL. |
| 2 | |
| 3 | Hermes (NousResearch/hermes-agent, tools/skills_guard.py) blocks community-tier |
| 4 | installs on a `dangerous` verdict, which any single CRITICAL finding produces. |
| 5 | Issue #513 was caused by 14 CRITICAL false positives; the fix removed them so the |
| 6 | verdict is `caution` (--force installable). This test replicates the scanner's |
| 7 | CRITICAL-severity regexes exactly and asserts none match in the scanned subtree, |
| 8 | so a future edit that reintroduces a blocking pattern fails CI instead of |
| 9 | silently re-blocking every Hermes user. |
| 10 | |
| 11 | This is a self-contained replica (no Hermes dependency). The rule regexes below |
| 12 | are copied verbatim from skills_guard.py's THREAT_PATTERNS; keep them in sync if |
| 13 | Hermes changes them. HIGH/MEDIUM findings are intentionally NOT checked here -- |
| 14 | they do not gate the `caution` verdict (see docs/plans hermes-scan plan). |
| 15 | """ |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import re |
| 19 | from pathlib import Path |
| 20 | |
| 21 | # scan root == the skill directory (where SKILL.md lives), matching how Hermes |
| 22 | # resolves owner/repo -> skills/<name>/. |
| 23 | SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "last30days" |
| 24 | |
| 25 | # CRITICAL-severity exfiltration/injection rules from skills_guard.py, verbatim. |
| 26 | CRITICAL_RULES = [ |
| 27 | (r'fetch\s*\([^\n]*\$\{?\w*(KEY|TOKEN|SECRET|PASSWORD|API)', "env_exfil_fetch"), |
| 28 | (r'httpx?\.(get|post|put|patch)\s*\([^\n]*(KEY|TOKEN|SECRET|PASSWORD)', "env_exfil_httpx"), |
| 29 | (r'requests\.(get|post|put|patch)\s*\([^\n]*(KEY|TOKEN|SECRET|PASSWORD)', "env_exfil_requests"), |
| 30 | (r'os\.environ\s*\.get\s*\(\s*["\'][^"\']*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)', "python_environ_get_secret"), |
| 31 | (r'os\.getenv\s*\(\s*[^\)]*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)', "python_getenv_secret"), |
| 32 | (r'ENV\[.*(?:KEY|TOKEN|SECRET|PASSWORD)', "ruby_env_secret"), |
| 33 | (r'do\s+not\s+(?:\w+\s+)*tell\s+(?:\w+\s+)*the\s+user', "deception_hide"), |
| 34 | ] |
| 35 | |
| 36 | # Scan-root .skillignore excludes (directory prefixes + explicit files). Mirrors |
| 37 | # skills/last30days/.skillignore so this test scans exactly what Hermes scans. |
| 38 | IGNORE_DIRS = ("assets/", "agents/", "scripts/lib/vendor/") |
| 39 | IGNORE_FILES = { |
| 40 | "scripts/build-skill.sh", "scripts/compare.sh", "scripts/evaluate_search_quality.py", |
| 41 | "scripts/test_device_auth.py", "scripts/test-v1-vs-v2.sh", "scripts/verify_v3.py", |
| 42 | } |
| 43 | |
| 44 | |
| 45 | def _scanned_files(): |
| 46 | for p in SKILL_ROOT.rglob("*"): |
| 47 | if not p.is_file(): |
| 48 | continue |
| 49 | rel = p.relative_to(SKILL_ROOT).as_posix() |
| 50 | if any(rel.startswith(d) for d in IGNORE_DIRS) or rel in IGNORE_FILES: |
| 51 | continue |
| 52 | # binary/asset extensions the scanner skips for text rules |
| 53 | if p.suffix.lower() in {".jpg", ".jpeg", ".png", ".gif", ".mp3", ".json"}: |
| 54 | continue |
| 55 | yield rel, p |
| 56 | |
| 57 | |
| 58 | def test_zero_critical_scanner_findings(): |
| 59 | compiled = [(re.compile(rx, re.IGNORECASE), name) for rx, name in CRITICAL_RULES] |
| 60 | hits = [] |
| 61 | for rel, path in _scanned_files(): |
| 62 | try: |
| 63 | text = path.read_text(encoding="utf-8", errors="replace") |
| 64 | except OSError: |
| 65 | continue |
| 66 | for i, line in enumerate(text.splitlines(), 1): |
| 67 | for rx, name in compiled: |
| 68 | if rx.search(line): |
| 69 | hits.append(f"{name} {rel}:{i} {line.strip()[:90]}") |
| 70 | assert not hits, ( |
| 71 | "Hermes scanner CRITICAL patterns reappeared in the scanned subtree " |
| 72 | "(this re-blocks every community install). Findings:\n " + "\n ".join(hits) |
| 73 | ) |
| 74 |